diff --git a/.flake8 b/.flake8
new file mode 100644
index 0000000000..c72f249028
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,5 @@
+[flake8]
+ignore = E203, E266, E501, W503, F403, F401, E711
+max-line-length = 79
+max-complexity = 18
+select = B,C,E,F,W,T4,B9
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 57f424d17b..0fe8a7c92a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,6 +64,7 @@ packaging/msi/obj
 simfdb
 tests/oldBinaries
 trace.*.xml
+trace.*.json
 .venv
 
 # Editor files
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000000..a843a71aac
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,9 @@
+repos:
+-   repo: https://github.com/psf/black
+    rev: 2018e667a6a36ee3fbfa8041cd36512f92f60d49  # frozen: 22.8.0
+    hooks:
+    - id: black
+-   repo: https://github.com/pycqa/flake8
+    rev: f8e1b317742036ff11ff86356fd2b68147e169f7  # frozen: 5.0.4
+    hooks:
+    - id: flake8
\ No newline at end of file
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 59161d79b6..4b631b5df4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,6 +22,11 @@ else()
   cmake_minimum_required(VERSION 3.13)
 endif()
 
+# silence deprecation warnings in newer versions of cmake
+if(POLICY CMP0135)
+  cmake_policy(SET CMP0135 NEW)
+endif()
+
 project(foundationdb
   VERSION 7.2.0
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
@@ -206,7 +211,7 @@ endif()
 if (CMAKE_EXPORT_COMPILE_COMMANDS AND WITH_PYTHON)
   add_custom_command(
 	  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json
-	  COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py
+	  COMMAND $<TARGET_FILE:Python3::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py
 	  ARGS -b ${CMAKE_CURRENT_BINARY_DIR} -s ${CMAKE_CURRENT_SOURCE_DIR} -o ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json
 	  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json
 	  COMMENT "Build compile commands for IDE"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9549297305..f142b3dfae 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -20,7 +20,7 @@ If you have questions, we encourage you to engage in discussion on the [communit
 
 ## Before you get started
 ### Community Guidelines
-We want the FoundationDB community to be as welcoming and inclusive as possible, and have adopted a [Code of Conduct](CODE_OF_CONDUCT.md) that we ask all community members to read and observe.
+We want the FoundationDB community to be as welcoming and inclusive as possible, and have adopted a [Code of Conduct](CODE_OF_CONDUCT.md) that we ask all community members to read and abide by.
 
 ### Project Licensing
 By submitting a pull request, you represent that you have the right to license your contribution to Apple and the community, and agree by submitting the patch that your contributions are licensed under the Apache 2.0 license.
@@ -34,10 +34,13 @@ Members of the Apple FoundationDB team are part of the core committers helping r
 
 ## Contributing
 ### Opening a Pull Request
-We love pull requests! For minor changes, feel free to open up a PR directly. For larger feature development and any changes that may require community discussion, we ask that you discuss your ideas on the [community forums](https://forums.foundationdb.org) prior to opening a PR, and then reference that thread within your PR comment. Please refer to [FoundationDB Commit Process](https://github.com/apple/foundationdb/wiki/FoundationDB-Commit-Process) for more detailed guidelines.
+We love pull requests! For minor changes, feel free to open up a PR directly. For larger feature development and any changes that may require community discussion, we ask that you discuss your ideas on the [community forums](https://forums.foundationdb.org) prior to opening a PR, and then reference that thread within your PR comment. Please refer to the [FoundationDB Commit Process](https://github.com/apple/foundationdb/wiki/FoundationDB-Commit-Process) for more detailed guidelines.
 
 CI will be run automatically for core committers, and for community PRs it will be initiated by the request of a core committer.  Tests can also be run locally via `ctest`, and core committers can run additional validation on pull requests prior to merging them.
 
+### Python pre-commit
+We use a pre-commit pipeline with black and flake8 to enforce python best coding practices. Install pre-commit ```pip install pre-commit```. Install it in your FoundationDB directory ```pre-commit install```.
+
 ### Reporting issues
 Please refer to the section below on [using GitHub issues and the community forums](#using-github-issues-and-community-forums) for more info.
 
@@ -46,10 +49,10 @@ To report a security issue, please **DO NOT** start by filing a public issue or
 
 ## Project Communication
 ### Community Forums
-We encourage your participation asking questions and helping improve the FoundationDB project. Check out the [FoundationDB community forums](https://forums.foundationdb.org), which serve a similar function as mailing lists in many open source projects. The forums are organized into three sections:
+We encourage your participation asking questions and helping improve the FoundationDB project. Check out the [FoundationDB community forums](https://forums.foundationdb.org), which serve a similar function as mailing lists in many open source projects. The forums are organized into three categories:
 
 * [Development](https://forums.foundationdb.org/c/development): For discussing the internals and development of the FoundationDB core, as well as layers.
-* [Using FoundationDB](https://forums.foundationdb.org/c/using-foundationdb): For discussing user-facing topics. Getting started and have a question? This is the place for you.
+* [Using FoundationDB](https://forums.foundationdb.org/c/using-foundationdb): For discussing user-facing topics. Getting started and have a question? This is the category for you.
 * [Site Feedback](https://forums.foundationdb.org/c/site-feedback): A category for discussing the forums and the OSS project, its organization, how it works, and how we can improve it.
 
 ### Using GitHub Issues and Community Forums
@@ -63,4 +66,4 @@ GitHub Issues should be used for tracking tasks. If you know the specific code t
 * Implementing an agreed upon feature: *GitHub Issues*
 
 ### Project and Development Updates
-Stay connected to the project and the community! For project and community updates, follow the [FoundationDB project blog](https://www.foundationdb.org/blog/). Development announcements will be made via the community forums' [dev-announce](https://forums.foundationdb.org/c/development/dev-announce) section.
+Stay connected to the project and the community! For project and community updates, follow the [FoundationDB project blog](https://www.foundationdb.org/blog/). Development announcements will be made via the community forums' [dev-announce](https://forums.foundationdb.org/c/development/dev-announce) category.
diff --git a/FDBLibTLS/FDBLibTLSPolicy.cpp b/FDBLibTLS/FDBLibTLSPolicy.cpp
index 6f81f91335..d97932659b 100644
--- a/FDBLibTLS/FDBLibTLSPolicy.cpp
+++ b/FDBLibTLS/FDBLibTLSPolicy.cpp
@@ -34,6 +34,7 @@
 #include <algorithm>
 #include <exception>
 #include <map>
+#include <stdexcept>
 #include <string>
 #include <vector>
 
diff --git a/FDBLibTLS/FDBLibTLSVerify.cpp b/FDBLibTLS/FDBLibTLSVerify.cpp
index 216966f4c0..4aeea07c15 100644
--- a/FDBLibTLS/FDBLibTLSVerify.cpp
+++ b/FDBLibTLS/FDBLibTLSVerify.cpp
@@ -28,6 +28,7 @@
 #include <algorithm>
 #include <exception>
 #include <cstring>
+#include <stdexcept>
 
 static int hexValue(char c) {
 	static char const digits[] = "0123456789ABCDEF";
diff --git a/bindings/bindingtester/bindingtester.py b/bindings/bindingtester/bindingtester.py
index 508ede8998..a5de827fe9 100755
--- a/bindings/bindingtester/bindingtester.py
+++ b/bindings/bindingtester/bindingtester.py
@@ -49,6 +49,17 @@ from bindingtester.known_testers import Tester
 import fdb
 import fdb.tuple
 
+
+API_VERSIONS = [
+    13, 14, 16, 21, 22, 23,
+    100, 200, 300,
+    400, 410, 420, 430, 440, 450, 460,
+    500, 510, 520,
+    600, 610, 620, 630,
+    700, 710, 720,
+]
+
+
 fdb.api_version(FDB_API_VERSION)
 
 
@@ -156,8 +167,7 @@ def choose_api_version(selected_api_version, tester_min_version, tester_max_vers
         elif random.random() < 0.7:
             api_version = min_version
         elif random.random() < 0.9:
-            api_version = random.choice([v for v in [13, 14, 16, 21, 22, 23, 100, 200, 300, 400, 410, 420, 430,
-                                                     440, 450, 460, 500, 510, 520, 600, 610, 620, 630, 700, 710, 720] if v >= min_version and v <= max_version])
+            api_version = random.choice([v for v in API_VERSIONS if v >= min_version and v <= max_version])
         else:
             api_version = random.randint(min_version, max_version)
 
diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index 00803634e7..e0a1fc31bb 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -1,8 +1,8 @@
 set(FDB_C_SRCS
-        fdb_c.cpp
-        foundationdb/fdb_c.h
-        foundationdb/fdb_c_internal.h
-        foundationdb/fdb_c_types.h)
+  fdb_c.cpp
+  foundationdb/fdb_c.h
+  foundationdb/fdb_c_internal.h
+  foundationdb/fdb_c_types.h)
 
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/foundationdb)
 
@@ -10,50 +10,50 @@ set(asm_file ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.g.S)
 
 set(os "linux")
 set(cpu "intel")
-if (APPLE)
+if(APPLE)
   set(os "osx")
-elseif (WIN32)
+elseif(WIN32)
   set(os "windows")
   set(asm_file ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.g.asm)
-endif ()
+endif()
 
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
   set(cpu "aarch64")
-elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le|powerpc64le)")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le|powerpc64le)")
   set(cpu "ppc64le")
-endif ()
+endif()
 
 set(IS_ARM_MAC NO)
-if (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
+if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
   set(IS_ARM_MAC YES)
-endif ()
+endif()
 
 add_custom_command(OUTPUT ${asm_file} ${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h
-        COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/generate_asm.py ${os} ${cpu}
-        ${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.cpp
-        ${asm_file}
-        ${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h
-        DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/generate_asm.py ${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.cpp
-        COMMENT "Generate C bindings")
+  COMMAND $<TARGET_FILE:Python3::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/generate_asm.py ${os} ${cpu}
+  ${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.cpp
+  ${asm_file}
+  ${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/generate_asm.py ${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.cpp
+  COMMENT "Generate C bindings")
 add_custom_target(fdb_c_generated DEPENDS ${asm_file}
-        ${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h)
+  ${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h)
 
 vexillographer_compile(TARGET fdb_c_options LANG c OUT ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h
-        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h)
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h)
 
 include(GenerateExportHeader)
 
-if (OPEN_FOR_IDE)
+if(OPEN_FOR_IDE)
   add_library(fdb_c OBJECT ${FDB_C_SRCS} ${asm_file})
-else ()
+else()
   add_library(fdb_c SHARED ${FDB_C_SRCS} ${asm_file})
   strip_debug_symbols(fdb_c)
-endif ()
+endif()
 add_dependencies(fdb_c fdb_c_generated fdb_c_options)
 add_dependencies(fdbclient fdb_c_options)
 add_dependencies(fdbclient_sampling fdb_c_options)
 target_link_libraries(fdb_c PRIVATE $<BUILD_INTERFACE:fdbclient>)
-if (USE_UBSAN)
+if(USE_UBSAN)
   # The intent of this hack is to force c targets that depend on fdb_c to use
   # c++ as their linker language. Otherwise you see undefined references to c++
   # specific ubsan symbols.
@@ -61,104 +61,108 @@ if (USE_UBSAN)
   set_property(TARGET force_cxx_linker PROPERTY IMPORTED_LOCATION /dev/null)
   set_target_properties(force_cxx_linker PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES CXX)
   target_link_libraries(fdb_c PUBLIC $<BUILD_INTERFACE:force_cxx_linker>)
-endif ()
-if (APPLE)
+endif()
+if(APPLE)
   set(symbols ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.symbols)
   add_custom_command(OUTPUT ${symbols}
-          COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py
-          ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h
-          ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c_internal.h
-          ${symbols}
-          DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c_internal.h
-          COMMENT "Generate exported_symbols_list")
+    COMMAND $<TARGET_FILE:Python3::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py
+    ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c_internal.h
+    ${symbols}
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c_internal.h
+    COMMENT "Generate exported_symbols_list")
   add_custom_target(exported_symbols_list DEPENDS ${symbols})
   add_dependencies(fdb_c exported_symbols_list)
   target_link_options(fdb_c PRIVATE "LINKER:-no_weak_exports,-exported_symbols_list,${symbols}")
-elseif (WIN32)
-else ()
-  if (NOT USE_UBSAN)
+elseif(WIN32)
+else()
+  if(NOT USE_UBSAN)
     # For ubsan we need to export type information for the vptr check to work.
     # Otherwise we only want to export fdb symbols in the fdb c api.
     target_link_options(fdb_c PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.map")
-  endif ()
+  endif()
   target_link_options(fdb_c PRIVATE "LINKER:-z,nodelete,-z,noexecstack")
-endif ()
+endif()
 target_include_directories(fdb_c PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/foundationdb>)
-if (WIN32)
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/foundationdb>)
+if(WIN32)
   enable_language(ASM_MASM)
   set_property(SOURCE ${asm_file} PROPERTY LANGUAGE ASM_MASM)
-endif ()
+endif()
 
 # The tests don't build on windows
-if (NOT WIN32)
+if(NOT WIN32)
   set(MAKO_SRCS
-          test/mako/async.hpp
-          test/mako/async.cpp
-          test/mako/blob_granules.hpp
-          test/mako/blob_granules.cpp
-          test/mako/future.hpp
-          test/mako/limit.hpp
-          test/mako/logger.hpp
-          test/mako/mako.cpp
-          test/mako/mako.hpp
-          test/mako/operations.hpp
-          test/mako/operations.cpp
-          test/mako/process.hpp
-          test/mako/shm.hpp
-          test/mako/stats.hpp
-          test/mako/time.hpp
-          test/mako/utils.cpp
-          test/mako/utils.hpp)
+    test/mako/async.hpp
+    test/mako/async.cpp
+    test/mako/blob_granules.hpp
+    test/mako/blob_granules.cpp
+    test/mako/future.hpp
+    test/mako/limit.hpp
+    test/mako/logger.hpp
+    test/mako/mako.cpp
+    test/mako/mako.hpp
+    test/mako/operations.hpp
+    test/mako/operations.cpp
+    test/mako/process.hpp
+    test/mako/shm.hpp
+    test/mako/stats.hpp
+    test/mako/time.hpp
+    test/mako/utils.cpp
+    test/mako/utils.hpp)
   add_subdirectory(test/unit/third_party)
   find_package(Threads REQUIRED)
   set(UNIT_TEST_SRCS
-          test/unit/unit_tests.cpp
-          test/unit/fdb_api.cpp
-          test/unit/fdb_api.hpp)
+    test/unit/unit_tests.cpp
+    test/unit/fdb_api.cpp
+    test/unit/fdb_api.hpp)
 
   set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp)
   set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp)
   set(DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS
-          test/unit/disconnected_timeout_tests.cpp
-          test/unit/fdb_api.cpp
-          test/unit/fdb_api.hpp)
+    test/unit/disconnected_timeout_tests.cpp
+    test/unit/fdb_api.cpp
+    test/unit/fdb_api.hpp)
 
   add_library(fdb_cpp INTERFACE test/fdb_api.hpp)
   target_sources(fdb_cpp INTERFACE)
   target_include_directories(fdb_cpp INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/test)
-  target_link_libraries(fdb_cpp INTERFACE fmt::fmt)
+  target_link_libraries(fdb_cpp INTERFACE fdb_c fmt::fmt)
 
   set(API_TESTER_SRCS
-          test/apitester/fdb_c_api_tester.cpp
-          test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp
-          test/apitester/TesterApiWorkload.cpp
-          test/apitester/TesterApiWorkload.h
-          test/apitester/TesterTestSpec.cpp
-          test/apitester/TesterTestSpec.h
-          test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp
-          test/apitester/TesterCancelTransactionWorkload.cpp
-          test/apitester/TesterCorrectnessWorkload.cpp
-          test/apitester/TesterKeyValueStore.cpp
-          test/apitester/TesterKeyValueStore.h
-          test/apitester/TesterOptions.h
-          test/apitester/TesterScheduler.cpp
-          test/apitester/TesterScheduler.h
-          test/apitester/TesterTransactionExecutor.cpp
-          test/apitester/TesterTransactionExecutor.h
-          test/apitester/TesterUtil.cpp
-          test/apitester/TesterUtil.h
-          test/apitester/TesterWatchAndWaitWorkload.cpp
-          test/apitester/TesterWorkload.cpp
-          test/apitester/TesterWorkload.h
-          )
+    test/apitester/fdb_c_api_tester.cpp
+    test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp
+    test/apitester/TesterApiWorkload.cpp
+    test/apitester/TesterApiWorkload.h
+    test/apitester/TesterTestSpec.cpp
+    test/apitester/TesterTestSpec.h
+    test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp
+    test/apitester/TesterBlobGranuleErrorsWorkload.cpp
+    test/apitester/TesterBlobGranuleUtil.cpp
+    test/apitester/TesterBlobGranuleUtil.h
+    test/apitester/TesterCancelTransactionWorkload.cpp
+    test/apitester/TesterCorrectnessWorkload.cpp
+    test/apitester/TesterExampleWorkload.cpp
+    test/apitester/TesterKeyValueStore.cpp
+    test/apitester/TesterKeyValueStore.h
+    test/apitester/TesterOptions.h
+    test/apitester/TesterScheduler.cpp
+    test/apitester/TesterScheduler.h
+    test/apitester/TesterTransactionExecutor.cpp
+    test/apitester/TesterTransactionExecutor.h
+    test/apitester/TesterUtil.cpp
+    test/apitester/TesterUtil.h
+    test/apitester/TesterWatchAndWaitWorkload.cpp
+    test/apitester/TesterWorkload.cpp
+    test/apitester/TesterWorkload.h
+    )
 
   add_library(fdb_c_unit_tests_impl OBJECT ${UNIT_TEST_SRCS})
   add_library(fdb_c_api_tester_impl OBJECT ${API_TESTER_SRCS})
 
-  if (OPEN_FOR_IDE)
+  if(OPEN_FOR_IDE)
     add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h)
     add_library(fdb_c_ryw_benchmark OBJECT test/ryw_benchmark.c test/test.h)
     add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h)
@@ -168,7 +172,7 @@ if (NOT WIN32)
     add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS})
     add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
     add_library(disconnected_timeout_unit_tests OBJECT ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
-  else ()
+  else()
     add_executable(fdb_c_performance_test test/performance_test.c test/test.h)
     add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h)
     add_executable(fdb_c_txn_size_test test/txn_size_test.c test/test.h)
@@ -186,7 +190,7 @@ if (NOT WIN32)
     strip_debug_symbols(fdb_c_ryw_benchmark)
     strip_debug_symbols(fdb_c_txn_size_test)
     strip_debug_symbols(fdb_c_client_memory_test)
-  endif ()
+  endif()
 
   target_link_libraries(fdb_c_performance_test PRIVATE fdb_c Threads::Threads)
   target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c Threads::Threads)
@@ -194,11 +198,10 @@ if (NOT WIN32)
   target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads)
 
   target_include_directories(fdb_c_api_tester_impl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include ${CMAKE_BINARY_DIR}/flow/include)
-  if (USE_SANITIZER)
-    target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_asan)
-  else ()
-    target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
-  endif ()
+  target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
+  if(NOT APPLE)
+    target_link_libraries(fdb_c_api_tester_impl PRIVATE stdc++fs)
+  endif()
   target_link_libraries(fdb_c_api_tester_impl PRIVATE SimpleOpt)
 
   target_include_directories(fdb_c_unit_tests_impl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/)
@@ -211,253 +214,305 @@ if (NOT WIN32)
 
   # do not set RPATH for mako
   set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
-  if (USE_SANITIZER)
-    target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_asan rapidjson)
-  else ()
-    target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_target rapidjson)
-  endif ()
+  target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_target rapidjson)
 
-  if (NOT OPEN_FOR_IDE)
+  if(NOT OPEN_FOR_IDE)
     # Make sure that fdb_c.h is compatible with c90
     add_executable(fdb_c90_test test/fdb_c90_test.c)
     set_property(TARGET fdb_c90_test PROPERTY C_STANDARD 90)
     target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Werror)
     target_link_libraries(fdb_c90_test PRIVATE fdb_c)
-  endif ()
+  endif()
 
-  if (OPEN_FOR_IDE)
+  if(OPEN_FOR_IDE)
     set(FDB_C_TARGET $<TARGET_OBJECTS:fdb_c>)
-  else ()
+  else()
     set(FDB_C_TARGET $<TARGET_FILE:fdb_c>)
     add_custom_command(
-            OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
-            COMMAND ${CMAKE_COMMAND} -E copy ${FDB_C_TARGET} ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
-            DEPENDS fdb_c
-            COMMENT "Copy libfdb_c to use as external client for test")
+      OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+      COMMAND ${CMAKE_COMMAND} -E copy ${FDB_C_TARGET} ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+      DEPENDS fdb_c
+      COMMENT "Copy libfdb_c to use as external client for test")
     add_custom_target(external_client DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so)
     add_dependencies(fdb_c_unit_tests_impl external_client)
     add_dependencies(disconnected_timeout_unit_tests external_client)
     add_dependencies(fdb_c_api_tester_impl external_client)
 
     add_fdbclient_test(
-            NAME fdb_c_setup_tests
-            COMMAND $<TARGET_FILE:fdb_c_setup_tests>)
+      NAME fdb_c_setup_tests
+      COMMAND $<TARGET_FILE:fdb_c_setup_tests>)
     add_fdbclient_test(
-            NAME fdb_c_unit_tests
-            COMMAND $<TARGET_FILE:fdb_c_unit_tests>
-            @CLUSTER_FILE@
-            fdb)
+      NAME fdb_c_unit_tests
+      COMMAND $<TARGET_FILE:fdb_c_unit_tests>
+      @CLUSTER_FILE@
+      fdb)
     add_fdbclient_test(
-            NAME fdb_c_unit_tests_version_510
-            COMMAND $<TARGET_FILE:fdb_c_unit_tests_version_510>
-            @CLUSTER_FILE@
-            fdb)
+      NAME fdb_c_unit_tests_version_510
+      COMMAND $<TARGET_FILE:fdb_c_unit_tests_version_510>
+      @CLUSTER_FILE@
+      fdb)
     add_fdbclient_test(
-            NAME trace_partial_file_suffix_test
-            COMMAND $<TARGET_FILE:trace_partial_file_suffix_test>
-            @CLUSTER_FILE@
-            fdb)
+      NAME trace_partial_file_suffix_test
+      COMMAND $<TARGET_FILE:trace_partial_file_suffix_test>
+      @CLUSTER_FILE@
+      fdb)
     add_fdbclient_test(
-            NAME fdb_c_external_client_unit_tests
-            COMMAND $<TARGET_FILE:fdb_c_unit_tests>
-            @CLUSTER_FILE@
-            fdb
-            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+      NAME fdb_c_external_client_unit_tests
+      COMMAND $<TARGET_FILE:fdb_c_unit_tests>
+      @CLUSTER_FILE@
+      fdb
+      ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
     )
     add_unavailable_fdbclient_test(
-            NAME disconnected_timeout_unit_tests
-            COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
-            @CLUSTER_FILE@
+      NAME disconnected_timeout_unit_tests
+      COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
+      @CLUSTER_FILE@
     )
     add_unavailable_fdbclient_test(
-            NAME disconnected_timeout_external_client_unit_tests
-            COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
-            @CLUSTER_FILE@
-            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+      NAME disconnected_timeout_external_client_unit_tests
+      COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
+      @CLUSTER_FILE@
+      ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
     )
     add_fdbclient_test(
-            NAME fdb_c_api_tests
-            DISABLE_LOG_DUMP
-            COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
-            --cluster-file
-            @CLUSTER_FILE@
-            --tester-binary
-            $<TARGET_FILE:fdb_c_api_tester>
-            --external-client-library
-            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
-            --test-dir
-            ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
-            --tmp-dir
-            @TMP_DIR@
-            --log-dir
-            @LOG_DIR@
+      NAME fdb_c_api_tests
+      DISABLE_LOG_DUMP
+      COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
+      --cluster-file
+      @CLUSTER_FILE@
+      --tester-binary
+      $<TARGET_FILE:fdb_c_api_tester>
+      --external-client-library
+      ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+      --test-dir
+      ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
+      --tmp-dir
+      @TMP_DIR@
+      --log-dir
+      @LOG_DIR@
     )
 
     add_fdbclient_test(
-            NAME fdb_c_api_tests_blob_granule
-            DISABLE_LOG_DUMP
-            API_TEST_BLOB_GRANULES_ENABLED
-            COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
-            --cluster-file
-            @CLUSTER_FILE@
-            --tester-binary
-            $<TARGET_FILE:fdb_c_api_tester>
-            --external-client-library
-            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
-            --test-dir
-            ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/blobgranuletests
-            --blob-granule-local-file-path
-            @DATA_DIR@/fdbblob/
-            --tmp-dir
-            @TMP_DIR@
-            --log-dir
-            @LOG_DIR@
+      NAME fdb_c_api_tests_local_only
+      DISABLE_LOG_DUMP
+      COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
+      --cluster-file
+      @CLUSTER_FILE@
+      --tester-binary
+      $<TARGET_FILE:fdb_c_api_tester>
+      --test-dir
+      ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/local_tests
+      --tmp-dir
+      @TMP_DIR@
+      --log-dir
+      @LOG_DIR@
     )
 
     add_fdbclient_test(
-            NAME fdb_c_api_tests_with_tls
-            DISABLE_LOG_DUMP
-            TLS_ENABLED
-            COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
-            --cluster-file
-            @CLUSTER_FILE@
-            --tester-binary
-            $<TARGET_FILE:fdb_c_api_tester>
-            --external-client-library
-            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
-            --test-dir
-            ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
-            --tmp-dir
-            @TMP_DIR@
-            --log-dir
-            @LOG_DIR@
-            --tls-cert-file
-            @CLIENT_CERT_FILE@
-            --tls-key-file
-            @CLIENT_KEY_FILE@
-            --tls-ca-file
-            @SERVER_CA_FILE@
+      NAME fdb_c_api_tests_blob_granule
+      DISABLE_LOG_DUMP
+      API_TEST_BLOB_GRANULES_ENABLED
+      COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
+      --cluster-file
+      @CLUSTER_FILE@
+      --tester-binary
+      $<TARGET_FILE:fdb_c_api_tester>
+      --external-client-library
+      ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+      --test-dir
+      ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/blobgranuletests
+      --blob-granule-local-file-path
+      @DATA_DIR@/fdbblob/
+      --tmp-dir
+      @TMP_DIR@
+      --log-dir
+      @LOG_DIR@
     )
 
-    if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT USE_SANITIZER)
+    add_fdbclient_test(
+      NAME fdb_c_api_tests_with_tls
+      DISABLE_LOG_DUMP
+      TLS_ENABLED
+      COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
+      --cluster-file
+      @CLUSTER_FILE@
+      --tester-binary
+      $<TARGET_FILE:fdb_c_api_tester>
+      --external-client-library
+      ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+      --test-dir
+      ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
+      --tmp-dir
+      @TMP_DIR@
+      --log-dir
+      @LOG_DIR@
+      --tls-cert-file
+      @CLIENT_CERT_FILE@
+      --tls-key-file
+      @CLIENT_KEY_FILE@
+      --tls-ca-file
+      @SERVER_CA_FILE@
+    )
+
+    add_test(NAME fdb_c_upgrade_to_future_version
+      COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+      --build-dir ${CMAKE_BINARY_DIR}
+      --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+      --upgrade-path "7.2.0" "7.3.0" "7.2.0"
+      --process-number 3
+      )
+    set_tests_properties("fdb_c_upgrade_to_future_version" PROPERTIES ENVIRONMENT "${SANITIZER_OPTIONS}")
+
+    add_test(NAME fdb_c_upgrade_to_future_version_blob_granules
+      COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+      --build-dir ${CMAKE_BINARY_DIR}
+      --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/ApiBlobGranulesCorrectness.toml
+      --upgrade-path "7.2.0" "7.3.0" "7.2.0"
+      --blob-granules-enabled
+      --process-number 3
+      )
+
+    if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT USE_SANITIZER)
       add_test(NAME fdb_c_upgrade_single_threaded_630api
-              COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-              --build-dir ${CMAKE_BINARY_DIR}
-              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
-              --upgrade-path "6.3.23" "7.0.0" "7.1.9" "7.2.0"
-              --process-number 1
-              )
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+        --build-dir ${CMAKE_BINARY_DIR}
+        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
+        --upgrade-path "6.3.23" "7.0.0" "7.1.9" "7.2.0"
+        --process-number 1
+        )
 
       add_test(NAME fdb_c_upgrade_single_threaded_700api
-              COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-              --build-dir ${CMAKE_BINARY_DIR}
-              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
-              --upgrade-path "7.0.0" "7.1.9" "7.2.0"
-              --process-number 1
-              )
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+        --build-dir ${CMAKE_BINARY_DIR}
+        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
+        --upgrade-path "7.0.0" "7.1.9" "7.2.0"
+        --process-number 1
+        )
 
       add_test(NAME fdb_c_upgrade_multi_threaded_630api
-              COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-              --build-dir ${CMAKE_BINARY_DIR}
-              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-              --upgrade-path "6.3.23" "7.0.0" "7.1.9" "7.2.0" "7.1.9"
-              --process-number 3
-              )
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+        --build-dir ${CMAKE_BINARY_DIR}
+        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+        --upgrade-path "6.3.23" "7.0.0" "7.1.9" "7.2.0" "7.1.9"
+        --process-number 3
+        )
 
       add_test(NAME fdb_c_upgrade_multi_threaded_700api
-              COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-              --build-dir ${CMAKE_BINARY_DIR}
-              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-              --upgrade-path "7.0.0" "7.1.9" "7.2.0" "7.1.9"
-              --process-number 3
-              )
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+        --build-dir ${CMAKE_BINARY_DIR}
+        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+        --upgrade-path "7.0.0" "7.1.9" "7.2.0" "7.1.9"
+        --process-number 3
+        )
 
       add_test(NAME fdb_c_upgrade_multi_threaded_710api
-              COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-              --build-dir ${CMAKE_BINARY_DIR}
-              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-              --upgrade-path "7.1.9" "7.2.0" "7.1.9"
-              --process-number 3
-              )
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+        --build-dir ${CMAKE_BINARY_DIR}
+        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+        --upgrade-path "7.1.9" "7.2.0" "7.1.9"
+        --process-number 3
+        )
 
       add_test(NAME fdb_c_cluster_wiggle
-              COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-              --build-dir ${CMAKE_BINARY_DIR}
-              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-              --upgrade-path "7.2.0" "wiggle"
-              --disable-log-dump
-              --process-number 3
-              --redundancy double
-              )
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+        --build-dir ${CMAKE_BINARY_DIR}
+        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+        --upgrade-path "7.2.0" "wiggle"
+        --disable-log-dump
+        --process-number 3
+        --redundancy double
+        )
 
       add_test(NAME fdb_c_wiggle_and_upgrade_latest
-              COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-              --build-dir ${CMAKE_BINARY_DIR}
-              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-              --upgrade-path "7.1.9" "wiggle" "7.2.0"
-              --disable-log-dump
-              --process-number 3
-              --redundancy double
-              )
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+        --build-dir ${CMAKE_BINARY_DIR}
+        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+        --upgrade-path "7.1.9" "wiggle" "7.2.0"
+        --disable-log-dump
+        --process-number 3
+        --redundancy double
+        )
 
       add_test(NAME fdb_c_wiggle_and_upgrade_63
-              COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-              --build-dir ${CMAKE_BINARY_DIR}
-              --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-              --upgrade-path "6.3.24" "wiggle" "7.0.0"
-              --disable-log-dump
-              --process-number 3
-              --redundancy double
-              )
+        COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+        --build-dir ${CMAKE_BINARY_DIR}
+        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+        --upgrade-path "6.3.24" "wiggle" "7.0.0"
+        --disable-log-dump
+        --process-number 3
+        --redundancy double
+        )
 
-    endif ()
-
-  endif ()
-endif ()
+    endif()
+  endif()
 
+endif()
 
 set(c_workloads_srcs
-        test/workloads/workloads.cpp
-        test/workloads/workloads.h
-        test/workloads/SimpleWorkload.cpp)
+  test/workloads/workloads.cpp
+  test/workloads/workloads.h
+  test/workloads/SimpleWorkload.cpp)
 
-if (OPEN_FOR_IDE)
+if(OPEN_FOR_IDE)
   add_library(c_workloads OBJECT ${c_workloads_srcs})
-else ()
+else()
   add_library(c_workloads SHARED ${c_workloads_srcs})
-endif ()
+endif()
 set_target_properties(c_workloads PROPERTIES
-        LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/share/foundationdb")
+  LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/share/foundationdb")
 target_link_libraries(c_workloads PUBLIC fdb_c)
 
-if (NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE)
+if(NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE)
   target_link_options(c_workloads PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/external_workload.map,-z,nodelete")
-endif ()
+endif()
 
 # Generate shim library in Linux builds
-if (OPEN_FOR_IDE)
+if(OPEN_FOR_IDE)
 
-  add_library(fdb_c_shim OBJECT fdb_c_shim.cpp)
+  add_library(fdb_c_shim OBJECT foundationdb/fdb_c_shim.h fdb_c_shim.cpp)
   target_link_libraries(fdb_c_shim PUBLIC dl)
+  target_include_directories(fdb_c_shim PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/foundationdb>)
 
-elseif (NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE) # Linux Only
+  add_library(fdb_c_shim_lib_tester OBJECT test/shim_lib_tester.cpp)
+  target_link_libraries(fdb_c_shim_lib_tester PRIVATE fdb_c_shim SimpleOpt fdb_cpp Threads::Threads)
+  target_include_directories(fdb_c_shim_lib_tester PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include)
+
+elseif(NOT WIN32 AND NOT APPLE AND NOT USE_SANITIZER) # Linux Only, non-santizer only
 
   set(SHIM_LIB_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
   set(SHIM_LIB_GEN_SRC
-          ${SHIM_LIB_OUTPUT_DIR}/libfdb_c.so.init.c
-          ${SHIM_LIB_OUTPUT_DIR}/libfdb_c.so.tramp.S)
+    ${SHIM_LIB_OUTPUT_DIR}/libfdb_c.so.init.c
+    ${SHIM_LIB_OUTPUT_DIR}/libfdb_c.so.tramp.S)
+
+  set(IMPLIBSO_SRC_DIR ${CMAKE_SOURCE_DIR}/contrib/Implib.so)
+  set(IMPLIBSO_SRC
+    ${IMPLIBSO_SRC_DIR}/implib-gen.py
+    ${IMPLIBSO_SRC_DIR}/arch/common/init.c.tpl
+    ${IMPLIBSO_SRC_DIR}/arch/${CMAKE_SYSTEM_PROCESSOR}/config.ini
+    ${IMPLIBSO_SRC_DIR}/arch/${CMAKE_SYSTEM_PROCESSOR}/table.S.tpl
+    ${IMPLIBSO_SRC_DIR}/arch/${CMAKE_SYSTEM_PROCESSOR}/trampoline.S.tpl
+    )
 
   add_custom_command(OUTPUT ${SHIM_LIB_GEN_SRC}
-          COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_SOURCE_DIR}/contrib/Implib.so/implib-gen.py
-          --target ${CMAKE_SYSTEM_PROCESSOR}
-          --outdir ${SHIM_LIB_OUTPUT_DIR}
-          --dlopen-callback=fdb_shim_dlopen_callback
-          $<TARGET_FILE:fdb_c>)
+    COMMAND $<TARGET_FILE:Python3::Interpreter> ${IMPLIBSO_SRC_DIR}/implib-gen.py
+    --target ${CMAKE_SYSTEM_PROCESSOR}
+    --outdir ${SHIM_LIB_OUTPUT_DIR}
+    --dlopen-callback=fdb_shim_dlopen_callback
+    $<TARGET_FILE:fdb_c>
+    DEPENDS ${IMPLIBSO_SRC} fdb_c
+    COMMENT "Generating source code for C shim library")
 
-  add_library(fdb_c_shim SHARED ${SHIM_LIB_GEN_SRC} fdb_c_shim.cpp)
+  add_library(fdb_c_shim STATIC ${SHIM_LIB_GEN_SRC} foundationdb/fdb_c_shim.h fdb_c_shim.cpp)
   target_link_options(fdb_c_shim PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.map,-z,nodelete,-z,noexecstack")
   target_link_libraries(fdb_c_shim PUBLIC dl)
+  target_include_directories(fdb_c_shim PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/foundationdb>)
 
   add_executable(fdb_c_shim_unit_tests)
   target_link_libraries(fdb_c_shim_unit_tests PRIVATE fdb_c_shim fdb_c_unit_tests_impl)
@@ -465,15 +520,20 @@ elseif (NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE) # Linux Only
   add_executable(fdb_c_shim_api_tester)
   target_link_libraries(fdb_c_shim_api_tester PRIVATE fdb_c_shim fdb_c_api_tester_impl)
 
-  add_test(NAME fdb_c_shim_library_tests
-          COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/test/fdb_c_shim_tests.py
-          --build-dir ${CMAKE_BINARY_DIR}
-          --unit-tests-bin $<TARGET_FILE:fdb_c_shim_unit_tests>
-          --api-tester-bin $<TARGET_FILE:fdb_c_shim_api_tester>
-          --api-test-dir ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
-          )
+  add_executable(fdb_c_shim_lib_tester test/shim_lib_tester.cpp)
+  target_link_libraries(fdb_c_shim_lib_tester PRIVATE fdb_c_shim SimpleOpt fdb_cpp Threads::Threads)
+  target_include_directories(fdb_c_shim_lib_tester PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include)
 
-endif () # End Linux only
+  add_test(NAME fdb_c_shim_library_tests
+    COMMAND $<TARGET_FILE:Python3::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/test/fdb_c_shim_tests.py
+    --build-dir ${CMAKE_BINARY_DIR}
+    --unit-tests-bin $<TARGET_FILE:fdb_c_shim_unit_tests>
+    --api-tester-bin $<TARGET_FILE:fdb_c_shim_api_tester>
+    --shim-lib-tester-bin $<TARGET_FILE:fdb_c_shim_lib_tester>
+    --api-test-dir ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
+    )
+
+endif() # End Linux only, non-sanitizer only
 
 # TODO: re-enable once the old vcxproj-based build system is removed.
 #generate_export_header(fdb_c EXPORT_MACRO_NAME "DLLEXPORT"
@@ -485,35 +545,51 @@ set(version_config "${generated_dir}/${targets_export_name}ConfigVersion.cmake")
 set(project_config "${generated_dir}/${targets_export_name}Config.cmake")
 include(CMakePackageConfigHelpers)
 write_basic_package_version_file(
-        "${version_config}" VERSION ${GENERIC_LIB_VERSION} COMPATIBILITY AnyNewerVersion
+  "${version_config}" VERSION ${GENERIC_LIB_VERSION} COMPATIBILITY AnyNewerVersion
 )
 configure_file("${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in" "${project_config}" @ONLY)
 
 fdb_install(
-        TARGETS fdb_c
-        EXPORT ${targets_export_name}
-        DESTINATION lib
-        COMPONENT clients)
+  TARGETS fdb_c
+  EXPORT ${targets_export_name}
+  DESTINATION lib
+  COMPONENT clients)
 fdb_install(
-        FILES foundationdb/fdb_c.h
-        ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h
-        ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options
-        ${CMAKE_SOURCE_DIR}/bindings/c/foundationdb/fdb_c_types.h
-        DESTINATION include
-        DESTINATION_SUFFIX /foundationdb
-        COMPONENT clients)
+  FILES foundationdb/fdb_c.h
+  ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h
+  ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options
+  ${CMAKE_SOURCE_DIR}/bindings/c/foundationdb/fdb_c_types.h
+  DESTINATION include
+  DESTINATION_SUFFIX /foundationdb
+  COMPONENT clients)
 fdb_install(
-        FILES "${project_config}" "${version_config}"
-        DESTINATION lib
-        DESTINATION_SUFFIX "/cmake/${targets_export_name}"
-        COMPONENT clients)
+  FILES "${project_config}" "${version_config}"
+  DESTINATION lib
+  DESTINATION_SUFFIX "/cmake/${targets_export_name}"
+  COMPONENT clients)
 fdb_configure_and_install(
-        FILE "${PROJECT_SOURCE_DIR}/cmake/foundationdb-client.pc.in"
-        DESTINATION lib
-        DESTINATION_SUFFIX "/pkgconfig"
-        COMPONENT clients)
+  FILE "${PROJECT_SOURCE_DIR}/cmake/foundationdb-client.pc.in"
+  DESTINATION lib
+  DESTINATION_SUFFIX "/pkgconfig"
+  COMPONENT clients)
 fdb_install(
-        EXPORT ${targets_export_name}
-        DESTINATION lib
-        DESTINATION_SUFFIX "/cmake/${targets_export_name}"
-        COMPONENT clients)
+  EXPORT ${targets_export_name}
+  DESTINATION lib
+  DESTINATION_SUFFIX "/cmake/${targets_export_name}"
+  COMPONENT clients)
+
+if(NOT WIN32 AND NOT APPLE AND NOT USE_SANITIZER) # Linux Only, non-sanitizer only
+
+  fdb_install(
+    FILES foundationdb/fdb_c_shim.h
+    DESTINATION include
+    DESTINATION_SUFFIX /foundationdb
+    COMPONENT clients)
+
+  fdb_install(
+    TARGETS fdb_c_shim
+    EXPORT ${targets_export_name}
+    DESTINATION lib
+    COMPONENT clients)
+
+endif() # End Linux only, non-ubsan only
diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp
index c97604b98c..4b225ddd80 100644
--- a/bindings/c/fdb_c.cpp
+++ b/bindings/c/fdb_c.cpp
@@ -79,9 +79,10 @@ extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_erro
 	if (predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) {
 		return code == error_code_not_committed || code == error_code_transaction_too_old ||
 		       code == error_code_future_version || code == error_code_database_locked ||
-		       code == error_code_proxy_memory_limit_exceeded || code == error_code_batch_transaction_throttled ||
-		       code == error_code_process_behind || code == error_code_tag_throttled ||
-		       code == error_code_unknown_tenant;
+		       code == error_code_grv_proxy_memory_limit_exceeded ||
+		       code == error_code_commit_proxy_memory_limit_exceeded ||
+		       code == error_code_batch_transaction_throttled || code == error_code_process_behind ||
+		       code == error_code_tag_throttled || code == error_code_unknown_tenant;
 	}
 	return false;
 }
@@ -238,6 +239,10 @@ fdb_error_t fdb_future_get_version_v619(FDBFuture* f, int64_t* out_version) {
 	CATCH_AND_RETURN(*out_version = TSAV(Version, f)->get(););
 }
 
+extern "C" DLLEXPORT fdb_error_t fdb_future_get_bool(FDBFuture* f, fdb_bool_t* out_value) {
+	CATCH_AND_RETURN(*out_value = TSAV(bool, f)->get(););
+}
+
 extern "C" DLLEXPORT fdb_error_t fdb_future_get_int64(FDBFuture* f, int64_t* out_value) {
 	CATCH_AND_RETURN(*out_value = TSAV(int64_t, f)->get(););
 }
@@ -319,6 +324,15 @@ extern "C" DLLEXPORT fdb_error_t fdb_future_get_key_array(FDBFuture* f, FDBKey c
 	                 *out_count = na.size(););
 }
 
+extern "C" DLLEXPORT fdb_error_t fdb_future_get_granule_summary_array(FDBFuture* f,
+                                                                      FDBGranuleSummary const** out_ranges,
+                                                                      int* out_count) {
+	CATCH_AND_RETURN(Standalone<VectorRef<BlobGranuleSummaryRef>> na =
+	                     TSAV(Standalone<VectorRef<BlobGranuleSummaryRef>>, f)->get();
+	                 *out_ranges = (FDBGranuleSummary*)na.begin();
+	                 *out_count = na.size(););
+}
+
 extern "C" DLLEXPORT void fdb_result_destroy(FDBResult* r) {
 	CATCH_AND_DIE(TSAVB(r)->cancel(););
 }
@@ -493,6 +507,58 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_wait_purge_granules_complete(FDBDat
 	    FDBFuture*)(DB(db)->waitPurgeGranulesComplete(StringRef(purge_key_name, purge_key_name_length)).extractPtr());
 }
 
+extern "C" DLLEXPORT FDBFuture* fdb_database_blobbify_range(FDBDatabase* db,
+                                                            uint8_t const* begin_key_name,
+                                                            int begin_key_name_length,
+                                                            uint8_t const* end_key_name,
+                                                            int end_key_name_length) {
+	return (FDBFuture*)(DB(db)
+	                        ->blobbifyRange(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
+	                                                    StringRef(end_key_name, end_key_name_length)))
+	                        .extractPtr());
+}
+
+extern "C" DLLEXPORT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db,
+                                                              uint8_t const* begin_key_name,
+                                                              int begin_key_name_length,
+                                                              uint8_t const* end_key_name,
+                                                              int end_key_name_length) {
+	return (FDBFuture*)(DB(db)
+	                        ->unblobbifyRange(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
+	                                                      StringRef(end_key_name, end_key_name_length)))
+	                        .extractPtr());
+}
+
+extern "C" DLLEXPORT FDBFuture* fdb_database_list_blobbified_ranges(FDBDatabase* db,
+                                                                    uint8_t const* begin_key_name,
+                                                                    int begin_key_name_length,
+                                                                    uint8_t const* end_key_name,
+                                                                    int end_key_name_length,
+                                                                    int rangeLimit) {
+	return (FDBFuture*)(DB(db)
+	                        ->listBlobbifiedRanges(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
+	                                                           StringRef(end_key_name, end_key_name_length)),
+	                                               rangeLimit)
+	                        .extractPtr());
+}
+
+extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_verify_blob_range(FDBDatabase* db,
+                                                                                  uint8_t const* begin_key_name,
+                                                                                  int begin_key_name_length,
+                                                                                  uint8_t const* end_key_name,
+                                                                                  int end_key_name_length,
+                                                                                  int64_t version) {
+	Optional<Version> rv;
+	if (version != latestVersion) {
+		rv = version;
+	}
+	return (FDBFuture*)(DB(db)
+	                        ->verifyBlobRange(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
+	                                                      StringRef(end_key_name, end_key_name_length)),
+	                                          rv)
+	                        .extractPtr());
+}
+
 extern "C" DLLEXPORT fdb_error_t fdb_tenant_create_transaction(FDBTenant* tenant, FDBTransaction** out_transaction) {
 	CATCH_AND_RETURN(*out_transaction = (FDBTransaction*)TENANT(tenant)->createTransaction().extractPtr(););
 }
@@ -855,11 +921,12 @@ extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_blob_granule_ranges(FDBTrans
                                                                         uint8_t const* begin_key_name,
                                                                         int begin_key_name_length,
                                                                         uint8_t const* end_key_name,
-                                                                        int end_key_name_length) {
+                                                                        int end_key_name_length,
+                                                                        int rangeLimit) {
 	RETURN_FUTURE_ON_ERROR(
 	    Standalone<VectorRef<KeyRangeRef>>,
 	    KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
-	    return (FDBFuture*)(TXN(tr)->getBlobGranuleRanges(range).extractPtr()););
+	    return (FDBFuture*)(TXN(tr)->getBlobGranuleRanges(range, rangeLimit).extractPtr()););
 }
 
 extern "C" DLLEXPORT FDBResult* fdb_transaction_read_blob_granules(FDBTransaction* tr,
@@ -889,6 +956,74 @@ extern "C" DLLEXPORT FDBResult* fdb_transaction_read_blob_granules(FDBTransactio
 	    return (FDBResult*)(TXN(tr)->readBlobGranules(range, beginVersion, rv, context).extractPtr()););
 }
 
+extern "C" DLLEXPORT FDBFuture* fdb_transaction_read_blob_granules_start(FDBTransaction* tr,
+                                                                         uint8_t const* begin_key_name,
+                                                                         int begin_key_name_length,
+                                                                         uint8_t const* end_key_name,
+                                                                         int end_key_name_length,
+                                                                         int64_t beginVersion,
+                                                                         int64_t readVersion,
+                                                                         int64_t* readVersionOut) {
+	Optional<Version> rv;
+	if (readVersion != latestVersion) {
+		rv = readVersion;
+	}
+	return (FDBFuture*)(TXN(tr)
+	                        ->readBlobGranulesStart(KeyRangeRef(KeyRef(begin_key_name, begin_key_name_length),
+	                                                            KeyRef(end_key_name, end_key_name_length)),
+	                                                beginVersion,
+	                                                rv,
+	                                                readVersionOut)
+	                        .extractPtr());
+}
+
+extern "C" DLLEXPORT FDBResult* fdb_transaction_read_blob_granules_finish(FDBTransaction* tr,
+                                                                          FDBFuture* f,
+                                                                          uint8_t const* begin_key_name,
+                                                                          int begin_key_name_length,
+                                                                          uint8_t const* end_key_name,
+                                                                          int end_key_name_length,
+                                                                          int64_t beginVersion,
+                                                                          int64_t readVersion,
+                                                                          FDBReadBlobGranuleContext* granule_context) {
+	// FIXME: better way to convert?
+	ReadBlobGranuleContext context;
+	context.userContext = granule_context->userContext;
+	context.start_load_f = granule_context->start_load_f;
+	context.get_load_f = granule_context->get_load_f;
+	context.free_load_f = granule_context->free_load_f;
+	context.debugNoMaterialize = granule_context->debugNoMaterialize;
+	context.granuleParallelism = granule_context->granuleParallelism;
+	ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> startFuture(
+	    TSAV(Standalone<VectorRef<BlobGranuleChunkRef>>, f));
+
+	return (FDBResult*)(TXN(tr)
+	                        ->readBlobGranulesFinish(startFuture,
+	                                                 KeyRangeRef(KeyRef(begin_key_name, begin_key_name_length),
+	                                                             KeyRef(end_key_name, end_key_name_length)),
+	                                                 beginVersion,
+	                                                 readVersion,
+	                                                 context)
+	                        .extractPtr());
+}
+
+extern "C" DLLEXPORT FDBFuture* fdb_transaction_summarize_blob_granules(FDBTransaction* tr,
+                                                                        uint8_t const* begin_key_name,
+                                                                        int begin_key_name_length,
+                                                                        uint8_t const* end_key_name,
+                                                                        int end_key_name_length,
+                                                                        int64_t summaryVersion,
+                                                                        int rangeLimit) {
+	RETURN_FUTURE_ON_ERROR(
+	    Standalone<VectorRef<BlobGranuleSummaryRef>>,
+	    KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
+
+	    Optional<Version> sv;
+	    if (summaryVersion != latestVersion) { sv = summaryVersion; }
+
+	    return (FDBFuture*)(TXN(tr)->summarizeBlobGranules(range, sv, rangeLimit).extractPtr()););
+}
+
 #include "fdb_c_function_pointers.g.h"
 
 #define FDB_API_CHANGED(func, ver)                                                                                     \
@@ -964,6 +1099,10 @@ extern "C" DLLEXPORT const char* fdb_get_client_version() {
 	return API->getClientVersion();
 }
 
+extern "C" DLLEXPORT void fdb_use_future_protocol_version() {
+	API->useFutureProtocolVersion();
+}
+
 #if defined(__APPLE__)
 #include <dlfcn.h>
 __attribute__((constructor)) static void initialize() {
diff --git a/bindings/c/fdb_c_shim.cpp b/bindings/c/fdb_c_shim.cpp
index 01c6bf9e6c..855db0313f 100644
--- a/bindings/c/fdb_c_shim.cpp
+++ b/bindings/c/fdb_c_shim.cpp
@@ -20,25 +20,42 @@
 
 #if (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
 
+#define DLLEXPORT __attribute__((visibility("default")))
+
+#include "foundationdb/fdb_c_shim.h"
+
 #include <dlfcn.h>
-#include <stdio.h>
-#include <stdlib.h>
 #include <string>
 
-static const char* FDB_C_CLIENT_LIBRARY_PATH = "FDB_C_CLIENT_LIBRARY_PATH";
+namespace {
 
-// Callback that tries different library names
+const char* FDB_LOCAL_CLIENT_LIBRARY_PATH_ENVVAR = "FDB_LOCAL_CLIENT_LIBRARY_PATH";
+std::string g_fdbLocalClientLibraryPath;
+
+} // namespace
+
+extern "C" DLLEXPORT void fdb_shim_set_local_client_library_path(const char* filePath) {
+	g_fdbLocalClientLibraryPath = filePath;
+}
+
+/* The callback of the fdb_c_shim layer that determines the path
+   of the fdb_c library to be dynamically loaded
+ */
 extern "C" void* fdb_shim_dlopen_callback(const char* libName) {
 	std::string libPath;
-	char* val = getenv(FDB_C_CLIENT_LIBRARY_PATH);
-	if (val) {
-		libPath = val;
+	if (!g_fdbLocalClientLibraryPath.empty()) {
+		libPath = g_fdbLocalClientLibraryPath;
 	} else {
-		libPath = libName;
+		char* val = getenv(FDB_LOCAL_CLIENT_LIBRARY_PATH_ENVVAR);
+		if (val) {
+			libPath = val;
+		} else {
+			libPath = libName;
+		}
 	}
 	return dlopen(libPath.c_str(), RTLD_LAZY | RTLD_GLOBAL);
 }
 
 #else
 #error Port me!
-#endif
\ No newline at end of file
+#endif
diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h
index 8b874c7ddf..2e4e977d76 100644
--- a/bindings/c/foundationdb/fdb_c.h
+++ b/bindings/c/foundationdb/fdb_c.h
@@ -70,17 +70,26 @@ DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code);
 
 #define /* fdb_error_t */ fdb_select_api_version(v) fdb_select_api_version_impl(v, FDB_API_VERSION)
 
+/*
+ * A variant of fdb_select_api_version that caps the header API version by the maximum API version
+ * supported by the client library. It is intended mainly for use in combination with the shim
+ * layer, which loads the client library dynamically.
+ */
+#define /* fdb_error_t */ fdb_select_api_version_capped(v)                                                             \
+	fdb_select_api_version_impl(                                                                                       \
+	    v, FDB_API_VERSION < fdb_get_max_api_version() ? FDB_API_VERSION : fdb_get_max_api_version())
+
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_network_set_option(FDBNetworkOption option,
                                                                 uint8_t const* value,
                                                                 int value_length);
 
 #if FDB_API_VERSION >= 14
-DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_setup_network();
+DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_setup_network(void);
 #endif
 
-DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_run_network();
+DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_run_network(void);
 
-DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_stop_network();
+DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_stop_network(void);
 
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*),
                                                                                 void* hook_parameter);
@@ -170,6 +179,14 @@ typedef struct keyrange {
 	const uint8_t* end_key;
 	int end_key_length;
 } FDBKeyRange;
+
+typedef struct granulesummary {
+	FDBKeyRange key_range;
+	int64_t snapshot_version;
+	int64_t snapshot_size;
+	int64_t delta_version;
+	int64_t delta_size;
+} FDBGranuleSummary;
 #pragma pack(pop)
 
 typedef struct readgranulecontext {
@@ -218,6 +235,8 @@ DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_set_callback(FDBFuture* f,
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_error(FDBFuture* f);
 #endif
 
+DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_bool(FDBFuture* f, fdb_bool_t* out);
+
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_int64(FDBFuture* f, int64_t* out);
 
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_uint64(FDBFuture* f, uint64_t* out);
@@ -253,6 +272,10 @@ DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_keyrange_array(FDBFuture
                                                                        FDBKeyRange const** out_ranges,
                                                                        int* out_count);
 
+DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_granule_summary_array(FDBFuture* f,
+                                                                              FDBGranuleSummary const** out_summaries,
+                                                                              int* out_count);
+
 /* FDBResult is a synchronous computation result, as opposed to a future that is asynchronous. */
 DLLEXPORT void fdb_result_destroy(FDBResult* r);
 
@@ -312,6 +335,32 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_wait_purge_granules_complet
                                                                                   uint8_t const* purge_key_name,
                                                                                   int purge_key_name_length);
 
+DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_blobbify_range(FDBDatabase* db,
+                                                                    uint8_t const* begin_key_name,
+                                                                    int begin_key_name_length,
+                                                                    uint8_t const* end_key_name,
+                                                                    int end_key_name_length);
+
+DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db,
+                                                                      uint8_t const* begin_key_name,
+                                                                      int begin_key_name_length,
+                                                                      uint8_t const* end_key_name,
+                                                                      int end_key_name_length);
+
+DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_list_blobbified_ranges(FDBDatabase* db,
+                                                                            uint8_t const* begin_key_name,
+                                                                            int begin_key_name_length,
+                                                                            uint8_t const* end_key_name,
+                                                                            int end_key_name_length,
+                                                                            int rangeLimit);
+
+DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_verify_blob_range(FDBDatabase* db,
+                                                                       uint8_t const* begin_key_name,
+                                                                       int begin_key_name_length,
+                                                                       uint8_t const* end_key_name,
+                                                                       int end_key_name_length,
+                                                                       int64_t version);
+
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_tenant_create_transaction(FDBTenant* tenant,
                                                                        FDBTransaction** out_transaction);
 
@@ -470,7 +519,8 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_blob_granule_ranges(
                                                                                 uint8_t const* begin_key_name,
                                                                                 int begin_key_name_length,
                                                                                 uint8_t const* end_key_name,
-                                                                                int end_key_name_length);
+                                                                                int end_key_name_length,
+                                                                                int rangeLimit);
 
 /* LatestVersion (-2) for readVersion means get read version from transaction
    Separated out as optional because BG reads can support longer-lived reads than normal FDB transactions */
@@ -483,6 +533,14 @@ DLLEXPORT WARN_UNUSED_RESULT FDBResult* fdb_transaction_read_blob_granules(FDBTr
                                                                            int64_t readVersion,
                                                                            FDBReadBlobGranuleContext granuleContext);
 
+DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_summarize_blob_granules(FDBTransaction* tr,
+                                                                                uint8_t const* begin_key_name,
+                                                                                int begin_key_name_length,
+                                                                                uint8_t const* end_key_name,
+                                                                                int end_key_name_length,
+                                                                                int64_t summaryVersion,
+                                                                                int rangeLimit);
+
 #define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0
 #define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0
 #define FDB_KEYSEL_FIRST_GREATER_THAN(k, l) k, l, 1, 1
@@ -490,8 +548,8 @@ DLLEXPORT WARN_UNUSED_RESULT FDBResult* fdb_transaction_read_blob_granules(FDBTr
 
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_select_api_version_impl(int runtime_version, int header_version);
 
-DLLEXPORT int fdb_get_max_api_version();
-DLLEXPORT const char* fdb_get_client_version();
+DLLEXPORT int fdb_get_max_api_version(void);
+DLLEXPORT const char* fdb_get_client_version(void);
 
 /* LEGACY API VERSIONS */
 
diff --git a/bindings/c/foundationdb/fdb_c_internal.h b/bindings/c/foundationdb/fdb_c_internal.h
index 2b1a2163c7..62b77f354e 100644
--- a/bindings/c/foundationdb/fdb_c_internal.h
+++ b/bindings/c/foundationdb/fdb_c_internal.h
@@ -49,6 +49,29 @@ DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_shared_state(FDBFuture*
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_create_database_from_connection_string(const char* connection_string,
                                                                                     FDBDatabase** out_database);
 
+DLLEXPORT void fdb_use_future_protocol_version();
+
+// the logical read_blob_granules is broken out (at different points depending on the client type) into the asynchronous
+// start() that happens on the fdb network thread, and synchronous finish() that happens off it
+DLLEXPORT FDBFuture* fdb_transaction_read_blob_granules_start(FDBTransaction* tr,
+                                                              uint8_t const* begin_key_name,
+                                                              int begin_key_name_length,
+                                                              uint8_t const* end_key_name,
+                                                              int end_key_name_length,
+                                                              int64_t beginVersion,
+                                                              int64_t readVersion,
+                                                              int64_t* readVersionOut);
+
+DLLEXPORT FDBResult* fdb_transaction_read_blob_granules_finish(FDBTransaction* tr,
+                                                               FDBFuture* f,
+                                                               uint8_t const* begin_key_name,
+                                                               int begin_key_name_length,
+                                                               uint8_t const* end_key_name,
+                                                               int end_key_name_length,
+                                                               int64_t beginVersion,
+                                                               int64_t readVersion,
+                                                               FDBReadBlobGranuleContext* granuleContext);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/bindings/c/foundationdb/fdb_c_shim.h b/bindings/c/foundationdb/fdb_c_shim.h
new file mode 100644
index 0000000000..44dfdc7348
--- /dev/null
+++ b/bindings/c/foundationdb/fdb_c_shim.h
@@ -0,0 +1,47 @@
+/*
+ * fdb_shim_c.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDB_SHIM_C_H
+#define FDB_SHIM_C_H
+#pragma once
+
+#ifndef DLLEXPORT
+#define DLLEXPORT
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Specify the path of the local libfdb_c.so library to be dynamically loaded by the shim layer
+ *
+ * This enables running the same application code with different client library versions,
+ * e.g. using the latest development build for testing new features, but still using the latest
+ * stable release in production deployments.
+ *
+ * The given path overrides the environment variable FDB_LOCAL_CLIENT_LIBRARY_PATH
+ */
+DLLEXPORT void fdb_shim_set_local_client_library_path(const char* filePath);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/bindings/c/test/apitester/TesterApiWorkload.cpp b/bindings/c/test/apitester/TesterApiWorkload.cpp
index c1499adb0c..90d4e18bd2 100644
--- a/bindings/c/test/apitester/TesterApiWorkload.cpp
+++ b/bindings/c/test/apitester/TesterApiWorkload.cpp
@@ -41,6 +41,10 @@ ApiWorkload::ApiWorkload(const WorkloadConfig& config) : WorkloadBase(config) {
 	stopReceived = false;
 	checkingProgress = false;
 	apiVersion = config.apiVersion;
+
+	for (int i = 0; i < config.numTenants; ++i) {
+		tenants.push_back(fdb::ByteString(fdb::toBytesRef("tenant" + std::to_string(i))));
+	}
 }
 
 IWorkloadControlIfc* ApiWorkload::getControlIfc() {
@@ -107,49 +111,57 @@ void ApiWorkload::randomOperation(TTaskFct cont) {
 }
 
 fdb::Key ApiWorkload::randomKeyName() {
-	return keyPrefix + Random::get().randomStringLowerCase(minKeyLength, maxKeyLength);
+	return keyPrefix + Random::get().randomByteStringLowerCase(minKeyLength, maxKeyLength);
 }
 
 fdb::Value ApiWorkload::randomValue() {
-	return Random::get().randomStringLowerCase(minValueLength, maxValueLength);
+	return Random::get().randomByteStringLowerCase(minValueLength, maxValueLength);
 }
 
-fdb::Key ApiWorkload::randomNotExistingKey() {
+fdb::Key ApiWorkload::randomNotExistingKey(std::optional<int> tenantId) {
 	while (true) {
 		fdb::Key key = randomKeyName();
-		if (!store.exists(key)) {
+		if (!stores[tenantId].exists(key)) {
 			return key;
 		}
 	}
 }
 
-fdb::Key ApiWorkload::randomExistingKey() {
+fdb::Key ApiWorkload::randomExistingKey(std::optional<int> tenantId) {
 	fdb::Key genKey = randomKeyName();
-	fdb::Key key = store.getKey(genKey, true, 1);
-	if (key != store.endKey()) {
+	fdb::Key key = stores[tenantId].getKey(genKey, true, 1);
+	if (key != stores[tenantId].endKey()) {
 		return key;
 	}
-	key = store.getKey(genKey, true, 0);
-	if (key != store.startKey()) {
+	key = stores[tenantId].getKey(genKey, true, 0);
+	if (key != stores[tenantId].startKey()) {
 		return key;
 	}
 	info("No existing key found, using a new random key.");
 	return genKey;
 }
 
-fdb::Key ApiWorkload::randomKey(double existingKeyRatio) {
+fdb::Key ApiWorkload::randomKey(double existingKeyRatio, std::optional<int> tenantId) {
 	if (Random::get().randomBool(existingKeyRatio)) {
-		return randomExistingKey();
+		return randomExistingKey(tenantId);
 	} else {
-		return randomNotExistingKey();
+		return randomNotExistingKey(tenantId);
 	}
 }
 
-void ApiWorkload::populateDataTx(TTaskFct cont) {
+std::optional<int> ApiWorkload::randomTenant() {
+	if (tenants.size() > 0) {
+		return Random::get().randomInt(0, tenants.size() - 1);
+	} else {
+		return {};
+	}
+}
+
+void ApiWorkload::populateDataTx(TTaskFct cont, std::optional<int> tenantId) {
 	int numKeys = maxKeysPerTransaction;
 	auto kvPairs = std::make_shared<std::vector<fdb::KeyValue>>();
 	for (int i = 0; i < numKeys; i++) {
-		kvPairs->push_back(fdb::KeyValue{ randomNotExistingKey(), randomValue() });
+		kvPairs->push_back(fdb::KeyValue{ randomNotExistingKey(tenantId), randomValue() });
 	}
 	execTransaction(
 	    [kvPairs](auto ctx) {
@@ -158,12 +170,29 @@ void ApiWorkload::populateDataTx(TTaskFct cont) {
 		    }
 		    ctx->commit();
 	    },
-	    [this, kvPairs, cont]() {
+	    [this, tenantId, kvPairs, cont]() {
 		    for (const fdb::KeyValue& kv : *kvPairs) {
-			    store.set(kv.key, kv.value);
+			    stores[tenantId].set(kv.key, kv.value);
 		    }
 		    schedule(cont);
-	    });
+	    },
+	    getTenant(tenantId));
+}
+
+void ApiWorkload::clearTenantData(TTaskFct cont, std::optional<int> tenantId) {
+	execTransaction(
+	    [this](auto ctx) {
+		    ctx->tx().clearRange(keyPrefix, keyPrefix + fdb::Key(1, '\xff'));
+		    ctx->commit();
+	    },
+	    [this, tenantId, cont]() {
+		    if (tenantId && tenantId.value() < tenants.size() - 1) {
+			    clearTenantData(cont, tenantId.value() + 1);
+		    } else {
+			    schedule(cont);
+		    }
+	    },
+	    getTenant(tenantId));
 }
 
 void ApiWorkload::clearData(TTaskFct cont) {
@@ -175,20 +204,51 @@ void ApiWorkload::clearData(TTaskFct cont) {
 	    [this, cont]() { schedule(cont); });
 }
 
-void ApiWorkload::populateData(TTaskFct cont) {
-	if (store.size() < initialSize) {
-		populateDataTx([this, cont]() { populateData(cont); });
-	} else {
+void ApiWorkload::populateTenantData(TTaskFct cont, std::optional<int> tenantId) {
+	while (stores[tenantId].size() >= initialSize && tenantId && tenantId.value() < tenants.size()) {
+		++tenantId.value();
+	}
+
+	if (tenantId >= tenants.size() || stores[tenantId].size() >= initialSize) {
 		info("Data population completed");
 		schedule(cont);
+	} else {
+		populateDataTx([this, cont, tenantId]() { populateTenantData(cont, tenantId); }, tenantId);
 	}
 }
 
-void ApiWorkload::randomInsertOp(TTaskFct cont) {
+void ApiWorkload::createTenants(TTaskFct cont) {
+	execTransaction(
+	    [this](auto ctx) {
+		    auto futures = std::make_shared<std::vector<fdb::Future>>();
+		    for (auto tenant : tenants) {
+			    futures->push_back(fdb::Tenant::getTenant(ctx->tx(), tenant));
+		    }
+		    ctx->continueAfterAll(*futures, [this, ctx, futures]() {
+			    for (int i = 0; i < futures->size(); ++i) {
+				    if (!(*futures)[i].get<fdb::future_var::ValueRef>()) {
+					    fdb::Tenant::createTenant(ctx->tx(), tenants[i]);
+				    }
+			    }
+			    ctx->commit();
+		    });
+	    },
+	    [this, cont]() { schedule(cont); });
+}
+
+void ApiWorkload::populateData(TTaskFct cont) {
+	if (tenants.size() > 0) {
+		createTenants([this, cont]() { populateTenantData(cont, std::make_optional(0)); });
+	} else {
+		populateTenantData(cont, {});
+	}
+}
+
+void ApiWorkload::randomInsertOp(TTaskFct cont, std::optional<int> tenantId) {
 	int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
 	auto kvPairs = std::make_shared<std::vector<fdb::KeyValue>>();
 	for (int i = 0; i < numKeys; i++) {
-		kvPairs->push_back(fdb::KeyValue{ randomNotExistingKey(), randomValue() });
+		kvPairs->push_back(fdb::KeyValue{ randomNotExistingKey(tenantId), randomValue() });
 	}
 	execTransaction(
 	    [kvPairs](auto ctx) {
@@ -197,19 +257,20 @@ void ApiWorkload::randomInsertOp(TTaskFct cont) {
 		    }
 		    ctx->commit();
 	    },
-	    [this, kvPairs, cont]() {
+	    [this, kvPairs, cont, tenantId]() {
 		    for (const fdb::KeyValue& kv : *kvPairs) {
-			    store.set(kv.key, kv.value);
+			    stores[tenantId].set(kv.key, kv.value);
 		    }
 		    schedule(cont);
-	    });
+	    },
+	    getTenant(tenantId));
 }
 
-void ApiWorkload::randomClearOp(TTaskFct cont) {
+void ApiWorkload::randomClearOp(TTaskFct cont, std::optional<int> tenantId) {
 	int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
 	auto keys = std::make_shared<std::vector<fdb::Key>>();
 	for (int i = 0; i < numKeys; i++) {
-		keys->push_back(randomExistingKey());
+		keys->push_back(randomExistingKey(tenantId));
 	}
 	execTransaction(
 	    [keys](auto ctx) {
@@ -218,15 +279,16 @@ void ApiWorkload::randomClearOp(TTaskFct cont) {
 		    }
 		    ctx->commit();
 	    },
-	    [this, keys, cont]() {
+	    [this, keys, cont, tenantId]() {
 		    for (const auto& key : *keys) {
-			    store.clear(key);
+			    stores[tenantId].clear(key);
 		    }
 		    schedule(cont);
-	    });
+	    },
+	    getTenant(tenantId));
 }
 
-void ApiWorkload::randomClearRangeOp(TTaskFct cont) {
+void ApiWorkload::randomClearRangeOp(TTaskFct cont, std::optional<int> tenantId) {
 	fdb::Key begin = randomKeyName();
 	fdb::Key end = randomKeyName();
 	if (begin > end) {
@@ -237,10 +299,19 @@ void ApiWorkload::randomClearRangeOp(TTaskFct cont) {
 		    ctx->tx().clearRange(begin, end);
 		    ctx->commit();
 	    },
-	    [this, begin, end, cont]() {
-		    store.clear(begin, end);
+	    [this, begin, end, cont, tenantId]() {
+		    stores[tenantId].clear(begin, end);
 		    schedule(cont);
-	    });
+	    },
+	    getTenant(tenantId));
+}
+
+std::optional<fdb::BytesRef> ApiWorkload::getTenant(std::optional<int> tenantId) {
+	if (tenantId) {
+		return tenants[*tenantId];
+	} else {
+		return {};
+	}
 }
 
 } // namespace FdbApiTester
diff --git a/bindings/c/test/apitester/TesterApiWorkload.h b/bindings/c/test/apitester/TesterApiWorkload.h
index fd3630ceee..a3a13e964d 100644
--- a/bindings/c/test/apitester/TesterApiWorkload.h
+++ b/bindings/c/test/apitester/TesterApiWorkload.h
@@ -96,17 +96,23 @@ protected:
 	// Key prefix
 	fdb::Key keyPrefix;
 
+	// The number of tenants to configure in the cluster
+	std::vector<fdb::ByteString> tenants;
+
 	// In-memory store maintaining expected database state
-	KeyValueStore store;
+	std::unordered_map<std::optional<int>, KeyValueStore> stores;
 
 	ApiWorkload(const WorkloadConfig& config);
 
 	// Methods for generating random keys and values
 	fdb::Key randomKeyName();
 	fdb::Value randomValue();
-	fdb::Key randomNotExistingKey();
-	fdb::Key randomExistingKey();
-	fdb::Key randomKey(double existingKeyRatio);
+	fdb::Key randomNotExistingKey(std::optional<int> tenantId);
+	fdb::Key randomExistingKey(std::optional<int> tenantId);
+	fdb::Key randomKey(double existingKeyRatio, std::optional<int> tenantId);
+
+	// Chooses a random tenant from the available tenants (or an empty optional if tenants aren't used in the test)
+	std::optional<int> randomTenant();
 
 	// Generate initial random data for the workload
 	void populateData(TTaskFct cont);
@@ -115,12 +121,18 @@ protected:
 	void clearData(TTaskFct cont);
 
 	// common operations
-	void randomInsertOp(TTaskFct cont);
-	void randomClearOp(TTaskFct cont);
-	void randomClearRangeOp(TTaskFct cont);
+	void randomInsertOp(TTaskFct cont, std::optional<int> tenantId);
+	void randomClearOp(TTaskFct cont, std::optional<int> tenantId);
+	void randomClearRangeOp(TTaskFct cont, std::optional<int> tenantId);
+
+	std::optional<fdb::BytesRef> getTenant(std::optional<int> tenantId);
 
 private:
-	void populateDataTx(TTaskFct cont);
+	void populateDataTx(TTaskFct cont, std::optional<int> tenantId);
+	void populateTenantData(TTaskFct cont, std::optional<int> tenantId);
+	void createTenants(TTaskFct cont);
+
+	void clearTenantData(TTaskFct cont, std::optional<int> tenantId);
 
 	void randomOperations();
 };
diff --git a/bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp b/bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp
index 52d8ddc651..f6164296da 100644
--- a/bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp
+++ b/bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp
@@ -18,61 +18,13 @@
  * limitations under the License.
  */
 #include "TesterApiWorkload.h"
+#include "TesterBlobGranuleUtil.h"
 #include "TesterUtil.h"
 #include <memory>
 #include <fmt/format.h>
 
 namespace FdbApiTester {
 
-class TesterGranuleContext {
-public:
-	std::unordered_map<int64_t, uint8_t*> loadsInProgress;
-	int64_t nextId = 0;
-	std::string basePath;
-
-	~TesterGranuleContext() {
-		// if there was an error or not all loads finished, delete data
-		for (auto& it : loadsInProgress) {
-			uint8_t* dataToFree = it.second;
-			delete[] dataToFree;
-		}
-	}
-};
-
-static int64_t granule_start_load(const char* filename,
-                                  int filenameLength,
-                                  int64_t offset,
-                                  int64_t length,
-                                  int64_t fullFileLength,
-                                  void* context) {
-
-	TesterGranuleContext* ctx = (TesterGranuleContext*)context;
-	int64_t loadId = ctx->nextId++;
-
-	uint8_t* buffer = new uint8_t[length];
-	std::ifstream fin(ctx->basePath + std::string(filename, filenameLength), std::ios::in | std::ios::binary);
-	fin.seekg(offset);
-	fin.read((char*)buffer, length);
-
-	ctx->loadsInProgress.insert({ loadId, buffer });
-
-	return loadId;
-}
-
-static uint8_t* granule_get_load(int64_t loadId, void* context) {
-	TesterGranuleContext* ctx = (TesterGranuleContext*)context;
-	return ctx->loadsInProgress.at(loadId);
-}
-
-static void granule_free_load(int64_t loadId, void* context) {
-	TesterGranuleContext* ctx = (TesterGranuleContext*)context;
-	auto it = ctx->loadsInProgress.find(loadId);
-	uint8_t* dataToFree = it->second;
-	delete[] dataToFree;
-
-	ctx->loadsInProgress.erase(it);
-}
-
 class ApiBlobGranuleCorrectnessWorkload : public ApiWorkload {
 public:
 	ApiBlobGranuleCorrectnessWorkload(const WorkloadConfig& config) : ApiWorkload(config) {
@@ -83,34 +35,39 @@ public:
 	}
 
 private:
-	enum OpType { OP_INSERT, OP_CLEAR, OP_CLEAR_RANGE, OP_READ, OP_GET_RANGES, OP_LAST = OP_GET_RANGES };
+	// FIXME: use other new blob granule apis!
+	enum OpType {
+		OP_INSERT,
+		OP_CLEAR,
+		OP_CLEAR_RANGE,
+		OP_READ,
+		OP_GET_GRANULES,
+		OP_SUMMARIZE,
+		OP_GET_BLOB_RANGES,
+		OP_VERIFY,
+		OP_LAST = OP_VERIFY
+	};
 	std::vector<OpType> excludedOpTypes;
 
 	// Allow reads at the start to get blob_granule_transaction_too_old if BG data isn't initialized yet
 	// FIXME: should still guarantee a read succeeds eventually somehow
 	bool seenReadSuccess = false;
 
-	void randomReadOp(TTaskFct cont) {
+	void randomReadOp(TTaskFct cont, std::optional<int> tenantId) {
 		fdb::Key begin = randomKeyName();
 		fdb::Key end = randomKeyName();
-		auto results = std::make_shared<std::vector<fdb::KeyValue>>();
-		auto tooOld = std::make_shared<bool>(false);
 		if (begin > end) {
 			std::swap(begin, end);
 		}
+
+		auto results = std::make_shared<std::vector<fdb::KeyValue>>();
+		auto tooOld = std::make_shared<bool>(false);
+
 		execTransaction(
 		    [this, begin, end, results, tooOld](auto ctx) {
 			    ctx->tx().setOption(FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE);
-			    TesterGranuleContext testerContext;
-			    testerContext.basePath = ctx->getBGBasePath();
-
-			    fdb::native::FDBReadBlobGranuleContext granuleContext;
-			    granuleContext.userContext = &testerContext;
-			    granuleContext.debugNoMaterialize = false;
-			    granuleContext.granuleParallelism = 1;
-			    granuleContext.start_load_f = &granule_start_load;
-			    granuleContext.get_load_f = &granule_get_load;
-			    granuleContext.free_load_f = &granule_free_load;
+			    TesterGranuleContext testerContext(ctx->getBGBasePath());
+			    fdb::native::FDBReadBlobGranuleContext granuleContext = createGranuleContext(&testerContext);
 
 			    fdb::Result res = ctx->tx().readBlobGranules(
 			        begin, end, 0 /* beginVersion */, -2 /* latest read version */, granuleContext);
@@ -124,8 +81,10 @@ private:
 			    } else if (err.code() != error_code_success) {
 				    ctx->onError(err);
 			    } else {
-				    auto& [out_kv, out_count, out_more] = out;
+				    auto resCopy = copyKeyValueArray(out);
+				    auto& [resVector, out_more] = resCopy;
 				    ASSERT(!out_more);
+				    results.get()->assign(resVector.begin(), resVector.end());
 				    if (!seenReadSuccess) {
 					    info("BlobGranuleCorrectness::randomReadOp first success\n");
 				    }
@@ -133,9 +92,10 @@ private:
 				    ctx->done();
 			    }
 		    },
-		    [this, begin, end, results, tooOld, cont]() {
+		    [this, begin, end, results, tooOld, cont, tenantId]() {
 			    if (!*tooOld) {
-				    std::vector<fdb::KeyValue> expected = store.getRange(begin, end, store.size(), false);
+				    std::vector<fdb::KeyValue> expected =
+				        stores[tenantId].getRange(begin, end, stores[tenantId].size(), false);
 				    if (results->size() != expected.size()) {
 					    error(fmt::format("randomReadOp result size mismatch. expected: {} actual: {}",
 					                      expected.size(),
@@ -166,19 +126,21 @@ private:
 				    }
 			    }
 			    schedule(cont);
-		    });
+		    },
+		    getTenant(tenantId));
 	}
 
-	void randomGetRangesOp(TTaskFct cont) {
+	void randomGetGranulesOp(TTaskFct cont, std::optional<int> tenantId) {
 		fdb::Key begin = randomKeyName();
 		fdb::Key end = randomKeyName();
-		auto results = std::make_shared<std::vector<fdb::KeyRange>>();
 		if (begin > end) {
 			std::swap(begin, end);
 		}
+		auto results = std::make_shared<std::vector<fdb::KeyRange>>();
+
 		execTransaction(
 		    [begin, end, results](auto ctx) {
-			    fdb::Future f = ctx->tx().getBlobGranuleRanges(begin, end).eraseType();
+			    fdb::Future f = ctx->tx().getBlobGranuleRanges(begin, end, 1000).eraseType();
 			    ctx->continueAfter(
 			        f,
 			        [ctx, f, results]() {
@@ -188,46 +150,180 @@ private:
 			        true);
 		    },
 		    [this, begin, end, results, cont]() {
-			    if (seenReadSuccess) {
-				    ASSERT(results->size() > 0);
-				    ASSERT(results->front().beginKey <= begin);
-				    ASSERT(results->back().endKey >= end);
-			    }
+			    this->validateRanges(results, begin, end, seenReadSuccess);
+			    schedule(cont);
+		    },
+		    getTenant(tenantId));
+	}
+
+	void randomSummarizeOp(TTaskFct cont, std::optional<int> tenantId) {
+		if (!seenReadSuccess) {
+			// tester can't handle this throwing bg_txn_too_old, so just don't call it unless we have already seen a
+			// read success
+			schedule(cont);
+			return;
+		}
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		auto results = std::make_shared<std::vector<fdb::GranuleSummary>>();
+		execTransaction(
+		    [begin, end, results](auto ctx) {
+			    fdb::Future f = ctx->tx().summarizeBlobGranules(begin, end, -2 /*latest version*/, 1000).eraseType();
+			    ctx->continueAfter(
+			        f,
+			        [ctx, f, results]() {
+				        *results = copyGranuleSummaryArray(f.get<fdb::future_var::GranuleSummaryRefArray>());
+				        ctx->done();
+			        },
+			        true);
+		    },
+		    [this, begin, end, results, cont]() {
+			    ASSERT(results->size() > 0);
+			    ASSERT(results->front().keyRange.beginKey <= begin);
+			    ASSERT(results->back().keyRange.endKey >= end);
 
 			    for (int i = 0; i < results->size(); i++) {
-				    // no empty or inverted ranges
-				    ASSERT((*results)[i].beginKey < (*results)[i].endKey);
+				    // TODO: could do validation of subsequent calls and ensure snapshot version never decreases
+				    ASSERT((*results)[i].keyRange.beginKey < (*results)[i].keyRange.endKey);
+				    ASSERT((*results)[i].snapshotVersion <= (*results)[i].deltaVersion);
+				    ASSERT((*results)[i].snapshotSize > 0);
+				    ASSERT((*results)[i].deltaSize >= 0);
 			    }
 
 			    for (int i = 1; i < results->size(); i++) {
 				    // ranges contain entire requested key range
-				    ASSERT((*results)[i].beginKey == (*results)[i - 1].endKey);
+				    ASSERT((*results)[i].keyRange.beginKey == (*results)[i - 1].keyRange.endKey);
 			    }
 
 			    schedule(cont);
-		    });
+		    },
+		    getTenant(tenantId));
+	}
+
+	void validateRanges(std::shared_ptr<std::vector<fdb::KeyRange>> results,
+	                    fdb::Key begin,
+	                    fdb::Key end,
+	                    bool shouldBeRanges) {
+		if (shouldBeRanges) {
+			ASSERT(results->size() > 0);
+			ASSERT(results->front().beginKey <= begin);
+			ASSERT(results->back().endKey >= end);
+		}
+		for (int i = 0; i < results->size(); i++) {
+			// no empty or inverted ranges
+			if ((*results)[i].beginKey >= (*results)[i].endKey) {
+				error(fmt::format("Empty/inverted range [{0} - {1}) for getBlobGranuleRanges({2} - {3})",
+				                  fdb::toCharsRef((*results)[i].beginKey),
+				                  fdb::toCharsRef((*results)[i].endKey),
+				                  fdb::toCharsRef(begin),
+				                  fdb::toCharsRef(end)));
+			}
+			ASSERT((*results)[i].beginKey < (*results)[i].endKey);
+		}
+
+		for (int i = 1; i < results->size(); i++) {
+			// ranges contain entire requested key range
+			if ((*results)[i].beginKey != (*results)[i].endKey) {
+				error(fmt::format("Non-contiguous range [{0} - {1}) for getBlobGranuleRanges({2} - {3})",
+				                  fdb::toCharsRef((*results)[i].beginKey),
+				                  fdb::toCharsRef((*results)[i].endKey),
+				                  fdb::toCharsRef(begin),
+				                  fdb::toCharsRef(end)));
+			}
+			ASSERT((*results)[i].beginKey == (*results)[i - 1].endKey);
+		}
+	}
+
+	void randomGetBlobRangesOp(TTaskFct cont) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		auto results = std::make_shared<std::vector<fdb::KeyRange>>();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execOperation(
+		    [begin, end, results](auto ctx) {
+			    fdb::Future f = ctx->db().listBlobbifiedRanges(begin, end, 1000).eraseType();
+			    ctx->continueAfter(f, [ctx, f, results]() {
+				    *results = copyKeyRangeArray(f.get<fdb::future_var::KeyRangeRefArray>());
+				    ctx->done();
+			    });
+		    },
+		    [this, begin, end, results, cont]() {
+			    this->validateRanges(results, begin, end, seenReadSuccess);
+			    schedule(cont);
+		    },
+		    /* failOnError = */ false);
+	}
+
+	void randomVerifyOp(TTaskFct cont) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+
+		auto verifyVersion = std::make_shared<int64_t>(false);
+		// info("Verify op starting");
+
+		execOperation(
+		    [begin, end, verifyVersion](auto ctx) {
+			    fdb::Future f = ctx->db().verifyBlobRange(begin, end, -2 /* latest version*/).eraseType();
+			    ctx->continueAfter(f, [ctx, verifyVersion, f]() {
+				    *verifyVersion = f.get<fdb::future_var::Int64>();
+				    ctx->done();
+			    });
+		    },
+		    [this, begin, end, verifyVersion, cont]() {
+			    if (*verifyVersion == -1) {
+				    ASSERT(!seenReadSuccess);
+			    } else {
+				    if (!seenReadSuccess) {
+					    info("BlobGranuleCorrectness::randomVerifyOp first success");
+				    }
+				    seenReadSuccess = true;
+			    }
+			    // info(fmt::format("verify op done @ {}", *verifyVersion));
+			    schedule(cont);
+		    },
+		    /* failOnError = */ false);
 	}
 
 	void randomOperation(TTaskFct cont) {
-		OpType txType = (store.size() == 0) ? OP_INSERT : (OpType)Random::get().randomInt(0, OP_LAST);
+		std::optional<int> tenantId = randomTenant();
+
+		OpType txType = (stores[tenantId].size() == 0) ? OP_INSERT : (OpType)Random::get().randomInt(0, OP_LAST);
 		while (std::count(excludedOpTypes.begin(), excludedOpTypes.end(), txType)) {
 			txType = (OpType)Random::get().randomInt(0, OP_LAST);
 		}
+
 		switch (txType) {
 		case OP_INSERT:
-			randomInsertOp(cont);
+			randomInsertOp(cont, tenantId);
 			break;
 		case OP_CLEAR:
-			randomClearOp(cont);
+			randomClearOp(cont, tenantId);
 			break;
 		case OP_CLEAR_RANGE:
-			randomClearRangeOp(cont);
+			randomClearRangeOp(cont, tenantId);
 			break;
 		case OP_READ:
-			randomReadOp(cont);
+			randomReadOp(cont, tenantId);
 			break;
-		case OP_GET_RANGES:
-			randomGetRangesOp(cont);
+		case OP_GET_GRANULES:
+			randomGetGranulesOp(cont, tenantId);
+			break;
+		case OP_SUMMARIZE:
+			randomSummarizeOp(cont, tenantId);
+			break;
+		case OP_GET_BLOB_RANGES:
+			randomGetBlobRangesOp(cont);
+			break;
+		case OP_VERIFY:
+			randomVerifyOp(cont);
 			break;
 		}
 	}
diff --git a/bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp b/bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp
new file mode 100644
index 0000000000..b4bcaacdc6
--- /dev/null
+++ b/bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp
@@ -0,0 +1,316 @@
+/*
+ * TesterBlobGranuleErrorsWorkload.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "TesterApiWorkload.h"
+#include "TesterBlobGranuleUtil.h"
+#include "TesterUtil.h"
+#include <memory>
+#include <fmt/format.h>
+
+namespace FdbApiTester {
+
+class BlobGranuleErrorsWorkload : public ApiWorkload {
+public:
+	BlobGranuleErrorsWorkload(const WorkloadConfig& config) : ApiWorkload(config) {}
+
+private:
+	enum OpType {
+		OP_READ_NO_MATERIALIZE,
+		OP_READ_FILE_LOAD_ERROR,
+		OP_READ_TOO_OLD,
+		OP_PURGE_UNALIGNED,
+		OP_BLOBBIFY_UNALIGNED,
+		OP_UNBLOBBIFY_UNALIGNED,
+		OP_CANCEL_GET_GRANULES,
+		OP_CANCEL_GET_RANGES,
+		OP_CANCEL_VERIFY,
+		OP_CANCEL_SUMMARIZE,
+		OP_CANCEL_BLOBBIFY,
+		OP_CANCEL_UNBLOBBIFY,
+		OP_CANCEL_PURGE,
+		OP_LAST = OP_CANCEL_PURGE
+	};
+
+	// could add summarize too old and verify too old as ops if desired but those are lower value
+
+	// Allow reads at the start to get blob_granule_transaction_too_old if BG data isn't initialized yet
+	// FIXME: should still guarantee a read succeeds eventually somehow
+	bool seenReadSuccess = false;
+
+	void doErrorOp(TTaskFct cont,
+	               std::string basePathAddition,
+	               bool doMaterialize,
+	               int64_t readVersion,
+	               fdb::native::fdb_error_t expectedError) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = begin;
+		// [K - K) empty range will succeed read because there is trivially nothing to do, so don't do it
+		while (end == begin) {
+			end = randomKeyName();
+		}
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+
+		execTransaction(
+		    [this, begin, end, basePathAddition, doMaterialize, readVersion, expectedError](auto ctx) {
+			    ctx->tx().setOption(FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE);
+
+			    TesterGranuleContext testerContext(ctx->getBGBasePath() + basePathAddition);
+			    fdb::native::FDBReadBlobGranuleContext granuleContext = createGranuleContext(&testerContext);
+			    granuleContext.debugNoMaterialize = !doMaterialize;
+
+			    fdb::Result res =
+			        ctx->tx().readBlobGranules(begin, end, 0 /* beginVersion */, readVersion, granuleContext);
+			    auto out = fdb::Result::KeyValueRefArray{};
+			    fdb::Error err = res.getKeyValueArrayNothrow(out);
+
+			    if (err.code() == error_code_success) {
+				    error(fmt::format("Operation succeeded in error test!"));
+			    }
+			    ASSERT(err.code() != error_code_success);
+			    if (err.code() != expectedError) {
+				    info(fmt::format("incorrect error. Expected {}, Got {}", expectedError, err.code()));
+				    if (err.code() == error_code_blob_granule_transaction_too_old) {
+					    ASSERT(!seenReadSuccess);
+					    ctx->done();
+				    } else {
+					    ctx->onError(err);
+				    }
+			    } else {
+				    if (err.code() != error_code_blob_granule_transaction_too_old) {
+					    seenReadSuccess = true;
+				    }
+				    ctx->done();
+			    }
+		    },
+		    [this, cont]() { schedule(cont); });
+	}
+
+	void randomOpReadNoMaterialize(TTaskFct cont) {
+		// ensure setting noMaterialize flag produces blob_granule_not_materialized
+		doErrorOp(cont, "", false, -2 /*latest read version */, error_code_blob_granule_not_materialized);
+	}
+
+	void randomOpReadFileLoadError(TTaskFct cont) {
+		// point to a file path that doesn't exist by adding an extra suffix
+		doErrorOp(cont, "extrapath/", true, -2 /*latest read version */, error_code_blob_granule_file_load_error);
+	}
+
+	void randomOpReadTooOld(TTaskFct cont) {
+		// read at a version (1) that should predate granule data
+		doErrorOp(cont, "", true, 1, error_code_blob_granule_transaction_too_old);
+	}
+
+	void randomPurgeUnalignedOp(TTaskFct cont) {
+		// blobbify/unblobbify need to be aligned to blob range boundaries, so this should always fail
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execOperation(
+		    [this, begin, end](auto ctx) {
+			    fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType();
+			    ctx->continueAfter(
+			        f,
+			        [this, ctx, f]() {
+				        info(fmt::format("unaligned purge got {}", f.error().code()));
+				        ASSERT(f.error().code() == error_code_unsupported_operation);
+				        ctx->done();
+			        },
+			        true);
+		    },
+		    [this, cont]() { schedule(cont); });
+	}
+
+	void randomBlobbifyUnalignedOp(bool blobbify, TTaskFct cont) {
+		// blobbify/unblobbify need to be aligned to blob range boundaries, so this should always return false
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		auto success = std::make_shared<bool>(false);
+		execOperation(
+		    [begin, end, blobbify, success](auto ctx) {
+			    fdb::Future f = blobbify ? ctx->db().blobbifyRange(begin, end).eraseType()
+			                             : ctx->db().unblobbifyRange(begin, end).eraseType();
+			    ctx->continueAfter(
+			        f,
+			        [ctx, f, success]() {
+				        *success = f.get<fdb::future_var::Bool>();
+				        ctx->done();
+			        },
+			        true);
+		    },
+		    [this, cont, success]() {
+			    ASSERT(!(*success));
+			    schedule(cont);
+		    });
+	}
+
+	void randomCancelGetGranulesOp(TTaskFct cont) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execTransaction(
+		    [begin, end](auto ctx) {
+			    fdb::Future f = ctx->tx().getBlobGranuleRanges(begin, end, 1000).eraseType();
+			    ctx->done();
+		    },
+		    [this, cont]() { schedule(cont); });
+	}
+
+	void randomCancelGetRangesOp(TTaskFct cont) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execOperation(
+		    [begin, end](auto ctx) {
+			    fdb::Future f = ctx->db().listBlobbifiedRanges(begin, end, 1000).eraseType();
+			    ctx->done();
+		    },
+		    [this, cont]() { schedule(cont); });
+	}
+
+	void randomCancelVerifyOp(TTaskFct cont) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execOperation(
+		    [begin, end](auto ctx) {
+			    fdb::Future f = ctx->db().verifyBlobRange(begin, end, -2 /* latest version*/).eraseType();
+			    ctx->done();
+		    },
+		    [this, cont]() { schedule(cont); });
+	}
+
+	void randomCancelSummarizeOp(TTaskFct cont) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execTransaction(
+		    [begin, end](auto ctx) {
+			    fdb::Future f = ctx->tx().summarizeBlobGranules(begin, end, -2, 1000).eraseType();
+			    ctx->done();
+		    },
+		    [this, cont]() { schedule(cont); });
+	}
+
+	void randomCancelBlobbifyOp(TTaskFct cont) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execOperation(
+		    [begin, end](auto ctx) {
+			    fdb::Future f = ctx->db().blobbifyRange(begin, end).eraseType();
+			    ctx->done();
+		    },
+		    [this, cont]() { schedule(cont); });
+	}
+
+	void randomCancelUnblobbifyOp(TTaskFct cont) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execOperation(
+		    [begin, end](auto ctx) {
+			    fdb::Future f = ctx->db().unblobbifyRange(begin, end).eraseType();
+			    ctx->done();
+		    },
+		    [this, cont]() { schedule(cont); });
+	}
+
+	void randomCancelPurgeOp(TTaskFct cont) {
+		fdb::Key begin = randomKeyName();
+		fdb::Key end = randomKeyName();
+		if (begin > end) {
+			std::swap(begin, end);
+		}
+		execOperation(
+		    [begin, end](auto ctx) {
+			    fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType();
+			    ctx->done();
+		    },
+		    [this, cont]() { schedule(cont); });
+	}
+
+	void randomOperation(TTaskFct cont) override {
+		OpType txType = (OpType)Random::get().randomInt(0, OP_LAST);
+		switch (txType) {
+		case OP_READ_NO_MATERIALIZE:
+			randomOpReadNoMaterialize(cont);
+			break;
+		case OP_READ_FILE_LOAD_ERROR:
+			randomOpReadFileLoadError(cont);
+			break;
+		case OP_READ_TOO_OLD:
+			randomOpReadTooOld(cont);
+			break;
+		case OP_PURGE_UNALIGNED:
+			// gets the correct error but it doesn't propagate properly in the test
+			// randomPurgeUnalignedOp(cont);
+			break;
+		case OP_BLOBBIFY_UNALIGNED:
+			randomBlobbifyUnalignedOp(true, cont);
+			break;
+		case OP_UNBLOBBIFY_UNALIGNED:
+			randomBlobbifyUnalignedOp(false, cont);
+			break;
+		case OP_CANCEL_GET_GRANULES:
+			randomCancelGetGranulesOp(cont);
+			break;
+		case OP_CANCEL_GET_RANGES:
+			randomCancelGetRangesOp(cont);
+			break;
+		case OP_CANCEL_VERIFY:
+			randomCancelVerifyOp(cont);
+			break;
+		case OP_CANCEL_SUMMARIZE:
+			randomCancelSummarizeOp(cont);
+			break;
+		case OP_CANCEL_BLOBBIFY:
+			randomCancelBlobbifyOp(cont);
+			break;
+		case OP_CANCEL_UNBLOBBIFY:
+			randomCancelUnblobbifyOp(cont);
+			break;
+		case OP_CANCEL_PURGE:
+			randomCancelPurgeOp(cont);
+			break;
+		}
+	}
+};
+
+WorkloadFactory<BlobGranuleErrorsWorkload> BlobGranuleErrorsWorkloadFactory("BlobGranuleErrors");
+
+} // namespace FdbApiTester
diff --git a/bindings/c/test/apitester/TesterBlobGranuleUtil.cpp b/bindings/c/test/apitester/TesterBlobGranuleUtil.cpp
new file mode 100644
index 0000000000..a908a9c0bf
--- /dev/null
+++ b/bindings/c/test/apitester/TesterBlobGranuleUtil.cpp
@@ -0,0 +1,80 @@
+/*
+ * TesterBlobGranuleUtil.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "TesterBlobGranuleUtil.h"
+#include "TesterUtil.h"
+#include <fstream>
+
+namespace FdbApiTester {
+
+// FIXME: avoid duplicating this between files!
+static int64_t granule_start_load(const char* filename,
+                                  int filenameLength,
+                                  int64_t offset,
+                                  int64_t length,
+                                  int64_t fullFileLength,
+                                  void* context) {
+
+	TesterGranuleContext* ctx = (TesterGranuleContext*)context;
+	int64_t loadId = ctx->nextId++;
+
+	uint8_t* buffer = new uint8_t[length];
+	std::ifstream fin(ctx->basePath + std::string(filename, filenameLength), std::ios::in | std::ios::binary);
+	if (fin.fail()) {
+		delete[] buffer;
+		buffer = nullptr;
+	} else {
+		fin.seekg(offset);
+		fin.read((char*)buffer, length);
+	}
+
+	ctx->loadsInProgress.insert({ loadId, buffer });
+
+	return loadId;
+}
+
+static uint8_t* granule_get_load(int64_t loadId, void* context) {
+	TesterGranuleContext* ctx = (TesterGranuleContext*)context;
+	return ctx->loadsInProgress.at(loadId);
+}
+
+static void granule_free_load(int64_t loadId, void* context) {
+	TesterGranuleContext* ctx = (TesterGranuleContext*)context;
+	auto it = ctx->loadsInProgress.find(loadId);
+	uint8_t* dataToFree = it->second;
+	delete[] dataToFree;
+
+	ctx->loadsInProgress.erase(it);
+}
+
+fdb::native::FDBReadBlobGranuleContext createGranuleContext(const TesterGranuleContext* testerContext) {
+	fdb::native::FDBReadBlobGranuleContext granuleContext;
+
+	granuleContext.userContext = (void*)testerContext;
+	granuleContext.debugNoMaterialize = false;
+	granuleContext.granuleParallelism = 1 + Random::get().randomInt(0, 3);
+	granuleContext.start_load_f = &granule_start_load;
+	granuleContext.get_load_f = &granule_get_load;
+	granuleContext.free_load_f = &granule_free_load;
+
+	return granuleContext;
+}
+
+} // namespace FdbApiTester
\ No newline at end of file
diff --git a/bindings/c/test/apitester/TesterBlobGranuleUtil.h b/bindings/c/test/apitester/TesterBlobGranuleUtil.h
new file mode 100644
index 0000000000..7b4b0dba81
--- /dev/null
+++ b/bindings/c/test/apitester/TesterBlobGranuleUtil.h
@@ -0,0 +1,49 @@
+/*
+ * TesterBlobGranuleUtil.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef APITESTER_BLOBGRANULE_UTIL_H
+#define APITESTER_BLOBGRANULE_UTIL_H
+#include "TesterUtil.h"
+#include "test/fdb_api.hpp"
+#include <unordered_map>
+
+namespace FdbApiTester {
+
+class TesterGranuleContext {
+public:
+	std::unordered_map<int64_t, uint8_t*> loadsInProgress;
+	std::string basePath;
+	int64_t nextId;
+
+	TesterGranuleContext(const std::string& basePath) : basePath(basePath), nextId(0) {}
+
+	~TesterGranuleContext() {
+		// this should now never happen with proper memory management
+		ASSERT(loadsInProgress.empty());
+	}
+};
+
+fdb::native::FDBReadBlobGranuleContext createGranuleContext(const TesterGranuleContext* testerContext);
+
+} // namespace FdbApiTester
+
+#endif
diff --git a/bindings/c/test/apitester/TesterCancelTransactionWorkload.cpp b/bindings/c/test/apitester/TesterCancelTransactionWorkload.cpp
index b569cdb35f..b4cd205143 100644
--- a/bindings/c/test/apitester/TesterCancelTransactionWorkload.cpp
+++ b/bindings/c/test/apitester/TesterCancelTransactionWorkload.cpp
@@ -31,11 +31,11 @@ private:
 	enum OpType { OP_CANCEL_GET, OP_CANCEL_AFTER_FIRST_GET, OP_LAST = OP_CANCEL_AFTER_FIRST_GET };
 
 	// Start multiple concurrent gets and cancel the transaction
-	void randomCancelGetTx(TTaskFct cont) {
+	void randomCancelGetTx(TTaskFct cont, std::optional<int> tenantId) {
 		int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
 		auto keys = std::make_shared<std::vector<fdb::Key>>();
 		for (int i = 0; i < numKeys; i++) {
-			keys->push_back(randomKey(readExistingKeysRatio));
+			keys->push_back(randomKey(readExistingKeysRatio, tenantId));
 		}
 		execTransaction(
 		    [keys](auto ctx) {
@@ -45,25 +45,26 @@ private:
 			    }
 			    ctx->done();
 		    },
-		    [this, cont]() { schedule(cont); });
+		    [this, cont]() { schedule(cont); },
+		    getTenant(tenantId));
 	}
 
 	// Start multiple concurrent gets and cancel the transaction after the first get returns
-	void randomCancelAfterFirstResTx(TTaskFct cont) {
+	void randomCancelAfterFirstResTx(TTaskFct cont, std::optional<int> tenantId) {
 		int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
 		auto keys = std::make_shared<std::vector<fdb::Key>>();
 		for (int i = 0; i < numKeys; i++) {
-			keys->push_back(randomKey(readExistingKeysRatio));
+			keys->push_back(randomKey(readExistingKeysRatio, tenantId));
 		}
 		execTransaction(
-		    [this, keys](auto ctx) {
+		    [this, keys, tenantId](auto ctx) {
 			    std::vector<fdb::Future> futures;
 			    for (const auto& key : *keys) {
 				    futures.push_back(ctx->tx().get(key, false).eraseType());
 			    }
 			    for (int i = 0; i < keys->size(); i++) {
 				    fdb::Future f = futures[i];
-				    auto expectedVal = store.get((*keys)[i]);
+				    auto expectedVal = stores[tenantId].get((*keys)[i]);
 				    ctx->continueAfter(f, [expectedVal, f, this, ctx]() {
 					    auto val = f.get<fdb::future_var::ValueRef>();
 					    if (expectedVal != val) {
@@ -75,17 +76,20 @@ private:
 				    });
 			    }
 		    },
-		    [this, cont]() { schedule(cont); });
+		    [this, cont]() { schedule(cont); },
+		    getTenant(tenantId));
 	}
 
 	void randomOperation(TTaskFct cont) override {
+		std::optional<int> tenantId = randomTenant();
 		OpType txType = (OpType)Random::get().randomInt(0, OP_LAST);
+
 		switch (txType) {
 		case OP_CANCEL_GET:
-			randomCancelGetTx(cont);
+			randomCancelGetTx(cont, tenantId);
 			break;
 		case OP_CANCEL_AFTER_FIRST_GET:
-			randomCancelAfterFirstResTx(cont);
+			randomCancelAfterFirstResTx(cont, tenantId);
 			break;
 		}
 	}
diff --git a/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp b/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp
index 9219bb7056..4486abdf97 100644
--- a/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp
+++ b/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp
@@ -41,11 +41,11 @@ private:
 		OP_LAST = OP_COMMIT_READ
 	};
 
-	void randomCommitReadOp(TTaskFct cont) {
+	void randomCommitReadOp(TTaskFct cont, std::optional<int> tenantId) {
 		int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
 		auto kvPairs = std::make_shared<std::vector<fdb::KeyValue>>();
 		for (int i = 0; i < numKeys; i++) {
-			kvPairs->push_back(fdb::KeyValue{ randomKey(readExistingKeysRatio), randomValue() });
+			kvPairs->push_back(fdb::KeyValue{ randomKey(readExistingKeysRatio, tenantId), randomValue() });
 		}
 		execTransaction(
 		    [kvPairs](auto ctx) {
@@ -54,9 +54,9 @@ private:
 			    }
 			    ctx->commit();
 		    },
-		    [this, kvPairs, cont]() {
+		    [this, kvPairs, cont, tenantId]() {
 			    for (const fdb::KeyValue& kv : *kvPairs) {
-				    store.set(kv.key, kv.value);
+				    stores[tenantId].set(kv.key, kv.value);
 			    }
 			    auto results = std::make_shared<std::vector<std::optional<fdb::Value>>>();
 			    execTransaction(
@@ -78,10 +78,10 @@ private:
 					        ctx->done();
 				        });
 			        },
-			        [this, kvPairs, results, cont]() {
+			        [this, kvPairs, results, cont, tenantId]() {
 				        ASSERT(results->size() == kvPairs->size());
 				        for (int i = 0; i < kvPairs->size(); i++) {
-					        auto expected = store.get((*kvPairs)[i].key);
+					        auto expected = stores[tenantId].get((*kvPairs)[i].key);
 					        auto actual = (*results)[i];
 					        if (actual != expected) {
 						        error(
@@ -93,16 +93,18 @@ private:
 					        }
 				        }
 				        schedule(cont);
-			        });
-		    });
+			        },
+			        getTenant(tenantId));
+		    },
+		    getTenant(tenantId));
 	}
 
-	void randomGetOp(TTaskFct cont) {
+	void randomGetOp(TTaskFct cont, std::optional<int> tenantId) {
 		int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
 		auto keys = std::make_shared<std::vector<fdb::Key>>();
 		auto results = std::make_shared<std::vector<std::optional<fdb::Value>>>();
 		for (int i = 0; i < numKeys; i++) {
-			keys->push_back(randomKey(readExistingKeysRatio));
+			keys->push_back(randomKey(readExistingKeysRatio, tenantId));
 		}
 		execTransaction(
 		    [keys, results](auto ctx) {
@@ -119,10 +121,10 @@ private:
 				    ctx->done();
 			    });
 		    },
-		    [this, keys, results, cont]() {
+		    [this, keys, results, cont, tenantId]() {
 			    ASSERT(results->size() == keys->size());
 			    for (int i = 0; i < keys->size(); i++) {
-				    auto expected = store.get((*keys)[i]);
+				    auto expected = stores[tenantId].get((*keys)[i]);
 				    if ((*results)[i] != expected) {
 					    error(fmt::format("randomGetOp mismatch. key: {} expected: {:.80} actual: {:.80}",
 					                      fdb::toCharsRef((*keys)[i]),
@@ -131,16 +133,17 @@ private:
 				    }
 			    }
 			    schedule(cont);
-		    });
+		    },
+		    getTenant(tenantId));
 	}
 
-	void randomGetKeyOp(TTaskFct cont) {
+	void randomGetKeyOp(TTaskFct cont, std::optional<int> tenantId) {
 		int numKeys = Random::get().randomInt(1, maxKeysPerTransaction);
 		auto keysWithSelectors = std::make_shared<std::vector<std::pair<fdb::Key, fdb::KeySelector>>>();
 		auto results = std::make_shared<std::vector<fdb::Key>>();
 		keysWithSelectors->reserve(numKeys);
 		for (int i = 0; i < numKeys; i++) {
-			auto key = randomKey(readExistingKeysRatio);
+			auto key = randomKey(readExistingKeysRatio, tenantId);
 			fdb::KeySelector selector;
 			selector.keyLength = key.size();
 			selector.orEqual = Random::get().randomBool(0.5);
@@ -169,20 +172,20 @@ private:
 				    ctx->done();
 			    });
 		    },
-		    [this, keysWithSelectors, results, cont]() {
+		    [this, keysWithSelectors, results, cont, tenantId]() {
 			    ASSERT(results->size() == keysWithSelectors->size());
 			    for (int i = 0; i < keysWithSelectors->size(); i++) {
 				    auto const& key = (*keysWithSelectors)[i].first;
 				    auto const& selector = (*keysWithSelectors)[i].second;
-				    auto expected = store.getKey(key, selector.orEqual, selector.offset);
+				    auto expected = stores[tenantId].getKey(key, selector.orEqual, selector.offset);
 				    auto actual = (*results)[i];
 				    // Local store only contains data for the current client, while fdb contains data from multiple
 				    // clients. If getKey returned a key outside of the range for the current client, adjust the result
 				    // to match what would be expected in the local store.
 				    if (actual.substr(0, keyPrefix.size()) < keyPrefix) {
-					    actual = store.startKey();
+					    actual = stores[tenantId].startKey();
 				    } else if ((*results)[i].substr(0, keyPrefix.size()) > keyPrefix) {
-					    actual = store.endKey();
+					    actual = stores[tenantId].endKey();
 				    }
 				    if (actual != expected) {
 					    error(fmt::format("randomGetKeyOp mismatch. key: {}, orEqual: {}, offset: {}, expected: {} "
@@ -195,37 +198,38 @@ private:
 				    }
 			    }
 			    schedule(cont);
-		    });
+		    },
+		    getTenant(tenantId));
 	}
 
 	void getRangeLoop(std::shared_ptr<ITransactionContext> ctx,
 	                  fdb::KeySelector begin,
-	                  fdb::KeySelector end,
+	                  fdb::Key endKey,
 	                  std::shared_ptr<std::vector<fdb::KeyValue>> results) {
 		auto f = ctx->tx().getRange(begin,
-		                            end,
+		                            fdb::key_select::firstGreaterOrEqual(endKey),
 		                            0 /*limit*/,
 		                            0 /*target_bytes*/,
 		                            FDB_STREAMING_MODE_WANT_ALL,
 		                            0 /*iteration*/,
 		                            false /*snapshot*/,
 		                            false /*reverse*/);
-		ctx->continueAfter(f, [this, ctx, f, end, results]() {
+		ctx->continueAfter(f, [this, ctx, f, endKey, results]() {
 			auto out = copyKeyValueArray(f.get());
 			results->insert(results->end(), out.first.begin(), out.first.end());
 			const bool more = out.second;
 			if (more) {
 				// Fetch the remaining results.
-				getRangeLoop(ctx, fdb::key_select::firstGreaterThan(results->back().key), end, results);
+				getRangeLoop(ctx, fdb::key_select::firstGreaterThan(results->back().key), endKey, results);
 			} else {
 				ctx->done();
 			}
 		});
 	}
 
-	void randomGetRangeOp(TTaskFct cont) {
-		auto begin = randomKey(readExistingKeysRatio);
-		auto end = randomKey(readExistingKeysRatio);
+	void randomGetRangeOp(TTaskFct cont, std::optional<int> tenantId) {
+		auto begin = randomKey(readExistingKeysRatio, tenantId);
+		auto end = randomKey(readExistingKeysRatio, tenantId);
 		auto results = std::make_shared<std::vector<fdb::KeyValue>>();
 
 		execTransaction(
@@ -233,13 +237,10 @@ private:
 			    // Clear the results vector, in case the transaction is retried.
 			    results->clear();
 
-			    getRangeLoop(ctx,
-			                 fdb::key_select::firstGreaterOrEqual(begin),
-			                 fdb::key_select::firstGreaterOrEqual(end),
-			                 results);
+			    getRangeLoop(ctx, fdb::key_select::firstGreaterOrEqual(begin), end, results);
 		    },
-		    [this, begin, end, results, cont]() {
-			    auto expected = store.getRange(begin, end, results->size() + 10, false);
+		    [this, begin, end, results, cont, tenantId]() {
+			    auto expected = stores[tenantId].getRange(begin, end, results->size() + 10, false);
 			    if (results->size() != expected.size()) {
 				    error(fmt::format("randomGetRangeOp mismatch. expected {} keys, actual {} keys",
 				                      expected.size(),
@@ -260,32 +261,35 @@ private:
 				    }
 			    }
 			    schedule(cont);
-		    });
+		    },
+		    getTenant(tenantId));
 	}
 
 	void randomOperation(TTaskFct cont) {
-		OpType txType = (store.size() == 0) ? OP_INSERT : (OpType)Random::get().randomInt(0, OP_LAST);
+		std::optional<int> tenantId = randomTenant();
+		OpType txType = (stores[tenantId].size() == 0) ? OP_INSERT : (OpType)Random::get().randomInt(0, OP_LAST);
+
 		switch (txType) {
 		case OP_INSERT:
-			randomInsertOp(cont);
+			randomInsertOp(cont, tenantId);
 			break;
 		case OP_GET:
-			randomGetOp(cont);
+			randomGetOp(cont, tenantId);
 			break;
 		case OP_GET_KEY:
-			randomGetKeyOp(cont);
+			randomGetKeyOp(cont, tenantId);
 			break;
 		case OP_CLEAR:
-			randomClearOp(cont);
+			randomClearOp(cont, tenantId);
 			break;
 		case OP_GET_RANGE:
-			randomGetRangeOp(cont);
+			randomGetRangeOp(cont, tenantId);
 			break;
 		case OP_CLEAR_RANGE:
-			randomClearRangeOp(cont);
+			randomClearRangeOp(cont, tenantId);
 			break;
 		case OP_COMMIT_READ:
-			randomCommitReadOp(cont);
+			randomCommitReadOp(cont, tenantId);
 			break;
 		}
 	}
diff --git a/bindings/c/test/apitester/TesterExampleWorkload.cpp b/bindings/c/test/apitester/TesterExampleWorkload.cpp
new file mode 100644
index 0000000000..882fdc62e4
--- /dev/null
+++ b/bindings/c/test/apitester/TesterExampleWorkload.cpp
@@ -0,0 +1,65 @@
+/*
+ * TesterExampleWorkload.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "TesterWorkload.h"
+#include "TesterUtil.h"
+
+namespace FdbApiTester {
+
+class SetAndGetWorkload : public WorkloadBase {
+public:
+	fdb::Key keyPrefix;
+	Random random;
+
+	SetAndGetWorkload(const WorkloadConfig& config) : WorkloadBase(config) {
+		keyPrefix = fdb::toBytesRef(fmt::format("{}/", workloadId));
+	}
+
+	void start() override { setAndGet(NO_OP_TASK); }
+
+	void setAndGet(TTaskFct cont) {
+		fdb::Key key = keyPrefix + random.randomByteStringLowerCase(10, 100);
+		fdb::Value value = random.randomByteStringLowerCase(10, 1000);
+		execTransaction(
+		    [key, value](auto ctx) {
+			    ctx->tx().set(key, value);
+			    ctx->commit();
+		    },
+		    [this, key, value, cont]() {
+			    execTransaction(
+			        [this, key, value](auto ctx) {
+				        auto future = ctx->tx().get(key, false);
+				        ctx->continueAfter(future, [this, ctx, future, value]() {
+					        std::optional<fdb::Value> res = copyValueRef(future.get());
+					        if (res != value) {
+						        error(fmt::format(
+						            "expected: {} actual: {}", fdb::toCharsRef(value), fdb::toCharsRef(res.value())));
+					        }
+					        ctx->done();
+				        });
+			        },
+			        cont);
+		    });
+	}
+};
+
+WorkloadFactory<SetAndGetWorkload> SetAndGetWorkloadFactory("SetAndGet");
+
+} // namespace FdbApiTester
diff --git a/bindings/c/test/apitester/TesterOptions.h b/bindings/c/test/apitester/TesterOptions.h
index 3ff57ec183..7c7d0fc948 100644
--- a/bindings/c/test/apitester/TesterOptions.h
+++ b/bindings/c/test/apitester/TesterOptions.h
@@ -38,6 +38,7 @@ public:
 	std::string logGroup;
 	std::string externalClientLibrary;
 	std::string externalClientDir;
+	std::string futureVersionClientLibrary;
 	std::string tmpDir;
 	bool disableLocalClient = false;
 	std::string testFile;
@@ -48,6 +49,7 @@ public:
 	int numClientThreads;
 	int numDatabases;
 	int numClients;
+	int numTenants = -1;
 	int statsIntervalMs = 0;
 	std::vector<std::pair<std::string, std::string>> knobs;
 	TestSpec testSpec;
diff --git a/bindings/c/test/apitester/TesterTestSpec.cpp b/bindings/c/test/apitester/TesterTestSpec.cpp
index 86a89c9116..1048aab493 100644
--- a/bindings/c/test/apitester/TesterTestSpec.cpp
+++ b/bindings/c/test/apitester/TesterTestSpec.cpp
@@ -65,6 +65,10 @@ std::unordered_map<std::string, std::function<void(const std::string& value, Tes
 	  [](const std::string& value, TestSpec* spec) { //
 	      spec->databasePerTransaction = (value == "true");
 	  } },
+	{ "tamperClusterFile",
+	  [](const std::string& value, TestSpec* spec) { //
+	      spec->tamperClusterFile = (value == "true");
+	  } },
 	{ "minFdbThreads",
 	  [](const std::string& value, TestSpec* spec) { //
 	      processIntOption(value, "minFdbThreads", spec->minFdbThreads, 1, 1000);
@@ -96,6 +100,18 @@ std::unordered_map<std::string, std::function<void(const std::string& value, Tes
 	{ "maxClients",
 	  [](const std::string& value, TestSpec* spec) { //
 	      processIntOption(value, "maxClients", spec->maxClients, 1, 1000);
+	  } },
+	{ "disableClientBypass",
+	  [](const std::string& value, TestSpec* spec) { //
+	      spec->disableClientBypass = (value == "true");
+	  } },
+	{ "minTenants",
+	  [](const std::string& value, TestSpec* spec) { //
+	      processIntOption(value, "minTenants", spec->minTenants, 1, 1000);
+	  } },
+	{ "maxTenants",
+	  [](const std::string& value, TestSpec* spec) { //
+	      processIntOption(value, "maxTenants", spec->maxTenants, 1, 1000);
 	  } }
 };
 
diff --git a/bindings/c/test/apitester/TesterTestSpec.h b/bindings/c/test/apitester/TesterTestSpec.h
index be7a573033..c0e9c0caf1 100644
--- a/bindings/c/test/apitester/TesterTestSpec.h
+++ b/bindings/c/test/apitester/TesterTestSpec.h
@@ -58,6 +58,9 @@ struct TestSpec {
 	// Execute each transaction in a separate database instance
 	bool databasePerTransaction = false;
 
+	// Test tampering the cluster file
+	bool tamperClusterFile = false;
+
 	// Size of the FDB client thread pool (a random number in the [min,max] range)
 	int minFdbThreads = 1;
 	int maxFdbThreads = 1;
@@ -75,6 +78,13 @@ struct TestSpec {
 	int minClients = 1;
 	int maxClients = 10;
 
+	// Disable the ability to bypass the MVC API, for
+	// cases when there are no external clients
+	bool disableClientBypass = false;
+	// Number of tenants (a random number in the [min,max] range)
+	int minTenants = 0;
+	int maxTenants = 0;
+
 	// List of workloads with their options
 	std::vector<WorkloadSpec> workloads;
 };
diff --git a/bindings/c/test/apitester/TesterTransactionExecutor.cpp b/bindings/c/test/apitester/TesterTransactionExecutor.cpp
index 221774854d..27acef0e14 100644
--- a/bindings/c/test/apitester/TesterTransactionExecutor.cpp
+++ b/bindings/c/test/apitester/TesterTransactionExecutor.cpp
@@ -23,25 +23,23 @@
 #include "foundationdb/fdb_c_types.h"
 #include "test/apitester/TesterScheduler.h"
 #include "test/fdb_api.hpp"
+#include <cstddef>
 #include <memory>
 #include <stdexcept>
+#include <string>
 #include <unordered_map>
 #include <mutex>
 #include <atomic>
 #include <chrono>
 #include <thread>
 #include <fmt/format.h>
+#include <filesystem>
 
 namespace FdbApiTester {
 
 constexpr int LONG_WAIT_TIME_US = 2000000;
 constexpr int LARGE_NUMBER_OF_RETRIES = 10;
 
-void TransactionActorBase::complete(fdb::Error err) {
-	error = err;
-	context = {};
-}
-
 void ITransactionContext::continueAfterAll(std::vector<fdb::Future> futures, TTaskFct cont) {
 	auto counter = std::make_shared<std::atomic<int>>(futures.size());
 	auto errorCode = std::make_shared<std::atomic<fdb::Error>>(fdb::Error::success());
@@ -72,20 +70,44 @@ void ITransactionContext::continueAfterAll(std::vector<fdb::Future> futures, TTa
  */
 class TransactionContextBase : public ITransactionContext {
 public:
-	TransactionContextBase(fdb::Transaction tx,
-	                       std::shared_ptr<ITransactionActor> txActor,
-	                       TTaskFct cont,
+	TransactionContextBase(ITransactionExecutor* executor,
+	                       TOpStartFct startFct,
+	                       TOpContFct cont,
 	                       IScheduler* scheduler,
 	                       int retryLimit,
-	                       std::string bgBasePath)
-	  : fdbTx(tx), txActor(txActor), contAfterDone(cont), scheduler(scheduler), retryLimit(retryLimit),
-	    txState(TxState::IN_PROGRESS), commitCalled(false), bgBasePath(bgBasePath) {}
+	                       std::string bgBasePath,
+	                       std::optional<fdb::BytesRef> tenantName,
+	                       bool transactional)
+	  : executor(executor), startFct(startFct), contAfterDone(cont), scheduler(scheduler), retryLimit(retryLimit),
+	    txState(TxState::IN_PROGRESS), commitCalled(false), bgBasePath(bgBasePath), tenantName(tenantName),
+	    transactional(transactional) {
+		databaseCreateErrorInjected = executor->getOptions().injectDatabaseCreateErrors &&
+		                              Random::get().randomBool(executor->getOptions().databaseCreateErrorRatio);
+		if (databaseCreateErrorInjected) {
+			fdbDb = fdb::Database(executor->getClusterFileForErrorInjection());
+		} else {
+			fdbDb = executor->selectDatabase();
+		}
+
+		if (transactional) {
+			if (tenantName) {
+				fdb::Tenant tenant = fdbDb.openTenant(*tenantName);
+				fdbTx = tenant.createTransaction();
+			} else {
+				fdbTx = fdbDb.createTransaction();
+			}
+		}
+	}
+
+	virtual ~TransactionContextBase() { ASSERT(txState == TxState::DONE); }
 
 	// A state machine:
 	// IN_PROGRESS -> (ON_ERROR -> IN_PROGRESS)* [-> ON_ERROR] -> DONE
 	enum class TxState { IN_PROGRESS, ON_ERROR, DONE };
 
-	fdb::Transaction tx() override { return fdbTx; }
+	fdb::Database db() override { return fdbDb.atomic_load(); }
+
+	fdb::Transaction tx() override { return fdbTx.atomic_load(); }
 
 	// Set a continuation to be executed when a future gets ready
 	void continueAfter(fdb::Future f, TTaskFct cont, bool retryOnError) override {
@@ -94,6 +116,7 @@ public:
 
 	// Complete the transaction with a commit
 	void commit() override {
+		ASSERT(transactional);
 		std::unique_lock<std::mutex> lock(mutex);
 		if (txState != TxState::IN_PROGRESS) {
 			return;
@@ -114,31 +137,90 @@ public:
 		}
 		txState = TxState::DONE;
 		lock.unlock();
+
+		// No need for lock from here on, because only one thread
+		// can enter DONE state and handle it
+
 		if (retriedErrors.size() >= LARGE_NUMBER_OF_RETRIES) {
 			fmt::print("Transaction succeeded after {} retries on errors: {}\n",
 			           retriedErrors.size(),
 			           fmt::join(retriedErrorCodes(), ", "));
 		}
-		// cancel transaction so that any pending operations on it
-		// fail gracefully
-		fdbTx.cancel();
-		txActor->complete(fdb::Error::success());
+
+		if (transactional) {
+			// cancel transaction so that any pending operations on it
+			// fail gracefully
+			fdbTx.cancel();
+		}
 		cleanUp();
-		contAfterDone();
+		ASSERT(txState == TxState::DONE);
+		contAfterDone(fdb::Error::success());
 	}
 
 	std::string getBGBasePath() override { return bgBasePath; }
 
+	virtual void onError(fdb::Error err) override {
+		std::unique_lock<std::mutex> lock(mutex);
+		if (txState != TxState::IN_PROGRESS) {
+			// Ignore further errors, if the transaction is in the error handing mode or completed
+			return;
+		}
+		txState = TxState::ON_ERROR;
+		lock.unlock();
+
+		// No need to hold the lock from here on, because ON_ERROR state is handled sequentially, and
+		// other callbacks are simply ignored while it stays in this state
+
+		if (!canRetry(err)) {
+			return;
+		}
+
+		ASSERT(!onErrorFuture);
+
+		if (databaseCreateErrorInjected && canBeInjectedDatabaseCreateError(err.code())) {
+			// Failed to create a database because of failure injection
+			// Restart by recreating the transaction in a valid database
+			auto thisRef = std::static_pointer_cast<TransactionContextBase>(shared_from_this());
+			scheduler->schedule([thisRef]() {
+				fdb::Database db = thisRef->executor->selectDatabase();
+				thisRef->fdbDb.atomic_store(db);
+				if (thisRef->transactional) {
+					if (thisRef->tenantName) {
+						fdb::Tenant tenant = db.openTenant(*thisRef->tenantName);
+						thisRef->fdbTx.atomic_store(tenant.createTransaction());
+					} else {
+						thisRef->fdbTx.atomic_store(db.createTransaction());
+					}
+				}
+				thisRef->restartTransaction();
+			});
+		} else if (transactional) {
+			onErrorArg = err;
+			onErrorFuture = tx().onError(err);
+			handleOnErrorFuture();
+		} else {
+			transactionFailed(err);
+		}
+	}
+
 protected:
 	virtual void doContinueAfter(fdb::Future f, TTaskFct cont, bool retryOnError) = 0;
 
+	virtual void handleOnErrorFuture() = 0;
+
 	// Clean up transaction state after completing the transaction
 	// Note that the object may live longer, because it is referenced
 	// by not yet triggered callbacks
-	virtual void cleanUp() {
+	void cleanUp() {
 		ASSERT(txState == TxState::DONE);
 		ASSERT(!onErrorFuture);
-		txActor = {};
+		cancelPendingFutures();
+	}
+
+	virtual void cancelPendingFutures() {}
+
+	bool canBeInjectedDatabaseCreateError(fdb::Error::CodeType errCode) {
+		return errCode == error_code_no_cluster_file_found || errCode == error_code_connection_string_invalid;
 	}
 
 	// Complete the transaction with an (unretriable) error
@@ -150,9 +232,12 @@ protected:
 		}
 		txState = TxState::DONE;
 		lock.unlock();
-		txActor->complete(err);
+
+		// No need for lock from here on, because only one thread
+		// can enter DONE state and handle it
+
 		cleanUp();
-		contAfterDone();
+		contAfterDone(err);
 	}
 
 	// Handle result of an a transaction onError call
@@ -163,14 +248,20 @@ protected:
 		if (err) {
 			transactionFailed(err);
 		} else {
-			std::unique_lock<std::mutex> lock(mutex);
-			txState = TxState::IN_PROGRESS;
-			commitCalled = false;
-			lock.unlock();
-			txActor->start();
+			restartTransaction();
 		}
 	}
 
+	void restartTransaction() {
+		ASSERT(txState == TxState::ON_ERROR);
+		cancelPendingFutures();
+		std::unique_lock<std::mutex> lock(mutex);
+		txState = TxState::IN_PROGRESS;
+		commitCalled = false;
+		lock.unlock();
+		startFct(shared_from_this());
+	}
+
 	// Checks if a transaction can be retried. Fails the transaction if the check fails
 	bool canRetry(fdb::Error lastErr) {
 		ASSERT(txState == TxState::ON_ERROR);
@@ -196,44 +287,77 @@ protected:
 		return retriedErrorCodes;
 	}
 
+	// Pointer to the transaction executor interface
+	// Set in contructor, stays immutable
+	ITransactionExecutor* const executor;
+
+	// FDB database
+	// Provides a thread safe interface by itself (no need for mutex)
+	fdb::Database fdbDb;
+
 	// FDB transaction
+	// Provides a thread safe interface by itself (no need for mutex)
 	fdb::Transaction fdbTx;
 
-	// Actor implementing the transaction worklflow
-	std::shared_ptr<ITransactionActor> txActor;
+	// The function implementing the starting point of the transaction
+	// Set in constructor and reset on cleanup (no need for mutex)
+	TOpStartFct startFct;
 
 	// Mutex protecting access to shared mutable state
+	// Only the state that is accessible unter IN_PROGRESS state
+	// must be protected by mutex
 	std::mutex mutex;
 
 	// Continuation to be called after completion of the transaction
-	TTaskFct contAfterDone;
+	// Set in contructor, stays immutable
+	const TOpContFct contAfterDone;
 
 	// Reference to the scheduler
-	IScheduler* scheduler;
+	// Set in contructor, stays immutable
+	// Cannot be accessed in DONE state, workloads can be completed and the scheduler deleted
+	IScheduler* const scheduler;
 
 	// Retry limit
-	int retryLimit;
+	// Set in contructor, stays immutable
+	const int retryLimit;
 
 	// Transaction execution state
+	// Must be accessed under mutex
 	TxState txState;
 
-	// onError future used in ON_ERROR state
+	// onError future
+	// used only in ON_ERROR state (no need for mutex)
 	fdb::Future onErrorFuture;
 
 	// The error code on which onError was called
+	// used only in ON_ERROR state (no need for mutex)
 	fdb::Error onErrorArg;
 
 	// The time point of calling onError
+	// used only in ON_ERROR state (no need for mutex)
 	TimePoint onErrorCallTimePoint;
 
 	// Transaction is committed or being committed
+	// Must be accessed under mutex
 	bool commitCalled;
 
 	// A history of errors on which the transaction was retried
+	// used only in ON_ERROR and DONE states (no need for mutex)
 	std::vector<fdb::Error> retriedErrors;
 
 	// blob granule base path
-	std::string bgBasePath;
+	// Set in contructor, stays immutable
+	const std::string bgBasePath;
+
+	// Indicates if the database error was injected
+	// Accessed on initialization and in ON_ERROR state only (no need for mutex)
+	bool databaseCreateErrorInjected;
+
+	// The tenant that we will run this transaction in
+	const std::optional<fdb::BytesRef> tenantName;
+
+	// Specifies whether the operation is transactional
+	const bool transactional;
 };
 
 /**
@@ -241,13 +365,16 @@ protected:
  */
 class BlockingTransactionContext : public TransactionContextBase {
 public:
-	BlockingTransactionContext(fdb::Transaction tx,
-	                           std::shared_ptr<ITransactionActor> txActor,
-	                           TTaskFct cont,
+	BlockingTransactionContext(ITransactionExecutor* executor,
+	                           TOpStartFct startFct,
+	                           TOpContFct cont,
 	                           IScheduler* scheduler,
 	                           int retryLimit,
-	                           std::string bgBasePath)
-	  : TransactionContextBase(tx, txActor, cont, scheduler, retryLimit, bgBasePath) {}
+	                           std::string bgBasePath,
+	                           std::optional<fdb::BytesRef> tenantName,
+	                           bool transactional)
+	  : TransactionContextBase(executor, startFct, cont, scheduler, retryLimit, bgBasePath, tenantName, transactional) {
+	}
 
 protected:
 	void doContinueAfter(fdb::Future f, TTaskFct cont, bool retryOnError) override {
@@ -288,22 +415,8 @@ protected:
 		onError(err);
 	}
 
-	virtual void onError(fdb::Error err) override {
-		std::unique_lock<std::mutex> lock(mutex);
-		if (txState != TxState::IN_PROGRESS) {
-			// Ignore further errors, if the transaction is in the error handing mode or completed
-			return;
-		}
-		txState = TxState::ON_ERROR;
-		lock.unlock();
-
-		if (!canRetry(err)) {
-			return;
-		}
-
-		ASSERT(!onErrorFuture);
-		onErrorFuture = fdbTx.onError(err);
-		onErrorArg = err;
+	virtual void handleOnErrorFuture() override {
+		ASSERT(txState == TxState::ON_ERROR);
 
 		auto start = timeNow();
 		fdb::Error err2 = onErrorFuture.blockUntilReady();
@@ -330,13 +443,16 @@ protected:
  */
 class AsyncTransactionContext : public TransactionContextBase {
 public:
-	AsyncTransactionContext(fdb::Transaction tx,
-	                        std::shared_ptr<ITransactionActor> txActor,
-	                        TTaskFct cont,
+	AsyncTransactionContext(ITransactionExecutor* executor,
+	                        TOpStartFct startFct,
+	                        TOpContFct cont,
 	                        IScheduler* scheduler,
 	                        int retryLimit,
-	                        std::string bgBasePath)
-	  : TransactionContextBase(tx, txActor, cont, scheduler, retryLimit, bgBasePath) {}
+	                        std::string bgBasePath,
+	                        std::optional<fdb::BytesRef> tenantName,
+	                        bool transactional)
+	  : TransactionContextBase(executor, startFct, cont, scheduler, retryLimit, bgBasePath, tenantName, transactional) {
+	}
 
 protected:
 	void doContinueAfter(fdb::Future f, TTaskFct cont, bool retryOnError) override {
@@ -344,7 +460,7 @@ protected:
 		if (txState != TxState::IN_PROGRESS) {
 			return;
 		}
-		callbackMap[f] = CallbackInfo{ f, cont, shared_from_this(), retryOnError, timeNow() };
+		callbackMap[f] = CallbackInfo{ f, cont, shared_from_this(), retryOnError, timeNow(), false };
 		lock.unlock();
 		try {
 			f.then([this](fdb::Future f) { futureReadyCallback(f, this); });
@@ -383,7 +499,6 @@ protected:
 		if (txState != TxState::IN_PROGRESS) {
 			return;
 		}
-		lock.unlock();
 		fdb::Error err = f.error();
 		auto waitTimeUs = timeElapsedInUs(cbInfo.startTime, endTime);
 		if (waitTimeUs > LONG_WAIT_TIME_US) {
@@ -392,32 +507,23 @@ protected:
 			           err.code(),
 			           err.what());
 		}
-		if (err.code() == error_code_transaction_cancelled) {
+		if (err.code() == error_code_transaction_cancelled || cbInfo.cancelled) {
 			return;
 		}
 		if (err.code() == error_code_success || !cbInfo.retryOnError) {
 			scheduler->schedule(cbInfo.cont);
 			return;
 		}
+		// We keep lock until here to prevent transitions from the IN_PROGRESS state
+		// which could possibly lead to completion of the workload and destruction
+		// of the scheduler
+		lock.unlock();
 		onError(err);
 	}
 
-	virtual void onError(fdb::Error err) override {
-		std::unique_lock<std::mutex> lock(mutex);
-		if (txState != TxState::IN_PROGRESS) {
-			// Ignore further errors, if the transaction is in the error handing mode or completed
-			return;
-		}
-		txState = TxState::ON_ERROR;
-		lock.unlock();
+	virtual void handleOnErrorFuture() override {
+		ASSERT(txState == TxState::ON_ERROR);
 
-		if (!canRetry(err)) {
-			return;
-		}
-
-		ASSERT(!onErrorFuture);
-		onErrorArg = err;
-		onErrorFuture = tx().onError(err);
 		onErrorCallTimePoint = timeNow();
 		onErrorThisRef = std::static_pointer_cast<AsyncTransactionContext>(shared_from_this());
 		try {
@@ -457,17 +563,17 @@ protected:
 		scheduler->schedule([thisRef]() { thisRef->handleOnErrorResult(); });
 	}
 
-	void cleanUp() override {
-		TransactionContextBase::cleanUp();
-
+	void cancelPendingFutures() override {
 		// Cancel all pending operations
 		// Note that the callbacks of the cancelled futures will still be called
 		std::unique_lock<std::mutex> lock(mutex);
 		std::vector<fdb::Future> futures;
 		for (auto& iter : callbackMap) {
+			iter.second.cancelled = true;
 			futures.push_back(iter.second.future);
 		}
 		lock.unlock();
+
 		for (auto& f : futures) {
 			f.cancel();
 		}
@@ -487,12 +593,16 @@ protected:
 		std::shared_ptr<ITransactionContext> thisRef;
 		bool retryOnError;
 		TimePoint startTime;
+		bool cancelled;
 	};
 
 	// Map for keeping track of future waits and holding necessary object references
+	// It can be accessed at any time when callbacks are triggered, so it mus always
+	// be mutex protected
 	std::unordered_map<fdb::Future, CallbackInfo> callbackMap;
 
 	// Holding reference to this for onError future C callback
+	// Accessed only in ON_ERROR state (no need for mutex)
 	std::shared_ptr<AsyncTransactionContext> onErrorThisRef;
 };
 
@@ -503,30 +613,86 @@ class TransactionExecutorBase : public ITransactionExecutor {
 public:
 	TransactionExecutorBase(const TransactionExecutorOptions& options) : options(options), scheduler(nullptr) {}
 
+	~TransactionExecutorBase() {
+		if (tamperClusterFileThread.joinable()) {
+			tamperClusterFileThread.join();
+		}
+	}
+
 	void init(IScheduler* scheduler, const char* clusterFile, const std::string& bgBasePath) override {
 		this->scheduler = scheduler;
 		this->clusterFile = clusterFile;
 		this->bgBasePath = bgBasePath;
+
+		ASSERT(!options.tmpDir.empty());
+		emptyClusterFile.create(options.tmpDir, "fdbempty.cluster");
+		invalidClusterFile.create(options.tmpDir, "fdbinvalid.cluster");
+		invalidClusterFile.write(Random().get().randomStringLowerCase<std::string>(1, 100));
+
+		emptyListClusterFile.create(options.tmpDir, "fdbemptylist.cluster");
+		emptyListClusterFile.write(fmt::format("{}:{}@",
+		                                       Random().get().randomStringLowerCase<std::string>(3, 8),
+		                                       Random().get().randomStringLowerCase<std::string>(1, 100)));
+
+		if (options.tamperClusterFile) {
+			tamperedClusterFile.create(options.tmpDir, "fdb.cluster");
+			originalClusterFile = clusterFile;
+			this->clusterFile = tamperedClusterFile.getFileName();
+
+			// begin with a valid cluster file, but with non existing address
+			tamperedClusterFile.write(fmt::format("{}:{}@192.168.{}.{}:{}",
+			                                      Random().get().randomStringLowerCase<std::string>(3, 8),
+			                                      Random().get().randomStringLowerCase<std::string>(1, 100),
+			                                      Random().get().randomInt(1, 254),
+			                                      Random().get().randomInt(1, 254),
+			                                      Random().get().randomInt(2000, 10000)));
+
+			tamperClusterFileThread = std::thread([this]() {
+				std::this_thread::sleep_for(std::chrono::seconds(2));
+				// now write an invalid connection string
+				tamperedClusterFile.write(fmt::format("{}:{}@",
+				                                      Random().get().randomStringLowerCase<std::string>(3, 8),
+				                                      Random().get().randomStringLowerCase<std::string>(1, 100)));
+				std::this_thread::sleep_for(std::chrono::seconds(2));
+				// finally use correct cluster file contents
+				std::filesystem::copy_file(std::filesystem::path(originalClusterFile),
+				                           std::filesystem::path(tamperedClusterFile.getFileName()),
+				                           std::filesystem::copy_options::overwrite_existing);
+			});
+		}
 	}
 
-protected:
-	// Execute the transaction on the given database instance
-	void executeOnDatabase(fdb::Database db, std::shared_ptr<ITransactionActor> txActor, TTaskFct cont) {
+	const TransactionExecutorOptions& getOptions() override { return options; }
+
+	void execute(TOpStartFct startFct,
+	             TOpContFct cont,
+	             std::optional<fdb::BytesRef> tenantName,
+	             bool transactional) override {
 		try {
-			fdb::Transaction tx = db.createTransaction();
 			std::shared_ptr<ITransactionContext> ctx;
 			if (options.blockOnFutures) {
 				ctx = std::make_shared<BlockingTransactionContext>(
-				    tx, txActor, cont, scheduler, options.transactionRetryLimit, bgBasePath);
+				    this, startFct, cont, scheduler, options.transactionRetryLimit, bgBasePath, tenantName, true);
 			} else {
 				ctx = std::make_shared<AsyncTransactionContext>(
-				    tx, txActor, cont, scheduler, options.transactionRetryLimit, bgBasePath);
+				    this, startFct, cont, scheduler, options.transactionRetryLimit, bgBasePath, tenantName, true);
 			}
-			txActor->init(ctx);
-			txActor->start();
+			startFct(ctx);
 		} catch (...) {
-			txActor->complete(fdb::Error(error_code_operation_failed));
-			cont();
+			cont(fdb::Error(error_code_operation_failed));
+		}
+	}
+
+	std::string getClusterFileForErrorInjection() override {
+		switch (Random::get().randomInt(0, 3)) {
+		case 0:
+			return fmt::format("{}{}", "not-existing-file", Random::get().randomStringLowerCase<std::string>(0, 2));
+		case 1:
+			return emptyClusterFile.getFileName();
+		case 2:
+			return invalidClusterFile.getFileName();
+		default: // case 3
+			return emptyListClusterFile.getFileName();
 		}
 	}
 
@@ -535,6 +701,12 @@ protected:
 	std::string bgBasePath;
 	std::string clusterFile;
 	IScheduler* scheduler;
+	TmpFile emptyClusterFile;
+	TmpFile invalidClusterFile;
+	TmpFile emptyListClusterFile;
+	TmpFile tamperedClusterFile;
+	std::thread tamperClusterFileThread;
+	std::string originalClusterFile;
 };
 
 /**
@@ -549,19 +721,19 @@ public:
 	void init(IScheduler* scheduler, const char* clusterFile, const std::string& bgBasePath) override {
 		TransactionExecutorBase::init(scheduler, clusterFile, bgBasePath);
 		for (int i = 0; i < options.numDatabases; i++) {
-			fdb::Database db(clusterFile);
+			fdb::Database db(this->clusterFile);
 			databases.push_back(db);
 		}
 	}
 
-	void execute(std::shared_ptr<ITransactionActor> txActor, TTaskFct cont) override {
+	fdb::Database selectDatabase() override {
 		int idx = Random::get().randomInt(0, options.numDatabases - 1);
-		executeOnDatabase(databases[idx], txActor, cont);
+		return databases[idx];
 	}
 
+private:
 	void release() { databases.clear(); }
 
-private:
 	std::vector<fdb::Database> databases;
 };
 
@@ -572,10 +744,7 @@ class DBPerTransactionExecutor : public TransactionExecutorBase {
 public:
 	DBPerTransactionExecutor(const TransactionExecutorOptions& options) : TransactionExecutorBase(options) {}
 
-	void execute(std::shared_ptr<ITransactionActor> txActor, TTaskFct cont) override {
-		fdb::Database db(clusterFile.c_str());
-		executeOnDatabase(db, txActor, cont);
-	}
+	fdb::Database selectDatabase() override { return fdb::Database(clusterFile.c_str()); }
 };
 
 std::unique_ptr<ITransactionExecutor> createTransactionExecutor(const TransactionExecutorOptions& options) {
diff --git a/bindings/c/test/apitester/TesterTransactionExecutor.h b/bindings/c/test/apitester/TesterTransactionExecutor.h
index 31f6f3bc84..b0e5268d14 100644
--- a/bindings/c/test/apitester/TesterTransactionExecutor.h
+++ b/bindings/c/test/apitester/TesterTransactionExecutor.h
@@ -38,6 +38,9 @@ class ITransactionContext : public std::enable_shared_from_this<ITransactionCont
 public:
 	virtual ~ITransactionContext() {}
 
+	// Current FDB database
+	virtual fdb::Database db() = 0;
+
 	// Current FDB transaction
 	virtual fdb::Transaction tx() = 0;
 
@@ -62,57 +65,11 @@ public:
 	virtual void continueAfterAll(std::vector<fdb::Future> futures, TTaskFct cont);
 };
 
-/**
- * Interface of an actor object implementing a concrete transaction
- */
-class ITransactionActor {
-public:
-	virtual ~ITransactionActor() {}
+// Type of the lambda functions implementing a database operation
+using TOpStartFct = std::function<void(std::shared_ptr<ITransactionContext>)>;
 
-	// Initialize with the given transaction context
-	virtual void init(std::shared_ptr<ITransactionContext> ctx) = 0;
-
-	// Start execution of the transaction, also called on retries
-	virtual void start() = 0;
-
-	// Transaction completion result (error_code_success in case of success)
-	virtual fdb::Error getError() = 0;
-
-	// Notification about the completion of the transaction
-	virtual void complete(fdb::Error err) = 0;
-};
-
-/**
- * A helper base class for transaction actors
- */
-class TransactionActorBase : public ITransactionActor {
-public:
-	void init(std::shared_ptr<ITransactionContext> ctx) override { context = ctx; }
-	fdb::Error getError() override { return error; }
-	void complete(fdb::Error err) override;
-
-protected:
-	std::shared_ptr<ITransactionContext> ctx() { return context; }
-
-private:
-	std::shared_ptr<ITransactionContext> context;
-	fdb::Error error = fdb::Error::success();
-};
-
-// Type of the lambda functions implementing a transaction
-using TTxStartFct = std::function<void(std::shared_ptr<ITransactionContext>)>;
-
-/**
- * A wrapper class for transactions implemented by lambda functions
- */
-class TransactionFct : public TransactionActorBase {
-public:
-	TransactionFct(TTxStartFct startFct) : startFct(startFct) {}
-	void start() override { startFct(this->ctx()); }
-
-private:
-	TTxStartFct startFct;
-};
+// Type of the lambda functions implementing a database operation
+using TOpContFct = std::function<void(fdb::Error)>;
 
 /**
  * Configuration of transaction execution mode
@@ -124,11 +81,27 @@ struct TransactionExecutorOptions {
 	// Create each transaction in a separate database instance
 	bool databasePerTransaction = false;
 
+	// Enable injection of database create errors
+	bool injectDatabaseCreateErrors = false;
+
+	// Test tampering cluster file contents
+	bool tamperClusterFile = false;
+
+	// The probability of injected database create errors
+	// Used if injectDatabaseCreateErrors = true
+	double databaseCreateErrorRatio = 0.1;
+
 	// The size of the database instance pool
 	int numDatabases = 1;
 
+	// The number of tenants to create in the cluster. If 0, no tenants are used.
+	int numTenants = 0;
+
 	// Maximum number of retries per transaction (0 - unlimited)
 	int transactionRetryLimit = 0;
+
+	// Temporary directory
+	std::string tmpDir;
 };
 
 /**
@@ -140,7 +113,13 @@ class ITransactionExecutor {
 public:
 	virtual ~ITransactionExecutor() {}
 	virtual void init(IScheduler* sched, const char* clusterFile, const std::string& bgBasePath) = 0;
-	virtual void execute(std::shared_ptr<ITransactionActor> tx, TTaskFct cont) = 0;
+	virtual void execute(TOpStartFct start,
+	                     TOpContFct cont,
+	                     std::optional<fdb::BytesRef> tenantName,
+	                     bool transactional) = 0;
+	virtual fdb::Database selectDatabase() = 0;
+	virtual std::string getClusterFileForErrorInjection() = 0;
+	virtual const TransactionExecutorOptions& getOptions() = 0;
 };
 
 // Create a transaction executor for the given options
diff --git a/bindings/c/test/apitester/TesterUtil.cpp b/bindings/c/test/apitester/TesterUtil.cpp
index 0e19081180..6ec9f76f04 100644
--- a/bindings/c/test/apitester/TesterUtil.cpp
+++ b/bindings/c/test/apitester/TesterUtil.cpp
@@ -23,6 +23,9 @@
 #include <algorithm>
 #include <ctype.h>
 #include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <string>
 
 namespace FdbApiTester {
 
@@ -46,16 +49,6 @@ Random& Random::get() {
 	return random;
 }
 
-fdb::ByteString Random::randomStringLowerCase(int minLength, int maxLength) {
-	int length = randomInt(minLength, maxLength);
-	fdb::ByteString str;
-	str.reserve(length);
-	for (int i = 0; i < length; i++) {
-		str += (char)randomInt('a', 'z');
-	}
-	return str;
-}
-
 bool Random::randomBool(double trueRatio) {
 	return std::uniform_real_distribution<double>(0.0, 1.0)(random) <= trueRatio;
 }
@@ -106,4 +99,52 @@ KeyRangeArray copyKeyRangeArray(fdb::future_var::KeyRangeRefArray::Type array) {
 	return out;
 };
 
+GranuleSummaryArray copyGranuleSummaryArray(fdb::future_var::GranuleSummaryRefArray::Type array) {
+	auto& [in_summaries, in_count] = array;
+
+	GranuleSummaryArray out;
+
+	for (int i = 0; i < in_count; ++i) {
+		fdb::native::FDBGranuleSummary nativeSummary = *in_summaries++;
+		fdb::GranuleSummary summary(nativeSummary);
+		out.push_back(summary);
+	}
+	return out;
+};
+
+TmpFile::~TmpFile() {
+	if (!filename.empty()) {
+		remove();
+	}
+}
+
+void TmpFile::create(std::string_view dir, std::string_view prefix) {
+	while (true) {
+		filename = fmt::format("{}/{}-{}", dir, prefix, Random::get().randomStringLowerCase<std::string>(6, 6));
+		if (!std::filesystem::exists(std::filesystem::path(filename))) {
+			break;
+		}
+	}
+
+	// Create an empty tmp file
+	std::fstream tmpFile(filename, std::fstream::out);
+	if (!tmpFile.good()) {
+		throw TesterError(fmt::format("Failed to create temporary file {}\n", filename));
+	}
+}
+
+void TmpFile::write(std::string_view data) {
+	std::ofstream ofs(filename, std::fstream::out | std::fstream::binary);
+	if (!ofs.good()) {
+		throw TesterError(fmt::format("Failed to write to the temporary file {}\n", filename));
+	}
+	ofs.write(data.data(), data.size());
+}
+
+void TmpFile::remove() {
+	if (!std::filesystem::remove(std::filesystem::path(filename))) {
+		fmt::print(stderr, "Failed to remove file {}\n", filename);
+	}
+}
+
 } // namespace FdbApiTester
\ No newline at end of file
diff --git a/bindings/c/test/apitester/TesterUtil.h b/bindings/c/test/apitester/TesterUtil.h
index de5e5c8990..1ace2c9721 100644
--- a/bindings/c/test/apitester/TesterUtil.h
+++ b/bindings/c/test/apitester/TesterUtil.h
@@ -66,7 +66,20 @@ public:
 
 	int randomInt(int min, int max);
 
-	fdb::ByteString randomStringLowerCase(int minLength, int maxLength);
+	template <class StringType>
+	StringType randomStringLowerCase(int minLength, int maxLength) {
+		int length = randomInt(minLength, maxLength);
+		StringType str;
+		str.reserve(length);
+		for (int i = 0; i < length; i++) {
+			str += (char)randomInt('a', 'z');
+		}
+		return str;
+	}
+
+	fdb::ByteString randomByteStringLowerCase(int minLength, int maxLength) {
+		return randomStringLowerCase<fdb::ByteString>(minLength, maxLength);
+	}
 
 	bool randomBool(double trueRatio);
 
@@ -120,6 +133,9 @@ KeyValueArray copyKeyValueArray(fdb::future_var::KeyValueRefArray::Type array);
 using KeyRangeArray = std::vector<fdb::KeyRange>;
 KeyRangeArray copyKeyRangeArray(fdb::future_var::KeyRangeRefArray::Type array);
 
+using GranuleSummaryArray = std::vector<fdb::GranuleSummary>;
+GranuleSummaryArray copyGranuleSummaryArray(fdb::future_var::GranuleSummaryRefArray::Type array);
+
 static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "Do not support non-little-endian systems");
 
 // Converts a little-endian encoded number into an integral type.
@@ -139,6 +155,19 @@ static fdb::ByteString toByteString(T value) {
 	return output;
 }
 
+// Creates a temporary file; file gets destroyed/deleted along with object destruction.
+struct TmpFile {
+public:
+	~TmpFile();
+	void create(std::string_view dir, std::string_view prefix);
+	void write(std::string_view data);
+	void remove();
+	const std::string& getFileName() const { return filename; }
+
+private:
+	std::string filename;
+};
+
 } // namespace FdbApiTester
 
 #endif
diff --git a/bindings/c/test/apitester/TesterWorkload.cpp b/bindings/c/test/apitester/TesterWorkload.cpp
index cbce118f10..8e7289f437 100644
--- a/bindings/c/test/apitester/TesterWorkload.cpp
+++ b/bindings/c/test/apitester/TesterWorkload.cpp
@@ -80,13 +80,14 @@ bool WorkloadConfig::getBoolOption(const std::string& name, bool defaultVal) con
 
 WorkloadBase::WorkloadBase(const WorkloadConfig& config)
   : manager(nullptr), tasksScheduled(0), numErrors(0), clientId(config.clientId), numClients(config.numClients),
-    failed(false), numTxCompleted(0) {
+    failed(false), numTxCompleted(0), numTxStarted(0), inProgress(false) {
 	maxErrors = config.getIntOption("maxErrors", 10);
 	workloadId = fmt::format("{}{}", config.name, clientId);
 }
 
 void WorkloadBase::init(WorkloadManager* manager) {
 	this->manager = manager;
+	inProgress = true;
 }
 
 void WorkloadBase::printStats() {
@@ -94,6 +95,7 @@ void WorkloadBase::printStats() {
 }
 
 void WorkloadBase::schedule(TTaskFct task) {
+	ASSERT(inProgress);
 	if (failed) {
 		return;
 	}
@@ -104,28 +106,49 @@ void WorkloadBase::schedule(TTaskFct task) {
 	});
 }
 
-void WorkloadBase::execTransaction(std::shared_ptr<ITransactionActor> tx, TTaskFct cont, bool failOnError) {
+void WorkloadBase::execTransaction(TOpStartFct startFct,
+                                   TTaskFct cont,
+                                   std::optional<fdb::BytesRef> tenant,
+                                   bool failOnError) {
+	doExecute(startFct, cont, tenant, failOnError, true);
+}
+
+// Execute a non-transactional database operation within the workload
+void WorkloadBase::execOperation(TOpStartFct startFct, TTaskFct cont, bool failOnError) {
+	doExecute(startFct, cont, {}, failOnError, false);
+}
+
+void WorkloadBase::doExecute(TOpStartFct startFct,
+                             TTaskFct cont,
+                             std::optional<fdb::BytesRef> tenant,
+                             bool failOnError,
+                             bool transactional) {
+	ASSERT(inProgress);
 	if (failed) {
 		return;
 	}
 	tasksScheduled++;
-	manager->txExecutor->execute(tx, [this, tx, cont, failOnError]() {
-		numTxCompleted++;
-		fdb::Error err = tx->getError();
-		if (err.code() == error_code_success) {
-			cont();
-		} else {
-			std::string msg = fmt::format("Transaction failed with error: {} ({})", err.code(), err.what());
-			if (failOnError) {
-				error(msg);
-				failed = true;
-			} else {
-				info(msg);
-				cont();
-			}
-		}
-		scheduledTaskDone();
-	});
+	numTxStarted++;
+	manager->txExecutor->execute(
+	    startFct,
+	    [this, startFct, cont, failOnError](fdb::Error err) {
+		    numTxCompleted++;
+		    if (err.code() == error_code_success) {
+			    cont();
+		    } else {
+			    std::string msg = fmt::format("Transaction failed with error: {} ({})", err.code(), err.what());
+			    if (failOnError) {
+				    error(msg);
+				    failed = true;
+			    } else {
+				    info(msg);
+				    cont();
+			    }
+		    }
+		    scheduledTaskDone();
+	    },
+	    tenant,
+	    transactional);
 }
 
 void WorkloadBase::info(const std::string& msg) {
@@ -143,11 +166,13 @@ void WorkloadBase::error(const std::string& msg) {
 
 void WorkloadBase::scheduledTaskDone() {
 	if (--tasksScheduled == 0) {
+		inProgress = false;
 		if (numErrors > 0) {
 			error(fmt::format("Workload failed with {} errors", numErrors.load()));
 		} else {
 			info("Workload successfully completed");
 		}
+		ASSERT(numTxStarted == numTxCompleted);
 		manager->workloadDone(this, numErrors > 0);
 	}
 }
@@ -165,8 +190,11 @@ void WorkloadManager::add(std::shared_ptr<IWorkload> workload, TTaskFct cont) {
 
 void WorkloadManager::run() {
 	std::vector<std::shared_ptr<IWorkload>> initialWorkloads;
-	for (auto iter : workloads) {
-		initialWorkloads.push_back(iter.second.ref);
+	{
+		std::unique_lock<std::mutex> lock(mutex);
+		for (auto iter : workloads) {
+			initialWorkloads.push_back(iter.second.ref);
+		}
 	}
 	for (auto iter : initialWorkloads) {
 		iter->init(this);
@@ -324,4 +352,4 @@ std::unordered_map<std::string, IWorkloadFactory*>& IWorkloadFactory::factories(
 	return theFactories;
 }
 
-} // namespace FdbApiTester
\ No newline at end of file
+} // namespace FdbApiTester
diff --git a/bindings/c/test/apitester/TesterWorkload.h b/bindings/c/test/apitester/TesterWorkload.h
index beb3082c5c..ea1c6816f9 100644
--- a/bindings/c/test/apitester/TesterWorkload.h
+++ b/bindings/c/test/apitester/TesterWorkload.h
@@ -82,6 +82,9 @@ struct WorkloadConfig {
 	// Total number of clients
 	int numClients;
 
+	// Number of Tenants
+	int numTenants;
+
 	// Selected FDB API version
 	int apiVersion;
 
@@ -116,12 +119,13 @@ protected:
 	void schedule(TTaskFct task);
 
 	// Execute a transaction within the workload
-	void execTransaction(std::shared_ptr<ITransactionActor> tx, TTaskFct cont, bool failOnError = true);
+	void execTransaction(TOpStartFct startFct,
+	                     TTaskFct cont,
+	                     std::optional<fdb::BytesRef> tenant = std::optional<fdb::BytesRef>(),
+	                     bool failOnError = true);
 
-	// Execute a transaction within the workload, a convenience method for a tranasaction defined by a lambda function
-	void execTransaction(TTxStartFct start, TTaskFct cont, bool failOnError = true) {
-		execTransaction(std::make_shared<TransactionFct>(start), cont, failOnError);
-	}
+	// Execute a non-transactional database operation within the workload
+	void execOperation(TOpStartFct startFct, TTaskFct cont, bool failOnError = true);
 
 	// Log an error message, increase error counter
 	void error(const std::string& msg);
@@ -135,6 +139,12 @@ protected:
 private:
 	WorkloadManager* manager;
 
+	void doExecute(TOpStartFct startFct,
+	               TTaskFct cont,
+	               std::optional<fdb::BytesRef> tenant,
+	               bool failOnError,
+	               bool transactional);
+
 	// Decrease scheduled task counter, notify the workload manager
 	// that the task is done if no more tasks schedule
 	void scheduledTaskDone();
@@ -164,6 +174,12 @@ protected:
 
 	// Number of completed transactions
 	std::atomic<int> numTxCompleted;
+
+	// Number of started transactions
+	std::atomic<int> numTxStarted;
+
+	// Workload is in progress (intialized, but not completed)
+	std::atomic<bool> inProgress;
 };
 
 // Workload manager
diff --git a/bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleErrorsMultiThr.toml b/bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleErrorsMultiThr.toml
new file mode 100644
index 0000000000..788bd04d85
--- /dev/null
+++ b/bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleErrorsMultiThr.toml
@@ -0,0 +1,22 @@
+[[test]]
+title = 'Blob Granule Errors Multi Threaded'
+multiThreaded = true
+buggify = true
+minFdbThreads = 2
+maxFdbThreads = 8
+minDatabases = 2
+maxDatabases = 8
+minClientThreads = 2
+maxClientThreads = 8
+minClients = 2
+maxClients = 8
+
+	[[test.workload]]
+    name = 'BlobGranuleErrors'
+    minKeyLength = 1
+	maxKeyLength = 64
+	minValueLength = 1
+	maxValueLength = 1000
+	maxKeysPerTransaction = 50
+	initialSize = 100
+	numRandomOperations = 100
\ No newline at end of file
diff --git a/bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleErrorsOnExternalThread.toml b/bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleErrorsOnExternalThread.toml
new file mode 100644
index 0000000000..788bd04d85
--- /dev/null
+++ b/bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleErrorsOnExternalThread.toml
@@ -0,0 +1,22 @@
+[[test]]
+title = 'Blob Granule Errors Multi Threaded'
+multiThreaded = true
+buggify = true
+minFdbThreads = 2
+maxFdbThreads = 8
+minDatabases = 2
+maxDatabases = 8
+minClientThreads = 2
+maxClientThreads = 8
+minClients = 2
+maxClients = 8
+
+	[[test.workload]]
+    name = 'BlobGranuleErrors'
+    minKeyLength = 1
+	maxKeyLength = 64
+	minValueLength = 1
+	maxValueLength = 1000
+	maxKeysPerTransaction = 50
+	initialSize = 100
+	numRandomOperations = 100
\ No newline at end of file
diff --git a/bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleErrorsSingleThr.toml b/bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleErrorsSingleThr.toml
new file mode 100644
index 0000000000..85e78975f6
--- /dev/null
+++ b/bindings/c/test/apitester/blobgranuletests/CApiBlobGranuleErrorsSingleThr.toml
@@ -0,0 +1,15 @@
+[[test]]
+title = 'Blob Granule Errors Single Threaded'
+minClients = 1
+maxClients = 3
+multiThreaded = false
+
+	[[test.workload]]
+    name = 'BlobGranuleErrors'
+    minKeyLength = 1
+	maxKeyLength = 64
+	minValueLength = 1
+	maxValueLength = 1000
+	maxKeysPerTransaction = 50
+	initialSize = 100
+	numRandomOperations = 100
\ No newline at end of file
diff --git a/bindings/c/test/apitester/fdb_c_api_tester.cpp b/bindings/c/test/apitester/fdb_c_api_tester.cpp
index 5a86678cec..1d79dd754c 100644
--- a/bindings/c/test/apitester/fdb_c_api_tester.cpp
+++ b/bindings/c/test/apitester/fdb_c_api_tester.cpp
@@ -36,6 +36,8 @@ namespace FdbApiTester {
 
 namespace {
 
+#define API_VERSION_CLIENT_TMP_DIR 720
+
 enum TesterOptionId {
 	OPT_CONNFILE,
 	OPT_HELP,
@@ -46,6 +48,7 @@ enum TesterOptionId {
 	OPT_KNOB,
 	OPT_EXTERNAL_CLIENT_LIBRARY,
 	OPT_EXTERNAL_CLIENT_DIRECTORY,
+	OPT_FUTURE_VERSION_CLIENT_LIBRARY,
 	OPT_TMP_DIR,
 	OPT_DISABLE_LOCAL_CLIENT,
 	OPT_TEST_FILE,
@@ -72,6 +75,7 @@ CSimpleOpt::SOption TesterOptionDefs[] = //
 	  { OPT_KNOB, "--knob-", SO_REQ_SEP },
 	  { OPT_EXTERNAL_CLIENT_LIBRARY, "--external-client-library", SO_REQ_SEP },
 	  { OPT_EXTERNAL_CLIENT_DIRECTORY, "--external-client-dir", SO_REQ_SEP },
+	  { OPT_FUTURE_VERSION_CLIENT_LIBRARY, "--future-version-client-library", SO_REQ_SEP },
 	  { OPT_TMP_DIR, "--tmp-dir", SO_REQ_SEP },
 	  { OPT_DISABLE_LOCAL_CLIENT, "--disable-local-client", SO_NONE },
 	  { OPT_TEST_FILE, "-f", SO_REQ_SEP },
@@ -110,6 +114,8 @@ void printProgramUsage(const char* execName) {
 	       "                 Path to the external client library.\n"
 	       "  --external-client-dir DIR\n"
 	       "                 Directory containing external client libraries.\n"
+	       "  --future-version-client-library FILE\n"
+	       "                 Path to a client library to be used with a future protocol version.\n"
 	       "  --tmp-dir DIR\n"
 	       "                 Directory for temporary files of the client.\n"
 	       "  --disable-local-client DIR\n"
@@ -204,6 +210,9 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) {
 	case OPT_EXTERNAL_CLIENT_DIRECTORY:
 		options.externalClientDir = args.OptionArg();
 		break;
+	case OPT_FUTURE_VERSION_CLIENT_LIBRARY:
+		options.futureVersionClientLibrary = args.OptionArg();
+		break;
 	case OPT_TMP_DIR:
 		options.tmpDir = args.OptionArg();
 		break;
@@ -278,7 +287,7 @@ void fdb_check(fdb::Error e) {
 }
 
 void applyNetworkOptions(TesterOptions& options) {
-	if (!options.tmpDir.empty()) {
+	if (!options.tmpDir.empty() && options.apiVersion >= API_VERSION_CLIENT_TMP_DIR) {
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_CLIENT_TMP_DIR, options.tmpDir);
 	}
 	if (!options.externalClientLibrary.empty()) {
@@ -296,6 +305,11 @@ void applyNetworkOptions(TesterOptions& options) {
 		}
 	}
 
+	if (!options.futureVersionClientLibrary.empty()) {
+		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_FUTURE_VERSION_CLIENT_LIBRARY,
+		                        options.futureVersionClientLibrary);
+	}
+
 	if (options.testSpec.multiThreaded) {
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_CLIENT_THREADS_PER_VERSION, options.numFdbThreads);
 	}
@@ -308,6 +322,10 @@ void applyNetworkOptions(TesterOptions& options) {
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_CLIENT_BUGGIFY_ENABLE);
 	}
 
+	if (options.testSpec.disableClientBypass && options.apiVersion >= 720) {
+		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_DISABLE_CLIENT_BYPASS);
+	}
+
 	if (options.trace) {
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE, options.traceDir);
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_FORMAT, options.traceFormat);
@@ -338,6 +356,12 @@ void randomizeOptions(TesterOptions& options) {
 	options.numClientThreads = random.randomInt(options.testSpec.minClientThreads, options.testSpec.maxClientThreads);
 	options.numDatabases = random.randomInt(options.testSpec.minDatabases, options.testSpec.maxDatabases);
 	options.numClients = random.randomInt(options.testSpec.minClients, options.testSpec.maxClients);
+
+	// Choose a random number of tenants. If a test is configured to allow 0 tenants, then use 0 tenants half the time.
+	if (options.testSpec.maxTenants >= options.testSpec.minTenants &&
+	    (options.testSpec.minTenants > 0 || random.randomBool(0.5))) {
+		options.numTenants = random.randomInt(options.testSpec.minTenants, options.testSpec.maxTenants);
+	}
 }
 
 bool runWorkloads(TesterOptions& options) {
@@ -346,7 +370,12 @@ bool runWorkloads(TesterOptions& options) {
 		txExecOptions.blockOnFutures = options.testSpec.blockOnFutures;
 		txExecOptions.numDatabases = options.numDatabases;
 		txExecOptions.databasePerTransaction = options.testSpec.databasePerTransaction;
+		// 7.1 and older releases crash on database create errors
+		txExecOptions.injectDatabaseCreateErrors = options.testSpec.buggify && options.apiVersion > 710;
 		txExecOptions.transactionRetryLimit = options.transactionRetryLimit;
+		txExecOptions.tmpDir = options.tmpDir.empty() ? std::string("/tmp") : options.tmpDir;
+		txExecOptions.tamperClusterFile = options.testSpec.tamperClusterFile;
+		txExecOptions.numTenants = options.numTenants;
 
 		std::vector<std::shared_ptr<IWorkload>> workloads;
 		workloads.reserve(options.testSpec.workloads.size() * options.numClients);
@@ -358,6 +387,7 @@ bool runWorkloads(TesterOptions& options) {
 				config.options = workloadSpec.options;
 				config.clientId = i;
 				config.numClients = options.numClients;
+				config.numTenants = options.numTenants;
 				config.apiVersion = options.apiVersion;
 				std::shared_ptr<IWorkload> workload = IWorkloadFactory::create(workloadSpec.name, config);
 				if (!workload) {
@@ -419,7 +449,7 @@ int main(int argc, char** argv) {
 		}
 		randomizeOptions(options);
 
-		fdb::selectApiVersion(options.apiVersion);
+		fdb::selectApiVersionCapped(options.apiVersion);
 		applyNetworkOptions(options);
 		fdb::network::setup();
 
diff --git a/bindings/c/test/apitester/local_tests/CApiCorrectnessSingleThr.toml b/bindings/c/test/apitester/local_tests/CApiCorrectnessSingleThr.toml
new file mode 100644
index 0000000000..9e6fc350ea
--- /dev/null
+++ b/bindings/c/test/apitester/local_tests/CApiCorrectnessSingleThr.toml
@@ -0,0 +1,29 @@
+[[test]]
+title = 'API Correctness Single Threaded'
+minClients = 1
+maxClients = 3
+minDatabases = 1
+maxDatabases = 3
+multiThreaded = false
+disableClientBypass = true
+
+    [[test.workload]]
+    name = 'ApiCorrectness'
+    minKeyLength = 1
+    maxKeyLength = 64
+    minValueLength = 1
+    maxValueLength = 1000
+    maxKeysPerTransaction = 50
+    initialSize = 100
+    numRandomOperations = 100
+    readExistingKeysRatio = 0.9
+
+    [[test.workload]]
+    name = 'AtomicOpsCorrectness'
+    initialSize = 0
+    numRandomOperations = 100
+
+    [[test.workload]]
+    name = 'WatchAndWait'
+    initialSize = 0
+    numRandomOperations = 10
diff --git a/bindings/c/test/apitester/tests/CApiMultiTenantCorrectnessMultiThr.toml b/bindings/c/test/apitester/tests/CApiMultiTenantCorrectnessMultiThr.toml
new file mode 100644
index 0000000000..2a5a0d30e1
--- /dev/null
+++ b/bindings/c/test/apitester/tests/CApiMultiTenantCorrectnessMultiThr.toml
@@ -0,0 +1,21 @@
+[[test]]
+title = 'Multi-tenant API Correctness Multi Threaded'
+multiThreaded = true
+buggify = true
+minFdbThreads = 2
+maxFdbThreads = 8
+minClients = 2
+maxClients = 8
+minTenants = 2
+maxTenants = 5
+
+    [[test.workload]]
+    name = 'ApiCorrectness'
+    minKeyLength = 1
+	maxKeyLength = 64
+	minValueLength = 1
+	maxValueLength = 1000
+	maxKeysPerTransaction = 5
+	initialSize = 100
+	numRandomOperations = 200
+	readExistingKeysRatio = 0.9
\ No newline at end of file
diff --git a/bindings/c/test/apitester/tests/CApiTamperClusterFile.toml b/bindings/c/test/apitester/tests/CApiTamperClusterFile.toml
new file mode 100644
index 0000000000..60a9715bd8
--- /dev/null
+++ b/bindings/c/test/apitester/tests/CApiTamperClusterFile.toml
@@ -0,0 +1,24 @@
+[[test]]
+title = 'Test tampering the cluster file'
+multiThreaded = true
+buggify = true
+tamperClusterFile = true
+minFdbThreads = 2
+maxFdbThreads = 4
+minDatabases = 2
+maxDatabases = 4
+minClientThreads = 2
+maxClientThreads = 4
+minClients = 2
+maxClients = 4
+
+    [[test.workload]]
+    name = 'ApiCorrectness'
+    minKeyLength = 1
+    maxKeyLength = 64
+    minValueLength = 1
+    maxValueLength = 1000
+    maxKeysPerTransaction = 50
+    initialSize = 100
+    numRandomOperations = 100
+    readExistingKeysRatio = 0.9
\ No newline at end of file
diff --git a/bindings/c/test/apitester/tests/upgrade/ApiBlobGranulesCorrectness.toml b/bindings/c/test/apitester/tests/upgrade/ApiBlobGranulesCorrectness.toml
new file mode 100644
index 0000000000..84531ea9c8
--- /dev/null
+++ b/bindings/c/test/apitester/tests/upgrade/ApiBlobGranulesCorrectness.toml
@@ -0,0 +1,23 @@
+[[test]]
+title = 'Mixed Workload for Upgrade Tests with a Multi-Threaded Client'
+multiThreaded = true
+buggify = true
+databasePerTransaction = false
+minFdbThreads = 2
+maxFdbThreads = 8
+minDatabases = 2
+maxDatabases = 8
+minClientThreads = 2
+maxClientThreads = 8
+minClients = 2
+maxClients = 8
+
+    [[test.workload]]
+    name = 'ApiBlobGranuleCorrectness'
+    minKeyLength = 1
+	maxKeyLength = 64
+	minValueLength = 1
+	maxValueLength = 1000
+	maxKeysPerTransaction = 50
+	initialSize = 100
+	runUntilStop = true
\ No newline at end of file
diff --git a/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml b/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
index 86e65c5918..94bf4e0509 100644
--- a/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
+++ b/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
@@ -32,4 +32,14 @@ maxClients = 8
     maxKeysPerTransaction = 50
     initialSize = 100
     runUntilStop = true
-    readExistingKeysRatio = 0.9
\ No newline at end of file
+    readExistingKeysRatio = 0.9
+
+    [[test.workload]]
+    name = 'AtomicOpsCorrectness'
+    initialSize = 0
+    runUntilStop = true
+
+    [[test.workload]]
+    name = 'WatchAndWait'
+    initialSize = 0
+    runUntilStop = true
\ No newline at end of file
diff --git a/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml b/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
index 42df76521b..daf070b31b 100644
--- a/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
+++ b/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
@@ -30,4 +30,14 @@ maxClients = 8
     maxKeysPerTransaction = 50
     initialSize = 100
     runUntilStop = true
-    readExistingKeysRatio = 0.9
\ No newline at end of file
+    readExistingKeysRatio = 0.9
+
+    [[test.workload]]
+    name = 'AtomicOpsCorrectness'
+    initialSize = 0
+    runUntilStop = true
+
+    [[test.workload]]
+    name = 'WatchAndWait'
+    initialSize = 0
+    runUntilStop = true
\ No newline at end of file
diff --git a/bindings/c/test/client_memory_test.cpp b/bindings/c/test/client_memory_test.cpp
index c6cff85574..3ea2f74a8a 100644
--- a/bindings/c/test/client_memory_test.cpp
+++ b/bindings/c/test/client_memory_test.cpp
@@ -44,7 +44,7 @@ int main(int argc, char** argv) {
 	if (argc != 2) {
 		printf("Usage: %s <cluster_file>", argv[0]);
 	}
-	fdb_check(fdb_select_api_version(720));
+	fdb_check(fdb_select_api_version(FDB_API_VERSION));
 	fdb_check(fdb_setup_network());
 	std::thread network_thread{ &fdb_run_network };
 
diff --git a/bindings/c/test/fdb_api.hpp b/bindings/c/test/fdb_api.hpp
index 5dede1bbf3..5403f59b78 100644
--- a/bindings/c/test/fdb_api.hpp
+++ b/bindings/c/test/fdb_api.hpp
@@ -46,6 +46,8 @@ namespace native {
 #include <foundationdb/fdb_c.h>
 }
 
+#define TENANT_API_VERSION_GUARD 720
+
 using ByteString = std::basic_string<uint8_t>;
 using BytesRef = std::basic_string_view<uint8_t>;
 using CharsRef = std::string_view;
@@ -62,6 +64,22 @@ struct KeyRange {
 	Key beginKey;
 	Key endKey;
 };
+struct GranuleSummary {
+	KeyRange keyRange;
+	int64_t snapshotVersion;
+	int64_t snapshotSize;
+	int64_t deltaVersion;
+	int64_t deltaSize;
+
+	GranuleSummary(const native::FDBGranuleSummary& nativeSummary) {
+		keyRange.beginKey = fdb::Key(nativeSummary.key_range.begin_key, nativeSummary.key_range.begin_key_length);
+		keyRange.endKey = fdb::Key(nativeSummary.key_range.end_key, nativeSummary.key_range.end_key_length);
+		snapshotVersion = nativeSummary.snapshot_version;
+		snapshotSize = nativeSummary.snapshot_size;
+		deltaVersion = nativeSummary.delta_version;
+		deltaSize = nativeSummary.delta_size;
+	}
+};
 
 inline uint8_t const* toBytePtr(char const* ptr) noexcept {
 	return reinterpret_cast<uint8_t const*>(ptr);
@@ -114,7 +132,7 @@ public:
 
 	explicit Error(CodeType err) noexcept : err(err) {}
 
-	char const* what() noexcept { return native::fdb_get_error(err); }
+	char const* what() const noexcept { return native::fdb_get_error(err); }
 
 	explicit operator bool() const noexcept { return err != 0; }
 
@@ -137,6 +155,13 @@ struct None {
 	struct Type {};
 	static Error extract(native::FDBFuture*, Type&) noexcept { return Error(0); }
 };
+struct Bool {
+	using Type = native::fdb_bool_t;
+	static Error extract(native::FDBFuture* f, Type& out) noexcept {
+		auto err = native::fdb_future_get_bool(f, &out);
+		return Error(err);
+	}
+};
 struct Int64 {
 	using Type = int64_t;
 	static Error extract(native::FDBFuture* f, Type& out) noexcept {
@@ -200,6 +225,27 @@ struct KeyRangeRefArray {
 	}
 };
 
+struct GranuleSummaryRef : native::FDBGranuleSummary {
+	fdb::KeyRef beginKey() const noexcept {
+		return fdb::KeyRef(native::FDBGranuleSummary::key_range.begin_key,
+		                   native::FDBGranuleSummary::key_range.begin_key_length);
+	}
+	fdb::KeyRef endKey() const noexcept {
+		return fdb::KeyRef(native::FDBGranuleSummary::key_range.end_key,
+		                   native::FDBGranuleSummary::key_range.end_key_length);
+	}
+};
+
+struct GranuleSummaryRefArray {
+	using Type = std::tuple<GranuleSummaryRef const*, int>;
+	static Error extract(native::FDBFuture* f, Type& out) noexcept {
+		auto& [out_summaries, out_count] = out;
+		auto err = native::fdb_future_get_granule_summary_array(
+		    f, reinterpret_cast<const native::FDBGranuleSummary**>(&out_summaries), &out_count);
+		return Error(err);
+	}
+};
+
 } // namespace future_var
 
 [[noreturn]] inline void throwError(std::string_view preamble, Error err) {
@@ -310,6 +356,7 @@ public:
 class Future {
 protected:
 	friend class Transaction;
+	friend class Database;
 	friend std::hash<Future>;
 	std::shared_ptr<native::FDBFuture> f;
 
@@ -468,6 +515,14 @@ public:
 	Transaction(const Transaction&) noexcept = default;
 	Transaction& operator=(const Transaction&) noexcept = default;
 
+	void atomic_store(Transaction other) { std::atomic_store(&tr, other.tr); }
+
+	Transaction atomic_load() {
+		Transaction retVal;
+		retVal.tr = std::atomic_load(&tr);
+		return retVal;
+	}
+
 	bool valid() const noexcept { return tr != nullptr; }
 
 	explicit operator bool() const noexcept { return valid(); }
@@ -559,9 +614,9 @@ public:
 		                                         reverse);
 	}
 
-	TypedFuture<future_var::KeyRangeRefArray> getBlobGranuleRanges(KeyRef begin, KeyRef end) {
+	TypedFuture<future_var::KeyRangeRefArray> getBlobGranuleRanges(KeyRef begin, KeyRef end, int rangeLimit) {
 		return native::fdb_transaction_get_blob_granule_ranges(
-		    tr.get(), begin.data(), intSize(begin), end.data(), intSize(end));
+		    tr.get(), begin.data(), intSize(begin), end.data(), intSize(end), rangeLimit);
 	}
 
 	Result readBlobGranules(KeyRef begin,
@@ -573,6 +628,14 @@ public:
 		    tr.get(), begin.data(), intSize(begin), end.data(), intSize(end), begin_version, read_version, context));
 	}
 
+	TypedFuture<future_var::GranuleSummaryRefArray> summarizeBlobGranules(KeyRef begin,
+	                                                                      KeyRef end,
+	                                                                      int64_t summaryVersion,
+	                                                                      int rangeLimit) {
+		return native::fdb_transaction_summarize_blob_granules(
+		    tr.get(), begin.data(), intSize(begin), end.data(), intSize(end), summaryVersion, rangeLimit);
+	}
+
 	TypedFuture<future_var::None> watch(KeyRef key) {
 		return native::fdb_transaction_watch(tr.get(), key.data(), intSize(key));
 	}
@@ -621,6 +684,7 @@ public:
 	static void createTenant(Transaction tr, BytesRef name) {
 		tr.setOption(FDBTransactionOption::FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, BytesRef());
 		tr.setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE, BytesRef());
+		tr.setOption(FDBTransactionOption::FDB_TR_OPTION_RAW_ACCESS, BytesRef());
 		tr.set(toBytesRef(fmt::format("{}{}", tenantManagementMapPrefix, toCharsRef(name))), BytesRef());
 	}
 
@@ -662,6 +726,14 @@ public:
 	}
 	Database() noexcept : db(nullptr) {}
 
+	void atomic_store(Database other) { std::atomic_store(&db, other.db); }
+
+	Database atomic_load() {
+		Database retVal;
+		retVal.db = std::atomic_load(&db);
+		return retVal;
+	}
+
 	Error setOptionNothrow(FDBDatabaseOption option, int64_t value) noexcept {
 		return Error(native::fdb_database_set_option(
 		    db.get(), option, reinterpret_cast<const uint8_t*>(&value), static_cast<int>(sizeof(value))));
@@ -707,10 +779,50 @@ public:
 			throwError("Failed to create transaction: ", err);
 		return Transaction(tx_native);
 	}
+
+	TypedFuture<future_var::KeyRangeRefArray> listBlobbifiedRanges(KeyRef begin, KeyRef end, int rangeLimit) {
+		if (!db)
+			throw std::runtime_error("listBlobbifiedRanges from null database");
+		return native::fdb_database_list_blobbified_ranges(
+		    db.get(), begin.data(), intSize(begin), end.data(), intSize(end), rangeLimit);
+	}
+
+	TypedFuture<future_var::Int64> verifyBlobRange(KeyRef begin, KeyRef end, int64_t version) {
+		if (!db)
+			throw std::runtime_error("verifyBlobRange from null database");
+		return native::fdb_database_verify_blob_range(
+		    db.get(), begin.data(), intSize(begin), end.data(), intSize(end), version);
+	}
+
+	TypedFuture<future_var::Bool> blobbifyRange(KeyRef begin, KeyRef end) {
+		if (!db)
+			throw std::runtime_error("blobbifyRange from null database");
+		return native::fdb_database_blobbify_range(db.get(), begin.data(), intSize(begin), end.data(), intSize(end));
+	}
+
+	TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) {
+		if (!db)
+			throw std::runtime_error("unblobbifyRange from null database");
+		return native::fdb_database_unblobbify_range(db.get(), begin.data(), intSize(begin), end.data(), intSize(end));
+	}
+
+	TypedFuture<future_var::KeyRef> purgeBlobGranules(KeyRef begin, KeyRef end, int64_t version, bool force) {
+		if (!db)
+			throw std::runtime_error("purgeBlobGranules from null database");
+		native::fdb_bool_t forceBool = force;
+		return native::fdb_database_purge_blob_granules(
+		    db.get(), begin.data(), intSize(begin), end.data(), intSize(end), version, forceBool);
+	}
+
+	TypedFuture<future_var::None> waitPurgeGranulesComplete(KeyRef purgeKey) {
+		if (!db)
+			throw std::runtime_error("purgeBlobGranules from null database");
+		return native::fdb_database_wait_purge_granules_complete(db.get(), purgeKey.data(), intSize(purgeKey));
+	}
 };
 
 inline Error selectApiVersionNothrow(int version) {
-	if (version < 720) {
+	if (version < TENANT_API_VERSION_GUARD) {
 		Tenant::tenantManagementMapPrefix = "\xff\xff/management/tenant_map/";
 	}
 	return Error(native::fdb_select_api_version(version));
@@ -722,6 +834,20 @@ inline void selectApiVersion(int version) {
 	}
 }
 
+inline Error selectApiVersionCappedNothrow(int version) {
+	if (version < TENANT_API_VERSION_GUARD) {
+		Tenant::tenantManagementMapPrefix = "\xff\xff/management/tenant_map/";
+	}
+	return Error(
+	    native::fdb_select_api_version_impl(version, std::min(native::fdb_get_max_api_version(), FDB_API_VERSION)));
+}
+
+inline void selectApiVersionCapped(int version) {
+	if (auto err = selectApiVersionCappedNothrow(version)) {
+		throwError(fmt::format("ERROR: fdb_select_api_version_capped({}): ", version), err);
+	}
+}
+
 } // namespace fdb
 
 template <>
diff --git a/bindings/c/test/fdb_c90_test.c b/bindings/c/test/fdb_c90_test.c
index e2011286ed..1bcdf63284 100644
--- a/bindings/c/test/fdb_c90_test.c
+++ b/bindings/c/test/fdb_c90_test.c
@@ -4,6 +4,6 @@
 int main(int argc, char* argv[]) {
 	(void)argc;
 	(void)argv;
-	fdb_select_api_version(720);
+	fdb_select_api_version(FDB_API_VERSION);
 	return 0;
 }
diff --git a/bindings/c/test/fdb_c_shim_tests.py b/bindings/c/test/fdb_c_shim_tests.py
index 138d94415f..b18ca3968d 100644
--- a/bindings/c/test/fdb_c_shim_tests.py
+++ b/bindings/c/test/fdb_c_shim_tests.py
@@ -7,13 +7,18 @@ import subprocess
 import sys
 import os
 
-sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tests', 'TestRunner')]
+sys.path[:0] = [os.path.join(os.path.dirname(
+    __file__), '..', '..', '..', 'tests', 'TestRunner')]
+
+# fmt: off
 from binary_download import FdbBinaryDownloader, CURRENT_VERSION
 from local_cluster import LocalCluster, random_secret_string
+# fmt: on
 
 LAST_RELEASE_VERSION = "7.1.5"
 TESTER_STATS_INTERVAL_SEC = 5
 DEFAULT_TEST_FILE = "CApiCorrectnessMultiThr.toml"
+IMPLIBSO_ERROR_CODE = -6  # SIGABORT
 
 
 def version_from_str(ver_str):
@@ -55,7 +60,8 @@ class TestEnv(LocalCluster):
         self.set_env_var("LD_LIBRARY_PATH", self.downloader.lib_dir(version))
         client_lib = self.downloader.lib_path(version)
         assert client_lib.exists(), "{} does not exist".format(client_lib)
-        self.client_lib_external = self.tmp_dir.joinpath("libfdb_c_external.so")
+        self.client_lib_external = self.tmp_dir.joinpath(
+            "libfdb_c_external.so")
         shutil.copyfile(client_lib, self.client_lib_external)
 
     def __enter__(self):
@@ -91,6 +97,9 @@ class FdbCShimTests:
         assert self.unit_tests_bin.exists(), "{} does not exist".format(self.unit_tests_bin)
         self.api_tester_bin = Path(args.api_tester_bin).resolve()
         assert self.api_tester_bin.exists(), "{} does not exist".format(self.api_tests_bin)
+        self.shim_lib_tester_bin = Path(args.shim_lib_tester_bin).resolve()
+        assert self.shim_lib_tester_bin.exists(
+        ), "{} does not exist".format(self.shim_lib_tester_bin)
         self.api_test_dir = Path(args.api_test_dir).resolve()
         assert self.api_test_dir.exists(), "{} does not exist".format(self.api_test_dir)
         self.downloader = FdbBinaryDownloader(args.build_dir)
@@ -98,6 +107,7 @@ class FdbCShimTests:
         self.platform = platform.machine()
         if (self.platform == "x86_64"):
             self.downloader.download_old_binaries(LAST_RELEASE_VERSION)
+            self.downloader.download_old_binaries("7.0.0")
 
     def build_c_api_tester_args(self, test_env, test_file):
         test_file_path = self.api_test_dir.joinpath(test_file)
@@ -128,7 +138,8 @@ class FdbCShimTests:
         with TestEnv(self.build_dir, self.downloader, version) as test_env:
             cmd_args = self.build_c_api_tester_args(test_env, test_file)
             env_vars = os.environ.copy()
-            env_vars["LD_LIBRARY_PATH"] = self.downloader.lib_dir(version)
+            env_vars["FDB_LOCAL_CLIENT_LIBRARY_PATH"] = self.downloader.lib_path(
+                version)
             test_env.exec_client_command(cmd_args, env_vars)
 
     def run_c_unit_tests(self, version):
@@ -143,38 +154,118 @@ class FdbCShimTests:
                 test_env.client_lib_external
             ]
             env_vars = os.environ.copy()
-            env_vars["LD_LIBRARY_PATH"] = self.downloader.lib_dir(version)
+            env_vars["FDB_LOCAL_CLIENT_LIBRARY_PATH"] = self.downloader.lib_path(
+                version)
             test_env.exec_client_command(cmd_args, env_vars)
 
-    def test_invalid_c_client_lib_env_var(self, version):
+    def run_c_shim_lib_tester(
+        self,
+        version,
+        test_env,
+        api_version=None,
+        invalid_lib_path=False,
+        call_set_path=False,
+        set_env_path=False,
+        set_ld_lib_path=False,
+        use_external_lib=True,
+        expected_ret_code=0
+    ):
         print('-' * 80)
-        print("Test invalid FDB_C_CLIENT_LIBRARY_PATH value")
+        if api_version is None:
+            api_version = api_version_from_str(version)
+        test_flags = []
+        if invalid_lib_path:
+            test_flags.append("invalid_lib_path")
+        if call_set_path:
+            test_flags.append("call_set_path")
+        if set_ld_lib_path:
+            test_flags.append("set_ld_lib_path")
+        if use_external_lib:
+            test_flags.append("use_external_lib")
+        else:
+            test_flags.append("use_local_lib")
+        print("C Shim Tests - version: {}, API version: {}, {}".format(version,
+              api_version, ", ".join(test_flags)))
         print('-' * 80)
-        with TestEnv(self.build_dir, self.downloader, version) as test_env:
-            cmd_args = self.build_c_api_tester_args(test_env, DEFAULT_TEST_FILE)
-            env_vars = os.environ.copy()
-            env_vars["FDB_C_CLIENT_LIBRARY_PATH"] = "dummy"
-            test_env.exec_client_command(cmd_args, env_vars, 1)
-
-    def test_valid_c_client_lib_env_var(self, version):
-        print('-' * 80)
-        print("Test valid FDB_C_CLIENT_LIBRARY_PATH value")
-        print('-' * 80)
-        with TestEnv(self.build_dir, self.downloader, version) as test_env:
-            cmd_args = self.build_c_api_tester_args(test_env, DEFAULT_TEST_FILE)
-            env_vars = os.environ.copy()
-            env_vars["FDB_C_CLIENT_LIBRARY_PATH"] = self.downloader.lib_path(version)
-            test_env.exec_client_command(cmd_args, env_vars)
+        cmd_args = [
+            self.shim_lib_tester_bin,
+            "--cluster-file",
+            test_env.cluster_file,
+            "--api-version",
+            str(api_version),
+        ]
+        if call_set_path:
+            cmd_args = cmd_args + [
+                "--local-client-library",
+                ("dummy" if invalid_lib_path else self.downloader.lib_path(version))
+            ]
+        if use_external_lib:
+            cmd_args = cmd_args + [
+                "--disable-local-client",
+                "--external-client-library",
+                test_env.client_lib_external
+            ]
+        env_vars = os.environ.copy()
+        env_vars["LD_LIBRARY_PATH"] = (
+            self.downloader.lib_dir(version) if set_ld_lib_path else "")
+        if set_env_path:
+            env_vars["FDB_LOCAL_CLIENT_LIBRARY_PATH"] = (
+                "dummy" if invalid_lib_path else self.downloader.lib_path(version))
+        test_env.exec_client_command(cmd_args, env_vars, expected_ret_code)
 
     def run_tests(self):
+        # Test the API workload with the dev version
+        self.run_c_api_test(CURRENT_VERSION, DEFAULT_TEST_FILE)
+
+        # Run unit tests with the dev version
+        self.run_c_unit_tests(CURRENT_VERSION)
+
+        with TestEnv(self.build_dir, self.downloader, CURRENT_VERSION) as test_env:
+            # Test lookup of the client library over LD_LIBRARY_PATH
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, set_ld_lib_path=True)
+
+            # Test setting the client library path over an API call
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, call_set_path=True)
+
+            # Test setting the client library path over an environment variable
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, set_env_path=True)
+
+            # Test using the loaded client library as the local client
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, call_set_path=True, use_external_lib=False)
+
+            # Test setting an invalid client library path over an API call
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, call_set_path=True, invalid_lib_path=True, expected_ret_code=IMPLIBSO_ERROR_CODE)
+
+            # Test setting an invalid client library path over an environment variable
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, set_env_path=True, invalid_lib_path=True, expected_ret_code=IMPLIBSO_ERROR_CODE)
+
+            # Test calling a function that exists in the loaded library, but not for the selected API version
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, call_set_path=True, api_version=700)
+
         # binary downloads are currently available only for x86_64
-        if (self.platform == "x86_64"):
+        if self.platform == "x86_64":
+            # Test the API workload with the release version
             self.run_c_api_test(LAST_RELEASE_VERSION, DEFAULT_TEST_FILE)
 
-        self.run_c_api_test(CURRENT_VERSION, DEFAULT_TEST_FILE)
-        self.run_c_unit_tests(CURRENT_VERSION)
-        self.test_invalid_c_client_lib_env_var(CURRENT_VERSION)
-        self.test_valid_c_client_lib_env_var(CURRENT_VERSION)
+            with TestEnv(self.build_dir, self.downloader, LAST_RELEASE_VERSION) as test_env:
+                # Test using the loaded client library as the local client
+                self.run_c_shim_lib_tester(
+                    LAST_RELEASE_VERSION, test_env, call_set_path=True, use_external_lib=False)
+
+                # Test the client library of the release version in combination with the dev API version
+                self.run_c_shim_lib_tester(
+                    LAST_RELEASE_VERSION, test_env, call_set_path=True, api_version=api_version_from_str(CURRENT_VERSION), expected_ret_code=1)
+
+                # Test calling a function that does not exist in the loaded library
+                self.run_c_shim_lib_tester(
+                    "7.0.0", test_env, call_set_path=True, api_version=700, expected_ret_code=IMPLIBSO_ERROR_CODE)
 
 
 if __name__ == "__main__":
@@ -194,12 +285,26 @@ if __name__ == "__main__":
         help="FDB build directory",
         required=True,
     )
-    parser.add_argument('--unit-tests-bin', type=str,
-                        help='Path to the fdb_c_shim_unit_tests executable.')
-    parser.add_argument('--api-tester-bin', type=str,
-                        help='Path to the fdb_c_shim_api_tester executable.')
-    parser.add_argument('--api-test-dir', type=str,
-                        help='Path to a directory with api test definitions.')
+    parser.add_argument(
+        '--unit-tests-bin',
+        type=str,
+        help='Path to the fdb_c_shim_unit_tests executable.',
+        required=True)
+    parser.add_argument(
+        '--api-tester-bin',
+        type=str,
+        help='Path to the fdb_c_shim_api_tester executable.',
+        required=True)
+    parser.add_argument(
+        '--shim-lib-tester-bin',
+        type=str,
+        help='Path to the fdb_c_shim_lib_tester executable.',
+        required=True)
+    parser.add_argument(
+        '--api-test-dir',
+        type=str,
+        help='Path to a directory with api test definitions.',
+        required=True)
     args = parser.parse_args()
     test = FdbCShimTests(args)
     test.run_tests()
diff --git a/bindings/c/test/mako/blob_granules.cpp b/bindings/c/test/mako/blob_granules.cpp
index af805f2e56..1071737211 100644
--- a/bindings/c/test/mako/blob_granules.cpp
+++ b/bindings/c/test/mako/blob_granules.cpp
@@ -26,6 +26,9 @@
 
 extern thread_local mako::Logger logr;
 
+// FIXME: use the same implementation as the api tester! this implementation was from back when mako was written in C
+// and is inferior.
+
 namespace mako::blob_granules::local_file {
 
 int64_t startLoad(const char* filename,
diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c
index 5cd9f64bc0..ab90395e05 100644
--- a/bindings/c/test/performance_test.c
+++ b/bindings/c/test/performance_test.c
@@ -641,7 +641,7 @@ void runTests(struct ResultSet* rs) {
 int main(int argc, char** argv) {
 	srand(time(NULL));
 	struct ResultSet* rs = newResultSet();
-	checkError(fdb_select_api_version(720), "select API version", rs);
+	checkError(fdb_select_api_version(FDB_API_VERSION), "select API version", rs);
 	printf("Running performance test at client version: %s\n", fdb_get_client_version());
 
 	valueStr = (uint8_t*)malloc((sizeof(uint8_t)) * valueSize);
diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c
index cf2754bcec..731a2ce0a2 100644
--- a/bindings/c/test/ryw_benchmark.c
+++ b/bindings/c/test/ryw_benchmark.c
@@ -285,7 +285,7 @@ void runTests(struct ResultSet* rs) {
 int main(int argc, char** argv) {
 	srand(time(NULL));
 	struct ResultSet* rs = newResultSet();
-	checkError(fdb_select_api_version(720), "select API version", rs);
+	checkError(fdb_select_api_version(FDB_API_VERSION), "select API version", rs);
 	printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version());
 
 	keys = generateKeys(numKeys, keySize);
diff --git a/bindings/c/test/shim_lib_tester.cpp b/bindings/c/test/shim_lib_tester.cpp
new file mode 100644
index 0000000000..c8813c45d0
--- /dev/null
+++ b/bindings/c/test/shim_lib_tester.cpp
@@ -0,0 +1,253 @@
+/*
+ * shim_lib_tester.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * A utility for testing shim library usage with various valid and invalid configurations
+ */
+
+#include "fmt/core.h"
+#include "test/fdb_api.hpp"
+#include "SimpleOpt/SimpleOpt.h"
+#include <thread>
+#include <string_view>
+#include "foundationdb/fdb_c_shim.h"
+
+#undef ERROR
+#define ERROR(name, number, description) enum { error_code_##name = number };
+
+#include "flow/error_definitions.h"
+
+using namespace std::string_view_literals;
+
+namespace {
+
+enum TesterOptionId {
+	OPT_HELP,
+	OPT_CONNFILE,
+	OPT_LOCAL_CLIENT_LIBRARY,
+	OPT_EXTERNAL_CLIENT_LIBRARY,
+	OPT_EXTERNAL_CLIENT_DIRECTORY,
+	OPT_DISABLE_LOCAL_CLIENT,
+	OPT_API_VERSION
+};
+
+const int MIN_TESTABLE_API_VERSION = 400;
+
+CSimpleOpt::SOption TesterOptionDefs[] = //
+    { { OPT_HELP, "-h", SO_NONE },
+	  { OPT_HELP, "--help", SO_NONE },
+	  { OPT_CONNFILE, "-C", SO_REQ_SEP },
+	  { OPT_CONNFILE, "--cluster-file", SO_REQ_SEP },
+	  { OPT_LOCAL_CLIENT_LIBRARY, "--local-client-library", SO_REQ_SEP },
+	  { OPT_EXTERNAL_CLIENT_LIBRARY, "--external-client-library", SO_REQ_SEP },
+	  { OPT_EXTERNAL_CLIENT_DIRECTORY, "--external-client-dir", SO_REQ_SEP },
+	  { OPT_DISABLE_LOCAL_CLIENT, "--disable-local-client", SO_NONE },
+	  { OPT_API_VERSION, "--api-version", SO_REQ_SEP },
+	  SO_END_OF_OPTIONS };
+
+class TesterOptions {
+public:
+	// FDB API version, using the latest version by default
+	int apiVersion = FDB_API_VERSION;
+	std::string clusterFile;
+	std::string localClientLibrary;
+	std::string externalClientLibrary;
+	std::string externalClientDir;
+	bool disableLocalClient = false;
+};
+
+void printProgramUsage(const char* execName) {
+	printf("usage: %s [OPTIONS]\n"
+	       "\n",
+	       execName);
+	printf("  -C, --cluster-file FILE\n"
+	       "                 The path of a file containing the connection string for the\n"
+	       "                 FoundationDB cluster. The default is `fdb.cluster'\n"
+	       "  --local-client-library FILE\n"
+	       "                 Path to the local client library.\n"
+	       "  --external-client-library FILE\n"
+	       "                 Path to the external client library.\n"
+	       "  --external-client-dir DIR\n"
+	       "                 Directory containing external client libraries.\n"
+	       "  --disable-local-client DIR\n"
+	       "                 Disable the local client, i.e. use only external client libraries.\n"
+	       "  --api-version VERSION\n"
+	       "                 Required FDB API version (default %d).\n"
+	       "  -h, --help     Display this help and exit.\n",
+	       FDB_API_VERSION);
+}
+
+bool processIntOption(const std::string& optionName, const std::string& value, int minValue, int maxValue, int& res) {
+	char* endptr;
+	res = strtol(value.c_str(), &endptr, 10);
+	if (*endptr != '\0') {
+		fmt::print(stderr, "Invalid value {} for {}", value, optionName);
+		return false;
+	}
+	if (res < minValue || res > maxValue) {
+		fmt::print(stderr, "Value for {} must be between {} and {}", optionName, minValue, maxValue);
+		return false;
+	}
+	return true;
+}
+
+bool processArg(TesterOptions& options, const CSimpleOpt& args) {
+	switch (args.OptionId()) {
+	case OPT_CONNFILE:
+		options.clusterFile = args.OptionArg();
+		break;
+	case OPT_LOCAL_CLIENT_LIBRARY:
+		options.localClientLibrary = args.OptionArg();
+		break;
+	case OPT_EXTERNAL_CLIENT_LIBRARY:
+		options.externalClientLibrary = args.OptionArg();
+		break;
+	case OPT_EXTERNAL_CLIENT_DIRECTORY:
+		options.externalClientDir = args.OptionArg();
+		break;
+	case OPT_DISABLE_LOCAL_CLIENT:
+		options.disableLocalClient = true;
+		break;
+	case OPT_API_VERSION:
+		if (!processIntOption(
+		        args.OptionText(), args.OptionArg(), MIN_TESTABLE_API_VERSION, FDB_API_VERSION, options.apiVersion)) {
+			return false;
+		}
+		break;
+	}
+	return true;
+}
+
+bool parseArgs(TesterOptions& options, int argc, char** argv) {
+	// declare our options parser, pass in the arguments from main
+	// as well as our array of valid options.
+	CSimpleOpt args(argc, argv, TesterOptionDefs);
+
+	// while there are arguments left to process
+	while (args.Next()) {
+		if (args.LastError() == SO_SUCCESS) {
+			if (args.OptionId() == OPT_HELP) {
+				printProgramUsage(argv[0]);
+				return false;
+			}
+			if (!processArg(options, args)) {
+				return false;
+			}
+		} else {
+			fmt::print(stderr, "ERROR: Invalid argument: {}\n", args.OptionText());
+			printProgramUsage(argv[0]);
+			return false;
+		}
+	}
+	return true;
+}
+
+void fdb_check(fdb::Error e, std::string_view msg, fdb::Error::CodeType expectedError = error_code_success) {
+	if (e.code()) {
+		fmt::print(stderr, "{}, Error: {}({})\n", msg, e.code(), e.what());
+		std::abort();
+	}
+}
+
+void applyNetworkOptions(TesterOptions& options) {
+	if (!options.externalClientLibrary.empty()) {
+		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT);
+		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_LIBRARY,
+		                        options.externalClientLibrary);
+	} else if (!options.externalClientDir.empty()) {
+		if (options.disableLocalClient) {
+			fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT);
+		}
+		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_DIRECTORY, options.externalClientDir);
+	} else {
+		if (options.disableLocalClient) {
+			fmt::print(stderr, "Invalid options: Cannot disable local client if no external library is provided");
+			exit(1);
+		}
+	}
+}
+
+void testBasicApi(const TesterOptions& options) {
+	fdb::Database db(options.clusterFile);
+	fdb::Transaction tx = db.createTransaction();
+	while (true) {
+		try {
+			// Set a time out to avoid long delays when testing invalid configurations
+			tx.setOption(FDB_TR_OPTION_TIMEOUT, 1000);
+			tx.set(fdb::toBytesRef("key1"sv), fdb::toBytesRef("val1"sv));
+			fdb_check(tx.commit().blockUntilReady(), "Wait on commit failed");
+			break;
+		} catch (const fdb::Error& err) {
+			if (err.code() == error_code_timed_out) {
+				exit(1);
+			}
+			auto onErrorFuture = tx.onError(err);
+			fdb_check(onErrorFuture.blockUntilReady(), "Wait on onError failed");
+			fdb_check(onErrorFuture.error(), "onError failed");
+		}
+	}
+}
+
+void test710Api(const TesterOptions& options) {
+	fdb::Database db(options.clusterFile);
+	try {
+		db.openTenant(fdb::toBytesRef("not_existing_tenant"sv));
+	} catch (const fdb::Error& err) {
+		fdb_check(err, "Tenant not found expected", error_code_tenant_not_found);
+	}
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+	int retCode = 0;
+	try {
+		TesterOptions options;
+		if (!parseArgs(options, argc, argv)) {
+			return 1;
+		}
+
+		if (!options.localClientLibrary.empty()) {
+			// Must be called before the first FDB API call
+			fdb_shim_set_local_client_library_path(options.localClientLibrary.c_str());
+		}
+
+		fdb::selectApiVersionCapped(options.apiVersion);
+		applyNetworkOptions(options);
+		fdb::network::setup();
+
+		std::thread network_thread{ &fdb::network::run };
+
+		// Try calling some basic functionality that is available
+		// in all recent API versions
+		testBasicApi(options);
+
+		// Try calling 710-specific API. This enables testing what
+		// happens if a library is missing a function
+		test710Api(options);
+
+		fdb_check(fdb::network::stop(), "Stop network failed");
+		network_thread.join();
+	} catch (const std::runtime_error& err) {
+		fmt::print(stderr, "runtime error caught: {}\n", err.what());
+		retCode = 1;
+	}
+	return retCode;
+}
diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c
index 97081f24a6..57c74a9bca 100644
--- a/bindings/c/test/txn_size_test.c
+++ b/bindings/c/test/txn_size_test.c
@@ -97,7 +97,7 @@ void runTests(struct ResultSet* rs) {
 int main(int argc, char** argv) {
 	srand(time(NULL));
 	struct ResultSet* rs = newResultSet();
-	checkError(fdb_select_api_version(720), "select API version", rs);
+	checkError(fdb_select_api_version(FDB_API_VERSION), "select API version", rs);
 	printf("Running performance test at client version: %s\n", fdb_get_client_version());
 
 	keys = generateKeys(numKeys, KEY_SIZE);
diff --git a/bindings/c/test/unit/disconnected_timeout_tests.cpp b/bindings/c/test/unit/disconnected_timeout_tests.cpp
index b1c6b72730..7d006faa23 100644
--- a/bindings/c/test/unit/disconnected_timeout_tests.cpp
+++ b/bindings/c/test/unit/disconnected_timeout_tests.cpp
@@ -255,7 +255,7 @@ int main(int argc, char** argv) {
 		          << std::endl;
 		return 1;
 	}
-	fdb_check(fdb_select_api_version(720));
+	fdb_check(fdb_select_api_version(FDB_API_VERSION));
 	if (argc >= 3) {
 		std::string externalClientLibrary = argv[2];
 		if (externalClientLibrary.substr(0, 2) != "--") {
diff --git a/bindings/c/test/unit/fdb_api.cpp b/bindings/c/test/unit/fdb_api.cpp
index d3c1dec30d..1376fc77c7 100644
--- a/bindings/c/test/unit/fdb_api.cpp
+++ b/bindings/c/test/unit/fdb_api.cpp
@@ -84,6 +84,12 @@ void Future::cancel() {
 	return fdb_future_get_keyrange_array(future_, out_keyranges, out_count);
 }
 
+// GranuleSummaryArrayFuture
+
+[[nodiscard]] fdb_error_t GranuleSummaryArrayFuture::get(const FDBGranuleSummary** out_summaries, int* out_count) {
+	return fdb_future_get_granule_summary_array(future_, out_summaries, out_count);
+}
+
 // KeyValueArrayFuture
 
 [[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more) {
@@ -356,10 +362,17 @@ fdb_error_t Transaction::add_conflict_range(std::string_view begin_key,
 	    tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size(), type);
 }
 
-KeyRangeArrayFuture Transaction::get_blob_granule_ranges(std::string_view begin_key, std::string_view end_key) {
-	return KeyRangeArrayFuture(fdb_transaction_get_blob_granule_ranges(
-	    tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size()));
+KeyRangeArrayFuture Transaction::get_blob_granule_ranges(std::string_view begin_key,
+                                                         std::string_view end_key,
+                                                         int rangeLimit) {
+	return KeyRangeArrayFuture(fdb_transaction_get_blob_granule_ranges(tr_,
+	                                                                   (const uint8_t*)begin_key.data(),
+	                                                                   begin_key.size(),
+	                                                                   (const uint8_t*)end_key.data(),
+	                                                                   end_key.size(),
+	                                                                   rangeLimit));
 }
+
 KeyValueArrayResult Transaction::read_blob_granules(std::string_view begin_key,
                                                     std::string_view end_key,
                                                     int64_t beginVersion,
@@ -375,4 +388,17 @@ KeyValueArrayResult Transaction::read_blob_granules(std::string_view begin_key,
 	                                                              granuleContext));
 }
 
+GranuleSummaryArrayFuture Transaction::summarize_blob_granules(std::string_view begin_key,
+                                                               std::string_view end_key,
+                                                               int64_t summary_version,
+                                                               int rangeLimit) {
+	return GranuleSummaryArrayFuture(fdb_transaction_summarize_blob_granules(tr_,
+	                                                                         (const uint8_t*)begin_key.data(),
+	                                                                         begin_key.size(),
+	                                                                         (const uint8_t*)end_key.data(),
+	                                                                         end_key.size(),
+	                                                                         summary_version,
+	                                                                         rangeLimit));
+}
+
 } // namespace fdb
diff --git a/bindings/c/test/unit/fdb_api.hpp b/bindings/c/test/unit/fdb_api.hpp
index 7d44a30a9a..137083a90c 100644
--- a/bindings/c/test/unit/fdb_api.hpp
+++ b/bindings/c/test/unit/fdb_api.hpp
@@ -161,6 +161,18 @@ private:
 	KeyRangeArrayFuture(FDBFuture* f) : Future(f) {}
 };
 
+class GranuleSummaryArrayFuture : public Future {
+public:
+	// Call this function instead of fdb_future_get_granule_summary_array when using
+	// the GranuleSummaryArrayFuture type. It's behavior is identical to
+	// fdb_future_get_granule_summary_array.
+	fdb_error_t get(const FDBGranuleSummary** out_summaries, int* out_count);
+
+private:
+	friend class Transaction;
+	GranuleSummaryArrayFuture(FDBFuture* f) : Future(f) {}
+};
+
 class EmptyFuture : public Future {
 private:
 	friend class Transaction;
@@ -348,12 +360,16 @@ public:
 	// Wrapper around fdb_transaction_add_conflict_range.
 	fdb_error_t add_conflict_range(std::string_view begin_key, std::string_view end_key, FDBConflictRangeType type);
 
-	KeyRangeArrayFuture get_blob_granule_ranges(std::string_view begin_key, std::string_view end_key);
+	KeyRangeArrayFuture get_blob_granule_ranges(std::string_view begin_key, std::string_view end_key, int rangeLimit);
 	KeyValueArrayResult read_blob_granules(std::string_view begin_key,
 	                                       std::string_view end_key,
 	                                       int64_t beginVersion,
 	                                       int64_t endVersion,
 	                                       FDBReadBlobGranuleContext granule_context);
+	GranuleSummaryArrayFuture summarize_blob_granules(std::string_view begin_key,
+	                                                  std::string_view end_key,
+	                                                  int64_t summaryVersion,
+	                                                  int rangeLimit);
 
 private:
 	FDBTransaction* tr_;
diff --git a/bindings/c/test/unit/setup_tests.cpp b/bindings/c/test/unit/setup_tests.cpp
index 6ac65b7850..2e96eb00b9 100644
--- a/bindings/c/test/unit/setup_tests.cpp
+++ b/bindings/c/test/unit/setup_tests.cpp
@@ -42,13 +42,13 @@ TEST_CASE("setup") {
 	CHECK(err);
 
 	// Select current API version
-	fdb_check(fdb_select_api_version(720));
+	fdb_check(fdb_select_api_version(FDB_API_VERSION));
 
 	// Error to call again after a successful return
-	err = fdb_select_api_version(720);
+	err = fdb_select_api_version(FDB_API_VERSION);
 	CHECK(err);
 
-	CHECK(fdb_get_max_api_version() >= 720);
+	CHECK(fdb_get_max_api_version() >= FDB_API_VERSION);
 
 	fdb_check(fdb_setup_network());
 	// Calling a second time should fail
diff --git a/bindings/c/test/unit/trace_partial_file_suffix_test.cpp b/bindings/c/test/unit/trace_partial_file_suffix_test.cpp
index 810cc066fd..73dc8132a5 100644
--- a/bindings/c/test/unit/trace_partial_file_suffix_test.cpp
+++ b/bindings/c/test/unit/trace_partial_file_suffix_test.cpp
@@ -53,7 +53,7 @@ bool file_exists(const char* path) {
 }
 
 int main(int argc, char** argv) {
-	fdb_check(fdb_select_api_version(720));
+	fdb_check(fdb_select_api_version(FDB_API_VERSION));
 
 	std::string file_identifier = "trace_partial_file_suffix_test" + std::to_string(std::random_device{}());
 	std::string trace_partial_file_suffix = ".tmp";
diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp
index 60af78c885..2d8dc2456f 100644
--- a/bindings/c/test/unit/unit_tests.cpp
+++ b/bindings/c/test/unit/unit_tests.cpp
@@ -941,13 +941,13 @@ static Value dataOfRecord(const int i) {
 	return Value(format("data-of-record-%08d", i));
 }
 static std::string indexEntryKey(const int i) {
-	return Tuple().append(StringRef(prefix)).append(INDEX).append(indexKey(i)).append(primaryKey(i)).pack().toString();
+	return Tuple::makeTuple(prefix, INDEX, indexKey(i), primaryKey(i)).pack().toString();
 }
 static std::string recordKey(const int i, const int split) {
-	return Tuple().append(prefix).append(RECORD).append(primaryKey(i)).append(split).pack().toString();
+	return Tuple::makeTuple(prefix, RECORD, primaryKey(i), split).pack().toString();
 }
 static std::string recordValue(const int i, const int split) {
-	return Tuple().append(dataOfRecord(i)).append(split).pack().toString();
+	return Tuple::makeTuple(dataOfRecord(i), split).pack().toString();
 }
 
 const static int SPLIT_SIZE = 3;
@@ -993,13 +993,8 @@ GetMappedRangeResult getMappedIndexEntries(int beginId,
                                            fdb::Transaction& tr,
                                            int matchIndex,
                                            bool allMissing) {
-	std::string mapper = Tuple()
-	                         .append(prefix)
-	                         .append(RECORD)
-	                         .append(allMissing ? "{K[2]}"_sr : "{K[3]}"_sr)
-	                         .append("{...}"_sr)
-	                         .pack()
-	                         .toString();
+	std::string mapper =
+	    Tuple::makeTuple(prefix, RECORD, (allMissing ? "{K[2]}"_sr : "{K[3]}"_sr), "{...}"_sr).pack().toString();
 	return getMappedIndexEntries(beginId, endId, tr, mapper, matchIndex);
 }
 
@@ -1037,7 +1032,7 @@ TEST_CASE("tuple_support_versionstamp") {
 	// a random 12 bytes long StringRef as a versionstamp
 	StringRef str = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12"_sr;
 	Versionstamp vs(str);
-	const Tuple t = Tuple().append(prefix).append(RECORD).appendVersionstamp(vs).append("{K[3]}"_sr).append("{...}"_sr);
+	const Tuple t = Tuple::makeTuple(prefix, RECORD, vs, "{K[3]}"_sr, "{...}"_sr);
 	ASSERT(t.getVersionstamp(2) == vs);
 
 	// verify the round-way pack-unpack path for a Tuple containing a versionstamp
@@ -1181,7 +1176,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_missing_all_secondary") {
 }
 
 TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
-	std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).pack().toString();
+	std::string mapper = Tuple::makeTuple(prefix, RECORD, "{K[3]}"_sr).pack().toString();
 	fdb::Transaction tr(db);
 	auto result = get_mapped_range(
 	    tr,
@@ -1200,7 +1195,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
 }
 
 TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_ryw_enable") {
-	std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).pack().toString();
+	std::string mapper = Tuple::makeTuple(prefix, RECORD, "{K[3]}"_sr).pack().toString();
 	fdb::Transaction tr(db);
 	fdb_check(tr.set_option(FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE, nullptr, 0)); // Not disable RYW
 	auto result = get_mapped_range(
@@ -2766,6 +2761,7 @@ TEST_CASE("Blob Granule Functions") {
 	auto confValue =
 	    get_value("\xff/conf/blob_granules_enabled", /* snapshot */ false, { FDB_TR_OPTION_READ_SYSTEM_KEYS });
 	if (!confValue.has_value() || confValue.value() != "1") {
+		// std::cout << "skipping blob granule test" << std::endl;
 		return;
 	}
 
@@ -2822,7 +2818,6 @@ TEST_CASE("Blob Granule Functions") {
 		fdb::KeyValueArrayResult r =
 		    tr.read_blob_granules(key("bg"), key("bh"), originalReadVersion, -2, granuleContext);
 		fdb_error_t err = r.get(&out_kv, &out_count, &out_more);
-		;
 		if (err && err != 2037 /* blob_granule_not_materialized */) {
 			fdb::EmptyFuture f2 = tr.on_error(err);
 			fdb_check(wait_future(f2));
@@ -2858,7 +2853,7 @@ TEST_CASE("Blob Granule Functions") {
 	// test ranges
 
 	while (1) {
-		fdb::KeyRangeArrayFuture f = tr.get_blob_granule_ranges(key("bg"), key("bh"));
+		fdb::KeyRangeArrayFuture f = tr.get_blob_granule_ranges(key("bg"), key("bh"), 1000);
 		fdb_error_t err = wait_future(f);
 		if (err) {
 			fdb::EmptyFuture f2 = tr.on_error(err);
@@ -2870,6 +2865,10 @@ TEST_CASE("Blob Granule Functions") {
 		int out_count;
 		fdb_check(f.get(&out_kr, &out_count));
 
+		CHECK(std::string((const char*)out_kr[0].begin_key, out_kr[0].begin_key_length) <= key("bg"));
+		CHECK(std::string((const char*)out_kr[out_count - 1].end_key, out_kr[out_count - 1].end_key_length) >=
+		      key("bh"));
+
 		CHECK(out_count >= 1);
 		// check key ranges are in order
 		for (int i = 0; i < out_count; i++) {
@@ -2877,9 +2876,9 @@ TEST_CASE("Blob Granule Functions") {
 			CHECK(std::string((const char*)out_kr[i].begin_key, out_kr[i].begin_key_length) <
 			      std::string((const char*)out_kr[i].end_key, out_kr[i].end_key_length));
 		}
-		// Ranges themselves are sorted
+		// Ranges themselves are sorted and contiguous
 		for (int i = 0; i < out_count - 1; i++) {
-			CHECK(std::string((const char*)out_kr[i].end_key, out_kr[i].end_key_length) <=
+			CHECK(std::string((const char*)out_kr[i].end_key, out_kr[i].end_key_length) ==
 			      std::string((const char*)out_kr[i + 1].begin_key, out_kr[i + 1].begin_key_length));
 		}
 
@@ -2905,7 +2904,6 @@ TEST_CASE("Blob Granule Functions") {
 	fdb_check(wait_future(waitPurgeFuture));
 
 	// re-read again at the purge version to make sure it is still valid
-
 	while (1) {
 		fdb_check(tr.set_option(FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE, nullptr, 0));
 		fdb::KeyValueArrayResult r =
@@ -2922,6 +2920,56 @@ TEST_CASE("Blob Granule Functions") {
 		tr.reset();
 		break;
 	}
+
+	// check granule summary
+	while (1) {
+		fdb::GranuleSummaryArrayFuture f = tr.summarize_blob_granules(key("bg"), key("bh"), originalReadVersion, 100);
+		fdb_error_t err = wait_future(f);
+		if (err) {
+			fdb::EmptyFuture f2 = tr.on_error(err);
+			fdb_check(wait_future(f2));
+			continue;
+		}
+
+		const FDBGranuleSummary* out_summaries;
+		int out_count;
+		fdb_check(f.get(&out_summaries, &out_count));
+
+		CHECK(out_count >= 1);
+		CHECK(out_count <= 100);
+
+		// check that ranges cover requested range
+		CHECK(std::string((const char*)out_summaries[0].key_range.begin_key,
+		                  out_summaries[0].key_range.begin_key_length) <= key("bg"));
+		CHECK(std::string((const char*)out_summaries[out_count - 1].key_range.end_key,
+		                  out_summaries[out_count - 1].key_range.end_key_length) >= key("bh"));
+
+		// check key ranges are in order
+		for (int i = 0; i < out_count; i++) {
+			// key range start < end
+			CHECK(std::string((const char*)out_summaries[i].key_range.begin_key,
+			                  out_summaries[i].key_range.begin_key_length) <
+			      std::string((const char*)out_summaries[i].key_range.end_key,
+			                  out_summaries[i].key_range.end_key_length));
+			// sanity check versions and sizes
+			CHECK(out_summaries[i].snapshot_version <= originalReadVersion);
+			CHECK(out_summaries[i].delta_version <= originalReadVersion);
+			CHECK(out_summaries[i].snapshot_version <= out_summaries[i].delta_version);
+			CHECK(out_summaries[i].snapshot_size > 0);
+			CHECK(out_summaries[i].delta_size >= 0);
+		}
+
+		// Ranges themselves are sorted and contiguous
+		for (int i = 0; i < out_count - 1; i++) {
+			CHECK(std::string((const char*)out_summaries[i].key_range.end_key,
+			                  out_summaries[i].key_range.end_key_length) ==
+			      std::string((const char*)out_summaries[i + 1].key_range.begin_key,
+			                  out_summaries[i + 1].key_range.begin_key_length));
+		}
+
+		tr.reset();
+		break;
+	}
 }
 
 int main(int argc, char** argv) {
@@ -2931,7 +2979,7 @@ int main(int argc, char** argv) {
 		          << std::endl;
 		return 1;
 	}
-	fdb_check(fdb_select_api_version(720));
+	fdb_check(fdb_select_api_version(FDB_API_VERSION));
 	if (argc >= 4) {
 		std::string externalClientLibrary = argv[3];
 		if (externalClientLibrary.substr(0, 2) != "--") {
diff --git a/bindings/c/test/workloads/SimpleWorkload.cpp b/bindings/c/test/workloads/SimpleWorkload.cpp
index 95be0ddd59..4dd7a800fe 100644
--- a/bindings/c/test/workloads/SimpleWorkload.cpp
+++ b/bindings/c/test/workloads/SimpleWorkload.cpp
@@ -266,7 +266,7 @@ struct SimpleWorkload final : FDBWorkload {
 		insertsPerTx = context->getOption("insertsPerTx", 100ul);
 		opsPerTx = context->getOption("opsPerTx", 100ul);
 		runFor = context->getOption("runFor", 10.0);
-		auto err = fdb_select_api_version(720);
+		auto err = fdb_select_api_version(FDB_API_VERSION);
 		if (err) {
 			context->trace(
 			    FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } });
diff --git a/bindings/flow/DirectoryLayer.actor.cpp b/bindings/flow/DirectoryLayer.actor.cpp
index 056b203a2e..13d3970ed3 100644
--- a/bindings/flow/DirectoryLayer.actor.cpp
+++ b/bindings/flow/DirectoryLayer.actor.cpp
@@ -23,17 +23,17 @@
 
 namespace FDB {
 const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = { 1, 0, 0, 0, 0, 0, 0, 0 };
-const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca");
-const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer");
-const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version");
+const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = "hca"_sr;
+const StringRef DirectoryLayer::LAYER_KEY = "layer"_sr;
+const StringRef DirectoryLayer::VERSION_KEY = "version"_sr;
 const int64_t DirectoryLayer::SUB_DIR_KEY = 0;
 
 const uint32_t DirectoryLayer::VERSION[3] = { 1, 0, 0 };
 
-const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe");
+const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = "\xfe"_sr;
 const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX);
 const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
-const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
+const StringRef DirectoryLayer::PARTITION_LAYER = "partition"_sr;
 
 DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes)
   : rootNode(nodeSubspace.get(nodeSubspace.key())), nodeSubspace(nodeSubspace), contentSubspace(contentSubspace),
diff --git a/bindings/flow/FDBLoanerTypes.h b/bindings/flow/FDBLoanerTypes.h
index 01000f6e27..ddd9a577b5 100644
--- a/bindings/flow/FDBLoanerTypes.h
+++ b/bindings/flow/FDBLoanerTypes.h
@@ -31,7 +31,7 @@ typedef Standalone<KeyRef> Key;
 typedef Standalone<ValueRef> Value;
 
 inline Key keyAfter(const KeyRef& key) {
-	if (key == LiteralStringRef("\xff\xff"))
+	if (key == "\xff\xff"_sr)
 		return key;
 
 	Standalone<StringRef> r;
@@ -43,7 +43,7 @@ inline Key keyAfter(const KeyRef& key) {
 }
 
 inline KeyRef keyAfter(const KeyRef& key, Arena& arena) {
-	if (key == LiteralStringRef("\xff\xff"))
+	if (key == "\xff\xff"_sr)
 		return key;
 	uint8_t* t = new (arena) uint8_t[key.size() + 1];
 	memcpy(t, key.begin(), key.size());
diff --git a/bindings/flow/Tuple.h b/bindings/flow/Tuple.h
index 4d903e5bb8..a7feab9419 100644
--- a/bindings/flow/Tuple.h
+++ b/bindings/flow/Tuple.h
@@ -63,7 +63,9 @@ struct Tuple {
 	Tuple& appendNull();
 	Tuple& appendVersionstamp(Versionstamp const&);
 
-	StringRef pack() const { return StringRef(data.begin(), data.size()); }
+	Standalone<StringRef> pack() const {
+		return Standalone<StringRef>(StringRef(data.begin(), data.size()), data.arena());
+	}
 
 	template <typename T>
 	Tuple& operator<<(T const& t) {
diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp
index 72ee49dcf4..d6e1431c77 100644
--- a/bindings/flow/fdb_flow.actor.cpp
+++ b/bindings/flow/fdb_flow.actor.cpp
@@ -38,7 +38,7 @@ THREAD_FUNC networkThread(void* fdb) {
 }
 
 ACTOR Future<Void> _test() {
-	API* fdb = FDB::API::selectAPIVersion(720);
+	API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION);
 	auto db = fdb->createDatabase();
 	state Reference<Transaction> tr = db->createTransaction();
 
@@ -63,15 +63,14 @@ ACTOR Future<Void> _test() {
 	// wait( waitForAllReady( versions ) );
 	printf("Elapsed: %lf\n", timer_monotonic() - starttime);
 
-	tr->set(LiteralStringRef("foo"), LiteralStringRef("bar"));
+	tr->set("foo"_sr, "bar"_sr);
 
-	Optional<FDBStandalone<ValueRef>> v = wait(tr->get(LiteralStringRef("foo")));
+	Optional<FDBStandalone<ValueRef>> v = wait(tr->get("foo"_sr));
 	if (v.present()) {
 		printf("%s\n", v.get().toString().c_str());
 	}
 
-	FDBStandalone<RangeResultRef> r =
-	    wait(tr->getRange(KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z")), 100));
+	FDBStandalone<RangeResultRef> r = wait(tr->getRange(KeyRangeRef("a"_sr, "z"_sr), 100));
 
 	for (auto kv : r) {
 		printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str());
@@ -82,7 +81,7 @@ ACTOR Future<Void> _test() {
 }
 
 void fdb_flow_test() {
-	API* fdb = FDB::API::selectAPIVersion(720);
+	API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION);
 	fdb->setupNetwork();
 	startThread(networkThread, fdb);
 
diff --git a/bindings/flow/tester/DirectoryTester.actor.cpp b/bindings/flow/tester/DirectoryTester.actor.cpp
index b21da1097c..a8fabdca4c 100644
--- a/bindings/flow/tester/DirectoryTester.actor.cpp
+++ b/bindings/flow/tester/DirectoryTester.actor.cpp
@@ -545,11 +545,10 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
 			pathTuple.append(p, true);
 		}
 
-		instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack());
-		instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true),
-		                     Tuple().append(directory->getLayer()).pack());
-		instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack());
-		instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack());
+		instruction->tr->set(logSubspace.pack("path"_sr, true), pathTuple.pack());
+		instruction->tr->set(logSubspace.pack("layer"_sr, true), Tuple().append(directory->getLayer()).pack());
+		instruction->tr->set(logSubspace.pack("exists"_sr, true), Tuple().append(exists ? 1 : 0).pack());
+		instruction->tr->set(logSubspace.pack("children"_sr, true), childrenTuple.pack());
 
 		return Void();
 	}
diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp
index 941e1b97b2..f300127e5d 100644
--- a/bindings/flow/tester/Tester.actor.cpp
+++ b/bindings/flow/tester/Tester.actor.cpp
@@ -470,12 +470,12 @@ ACTOR Future<Standalone<StringRef>> waitForVoid(Future<Void> f) {
 	try {
 		wait(f);
 		Tuple t;
-		t.append(LiteralStringRef("RESULT_NOT_PRESENT"));
+		t.append("RESULT_NOT_PRESENT"_sr);
 		return t.pack();
 	} catch (Error& e) {
 		// printf("FDBError1:%d\n", e.code());
 		Tuple t;
-		t.append(LiteralStringRef("ERROR"));
+		t.append("ERROR"_sr);
 		t.append(format("%d", e.code()));
 		// pack above as error string into another tuple
 		Tuple ret;
@@ -493,7 +493,7 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future<FDBStandalone<KeyRef>> f
 	} catch (Error& e) {
 		// printf("FDBError2:%d\n", e.code());
 		Tuple t;
-		t.append(LiteralStringRef("ERROR"));
+		t.append("ERROR"_sr);
 		t.append(format("%d", e.code()));
 		// pack above as error string into another tuple
 		Tuple ret;
@@ -509,7 +509,7 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future<Optional<FDBStandalone<V
 		if (value.present())
 			str = value.get();
 		else
-			str = LiteralStringRef("RESULT_NOT_PRESENT");
+			str = "RESULT_NOT_PRESENT"_sr;
 
 		Tuple t;
 		t.append(str);
@@ -517,7 +517,7 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future<Optional<FDBStandalone<V
 	} catch (Error& e) {
 		// printf("FDBError3:%d\n", e.code());
 		Tuple t;
-		t.append(LiteralStringRef("ERROR"));
+		t.append("ERROR"_sr);
 		t.append(format("%d", e.code()));
 		// pack above as error string into another tuple
 		Tuple ret;
@@ -543,7 +543,7 @@ ACTOR Future<Standalone<StringRef>> getKey(Future<FDBStandalone<KeyRef>> f, Stan
 	} catch (Error& e) {
 		// printf("FDBError4:%d\n", e.code());
 		Tuple t;
-		t.append(LiteralStringRef("ERROR"));
+		t.append("ERROR"_sr);
 		t.append(format("%d", e.code()));
 		// pack above as error string into another tuple
 		Tuple ret;
@@ -670,7 +670,7 @@ struct GetEstimatedRangeSize : InstructionFunc {
 		state Standalone<StringRef> endKey = Tuple::unpack(s2).getString(0);
 		Future<int64_t> fsize = instruction->tr->getEstimatedRangeSizeBytes(KeyRangeRef(beginKey, endKey));
 		int64_t size = wait(fsize);
-		data->stack.pushTuple(LiteralStringRef("GOT_ESTIMATED_RANGE_SIZE"));
+		data->stack.pushTuple("GOT_ESTIMATED_RANGE_SIZE"_sr);
 
 		return Void();
 	}
@@ -698,7 +698,7 @@ struct GetRangeSplitPoints : InstructionFunc {
 		Future<FDBStandalone<VectorRef<KeyRef>>> fsplitPoints =
 		    instruction->tr->getRangeSplitPoints(KeyRangeRef(beginKey, endKey), chunkSize);
 		FDBStandalone<VectorRef<KeyRef>> splitPoints = wait(fsplitPoints);
-		data->stack.pushTuple(LiteralStringRef("GOT_RANGE_SPLIT_POINTS"));
+		data->stack.pushTuple("GOT_RANGE_SPLIT_POINTS"_sr);
 
 		return Void();
 	}
@@ -743,7 +743,7 @@ struct GetReadVersionFunc : InstructionFunc {
 	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
 		Version v = wait(instruction->tr->getReadVersion());
 		data->lastVersion = v;
-		data->stack.pushTuple(LiteralStringRef("GOT_READ_VERSION"));
+		data->stack.pushTuple("GOT_READ_VERSION"_sr);
 		return Void();
 	}
 };
@@ -767,7 +767,7 @@ struct GetCommittedVersionFunc : InstructionFunc {
 
 	static Future<Void> call(Reference<FlowTesterData> const& data, Reference<InstructionData> const& instruction) {
 		data->lastVersion = instruction->tr->getCommittedVersion();
-		data->stack.pushTuple(LiteralStringRef("GOT_COMMITTED_VERSION"));
+		data->stack.pushTuple("GOT_COMMITTED_VERSION"_sr);
 		return Void();
 	}
 };
@@ -781,7 +781,7 @@ struct GetApproximateSizeFunc : InstructionFunc {
 	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
 		int64_t _ = wait(instruction->tr->getApproximateSize());
 		(void)_; // disable unused variable warning
-		data->stack.pushTuple(LiteralStringRef("GOT_APPROXIMATE_SIZE"));
+		data->stack.pushTuple("GOT_APPROXIMATE_SIZE"_sr);
 		return Void();
 	}
 };
@@ -1485,7 +1485,7 @@ struct ReadConflictKeyFunc : InstructionFunc {
 		// printf("=========READ_CONFLICT_KEY:%s\n", printable(key).c_str());
 		instruction->tr->addReadConflictKey(key);
 
-		data->stack.pushTuple(LiteralStringRef("SET_CONFLICT_KEY"));
+		data->stack.pushTuple("SET_CONFLICT_KEY"_sr);
 		return Void();
 	}
 };
@@ -1506,7 +1506,7 @@ struct WriteConflictKeyFunc : InstructionFunc {
 		// printf("=========WRITE_CONFLICT_KEY:%s\n", printable(key).c_str());
 		instruction->tr->addWriteConflictKey(key);
 
-		data->stack.pushTuple(LiteralStringRef("SET_CONFLICT_KEY"));
+		data->stack.pushTuple("SET_CONFLICT_KEY"_sr);
 		return Void();
 	}
 };
@@ -1529,7 +1529,7 @@ struct ReadConflictRangeFunc : InstructionFunc {
 
 		// printf("=========READ_CONFLICT_RANGE:%s:%s\n", printable(begin).c_str(), printable(end).c_str());
 		instruction->tr->addReadConflictRange(KeyRange(KeyRangeRef(begin, end)));
-		data->stack.pushTuple(LiteralStringRef("SET_CONFLICT_RANGE"));
+		data->stack.pushTuple("SET_CONFLICT_RANGE"_sr);
 		return Void();
 	}
 };
@@ -1553,7 +1553,7 @@ struct WriteConflictRangeFunc : InstructionFunc {
 		// printf("=========WRITE_CONFLICT_RANGE:%s:%s\n", printable(begin).c_str(), printable(end).c_str());
 		instruction->tr->addWriteConflictRange(KeyRange(KeyRangeRef(begin, end)));
 
-		data->stack.pushTuple(LiteralStringRef("SET_CONFLICT_RANGE"));
+		data->stack.pushTuple("SET_CONFLICT_RANGE"_sr);
 		return Void();
 	}
 };
@@ -1643,10 +1643,8 @@ struct UnitTestsFunc : InstructionFunc {
 		                            Optional<StringRef>(StringRef((const uint8_t*)&locationCacheSize, 8)));
 		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES,
 		                            Optional<StringRef>(StringRef((const uint8_t*)&maxWatches, 8)));
-		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID,
-		                            Optional<StringRef>(LiteralStringRef("dc_id")));
-		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID,
-		                            Optional<StringRef>(LiteralStringRef("machine_id")));
+		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, Optional<StringRef>("dc_id"_sr));
+		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, Optional<StringRef>("machine_id"_sr));
 		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE);
 		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE);
 		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH,
@@ -1685,13 +1683,13 @@ struct UnitTestsFunc : InstructionFunc {
 		              Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8)));
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_USED_DURING_COMMIT_PROTECTION_DISABLE);
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE,
-		              Optional<StringRef>(LiteralStringRef("my_transaction")));
+		              Optional<StringRef>("my_transaction"_sr));
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_LOCK_AWARE);
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE);
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS);
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_REPORT_CONFLICTING_KEYS);
 
-		Optional<FDBStandalone<ValueRef>> _ = wait(tr->get(LiteralStringRef("\xff")));
+		Optional<FDBStandalone<ValueRef>> _ = wait(tr->get("\xff"_sr));
 		tr->cancel();
 
 		return Void();
@@ -1724,13 +1722,13 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
 		Tuple opTuple = Tuple::unpack(data->instructions[idx].value);
 		state Standalone<StringRef> op = opTuple.getString(0);
 
-		state bool isDatabase = op.endsWith(LiteralStringRef("_DATABASE"));
-		state bool isSnapshot = op.endsWith(LiteralStringRef("_SNAPSHOT"));
-		state bool isDirectory = op.startsWith(LiteralStringRef("DIRECTORY_"));
+		state bool isDatabase = op.endsWith("_DATABASE"_sr);
+		state bool isSnapshot = op.endsWith("_SNAPSHOT"_sr);
+		state bool isDirectory = op.startsWith("DIRECTORY_"_sr);
 
 		try {
 			if (LOG_INSTRUCTIONS) {
-				if (op != LiteralStringRef("SWAP") && op != LiteralStringRef("PUSH")) {
+				if (op != "SWAP"_sr && op != "PUSH"_sr) {
 					printf("%zu. %s\n", idx, tupleToString(opTuple).c_str());
 					fflush(stdout);
 				}
@@ -1773,7 +1771,7 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
 				if (opsThatCreateDirectories.count(op.toString())) {
 					data->directoryData.directoryList.push_back(DirectoryOrSubspace());
 				}
-				data->stack.pushTuple(LiteralStringRef("DIRECTORY_ERROR"));
+				data->stack.pushTuple("DIRECTORY_ERROR"_sr);
 			} else {
 				data->stack.pushError(e.code());
 			}
@@ -1873,7 +1871,7 @@ ACTOR void _test_versionstamp() {
 	try {
 		g_network = newNet2(TLSConfig());
 
-		API* fdb = FDB::API::selectAPIVersion(720);
+		API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION);
 
 		fdb->setupNetwork();
 		startThread(networkThread, fdb);
@@ -1883,15 +1881,14 @@ ACTOR void _test_versionstamp() {
 
 		state Future<FDBStandalone<StringRef>> ftrVersion = tr->getVersionstamp();
 
-		tr->atomicOp(LiteralStringRef("foo"),
-		             LiteralStringRef("blahblahbl\x00\x00\x00\x00"),
-		             FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
+		tr->atomicOp(
+		    "foo"_sr, "blahblahbl\x00\x00\x00\x00"_sr, FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
 
 		wait(tr->commit()); // should use retry loop
 
 		tr->reset();
 
-		Optional<FDBStandalone<StringRef>> optionalDbVersion = wait(tr->get(LiteralStringRef("foo")));
+		Optional<FDBStandalone<StringRef>> optionalDbVersion = wait(tr->get("foo"_sr));
 		state FDBStandalone<StringRef> dbVersion = optionalDbVersion.get();
 		FDBStandalone<StringRef> trVersion = wait(ftrVersion);
 
diff --git a/bindings/flow/tester/Tester.actor.h b/bindings/flow/tester/Tester.actor.h
index 63fc9fe9a3..f42f8d74ee 100644
--- a/bindings/flow/tester/Tester.actor.h
+++ b/bindings/flow/tester/Tester.actor.h
@@ -71,7 +71,7 @@ struct FlowTesterStack {
 
 	void pushError(int errorCode) {
 		FDB::Tuple t;
-		t.append(LiteralStringRef("ERROR"));
+		t.append("ERROR"_sr);
 		t.append(format("%d", errorCode));
 		// pack above as error string into another tuple
 		pushTuple(t.pack().toString());
diff --git a/bindings/go/src/fdb/fdb.go b/bindings/go/src/fdb/fdb.go
index 7800dd9bf5..e308049be0 100644
--- a/bindings/go/src/fdb/fdb.go
+++ b/bindings/go/src/fdb/fdb.go
@@ -128,7 +128,7 @@ func APIVersion(version int) error {
 		return errAPIVersionAlreadySet
 	}
 
-	if version < 200 || version > 720 {
+	if version < 200 || version > headerVersion {
 		return errAPIVersionNotSupported
 	}
 
diff --git a/bindings/go/src/fdb/fdb_test.go b/bindings/go/src/fdb/fdb_test.go
index 976a3ec9d0..00b3f41304 100644
--- a/bindings/go/src/fdb/fdb_test.go
+++ b/bindings/go/src/fdb/fdb_test.go
@@ -29,10 +29,12 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb"
 )
 
+const API_VERSION int = 720
+
 func ExampleOpenDefault() {
 	var e error
 
-	e = fdb.APIVersion(720)
+	e = fdb.APIVersion(API_VERSION)
 	if e != nil {
 		fmt.Printf("Unable to set API version: %v\n", e)
 		return
@@ -52,7 +54,7 @@ func ExampleOpenDefault() {
 }
 
 func TestVersionstamp(t *testing.T) {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 	db := fdb.MustOpenDefault()
 
 	setVs := func(t fdb.Transactor, key fdb.Key) (fdb.FutureKey, error) {
@@ -98,7 +100,7 @@ func TestVersionstamp(t *testing.T) {
 }
 
 func TestReadTransactionOptions(t *testing.T) {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 	db := fdb.MustOpenDefault()
 	_, e := db.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) {
 		rtr.Options().SetAccessSystemKeys()
@@ -110,7 +112,7 @@ func TestReadTransactionOptions(t *testing.T) {
 }
 
 func ExampleTransactor() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 	db := fdb.MustOpenDefault()
 
 	setOne := func(t fdb.Transactor, key fdb.Key, value []byte) error {
@@ -161,7 +163,7 @@ func ExampleTransactor() {
 }
 
 func ExampleReadTransactor() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 	db := fdb.MustOpenDefault()
 
 	getOne := func(rt fdb.ReadTransactor, key fdb.Key) ([]byte, error) {
@@ -214,7 +216,7 @@ func ExampleReadTransactor() {
 }
 
 func ExamplePrefixRange() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 	db := fdb.MustOpenDefault()
 
 	tr, e := db.CreateTransaction()
@@ -253,7 +255,7 @@ func ExamplePrefixRange() {
 }
 
 func ExampleRangeIterator() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 	db := fdb.MustOpenDefault()
 
 	tr, e := db.CreateTransaction()
diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index a3c0674e64..b765e09508 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -102,6 +102,11 @@ func (o NetworkOptions) SetTraceFileIdentifier(param string) error {
 	return o.setOpt(36, []byte(param))
 }
 
+// Use the same base trace file name for all client threads as it did before version 7.2. The current default behavior is to use distinct trace file names for client threads by including their version and thread index.
+func (o NetworkOptions) SetTraceShareAmongClientThreads() error {
+	return o.setOpt(37, nil)
+}
+
 // Set file suffix for partially written log files.
 //
 // Parameter: Append this suffix to partially written log files. When a log file is complete, it is renamed to remove the suffix. No separator is added between the file and the suffix. If you want to add a file extension, you should include the separator - e.g. '.tmp' instead of 'tmp' to add the 'tmp' extension.
@@ -239,6 +244,13 @@ func (o NetworkOptions) SetClientThreadsPerVersion(param int64) error {
 	return o.setOpt(65, int64ToBytes(param))
 }
 
+// Adds an external client library to be used with a future version protocol. This option can be used testing purposes only!
+//
+// Parameter: path to client library
+func (o NetworkOptions) SetFutureVersionClientLibrary(param string) error {
+	return o.setOpt(66, []byte(param))
+}
+
 // Disables logging of client statistics, such as sampled transaction activity.
 func (o NetworkOptions) SetDisableClientStatisticsLogging() error {
 	return o.setOpt(70, nil)
@@ -254,6 +266,11 @@ func (o NetworkOptions) SetEnableRunLoopProfiling() error {
 	return o.setOpt(71, nil)
 }
 
+// Prevents the multi-version client API from being disabled, even if no external clients are configured. This option is required to use GRV caching.
+func (o NetworkOptions) SetDisableClientBypass() error {
+	return o.setOpt(72, nil)
+}
+
 // Enable client buggify - will make requests randomly fail (intended for client testing)
 func (o NetworkOptions) SetClientBuggifyEnable() error {
 	return o.setOpt(80, nil)
@@ -610,11 +627,18 @@ func (o TransactionOptions) SetBypassUnreadable() error {
 	return o.setOpt(1100, nil)
 }
 
-// Allows this transaction to use cached GRV from the database context. Defaults to off. Upon first usage, starts a background updater to periodically update the cache to avoid stale read versions.
+// Allows this transaction to use cached GRV from the database context. Defaults to off. Upon first usage, starts a background updater to periodically update the cache to avoid stale read versions. The disable_client_bypass option must also be set.
 func (o TransactionOptions) SetUseGrvCache() error {
 	return o.setOpt(1101, nil)
 }
 
+// Attach given authorization token to the transaction such that subsequent tenant-aware requests are authorized
+//
+// Parameter: A JSON Web Token authorized to access data belonging to one or more tenants, indicated by 'tenants' claim of the token's payload.
+func (o TransactionOptions) SetAuthorizationToken(param string) error {
+	return o.setOpt(2000, []byte(param))
+}
+
 type StreamingMode int
 
 const (
diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt
index 22564dccc8..7057f22384 100644
--- a/bindings/java/CMakeLists.txt
+++ b/bindings/java/CMakeLists.txt
@@ -34,9 +34,11 @@ set(JAVA_BINDING_SRCS
   src/main/com/apple/foundationdb/FDBDatabase.java
   src/main/com/apple/foundationdb/FDBTenant.java
   src/main/com/apple/foundationdb/FDBTransaction.java
+  src/main/com/apple/foundationdb/FutureBool.java
   src/main/com/apple/foundationdb/FutureInt64.java
   src/main/com/apple/foundationdb/FutureKey.java
   src/main/com/apple/foundationdb/FutureKeyArray.java
+  src/main/com/apple/foundationdb/FutureKeyRangeArray.java
   src/main/com/apple/foundationdb/FutureResult.java
   src/main/com/apple/foundationdb/FutureResults.java
   src/main/com/apple/foundationdb/FutureMappedResults.java
@@ -56,6 +58,7 @@ set(JAVA_BINDING_SRCS
   src/main/com/apple/foundationdb/RangeQuery.java
   src/main/com/apple/foundationdb/MappedRangeQuery.java
   src/main/com/apple/foundationdb/KeyArrayResult.java
+  src/main/com/apple/foundationdb/KeyRangeArrayResult.java
   src/main/com/apple/foundationdb/RangeResult.java
   src/main/com/apple/foundationdb/MappedRangeResult.java
   src/main/com/apple/foundationdb/RangeResultInfo.java
diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp
index e9bf309fa4..6c65313f4b 100644
--- a/bindings/java/JavaWorkload.cpp
+++ b/bindings/java/JavaWorkload.cpp
@@ -379,7 +379,7 @@ struct JVM {
 		jmethodID selectMethod =
 		    env->GetStaticMethodID(fdbClass, "selectAPIVersion", "(I)Lcom/apple/foundationdb/FDB;");
 		checkException();
-		auto fdbInstance = env->CallStaticObjectMethod(fdbClass, selectMethod, jint(720));
+		auto fdbInstance = env->CallStaticObjectMethod(fdbClass, selectMethod, jint(FDB_API_VERSION));
 		checkException();
 		env->CallObjectMethod(fdbInstance, getMethod(fdbClass, "disableShutdownHook", "()V"));
 		checkException();
diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp
index 04c87e3b19..660e6945fa 100644
--- a/bindings/java/fdbJNI.cpp
+++ b/bindings/java/fdbJNI.cpp
@@ -25,9 +25,11 @@
 #include "com_apple_foundationdb_FDB.h"
 #include "com_apple_foundationdb_FDBDatabase.h"
 #include "com_apple_foundationdb_FDBTransaction.h"
+#include "com_apple_foundationdb_FutureBool.h"
 #include "com_apple_foundationdb_FutureInt64.h"
 #include "com_apple_foundationdb_FutureKey.h"
 #include "com_apple_foundationdb_FutureKeyArray.h"
+#include "com_apple_foundationdb_FutureKeyRangeArray.h"
 #include "com_apple_foundationdb_FutureResult.h"
 #include "com_apple_foundationdb_FutureResults.h"
 #include "com_apple_foundationdb_FutureStrings.h"
@@ -55,7 +57,11 @@ static jclass mapped_range_result_class;
 static jclass mapped_key_value_class;
 static jclass string_class;
 static jclass key_array_result_class;
+static jclass keyrange_class;
+static jclass keyrange_array_result_class;
 static jmethodID key_array_result_init;
+static jmethodID keyrange_init;
+static jmethodID keyrange_array_result_init;
 static jmethodID range_result_init;
 static jmethodID mapped_range_result_init;
 static jmethodID mapped_key_value_from_bytes;
@@ -278,6 +284,23 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseM
 	fdb_future_release_memory(var);
 }
 
+JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FutureBool_FutureBool_1get(JNIEnv* jenv, jobject, jlong future) {
+	if (!future) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+	FDBFuture* f = (FDBFuture*)future;
+
+	fdb_bool_t value = false;
+	fdb_error_t err = fdb_future_get_bool(f, &value);
+	if (err) {
+		safeThrow(jenv, getThrowable(jenv, err));
+		return 0;
+	}
+
+	return (jboolean)value;
+}
+
 JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get(JNIEnv* jenv, jobject, jlong future) {
 	if (!future) {
 		throwParamNotNull(jenv);
@@ -407,6 +430,61 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyArray_FutureKeyAr
 	return result;
 }
 
+JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyRangeArray_FutureKeyRangeArray_1get(JNIEnv* jenv,
+                                                                                                   jobject,
+                                                                                                   jlong future) {
+	if (!future) {
+		throwParamNotNull(jenv);
+		return JNI_NULL;
+	}
+
+	FDBFuture* f = (FDBFuture*)future;
+
+	const FDBKeyRange* fdbKr;
+	int count;
+	fdb_error_t err = fdb_future_get_keyrange_array(f, &fdbKr, &count);
+	if (err) {
+		safeThrow(jenv, getThrowable(jenv, err));
+		return JNI_NULL;
+	}
+
+	jobjectArray kr_values = jenv->NewObjectArray(count, keyrange_class, NULL);
+	if (!kr_values) {
+		if (!jenv->ExceptionOccurred())
+			throwOutOfMem(jenv);
+		return JNI_NULL;
+	}
+
+	for (int i = 0; i < count; i++) {
+		jbyteArray beginArr = jenv->NewByteArray(fdbKr[i].begin_key_length);
+		if (!beginArr) {
+			if (!jenv->ExceptionOccurred())
+				throwOutOfMem(jenv);
+			return JNI_NULL;
+		}
+		jbyteArray endArr = jenv->NewByteArray(fdbKr[i].end_key_length);
+		if (!endArr) {
+			if (!jenv->ExceptionOccurred())
+				throwOutOfMem(jenv);
+			return JNI_NULL;
+		}
+		jenv->SetByteArrayRegion(beginArr, 0, fdbKr[i].begin_key_length, (const jbyte*)fdbKr[i].begin_key);
+		jenv->SetByteArrayRegion(endArr, 0, fdbKr[i].end_key_length, (const jbyte*)fdbKr[i].end_key);
+
+		jobject kr = jenv->NewObject(keyrange_class, keyrange_init, beginArr, endArr);
+		if (jenv->ExceptionOccurred())
+			return JNI_NULL;
+		jenv->SetObjectArrayElement(kr_values, i, kr);
+		if (jenv->ExceptionOccurred())
+			return JNI_NULL;
+	}
+	jobject krarr = jenv->NewObject(keyrange_array_result_class, keyrange_array_result_init, kr_values);
+	if (jenv->ExceptionOccurred())
+		return JNI_NULL;
+
+	return krarr;
+}
+
 // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers
 JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv* jenv,
                                                                                        jobject,
@@ -765,6 +843,207 @@ JNIEXPORT jdouble JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1getM
 	return (jdouble)fdb_database_get_main_thread_busyness(database);
 }
 
+JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1purgeBlobGranules(JNIEnv* jenv,
+                                                                                            jobject,
+                                                                                            jlong dbPtr,
+                                                                                            jbyteArray beginKeyBytes,
+                                                                                            jbyteArray endKeyBytes,
+                                                                                            jlong purgeVersion,
+                                                                                            jboolean force) {
+	if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+
+	FDBDatabase* database = (FDBDatabase*)dbPtr;
+
+	uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
+	if (!beginKeyArr) {
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	if (!endKeyArr) {
+		jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	FDBFuture* f = fdb_database_purge_blob_granules(database,
+	                                                beginKeyArr,
+	                                                jenv->GetArrayLength(beginKeyBytes),
+	                                                endKeyArr,
+	                                                jenv->GetArrayLength(endKeyBytes),
+	                                                purgeVersion,
+	                                                (fdb_bool_t)force);
+	jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
+	jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT);
+	return (jlong)f;
+}
+
+JNIEXPORT jlong JNICALL
+Java_com_apple_foundationdb_FDBDatabase_Database_1waitPurgeGranulesComplete(JNIEnv* jenv,
+                                                                            jobject,
+                                                                            jlong dbPtr,
+                                                                            jbyteArray purgeKeyBytes) {
+	if (!dbPtr || !purgeKeyBytes) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+	FDBDatabase* database = (FDBDatabase*)dbPtr;
+	uint8_t* purgeKeyArr = (uint8_t*)jenv->GetByteArrayElements(purgeKeyBytes, JNI_NULL);
+
+	if (!purgeKeyArr) {
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+	FDBFuture* f =
+	    fdb_database_wait_purge_granules_complete(database, purgeKeyArr, jenv->GetArrayLength(purgeKeyBytes));
+	jenv->ReleaseByteArrayElements(purgeKeyBytes, (jbyte*)purgeKeyArr, JNI_ABORT);
+
+	return (jlong)f;
+}
+
+JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1blobbifyRange(JNIEnv* jenv,
+                                                                                        jobject,
+                                                                                        jlong dbPtr,
+                                                                                        jbyteArray beginKeyBytes,
+                                                                                        jbyteArray endKeyBytes) {
+	if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+
+	FDBDatabase* database = (FDBDatabase*)dbPtr;
+
+	uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
+	if (!beginKeyArr) {
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	if (!endKeyArr) {
+		jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	FDBFuture* f = fdb_database_blobbify_range(
+	    database, beginKeyArr, jenv->GetArrayLength(beginKeyBytes), endKeyArr, jenv->GetArrayLength(endKeyBytes));
+	jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
+	jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT);
+	return (jlong)f;
+}
+
+JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1unblobbifyRange(JNIEnv* jenv,
+                                                                                          jobject,
+                                                                                          jlong dbPtr,
+                                                                                          jbyteArray beginKeyBytes,
+                                                                                          jbyteArray endKeyBytes) {
+	if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+
+	FDBDatabase* database = (FDBDatabase*)dbPtr;
+
+	uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
+	if (!beginKeyArr) {
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	if (!endKeyArr) {
+		jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	FDBFuture* f = fdb_database_unblobbify_range(
+	    database, beginKeyArr, jenv->GetArrayLength(beginKeyBytes), endKeyArr, jenv->GetArrayLength(endKeyBytes));
+	jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
+	jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT);
+	return (jlong)f;
+}
+
+JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1listBlobbifiedRanges(JNIEnv* jenv,
+                                                                                               jobject,
+                                                                                               jlong dbPtr,
+                                                                                               jbyteArray beginKeyBytes,
+                                                                                               jbyteArray endKeyBytes,
+                                                                                               jint rangeLimit) {
+	if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+	FDBDatabase* tr = (FDBDatabase*)dbPtr;
+
+	uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
+	if (!startKey) {
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	if (!endKey) {
+		jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	FDBFuture* f = fdb_database_list_blobbified_ranges(
+	    tr, startKey, jenv->GetArrayLength(beginKeyBytes), endKey, jenv->GetArrayLength(endKeyBytes), rangeLimit);
+	jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
+	jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
+	return (jlong)f;
+}
+
+JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1verifyBlobRange(JNIEnv* jenv,
+                                                                                          jobject,
+                                                                                          jlong dbPtr,
+                                                                                          jbyteArray beginKeyBytes,
+                                                                                          jbyteArray endKeyBytes,
+                                                                                          jlong version) {
+	if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+	FDBDatabase* tr = (FDBDatabase*)dbPtr;
+
+	uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
+	if (!startKey) {
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	if (!endKey) {
+		jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	FDBFuture* f = fdb_database_verify_blob_range(
+	    tr, startKey, jenv->GetArrayLength(beginKeyBytes), endKey, jenv->GetArrayLength(endKeyBytes), version);
+	jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
+	jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
+	return (jlong)f;
+}
+
 JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv,
                                                                             jobject,
                                                                             jint predicate,
@@ -1242,6 +1521,41 @@ Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRangeSplitPoints(JNIE
 	return (jlong)f;
 }
 
+JNIEXPORT jlong JNICALL
+Java_com_apple_foundationdb_FDBTransaction_Transaction_1getBlobGranuleRanges(JNIEnv* jenv,
+                                                                             jobject,
+                                                                             jlong tPtr,
+                                                                             jbyteArray beginKeyBytes,
+                                                                             jbyteArray endKeyBytes,
+                                                                             jint rowLimit) {
+	if (!tPtr || !beginKeyBytes || !endKeyBytes || !rowLimit) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+	FDBTransaction* tr = (FDBTransaction*)tPtr;
+
+	uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
+	if (!startKey) {
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	if (!endKey) {
+		jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	FDBFuture* f = fdb_transaction_get_blob_granule_ranges(
+	    tr, startKey, jenv->GetArrayLength(beginKeyBytes), endKey, jenv->GetArrayLength(endKeyBytes), rowLimit);
+	jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
+	jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
+	return (jlong)f;
+}
+
 JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv* jenv,
                                                                                    jobject,
                                                                                    jlong tPtr,
@@ -1681,6 +1995,15 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) {
 		key_array_result_init = env->GetMethodID(local_key_array_result_class, "<init>", "([B[I)V");
 		key_array_result_class = (jclass)(env)->NewGlobalRef(local_key_array_result_class);
 
+		jclass local_keyrange_class = env->FindClass("com/apple/foundationdb/Range");
+		keyrange_init = env->GetMethodID(local_keyrange_class, "<init>", "([B[B)V");
+		keyrange_class = (jclass)(env)->NewGlobalRef(local_keyrange_class);
+
+		jclass local_keyrange_array_result_class = env->FindClass("com/apple/foundationdb/KeyRangeArrayResult");
+		keyrange_array_result_init =
+		    env->GetMethodID(local_keyrange_array_result_class, "<init>", "([Lcom/apple/foundationdb/Range;)V");
+		keyrange_array_result_class = (jclass)(env)->NewGlobalRef(local_keyrange_array_result_class);
+
 		jclass local_range_result_summary_class = env->FindClass("com/apple/foundationdb/RangeResultSummary");
 		range_result_summary_init = env->GetMethodID(local_range_result_summary_class, "<init>", "([BIZ)V");
 		range_result_summary_class = (jclass)(env)->NewGlobalRef(local_range_result_summary_class);
@@ -1705,6 +2028,12 @@ void JNI_OnUnload(JavaVM* vm, void* reserved) {
 		if (range_result_class != JNI_NULL) {
 			env->DeleteGlobalRef(range_result_class);
 		}
+		if (keyrange_array_result_class != JNI_NULL) {
+			env->DeleteGlobalRef(keyrange_array_result_class);
+		}
+		if (keyrange_class != JNI_NULL) {
+			env->DeleteGlobalRef(keyrange_class);
+		}
 		if (mapped_range_result_class != JNI_NULL) {
 			env->DeleteGlobalRef(mapped_range_result_class);
 		}
diff --git a/bindings/java/src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java
index 5087361c43..80afa6c761 100644
--- a/bindings/java/src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java
+++ b/bindings/java/src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java
@@ -40,6 +40,8 @@ import org.junit.jupiter.api.Assertions;
  * This test is to verify the atomicity of transactions. 
  */
 public class CycleMultiClientIntegrationTest {
+    public static final int API_VERSION = 720;
+
     public static final MultiClientHelper clientHelper = new MultiClientHelper();
 
     // more write txn than validate txn, as parent thread waits only for validate txn.
@@ -51,7 +53,7 @@ public class CycleMultiClientIntegrationTest {
     private static List<String> expected = new ArrayList<>(Arrays.asList("0", "1", "2", "3"));
 
     public static void main(String[] args) throws Exception {
-        FDB fdb = FDB.selectAPIVersion(720);
+        FDB fdb = FDB.selectAPIVersion(API_VERSION);
         setupThreads(fdb);
         Collection<Database> dbs = clientHelper.openDatabases(fdb); // the clientHelper will close the databases for us
         System.out.println("Starting tests");
diff --git a/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java b/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java
index 59fbc3fe55..b91c9e7de3 100644
--- a/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java
+++ b/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java
@@ -40,7 +40,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
  */
 @ExtendWith(RequiresDatabase.class)
 class DirectoryTest {
-	private static final FDB fdb = FDB.selectAPIVersion(720);
+	public static final int API_VERSION = 720;
+	private static final FDB fdb = FDB.selectAPIVersion(API_VERSION);
 
 	@Test
 	void testCanCreateDirectory() throws Exception {
diff --git a/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java
index 063e9e276d..3aedef4d1e 100644
--- a/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java
+++ b/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java
@@ -41,7 +41,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
 
 @ExtendWith(RequiresDatabase.class)
 class MappedRangeQueryIntegrationTest {
-	private static final FDB fdb = FDB.selectAPIVersion(720);
+	public static final int API_VERSION = 720;
+	private static final FDB fdb = FDB.selectAPIVersion(API_VERSION);
 	public String databaseArg = null;
 	private Database openFDB() { return fdb.open(databaseArg); }
 
@@ -110,7 +111,7 @@ class MappedRangeQueryIntegrationTest {
 	boolean validate = true;
 	@Test
 	void comparePerformance() {
-		FDB fdb = FDB.selectAPIVersion(720);
+		FDB fdb = FDB.selectAPIVersion(API_VERSION);
 		try (Database db = openFDB()) {
 			insertRecordsWithIndexes(numRecords, db);
 			instrument(rangeQueryAndThenRangeQueries, "rangeQueryAndThenRangeQueries", db);
diff --git a/bindings/java/src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java
index bc64877199..fb6d3afd9f 100644
--- a/bindings/java/src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java
+++ b/bindings/java/src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java
@@ -41,7 +41,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
  */
 @ExtendWith(RequiresDatabase.class)
 class RangeQueryIntegrationTest {
-	private static final FDB fdb = FDB.selectAPIVersion(720);
+	public static final int API_VERSION = 720;
+	private static final FDB fdb = FDB.selectAPIVersion(API_VERSION);
 
 	@BeforeEach
 	@AfterEach
diff --git a/bindings/java/src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java b/bindings/java/src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java
index c11940d41a..ab8ab1364a 100644
--- a/bindings/java/src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java
+++ b/bindings/java/src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java
@@ -41,6 +41,8 @@ import org.junit.jupiter.api.Assertions;
  *   are still seeting the initialValue even after new transactions set them to a new value. 
  */
 public class RepeatableReadMultiThreadClientTest {
+    public static final int API_VERSION = 720;
+
     public static final MultiClientHelper clientHelper = new MultiClientHelper();
 
     private static final int oldValueReadCount = 30;
@@ -52,7 +54,7 @@ public class RepeatableReadMultiThreadClientTest {
     private static final Map<Thread, OldValueReader> threadToOldValueReaders = new HashMap<>();
 
     public static void main(String[] args) throws Exception {
-        FDB fdb = FDB.selectAPIVersion(720);
+        FDB fdb = FDB.selectAPIVersion(API_VERSION);
         setupThreads(fdb);
         Collection<Database> dbs = clientHelper.openDatabases(fdb); // the clientHelper will close the databases for us
         System.out.println("Starting tests");
diff --git a/bindings/java/src/integration/com/apple/foundationdb/RequiresDatabase.java b/bindings/java/src/integration/com/apple/foundationdb/RequiresDatabase.java
index 785008902c..ead1f499c1 100644
--- a/bindings/java/src/integration/com/apple/foundationdb/RequiresDatabase.java
+++ b/bindings/java/src/integration/com/apple/foundationdb/RequiresDatabase.java
@@ -47,6 +47,7 @@ import org.opentest4j.TestAbortedException;
  * be running a server and you don't want to deal with spurious test failures.
  */
 public class RequiresDatabase implements ExecutionCondition, BeforeAllCallback {
+	public static final int API_VERSION = 720;
 
 	public static boolean canRunIntegrationTest() {
 		String prop = System.getProperty("run.integration.tests");
@@ -80,7 +81,7 @@ public class RequiresDatabase implements ExecutionCondition, BeforeAllCallback {
 		 * assume that if we are here, then canRunIntegrationTest() is returning true and we don't have to bother
 		 * checking it.
 		 */
-		try (Database db = FDB.selectAPIVersion(720).open()) {
+		try (Database db = FDB.selectAPIVersion(API_VERSION).open()) {
 			db.run(tr -> {
 				CompletableFuture<byte[]> future = tr.get("test".getBytes());
 
diff --git a/bindings/java/src/integration/com/apple/foundationdb/SidebandMultiThreadClientTest.java b/bindings/java/src/integration/com/apple/foundationdb/SidebandMultiThreadClientTest.java
index 4a4736d566..30f86632eb 100644
--- a/bindings/java/src/integration/com/apple/foundationdb/SidebandMultiThreadClientTest.java
+++ b/bindings/java/src/integration/com/apple/foundationdb/SidebandMultiThreadClientTest.java
@@ -19,6 +19,8 @@ import org.junit.jupiter.api.Assertions;
  * This test is to verify the causal consistency of transactions for mutli-threaded client. 
  */
 public class SidebandMultiThreadClientTest {
+    public static final int API_VERSION = 720;
+
     public static final MultiClientHelper clientHelper = new MultiClientHelper();
 
     private static final Map<Database, BlockingQueue<String>> db2Queues = new HashMap<>();
@@ -26,7 +28,7 @@ public class SidebandMultiThreadClientTest {
     private static final int txnCnt = 1000;
 
     public static void main(String[] args) throws Exception {
-        FDB fdb = FDB.selectAPIVersion(720);
+        FDB fdb = FDB.selectAPIVersion(API_VERSION);
         setupThreads(fdb);
         Collection<Database> dbs = clientHelper.openDatabases(fdb); // the clientHelper will close the databases for us
         for (Database db : dbs) {
diff --git a/bindings/java/src/junit/com/apple/foundationdb/FDBLibraryRule.java b/bindings/java/src/junit/com/apple/foundationdb/FDBLibraryRule.java
index 6e53b179e5..fc54f6c333 100644
--- a/bindings/java/src/junit/com/apple/foundationdb/FDBLibraryRule.java
+++ b/bindings/java/src/junit/com/apple/foundationdb/FDBLibraryRule.java
@@ -29,6 +29,8 @@ import org.junit.jupiter.api.extension.ExtensionContext;
  * are not available for any reason.
  */
 public class FDBLibraryRule implements BeforeAllCallback {
+	public static final int CURRENT_API_VERSION = 720;
+
 	private final int apiVersion;
 
 	// because FDB is a singleton (currently), this isn't a super-useful cache,
@@ -37,7 +39,7 @@ public class FDBLibraryRule implements BeforeAllCallback {
 
 	public FDBLibraryRule(int apiVersion) { this.apiVersion = apiVersion; }
 
-	public static FDBLibraryRule current() { return new FDBLibraryRule(720); }
+	public static FDBLibraryRule current() { return new FDBLibraryRule(CURRENT_API_VERSION); }
 
 	public static FDBLibraryRule v63() { return new FDBLibraryRule(630); }
 
diff --git a/bindings/java/src/main/com/apple/foundationdb/Database.java b/bindings/java/src/main/com/apple/foundationdb/Database.java
index 8606d7ec39..5100dec392 100644
--- a/bindings/java/src/main/com/apple/foundationdb/Database.java
+++ b/bindings/java/src/main/com/apple/foundationdb/Database.java
@@ -161,6 +161,172 @@ public interface Database extends AutoCloseable, TransactionContext {
 	 */
 	double getMainThreadBusyness();
 
+	/**
+	 * Runs {@link #purgeBlobGranules(Function)} on the default executor.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param force if true delete all data, if not keep data >= purgeVersion
+	 *
+	 * @return the key to watch for purge complete
+	 */
+	default CompletableFuture<byte[]> purgeBlobGranules(byte[] beginKey, byte[] endKey, boolean force) {
+		return purgeBlobGranules(beginKey, endKey, -2, force, getExecutor());
+	}
+
+	/**
+	 * Runs {@link #purgeBlobGranules(Function)} on the default executor.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param purgeVersion version to purge at
+	 * @param force if true delete all data, if not keep data >= purgeVersion
+	 *
+	 * @return the key to watch for purge complete
+	 */
+	default CompletableFuture<byte[]> purgeBlobGranules(byte[] beginKey, byte[] endKey, long purgeVersion, boolean force) {
+		return purgeBlobGranules(beginKey, endKey, purgeVersion, force, getExecutor());
+	}
+
+	/**
+	 * Queues a purge of blob granules for the specified key range, at the specified version.
+     *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param purgeVersion version to purge at
+	 * @param force if true delete all data, if not keep data >= purgeVersion
+	 * @param e the {@link Executor} to use for asynchronous callbacks
+
+	 * @return the key to watch for purge complete
+	 */
+	CompletableFuture<byte[]> purgeBlobGranules(byte[] beginKey, byte[] endKey, long purgeVersion, boolean force, Executor e);
+
+
+	/**
+	 * Runs {@link #waitPurgeGranulesComplete(Function)} on the default executor.
+	 *
+	 * @param purgeKey key to watch
+	 */
+	default CompletableFuture<Void> waitPurgeGranulesComplete(byte[] purgeKey) {
+		return waitPurgeGranulesComplete(purgeKey, getExecutor());
+	}
+
+	/**
+	 * Wait for a previous call to purgeBlobGranules to complete.
+	 *
+	 * @param purgeKey key to watch
+	 * @param e the {@link Executor} to use for asynchronous callbacks
+	 */
+	CompletableFuture<Void> waitPurgeGranulesComplete(byte[] purgeKey, Executor e);
+
+	/**
+	 * Runs {@link #blobbifyRange(Function)} on the default executor.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+
+	 * @return if the recording of the range was successful
+	 */
+	default CompletableFuture<Boolean> blobbifyRange(byte[] beginKey, byte[] endKey) {
+		return blobbifyRange(beginKey, endKey, getExecutor());
+	}
+
+	/**
+	 * Sets a range to be blobbified in the database. Must be a completely unblobbified range.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param e the {@link Executor} to use for asynchronous callbacks
+
+	 * @return if the recording of the range was successful
+	 */
+	CompletableFuture<Boolean> blobbifyRange(byte[] beginKey, byte[] endKey, Executor e);
+
+	/**
+	 * Runs {@link #unblobbifyRange(Function)} on the default executor.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+
+	 * @return if the recording of the range was successful
+	 */
+	default CompletableFuture<Boolean> unblobbifyRange(byte[] beginKey, byte[] endKey) {
+		return unblobbifyRange(beginKey, endKey, getExecutor());
+	}
+
+	/**
+	 * Unsets a blobbified range in the database. The range must be aligned to known blob ranges.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param e the {@link Executor} to use for asynchronous callbacks
+
+	 * @return if the recording of the range was successful
+	 */
+	CompletableFuture<Boolean> unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e);
+
+	/**
+	 * Runs {@link #listBlobbifiedRanges(Function)} on the default executor.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param rangeLimit batch size
+	 * @param e the {@link Executor} to use for asynchronous callbacks
+
+	 * @return a future with the list of blobbified ranges: [lastLessThan(beginKey), firstGreaterThanOrEqual(endKey)]
+	 */
+	 default CompletableFuture<KeyRangeArrayResult> listBlobbifiedRanges(byte[] beginKey, byte[] endKey, int rangeLimit) {
+		return listBlobbifiedRanges(beginKey, endKey, rangeLimit, getExecutor());
+	 }
+
+	/**
+	 * Lists blobbified ranges in the database. There may be more if result.size() == rangeLimit.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param rangeLimit batch size
+	 * @param e the {@link Executor} to use for asynchronous callbacks
+
+	 * @return a future with the list of blobbified ranges: [lastLessThan(beginKey), firstGreaterThanOrEqual(endKey)]
+	 */
+	 CompletableFuture<KeyRangeArrayResult> listBlobbifiedRanges(byte[] beginKey, byte[] endKey, int rangeLimit, Executor e);
+
+	/**
+	 * Runs {@link #verifyBlobRange(Function)} on the default executor.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 *
+	 * @return a future with the version of the last blob granule.
+	 */
+	default CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey) {
+		return verifyBlobRange(beginKey, endKey, -2, getExecutor());
+	}
+
+	/**
+	 * Runs {@link #verifyBlobRange(Function)} on the default executor.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param version version to read at
+	 *
+	 * @return a future with the version of the last blob granule.
+	 */
+	default CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey, long version) {
+		return verifyBlobRange(beginKey, endKey, version, getExecutor());
+	}
+
+	/**
+	 * Checks if a blob range is blobbified.
+	 *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param version version to read at
+	 *
+	 * @return a future with the version of the last blob granule.
+	 */
+	CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e);
+
 	/**
 	 * Runs a read-only transactional function against this {@code Database} with retry logic.
 	 *  {@link Function#apply(Object) apply(ReadTransaction)} will be called on the
diff --git a/bindings/java/src/main/com/apple/foundationdb/FDB.java b/bindings/java/src/main/com/apple/foundationdb/FDB.java
index 5215d0836e..47ba2eead1 100644
--- a/bindings/java/src/main/com/apple/foundationdb/FDB.java
+++ b/bindings/java/src/main/com/apple/foundationdb/FDB.java
@@ -191,11 +191,6 @@ public class FDB {
 		Select_API_version(version);
 		singleton = new FDB(version);
 
-		if (version < 720) {
-			TenantManagement.TENANT_MAP_PREFIX = ByteArrayUtil.join(new byte[] { (byte)255, (byte)255 },
-					"/management/tenant_map/".getBytes());
-		}
-
 		return singleton;
 	}
 
diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java
index 5e0b808242..98c001a1b0 100644
--- a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java
+++ b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java
@@ -200,6 +200,66 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume
 		}
 	}
 
+	@Override
+	public CompletableFuture<byte[]> purgeBlobGranules(byte[] beginKey, byte[] endKey, long purgeVersion, boolean force, Executor e) {
+		pointerReadLock.lock();
+		try {
+			return new FutureKey(Database_purgeBlobGranules(getPtr(), beginKey, endKey, purgeVersion, force), e, eventKeeper);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
+	@Override
+	public CompletableFuture<Void> waitPurgeGranulesComplete(byte[] purgeKey, Executor e) {
+		pointerReadLock.lock();
+		try {
+			return new FutureVoid(Database_waitPurgeGranulesComplete(getPtr(), purgeKey), e);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
+	@Override
+	public CompletableFuture<Boolean> blobbifyRange(byte[] beginKey, byte[] endKey, Executor e) {
+		pointerReadLock.lock();
+		try {
+			return new FutureBool(Database_blobbifyRange(getPtr(), beginKey, endKey), e);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
+	@Override
+	public CompletableFuture<Boolean> unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e) {
+		pointerReadLock.lock();
+		try {
+			return new FutureBool(Database_unblobbifyRange(getPtr(), beginKey, endKey), e);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
+	@Override
+	public CompletableFuture<KeyRangeArrayResult> listBlobbifiedRanges(byte[] beginKey, byte[] endKey, int rangeLimit, Executor e) {
+		pointerReadLock.lock();
+		try {
+			return new FutureKeyRangeArray(Database_listBlobbifiedRanges(getPtr(), beginKey, endKey, rangeLimit), e);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
+	@Override
+	public CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e) {
+		pointerReadLock.lock();
+		try {
+			return new FutureInt64(Database_verifyBlobRange(getPtr(), beginKey, endKey, version), e);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
 	@Override
 	public Executor getExecutor() {
 		return executor;
@@ -215,4 +275,10 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume
 	private native void Database_dispose(long cPtr);
 	private native void Database_setOption(long cPtr, int code, byte[] value) throws FDBException;
 	private native double Database_getMainThreadBusyness(long cPtr);
+	private native long Database_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force);
+	private native long Database_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey);
+	private native long Database_blobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
+	private native long Database_unblobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
+	private native long Database_listBlobbifiedRanges(long cPtr, byte[] beginKey, byte[] endKey, int rangeLimit);
+	private native long Database_verifyBlobRange(long cPtr, byte[] beginKey, byte[] endKey, long version);
 }
\ No newline at end of file
diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java
index b35196c146..7943c5e9d1 100644
--- a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java
+++ b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java
@@ -97,6 +97,11 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 			return FDBTransaction.this.getRangeSplitPoints(range, chunkSize);
 		}
 
+		@Override
+		public CompletableFuture<KeyRangeArrayResult> getBlobGranuleRanges(byte[] begin, byte[] end, int rowLimit) {
+			return FDBTransaction.this.getBlobGranuleRanges(begin, end, rowLimit);
+		}
+
 		@Override
 		public AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper,
 		                                                    int limit, int matchIndex, boolean reverse,
@@ -352,6 +357,16 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 		return this.getRangeSplitPoints(range.begin, range.end, chunkSize);
 	}
 
+	@Override
+	public CompletableFuture<KeyRangeArrayResult> getBlobGranuleRanges(byte[] begin, byte[] end, int rowLimit) {
+		pointerReadLock.lock();
+		try {
+			return new FutureKeyRangeArray(Transaction_getBlobGranuleRanges(getPtr(), begin, end, rowLimit), executor);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
 	@Override
 	public AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper, int limit,
 	                                                    int matchIndex, boolean reverse, StreamingMode mode) {
@@ -842,4 +857,5 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 	private native long Transaction_getKeyLocations(long cPtr, byte[] key);
 	private native long Transaction_getEstimatedRangeSizeBytes(long cPtr, byte[] keyBegin, byte[] keyEnd);
 	private native long Transaction_getRangeSplitPoints(long cPtr, byte[] keyBegin, byte[] keyEnd, long chunkSize);
+	private native long Transaction_getBlobGranuleRanges(long cPtr, byte[] keyBegin, byte[] keyEnd, int rowLimit);
 }
diff --git a/bindings/java/src/main/com/apple/foundationdb/FutureBool.java b/bindings/java/src/main/com/apple/foundationdb/FutureBool.java
new file mode 100644
index 0000000000..ddbbd02649
--- /dev/null
+++ b/bindings/java/src/main/com/apple/foundationdb/FutureBool.java
@@ -0,0 +1,37 @@
+/*
+ * FutureBool.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb;
+
+import java.util.concurrent.Executor;
+
+class FutureBool extends NativeFuture<Boolean> {
+	FutureBool(long cPtr, Executor executor) {
+		super(cPtr);
+		registerMarshalCallback(executor);
+	}
+
+	@Override
+	protected Boolean getIfDone_internal(long cPtr) throws FDBException {
+		return FutureBool_get(cPtr);
+	}
+
+	private native boolean FutureBool_get(long cPtr) throws FDBException;
+}
diff --git a/bindings/java/src/main/com/apple/foundationdb/FutureKeyRangeArray.java b/bindings/java/src/main/com/apple/foundationdb/FutureKeyRangeArray.java
new file mode 100644
index 0000000000..d866e9fca4
--- /dev/null
+++ b/bindings/java/src/main/com/apple/foundationdb/FutureKeyRangeArray.java
@@ -0,0 +1,37 @@
+/*
+ * FutureKeyRangeArray.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb;
+
+import java.util.concurrent.Executor;
+
+class FutureKeyRangeArray extends NativeFuture<KeyRangeArrayResult> {
+	FutureKeyRangeArray(long cPtr, Executor executor) {
+		super(cPtr);
+		registerMarshalCallback(executor);
+	}
+
+	@Override
+	protected KeyRangeArrayResult getIfDone_internal(long cPtr) throws FDBException {
+		return FutureKeyRangeArray_get(cPtr);
+	}
+
+	private native KeyRangeArrayResult FutureKeyRangeArray_get(long cPtr) throws FDBException;
+}
diff --git a/bindings/java/src/main/com/apple/foundationdb/KeyRangeArrayResult.java b/bindings/java/src/main/com/apple/foundationdb/KeyRangeArrayResult.java
new file mode 100644
index 0000000000..7385b8fe0a
--- /dev/null
+++ b/bindings/java/src/main/com/apple/foundationdb/KeyRangeArrayResult.java
@@ -0,0 +1,36 @@
+/*
+ * KeyRangeArrayResult.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class KeyRangeArrayResult {
+	final List<Range> keyRanges;
+
+	public KeyRangeArrayResult(Range[] keyRangeArr) {
+		this.keyRanges = Arrays.asList(keyRangeArr);
+	}
+
+	public List<Range> getKeyRanges() {
+		return keyRanges;
+	}
+}
diff --git a/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java b/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java
index 11ed7e900c..04050de6fb 100644
--- a/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java
+++ b/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java
@@ -513,6 +513,17 @@ public interface ReadTransaction extends ReadTransactionContext {
 	 */
 	CompletableFuture<KeyArrayResult> getRangeSplitPoints(Range range, long chunkSize);
 
+	/**
+	 * Gets the blob granule ranges for a given region.
+	 * Returned in batches, requires calling again moving the begin key up.
+	 *
+	 * @param begin beginning of the range (inclusive)
+	 * @param end end of the range (exclusive)
+
+	 * @return list of blob granules in the given range. May not be all.
+	 */
+	 CompletableFuture<KeyRangeArrayResult> getBlobGranuleRanges(byte[] begin, byte[] end, int rowLimit);
+
 	
 	/**
 	 * Returns a set of options that can be set on a {@code Transaction}
diff --git a/bindings/java/src/main/com/apple/foundationdb/TenantManagement.java b/bindings/java/src/main/com/apple/foundationdb/TenantManagement.java
index 12aaf70322..58f223fa4b 100644
--- a/bindings/java/src/main/com/apple/foundationdb/TenantManagement.java
+++ b/bindings/java/src/main/com/apple/foundationdb/TenantManagement.java
@@ -262,7 +262,7 @@ public class TenantManagement {
 			this.begin = ByteArrayUtil.join(TENANT_MAP_PREFIX, begin);
 			this.end = ByteArrayUtil.join(TENANT_MAP_PREFIX, end);
 
-			tr.options().setReadSystemKeys();
+			tr.options().setRawAccess();
 			tr.options().setLockAware();
 
 			firstGet = tr.getRange(this.begin, this.end, limit);
diff --git a/bindings/java/src/main/overview.html.in b/bindings/java/src/main/overview.html.in
index 3154efbfc3..a37a8859f9 100644
--- a/bindings/java/src/main/overview.html.in
+++ b/bindings/java/src/main/overview.html.in
@@ -28,8 +28,10 @@ import com.apple.foundationdb.FDB;
 import com.apple.foundationdb.tuple.Tuple;
 
 public class Example {
+  public static final int apiVersion = 720;
+
   public static void main(String[] args) {
-    FDB fdb = FDB.selectAPIVersion(720);
+    FDB fdb = FDB.selectAPIVersion(apiVersion);
 
     try(Database db = fdb.open()) {
       // Run an operation on the database
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java
index a1d7a4d976..425c5d2369 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java
@@ -29,11 +29,13 @@ import com.apple.foundationdb.FDB;
 import com.apple.foundationdb.Transaction;
 
 public class BlockingBenchmark {
+	public static final int API_VERSION = 720;
+
 	private static final int REPS = 100000;
 	private static final int PARALLEL = 100;
 
 	public static void main(String[] args) throws InterruptedException {
-		FDB fdb = FDB.selectAPIVersion(720);
+		FDB fdb = FDB.selectAPIVersion(API_VERSION);
 
 		// The cluster file DOES NOT need to be valid, although it must exist.
 		//  This is because the database is never really contacted in this test.
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java
index 38be19a60f..0eabef64c0 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java
@@ -30,6 +30,8 @@ import com.apple.foundationdb.Database;
 import com.apple.foundationdb.FDB;
 
 public class ConcurrentGetSetGet {
+	public static final int API_VERSION = 720;
+
 	public static final Charset UTF8 = Charset.forName("UTF-8");
 
 	final Semaphore semaphore = new Semaphore(CONCURRENCY);
@@ -48,7 +50,7 @@ public class ConcurrentGetSetGet {
 	}
 
 	public static void main(String[] args) {
-		try(Database database = FDB.selectAPIVersion(720).open()) {
+		try(Database database = FDB.selectAPIVersion(API_VERSION).open()) {
 			new ConcurrentGetSetGet().apply(database);
 		}
 	}
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/Context.java b/bindings/java/src/test/com/apple/foundationdb/test/Context.java
index a594e088a1..dbbe7e73de 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/Context.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/Context.java
@@ -29,6 +29,7 @@ import java.util.Optional;
 
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
 
 import com.apple.foundationdb.Database;
 import com.apple.foundationdb.FDB;
@@ -64,7 +65,7 @@ abstract class Context implements Runnable, AutoCloseable {
 	private List<Thread> children = new LinkedList<>();
 	private static Map<String, TransactionState> transactionMap = new HashMap<>();
 	private static Map<Transaction, AtomicInteger> transactionRefCounts = new HashMap<>();
-	private static Map<byte[], Tenant> tenantMap = new HashMap<>();
+	private static Map<byte[], Tenant> tenantMap = new ConcurrentHashMap<>();
 
 	Context(Database db, byte[] prefix) {
 		this.db = db;
@@ -83,8 +84,8 @@ abstract class Context implements Runnable, AutoCloseable {
 		try {
 			executeOperations();
 		} catch(Throwable t) {
-			// EAT
 			t.printStackTrace();
+			System.exit(1);
 		}
 		while(children.size() > 0) {
 			//System.out.println("Shutting down...waiting on " + children.size() + " threads");
@@ -146,10 +147,11 @@ abstract class Context implements Runnable, AutoCloseable {
 	private static synchronized boolean newTransaction(Database db, Optional<Tenant> tenant, String trName, boolean allowReplace) {
 		TransactionState oldState = transactionMap.get(trName);
 		if (oldState != null) {
-			releaseTransaction(oldState.transaction);
-		}
-		else if (!allowReplace) {
-			return false;
+			if (allowReplace) {
+				releaseTransaction(oldState.transaction);
+			} else {
+				return false;
+			}
 		}
 
 		TransactionState newState = new TransactionState(createTransaction(db, tenant), tenant);
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/Example.java b/bindings/java/src/test/com/apple/foundationdb/test/Example.java
index da5bbfdc2a..d7f1336d51 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/Example.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/Example.java
@@ -25,8 +25,10 @@ import com.apple.foundationdb.FDB;
 import com.apple.foundationdb.tuple.Tuple;
 
 public class Example {
+	public static final int API_VERSION = 720;
+
 	public static void main(String[] args) {
-		FDB fdb = FDB.selectAPIVersion(720);
+		FDB fdb = FDB.selectAPIVersion(API_VERSION);
 
 		try(Database db = fdb.open()) {
 			// Run an operation on the database
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java
index 35adfa5e1f..2af2152cf3 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java
@@ -28,10 +28,12 @@ import com.apple.foundationdb.KeyValue;
 import com.apple.foundationdb.TransactionContext;
 
 public class IterableTest {
+	public static final int API_VERSION = 720;
+
 	public static void main(String[] args) throws InterruptedException {
 		final int reps = 1000;
 		try {
-			FDB fdb = FDB.selectAPIVersion(720);
+			FDB fdb = FDB.selectAPIVersion(API_VERSION);
 			try(Database db = fdb.open()) {
 				runTests(reps, db);
 			}
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java
index 6410165f27..969b6c75e3 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java
@@ -32,9 +32,10 @@ import com.apple.foundationdb.async.AsyncUtil;
 import com.apple.foundationdb.tuple.ByteArrayUtil;
 
 public class LocalityTests {
+	public static final int API_VERSION = 720;
 
 	public static void main(String[] args) {
-		FDB fdb = FDB.selectAPIVersion(720);
+		FDB fdb = FDB.selectAPIVersion(API_VERSION);
 		try(Database database = fdb.open(args[0])) {
 			try(Transaction tr = database.createTransaction()) {
 				String[] keyAddresses = LocalityUtil.getAddressesForKey(tr, "a".getBytes()).join();
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java
index 6518116324..e5e0c9f9e6 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java
@@ -36,6 +36,8 @@ import com.apple.foundationdb.async.AsyncIterator;
 import com.apple.foundationdb.tuple.ByteArrayUtil;
 
 public class ParallelRandomScan {
+	public static final int API_VERSION = 720;
+
 	private static final int ROWS = 1000000;
 	private static final int DURATION_MS = 2000;
 	private static final int PARALLELISM_MIN = 10;
@@ -43,7 +45,7 @@ public class ParallelRandomScan {
 	private static final int PARALLELISM_STEP = 5;
 
 	public static void main(String[] args) throws InterruptedException {
-		FDB api = FDB.selectAPIVersion(720);
+		FDB api = FDB.selectAPIVersion(API_VERSION);
 		try(Database database = api.open(args[0])) {
 			for(int i = PARALLELISM_MIN; i <= PARALLELISM_MAX; i += PARALLELISM_STEP) {
 				runTest(database, i, ROWS, DURATION_MS);
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java
index d847556cdc..e11f8b9793 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java
@@ -29,12 +29,14 @@ import com.apple.foundationdb.FDB;
 import com.apple.foundationdb.Transaction;
 
 public class SerialInsertion {
+	public static final int API_VERSION = 720;
+
 	private static final int THREAD_COUNT = 10;
 	private static final int BATCH_SIZE = 1000;
 	private static final int NODES = 1000000;
 
 	public static void main(String[] args) {
-		FDB api = FDB.selectAPIVersion(720);
+		FDB api = FDB.selectAPIVersion(API_VERSION);
 		try(Database database = api.open()) {
 			long start = System.currentTimeMillis();
 
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java
index 6e262e561f..f55af41c35 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java
@@ -34,12 +34,14 @@ import com.apple.foundationdb.Transaction;
 import com.apple.foundationdb.async.AsyncIterable;
 
 public class SerialIteration {
+	public static final int API_VERSION = 720;
+
 	private static final int ROWS = 1000000;
 	private static final int RUNS = 25;
 	private static final int THREAD_COUNT = 1;
 
 	public static void main(String[] args) throws InterruptedException {
-		FDB api = FDB.selectAPIVersion(720);
+		FDB api = FDB.selectAPIVersion(API_VERSION);
 		try(Database database = api.open(args[0])) {
 			for(int i = 1; i <= THREAD_COUNT; i++) {
 				runThreadedTest(database, i);
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java
index 9313543d02..ea3210e2de 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java
@@ -27,10 +27,12 @@ import com.apple.foundationdb.FDB;
 import com.apple.foundationdb.TransactionContext;
 
 public class SerialTest {
+	public static final int API_VERSION = 720;
+
 	public static void main(String[] args) throws InterruptedException {
 		final int reps = 1000;
 		try {
-			FDB fdb = FDB.selectAPIVersion(720);
+			FDB fdb = FDB.selectAPIVersion(API_VERSION);
 			try(Database db = fdb.open()) {
 				runTests(reps, db);
 			}
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java
index 1f3aec5501..6fdee20cad 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java
@@ -35,11 +35,13 @@ import com.apple.foundationdb.tuple.Tuple;
  * Some tests regarding conflict ranges to make sure they do what we expect.
  */
 public class SnapshotTransactionTest {
+	public static final int API_VERSION = 720;
+
 	private static final int CONFLICT_CODE = 1020;
 	private static final Subspace SUBSPACE = new Subspace(Tuple.from("test", "conflict_ranges"));
 
 	public static void main(String[] args) {
-		FDB fdb = FDB.selectAPIVersion(720);
+		FDB fdb = FDB.selectAPIVersion(API_VERSION);
 		try(Database db = fdb.open()) {
 			snapshotReadShouldNotConflict(db);
 			snapshotShouldNotAddConflictRange(db);
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java
index 4fa45f7cbe..2ce8e76343 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java
@@ -32,12 +32,14 @@ import com.apple.foundationdb.tuple.Tuple;
 import com.apple.foundationdb.tuple.Versionstamp;
 
 public class TupleTest {
+	public static final int API_VERSION = 720;
+
 	private static final byte FF = (byte)0xff;
 
 	public static void main(String[] args) throws NoSuchFieldException {
 		final int reps = 1000;
 		try {
-			FDB fdb = FDB.selectAPIVersion(720);
+			FDB fdb = FDB.selectAPIVersion(API_VERSION);
 			try(Database db = fdb.open()) {
 				runTests(reps, db);
 			}
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java
index b39744dd32..421db6c542 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java
@@ -31,8 +31,10 @@ import com.apple.foundationdb.tuple.Tuple;
 import com.apple.foundationdb.tuple.Versionstamp;
 
 public class VersionstampSmokeTest {
+	public static final int API_VERSION = 720;
+
 	public static void main(String[] args) {
-		FDB fdb = FDB.selectAPIVersion(720);
+		FDB fdb = FDB.selectAPIVersion(API_VERSION);
 		try(Database db = fdb.open()) {
 			db.run(tr -> {
 				tr.clear(Tuple.from("prefix").range());
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java
index 29e05db04e..ef8db81fd4 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java
@@ -32,9 +32,10 @@ import com.apple.foundationdb.FDBException;
 import com.apple.foundationdb.Transaction;
 
 public class WatchTest {
+	public static final int API_VERSION = 720;
 
 	public static void main(String[] args) {
-		FDB fdb = FDB.selectAPIVersion(720);
+		FDB fdb = FDB.selectAPIVersion(API_VERSION);
 		try(Database database = fdb.open(args[0])) {
 			database.options().setLocationCacheSize(42);
 			try(Transaction tr = database.createTransaction()) {
diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt
index 0f871d8c87..af281a7405 100644
--- a/bindings/python/CMakeLists.txt
+++ b/bindings/python/CMakeLists.txt
@@ -68,45 +68,10 @@ endif()
 set(setup_file_name foundationdb-${FDB_VERSION}.tar.gz)
 set(package_file ${CMAKE_BINARY_DIR}/packages/foundationdb-${FDB_VERSION}${not_fdb_release_string}.tar.gz)
 add_custom_command(OUTPUT ${package_file}
-  COMMAND $<TARGET_FILE:Python::Interpreter> setup.py sdist --formats=gztar &&
+  COMMAND $<TARGET_FILE:Python3::Interpreter> setup.py sdist --formats=gztar &&
           ${CMAKE_COMMAND} -E copy dist/${setup_file_name} ${package_file}
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   COMMENT "Create Python sdist package")
 add_custom_target(python_package DEPENDS ${package_file})
 add_dependencies(python_package python_binding)
 add_dependencies(packages python_package)
-
-if (NOT WIN32 AND NOT OPEN_FOR_IDE)
-  add_fdbclient_test(
-    NAME single_process_fdbcli_tests
-    COMMAND ${CMAKE_SOURCE_DIR}/bindings/python/tests/fdbcli_tests.py
-            ${CMAKE_BINARY_DIR}
-            @CLUSTER_FILE@
-            )
-  add_fdbclient_test(
-    NAME multi_process_fdbcli_tests
-    PROCESS_NUMBER 5
-    COMMAND ${CMAKE_SOURCE_DIR}/bindings/python/tests/fdbcli_tests.py
-            ${CMAKE_BINARY_DIR}
-            @CLUSTER_FILE@
-            5
-            )
-  if (TARGET external_client) # external_client copies fdb_c to bindings/c/libfdb_c_external.so
-    add_fdbclient_test(
-    NAME single_process_external_client_fdbcli_tests
-    COMMAND ${CMAKE_SOURCE_DIR}/bindings/python/tests/fdbcli_tests.py
-            ${CMAKE_BINARY_DIR}
-            @CLUSTER_FILE@
-            --external-client-library ${CMAKE_BINARY_DIR}/bindings/c/libfdb_c_external.so
-            )
-    add_fdbclient_test(
-    NAME multi_process_external_client_fdbcli_tests
-    PROCESS_NUMBER 5
-    COMMAND ${CMAKE_SOURCE_DIR}/bindings/python/tests/fdbcli_tests.py
-            ${CMAKE_BINARY_DIR}
-            @CLUSTER_FILE@
-            5
-            --external-client-library ${CMAKE_BINARY_DIR}/bindings/c/libfdb_c_external.so
-            )
-  endif()
-endif()
diff --git a/bindings/python/fdb/__init__.py b/bindings/python/fdb/__init__.py
index e7d1a8bc30..930ad35396 100644
--- a/bindings/python/fdb/__init__.py
+++ b/bindings/python/fdb/__init__.py
@@ -100,10 +100,8 @@ def api_version(ver):
 
     _add_symbols(fdb.impl, list)
 
-    if ver >= 710:
+    if ver >= 630:
         import fdb.tenant_management
-        if ver < 720:
-            fdb.tenant_management._tenant_map_prefix = b'\xff\xff/management/tenant_map/'
 
     if ver < 610:
         globals()["init"] = getattr(fdb.impl, "init")
diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py
index 51d67e5162..aa967ba25d 100644
--- a/bindings/python/fdb/impl.py
+++ b/bindings/python/fdb/impl.py
@@ -1359,7 +1359,7 @@ else:
     except:
         # The system python on OS X can't find the library installed to /usr/local/lib if SIP is enabled
         # find_library does find the location in /usr/local/lib, so if the above fails fallback to using it
-        lib_path = ctypes.util.find_library(capi_name)
+        lib_path = ctypes.util.find_library("fdb_c")
         if lib_path is not None:
             try:
                 _capi = ctypes.CDLL(lib_path)
diff --git a/bindings/python/fdb/tenant_management.py b/bindings/python/fdb/tenant_management.py
index 84c3a46d03..ebe36594a5 100644
--- a/bindings/python/fdb/tenant_management.py
+++ b/bindings/python/fdb/tenant_management.py
@@ -103,7 +103,7 @@ class FDBTenantList(object):
 # JSON strings of the tenant metadata
 @_impl.transactional
 def _list_tenants_impl(tr, begin, end, limit):
-    tr.options.set_read_system_keys()
+    tr.options.set_raw_access()
     begin_key = b'%s%s' % (_tenant_map_prefix, begin)
     end_key = b'%s%s' % (_tenant_map_prefix, end)
 
diff --git a/bindings/python/tests/size_limit_tests.py b/bindings/python/tests/size_limit_tests.py
index cd27f985b0..b94d7ea8e4 100644
--- a/bindings/python/tests/size_limit_tests.py
+++ b/bindings/python/tests/size_limit_tests.py
@@ -66,6 +66,9 @@ def test_size_limit_option(db):
     except fdb.FDBError as e:
         assert(e.code == 2101)  # Transaction exceeds byte limit (2101)
 
+    # Reset the size limit for future tests
+    db.options.set_transaction_size_limit(10000000)
+
 @fdb.transactional
 def test_get_approximate_size(tr):
     tr[b'key1'] = b'value1'
diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake
index c0440df5ef..786126359b 100644
--- a/cmake/AddFdbTest.cmake
+++ b/cmake/AddFdbTest.cmake
@@ -125,7 +125,7 @@ function(add_fdb_test)
   list(TRANSFORM ADD_FDB_TEST_TEST_FILES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
   if (ENABLE_SIMULATION_TESTS)
     add_test(NAME ${test_name}
-      COMMAND $<TARGET_FILE:Python::Interpreter> ${TestRunner}
+      COMMAND $<TARGET_FILE:Python3::Interpreter> ${TestRunner}
       -n ${test_name}
       -b ${PROJECT_BINARY_DIR}
       -t ${test_type}
@@ -142,7 +142,7 @@ function(add_fdb_test)
       ${VALGRIND_OPTION}
       ${ADD_FDB_TEST_TEST_FILES}
       WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-    set_tests_properties("${test_name}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
+    set_tests_properties("${test_name}" PROPERTIES ENVIRONMENT "${SANITIZER_OPTIONS}")
     get_filename_component(test_dir_full ${first_file} DIRECTORY)
     if(NOT ${test_dir_full} STREQUAL "")
       get_filename_component(test_dir ${test_dir_full} NAME)
@@ -172,8 +172,7 @@ function(stage_correctness_package)
   file(MAKE_DIRECTORY ${STAGE_OUT_DIR}/bin)
   string(LENGTH "${CMAKE_SOURCE_DIR}/tests/" base_length)
   foreach(test IN LISTS TEST_NAMES)
-    if(("${TEST_TYPE_${test}}" STREQUAL "simulation") AND
-        (${test} MATCHES ${TEST_PACKAGE_INCLUDE}) AND
+    if((${test} MATCHES ${TEST_PACKAGE_INCLUDE}) AND
         (NOT ${test} MATCHES ${TEST_PACKAGE_EXCLUDE}))
       foreach(file IN LISTS TEST_FILES_${test})
         string(SUBSTRING ${file} ${base_length} -1 rel_out_file)
@@ -199,16 +198,17 @@ function(stage_correctness_package)
       set(src_dir "${src_dir}/")
       string(SUBSTRING ${src_dir} ${dir_len} -1 dest_dir)
       string(SUBSTRING ${file} ${dir_len} -1 rel_out_file)
-	  set(out_file ${STAGE_OUT_DIR}/${rel_out_file})
+      set(out_file ${STAGE_OUT_DIR}/${rel_out_file})
       list(APPEND external_files ${out_file})
-	  add_custom_command(
+      add_custom_command(
         OUTPUT ${out_file}
-		DEPENDS ${file}
-		COMMAND ${CMAKE_COMMAND} -E copy ${file} ${out_file}
-		COMMENT "Copying ${STAGE_CONTEXT} external file ${file}"
-		)
+        DEPENDS ${file}
+        COMMAND ${CMAKE_COMMAND} -E copy ${file} ${out_file}
+        COMMENT "Copying ${STAGE_CONTEXT} external file ${file}"
+        )
     endforeach()
   endforeach()
+
   list(APPEND package_files ${STAGE_OUT_DIR}/bin/fdbserver
                             ${STAGE_OUT_DIR}/bin/coverage.fdbserver.xml
                             ${STAGE_OUT_DIR}/bin/coverage.fdbclient.xml
@@ -218,6 +218,7 @@ function(stage_correctness_package)
                             ${STAGE_OUT_DIR}/bin/TraceLogHelper.dll
                             ${STAGE_OUT_DIR}/CMakeCache.txt
     )
+
   add_custom_command(
     OUTPUT ${package_files}
     DEPENDS ${CMAKE_BINARY_DIR}/CMakeCache.txt
@@ -239,6 +240,20 @@ function(stage_correctness_package)
                                      ${STAGE_OUT_DIR}/bin
     COMMENT "Copying files for ${STAGE_CONTEXT} package"
     )
+
+  set(test_harness_dir "${CMAKE_SOURCE_DIR}/contrib/TestHarness2")
+  file(GLOB_RECURSE test_harness2_files RELATIVE "${test_harness_dir}" CONFIGURE_DEPENDS "${test_harness_dir}/*.py")
+  foreach(file IN LISTS test_harness2_files)
+    set(src_file "${test_harness_dir}/${file}")
+    set(out_file "${STAGE_OUT_DIR}/${file}")
+    get_filename_component(dir "${out_file}" DIRECTORY)
+    file(MAKE_DIRECTORY "${dir}")
+    add_custom_command(OUTPUT ${out_file}
+      COMMAND ${CMAKE_COMMAND} -E copy "${src_file}" "${out_file}"
+      DEPENDS "${src_file}")
+    list(APPEND package_files "${out_file}")
+  endforeach()
+
   list(APPEND package_files ${test_files} ${external_files})
   if(STAGE_OUT_FILES)
     set(${STAGE_OUT_FILES} ${package_files} PARENT_SCOPE)
@@ -404,7 +419,7 @@ endfunction()
 
 # Creates a single cluster before running the specified command (usually a ctest test)
 function(add_fdbclient_test)
-  set(options DISABLED ENABLED DISABLE_LOG_DUMP API_TEST_BLOB_GRANULES_ENABLED TLS_ENABLED)
+  set(options DISABLED ENABLED DISABLE_TENANTS DISABLE_LOG_DUMP API_TEST_BLOB_GRANULES_ENABLED TLS_ENABLED)
   set(oneValueArgs NAME PROCESS_NUMBER TEST_TIMEOUT WORKING_DIRECTORY)
   set(multiValueArgs COMMAND)
   cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
@@ -431,6 +446,9 @@ function(add_fdbclient_test)
   if(T_DISABLE_LOG_DUMP)
     list(APPEND TMP_CLUSTER_CMD --disable-log-dump)
   endif()
+  if(T_DISABLE_TENANTS)
+    list(APPEND TMP_CLUSTER_CMD --disable-tenants)
+  endif()
   if(T_API_TEST_BLOB_GRANULES_ENABLED)
     list(APPEND TMP_CLUSTER_CMD --blob-granules-enabled)
   endif()
@@ -440,16 +458,20 @@ function(add_fdbclient_test)
   message(STATUS "Adding Client test ${T_NAME}")
   add_test(NAME "${T_NAME}"
     WORKING_DIRECTORY ${T_WORKING_DIRECTORY}
-    COMMAND ${Python_EXECUTABLE} ${TMP_CLUSTER_CMD}
+    COMMAND ${Python3_EXECUTABLE} ${TMP_CLUSTER_CMD}
             --
             ${T_COMMAND})
   if (T_TEST_TIMEOUT)
     set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT ${T_TEST_TIMEOUT})
   else()
     # default timeout
-    set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 300)
+    if(USE_SANITIZER)
+      set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 1200)
+    else()
+      set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 300)
+    endif()
   endif()
-  set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
+  set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT "${SANITIZER_OPTIONS}")
 endfunction()
 
 # Creates a cluster file for a nonexistent cluster before running the specified command
@@ -473,7 +495,7 @@ function(add_unavailable_fdbclient_test)
   endif()
   message(STATUS "Adding unavailable client test ${T_NAME}")
   add_test(NAME "${T_NAME}"
-  COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/fake_cluster.py
+  COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/fake_cluster.py
           --output-dir ${CMAKE_BINARY_DIR}
           --
           ${T_COMMAND})
@@ -483,7 +505,7 @@ function(add_unavailable_fdbclient_test)
     # default timeout
     set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60)
   endif()
-  set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
+  set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT "${SANITIZER_OPTIONS}")
 endfunction()
 
 # Creates 3 distinct clusters before running the specified command.
@@ -508,7 +530,7 @@ function(add_multi_fdbclient_test)
   endif()
   message(STATUS "Adding Client test ${T_NAME}")
   add_test(NAME "${T_NAME}"
-    COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_multi_cluster.py
+    COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_multi_cluster.py
             --build-dir ${CMAKE_BINARY_DIR}
             --clusters 3
             --
diff --git a/cmake/CompileBoost.cmake b/cmake/CompileBoost.cmake
index 9952746972..47d3fb2996 100644
--- a/cmake/CompileBoost.cmake
+++ b/cmake/CompileBoost.cmake
@@ -36,9 +36,12 @@ function(compile_boost)
   set(B2_COMMAND "./b2")
   set(BOOST_COMPILER_FLAGS -fvisibility=hidden -fPIC -std=c++17 -w)
   set(BOOST_LINK_FLAGS "")
-  if(APPLE OR CLANG OR ICX OR USE_LIBCXX)
+  if(APPLE OR ICX OR USE_LIBCXX)
     list(APPEND BOOST_COMPILER_FLAGS -stdlib=libc++ -nostdlib++)
-    list(APPEND BOOST_LINK_FLAGS -static-libgcc -lc++ -lc++abi)
+    list(APPEND BOOST_LINK_FLAGS -lc++ -lc++abi)
+    if (NOT APPLE)
+      list(APPEND BOOST_LINK_FLAGS -static-libgcc)
+    endif()
   endif()
 
   # Update the user-config.jam
@@ -46,27 +49,35 @@ function(compile_boost)
   foreach(flag IN LISTS BOOST_COMPILER_FLAGS COMPILE_BOOST_CXXFLAGS)
     string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIONS "<cxxflags>${flag} ")
   endforeach()
-  #foreach(flag IN LISTS BOOST_LINK_FLAGS COMPILE_BOOST_LDFLAGS)
-  # string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIONS "<linkflags>${flag} ")
-  #endforeach()
+  foreach(flag IN LISTS BOOST_LINK_FLAGS COMPILE_BOOST_LDFLAGS)
+    string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIONS "<linkflags>${flag} ")
+  endforeach()
   configure_file(${CMAKE_SOURCE_DIR}/cmake/user-config.jam.cmake ${CMAKE_BINARY_DIR}/user-config.jam)
   set(USER_CONFIG_FLAG --user-config=${CMAKE_BINARY_DIR}/user-config.jam)
 
   # Build boost
   include(ExternalProject)
+
   set(BOOST_INSTALL_DIR "${CMAKE_BINARY_DIR}/boost_install")
   ExternalProject_add("${COMPILE_BOOST_TARGET}Project"
-    URL "https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.bz2"
-    URL_HASH SHA256=8681f175d4bdb26c52222665793eef08490d7758529330f98d3b29dd0735bccc
-    CONFIGURE_COMMAND ${BOOTSTRAP_COMMAND} ${BOOTSTRAP_ARGS} --with-libraries=${BOOTSTRAP_LIBRARIES} --with-toolset=${BOOST_TOOLSET}
-    BUILD_COMMAND ${B2_COMMAND} link=static ${COMPILE_BOOST_BUILD_ARGS} --prefix=${BOOST_INSTALL_DIR} ${USER_CONFIG_FLAG} install
-    BUILD_IN_SOURCE ON
-    INSTALL_COMMAND ""
-    UPDATE_COMMAND ""
-    BUILD_BYPRODUCTS "${BOOST_INSTALL_DIR}/boost/config.hpp"
-                     "${BOOST_INSTALL_DIR}/lib/libboost_context.a"
-                     "${BOOST_INSTALL_DIR}/lib/libboost_filesystem.a"
-                     "${BOOST_INSTALL_DIR}/lib/libboost_iostreams.a")
+    URL                "https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.bz2"
+    URL_HASH           SHA256=8681f175d4bdb26c52222665793eef08490d7758529330f98d3b29dd0735bccc
+    CONFIGURE_COMMAND  ${BOOTSTRAP_COMMAND}
+                       ${BOOTSTRAP_ARGS}
+                       --with-libraries=${BOOTSTRAP_LIBRARIES}
+                       --with-toolset=${BOOST_TOOLSET}
+    BUILD_COMMAND      ${B2_COMMAND}
+                       link=static
+                       ${COMPILE_BOOST_BUILD_ARGS}
+                       --prefix=${BOOST_INSTALL_DIR}
+                       ${USER_CONFIG_FLAG} install
+    BUILD_IN_SOURCE    ON
+    INSTALL_COMMAND    ""
+    UPDATE_COMMAND     ""
+    BUILD_BYPRODUCTS   "${BOOST_INSTALL_DIR}/boost/config.hpp"
+                       "${BOOST_INSTALL_DIR}/lib/libboost_context.a"
+                       "${BOOST_INSTALL_DIR}/lib/libboost_filesystem.a"
+                       "${BOOST_INSTALL_DIR}/lib/libboost_iostreams.a")
 
   add_library(${COMPILE_BOOST_TARGET}_context STATIC IMPORTED)
   add_dependencies(${COMPILE_BOOST_TARGET}_context ${COMPILE_BOOST_TARGET}Project)
@@ -92,10 +103,10 @@ if(USE_SANITIZER)
   endif()
   message(STATUS "A sanitizer is enabled, need to build boost from source")
   if (USE_VALGRIND)
-    compile_boost(TARGET boost_asan BUILD_ARGS valgrind=on
+    compile_boost(TARGET boost_target BUILD_ARGS valgrind=on
       CXXFLAGS ${SANITIZER_COMPILE_OPTIONS} LDFLAGS ${SANITIZER_LINK_OPTIONS})
   else()
-    compile_boost(TARGET boost_asan BUILD_ARGS context-impl=ucontext
+    compile_boost(TARGET boost_target BUILD_ARGS context-impl=ucontext
       CXXFLAGS ${SANITIZER_COMPILE_OPTIONS} LDFLAGS ${SANITIZER_LINK_OPTIONS})
   endif()
   return()
@@ -130,7 +141,7 @@ if(WIN32)
   return()
 endif()
 
-find_package(Boost 1.78.0 EXACT QUIET COMPONENTS context filesystem CONFIG PATHS ${BOOST_HINT_PATHS})
+find_package(Boost 1.78.0 EXACT QUIET COMPONENTS context filesystem iostreams CONFIG PATHS ${BOOST_HINT_PATHS})
 set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores any installed boost")
 
 if(Boost_FOUND AND Boost_filesystem_FOUND AND Boost_context_FOUND AND Boost_iostreams_FOUND AND NOT FORCE_BOOST_BUILD)
diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index 4634e57e7c..3fdea389ab 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -4,30 +4,42 @@ find_package(RocksDB 6.27.3)
 
 include(ExternalProject)
 
-if (RocksDB_FOUND)
+set(RocksDB_CMAKE_ARGS
+  -DUSE_RTTI=1
+  -DPORTABLE=${PORTABLE_ROCKSDB}
+  -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+  -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+  -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
+  -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+  -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+  -DCMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS}
+  -DCMAKE_STATIC_LINKER_FLAGS=${CMAKE_STATIC_LINKER_FLAGS}
+  -DCMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS}
+  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+  -DFAIL_ON_WARNINGS=OFF
+  -DWITH_GFLAGS=OFF
+  -DWITH_TESTS=OFF
+  -DWITH_TOOLS=OFF
+  -DWITH_CORE_TOOLS=OFF
+  -DWITH_BENCHMARK_TOOLS=OFF
+  -DWITH_BZ2=OFF
+  -DWITH_LZ4=ON
+  -DWITH_SNAPPY=OFF
+  -DWITH_ZLIB=OFF
+  -DWITH_ZSTD=OFF
+  -DWITH_LIBURING=${WITH_LIBURING}
+  -DWITH_TSAN=${USE_TSAN}
+  -DWITH_ASAN=${USE_ASAN}
+  -DWITH_UBSAN=${USE_UBSAN}
+  -DROCKSDB_BUILD_SHARED=OFF
+  -DCMAKE_POSITION_INDEPENDENT_CODE=True
+)
+
+if(ROCKSDB_FOUND)
   ExternalProject_Add(rocksdb
     SOURCE_DIR "${RocksDB_ROOT}"
     DOWNLOAD_COMMAND ""
-    CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB}
-               -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-               -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-               -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-               -DWITH_GFLAGS=OFF
-               -DWITH_TESTS=OFF
-               -DWITH_TOOLS=OFF
-               -DWITH_CORE_TOOLS=OFF
-               -DWITH_BENCHMARK_TOOLS=OFF
-               -DWITH_BZ2=OFF
-               -DWITH_LZ4=ON
-               -DWITH_SNAPPY=OFF
-               -DWITH_ZLIB=OFF
-               -DWITH_ZSTD=OFF
-               -DWITH_LIBURING=${WITH_LIBURING}
-               -DWITH_TSAN=${USE_TSAN}
-               -DWITH_ASAN=${USE_ASAN}
-               -DWITH_UBSAN=${USE_UBSAN}
-               -DROCKSDB_BUILD_SHARED=OFF
-               -DCMAKE_POSITION_INDEPENDENT_CODE=True
+    CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
     BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
     INSTALL_COMMAND ""
   )
@@ -37,28 +49,9 @@ if (RocksDB_FOUND)
       ${BINARY_DIR}/librocksdb.a)
 else()
   ExternalProject_Add(rocksdb
-    URL        https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz
-    URL_HASH   SHA256=ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58
-    CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB}
-               -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-               -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-               -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-               -DWITH_GFLAGS=OFF
-               -DWITH_TESTS=OFF
-               -DWITH_TOOLS=OFF
-               -DWITH_CORE_TOOLS=OFF
-               -DWITH_BENCHMARK_TOOLS=OFF
-               -DWITH_BZ2=OFF
-               -DWITH_LZ4=ON
-               -DWITH_SNAPPY=OFF
-               -DWITH_ZLIB=OFF
-               -DWITH_ZSTD=OFF
-               -DWITH_LIBURING=${WITH_LIBURING}
-               -DWITH_TSAN=${USE_TSAN}
-               -DWITH_ASAN=${USE_ASAN}
-               -DWITH_UBSAN=${USE_UBSAN}
-               -DROCKSDB_BUILD_SHARED=OFF
-               -DCMAKE_POSITION_INDEPENDENT_CODE=True
+    URL https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz
+    URL_HASH SHA256=ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58
+    CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
     BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
     INSTALL_COMMAND ""
   )
@@ -68,7 +61,7 @@ else()
       ${BINARY_DIR}/librocksdb.a)
 
   ExternalProject_Get_Property(rocksdb SOURCE_DIR)
-  set (ROCKSDB_INCLUDE_DIR "${SOURCE_DIR}/include")
+  set(ROCKSDB_INCLUDE_DIR "${SOURCE_DIR}/include")
 
   set(ROCKSDB_FOUND TRUE)
 endif()
diff --git a/cmake/CompileZstd.cmake b/cmake/CompileZstd.cmake
new file mode 100644
index 0000000000..968d09398f
--- /dev/null
+++ b/cmake/CompileZstd.cmake
@@ -0,0 +1,23 @@
+# Compile zstd
+
+function(compile_zstd)
+
+  include(FetchContent)
+
+  set(ZSTD_SOURCE_DIR ${CMAKE_BINARY_DIR}/zstd)
+
+  FetchContent_Declare(
+    ZSTD
+    GIT_REPOSITORY https://github.com/facebook/zstd.git
+    GIT_TAG        v1.5.2
+    SOURCE_DIR     ${ZSTD_SOURCE_DIR}
+    BINARY_DIR     ${ZSTD_SOURCE_DIR}
+    SOURCE_SUBDIR  "build/cmake"
+  )
+
+  FetchContent_MakeAvailable(ZSTD)
+
+  add_library(ZSTD::ZSTD STATIC IMPORTED)
+  set_target_properties(ZSTD::ZSTD PROPERTIES IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/lib/libzstd.a")
+  target_include_directories(ZSTD::ZSTD PUBLIC ${ZSTD_INCLUDE_DIRS})
+endfunction(compile_zstd)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 2deaa4d0bc..e38f333b58 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -25,6 +25,7 @@ env_set(STATIC_LINK_LIBCXX "${_static_link_libcxx}" BOOL "Statically link libstd
 env_set(TRACE_PC_GUARD_INSTRUMENTATION_LIB "" STRING "Path to a library containing an implementation for __sanitizer_cov_trace_pc_guard. See https://clang.llvm.org/docs/SanitizerCoverage.html for more info.")
 env_set(PROFILE_INSTR_GENERATE OFF BOOL "If set, build FDB as an instrumentation build to generate profiles")
 env_set(PROFILE_INSTR_USE "" STRING "If set, build FDB with profile")
+env_set(FULL_DEBUG_SYMBOLS OFF BOOL "Generate full debug symbols")
 
 set(USE_SANITIZER OFF)
 if(USE_ASAN OR USE_VALGRIND OR USE_MSAN OR USE_TSAN OR USE_UBSAN)
@@ -68,6 +69,8 @@ if(WIN32)
   add_definitions(-DBOOST_USE_WINDOWS_H)
   add_definitions(-DWIN32_LEAN_AND_MEAN)
   add_definitions(-D_ITERATOR_DEBUG_LEVEL=0)
+  add_definitions(-DNOGDI) # WinGDI.h defines macro ERROR
+  add_definitions(-D_USE_MATH_DEFINES) # Math constants
 endif()
 
 if (USE_CCACHE)
@@ -162,9 +165,20 @@ else()
   set(SANITIZER_COMPILE_OPTIONS)
   set(SANITIZER_LINK_OPTIONS)
 
-  # we always compile with debug symbols. CPack will strip them out
+  # we always compile with debug symbols. For release builds CPack will strip them out
   # and create a debuginfo rpm
-  add_compile_options(-ggdb -fno-omit-frame-pointer)
+  add_compile_options(-fno-omit-frame-pointer -gz)
+  add_link_options(-gz)
+  if(FDB_RELEASE OR FULL_DEBUG_SYMBOLS OR CMAKE_BUILD_TYPE STREQUAL "Debug")
+    # Configure with FULL_DEBUG_SYMBOLS=ON to generate all symbols for debugging with gdb
+    # Also generating full debug symbols in release builds, because they are packaged
+    # separately and installed optionally
+    add_compile_options(-ggdb)
+  else()
+    # Generating minimal debug symbols by default. They are sufficient for testing purposes
+    add_compile_options(-ggdb1)
+  endif()
+
   if(TRACE_PC_GUARD_INSTRUMENTATION_LIB)
       add_compile_options(-fsanitize-coverage=trace-pc-guard)
       link_libraries(${TRACE_PC_GUARD_INSTRUMENTATION_LIB})
@@ -190,6 +204,7 @@ else()
   endif()
 
   if(USE_GCOV)
+    add_compile_options(--coverage)
     add_link_options(--coverage)
   endif()
 
@@ -198,6 +213,8 @@ else()
       -fsanitize=undefined
       # TODO(atn34) Re-enable -fsanitize=alignment once https://github.com/apple/foundationdb/issues/1434 is resolved
       -fno-sanitize=alignment
+      # https://github.com/apple/foundationdb/issues/7955
+      -fno-sanitize=function
       -DBOOST_USE_UCONTEXT)
     list(APPEND SANITIZER_LINK_OPTIONS -fsanitize=undefined)
   endif()
@@ -275,16 +292,35 @@ else()
   #add_compile_options(-fno-builtin-memcpy)
 
   if (CLANG OR ICX)
-    add_compile_options()
     if (APPLE OR USE_LIBCXX)
-      add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
       if (NOT APPLE)
         if (STATIC_LINK_LIBCXX)
-          add_link_options(-static-libgcc -nostdlib++  -Wl,-Bstatic -lc++ -lc++abi -Wl,-Bdynamic)
+          set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -nostdlib++ -Wl,-Bstatic -lc++ -lc++abi -Wl,-Bdynamic")
+          set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libgcc -nostdlib++ -Wl,-Bstatic -lc++ -lc++abi -Wl,-Bdynamic")
+        else()
+          # Make sure that libc++ can be found be the platform's loader, so that thing's like cmake's "try_run" work.
+          find_library(LIBCXX_SO_PATH c++ /usr/local/lib)
+          if (LIBCXX_SO_PATH)
+            get_filename_component(LIBCXX_SO_DIR ${LIBCXX_SO_PATH} DIRECTORY)
+            if (APPLE)
+              set(ENV{DYLD_LIBRARY_PATH} "$ENV{DYLD_LIBRARY_PATH}:${LIBCXX_SO_DIR}")
+            elseif(WIN32)
+              set(ENV{PATH} "$ENV{PATH};${LIBCXX_SO_DIR}")
+            else()
+              set(ENV{LD_LIBRARY_PATH} "$ENV{LD_LIBRARY_PATH}:${LIBCXX_SO_DIR}")
+            endif()
+          endif()
         endif()
-        add_link_options(-stdlib=libc++ -Wl,-build-id=sha1)
+        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -Wl,-build-id=sha1")
+        set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -stdlib=libc++ -Wl,-build-id=sha1")
       endif()
     endif()
+    if (NOT APPLE AND NOT USE_LIBCXX)
+      message(STATUS "Linking libatomic")
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -latomic")
+      set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -latomic")
+    endif()
     if (OPEN_FOR_IDE)
       add_compile_options(
         -Wno-unknown-attributes)
@@ -302,11 +338,19 @@ else()
       -Wno-unknown-warning-option
       -Wno-unused-parameter
       -Wno-constant-logical-operand
+      # These need to be disabled for FDB's RocksDB storage server implementation
+      -Wno-deprecated-copy
+      -Wno-delete-non-abstract-non-virtual-dtor
+      -Wno-range-loop-construct
+      -Wno-reorder-ctor
+      # Needed for clang 13 (todo: Update above logic so that it figures out when to pass in -static-libstdc++ and when it will be ignored)
+      # When you remove this, you might need to move it back to the USE_CCACHE stanza.  It was (only) there before I moved it here.
+      -Wno-unused-command-line-argument
       )
     if (USE_CCACHE)
       add_compile_options(
         -Wno-register
-        -Wno-unused-command-line-argument)
+      )
     endif()
     if (PROFILE_INSTR_GENERATE)
       add_compile_options(-fprofile-instr-generate)
diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake
index 13e3f790a8..292d58c454 100644
--- a/cmake/FDBComponents.cmake
+++ b/cmake/FDBComponents.cmake
@@ -56,8 +56,8 @@ endif()
 # Python Bindings
 ################################################################################
 
-find_package(Python COMPONENTS Interpreter)
-if(Python_Interpreter_FOUND)
+find_package(Python3 COMPONENTS Interpreter)
+if(Python3_Interpreter_FOUND)
   set(WITH_PYTHON ON)
 else()
   message(WARNING "Could not found a suitable python interpreter")
@@ -178,7 +178,7 @@ set(PORTABLE_ROCKSDB ON CACHE BOOL "Compile RocksDB in portable mode") # Set thi
 set(WITH_LIBURING OFF CACHE BOOL "Build with liburing enabled") # Set this to ON to include liburing
 # RocksDB is currently enabled by default for GCC but does not build with the latest
 # Clang.
-if (SSD_ROCKSDB_EXPERIMENTAL AND GCC)
+if (SSD_ROCKSDB_EXPERIMENTAL AND NOT WIN32)
   set(WITH_ROCKSDB_EXPERIMENTAL ON)
 else()
   set(WITH_ROCKSDB_EXPERIMENTAL OFF)
@@ -200,6 +200,9 @@ else()
     URL "https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz"
     URL_HASH SHA256=bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d
     CMAKE_CACHE_ARGS
+      -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+      -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER}
+      -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER}
       -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/toml11
       -Dtoml11_BUILD_TEST:BOOL=OFF
     BUILD_ALWAYS ON)
@@ -229,7 +232,12 @@ set(COROUTINE_IMPL ${DEFAULT_COROUTINE_IMPL} CACHE STRING "Which coroutine imple
 
 set(BUILD_AWS_BACKUP OFF CACHE BOOL "Build AWS S3 SDK backup client")
 if (BUILD_AWS_BACKUP)
-  set(WITH_AWS_BACKUP ON)
+  if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+    set(WITH_AWS_BACKUP ON)
+  else()
+    message(WARNING "BUILD_AWS_BACKUP set but ignored ${CMAKE_SYSTEM_PROCESSOR} is not supported yet")
+    set(WITH_AWS_BACKUP OFF)
+  endif()
 else()
   set(WITH_AWS_BACKUP OFF)
 endif()
diff --git a/cmake/FlowCommands.cmake b/cmake/FlowCommands.cmake
index 4d7fa4f2c2..81ea36892d 100644
--- a/cmake/FlowCommands.cmake
+++ b/cmake/FlowCommands.cmake
@@ -9,6 +9,14 @@ define_property(TARGET PROPERTY COVERAGE_FILTERS
 expression in this list will be ignored when the coverage.target.xml file is \
 generated. This property is set through the add_flow_target function.")
 
+if(WIN32)
+  set(compilation_unit_macro_default OFF)
+else()
+  set(compilation_unit_macro_default ON)
+endif()
+
+set(PASS_COMPILATION_UNIT "${compilation_unit_macro_default}" CACHE BOOL
+  "Pass path to compilation unit as macro to each compilation unit (useful for code probes)")
 
 function(generate_coverage_xml)
   if(NOT (${ARGC} EQUAL "1"))
@@ -259,6 +267,11 @@ function(add_flow_target)
         endif()
       endif()
     endforeach()
+    if(PASS_COMPILATION_UNIT)
+      foreach(s IN LISTS sources)
+        set_source_files_properties("${s}" PROPERTIES COMPILE_DEFINITIONS "COMPILATION_UNIT=${s}")
+      endforeach()
+    endif()
     if(AFT_EXECUTABLE)
       set(strip_target ON)
       set(target_type exec)
diff --git a/cmake/Jemalloc.cmake b/cmake/Jemalloc.cmake
index bfdd2f5898..8d04ebccca 100644
--- a/cmake/Jemalloc.cmake
+++ b/cmake/Jemalloc.cmake
@@ -14,7 +14,7 @@ ExternalProject_add(Jemalloc_project
   BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
   "${JEMALLOC_DIR}/lib/libjemalloc.a"
   "${JEMALLOC_DIR}/lib/libjemalloc_pic.a"
-  CONFIGURE_COMMAND ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof
+  CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof
   BUILD_IN_SOURCE ON
   BUILD_COMMAND make
   INSTALL_DIR "${JEMALLOC_DIR}"
@@ -24,4 +24,4 @@ add_dependencies(im_jemalloc_pic Jemalloc_project)
 set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc_pic.a")
 set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.a")
 target_include_directories(jemalloc INTERFACE "${JEMALLOC_DIR}/include")
-target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc)
\ No newline at end of file
+target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc)
diff --git a/cmake/awssdk.cmake b/cmake/awssdk.cmake
index 88cb7c78e9..ab62f9b6d6 100644
--- a/cmake/awssdk.cmake
+++ b/cmake/awssdk.cmake
@@ -2,16 +2,14 @@ project(awssdk-download NONE)
 
 # Compile the sdk with clang and libc++, since otherwise we get libc++ vs libstdc++ link errors when compiling fdb with clang
 set(AWSSDK_COMPILER_FLAGS "")
-set(AWSSDK_LINK_FLAGS "")
-if(APPLE OR CLANG OR USE_LIBCXX)
-  set(AWSSDK_COMPILER_FLAGS -stdlib=libc++ -nostdlib++)
-  set(AWSSDK_LINK_FLAGS -stdlib=libc++ -lc++abi)
+if(APPLE OR USE_LIBCXX)
+  set(AWSSDK_COMPILER_FLAGS "-stdlib=libc++ -nostdlib++")
 endif()
 
 include(ExternalProject)
 ExternalProject_Add(awssdk_project
   GIT_REPOSITORY    https://github.com/aws/aws-sdk-cpp.git
-  GIT_TAG           2af3ce543c322cb259471b3b090829464f825972 # v1.9.200
+  GIT_TAG           e4b4b310d8631bc7e9a797b6ac03a73c6f210bf6 # v1.9.331
   SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/awssdk-src"
   BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build"
   GIT_CONFIG        advice.detachedHead=false
@@ -21,11 +19,11 @@ ExternalProject_Add(awssdk_project
                     -DSIMPLE_INSTALL=ON
                     -DCMAKE_INSTALL_PREFIX=install # need to specify an install prefix so it doesn't install in /usr/lib - FIXME: use absolute path
                     -DBYO_CRYPTO=ON                # we have our own crypto libraries that conflict if we let aws sdk build and link its own
-
+                    -DBUILD_CURL=ON
+                    -DBUILD_ZLIB=ON
                     
                     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-                    -DCMAKE_EXE_LINKER_FLAGS=${AWSSDK_COMPILER_FLAGS}
-                    -DCMAKE_CXX_FLAGS=${AWSSDK_LINK_FLAGS}
+                    -DCMAKE_CXX_FLAGS=${AWSSDK_COMPILER_FLAGS}
   TEST_COMMAND      ""
   # the sdk build produces a ton of artifacts, with their own dependency tree, so there is a very specific dependency order they must be linked in
   BUILD_BYPRODUCTS  "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a"
@@ -35,11 +33,14 @@ ExternalProject_Add(awssdk_project
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-event-stream.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-http.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-mqtt.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-sdkutils.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-cal.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/curl/lib/libcurl.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/zlib/lib/libz.a"
 )
 
 add_library(awssdk_core STATIC IMPORTED)
@@ -75,6 +76,10 @@ add_library(awssdk_c_io STATIC IMPORTED)
 add_dependencies(awssdk_c_io awssdk_project)
 set_target_properties(awssdk_c_io PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a")
 
+add_library(awssdk_c_sdkutils STATIC IMPORTED)
+add_dependencies(awssdk_c_sdkutils awssdk_project)
+set_target_properties(awssdk_c_sdkutils PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-sdkutils.a")
+
 add_library(awssdk_checksums STATIC IMPORTED)
 add_dependencies(awssdk_checksums awssdk_project)
 set_target_properties(awssdk_checksums PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a")
@@ -91,7 +96,15 @@ add_library(awssdk_c_common STATIC IMPORTED)
 add_dependencies(awssdk_c_common awssdk_project)
 set_target_properties(awssdk_c_common PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a")
 
+add_library(curl STATIC IMPORTED)
+add_dependencies(curl awssdk_project)
+set_property(TARGET curl PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/curl/lib/libcurl.a")
+
+add_library(zlib STATIC IMPORTED)
+add_dependencies(zlib awssdk_project)
+set_property(TARGET zlib PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/zlib/lib/libz.a")
+
 # link them all together in one interface target
 add_library(awssdk_target INTERFACE)
 target_include_directories(awssdk_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/include)
-target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl)
\ No newline at end of file
+target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_sdkutils awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl zlib)
diff --git a/cmake/user-config.jam.cmake b/cmake/user-config.jam.cmake
index 6d2883cc95..696bcdd831 100644
--- a/cmake/user-config.jam.cmake
+++ b/cmake/user-config.jam.cmake
@@ -1 +1,2 @@
 using @BOOST_TOOLSET@ : : @BOOST_CXX_COMPILER@ : @BOOST_ADDITIONAL_COMPILE_OPTIONS@ ;
+using zstd : 1.5.2 : <include>/@CMAKE_BINARY_DIR@/zstd/lib <search>/@CMAKE_BINARY_DIR@/lib ;
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index cce49cf76c..75ca06243f 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -8,6 +8,8 @@ add_subdirectory(rapidxml)
 add_subdirectory(sqlite)
 add_subdirectory(SimpleOpt)
 add_subdirectory(fmt-8.1.1)
+add_subdirectory(md5)
+add_subdirectory(libb64)
 if(NOT WIN32)
   add_subdirectory(linenoise)
   add_subdirectory(debug_determinism)
diff --git a/contrib/Implib.so/arch/common/init.c.tpl b/contrib/Implib.so/arch/common/init.c.tpl
index a71f8df44f..d8b9b3bbde 100644
--- a/contrib/Implib.so/arch/common/init.c.tpl
+++ b/contrib/Implib.so/arch/common/init.c.tpl
@@ -26,8 +26,7 @@ extern "C" {
 #define CHECK(cond, fmt, ...) do { \
     if(!(cond)) { \
       fprintf(stderr, "implib-gen: $load_name: " fmt "\n", ##__VA_ARGS__); \
-      assert(0 && "Assertion in generated code"); \
-      exit(1); \
+      abort(); \
     } \
   } while(0)
 
diff --git a/contrib/Joshua/scripts/bindingTestScript.sh b/contrib/Joshua/scripts/bindingTestScript.sh
index f4e0e8eb8b..2d6badbe9d 100755
--- a/contrib/Joshua/scripts/bindingTestScript.sh
+++ b/contrib/Joshua/scripts/bindingTestScript.sh
@@ -83,6 +83,7 @@ fi
 # Stop the cluster
 if stopCluster; then
 	unset FDBSERVERID
+	trap - EXIT
 fi
 
 exit "${status}"
diff --git a/contrib/Joshua/scripts/correctnessTest.sh b/contrib/Joshua/scripts/correctnessTest.sh
index a617d81088..bee09acf25 100755
--- a/contrib/Joshua/scripts/correctnessTest.sh
+++ b/contrib/Joshua/scripts/correctnessTest.sh
@@ -4,4 +4,6 @@
 export ASAN_OPTIONS="detect_leaks=0"
 
 OLDBINDIR="${OLDBINDIR:-/app/deploy/global_data/oldBinaries}"
-mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" false
+#mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" false
+
+python3 -m test_harness.app -s ${JOSHUA_SEED} --old-binaries-path ${OLDBINDIR}
diff --git a/contrib/Joshua/scripts/correctnessTimeout.sh b/contrib/Joshua/scripts/correctnessTimeout.sh
index 7917aae591..6bd0bfeee0 100755
--- a/contrib/Joshua/scripts/correctnessTimeout.sh
+++ b/contrib/Joshua/scripts/correctnessTimeout.sh
@@ -1,4 +1,4 @@
 #!/bin/bash -u
-for file in `find . -name 'trace*.xml'` ; do
-    mono ./bin/TestHarness.exe summarize "${file}" summary.xml "" JoshuaTimeout true
-done
+
+
+python3 -m test_harness.timeout
diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh
index abbf93abc5..500e106339 100644
--- a/contrib/Joshua/scripts/localClusterStart.sh
+++ b/contrib/Joshua/scripts/localClusterStart.sh
@@ -210,7 +210,7 @@ function stopCluster
   then
     # Ensure that process is dead
     if ! kill -0 "${FDBSERVERID}" 2> /dev/null; then
-      log "Killed cluster (${FDBSERVERID}) via cli"
+      log "Killed cluster (${FDBSERVERID}) via cli" "${DEBUGLEVEL}"
     elif ! kill -9 "${FDBSERVERID}"; then
       log "Failed to kill FDB Server process (${FDBSERVERID}) via cli or kill command"
       let status="${status} + 1"
diff --git a/contrib/Joshua/scripts/valgrindTest.sh b/contrib/Joshua/scripts/valgrindTest.sh
index 5409429691..820750f3b2 100755
--- a/contrib/Joshua/scripts/valgrindTest.sh
+++ b/contrib/Joshua/scripts/valgrindTest.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 OLDBINDIR="${OLDBINDIR:-/app/deploy/global_data/oldBinaries}"
-mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" true
+python3 -m test_harness.app -s ${JOSHUA_SEED} --old-binaries-path ${OLDBINDIR} --use-valgrind
diff --git a/contrib/Joshua/scripts/valgrindTimeout.sh b/contrib/Joshua/scripts/valgrindTimeout.sh
index b9d9e7ebad..2224598e43 100755
--- a/contrib/Joshua/scripts/valgrindTimeout.sh
+++ b/contrib/Joshua/scripts/valgrindTimeout.sh
@@ -1,6 +1,2 @@
 #!/bin/bash -u
-for file in `find . -name 'trace*.xml'` ; do
-    for valgrindFile in `find . -name 'valgrind*.xml'` ; do
-        mono ./bin/TestHarness.exe summarize "${file}" summary.xml "${valgrindFile}" JoshuaTimeout true
-    done
-done
+python3 -m test_harness.timeout --use-valgrind
diff --git a/contrib/TestHarness/Program.cs b/contrib/TestHarness/Program.cs
index 0b2bbc1127..b3e003dee5 100644
--- a/contrib/TestHarness/Program.cs
+++ b/contrib/TestHarness/Program.cs
@@ -19,6 +19,7 @@
  */
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -302,6 +303,7 @@ namespace SummarizeTest
                         uniqueFileSet.Add(file.Substring(0, file.LastIndexOf("-"))); // all restarting tests end with -1.txt or -2.txt
                     }
                     uniqueFiles = uniqueFileSet.ToArray();
+                    Array.Sort(uniqueFiles);
                     testFile = random.Choice(uniqueFiles);
                     // The on-disk format changed in 4.0.0, and 5.x can't load files from 3.x.
                     string oldBinaryVersionLowerBound = "4.0.0";
@@ -334,8 +336,9 @@ namespace SummarizeTest
                         // thus, by definition, if "until_" appears, we do not want to run with the current binary version
                         oldBinaries = oldBinaries.Concat(currentBinary);
                     }
-                    List<string> oldBinariesList = oldBinaries.ToList<string>();
-                    if (oldBinariesList.Count == 0) {
+                    string[] oldBinariesList = oldBinaries.ToArray<string>();
+                    Array.Sort(oldBinariesList);
+                    if (oldBinariesList.Count() == 0) {
                         // In theory, restarting tests are named to have at least one old binary version to run
                         // But if none of the provided old binaries fall in the range, we just skip the test
                         Console.WriteLine("No available old binary version from {0} to {1}", oldBinaryVersionLowerBound, oldBinaryVersionUpperBound);
@@ -347,6 +350,7 @@ namespace SummarizeTest
                 else
                 {
                     uniqueFiles = Directory.GetFiles(testDir);
+                    Array.Sort(uniqueFiles);
                     testFile = random.Choice(uniqueFiles);
                 }
             }
@@ -487,6 +491,16 @@ namespace SummarizeTest
                         useValgrind ? "on" : "off");
                     }
 
+                    IDictionary data = Environment.GetEnvironmentVariables();
+                    foreach (DictionaryEntry i in data)
+                    {
+                        string k=(string)i.Key;
+                        string v=(string)i.Value;
+                        if (k.StartsWith("FDB_KNOB")) {
+                           process.StartInfo.EnvironmentVariables[k]=v;
+                        }
+                    }
+
                     process.Start();
 
                     // SOMEDAY: Do we want to actually do anything with standard output or error?
@@ -718,7 +732,7 @@ namespace SummarizeTest
                         process.Refresh();
                         if (process.HasExited)
                             return;
-                        long mem = process.PrivateMemorySize64;
+                        long mem = process.PagedMemorySize64;
                         MaxMem = Math.Max(MaxMem, mem);
                         //Console.WriteLine(string.Format("Process used {0} bytes", MaxMem));
                         Thread.Sleep(1000);
@@ -744,16 +758,28 @@ namespace SummarizeTest
             AppendToSummary(summaryFileName, xout);
         }
 
-        // Parses the valgrind XML file and returns a list of "what" tags for each error.
+        static string ParseValgrindStack(XElement stackElement) {
+            string backtrace = "";
+            foreach (XElement frame in stackElement.Elements()) {
+                backtrace += " " + frame.Element("ip").Value.ToLower();
+            }
+            if (backtrace.Length > 0) {
+                backtrace = "addr2line -e fdbserver.debug -p -C -f -i" + backtrace;
+            }
+
+            return backtrace;
+        }
+
+        // Parses the valgrind XML file and returns a list of error elements.
         //  All errors for which the "kind" tag starts with "Leak" are ignored
-        static string[] ParseValgrindOutput(string valgrindOutputFileName, bool traceToStdout)
+        static XElement[] ParseValgrindOutput(string valgrindOutputFileName, bool traceToStdout)
         {
             if (!traceToStdout)
             {
                 Console.WriteLine("Reading vXML file: " + valgrindOutputFileName);
             }
 
-            ISet<string> whats = new HashSet<string>();
+            IList<XElement> errors = new List<XElement>();
             XElement xdoc = XDocument.Load(valgrindOutputFileName).Element("valgrindoutput");
             foreach(var elem in xdoc.Elements()) {
                 if (elem.Name != "error")
@@ -761,9 +787,29 @@ namespace SummarizeTest
                 string kind = elem.Element("kind").Value;
                 if(kind.StartsWith("Leak"))
                     continue;
-                whats.Add(elem.Element("what").Value);
+
+                XElement errorElement = new XElement("ValgrindError",
+                                new XAttribute("Severity", (int)Magnesium.Severity.SevError));
+
+                int num = 1;
+                string suffix = "";
+                foreach (XElement sub in elem.Elements()) {
+                    if (sub.Name == "what") {
+                        errorElement.SetAttributeValue("What", sub.Value);
+                    } else if (sub.Name == "auxwhat") {
+                        suffix = "Aux" + num++;
+                        errorElement.SetAttributeValue("What" + suffix, sub.Value);
+                    } else if (sub.Name == "stack") {
+                        errorElement.SetAttributeValue("Backtrace" + suffix, ParseValgrindStack(sub));
+                    } else if (sub.Name == "origin") {
+                        errorElement.SetAttributeValue("WhatOrigin", sub.Element("what").Value);
+                        errorElement.SetAttributeValue("BacktraceOrigin", ParseValgrindStack(sub.Element("stack")));
+                    }
+                }
+
+                errors.Add(errorElement);
             }
-            return whats.ToArray();
+            return errors.ToArray();
         }
 
         delegate IEnumerable<Magnesium.Event> parseDelegate(System.IO.Stream stream, string file,
@@ -788,8 +834,15 @@ namespace SummarizeTest
             string firstRetryableError = "";
             int stderrSeverity = (int)Magnesium.Severity.SevError;
 
+            xout.Add(new XAttribute("DeterminismCheck", expectedUnseed != -1 ? "1" : "0"));
+            xout.Add(new XAttribute("OldBinary", Path.GetFileName(oldBinaryName)));
+
+            if (traceFiles.Length == 0) {
+                xout.Add(new XElement("NoTraceFilesFound"));
+            }
+
             Dictionary<KeyValuePair<string, Magnesium.Severity>, Magnesium.Severity> severityMap = new Dictionary<KeyValuePair<string, Magnesium.Severity>, Magnesium.Severity>();
-            Dictionary<Tuple<string, string>, bool> codeCoverage = new Dictionary<Tuple<string, string>, bool>();
+            var codeCoverage = new Dictionary<Tuple<string, string, string>, bool>();
 
             foreach (var traceFileName in traceFiles)
             {
@@ -824,9 +877,7 @@ namespace SummarizeTest
                                     new XAttribute("RandomSeed", ev.Details.RandomSeed),
                                     new XAttribute("SourceVersion", ev.Details.SourceVersion),
                                     new XAttribute("Time", ev.Details.ActualTime),
-                                    new XAttribute("BuggifyEnabled", ev.Details.BuggifyEnabled),
-                                    new XAttribute("DeterminismCheck", expectedUnseed != -1 ? "1" : "0"),
-                                    new XAttribute("OldBinary", Path.GetFileName(oldBinaryName)));
+                                    new XAttribute("BuggifyEnabled", ev.Details.BuggifyEnabled));
                                 testBeginFound = true;
                                 if (ev.DDetails.ContainsKey("FaultInjectionEnabled"))
                                     xout.Add(new XAttribute("FaultInjectionEnabled", ev.Details.FaultInjectionEnabled));
@@ -902,12 +953,17 @@ namespace SummarizeTest
                             if (ev.Type == "CodeCoverage" && !willRestart)
                             {
                                 bool covered = true;
-                                if(ev.DDetails.ContainsKey("Covered"))
+                                if (ev.DDetails.ContainsKey("Covered"))
                                 {
                                     covered = int.Parse(ev.Details.Covered) != 0;
                                 }
 
-                                var key = new Tuple<string, string>(ev.Details.File, ev.Details.Line);
+                                var comment = "";
+                                if (ev.DDetails.ContainsKey("Comment"))
+                                {
+                                    comment = ev.Details.Comment;
+                                }
+                                var key = new Tuple<string, string, string>(ev.Details.File, ev.Details.Line, comment);
                                 if (covered || !codeCoverage.ContainsKey(key))
                                 {
                                     codeCoverage[key] = covered;
@@ -917,6 +973,10 @@ namespace SummarizeTest
                             {
                                 xout.Add(new XElement(ev.Type, new XAttribute("File", ev.Details.File), new XAttribute("Line", ev.Details.Line)));
                             }
+                            if (ev.Type == "RunningUnitTest") 
+                            {
+                                xout.Add(new XElement(ev.Type, new XAttribute("Name", ev.Details.Name), new XAttribute("File", ev.Details.File), new XAttribute("Line", ev.Details.Line)));
+                            }
                             if (ev.Type == "TestsExpectedToPass")
                                 testCount = int.Parse(ev.Details.Count);
                             if (ev.Type == "TestResults" && ev.Details.Passed == "1")
@@ -954,6 +1014,12 @@ namespace SummarizeTest
                 xout.Add(new XElement(externalError, new XAttribute("Severity", (int)Magnesium.Severity.SevError)));
             }
 
+            string joshuaSeed = System.Environment.GetEnvironmentVariable("JOSHUA_SEED");
+
+            if (joshuaSeed != null) {
+                xout.Add(new XAttribute("JoshuaSeed", joshuaSeed));
+            }
+
             foreach(var kv in codeCoverage)
             {
                 var element = new XElement("CodeCoverage", new XAttribute("File", kv.Key.Item1), new XAttribute("Line", kv.Key.Item2));
@@ -961,6 +1027,9 @@ namespace SummarizeTest
                 {
                     element.Add(new XAttribute("Covered", "0"));
                 }
+                if (kv.Key.Item3.Length > 0) {
+                    element.Add(new XAttribute("Comment", kv.Key.Item3));
+                }
 
                 xout.Add(element);
             }
@@ -1046,12 +1115,10 @@ namespace SummarizeTest
                 try
                 {
                     // If there are any errors reported "ok" will be set to false
-                    var whats = ParseValgrindOutput(valgrindOutputFileName, traceToStdout);
-                    foreach (var what in whats)
+                    var valgrindErrors = ParseValgrindOutput(valgrindOutputFileName, traceToStdout);
+                    foreach (var vError in valgrindErrors)
                     {
-                        xout.Add(new XElement("ValgrindError",
-                                new XAttribute("Severity", (int)Magnesium.Severity.SevError),
-                                new XAttribute("What", what)));
+                        xout.Add(vError);
                         ok = false;
                         error = true;
                     }
diff --git a/contrib/TestHarness2/.gitignore b/contrib/TestHarness2/.gitignore
new file mode 100644
index 0000000000..80682f9552
--- /dev/null
+++ b/contrib/TestHarness2/.gitignore
@@ -0,0 +1,2 @@
+/tmp/
+/venv
diff --git a/contrib/TestHarness2/test_harness/__init__.py b/contrib/TestHarness2/test_harness/__init__.py
new file mode 100644
index 0000000000..3cb95520ec
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/__init__.py
@@ -0,0 +1,2 @@
+# Currently this file is left intentionally empty. It's main job for now is to indicate that this directory
+# should be used as a module.
diff --git a/contrib/TestHarness2/test_harness/app.py b/contrib/TestHarness2/test_harness/app.py
new file mode 100644
index 0000000000..3e300c6bf4
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/app.py
@@ -0,0 +1,25 @@
+import argparse
+import sys
+import traceback
+
+from test_harness.config import config
+from test_harness.run import TestRunner
+from test_harness.summarize import SummaryTree
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser('TestHarness', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        config.build_arguments(parser)
+        args = parser.parse_args()
+        config.extract_args(args)
+        test_runner = TestRunner()
+        if not test_runner.run():
+            exit(1)
+    except Exception as e:
+        _, _, exc_traceback = sys.exc_info()
+        error = SummaryTree('TestHarnessError')
+        error.attributes['Severity'] = '40'
+        error.attributes['ErrorMessage'] = str(e)
+        error.attributes['Trace'] = repr(traceback.format_tb(exc_traceback))
+        error.dump(sys.stdout)
+        exit(1)
diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py
new file mode 100644
index 0000000000..191fab629d
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/config.py
@@ -0,0 +1,266 @@
+from __future__ import annotations
+
+import argparse
+import collections
+import copy
+import os
+import random
+from enum import Enum
+from pathlib import Path
+from typing import List, Any, OrderedDict, Dict
+
+
+class BuggifyOptionValue(Enum):
+    ON = 1
+    OFF = 2
+    RANDOM = 3
+
+
+class BuggifyOption:
+    def __init__(self, val: str | None = None):
+        self.value = BuggifyOptionValue.RANDOM
+        if val is not None:
+            v = val.lower()
+            if v in ['on', '1', 'true']:
+                self.value = BuggifyOptionValue.ON
+            elif v in ['off', '0', 'false']:
+                self.value = BuggifyOptionValue.OFF
+            elif v in ['random', 'rnd', 'r']:
+                pass
+            else:
+                assert False, 'Invalid value {} -- use true, false, or random'.format(v)
+
+
+class ConfigValue:
+    def __init__(self, name: str, **kwargs):
+        self.name = name
+        self.value = None
+        self.kwargs = kwargs
+        if 'default' in self.kwargs:
+            self.value = self.kwargs['default']
+
+    def get_arg_name(self) -> str:
+        if 'long_name' in self.kwargs:
+            return self.kwargs['long_name']
+        else:
+            return self.name
+
+    def add_to_args(self, parser: argparse.ArgumentParser):
+        kwargs = copy.copy(self.kwargs)
+        long_name = self.name
+        short_name = None
+        if 'long_name' in kwargs:
+            long_name = kwargs['long_name']
+            del kwargs['long_name']
+        if 'short_name' in kwargs:
+            short_name = kwargs['short_name']
+            del kwargs['short_name']
+        if 'action' in kwargs and kwargs['action'] in ['store_true', 'store_false']:
+            del kwargs['type']
+        long_name = long_name.replace('_', '-')
+        if short_name is None:
+            # line below is useful for debugging
+            # print('add_argument(\'--{}\', [{{{}}}])'.format(long_name, ', '.join(['\'{}\': \'{}\''.format(k, v)
+            #                                                                       for k, v in kwargs.items()])))
+            parser.add_argument('--{}'.format(long_name), **kwargs)
+        else:
+            # line below is useful for debugging
+            # print('add_argument(\'-{}\', \'--{}\', [{{{}}}])'.format(short_name, long_name,
+            #                                                          ', '.join(['\'{}\': \'{}\''.format(k, v)
+            #                                                                     for k, v in kwargs.items()])))
+            parser.add_argument('-{}'.format(short_name), '--{}'.format(long_name), **kwargs)
+
+    def get_value(self, args: argparse.Namespace) -> tuple[str, Any]:
+        return self.name, args.__getattribute__(self.get_arg_name())
+
+
+class Config:
+    """
+    This is the central configuration class for test harness. The values in this class are exposed globally through
+    a global variable test_harness.config.config. This class provides some "magic" to keep test harness flexible.
+    Each parameter can further be configured using an `_args` member variable which is expected to be a dictionary.
+    * The value of any variable can be set through the command line. For a variable named `variable_name` we will
+      by default create a new command line option `--variable-name` (`_` is automatically changed to `-`). This
+      default can be changed by setting the `'long_name'` property in the `_arg` dict.
+    * In addition the user can also optionally set a short-name. This can be achieved by setting the `'short_name'`
+      property in the `_arg` dictionary.
+    * All additional properties in `_args` are passed to `argparse.add_argument`.
+    * If the default of a variable is `None` the user should explicitly set the `'type'` property to an appropriate
+      type.
+    * In addition to command line flags, all configuration options can also be controlled through environment variables.
+      By default, `variable-name` can be changed by setting the environment variable `TH_VARIABLE_NAME`. This default
+      can be changed by setting the `'env_name'` property.
+    * Test harness comes with multiple executables. Each of these should use the config facility. For this,
+      `Config.build_arguments` should be called first with the `argparse` parser. Then `Config.extract_args` needs
+      to be called with the result of `argparse.ArgumentParser.parse_args`. A sample example could look like this:
+      ```
+      parser = argparse.ArgumentParser('TestHarness', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+      config.build_arguments(parser)
+      args = parser.parse_args()
+      config.extract_args(args)
+      ```
+    * Changing the default value for all executables might not always be desirable. If it should be only changed for
+      one executable Config.change_default should be used.
+    """
+    def __init__(self):
+        self.random = random.Random()
+        self.cluster_file: str | None = None
+        self.cluster_file_args = {'short_name': 'C', 'type': str, 'help': 'Path to fdb cluster file', 'required': False,
+                                  'env_name': 'JOSHUA_CLUSTER_FILE'}
+        self.joshua_dir: str | None = None
+        self.joshua_dir_args = {'type': str, 'help': 'Where to write FDB data to', 'required': False,
+                                'env_name': 'JOSHUA_APP_DIR'}
+        self.stats: str | None = None
+        self.stats_args = {'type': str, 'help': 'A base64 encoded list of statistics (used to reproduce runs)',
+                           'required': False}
+        self.random_seed: int | None = None
+        self.random_seed_args = {'type': int,
+                                 'help': 'Force given seed given to fdbserver -- mostly useful for debugging',
+                                 'required': False}
+        self.kill_seconds: int = 30 * 60
+        self.kill_seconds_args = {'help': 'Timeout for individual test'}
+        self.buggify_on_ratio: float = 0.8
+        self.buggify_on_ratio_args = {'help': 'Probability that buggify is turned on'}
+        self.write_run_times = False
+        self.write_run_times_args = {'help': 'Write back probabilities after each test run',
+                                     'action': 'store_true'}
+        self.unseed_check_ratio: float = 0.05
+        self.unseed_check_ratio_args = {'help': 'Probability for doing determinism check'}
+        self.test_dirs: List[str] = ['slow', 'fast', 'restarting', 'rare', 'noSim']
+        self.test_dirs_args: dict = {'nargs': '*', 'help': 'test_directories to look for files in'}
+        self.trace_format: str = 'json'
+        self.trace_format_args = {'choices': ['json', 'xml'], 'help': 'What format fdb should produce'}
+        self.crash_on_error: bool = True
+        self.crash_on_error_args = {'long_name': 'no_crash', 'action': 'store_false',
+                                    'help': 'Don\'t crash on first error'}
+        self.max_warnings: int = 10
+        self.max_warnings_args = {'short_name': 'W'}
+        self.max_errors: int = 10
+        self.max_errors_args = {'short_name': 'E'}
+        self.old_binaries_path: Path = Path('/app/deploy/global_data/oldBinaries/')
+        self.old_binaries_path_args = {'help': 'Path to the directory containing the old fdb binaries'}
+        self.tls_plugin_path: Path = Path('/app/deploy/runtime/.tls_5_1/FDBLibTLS.so')
+        self.tls_plugin_path_args = {'help': 'Path to the tls plugin used for binaries < 5.2.0'}
+        self.disable_kaio: bool = False
+        self.use_valgrind: bool = False
+        self.use_valgrind_args = {'action': 'store_true'}
+        self.buggify = BuggifyOption('random')
+        self.buggify_args = {'short_name': 'b', 'choices': ['on', 'off', 'random']}
+        self.pretty_print: bool = False
+        self.pretty_print_args = {'short_name': 'P', 'action': 'store_true'}
+        self.clean_up: bool = True
+        self.clean_up_args = {'long_name': 'no_clean_up', 'action': 'store_false'}
+        self.run_dir: Path = Path('tmp')
+        self.joshua_seed: int = random.randint(0, 2 ** 32 - 1)
+        self.joshua_seed_args = {'short_name': 's', 'help': 'A random seed', 'env_name': 'JOSHUA_SEED'}
+        self.print_coverage = False
+        self.print_coverage_args = {'action': 'store_true'}
+        self.binary = Path('bin') / ('fdbserver.exe' if os.name == 'nt' else 'fdbserver')
+        self.binary_args = {'help': 'Path to executable'}
+        self.hit_per_runs_ratio: int = 20000
+        self.hit_per_runs_ratio_args = {'help': 'Maximum test runs before each code probe hit at least once'}
+        self.output_format: str = 'xml'
+        self.output_format_args = {'short_name': 'O', 'choices': ['json', 'xml'],
+                                   'help': 'What format TestHarness should produce'}
+        self.include_test_files: str = r'.*'
+        self.include_test_files_args = {'help': 'Only consider test files whose path match against the given regex'}
+        self.exclude_test_files: str = r'.^'
+        self.exclude_test_files_args = {'help': 'Don\'t consider test files whose path match against the given regex'}
+        self.include_test_classes: str = r'.*'
+        self.include_test_classes_args = {'help': 'Only consider tests whose names match against the given regex'}
+        self.exclude_test_names: str = r'.^'
+        self.exclude_test_names_args = {'help': 'Don\'t consider tests whose names match against the given regex'}
+        self.details: bool = False
+        self.details_args = {'help': 'Print detailed results', 'short_name': 'c', 'action': 'store_true'}
+        self.success: bool = False
+        self.success_args = {'help': 'Print successful results', 'action': 'store_true'}
+        self.cov_include_files: str = r'.*'
+        self.cov_include_files_args = {'help': 'Only consider coverage traces that originated in files matching regex'}
+        self.cov_exclude_files: str = r'.^'
+        self.cov_exclude_files_args = {'help': 'Ignore coverage traces that originated in files matching regex'}
+        self.max_stderr_bytes: int = 10000
+        self.write_stats: bool = True
+        self.read_stats: bool = True
+        self.reproduce_prefix: str | None = None
+        self.reproduce_prefix_args = {'type': str, 'required': False,
+                                      'help': 'When printing the results, prepend this string to the command'}
+        self._env_names: Dict[str, str] = {}
+        self._config_map = self._build_map()
+        self._read_env()
+        self.random.seed(self.joshua_seed, version=2)
+
+    def change_default(self, attr: str, default_val):
+        assert attr in self._config_map, 'Unknown config attribute {}'.format(attr)
+        self.__setattr__(attr, default_val)
+        self._config_map[attr].kwargs['default'] = default_val
+
+    def _get_env_name(self, var_name: str) -> str:
+        return self._env_names.get(var_name, 'TH_{}'.format(var_name.upper()))
+
+    def dump(self):
+        for attr in dir(self):
+            obj = getattr(self, attr)
+            if attr == 'random' or attr.startswith('_') or callable(obj) or attr.endswith('_args'):
+                continue
+            print('config.{}: {} = {}'.format(attr, type(obj), obj))
+
+    def _build_map(self) -> OrderedDict[str, ConfigValue]:
+        config_map: OrderedDict[str, ConfigValue] = collections.OrderedDict()
+        for attr in dir(self):
+            obj = getattr(self, attr)
+            if attr == 'random' or attr.startswith('_') or callable(obj):
+                continue
+            if attr.endswith('_args'):
+                name = attr[0:-len('_args')]
+                assert name in config_map
+                assert isinstance(obj, dict)
+                for k, v in obj.items():
+                    if k == 'env_name':
+                        self._env_names[name] = v
+                    else:
+                        config_map[name].kwargs[k] = v
+            else:
+                # attribute_args has to be declared after the attribute
+                assert attr not in config_map
+                val_type = type(obj)
+                kwargs = {'type': val_type, 'default': obj}
+                config_map[attr] = ConfigValue(attr, **kwargs)
+        return config_map
+
+    def _read_env(self):
+        for attr in dir(self):
+            obj = getattr(self, attr)
+            if attr == 'random' or attr.startswith('_') or attr.endswith('_args') or callable(obj):
+                continue
+            env_name = self._get_env_name(attr)
+            attr_type = self._config_map[attr].kwargs['type']
+            assert type(None) != attr_type
+            e = os.getenv(env_name)
+            if e is not None:
+                # Use the env var to supply the default value, so that if the
+                # environment variable is set and the corresponding command line
+                # flag is not, the environment variable has an effect.
+                self._config_map[attr].kwargs['default'] = attr_type(e)
+
+    def build_arguments(self, parser: argparse.ArgumentParser):
+        for val in self._config_map.values():
+            val.add_to_args(parser)
+
+    def extract_args(self, args: argparse.Namespace):
+        for val in self._config_map.values():
+            k, v = val.get_value(args)
+            if v is not None:
+                config.__setattr__(k, v)
+        self.random.seed(self.joshua_seed, version=2)
+
+
+config = Config()
+
+if __name__ == '__main__':
+    # test the config setup
+    parser = argparse.ArgumentParser('TestHarness Config Tester',
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    config.build_arguments(parser)
+    args = parser.parse_args()
+    config.extract_args(args)
+    config.dump()
diff --git a/contrib/TestHarness2/test_harness/fdb.py b/contrib/TestHarness2/test_harness/fdb.py
new file mode 100644
index 0000000000..1e6afa3906
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/fdb.py
@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from typing import OrderedDict, Tuple, List
+
+import collections
+import fdb
+import fdb.tuple
+import struct
+
+from test_harness.run import StatFetcher, TestDescription
+from test_harness.config import config
+from test_harness.summarize import SummaryTree, Coverage
+
+# Before increasing this, make sure that all Joshua clusters (at Apple and Snowflake) have been upgraded.
+# This version needs to be changed if we either need newer features from FDB or the current API version is
+# getting retired.
+fdb.api_version(630)
+
+
+def str_to_tuple(s: str | None):
+    if s is None:
+        return s
+    return tuple(s.split(','))
+
+
+fdb_db = None
+
+
+def open_db(cluster_file: str | None):
+    global fdb_db
+    if fdb_db is None:
+        fdb_db = fdb.open(cluster_file)
+    return fdb_db
+
+
+def chunkify(iterable, sz: int):
+    res = []
+    for item in iterable:
+        res.append(item)
+        if len(res) >= sz:
+            yield res
+            res = []
+    if len(res) > 0:
+        yield res
+
+
+@fdb.transactional
+def write_coverage_chunk(tr, path: Tuple[str, ...], metadata: Tuple[str, ...],
+                         coverage: List[Tuple[Coverage, bool]], initialized: bool) -> bool:
+    cov_dir = fdb.directory.create_or_open(tr, path)
+    if not initialized:
+        metadata_dir = fdb.directory.create_or_open(tr, metadata)
+        v = tr[metadata_dir['initialized']]
+        initialized = v.present()
+    for cov, covered in coverage:
+        if not initialized or covered:
+            tr.add(cov_dir.pack((cov.file, cov.line, cov.comment)), struct.pack('<I', 1 if covered else 0))
+    return initialized
+
+
+@fdb.transactional
+def set_initialized(tr, metadata: Tuple[str, ...]):
+    metadata_dir = fdb.directory.create_or_open(tr, metadata)
+    tr[metadata_dir['initialized']] = fdb.tuple.pack((True,))
+
+
+def write_coverage(cluster_file: str | None, cov_path: Tuple[str, ...], metadata: Tuple[str, ...],
+                   coverage: OrderedDict[Coverage, bool]):
+    db = open_db(cluster_file)
+    assert config.joshua_dir is not None
+    initialized: bool = False
+    for chunk in chunkify(coverage.items(), 100):
+        initialized = write_coverage_chunk(db, cov_path, metadata, chunk, initialized)
+    if not initialized:
+        set_initialized(db, metadata)
+
+
+@fdb.transactional
+def _read_coverage(tr, cov_path: Tuple[str, ...]) -> OrderedDict[Coverage, int]:
+    res = collections.OrderedDict()
+    cov_dir = fdb.directory.create_or_open(tr, cov_path)
+    for k, v in tr[cov_dir.range()]:
+        file, line, comment = cov_dir.unpack(k)
+        count = struct.unpack('<I', v)[0]
+        res[Coverage(file, line, comment)] = count
+    return res
+
+
+def read_coverage(cluster_file: str | None, cov_path: Tuple[str, ...]) -> OrderedDict[Coverage, int]:
+    db = open_db(cluster_file)
+    return _read_coverage(db, cov_path)
+
+
+class TestStatistics:
+    def __init__(self, runtime: int, run_count: int):
+        self.runtime: int = runtime
+        self.run_count: int = run_count
+
+
+class Statistics:
+    def __init__(self, cluster_file: str | None, joshua_dir: Tuple[str, ...]):
+        self.db = open_db(cluster_file)
+        self.stats_dir = self.open_stats_dir(self.db, joshua_dir)
+        self.stats: OrderedDict[str, TestStatistics] = self.read_stats_from_db(self.db)
+
+    @fdb.transactional
+    def open_stats_dir(self, tr, app_dir: Tuple[str]):
+        stats_dir = app_dir + ('runtime_stats',)
+        return fdb.directory.create_or_open(tr, stats_dir)
+
+    @fdb.transactional
+    def read_stats_from_db(self, tr) -> OrderedDict[str, TestStatistics]:
+        result = collections.OrderedDict()
+        for k, v in tr[self.stats_dir.range()]:
+            test_name = self.stats_dir.unpack(k)[0]
+            runtime, run_count = struct.unpack('<II', v)
+            result[test_name] = TestStatistics(runtime, run_count)
+        return result
+
+    @fdb.transactional
+    def _write_runtime(self, tr, test_name: str, time: int) -> None:
+        key = self.stats_dir.pack((test_name,))
+        tr.add(key, struct.pack('<II', time, 1))
+
+    def write_runtime(self, test_name: str, time: int) -> None:
+        assert self.db is not None
+        self._write_runtime(self.db, test_name, time)
+
+
+class FDBStatFetcher(StatFetcher):
+    def __init__(self, tests: OrderedDict[str, TestDescription],
+                 joshua_dir: Tuple[str] = str_to_tuple(config.joshua_dir)):
+        super().__init__(tests)
+        self.statistics = Statistics(config.cluster_file, joshua_dir)
+
+    def read_stats(self):
+        for k, v in self.statistics.stats.items():
+            if k in self.tests.keys():
+                self.tests[k].total_runtime = v.runtime
+                self.tests[k].num_runs = v.run_count
+
+    def add_run_time(self, test_name: str, runtime: int, out: SummaryTree):
+        self.statistics.write_runtime(test_name, runtime)
+        super().add_run_time(test_name, runtime, out)
diff --git a/contrib/TestHarness2/test_harness/joshua.py b/contrib/TestHarness2/test_harness/joshua.py
new file mode 100644
index 0000000000..33c5881dcc
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/joshua.py
@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import collections
+import io
+import sys
+import xml.sax
+import xml.sax.handler
+from pathlib import Path
+from typing import List, OrderedDict, Set
+
+from joshua import joshua_model
+
+import test_harness.run
+from test_harness.config import config
+from test_harness.summarize import SummaryTree
+
+
+class ToSummaryTree(xml.sax.handler.ContentHandler):
+    def __init__(self):
+        super().__init__()
+        self.root: SummaryTree | None = None
+        self.stack: List[SummaryTree] = []
+
+    def result(self) -> SummaryTree:
+        assert len(self.stack) == 0 and self.root is not None, 'Parse Error'
+        return self.root
+
+    def startElement(self, name, attrs):
+        new_child = SummaryTree(name)
+        for k, v in attrs.items():
+            new_child.attributes[k] = v
+        self.stack.append(new_child)
+
+    def endElement(self, name):
+        closed = self.stack.pop()
+        assert closed.name == name
+        if len(self.stack) == 0:
+            self.root = closed
+        else:
+            self.stack[-1].children.append(closed)
+
+
+def _print_summary(summary: SummaryTree, commands: Set[str]):
+    cmd = []
+    if config.reproduce_prefix is not None:
+        cmd.append(config.reproduce_prefix)
+    cmd.append('fdbserver')
+    if 'TestFile' in summary.attributes:
+        file_name = summary.attributes['TestFile']
+        role = 'test' if test_harness.run.is_no_sim(Path(file_name)) else 'simulation'
+        cmd += ['-r', role, '-f', file_name]
+    else:
+        cmd += ['-r', 'simulation', '-f', '<ERROR>']
+    if 'RandomSeed' in summary.attributes:
+        cmd += ['-s', summary.attributes['RandomSeed']]
+    else:
+        cmd += ['-s', '<Error>']
+    if 'BuggifyEnabled' in summary.attributes:
+        arg = 'on'
+        if summary.attributes['BuggifyEnabled'].lower() in ['0', 'off', 'false']:
+            arg = 'off'
+        cmd += ['-b', arg]
+    else:
+        cmd += ['b', '<ERROR>']
+    cmd += ['--crash', '--trace_format', config.trace_format]
+    key = ' '.join(cmd)
+    count = 1
+    while key in commands:
+        key = '{} # {}'.format(' '.join(cmd), count)
+        count += 1
+    # we want the command as the first attribute
+    attributes = {'Command': ' '.join(cmd)}
+    for k, v in summary.attributes.items():
+        if k == 'Errors':
+            attributes['ErrorCount'] = v
+        else:
+            attributes[k] = v
+    summary.attributes = attributes
+    if config.details:
+        key = str(len(commands))
+        str_io = io.StringIO()
+        summary.dump(str_io, prefix=('  ' if config.pretty_print else ''))
+        if config.output_format == 'json':
+            sys.stdout.write('{}"Test{}": {}'.format('  ' if config.pretty_print else '',
+                                                     key, str_io.getvalue()))
+        else:
+            sys.stdout.write(str_io.getvalue())
+        if config.pretty_print:
+            sys.stdout.write('\n' if config.output_format == 'xml' else ',\n')
+        return key
+    error_count = 0
+    warning_count = 0
+    small_summary = SummaryTree('Test')
+    small_summary.attributes = attributes
+    errors = SummaryTree('Errors')
+    warnings = SummaryTree('Warnings')
+    buggifies: OrderedDict[str, List[int]] = collections.OrderedDict()
+    for child in summary.children:
+        if 'Severity' in child.attributes and child.attributes['Severity'] == '40' and error_count < config.max_errors:
+            error_count += 1
+            errors.append(child)
+        if 'Severity' in child.attributes and child.attributes[
+            'Severity'] == '30' and warning_count < config.max_warnings:
+            warning_count += 1
+            warnings.append(child)
+        if child.name == 'BuggifySection':
+            file = child.attributes['File']
+            line = int(child.attributes['Line'])
+            buggifies.setdefault(file, []).append(line)
+    buggifies_elem = SummaryTree('Buggifies')
+    for file, lines in buggifies.items():
+        lines.sort()
+        if config.output_format == 'json':
+            buggifies_elem.attributes[file] = ' '.join(str(line) for line in lines)
+        else:
+            child = SummaryTree('Buggify')
+            child.attributes['File'] = file
+            child.attributes['Lines'] = ' '.join(str(line) for line in lines)
+            small_summary.append(child)
+    small_summary.children.append(buggifies_elem)
+    if len(errors.children) > 0:
+        small_summary.children.append(errors)
+    if len(warnings.children) > 0:
+        small_summary.children.append(warnings)
+    output = io.StringIO()
+    small_summary.dump(output, prefix=('  ' if config.pretty_print else ''))
+    if config.output_format == 'json':
+        sys.stdout.write('{}"{}": {}'.format('  ' if config.pretty_print else '', key, output.getvalue().strip()))
+    else:
+        sys.stdout.write('{}{}'.format('  ' if config.pretty_print else '', output.getvalue().strip()))
+    sys.stdout.write('\n' if config.output_format == 'xml' else ',\n')
+
+
+def print_errors(ensemble_id: str):
+    joshua_model.open(config.cluster_file)
+    properties = joshua_model.get_ensemble_properties(ensemble_id)
+    compressed = properties["compressed"] if "compressed" in properties else False
+    for rec in joshua_model.tail_results(ensemble_id, errors_only=(not config.success), compressed=compressed):
+        if len(rec) == 5:
+            version_stamp, result_code, host, seed, output = rec
+        elif len(rec) == 4:
+            version_stamp, result_code, host, output = rec
+            seed = None
+        elif len(rec) == 3:
+            version_stamp, result_code, output = rec
+            host = None
+            seed = None
+        elif len(rec) == 2:
+            version_stamp, seed = rec
+            output = str(joshua_model.fdb.tuple.unpack(seed)[0]) + "\n"
+            result_code = None
+            host = None
+            seed = None
+        else:
+            raise Exception("Unknown result format")
+        lines = output.splitlines()
+        commands: Set[str] = set()
+        for line in lines:
+            summary = ToSummaryTree()
+            xml.sax.parseString(line, summary)
+            commands.add(_print_summary(summary.result(), commands))
diff --git a/contrib/TestHarness2/test_harness/results.py b/contrib/TestHarness2/test_harness/results.py
new file mode 100644
index 0000000000..486c497d35
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/results.py
@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import re
+import sys
+import test_harness.fdb
+
+from typing import List, Tuple, OrderedDict
+from test_harness.summarize import SummaryTree, Coverage
+from test_harness.config import config
+from xml.sax.saxutils import quoteattr
+
+
+class GlobalStatistics:
+    def __init__(self):
+        self.total_probes_hit: int = 0
+        self.total_cpu_time: int = 0
+        self.total_test_runs: int = 0
+        self.total_missed_probes: int = 0
+
+
+class EnsembleResults:
+    def __init__(self, cluster_file: str | None, ensemble_id: str):
+        self.global_statistics = GlobalStatistics()
+        self.fdb_path = ('joshua', 'ensembles', 'results', 'application', ensemble_id)
+        self.coverage_path = self.fdb_path + ('coverage',)
+        self.statistics = test_harness.fdb.Statistics(cluster_file, self.fdb_path)
+        coverage_dict: OrderedDict[Coverage, int] = test_harness.fdb.read_coverage(cluster_file, self.coverage_path)
+        self.coverage: List[Tuple[Coverage, int]] = []
+        self.min_coverage_hit: int | None = None
+        self.ratio = self.global_statistics.total_test_runs / config.hit_per_runs_ratio
+        for cov, count in coverage_dict.items():
+            if re.search(config.cov_include_files, cov.file) is None:
+                continue
+            if re.search(config.cov_exclude_files, cov.file) is not None:
+                continue
+            self.global_statistics.total_probes_hit += count
+            self.coverage.append((cov, count))
+            if count <= self.ratio:
+                self.global_statistics.total_missed_probes += 1
+            if self.min_coverage_hit is None or self.min_coverage_hit > count:
+                self.min_coverage_hit = count
+        self.coverage.sort(key=lambda x: (x[1], x[0].file, x[0].line))
+        self.stats: List[Tuple[str, int, int]] = []
+        for k, v in self.statistics.stats.items():
+            self.global_statistics.total_test_runs += v.run_count
+            self.global_statistics.total_cpu_time += v.runtime
+            self.stats.append((k, v.runtime, v.run_count))
+        self.stats.sort(key=lambda x: x[1], reverse=True)
+        if self.min_coverage_hit is not None:
+            self.coverage_ok = self.min_coverage_hit > self.ratio
+        else:
+            self.coverage_ok = False
+
+    def dump(self, prefix: str):
+        errors = 0
+        out = SummaryTree('EnsembleResults')
+        out.attributes['TotalRuntime'] = str(self.global_statistics.total_cpu_time)
+        out.attributes['TotalTestRuns'] = str(self.global_statistics.total_test_runs)
+        out.attributes['TotalProbesHit'] = str(self.global_statistics.total_probes_hit)
+        out.attributes['MinProbeHit'] = str(self.min_coverage_hit)
+        out.attributes['TotalProbes'] = str(len(self.coverage))
+        out.attributes['MissedProbes'] = str(self.global_statistics.total_missed_probes)
+
+        for cov, count in self.coverage:
+            severity = 10 if count > self.ratio else 40
+            if severity == 40:
+                errors += 1
+            if (severity == 40 and errors <= config.max_errors) or config.details:
+                child = SummaryTree('CodeProbe')
+                child.attributes['Severity'] = str(severity)
+                child.attributes['File'] = cov.file
+                child.attributes['Line'] = str(cov.line)
+                child.attributes['Comment'] = '' if cov.comment is None else cov.comment
+                child.attributes['HitCount'] = str(count)
+                out.append(child)
+
+        if config.details:
+            for k, runtime, run_count in self.stats:
+                child = SummaryTree('Test')
+                child.attributes['Name'] = k
+                child.attributes['Runtime'] = str(runtime)
+                child.attributes['RunCount'] = str(run_count)
+                out.append(child)
+        if errors > 0:
+            out.attributes['Errors'] = str(errors)
+        str_io = io.StringIO()
+        out.dump(str_io, prefix=prefix, new_line=config.pretty_print)
+        if config.output_format == 'xml':
+            sys.stdout.write(str_io.getvalue())
+        else:
+            sys.stdout.write('{}"EnsembleResults":{}{}'.format('  ' if config.pretty_print else '',
+                                                               '\n' if config.pretty_print else ' ',
+                                                               str_io.getvalue()))
+
+
+def write_header(ensemble_id: str):
+    if config.output_format == 'json':
+        if config.pretty_print:
+            print('{')
+            print('  "{}": {},\n'.format('ID', json.dumps(ensemble_id.strip())))
+        else:
+            sys.stdout.write('{{{}: {},'.format('ID', json.dumps(ensemble_id.strip())))
+    elif config.output_format == 'xml':
+        sys.stdout.write('<Ensemble ID={}>'.format(quoteattr(ensemble_id.strip())))
+        if config.pretty_print:
+            sys.stdout.write('\n')
+    else:
+        assert False, 'unknown output format {}'.format(config.output_format)
+
+
+def write_footer():
+    if config.output_format == 'xml':
+        sys.stdout.write('</Ensemble>\n')
+    elif config.output_format == 'json':
+        sys.stdout.write('}\n')
+    else:
+        assert False, 'unknown output format {}'.format(config.output_format)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser('TestHarness Results', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    config.change_default('pretty_print', True)
+    config.change_default('max_warnings', 0)
+    config.build_arguments(parser)
+    parser.add_argument('ensemble_id', type=str, help='The ensemble to fetch the result for')
+    args = parser.parse_args()
+    config.extract_args(args)
+    config.output_format = args.output_format
+    write_header(args.ensemble_id)
+    try:
+        import test_harness.joshua
+        test_harness.joshua.print_errors(args.ensemble_id)
+    except ModuleNotFoundError:
+        child = SummaryTree('JoshuaNotFound')
+        child.attributes['Severity'] = '30'
+        child.attributes['Message'] = 'Could not import Joshua -- set PYTHONPATH to joshua checkout dir'
+        child.dump(sys.stdout, prefix=('  ' if config.pretty_print else ''), new_line=config.pretty_print)
+    results = EnsembleResults(config.cluster_file, args.ensemble_id)
+    results.dump('  ' if config.pretty_print else '')
+    write_footer()
+    exit(0 if results.coverage_ok else 1)
diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py
new file mode 100644
index 0000000000..2cd24575fb
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/run.py
@@ -0,0 +1,477 @@
+from __future__ import annotations
+
+import array
+import base64
+import collections
+import math
+import os
+import resource
+import shutil
+import subprocess
+import re
+import sys
+import threading
+import time
+import uuid
+
+from functools import total_ordering
+from pathlib import Path
+from test_harness.version import Version
+from test_harness.config import config
+from typing import Dict, List, Pattern, OrderedDict
+
+from test_harness.summarize import Summary, SummaryTree
+
+
+@total_ordering
+class TestDescription:
+    def __init__(self, path: Path, name: str, priority: float):
+        self.paths: List[Path] = [path]
+        self.name = name
+        self.priority: float = priority
+        # we only measure in seconds. Otherwise, keeping determinism will be difficult
+        self.total_runtime: int = 0
+        self.num_runs: int = 0
+
+    def __lt__(self, other):
+        if isinstance(other, TestDescription):
+            return self.name < other.name
+        else:
+            return self.name < str(other)
+
+    def __eq__(self, other):
+        if isinstance(other, TestDescription):
+            return self.name < other.name
+        else:
+            return self.name < str(other.name)
+
+
+class StatFetcher:
+    def __init__(self, tests: OrderedDict[str, TestDescription]):
+        self.tests = tests
+
+    def read_stats(self):
+        pass
+
+    def add_run_time(self, test_name: str, runtime: int, out: SummaryTree):
+        self.tests[test_name].total_runtime += runtime
+
+
+class TestPicker:
+    def __init__(self, test_dir: Path):
+        if not test_dir.exists():
+            raise RuntimeError('{} is neither a directory nor a file'.format(test_dir))
+        self.include_files_regex = re.compile(config.include_test_files)
+        self.exclude_files_regex = re.compile(config.exclude_test_files)
+        self.include_tests_regex = re.compile(config.include_test_classes)
+        self.exclude_tests_regex = re.compile(config.exclude_test_names)
+        self.test_dir: Path = test_dir
+        self.tests: OrderedDict[str, TestDescription] = collections.OrderedDict()
+        self.restart_test: Pattern = re.compile(r".*-\d+\.(txt|toml)")
+        self.follow_test: Pattern = re.compile(r".*-[2-9]\d*\.(txt|toml)")
+
+        for subdir in self.test_dir.iterdir():
+            if subdir.is_dir() and subdir.name in config.test_dirs:
+                self.walk_test_dir(subdir)
+        self.stat_fetcher: StatFetcher
+        if config.stats is not None or config.joshua_dir is None:
+            self.stat_fetcher = StatFetcher(self.tests)
+        else:
+            from test_harness.fdb import FDBStatFetcher
+            self.stat_fetcher = FDBStatFetcher(self.tests)
+        if config.stats is not None:
+            self.load_stats(config.stats)
+        else:
+            self.fetch_stats()
+
+    def add_time(self, test_file: Path, run_time: int, out: SummaryTree) -> None:
+        # getting the test name is fairly inefficient. But since we only have 100s of tests, I won't bother
+        test_name: str | None = None
+        test_desc: TestDescription | None = None
+        for name, test in self.tests.items():
+            for p in test.paths:
+                test_files: List[Path]
+                if self.restart_test.match(p.name):
+                    test_files = self.list_restart_files(p)
+                else:
+                    test_files = [p]
+                for file in test_files:
+                    if file.absolute() == test_file.absolute():
+                        test_name = name
+                        test_desc = test
+                        break
+                if test_name is not None:
+                    break
+            if test_name is not None:
+                break
+        assert test_name is not None and test_desc is not None
+        self.stat_fetcher.add_run_time(test_name, run_time, out)
+        out.attributes['TotalTestTime'] = str(test_desc.total_runtime)
+        out.attributes['TestRunCount'] = str(test_desc.num_runs)
+
+    def dump_stats(self) -> str:
+        res = array.array('I')
+        for _, spec in self.tests.items():
+            res.append(spec.total_runtime)
+        return base64.standard_b64encode(res.tobytes()).decode('utf-8')
+
+    def fetch_stats(self):
+        self.stat_fetcher.read_stats()
+
+    def load_stats(self, serialized: str):
+        times = array.array('I')
+        times.frombytes(base64.standard_b64decode(serialized))
+        assert len(times) == len(self.tests.items())
+        for idx, (_, spec) in enumerate(self.tests.items()):
+            spec.total_runtime = times[idx]
+
+    def parse_txt(self, path: Path):
+        if self.include_files_regex.search(str(path)) is None or self.exclude_files_regex.search(str(path)) is not None:
+            return
+        with path.open('r') as f:
+            test_name: str | None = None
+            test_class: str | None = None
+            priority: float | None = None
+            for line in f:
+                line = line.strip()
+                kv = line.split('=')
+                if len(kv) != 2:
+                    continue
+                kv[0] = kv[0].strip()
+                kv[1] = kv[1].strip(' \r\n\t\'"')
+                if kv[0] == 'testTitle' and test_name is None:
+                    test_name = kv[1]
+                if kv[0] == 'testClass' and test_class is None:
+                    test_class = kv[1]
+                if kv[0] == 'testPriority' and priority is None:
+                    try:
+                        priority = float(kv[1])
+                    except ValueError:
+                        raise RuntimeError("Can't parse {} -- testPriority in {} should be set to a float".format(kv[1],
+                                                                                                                  path))
+                if test_name is not None and test_class is not None and priority is not None:
+                    break
+            if test_name is None:
+                return
+            if test_class is None:
+                test_class = test_name
+            if priority is None:
+                priority = 1.0
+            if self.include_tests_regex.search(test_class) is None \
+                    or self.exclude_tests_regex.search(test_class) is not None:
+                return
+            if test_class not in self.tests:
+                self.tests[test_class] = TestDescription(path, test_class, priority)
+            else:
+                self.tests[test_class].paths.append(path)
+
+    def walk_test_dir(self, test: Path):
+        if test.is_dir():
+            for file in test.iterdir():
+                self.walk_test_dir(file)
+        else:
+            # check whether we're looking at a restart test
+            if self.follow_test.match(test.name) is not None:
+                return
+            if test.suffix == '.txt' or test.suffix == '.toml':
+                self.parse_txt(test)
+
+    @staticmethod
+    def list_restart_files(start_file: Path) -> List[Path]:
+        name = re.sub(r'-\d+.(txt|toml)', '', start_file.name)
+        res: List[Path] = []
+        for test_file in start_file.parent.iterdir():
+            if test_file.name.startswith(name):
+                res.append(test_file)
+        assert len(res) > 1
+        res.sort()
+        return res
+
+    def choose_test(self) -> List[Path]:
+        min_runtime: float | None = None
+        candidates: List[TestDescription] = []
+        for _, v in self.tests.items():
+            this_time = v.total_runtime * v.priority
+            if min_runtime is None or this_time < min_runtime:
+                min_runtime = this_time
+                candidates = [v]
+            elif this_time == min_runtime:
+                candidates.append(v)
+        candidates.sort()
+        choice = config.random.randint(0, len(candidates) - 1)
+        test = candidates[choice]
+        result = test.paths[config.random.randint(0, len(test.paths) - 1)]
+        if self.restart_test.match(result.name):
+            return self.list_restart_files(result)
+        else:
+            return [result]
+
+
+class OldBinaries:
+    def __init__(self):
+        self.first_file_expr = re.compile(r'.*-1\.(txt|toml)')
+        self.old_binaries_path: Path = config.old_binaries_path
+        self.binaries: OrderedDict[Version, Path] = collections.OrderedDict()
+        if not self.old_binaries_path.exists() or not self.old_binaries_path.is_dir():
+            return
+        exec_pattern = re.compile(r'fdbserver-\d+\.\d+\.\d+(\.exe)?')
+        for file in self.old_binaries_path.iterdir():
+            if not file.is_file() or not os.access(file, os.X_OK):
+                continue
+            if exec_pattern.fullmatch(file.name) is not None:
+                self._add_file(file)
+
+    def _add_file(self, file: Path):
+        version_str = file.name.split('-')[1]
+        if version_str.endswith('.exe'):
+            version_str = version_str[0:-len('.exe')]
+        ver = Version.parse(version_str)
+        self.binaries[ver] = file
+
+    def choose_binary(self, test_file: Path) -> Path:
+        if len(self.binaries) == 0:
+            return config.binary
+        max_version = Version.max_version()
+        min_version = Version.parse('5.0.0')
+        dirs = test_file.parent.parts
+        if 'restarting' not in dirs:
+            return config.binary
+        version_expr = dirs[-1].split('_')
+        first_file = self.first_file_expr.match(test_file.name) is not None
+        if first_file and version_expr[0] == 'to':
+            # downgrade test -- first binary should be current one
+            return config.binary
+        if not first_file and version_expr[0] == 'from':
+            # upgrade test -- we only return an old version for the first test file
+            return config.binary
+        if version_expr[0] == 'from' or version_expr[0] == 'to':
+            min_version = Version.parse(version_expr[1])
+        if len(version_expr) == 4 and version_expr[2] == 'until':
+            max_version = Version.parse(version_expr[3])
+        candidates: List[Path] = []
+        for ver, binary in self.binaries.items():
+            if min_version <= ver <= max_version:
+                candidates.append(binary)
+        if len(candidates) == 0:
+            return config.binary
+        return config.random.choice(candidates)
+
+
+def is_restarting_test(test_file: Path):
+    for p in test_file.parts:
+        if p == 'restarting':
+            return True
+    return False
+
+
+def is_no_sim(test_file: Path):
+    return test_file.parts[-2] == 'noSim'
+
+
+class ResourceMonitor(threading.Thread):
+    def __init__(self):
+        super().__init__()
+        self.start_time = time.time()
+        self.end_time: float | None = None
+        self._stop_monitor = False
+        self.max_rss = 0
+
+    def run(self) -> None:
+        while not self._stop_monitor:
+            time.sleep(1)
+            resources = resource.getrusage(resource.RUSAGE_CHILDREN)
+            self.max_rss = max(resources.ru_maxrss, self.max_rss)
+
+    def stop(self):
+        self.end_time = time.time()
+        self._stop_monitor = True
+
+    def time(self):
+        return self.end_time - self.start_time
+
+
+class TestRun:
+    def __init__(self, binary: Path, test_file: Path, random_seed: int, uid: uuid.UUID,
+                 restarting: bool = False, test_determinism: bool = False, buggify_enabled: bool = False,
+                 stats: str | None = None, expected_unseed: int | None = None, will_restart: bool = False):
+        self.binary = binary
+        self.test_file = test_file
+        self.random_seed = random_seed
+        self.uid = uid
+        self.restarting = restarting
+        self.test_determinism = test_determinism
+        self.stats: str | None = stats
+        self.expected_unseed: int | None = expected_unseed
+        self.use_valgrind: bool = config.use_valgrind
+        self.old_binary_path: Path = config.old_binaries_path
+        self.buggify_enabled: bool = buggify_enabled
+        self.fault_injection_enabled: bool = True
+        self.trace_format: str | None = config.trace_format
+        if Version.of_binary(self.binary) < "6.1.0":
+            self.trace_format = None
+        self.use_tls_plugin = Version.of_binary(self.binary) < "5.2.0"
+        self.temp_path = config.run_dir / str(self.uid)
+        # state for the run
+        self.retryable_error: bool = False
+        self.summary: Summary = Summary(binary, uid=self.uid, stats=self.stats, expected_unseed=self.expected_unseed,
+                                        will_restart=will_restart)
+        self.run_time: int = 0
+        self.success = self.run()
+
+    def log_test_plan(self, out: SummaryTree):
+        test_plan: SummaryTree = SummaryTree('TestPlan')
+        test_plan.attributes['TestUID'] = str(self.uid)
+        test_plan.attributes['RandomSeed'] = str(self.random_seed)
+        test_plan.attributes['TestFile'] = str(self.test_file)
+        test_plan.attributes['Buggify'] = '1' if self.buggify_enabled else '0'
+        test_plan.attributes['FaultInjectionEnabled'] = '1' if self.fault_injection_enabled else '0'
+        test_plan.attributes['DeterminismCheck'] = '1' if self.test_determinism else '0'
+        out.append(test_plan)
+
+    def delete_simdir(self):
+        shutil.rmtree(self.temp_path / Path('simfdb'))
+
+    def run(self):
+        command: List[str] = []
+        env: Dict[str, str] = os.environ.copy()
+        valgrind_file: Path | None = None
+        if self.use_valgrind and self.binary == config.binary:
+            # Only run the binary under test under valgrind. There's nothing we
+            # can do about valgrind errors in old binaries anyway, and it makes
+            # the test take longer. Also old binaries weren't built with
+            # USE_VALGRIND=ON, and we have seen false positives with valgrind in
+            # such binaries.
+            command.append('valgrind')
+            valgrind_file = self.temp_path / Path('valgrind-{}.xml'.format(self.random_seed))
+            dbg_path = os.getenv('FDB_VALGRIND_DBGPATH')
+            if dbg_path is not None:
+                command.append('--extra-debuginfo-path={}'.format(dbg_path))
+            command += ['--xml=yes', '--xml-file={}'.format(valgrind_file.absolute()), '-q']
+        command += [str(self.binary.absolute()),
+                    '-r', 'test' if is_no_sim(self.test_file) else 'simulation',
+                    '-f', str(self.test_file),
+                    '-s', str(self.random_seed)]
+        if self.trace_format is not None:
+            command += ['--trace_format', self.trace_format]
+        if self.use_tls_plugin:
+            command += ['--tls_plugin', str(config.tls_plugin_path)]
+            env["FDB_TLS_PLUGIN"] = str(config.tls_plugin_path)
+        if config.disable_kaio:
+            command += ['--knob-disable-posix-kernel-aio=1']
+        if Version.of_binary(self.binary) >= '7.1.0':
+            command += ['-fi', 'on' if self.fault_injection_enabled else 'off']
+        if self.restarting:
+            command.append('--restarting')
+        if self.buggify_enabled:
+            command += ['-b', 'on']
+        if config.crash_on_error:
+            command.append('--crash')
+
+        self.temp_path.mkdir(parents=True, exist_ok=True)
+
+        # self.log_test_plan(out)
+        resources = ResourceMonitor()
+        resources.start()
+        process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, cwd=self.temp_path,
+                                   text=True, env=env)
+        did_kill = False
+        timeout = 20 * config.kill_seconds if self.use_valgrind else config.kill_seconds
+        err_out: str
+        try:
+            _, err_out = process.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            _, err_out = process.communicate()
+            did_kill = True
+        resources.stop()
+        resources.join()
+        # we're rounding times up, otherwise we will prefer running very short tests (<1s)
+        self.run_time = math.ceil(resources.time())
+        self.summary.runtime = resources.time()
+        self.summary.max_rss = resources.max_rss
+        self.summary.was_killed = did_kill
+        self.summary.valgrind_out_file = valgrind_file
+        self.summary.error_out = err_out
+        self.summary.summarize(self.temp_path, ' '.join(command))
+        return self.summary.ok()
+
+
+def decorate_summary(out: SummaryTree, test_file: Path, seed: int, buggify: bool):
+    """Sometimes a test can crash before ProgramStart is written to the traces. These
+    tests are then hard to reproduce (they can be reproduced through TestHarness but
+    require the user to run in the joshua docker container). To account for this we
+    will write the necessary information into the attributes if it is missing."""
+    if 'TestFile' not in out.attributes:
+        out.attributes['TestFile'] = str(test_file)
+    if 'RandomSeed' not in out.attributes:
+        out.attributes['RandomSeed'] = str(seed)
+    if 'BuggifyEnabled' not in out.attributes:
+        out.attributes['BuggifyEnabled'] = '1' if buggify else '0'
+
+
+class TestRunner:
+    def __init__(self):
+        self.uid = uuid.uuid4()
+        self.test_path: Path = Path('tests')
+        self.cluster_file: str | None = None
+        self.fdb_app_dir: str | None = None
+        self.binary_chooser = OldBinaries()
+        self.test_picker = TestPicker(self.test_path)
+
+    def backup_sim_dir(self, seed: int):
+        temp_dir = config.run_dir / str(self.uid)
+        src_dir = temp_dir / 'simfdb'
+        assert src_dir.is_dir()
+        dest_dir = temp_dir / 'simfdb.{}'.format(seed)
+        assert not dest_dir.exists()
+        shutil.copytree(src_dir, dest_dir)
+
+    def restore_sim_dir(self, seed: int):
+        temp_dir = config.run_dir / str(self.uid)
+        src_dir = temp_dir / 'simfdb.{}'.format(seed)
+        assert src_dir.exists()
+        dest_dir = temp_dir / 'simfdb'
+        shutil.rmtree(dest_dir)
+        shutil.move(src_dir, dest_dir)
+
+    def run_tests(self, test_files: List[Path], seed: int, test_picker: TestPicker) -> bool:
+        result: bool = True
+        for count, file in enumerate(test_files):
+            will_restart = count + 1 < len(test_files)
+            binary = self.binary_chooser.choose_binary(file)
+            unseed_check = not is_no_sim(file) and config.random.random() < config.unseed_check_ratio
+            buggify_enabled: bool = config.random.random() < config.buggify_on_ratio
+            if unseed_check and count != 0:
+                # for restarting tests we will need to restore the sim2 after the first run
+                self.backup_sim_dir(seed + count - 1)
+            run = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0,
+                          stats=test_picker.dump_stats(), will_restart=will_restart, buggify_enabled=buggify_enabled)
+            result = result and run.success
+            test_picker.add_time(test_files[0], run.run_time, run.summary.out)
+            decorate_summary(run.summary.out, file, seed + count, run.buggify_enabled)
+            if unseed_check and run.summary.unseed:
+                run.summary.out.append(run.summary.list_simfdb())
+            run.summary.out.dump(sys.stdout)
+            if not result:
+                return False
+            if unseed_check and run.summary.unseed is not None:
+                if count != 0:
+                    self.restore_sim_dir(seed + count - 1)
+                run2 = TestRun(binary, file.absolute(), seed + count, self.uid, restarting=count != 0,
+                               stats=test_picker.dump_stats(), expected_unseed=run.summary.unseed,
+                               will_restart=will_restart, buggify_enabled=buggify_enabled)
+                test_picker.add_time(file, run2.run_time, run.summary.out)
+                decorate_summary(run2.summary.out, file, seed + count, run.buggify_enabled)
+                run2.summary.out.dump(sys.stdout)
+                result = result and run2.success
+                if not result:
+                    return False
+        return result
+
+    def run(self) -> bool:
+        seed = config.random_seed if config.random_seed is not None else config.random.randint(0, 2 ** 32 - 1)
+        test_files = self.test_picker.choose_test()
+        success = self.run_tests(test_files, seed, self.test_picker)
+        if config.clean_up:
+            shutil.rmtree(config.run_dir / str(self.uid))
+        return success
diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py
new file mode 100644
index 0000000000..54b2f799b5
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/summarize.py
@@ -0,0 +1,620 @@
+from __future__ import annotations
+
+import collections
+import inspect
+import json
+import os
+import re
+import sys
+import traceback
+import uuid
+import xml.sax
+import xml.sax.handler
+import xml.sax.saxutils
+
+from pathlib import Path
+from typing import List, Dict, TextIO, Callable, Optional, OrderedDict, Any, Tuple, Iterator, Iterable
+
+from test_harness.config import config
+from test_harness.valgrind import parse_valgrind_output
+
+
+class SummaryTree:
+    def __init__(self, name: str):
+        self.name = name
+        self.children: List[SummaryTree] = []
+        self.attributes: Dict[str, str] = {}
+
+    def append(self, element: SummaryTree):
+        self.children.append(element)
+
+    def to_dict(self, add_name: bool = True) -> Dict[str, Any] | List[Any]:
+        if len(self.children) > 0 and len(self.attributes) == 0:
+            children = []
+            for child in self.children:
+                children.append(child.to_dict())
+            if add_name:
+                return {self.name: children}
+            else:
+                return children
+        res: Dict[str, Any] = {}
+        if add_name:
+            res['Type'] = self.name
+        for k, v in self.attributes.items():
+            res[k] = v
+        children = []
+        child_keys: Dict[str, int] = {}
+        for child in self.children:
+            if child.name in child_keys:
+                child_keys[child.name] += 1
+            else:
+                child_keys[child.name] = 1
+        for child in self.children:
+            if child_keys[child.name] == 1 and child.name not in self.attributes:
+                res[child.name] = child.to_dict(add_name=False)
+            else:
+                children.append(child.to_dict())
+        if len(children) > 0:
+            res['children'] = children
+        return res
+
+    def to_json(self, out: TextIO, prefix: str = ''):
+        res = json.dumps(self.to_dict(), indent=('  ' if config.pretty_print else None))
+        for line in res.splitlines(False):
+            out.write('{}{}\n'.format(prefix, line))
+
+    def to_xml(self, out: TextIO, prefix: str = ''):
+        # minidom doesn't support omitting the xml declaration which is a problem for joshua
+        # However, our xml is very simple and therefore serializing manually is easy enough
+        attrs = []
+        print_width = 120
+        try:
+            print_width, _ = os.get_terminal_size()
+        except OSError:
+            pass
+        for k, v in self.attributes.items():
+            attrs.append('{}={}'.format(k, xml.sax.saxutils.quoteattr(v)))
+        elem = '{}<{}{}'.format(prefix, self.name, ('' if len(attrs) == 0 else ' '))
+        out.write(elem)
+        if config.pretty_print:
+            curr_line_len = len(elem)
+            for i in range(len(attrs)):
+                attr_len = len(attrs[i])
+                if i == 0 or attr_len + curr_line_len + 1 <= print_width:
+                    if i != 0:
+                        out.write(' ')
+                    out.write(attrs[i])
+                    curr_line_len += attr_len
+                else:
+                    out.write('\n')
+                    out.write(' ' * len(elem))
+                    out.write(attrs[i])
+                    curr_line_len = len(elem) + attr_len
+        else:
+            out.write(' '.join(attrs))
+        if len(self.children) == 0:
+            out.write('/>')
+        else:
+            out.write('>')
+        for child in self.children:
+            if config.pretty_print:
+                out.write('\n')
+            child.to_xml(out, prefix=('  {}'.format(prefix) if config.pretty_print else prefix))
+        if len(self.children) > 0:
+            out.write('{}{}</{}>'.format(('\n' if config.pretty_print else ''), prefix, self.name))
+
+    def dump(self, out: TextIO, prefix: str = '', new_line: bool = True):
+        if config.output_format == 'json':
+            self.to_json(out, prefix=prefix)
+        else:
+            self.to_xml(out, prefix=prefix)
+        if new_line:
+            out.write('\n')
+
+
+ParserCallback = Callable[[Dict[str, str]], Optional[str]]
+
+
+class ParseHandler:
+    def __init__(self, out: SummaryTree):
+        self.out = out
+        self.events: OrderedDict[Optional[Tuple[str, Optional[str]]], List[ParserCallback]] = collections.OrderedDict()
+
+    def add_handler(self, attr: Tuple[str, Optional[str]], callback: ParserCallback) -> None:
+        self.events.setdefault(attr, []).append(callback)
+
+    def _call(self, callback: ParserCallback, attrs: Dict[str, str]) -> str | None:
+        try:
+            return callback(attrs)
+        except Exception as e:
+            _, _, exc_traceback = sys.exc_info()
+            child = SummaryTree('NonFatalParseError')
+            child.attributes['Severity'] = '30'
+            child.attributes['ErrorMessage'] = str(e)
+            child.attributes['Trace'] = repr(traceback.format_tb(exc_traceback))
+            self.out.append(child)
+            return None
+
+    def handle(self, attrs: Dict[str, str]):
+        if None in self.events:
+            for callback in self.events[None]:
+                self._call(callback, attrs)
+        for k, v in attrs.items():
+            if (k, None) in self.events:
+                for callback in self.events[(k, None)]:
+                    remap = self._call(callback, attrs)
+                    if remap is not None:
+                        v = remap
+                        attrs[k] = v
+            if (k, v) in self.events:
+                for callback in self.events[(k, v)]:
+                    remap = self._call(callback, attrs)
+                    if remap is not None:
+                        v = remap
+                        attrs[k] = v
+
+
+class Parser:
+    def parse(self, file: TextIO, handler: ParseHandler) -> None:
+        pass
+
+
+class XmlParser(Parser, xml.sax.handler.ContentHandler):
+    def __init__(self):
+        super().__init__()
+        self.handler: ParseHandler | None = None
+
+    def parse(self, file: TextIO, handler: ParseHandler) -> None:
+        xml.sax.parse(file, self)
+
+    def startElement(self, name, attrs) -> None:
+        attributes: Dict[str, str] = {}
+        for name in attrs.getNames():
+            attributes[name] = attrs.getValue(name)
+        assert self.handler is not None
+        self.handler.handle(attributes)
+
+
+class JsonParser(Parser):
+    def __init__(self):
+        super().__init__()
+
+    def parse(self, file: TextIO, handler: ParseHandler):
+        for line in file:
+            obj = json.loads(line)
+            handler.handle(obj)
+
+
+class Coverage:
+    def __init__(self, file: str, line: str | int, comment: str | None = None):
+        self.file = file
+        self.line = int(line)
+        self.comment = comment
+
+    def to_tuple(self) -> Tuple[str, int, str | None]:
+        return self.file, self.line, self.comment
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, tuple) and len(other) == 3:
+            return self.to_tuple() == other
+        elif isinstance(other, Coverage):
+            return self.to_tuple() == other.to_tuple()
+        else:
+            return False
+
+    def __lt__(self, other) -> bool:
+        if isinstance(other, tuple) and len(other) == 3:
+            return self.to_tuple() < other
+        elif isinstance(other, Coverage):
+            return self.to_tuple() < other.to_tuple()
+        else:
+            return False
+
+    def __le__(self, other) -> bool:
+        if isinstance(other, tuple) and len(other) == 3:
+            return self.to_tuple() <= other
+        elif isinstance(other, Coverage):
+            return self.to_tuple() <= other.to_tuple()
+        else:
+            return False
+
+    def __gt__(self, other: Coverage) -> bool:
+        if isinstance(other, tuple) and len(other) == 3:
+            return self.to_tuple() > other
+        elif isinstance(other, Coverage):
+            return self.to_tuple() > other.to_tuple()
+        else:
+            return False
+
+    def __ge__(self, other):
+        if isinstance(other, tuple) and len(other) == 3:
+            return self.to_tuple() >= other
+        elif isinstance(other, Coverage):
+            return self.to_tuple() >= other.to_tuple()
+        else:
+            return False
+
+    def __hash__(self):
+        return hash((self.file, self.line, self.comment))
+
+
+class TraceFiles:
+    def __init__(self, path: Path):
+        self.path: Path = path
+        self.timestamps: List[int] = []
+        self.runs: OrderedDict[int, List[Path]] = collections.OrderedDict()
+        trace_expr = re.compile(r'trace.*\.(json|xml)')
+        for file in self.path.iterdir():
+            if file.is_file() and trace_expr.match(file.name) is not None:
+                ts = int(file.name.split('.')[6])
+                if ts in self.runs:
+                    self.runs[ts].append(file)
+                else:
+                    self.timestamps.append(ts)
+                    self.runs[ts] = [file]
+        self.timestamps.sort(reverse=True)
+
+    def __getitem__(self, idx: int) -> List[Path]:
+        res = self.runs[self.timestamps[idx]]
+        res.sort()
+        return res
+
+    def __len__(self) -> int:
+        return len(self.runs)
+
+    def items(self) -> Iterator[List[Path]]:
+        class TraceFilesIterator(Iterable[List[Path]]):
+            def __init__(self, trace_files: TraceFiles):
+                self.current = 0
+                self.trace_files: TraceFiles = trace_files
+
+            def __iter__(self):
+                return self
+
+            def __next__(self) -> List[Path]:
+                if len(self.trace_files) <= self.current:
+                    raise StopIteration
+                self.current += 1
+                return self.trace_files[self.current - 1]
+        return TraceFilesIterator(self)
+
+
+class Summary:
+    def __init__(self, binary: Path, runtime: float = 0, max_rss: int | None = None,
+                 was_killed: bool = False, uid: uuid.UUID | None = None, expected_unseed: int | None = None,
+                 exit_code: int = 0, valgrind_out_file: Path | None = None, stats: str | None = None,
+                 error_out: str = None, will_restart: bool = False):
+        self.binary = binary
+        self.runtime: float = runtime
+        self.max_rss: int | None = max_rss
+        self.was_killed: bool = was_killed
+        self.expected_unseed: int | None = expected_unseed
+        self.exit_code: int = exit_code
+        self.out: SummaryTree = SummaryTree('Test')
+        self.test_begin_found: bool = False
+        self.test_end_found: bool = False
+        self.unseed: int | None = None
+        self.valgrind_out_file: Path | None = valgrind_out_file
+        self.severity_map: OrderedDict[tuple[str, int], int] = collections.OrderedDict()
+        self.error: bool = False
+        self.errors: int = 0
+        self.warnings: int = 0
+        self.coverage: OrderedDict[Coverage, bool] = collections.OrderedDict()
+        self.test_count: int = 0
+        self.tests_passed: int = 0
+        self.error_out = error_out
+        self.stderr_severity: str = '40'
+        self.will_restart: bool = will_restart
+        self.test_dir: Path | None = None
+
+        if uid is not None:
+            self.out.attributes['TestUID'] = str(uid)
+        if stats is not None:
+            self.out.attributes['Statistics'] = stats
+        self.out.attributes['JoshuaSeed'] = str(config.joshua_seed)
+        self.out.attributes['WillRestart'] = '1' if self.will_restart else '0'
+
+        self.handler = ParseHandler(self.out)
+        self.register_handlers()
+
+    def summarize_files(self, trace_files: List[Path]):
+        assert len(trace_files) > 0
+        for f in trace_files:
+            self.parse_file(f)
+        self.done()
+
+    def summarize(self, trace_dir: Path, command: str):
+        self.test_dir = trace_dir
+        trace_files = TraceFiles(trace_dir)
+        if len(trace_files) == 0:
+            self.error = True
+            child = SummaryTree('NoTracesFound')
+            child.attributes['Severity'] = '40'
+            child.attributes['Path'] = str(trace_dir.absolute())
+            child.attributes['Command'] = command
+            self.out.append(child)
+            return
+        self.summarize_files(trace_files[0])
+        if config.joshua_dir is not None:
+            import test_harness.fdb
+            test_harness.fdb.write_coverage(config.cluster_file,
+                                            test_harness.fdb.str_to_tuple(config.joshua_dir) + ('coverage',),
+                                            test_harness.fdb.str_to_tuple(config.joshua_dir) + ('coverage-metadata',),
+                                            self.coverage)
+
+    def list_simfdb(self) -> SummaryTree:
+        res = SummaryTree('SimFDB')
+        res.attributes['TestDir'] = str(self.test_dir)
+        if self.test_dir is None:
+            return res
+        simfdb = self.test_dir / Path('simfdb')
+        if not simfdb.exists():
+            res.attributes['NoSimDir'] = "simfdb doesn't exist"
+            return res
+        elif not simfdb.is_dir():
+            res.attributes['NoSimDir'] = 'simfdb is not a directory'
+            return res
+        for file in simfdb.iterdir():
+            child = SummaryTree('Directory' if file.is_dir() else 'File')
+            child.attributes['Name'] = file.name
+            res.append(child)
+        return res
+
+    def ok(self):
+        return not self.error
+
+    def done(self):
+        if config.print_coverage:
+            for k, v in self.coverage.items():
+                child = SummaryTree('CodeCoverage')
+                child.attributes['File'] = k.file
+                child.attributes['Line'] = str(k.line)
+                if not v:
+                    child.attributes['Covered'] = '0'
+                if k.comment is not None and len(k.comment):
+                    child.attributes['Comment'] = k.comment
+                self.out.append(child)
+        if self.warnings > config.max_warnings:
+            child = SummaryTree('WarningLimitExceeded')
+            child.attributes['Severity'] = '30'
+            child.attributes['WarningCount'] = str(self.warnings)
+            self.out.append(child)
+        if self.errors > config.max_errors:
+            child = SummaryTree('ErrorLimitExceeded')
+            child.attributes['Severity'] = '40'
+            child.attributes['ErrorCount'] = str(self.errors)
+            self.out.append(child)
+        if self.was_killed:
+            child = SummaryTree('ExternalTimeout')
+            child.attributes['Severity'] = '40'
+            self.out.append(child)
+            self.error = True
+        if self.max_rss is not None:
+            self.out.attributes['PeakMemory'] = str(self.max_rss)
+        if self.valgrind_out_file is not None:
+            try:
+                valgrind_errors = parse_valgrind_output(self.valgrind_out_file)
+                for valgrind_error in valgrind_errors:
+                    if valgrind_error.kind.startswith('Leak'):
+                        continue
+                    self.error = True
+                    child = SummaryTree('ValgrindError')
+                    child.attributes['Severity'] = '40'
+                    child.attributes['What'] = valgrind_error.what.what
+                    child.attributes['Backtrace'] = valgrind_error.what.backtrace
+                    aux_count = 0
+                    for aux in valgrind_error.aux:
+                        child.attributes['WhatAux{}'.format(aux_count)] = aux.what
+                        child.attributes['BacktraceAux{}'.format(aux_count)] = aux.backtrace
+                        aux_count += 1
+                    self.out.append(child)
+            except Exception as e:
+                self.error = True
+                child = SummaryTree('ValgrindParseError')
+                child.attributes['Severity'] = '40'
+                child.attributes['ErrorMessage'] = str(e)
+                _, _, exc_traceback = sys.exc_info()
+                child.attributes['Trace'] = repr(traceback.format_tb(exc_traceback))
+                self.out.append(child)
+        if not self.test_end_found:
+            child = SummaryTree('TestUnexpectedlyNotFinished')
+            child.attributes['Severity'] = '40'
+            self.out.append(child)
+        if self.error_out is not None and len(self.error_out) > 0:
+            lines = self.error_out.splitlines()
+            stderr_bytes = 0
+            for line in lines:
+                if line.endswith("WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!"):
+                    # When running ASAN we expect to see this message. Boost coroutine should be using the correct asan annotations so that it shouldn't produce any false positives.
+                    continue
+                if line.endswith("Warning: unimplemented fcntl command: 1036"):
+                    # Valgrind produces this warning when F_SET_RW_HINT is used
+                    continue
+                if self.stderr_severity == '40':
+                    self.error = True
+                remaining_bytes = config.max_stderr_bytes - stderr_bytes
+                if remaining_bytes > 0:
+                    out_err = line[0:remaining_bytes] + ('...' if len(line) > remaining_bytes else '')
+                    child = SummaryTree('StdErrOutput')
+                    child.attributes['Severity'] = self.stderr_severity
+                    child.attributes['Output'] = out_err
+                    self.out.append(child)
+                stderr_bytes += len(line)
+            if stderr_bytes > config.max_stderr_bytes:
+                child = SummaryTree('StdErrOutputTruncated')
+                child.attributes['Severity'] = self.stderr_severity
+                child.attributes['BytesRemaining'] = str(stderr_bytes - config.max_stderr_bytes)
+                self.out.append(child)
+
+        self.out.attributes['Ok'] = '1' if self.ok() else '0'
+        if not self.ok():
+            reason = 'Unknown'
+            if self.error:
+                reason = 'ProducedErrors'
+            elif not self.test_end_found:
+                reason = 'TestDidNotFinish'
+            elif self.tests_passed == 0:
+                reason = 'NoTestsPassed'
+            elif self.test_count != self.tests_passed:
+                reason = 'Expected {} tests to pass, but only {} did'.format(self.test_count, self.tests_passed)
+            self.out.attributes['FailReason'] = reason
+
+    def parse_file(self, file: Path):
+        parser: Parser
+        if file.suffix == '.json':
+            parser = JsonParser()
+        elif file.suffix == '.xml':
+            parser = XmlParser()
+        else:
+            child = SummaryTree('TestHarnessBug')
+            child.attributes['File'] = __file__
+            frame = inspect.currentframe()
+            if frame is not None:
+                child.attributes['Line'] = str(inspect.getframeinfo(frame).lineno)
+            child.attributes['Details'] = 'Unexpected suffix {} for file {}'.format(file.suffix, file.name)
+            self.error = True
+            self.out.append(child)
+            return
+        with file.open('r') as f:
+            try:
+                parser.parse(f, self.handler)
+            except Exception as e:
+                child = SummaryTree('SummarizationError')
+                child.attributes['Severity'] = '40'
+                child.attributes['ErrorMessage'] = str(e)
+                self.out.append(child)
+
+    def register_handlers(self):
+        def remap_event_severity(attrs):
+            if 'Type' not in attrs or 'Severity' not in attrs:
+                return None
+            k = (attrs['Type'], int(attrs['Severity']))
+            if k in self.severity_map:
+                return str(self.severity_map[k])
+
+        self.handler.add_handler(('Severity', None), remap_event_severity)
+
+        def program_start(attrs: Dict[str, str]):
+            if self.test_begin_found:
+                return
+            self.test_begin_found = True
+            self.out.attributes['RandomSeed'] = attrs['RandomSeed']
+            self.out.attributes['SourceVersion'] = attrs['SourceVersion']
+            self.out.attributes['Time'] = attrs['ActualTime']
+            self.out.attributes['BuggifyEnabled'] = attrs['BuggifyEnabled']
+            self.out.attributes['DeterminismCheck'] = '0' if self.expected_unseed is None else '1'
+            if self.binary.name != 'fdbserver':
+                self.out.attributes['OldBinary'] = self.binary.name
+            if 'FaultInjectionEnabled' in attrs:
+                self.out.attributes['FaultInjectionEnabled'] = attrs['FaultInjectionEnabled']
+
+        self.handler.add_handler(('Type', 'ProgramStart'), program_start)
+
+        def set_test_file(attrs: Dict[str, str]):
+            test_file = Path(attrs['TestFile'])
+            cwd = Path('.').absolute()
+            try:
+                test_file = test_file.relative_to(cwd)
+            except ValueError:
+                pass
+            self.out.attributes['TestFile'] = str(test_file)
+
+        self.handler.add_handler(('Type', 'Simulation'), set_test_file)
+        self.handler.add_handler(('Type', 'NonSimulationTest'), set_test_file)
+
+        def set_elapsed_time(attrs: Dict[str, str]):
+            if self.test_end_found:
+                return
+            self.test_end_found = True
+            self.unseed = int(attrs['RandomUnseed'])
+            if self.expected_unseed is not None and self.unseed != self.expected_unseed:
+                severity = 40 if ('UnseedMismatch', 40) not in self.severity_map \
+                    else self.severity_map[('UnseedMismatch', 40)]
+                if severity >= 30:
+                    child = SummaryTree('UnseedMismatch')
+                    child.attributes['Unseed'] = str(self.unseed)
+                    child.attributes['ExpectedUnseed'] = str(self.expected_unseed)
+                    child.attributes['Severity'] = str(severity)
+                    if severity >= 40:
+                        self.error = True
+                    self.out.append(child)
+            self.out.attributes['SimElapsedTime'] = attrs['SimTime']
+            self.out.attributes['RealElapsedTime'] = attrs['RealTime']
+            if self.unseed is not None:
+                self.out.attributes['RandomUnseed'] = str(self.unseed)
+
+        self.handler.add_handler(('Type', 'ElapsedTime'), set_elapsed_time)
+
+        def parse_warning(attrs: Dict[str, str]):
+            self.warnings += 1
+            if self.warnings > config.max_warnings:
+                return
+            child = SummaryTree(attrs['Type'])
+            for k, v in attrs.items():
+                if k != 'Type':
+                    child.attributes[k] = v
+            self.out.append(child)
+
+        self.handler.add_handler(('Severity', '30'), parse_warning)
+
+        def parse_error(attrs: Dict[str, str]):
+            self.errors += 1
+            self.error = True
+            if self.errors > config.max_errors:
+                return
+            child = SummaryTree(attrs['Type'])
+            for k, v in attrs.items():
+                child.attributes[k] = v
+            self.out.append(child)
+
+        self.handler.add_handler(('Severity', '40'), parse_error)
+
+        def coverage(attrs: Dict[str, str]):
+            covered = True
+            if 'Covered' in attrs:
+                covered = int(attrs['Covered']) != 0
+            comment = ''
+            if 'Comment' in attrs:
+                comment = attrs['Comment']
+            c = Coverage(attrs['File'], attrs['Line'], comment)
+            if covered or c not in self.coverage:
+                self.coverage[c] = covered
+
+        self.handler.add_handler(('Type', 'CodeCoverage'), coverage)
+
+        def expected_test_pass(attrs: Dict[str, str]):
+            self.test_count = int(attrs['Count'])
+
+        self.handler.add_handler(('Type', 'TestsExpectedToPass'), expected_test_pass)
+
+        def test_passed(attrs: Dict[str, str]):
+            if attrs['Passed'] == '1':
+                self.tests_passed += 1
+
+        self.handler.add_handler(('Type', 'TestResults'), test_passed)
+
+        def remap_event_severity(attrs: Dict[str, str]):
+            self.severity_map[(attrs['TargetEvent'], int(attrs['OriginalSeverity']))] = int(attrs['NewSeverity'])
+
+        self.handler.add_handler(('Type', 'RemapEventSeverity'), remap_event_severity)
+
+        def buggify_section(attrs: Dict[str, str]):
+            if attrs['Type'] == 'FaultInjected' or attrs.get('Activated', '0') == '1':
+                child = SummaryTree(attrs['Type'])
+                child.attributes['File'] = attrs['File']
+                child.attributes['Line'] = attrs['Line']
+                self.out.append(child)
+        self.handler.add_handler(('Type', 'BuggifySection'), buggify_section)
+        self.handler.add_handler(('Type', 'FaultInjected'), buggify_section)
+
+        def running_unit_test(attrs: Dict[str, str]):
+            child = SummaryTree('RunningUnitTest')
+            child.attributes['Name'] = attrs['Name']
+            child.attributes['File'] = attrs['File']
+            child.attributes['Line'] = attrs['Line']
+        self.handler.add_handler(('Type', 'RunningUnitTest'), running_unit_test)
+
+        def stderr_severity(attrs: Dict[str, str]):
+            if 'NewSeverity' in attrs:
+                self.stderr_severity = attrs['NewSeverity']
+        self.handler.add_handler(('Type', 'StderrSeverity'), stderr_severity)
diff --git a/contrib/TestHarness2/test_harness/test_valgrind_parser.py b/contrib/TestHarness2/test_harness/test_valgrind_parser.py
new file mode 100644
index 0000000000..0b36e8e6d5
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/test_valgrind_parser.py
@@ -0,0 +1,16 @@
+import sys
+
+from test_harness.valgrind import parse_valgrind_output
+from pathlib import Path
+
+
+if __name__ == '__main__':
+    errors = parse_valgrind_output(Path(sys.argv[1]))
+    for valgrind_error in errors:
+        print('ValgrindError: what={}, kind={}'.format(valgrind_error.what.what, valgrind_error.kind))
+        print('Backtrace: {}'.format(valgrind_error.what.backtrace))
+        counter = 0
+        for aux in valgrind_error.aux:
+            print('Aux {}:'.format(counter))
+            print('  What: {}'.format(aux.what))
+            print('  Backtrace: {}'.format(aux.backtrace))
diff --git a/contrib/TestHarness2/test_harness/timeout.py b/contrib/TestHarness2/test_harness/timeout.py
new file mode 100644
index 0000000000..90af7096fd
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/timeout.py
@@ -0,0 +1,60 @@
+import argparse
+import re
+import sys
+
+from pathlib import Path
+from test_harness.config import config
+from test_harness.summarize import Summary, TraceFiles
+from typing import Pattern, List
+
+
+def files_matching(path: Path, pattern: Pattern, recurse: bool = True) -> List[Path]:
+    res: List[Path] = []
+    for file in path.iterdir():
+        if file.is_file() and pattern.match(file.name) is not None:
+            res.append(file)
+        elif file.is_dir() and recurse:
+            res += files_matching(file, pattern, recurse)
+    return res
+
+
+def dirs_with_files_matching(path: Path, pattern: Pattern, recurse: bool = True) -> List[Path]:
+    res: List[Path] = []
+    sub_directories: List[Path] = []
+    has_file = False
+    for file in path.iterdir():
+        if file.is_file() and pattern.match(file.name) is not None:
+            has_file = True
+        elif file.is_dir() and recurse:
+            sub_directories.append(file)
+    if has_file:
+        res.append(path)
+    if recurse:
+        for file in sub_directories:
+            res += dirs_with_files_matching(file, pattern, recurse=True)
+    res.sort()
+    return res
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser('TestHarness Timeout', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    config.build_arguments(parser)
+    args = parser.parse_args()
+    config.extract_args(args)
+    valgrind_files: List[Path] = []
+    if config.use_valgrind:
+        valgrind_files = files_matching(Path.cwd(), re.compile(r'valgrind.*\.xml'))
+
+    for directory in dirs_with_files_matching(Path.cwd(), re.compile(r'trace.*\.(json|xml)'), recurse=True):
+        trace_files = TraceFiles(directory)
+        for files in trace_files.items():
+            if config.use_valgrind:
+                for valgrind_file in valgrind_files:
+                    summary = Summary(Path('bin/fdbserver'), was_killed=True)
+                    summary.valgrind_out_file = valgrind_file
+                    summary.summarize_files(files)
+                    summary.out.dump(sys.stdout)
+            else:
+                summary = Summary(Path('bin/fdbserver'), was_killed=True)
+                summary.summarize_files(files)
+                summary.out.dump(sys.stdout)
diff --git a/contrib/TestHarness2/test_harness/valgrind.py b/contrib/TestHarness2/test_harness/valgrind.py
new file mode 100644
index 0000000000..399b47c0cc
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/valgrind.py
@@ -0,0 +1,141 @@
+import enum
+import xml
+import xml.sax.handler
+from pathlib import Path
+from typing import List
+
+
+class ValgrindWhat:
+    def __init__(self):
+        self.what: str = ''
+        self.backtrace: str = ''
+
+
+class ValgrindError:
+    def __init__(self):
+        self.what: ValgrindWhat = ValgrindWhat()
+        self.kind: str = ''
+        self.aux: List[ValgrindWhat] = []
+
+
+# noinspection PyArgumentList
+class ValgrindParseState(enum.Enum):
+    ROOT = enum.auto()
+    ERROR = enum.auto()
+    ERROR_AUX = enum.auto()
+    KIND = enum.auto()
+    WHAT = enum.auto()
+    TRACE = enum.auto()
+    AUX_WHAT = enum.auto()
+    STACK = enum.auto()
+    STACK_AUX = enum.auto()
+    STACK_IP = enum.auto()
+    STACK_IP_AUX = enum.auto()
+
+
+class ValgrindHandler(xml.sax.handler.ContentHandler):
+    def __init__(self):
+        super().__init__()
+        self.stack: List[ValgrindError] = []
+        self.result: List[ValgrindError] = []
+        self.state_stack: List[ValgrindParseState] = []
+
+    def state(self) -> ValgrindParseState:
+        if len(self.state_stack) == 0:
+            return ValgrindParseState.ROOT
+        return self.state_stack[-1]
+
+    @staticmethod
+    def from_content(content):
+        # pdb.set_trace()
+        if isinstance(content, bytes):
+            return content.decode()
+        assert isinstance(content, str)
+        return content
+
+    def characters(self, content):
+        # pdb.set_trace()
+        state = self.state()
+        if len(self.state_stack) == 0:
+            return
+        else:
+            assert len(self.stack) > 0
+        if state is ValgrindParseState.KIND:
+            self.stack[-1].kind += self.from_content(content)
+        elif state is ValgrindParseState.WHAT:
+            self.stack[-1].what.what += self.from_content(content)
+        elif state is ValgrindParseState.AUX_WHAT:
+            self.stack[-1].aux[-1].what += self.from_content(content)
+        elif state is ValgrindParseState.STACK_IP:
+            self.stack[-1].what.backtrace += self.from_content(content)
+        elif state is ValgrindParseState.STACK_IP_AUX:
+            self.stack[-1].aux[-1].backtrace += self.from_content(content)
+
+    def startElement(self, name, attrs):
+        # pdb.set_trace()
+        if name == 'error':
+            self.stack.append(ValgrindError())
+            self.state_stack.append(ValgrindParseState.ERROR)
+        if len(self.stack) == 0:
+            return
+        if name == 'kind':
+            self.state_stack.append(ValgrindParseState.KIND)
+        elif name == 'what':
+            self.state_stack.append(ValgrindParseState.WHAT)
+        elif name == 'auxwhat':
+            assert self.state() in [ValgrindParseState.ERROR, ValgrindParseState.ERROR_AUX]
+            self.state_stack.pop()
+            self.state_stack.append(ValgrindParseState.ERROR_AUX)
+            self.state_stack.append(ValgrindParseState.AUX_WHAT)
+            self.stack[-1].aux.append(ValgrindWhat())
+        elif name == 'stack':
+            state = self.state()
+            assert state in [ValgrindParseState.ERROR, ValgrindParseState.ERROR_AUX]
+            if state == ValgrindParseState.ERROR:
+                self.state_stack.append(ValgrindParseState.STACK)
+            else:
+                self.state_stack.append(ValgrindParseState.STACK_AUX)
+        elif name == 'ip':
+            state = self.state()
+            assert state in [ValgrindParseState.STACK, ValgrindParseState.STACK_AUX]
+            if state == ValgrindParseState.STACK:
+                self.state_stack.append(ValgrindParseState.STACK_IP)
+                if len(self.stack[-1].what.backtrace) == 0:
+                    self.stack[-1].what.backtrace = 'addr2line -e fdbserver.debug -p -C -f -i '
+                else:
+                    self.stack[-1].what.backtrace += ' '
+            else:
+                self.state_stack.append(ValgrindParseState.STACK_IP_AUX)
+                if len(self.stack[-1].aux[-1].backtrace) == 0:
+                    self.stack[-1].aux[-1].backtrace = 'addr2line -e fdbserver.debug -p -C -f -i '
+                else:
+                    self.stack[-1].aux[-1].backtrace += ' '
+
+    def endElement(self, name):
+        # pdb.set_trace()
+        if name == 'error':
+            self.result.append(self.stack.pop())
+            self.state_stack.pop()
+        elif name == 'kind':
+            assert self.state() == ValgrindParseState.KIND
+            self.state_stack.pop()
+        elif name == 'what':
+            assert self.state() == ValgrindParseState.WHAT
+            self.state_stack.pop()
+        elif name == 'auxwhat':
+            assert self.state() == ValgrindParseState.AUX_WHAT
+            self.state_stack.pop()
+        elif name == 'stack':
+            assert self.state() in [ValgrindParseState.STACK, ValgrindParseState.STACK_AUX]
+            self.state_stack.pop()
+        elif name == 'ip':
+            self.state_stack.pop()
+            state = self.state()
+            assert state in [ValgrindParseState.STACK, ValgrindParseState.STACK_AUX]
+
+
+def parse_valgrind_output(valgrind_out_file: Path) -> List[ValgrindError]:
+    handler = ValgrindHandler()
+    with valgrind_out_file.open('r') as f:
+        xml.sax.parse(f, handler)
+        return handler.result
diff --git a/contrib/TestHarness2/test_harness/version.py b/contrib/TestHarness2/test_harness/version.py
new file mode 100644
index 0000000000..fe04206a8a
--- /dev/null
+++ b/contrib/TestHarness2/test_harness/version.py
@@ -0,0 +1,66 @@
+from functools import total_ordering
+from pathlib import Path
+from typing import Tuple
+
+
+@total_ordering
+class Version:
+    def __init__(self):
+        self.major: int = 0
+        self.minor: int = 0
+        self.patch: int = 0
+
+    def version_tuple(self):
+        return self.major, self.minor, self.patch
+
+    def _compare(self, other) -> int:
+        lhs: Tuple[int, int, int] = self.version_tuple()
+        rhs: Tuple[int, int, int]
+        if isinstance(other, Version):
+            rhs = other.version_tuple()
+        else:
+            rhs = Version.parse(str(other)).version_tuple()
+        if lhs < rhs:
+            return -1
+        elif lhs > rhs:
+            return 1
+        else:
+            return 0
+
+    def __eq__(self, other) -> bool:
+        return self._compare(other) == 0
+
+    def __lt__(self, other) -> bool:
+        return self._compare(other) < 0
+
+    def __hash__(self):
+        return hash(self.version_tuple())
+
+    def __str__(self):
+        return format('{}.{}.{}'.format(self.major, self.minor, self.patch))
+
+    @staticmethod
+    def of_binary(binary: Path):
+        parts = binary.name.split('-')
+        if len(parts) != 2:
+            return Version.max_version()
+        return Version.parse(parts[1])
+
+    @staticmethod
+    def parse(version: str):
+        version_tuple = version.split('.')
+        self = Version()
+        self.major = int(version_tuple[0])
+        if len(version_tuple) > 1:
+            self.minor = int(version_tuple[1])
+            if len(version_tuple) > 2:
+                self.patch = int(version_tuple[2])
+        return self
+
+    @staticmethod
+    def max_version():
+        self = Version()
+        self.major = 2**32 - 1
+        self.minor = 2**32 - 1
+        self.patch = 2**32 - 1
+        return self
diff --git a/contrib/crc32/CMakeLists.txt b/contrib/crc32/CMakeLists.txt
index f93697c1a9..3fb87e1901 100644
--- a/contrib/crc32/CMakeLists.txt
+++ b/contrib/crc32/CMakeLists.txt
@@ -1,2 +1,8 @@
 add_library(crc32 STATIC crc32.S crc32_wrapper.c crc32c.cpp)
+if (CLANG)
+	# This is necessary for clang since the compiler reports that crc32_align is
+	# defined but not used. With -Werror, crc32 will not compile.
+	# TODO: Remove this when the upstream issue is repaired.
+	target_compile_options(crc32 PUBLIC -Wno-unused-function)
+endif ()
 target_include_directories(crc32 PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
diff --git a/contrib/ddsketch_calc.py b/contrib/ddsketch_calc.py
new file mode 100644
index 0000000000..b113cb37dc
--- /dev/null
+++ b/contrib/ddsketch_calc.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+#
+# ddsketch_calc.py
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import numpy as np
+import math as m
+
+
+# Implements a DDSketch class as desrcibed in:
+# https://arxiv.org/pdf/1908.10693.pdf
+
+# This class has methods that use cubic interpolation to quickly compute log
+# and inverse log. The coefficients A,B,C as well as correctingFactor are
+# all constants used for interpolating. 
+
+# The implementation for interpolation was originally seen here in:
+# https://github.com/DataDog/sketches-java/
+# in the file CubicallyInterpolatedMapping.java
+
+class DDSketch(object):
+    A = 6.0 / 35.0
+    B = -3.0 / 5.0
+    C = 10.0 / 7.0
+    EPS = 1e-18
+    correctingFactor = 1.00988652862227438516
+    offset = 0
+    multiplier = 0
+    gamma = 0
+
+    def __init__(self, errorGuarantee):
+        self.gamma = (1 + errorGuarantee) / (1 - errorGuarantee)
+        self.multiplier = (self.correctingFactor * m.log(2)) / m.log(self.gamma)
+        self.offset = self.getIndex(1.0 / self.EPS)
+
+    def fastlog(self, value):
+        s = np.frexp(value)
+        e = s[1]
+        s = s[0]
+        s = s * 2 - 1
+        return ((self.A * s + self.B) * s + self.C) * s + e - 1
+
+    def reverseLog(self, index):
+        exponent = m.floor(index)
+        d0 = self.B * self.B - 3 * self.A * self.C
+        d1 = 2 * self.B * self.B * self.B - 9 * self.A * self.B * self.C - 27 * self.A * self.A * (index - exponent)
+        p = np.cbrt((d1 - np.sqrt(d1 * d1 - 4 * d0 * d0 * d0)) / 2)
+        significandPlusOne = - (self.B + p + d0 / p) / (3 * self.A) + 1
+        return np.ldexp(significandPlusOne / 2, exponent + 1)
+
+    def getIndex(self, sample):
+        return m.ceil(self.fastlog(sample) * self.multiplier) + self.offset
+
+    def getValue(self, idx):
+        return self.reverseLog((idx - self.offset) / self.multiplier) * 2.0 / (1 + self.gamma)
+
diff --git a/contrib/ddsketch_compare.py b/contrib/ddsketch_compare.py
new file mode 100644
index 0000000000..d3b5f9942f
--- /dev/null
+++ b/contrib/ddsketch_compare.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+#
+# ddsketch_compare.py
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import json
+import numpy as np
+
+
+# kullback-leibler divergence (or relative entropy)
+def relative_entropy(p, q):
+    difference = 0.0
+    for i in range(len(p)):
+        if p[i] != 0.0 and q[i] != 0.0:
+            difference += (p[i] * np.log2(p[i]/q[i]))
+    return difference
+
+# jensen-shannon divergence (or symmetric relative entropy)
+def relative_entropy_symmetric(dd1, dd2):
+    # normalize p, q into distribution
+    sum1 = sum(dd1)
+    sum2 = sum(dd2)
+
+    p = [dd1[i] / sum1 for i in range(len(dd1))]
+    q = [dd2[i] / sum2 for i in range(len(dd2))]
+    m = [0.5 * (p[i] + q[i]) for i in range(len(p))]
+
+    return 0.5 * relative_entropy(p, m) + 0.5 * relative_entropy(q, m)
+
+# setup cmdline args
+parser = argparse.ArgumentParser(description="Compares two DDSketch distributions")
+parser.add_argument('--txn1', help='Transaction type for first file', required=True, type=str)
+parser.add_argument('--txn2', help='Transaction type for second file', required=True, type=str)
+parser.add_argument('--file1', help='Path to first ddsketch json', required=True, type=str)
+parser.add_argument('--file2', help="Path to second ddsketch json'", required=True, type=str)
+parser.add_argument("--op", help='Operation name', type=str)
+args = parser.parse_args()
+
+f1 = open(args.file1)
+f2 = open(args.file2)
+data1 = json.load(f1)
+data2 = json.load(f2)
+
+if data1[args.txn1][args.op]["errorGuarantee"] != data2[args.txn2][args.op]["errorGuarantee"]:
+    print("ERROR: The sketches have different error guarantees and cannot be compared!")
+    exit()
+
+b1 = data1[args.txn1][args.op]["buckets"]
+b2 = data2[args.txn2][args.op]["buckets"]
+
+re = relative_entropy_symmetric(b1, b2)
+print("The similarity is: ", round(re, 8))
+print("1 means least alike, 0 means most alike")
\ No newline at end of file
diff --git a/contrib/ddsketch_conversion.py b/contrib/ddsketch_conversion.py
new file mode 100644
index 0000000000..5b9825f267
--- /dev/null
+++ b/contrib/ddsketch_conversion.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+#
+# ddsketch_conversion.py
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import ddsketch_calc as dd
+
+
+parser = argparse.ArgumentParser(description="Converts values to DDSketch buckets")
+parser.add_argument('-e', '--error_guarantee', help='Error guarantee (default is 0.005)', required=False, type=float)
+parser.add_argument('-v', '--value', help="Value", required=False, type=int)
+parser.add_argument('-b', '--bucket', help='Bucket index', required=False, type=int)
+args = parser.parse_args()
+
+error = 0.005
+
+if args.error_guarantee is not None:
+    error = args.error_guarantee
+
+sketch = dd.DDSketch(error)
+
+if args.value is not None:
+    print("Bucket index for ", args.value)
+    print(sketch.getIndex(args.value))
+
+if args.bucket is not None:
+    print("Value for bucket ", args.bucket)
+    print(sketch.getValue(args.bucket))
\ No newline at end of file
diff --git a/contrib/export_graph.py b/contrib/export_graph.py
new file mode 100644
index 0000000000..9cd369f7de
--- /dev/null
+++ b/contrib/export_graph.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+#
+# export_graph.py
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import json
+import matplotlib.pyplot as plt
+import argparse
+import ddsketch_calc as dd
+
+# setup cmdline args
+parser = argparse.ArgumentParser(description="Graphs DDSketch distribution")
+parser.add_argument('-t', '--txn', help='Transaction type (ex: g8ui)', required=True, type=str)
+parser.add_argument('--file', help='Path to ddsketch json', required=True, type=str)
+parser.add_argument('--title', help='Title for the graph', required=False, type=str)
+parser.add_argument('--savefig', help='Will save the plot to a file if set', type=str)
+parser.add_argument('--op', help='Which OP to plot (casing matters)', type=str)
+args = parser.parse_args()
+
+
+# Opening JSON file
+f = open(args.file)
+data = json.load(f)
+
+# parse json and init sketch
+buckets = data[args.t][args.op]["buckets"]
+error = data[args.t][args.op]["errorGuarantee"]
+sketch = dd.DDSketch(error)
+
+# trim the tails of the distribution
+ls = [i for i, e in enumerate(buckets) if e != 0]
+actual_data = buckets[ls[0]:ls[-1]+1]
+indices = range(ls[0], ls[-1]+1)
+actual_indices = [sketch.getValue(i) for i in indices]
+
+# configure the x-axis to make more sense
+fig, ax = plt.subplots()
+ax.ticklabel_format(useOffset=False, style='plain')
+plt.plot(actual_indices, actual_data)
+plt.xlabel("Latency (in us)")
+plt.ylabel("Frequency count")
+
+plt_title = "Title"
+if args.title is not None:
+    plt_title = args.title
+plt.title(plt_title)
+plt.xlim([actual_indices[0], actual_indices[-1]])
+if args.savefig is not None:
+    plt.savefig(args.savefig, format='png')
+else:   
+    plt.show()
\ No newline at end of file
diff --git a/contrib/libb64/CMakeLists.txt b/contrib/libb64/CMakeLists.txt
new file mode 100644
index 0000000000..1ef665f079
--- /dev/null
+++ b/contrib/libb64/CMakeLists.txt
@@ -0,0 +1,2 @@
+add_library(libb64 STATIC cdecode.c cencode.c)
+target_include_directories(libb64 PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
diff --git a/fdbclient/libb64/cdecode.c b/contrib/libb64/cdecode.c
similarity index 56%
rename from fdbclient/libb64/cdecode.c
rename to contrib/libb64/cdecode.c
index f10ecc3dcc..5a833ab689 100644
--- a/fdbclient/libb64/cdecode.c
+++ b/contrib/libb64/cdecode.c
@@ -5,18 +5,18 @@ This is part of the libb64 project, and has been placed in the public domain.
 For details, see http://sourceforge.net/projects/libb64
 */
 
-#include "fdbclient/libb64/cdecode.h"
+#include "libb64/cdecode.h"
 
-int base64_decode_value(char value_in) {
-	static const char decoding[] = { 62, -1, -1, -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1,
-		                             -1, -1, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17,
-		                             18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31,
-		                             32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 };
-	static const char decoding_size = sizeof(decoding);
+int base64_decode_value(int value_in) {
+	static const int decoding[] = { 62, -1, -1, -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1,
+		                            -1, -1, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17,
+		                            18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31,
+		                            32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 };
+	static const int decoding_size = sizeof(decoding) / sizeof(decoding[0]);
 	value_in -= 43;
-	if (value_in < 0 || value_in > decoding_size)
+	if (value_in < 0 || value_in >= decoding_size)
 		return -1;
-	return decoding[(int)value_in];
+	return decoding[value_in];
 }
 
 void base64_init_decodestate(base64_decodestate* state_in) {
@@ -27,7 +27,7 @@ void base64_init_decodestate(base64_decodestate* state_in) {
 int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in) {
 	const char* codechar = code_in;
 	char* plainchar = plaintext_out;
-	char fragment;
+	int fragment = 0;
 
 	*plainchar = state_in->plainchar;
 
@@ -40,9 +40,9 @@ int base64_decode_block(const char* code_in, const int length_in, char* plaintex
 					state_in->plainchar = *plainchar;
 					return plainchar - plaintext_out;
 				}
-				fragment = (char)base64_decode_value(*codechar++);
+				fragment = base64_decode_value(*codechar++);
 			} while (fragment < 0);
-			*plainchar = (fragment & 0x03f) << 2;
+			*plainchar = (char)((fragment & 0x03f) << 2);
 		case step_b:
 			do {
 				if (codechar == code_in + length_in) {
@@ -50,10 +50,10 @@ int base64_decode_block(const char* code_in, const int length_in, char* plaintex
 					state_in->plainchar = *plainchar;
 					return plainchar - plaintext_out;
 				}
-				fragment = (char)base64_decode_value(*codechar++);
+				fragment = base64_decode_value(*codechar++);
 			} while (fragment < 0);
-			*plainchar++ |= (fragment & 0x030) >> 4;
-			*plainchar = (fragment & 0x00f) << 4;
+			*plainchar++ |= (char)((fragment & 0x030) >> 4);
+			*plainchar = (char)((fragment & 0x00f) << 4);
 		case step_c:
 			do {
 				if (codechar == code_in + length_in) {
@@ -61,10 +61,10 @@ int base64_decode_block(const char* code_in, const int length_in, char* plaintex
 					state_in->plainchar = *plainchar;
 					return plainchar - plaintext_out;
 				}
-				fragment = (char)base64_decode_value(*codechar++);
+				fragment = base64_decode_value(*codechar++);
 			} while (fragment < 0);
-			*plainchar++ |= (fragment & 0x03c) >> 2;
-			*plainchar = (fragment & 0x003) << 6;
+			*plainchar++ |= (char)((fragment & 0x03c) >> 2);
+			*plainchar = (char)((fragment & 0x003) << 6);
 		case step_d:
 			do {
 				if (codechar == code_in + length_in) {
@@ -72,9 +72,9 @@ int base64_decode_block(const char* code_in, const int length_in, char* plaintex
 					state_in->plainchar = *plainchar;
 					return plainchar - plaintext_out;
 				}
-				fragment = (char)base64_decode_value(*codechar++);
+				fragment = base64_decode_value(*codechar++);
 			} while (fragment < 0);
-			*plainchar++ |= (fragment & 0x03f);
+			*plainchar++ |= (char)((fragment & 0x03f));
 		}
 	}
 	/* control should not reach here */
diff --git a/fdbclient/libb64/cencode.c b/contrib/libb64/cencode.c
similarity index 98%
rename from fdbclient/libb64/cencode.c
rename to contrib/libb64/cencode.c
index 7999b47d61..85e679c7c8 100644
--- a/fdbclient/libb64/cencode.c
+++ b/contrib/libb64/cencode.c
@@ -5,7 +5,7 @@ This is part of the libb64 project, and has been placed in the public domain.
 For details, see http://sourceforge.net/projects/libb64
 */
 
-#include "fdbclient/libb64/cencode.h"
+#include "libb64/cencode.h"
 
 const int CHARS_PER_LINE = 72;
 
diff --git a/fdbclient/include/fdbclient/libb64/cdecode.h b/contrib/libb64/include/libb64/cdecode.h
similarity index 93%
rename from fdbclient/include/fdbclient/libb64/cdecode.h
rename to contrib/libb64/include/libb64/cdecode.h
index 26d5873f22..04655b3a95 100644
--- a/fdbclient/include/fdbclient/libb64/cdecode.h
+++ b/contrib/libb64/include/libb64/cdecode.h
@@ -17,7 +17,7 @@ typedef struct {
 
 void base64_init_decodestate(base64_decodestate* state_in);
 
-int base64_decode_value(char value_in);
+int base64_decode_value(int value_in);
 
 int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in);
 
diff --git a/fdbclient/include/fdbclient/libb64/cencode.h b/contrib/libb64/include/libb64/cencode.h
similarity index 100%
rename from fdbclient/include/fdbclient/libb64/cencode.h
rename to contrib/libb64/include/libb64/cencode.h
diff --git a/fdbclient/include/fdbclient/libb64/decode.h b/contrib/libb64/include/libb64/decode.h
similarity index 98%
rename from fdbclient/include/fdbclient/libb64/decode.h
rename to contrib/libb64/include/libb64/decode.h
index 84a3998d7a..696e3f1a4d 100644
--- a/fdbclient/include/fdbclient/libb64/decode.h
+++ b/contrib/libb64/include/libb64/decode.h
@@ -9,6 +9,7 @@ For details, see http://sourceforge.net/projects/libb64
 #define BASE64_DECODE_H
 
 #include <iostream>
+#include "libb64/encode.h"
 
 namespace base64 {
 extern "C" {
diff --git a/fdbclient/include/fdbclient/libb64/encode.h b/contrib/libb64/include/libb64/encode.h
similarity index 100%
rename from fdbclient/include/fdbclient/libb64/encode.h
rename to contrib/libb64/include/libb64/encode.h
diff --git a/contrib/md5/CMakeLists.txt b/contrib/md5/CMakeLists.txt
new file mode 100644
index 0000000000..317065b5c1
--- /dev/null
+++ b/contrib/md5/CMakeLists.txt
@@ -0,0 +1,2 @@
+add_library(md5 STATIC md5.c)
+target_include_directories(md5 PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
diff --git a/fdbclient/include/fdbclient/md5/md5.h b/contrib/md5/include/md5/md5.h
similarity index 100%
rename from fdbclient/include/fdbclient/md5/md5.h
rename to contrib/md5/include/md5/md5.h
diff --git a/fdbclient/md5/md5.c b/contrib/md5/md5.c
similarity index 100%
rename from fdbclient/md5/md5.c
rename to contrib/md5/md5.c
diff --git a/contrib/observability_splunk_dashboard/details.xml b/contrib/observability_splunk_dashboard/details.xml
new file mode 100644
index 0000000000..70ff15883b
--- /dev/null
+++ b/contrib/observability_splunk_dashboard/details.xml
@@ -0,0 +1,431 @@
+<form theme="light">
+  <label>FoundationDB - Details</label>
+  <description>Details for FoundationDB Cluster</description>
+  <fieldset submitButton="false">
+    <input type="text" token="Index" searchWhenChanged="true">
+      <label>Index</label>
+      <default>*</default>
+    </input>
+    <input type="text" token="LogGroup" searchWhenChanged="true">
+      <label>LogGroup</label>
+      <default>*</default>
+    </input>
+    <input type="time" token="TimeRange" searchWhenChanged="true">
+      <label>Time Range</label>
+      <default>
+        <earliest>-60m@m</earliest>
+        <latest>now</latest>
+      </default>
+    </input>
+    <input type="dropdown" token="Span" searchWhenChanged="true">
+      <label>Timechart Resolution</label>
+      <choice value="bins=100">Default</choice>
+      <choice value="span=5s">5 seconds</choice>
+      <choice value="span=1m">1 minute</choice>
+      <choice value="span=10m">10 minutes</choice>
+      <choice value="span=1h">1 hour</choice>
+      <choice value="span=1d">1 day</choice>
+      <default>bins=100</default>
+      <initialValue>bins=100</initialValue>
+    </input>
+    <input type="dropdown" token="Roles" searchWhenChanged="true">
+      <label>Roles</label>
+      <choice value="">All</choice>
+      <choice value="Roles=*SS*">Storage Server</choice>
+      <choice value="Roles=*TL*">Transaction Log</choice>
+      <choice value="Roles=*MP*">Proxy</choice>
+      <choice value="Roles=*RV*">Resolver</choice>
+      <choice value="Roles=*MS*">Master</choice>
+      <choice value="Roles=*CC*">Cluster Controller</choice>
+      <choice value="Roles=*LR*">Log Router</choice>
+      <choice value="Roles=*DD*">Data Distributor</choice>
+      <choice value="Roles=*RK*">Ratekeeper</choice>
+      <choice value="Roles=*TS*">Tester</choice>
+      <default></default>
+    </input>
+    <input type="text" token="Host" searchWhenChanged="true">
+      <label>Host</label>
+      <default>*</default>
+    </input>
+    <input type="text" token="Machine" searchWhenChanged="true">
+      <label>Machine</label>
+      <default>*</default>
+    </input>
+  </fieldset>
+  <row>
+    <panel>
+      <chart>
+        <title>Storage Queue Size</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=StorageMetrics $Roles$ host=$Host$ Machine=$Machine$ TrackLatestType=Original | rex field=BytesInput "(?&lt;InputRate&gt;.*) (?&lt;InputRoughness&gt;.*) (?&lt;InputCounter&gt;.*)" | rex field=BytesDurable "(?&lt;DurableRate&gt;.*) (?&lt;DurableRoughness&gt;.*) (?&lt;DurableCounter&gt;.*)" | eval QueueSize=InputCounter-DurableCounter | timechart $Span$ avg(QueueSize) by Machine</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>Storage Input Rate</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=StorageMetrics $Roles$ host=$Host$ Machine=$Machine$ TrackLatestType=Original | rex field=BytesInput "(?&lt;InputRate&gt;.*) (?&lt;InputRoughness&gt;.*) (?&lt;InputCounter&gt;.*)" | timechart $Span$ avg(InputRate) by Machine</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>Storage Bytes Queried</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=StorageMetrics $Roles$ host=$Host$ Machine=$Machine$ TrackLatestType=Original | rex field=BytesQueried "(?&lt;Rate&gt;.*) (?&lt;Roughness&gt;.*) (?&lt;Counter&gt;.*)" | timechart $Span$ avg(Rate) by Machine</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <chart>
+        <title>Average Process CPU by Role (capped at 2; beware kernel bug)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics $Roles$ host=$Host$ Machine=$Machine$ TrackLatestType=Original | eval Cpu=CPUSeconds/Elapsed | timechart $Span$ avg(Cpu) by Roles</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.axisY.maximumNumber">2</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>Max Process CPU by Role (capped at 2; beware kernel bug)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics $Roles$ host=$Host$ Machine=$Machine$ TrackLatestType=Original | eval Cpu=CPUSeconds/Elapsed | timechart $Span$ max(Cpu) by Roles</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.axisY.maximumNumber">2</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>Disk Busyness</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ $Roles$ host=$Host$ Machine=$Machine$ Type=ProcessMetrics TrackLatestType=Original | eval DiskBusyPercentage=(Elapsed-DiskIdleSeconds)/Elapsed | timechart $Span$ avg(DiskBusyPercentage) by Machine</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <chart>
+        <title>Max Run Loop Busyness by Role (for &lt;=6.1, S2Pri1)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ $Roles$ host=$Host$ Machine=$Machine$ Type=NetworkMetrics NOT TrackLatestType=Rolled | eval Busyness=if(isnull(PriorityStarvedBelow1), if(isnull(PriorityBusy1), S2Pri1, PriorityBusy1/Elapsed), PriorityStarvedBelow1/Elapsed) | timechart $Span$ max(Busyness) by Roles</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>Max Run Loop Busyness by Priority (6.2+ only)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ $Roles$ host=$Host$ Machine=$Machine$ Type=NetworkMetrics TrackLatestType=Original | foreach PriorityBusy* [eval Busyness&lt;&lt;MATCHSTR&gt;&gt;=PriorityBusy&lt;&lt;MATCHSTR&gt;&gt;/Elapsed] | timechart $Span$ max(Busyness*)</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>TLog Queue Size</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=TLogMetrics $Roles$ host=$Host$ Machine=$Machine$ TrackLatestType=Original | eval QueueSize=SharedBytesInput-SharedBytesDurable | timechart $Span$ avg(QueueSize) by Machine</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <chart>
+        <title>Connection Timeouts (counted on both sides of connection)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ (Type=ConnectionTimeout OR Type=ConnectionTimedOut) $Roles$ host=$Host$ | eval WithAddr=if(Type=="ConnectionTimedOut", PeerAddr, WithAddr) | rex field=WithAddr "(?&lt;OtherAddr&gt;[^:]*:[^:]*).*" | eval Machine=Machine+","+OtherAddr | makemv delim="," Machine | search Machine=$Machine$ | eval Count=1+SuppressedEventCount | timechart sum(Count) by Machine useother=f</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.chart.nullValueMode">zero</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>Pairwise Connection Timeouts Between Datacenters</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ (Type=ConnectionTimeout OR Type=ConnectionTimedOut)  host=* Machine=* NOT TrackLatestType=Rolled 
+| eval WithAddr=if(Type=="ConnectionTimedOut", PeerAddr, WithAddr) 
+| rex field=host "(?&lt;Datacenter&gt;..).*" 
+| eval Datacenter=if(isnotnull(pie_work_unit), pie_work_unit, Datacenter) 
+| rex field=WithAddr "(?&lt;OtherIP&gt;[^:]*):.*" 
+| join OtherIP 
+    [search index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics NOT TrackLatestType=Rolled 
+    | rex field=Machine "(?&lt;OtherIP&gt;[^:]*):.*" 
+    | rex field=host "(?&lt;OtherDatacenter&gt;..).*"
+    | eval OtherDatacenter=if(isnotnull(pie_work_unit), pie_work_unit, OtherDatacenter)]
+| eval DC1=if(Datacenter&gt;OtherDatacenter, Datacenter, OtherDatacenter), DC2=if(Datacenter&gt;OtherDatacenter, OtherDatacenter, Datacenter) 
+| eval Connection=DC1+" &lt;-&gt; " + DC2 
+| eval Count=1+SuppressedEventCount 
+| timechart count by Connection</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <table>
+        <title>Pairwise Connection Timeouts Between Known Server Processes (Sorted by Count, descending)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ (Type=ConnectionTimeout OR Type=ConnectionTimedOut OR Type=ProcessMetrics) $Roles$ host=$Host$ Machine=$Machine$ NOT TrackLatestType=Rolled | eval WithAddr=if(Type=="ConnectionTimedOut", PeerAddr, WithAddr), Reason=if(Type=="ConnectionTimedOut", "Timed out trying to connect", "Established connection timed out") | rex field=Machine "(?&lt;IP&gt;[^:]*):.*" | rex field=host "(?&lt;Datacenter&gt;..).*" | rex field=WithAddr "(?&lt;OtherIP&gt;[^:]*):.*" | eventstats values(Roles) as Roles by IP | join OtherIP [search index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics NOT TrackLatestType=Rolled | rex field=Machine "(?&lt;OtherIP&gt;[^:]*):.*" | rex field=host "(?&lt;OtherDatacenter&gt;..).*" | stats values(Roles) as OtherRoles by OtherIP, OtherDatacenter | eval OtherRoles="("+mvjoin(OtherRoles,",")+")"] | eval Roles="("+mvjoin(Roles,",")+")" | eval IP=Datacenter+": "+IP+" "+Roles, OtherIP=OtherDatacenter+": "+OtherIP+" "+OtherRoles | eval Addr1=if(IP&gt;OtherIP, IP, OtherIP), Addr2=if(IP&gt;OtherIP, OtherIP, IP) | eval Connection=Addr1+" &lt;-&gt; " + Addr2 | eval Count=1+SuppressedEventCount | stats sum(Count) as Count, values(Reason) as Reasons by Connection | sort -Count</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <chart>
+        <title>Lazy Deletion Rate (making space available for reuse)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ $Roles$ host=$Host$ Machine=$Machine$ Type=SpringCleaningMetrics | eval Metric=LazyDeletePages | streamstats current=f global=f window=1 first(Metric) as NextMetric, first(Time), as NextTime by ID | eval Rate=4096*(NextMetric-Metric)/(NextTime-Time) | timechart $Span$ avg(Rate) by Machine</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>Vacuuming Rate (shrinking file)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ $Roles$ host=$Host$ Machine=$Machine$ Type=SpringCleaningMetrics | eval Metric=VacuumedPages | streamstats current=f global=f window=1 first(Metric) as NextMetric, first(Time), as NextTime by ID | eval Rate=4096*(NextMetric-Metric)/(NextTime-Time) | timechart $Span$ avg(Rate) by Machine</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>Roles</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ $Roles$ host=$Host$ Machine=$Machine$ NOT TrackLatestType=Rolled | makemv delim="," Roles | mvexpand Roles | timechart $Span$ distinct_count(Machine) by Roles</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <table>
+        <title>Slow Tasks (Sorted by Duration, Descending)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=SlowTask $Roles$ host=$Host$ Machine=$Machine$ | sort -Duration | table _time, Duration, Machine, TaskID, Roles</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+    <panel>
+      <table>
+        <title>Event Counts (Sorted by Severity and Count, Descending)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ $Roles$ host=$Host$ Machine=$Machine$ NOT TrackLatestType=Rolled | stats count as Count by Type, Severity | sort -Severity, -Count</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+    <panel>
+      <table>
+        <title>Errors</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Severity=40 $Roles$ host=$Host$ Machine=$Machine$ NOT TrackLatestType=Rolled | table _time, Type, Machine, Roles</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <table>
+        <title>Recoveries (Ignores Filters)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=MasterRecoveryState TrackLatestType=Original (StatusCode=0 OR StatusCode=11) | eval RecoveryResetInterval=10 | sort _time | streamstats earliest(_time) as RecoveryStart, count as EventCount reset_after="(StatusCode=11)" | where StatusCode=11 | eval EventCount=if(EventCount==1, 2, EventCount), RecoveryStart=if(RecoveryStart==_time, _time-RecoveryDuration, RecoveryStart) | sort -_time | streamstats current=f global=f window=1 first(RecoveryStart) as NextRecoveryStart | eval RecoverySpan=NextRecoveryStart-_time, FailedRecoveries=EventCount-2, SuccessfulRecoveries=1 | eval AvailableSeconds=if(RecoverySpan&lt;RecoveryResetInterval, RecoverySpan, 0) | sort _time | streamstats earliest(RecoveryStart) as RecoveryStart, sum(FailedRecoveries) as FailedRecoveryCount, sum(SuccessfulRecoveries) as SuccessfulRecoveryCount, sum(AvailableSeconds) as AvailableSeconds reset_after="(NOT RecoverySpan &lt; RecoveryResetInterval)"  | where NOT RecoverySpan &lt; RecoveryResetInterval | eval Duration=_time-RecoveryStart, StartTime=strftime(RecoveryStart, "%F %X.%Q"), ShortLivedRecoveryCount=SuccessfulRecoveryCount-1 | table StartTime, Duration, FailedRecoveryCount, ShortLivedRecoveryCount, AvailableSeconds | sort -StartTime</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+    <panel>
+      <table>
+        <title>Process (Re)starts</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=ProgramStart TrackLatestType=Original $Roles$ host=$Host$ Machine=$Machine$ | table _time, Machine | sort -_time</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+    <panel>
+      <chart>
+        <title>Failure Detection (Machine Filter Only)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=FailureDetectionStatus System=$Machine$ | sort _time | eval Failed=if(Status=="Failed", 1, 0) | streamstats current=t global=f window=2 first(Failed) as PrevFailed by System | where PrevFailed=1 OR Failed=1 | eval Failed=PrevFailed + "," + Failed | makemv delim="," Failed | mvexpand Failed | timechart $Span$ max(Failed) by System</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.axisY.maximumNumber">1</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <table>
+        <title>Storage Server Space Usage (Sorted by Available Space Percentage, Ascending)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=StorageMetrics $Roles$ host=$Host$ Machine=$Machine$ TrackLatestType=Original | eval AvailableSpacePercent=KvstoreBytesAvailable/KvstoreBytesTotal, FreeSpacePercent=KvstoreBytesFree/KvstoreBytesTotal, GBUsed=KvstoreBytesUsed/1e9, GBStored=BytesStored/1e9, Overhead=KvstoreBytesUsed/BytesStored, GBTotalSpace=KvstoreBytesTotal/1e9 | stats latest(AvailableSpacePercent) as AvailableSpacePercent, latest(FreeSpacePercent) as FreeSpacePercent, latest(GBStored) as GBStored, latest(GBUsed) as GBUsed, latest(Overhead) as OverheadFactor, latest(GBTotalSpace) as GBTotalSpace by Machine | sort AvailableSpacePercent</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <table>
+        <title>TLog Server Space Usage (Sorted by Available Space Percentage, Ascending)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$  Type=TLogMetrics  host=* Machine=* TrackLatestType=Original Roles=TL | eval AvailableSpacePercent=KvstoreBytesAvailable/KvstoreBytesTotal, FreeDiskSpacePercent=KvstoreBytesFree/KvstoreBytesTotal, GBUsed=KvstoreBytesUsed/1e9,  GBTotalSpace=KvstoreBytesTotal/1e9  | stats latest(AvailableSpacePercent) as AvailableSpacePercent, latest(FreeDiskSpacePercent) as FreeDiskSpacePercent,  latest(GBUsed) as GBUsed, latest(GBTotalSpace) as GBTotalSpace by Machine | sort AvailableSpacePercent</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <chart>
+        <title>Data Movement by Type (Log Scale, Ignores Filters)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=MovingData TrackLatestType=Original | timechart avg(Priority*) as *</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <chart>
+        <title>Storage Server Max Bytes Stored by Host</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=StorageMetrics $Roles$ host=$Host$ Machine=$Machine$ TrackLatestType=Original | eval GBStored=BytesStored/1e9 | timechart max(GBStored) by host limit=100</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <table>
+        <title>Master Failed Clients</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$  Type=WaitFailureClient 
+| stats count by FailedEndpoint</query>
+          <earliest>$TimeRange.earliest$</earliest>
+          <latest>$TimeRange.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+      </table>
+    </panel>
+  </row>
+</form>
\ No newline at end of file
diff --git a/contrib/observability_splunk_dashboard/performance_overview.xml b/contrib/observability_splunk_dashboard/performance_overview.xml
new file mode 100644
index 0000000000..0719e2bbab
--- /dev/null
+++ b/contrib/observability_splunk_dashboard/performance_overview.xml
@@ -0,0 +1,323 @@
+<form theme="dark">
+  <label>FoundationDB - Performance Overview (Dev WiP)</label>
+  <fieldset submitButton="false" autoRun="true">
+    <input type="text" token="Index" searchWhenChanged="true">
+      <label>Index</label>
+      <default>*</default>
+    </input>
+    <input type="text" token="LogGroup" searchWhenChanged="true">
+      <label>LogGroup</label>
+      <default></default>
+    </input>
+    <input type="time" token="TimeSpan" searchWhenChanged="true">
+      <label>TimeSpan</label>
+      <default>
+        <earliest>-60m@m</earliest>
+        <latest>now</latest>
+      </default>
+    </input>
+    <input type="dropdown" token="UpdateRateTypeToken" searchWhenChanged="true">
+      <label>RK: Normal or Batch Txn</label>
+      <choice value="">Normal</choice>
+      <choice value="Batch">Batch</choice>
+      <default></default>
+    </input>
+    <input type="text" token="ChartBinSizeToken" searchWhenChanged="true">
+      <label>Chart Bin Size</label>
+      <default>60s</default>
+    </input>
+  </fieldset>
+  <row>
+    <panel>
+      <title>Transaction Rate measured on Proxies</title>
+      <chart>
+        <title>Sum in $ChartBinSizeToken$ seconds</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ host=* Machine=*  (Type="ProxyMetrics" OR Type="GrvProxyMetrics") AND TrackLatestType="Original" 
+| makemv delim=" " TxnRequestIn | makemv delim=" " TxnRequestOut | makemv delim=" " TxnStartIn | makemv delim=" " TxnStartOut | makemv delim=" " TxnThrottled
+| eval TxnRequestInRate=mvindex(TxnRequestIn, 0), TxnRequestOutRate=mvindex(TxnRequestOut, 0), TxnStartInRate=mvindex(TxnStartIn, 0), TxnStartOutRate=mvindex(TxnStartOut, 0), TxnThrottledRate=mvindex(TxnThrottled, 0)
+| timechart span=$ChartBinSizeToken$ sum(TxnRequestInRate) as StartedTxnBatchRate, sum(TxnRequestOutRate) as FinishedTxnBatchRate, sum(TxnStartInRate) as StartedTxnRate, sum(TxnStartOutRate) as FinishedTxnRate, sum(TxnThrottledRate) as ThrottledTxnRate</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Read Rate measured on Storage Servers</title>
+      <chart>
+        <title>Average in $ChartBinSizeToken$ seconds</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=StorageMetrics TrackLatestType="Original" 
+| rex field=BytesQueried "(?&lt;RRate&gt;.*) (?&lt;RRoughness&gt;.*) (?&lt;RCounter&gt;.*)" 
+| rex field=RowsQueried "(?&lt;KRate&gt;.*) (?&lt;KRoughness&gt;.*) (?&lt;KCounter&gt;.*)" 
+| rex field=BytesInput "(?&lt;WRate&gt;.*) (?&lt;WRoughness&gt;.*) (?&lt;WCounter&gt;.*)" 
+| rex field=BytesFetched "(?&lt;FRate&gt;.*) (?&lt;FRoughness&gt;.*) (?&lt;FCounter&gt;.*)" 
+| timechart span=$ChartBinSizeToken$ avg(RRate) as BytesReadPerSecond, avg(KRate) as RowsReadPerSecond, avg(FRate) as DDReadPerSecond</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Write Rate measured on Proxies</title>
+      <chart>
+        <title>1min Average</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ host=* Machine=*  (Type="ProxyMetrics" OR Type="GrvProxyMetrics") AND TrackLatestType="Original" 
+| makemv delim=" " MutationBytes
+| makemv delim=" " Mutations
+| eval MutationBytesRate=mvindex(MutationBytes, 0), MutationsRate=mvindex(Mutations,0)
+| bucket span=5s _time
+| stats sum(MutationBytesRate) as MutationBytes, sum(MutationsRate) as Mutations by _time
+|eval MutationMB=MutationBytes/1024/1024, MutationsK=Mutations/1000
+| timechart span=$ChartBinSizeToken$ avg(MutationMB) as MutationMB, avg(MutationsK) as MutationsK</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.abbreviation">none</option>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="charting.layout.splitSeries">0</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Write Rate measured on Storage Servers</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=StorageMetrics TrackLatestType="Original" 
+| rex field=BytesInput "(?&lt;WRate&gt;.*) (?&lt;WRoughness&gt;.*) (?&lt;WCounter&gt;.*)" 
+| rex field=BytesFetched "(?&lt;FRate&gt;.*) (?&lt;FRoughness&gt;.*) (?&lt;FCounter&gt;.*)" 
+| timechart span=$ChartBinSizeToken$ avg(WRate) as BytesPerSecond, avg(FRate) as DDBytesWrittenPerSecond</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>GRV Latency measured on all Proxies</title>
+      <chart>
+        <title>Seconds</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=GRVLatencyMetrics AND TrackLatestType="Original"
+| timechart span=$ChartBinSizeToken$ avg(Max) as maxLatency, avg(Mean) as meanLatency, avg(P99) as P99Latency, avg(P99.9) as P999Latency, avg(P95) as P95Latency</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="charting.legend.placement">bottom</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Commit Latency measured on all Proxies</title>
+      <chart>
+        <title>Seconds</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$  Type=CommitLatencyMetrics AND TrackLatestType="Original"
+| timechart span=$ChartBinSizeToken$ avg(Max) as maxLatency, avg(Mean) as meanLatency, avg(P99) as P99Latency, avg(P99.9) as P999Latency, avg(P95) as P95Latency</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="charting.legend.placement">bottom</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Read Latency measured on all Storage Servers</title>
+      <chart>
+        <title>Seconds</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$  Type=ReadLatencyMetrics AND TrackLatestType="Original"
+| timechart span=$ChartBinSizeToken$ avg(Max) as maxLatency, avg(Mean) as meanLatency, avg(P99) as P99Latency, avg(P99.9) as P999Latency, avg(P95) as P95Latency</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="charting.legend.placement">bottom</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>RateKeeper: ReleasedTPS vs LimitTPS</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=RkUpdate$UpdateRateTypeToken$ AND TrackLatestType="Original" 
+| replace inf with 100000000000 
+| eval _time=Time
+| table _time ReleasedTPS TPSLimit
+| timechart span=$ChartBinSizeToken$ avg(ReleasedTPS) avg(TPSLimit)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="height">251</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>RateKeeper: Throttling Reason</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=RkUpdate$UpdateRateTypeToken$ AND TrackLatestType="Original" 
+| replace inf with 100000000000 
+| eval _time=Time
+| table _time Reason</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisLabelsY.majorUnit">1</option>
+        <option name="charting.axisY.abbreviation">none</option>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">area</option>
+        <option name="charting.drilldown">none</option>
+        <option name="charting.legend.mode">standard</option>
+        <option name="height">249</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>RateKeeper: Throttling Server</title>
+      <table>
+        <title>Ratekeeper: Limit Reason: ReasonServerID (Most recent 10 records)</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=RkUpdate AND TrackLatestType="Original" 
+| streamstats count as numOfEvents 
+| where numOfEvents &lt; 10
+| eval DateTime=strftime(Time, "%Y-%m-%dT%H:%M:%S")
+| table DateTime, ReasonServerID</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Disk Overhead = Disk Usage / Logical KV Size</title>
+      <chart>
+        <title>Y-axis is capped at 10</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ host=* Machine=*  (Type=StorageMetrics OR Type=DDTrackerStats) TrackLatestType=Original
+| bucket _time span=5s 
+| stats sum(KvstoreBytesUsed) as StorageDiskUsedBytes, sum(KvstoreBytesTotal) as StorageDiskTotalBytes, avg(TotalSizeBytes) as LogicalKVBytes by _time
+| eval overhead=StorageDiskUsedBytes/LogicalKVBytes
+| timechart avg(overhead)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.maximumNumber">10</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="charting.legend.placement">bottom</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>KV Data Size</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+Roles=*DD* host=* Machine=*  Type=DDTrackerStats TrackLatestType=Original
+| eval TotalKVGB=TotalSizeBytes/1024/1024/1024, SystemKVGB=SystemSizeBytes/1024/1024/1024
+|timechart avg(TotalKVGB), avg(SystemKVGB), avg(Shards)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="charting.legend.placement">bottom</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Disk Usage</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ host=* Machine=*  Type=StorageMetrics TrackLatestType=Original
+| bucket _time span=5s 
+| stats sum(KvstoreBytesUsed) as StorageDiskUsedBytes, sum(KvstoreBytesTotal) as StorageDiskTotalBytes by _time
+|eval StorageDiskTotalMB = StorageDiskTotalBytes/1024/1024, StorageDiskUsedMB=StorageDiskUsedBytes/1024/1024
+| timechart avg(StorageDiskTotalMB) as StorageDiskTotalMB, avg(StorageDiskUsedMB) as StorageDiskUsedMB</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="charting.legend.placement">bottom</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Cluster Roles</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics TrackLatestType="Original"
+| rex field=host "(?&lt;HostDC&gt;..).*-..(?&lt;HostConfig&gt;..).*"
+| eval HostDC=if(isnotnull(pie_work_unit), pie_work_unit, HostDC) 
+| makemv delim="," Roles
+| stats dc(Machine) as MachineCount by Roles, HostDC
+| stats list(HostDC), list(MachineCount) by Roles
+| sort Roles</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Storage Engine</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=Role Origination=Recruited As=StorageServer | table StorageEngine, OriginalDateTime, DateTime |head 2</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+    <panel>
+      <title>Cluster Generations</title>
+      <chart>
+        <title>Indicate FDB recoveries</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=TLogMetrics |timechart max(Generation)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+</form>
\ No newline at end of file
diff --git a/contrib/observability_splunk_dashboard/ratekeeper.xml b/contrib/observability_splunk_dashboard/ratekeeper.xml
new file mode 100644
index 0000000000..c4a31a8fbc
--- /dev/null
+++ b/contrib/observability_splunk_dashboard/ratekeeper.xml
@@ -0,0 +1,928 @@
+<form theme="dark">
+  <label>FoundationDB - RateKeeper (Dev)</label>
+  <fieldset submitButton="false">
+    <input type="text" token="Index" searchWhenChanged="true">
+      <label>Index</label>
+      <default>*</default>
+    </input>
+    <input type="text" token="LogGroup" searchWhenChanged="true">
+      <label>LogGroup</label>
+      <default></default>
+    </input>
+    <input type="time" token="TimeSpan" searchWhenChanged="true">
+      <label>TimeSpan</label>
+      <default>
+        <earliest>-60m@m</earliest>
+        <latest>now</latest>
+      </default>
+    </input>
+    <input type="dropdown" token="UpdateRateTypeToken" searchWhenChanged="true">
+      <label>RKChart: Normal or Batch</label>
+      <choice value="">Normal</choice>
+      <choice value="Batch">Batch</choice>
+      <default></default>
+    </input>
+    <input type="text" token="ChartBinSizeToken" searchWhenChanged="true">
+      <label>Chart Bin Size</label>
+      <default>30s</default>
+    </input>
+    <input type="dropdown" token="ChartByMachineToken" searchWhenChanged="true">
+      <label>ClusterStateMetric byMachine</label>
+      <choice value="by Machine">Yes</choice>
+      <choice value="">No</choice>
+      <default></default>
+    </input>
+    <input type="dropdown" token="RolePerformanceChartToken" searchWhenChanged="true">
+      <label>Role for Proc Perf Charts</label>
+      <choice value="MasterServer">MasterServer</choice>
+      <choice value="MasterProxyServer">MasterProxyServer</choice>
+      <choice value="StorageServer">StorageServer</choice>
+      <choice value="TLog">TLog</choice>
+      <choice value="Resolver">Resolver</choice>
+      <choice value="GrvProxyServer">GrvProxyServer</choice>
+      <choice value="CommitProxyServer">CommitProxyServer</choice>
+    </input>
+    <input type="dropdown" token="SourcePerfConnectionToken" searchWhenChanged="true">
+      <label>Source for Perf Connection</label>
+      <choice value="MasterServer">MasterServer</choice>
+      <choice value="MasterProxyServer">MasterProxyServer</choice>
+      <choice value="Resolver">Resolver</choice>
+      <choice value="TLog">TLog</choice>
+      <choice value="StorageServer">StorageServer</choice>
+      <choice value="GrvProxyServer">GrvProxyServer</choice>
+      <choice value="CommitProxyServer">CommitProxyServer</choice>
+    </input>
+    <input type="dropdown" token="DestinationPerfConnectionToken" searchWhenChanged="true">
+      <label>Dest for Perf Connection</label>
+      <choice value="MasterServer">MasterServer</choice>
+      <choice value="MasterProxyServer">MasterProxyServer</choice>
+      <choice value="Resolver">Resolver</choice>
+      <choice value="TLog">TLog</choice>
+      <choice value="StorageServer">StorageServer</choice>
+      <choice value="GrvProxyServer">GrvProxyServer</choice>
+      <choice value="CommitProxyServer">CommitProxyServer</choice>
+    </input>
+  </fieldset>
+  <row>
+    <panel>
+      <title>Aggregated Storage Server Bandwidth</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=StorageMetrics TrackLatestType="Original" 
+        | rex field=BytesQueried "(?&lt;RRate&gt;.*) (?&lt;RRoughness&gt;.*) (?&lt;RCounter&gt;.*)" 
+         |  rex field=BytesInput "(?&lt;WRate&gt;.*) (?&lt;WRoughness&gt;.*) (?&lt;WCounter&gt;.*)" 
+          | rex field=BytesFetched "(?&lt;FRate&gt;.*) (?&lt;FRoughness&gt;.*) (?&lt;FCounter&gt;.*)" 
+          | bin span=5s _time 
+          | stats sum(RRate) as ReadSum, sum(WRate) as WriteSum, sum(FRate) as FetchedKeyRate by _time
+          | eval ReadSpeedMB=ReadSum/1024/1024, WriteSpeedMB=WriteSum/1024/1024, FetchedKeyRateMB=FetchedKeyRate/1024/1024
+          |timechart avg(ReadSpeedMB), avg(WriteSpeedMB), avg(FetchedKeyRateMB)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Aggregated Proxy Bandwidth</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ (Type="ProxyMetrics" OR Type="GrvProxyMetrics") AND TrackLatestType="Original" 
+| makemv delim=" " TxnRequestIn | makemv delim=" " TxnRequestOut | makemv delim=" " TxnStartIn | makemv delim=" " TxnStartOut | makemv delim=" " MutationBytes
+| eval TxnRequestInRate=mvindex(TxnRequestIn, 0), TxnRequestOutRate=mvindex(TxnRequestOut, 0), TxnStartInRate=mvindex(TxnStartIn, 0), TxnStartOutRate=mvindex(TxnStartOut, 0), MutationBytesRate=mvindex(MutationBytes, 0)
+| bin span=60s _time
+| stats avg(TxnRequestInRate) as TxnRequestInRatePerHost, avg(TxnRequestOutRate) as TxnRequestOutRatePerHost, avg(TxnStartInRate) as TxnStartInRatePerHost, avg(TxnStartOutRate) as TxnStartOutRatePerHost, avg(MutationBytesRate) as MutationBytesRatePerHost by Machine,_time
+| eval WriteThroughputKB=sum(MutationBytesRatePerHost)/1000  
+| timechart span=1m sum(TxnRequestInRatePerHost), sum(TxnRequestOutRatePerHost), sum(TxnStartInRatePerHost), sum(TxnStartOutRatePerHost), sum(WriteThroughputKB)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 1: Overview - GRV Arrivals and Leaves per Second Seen by Proxies</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ (Type="ProxyMetrics" OR Type="GrvProxyMetrics") AND TrackLatestType="Original" 
+| eval TxnRequestIn=mvindex(TxnRequestIn, 0), TxnRequestOut=mvindex(TxnRequestOut, 0), TxnStartIn=mvindex(TxnStartIn, 0), TxnStartOut=mvindex(TxnStartOut, 0) 
+| timechart span=30s avg(TxnRequestIn) avg(TxnRequestOut) avg(TxnStartIn) avg(TxnStartOut) by Machine</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="height">249</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 2: RKOverview - Input ReleasedTPS and Output TPSLimit</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=RkUpdate$UpdateRateTypeToken$ AND TrackLatestType="Original" 
+| replace inf with 100000000000 
+| eval _time=Time
+| table _time ReleasedTPS TPSLimit
+| timechart span=$ChartBinSizeToken$ avg(ReleasedTPS) avg(TPSLimit)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="height">251</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 3: RKOverview - RKLimitReason</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=RkUpdate$UpdateRateTypeToken$ AND TrackLatestType="Original" 
+| replace inf with 100000000000 
+| eval _time=Time
+| table _time Reason</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisLabelsY.majorUnit">1</option>
+        <option name="charting.axisY.abbreviation">none</option>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">area</option>
+        <option name="charting.drilldown">none</option>
+        <option name="height">249</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 4: Don't Process Transactions - RkSSListFetchTimeout (TpsLimit = 0)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ 
+Type="RkSSListFetchTimeout" 
+| timechart span=1s count</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 5: Don't Process Transactions - RkTlogMinFreeSpaceZero (TpsLimit = 0)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ 
+Type="RkTlogMinFreeSpaceZero" 
+| timechart span=1s count</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 6: Don't Process Transactions - ProxyGRVThresholdExceeded</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ (Type="ProxyGRVThresholdExceeded*") AND TrackLatestType="Original" 
+| timechart span=1s count by Type</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 7: RKLimitReasonCandidate - LimitingStorageServerDurabilityLag (MVCCVersionInMemory)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=RkUpdate$UpdateRateTypeToken$ AND TrackLatestType="Original" 
+| replace inf with 100000000000 
+| timechart span=$ChartBinSizeToken$ avg(LimitingStorageServerDurabilityLag)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 8: RKLimitReasonCandidate - LimitingStorageServerVersionLag (TLogVer-SSVer)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=RkUpdate$UpdateRateTypeToken$ AND TrackLatestType="Original" 
+| replace inf with 100000000000 
+| timechart span=$ChartBinSizeToken$ avg(LimitingStorageServerVersionLag)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 9: RKLimitReasonCandidate - LimitingStorageServerQueue</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=RkUpdate$UpdateRateTypeToken$ AND TrackLatestType="Original" 
+| replace inf with 100000000000 
+| timechart span=$ChartBinSizeToken$ avg(LimitingStorageServerQueue)</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 10: Runtime Monitoring - StorageServer MVCCVersionInMemory (storage_server_durability_lag)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="StorageMetrics" AND TrackLatestType="Original" 
+| eval NonDurableVersions=Version-DurableVersion
+| timechart span=$ChartBinSizeToken$ limit=0 avg(NonDurableVersions) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="height">251</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 11: Runtime Monitoring - StorageServer LocalRate (higher MVCCVersionInMemory -&gt; lower LocalRate)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="StorageMetrics" 
+| timechart limit=0 avg(LocalRate) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 12: Runtime Monitoring - StorageServer ReadsRejected (lower LocalRate -&gt; higher probability of rejecting read))</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="StorageMetrics" 
+| timechart limit=0 avg(ReadsRejected) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 13: Runtime Monitoring - Version Lag between StorageServer and Tlog (storage_server_readable_behind)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="StorageMetrics" AND TrackLatestType="Original" 
+| eval SSFallBehindVersions=VersionLag
+| timechart span=$ChartBinSizeToken$ limit=0 avg(SSFallBehindVersions) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 14: Runtime Monitoring - StorageServerBytes (storage_server_write_queue_size)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="StorageMetrics" AND TrackLatestType="Original" 
+| makemv delim=" " BytesInput | makemv delim=" " BytesDurable | makemv delim=" " BytesFetched | makemv delim=" " MutationBytes
+| eval BytesInput=mvindex(BytesInput, 2), BytesDurable=mvindex(BytesDurable, 2), BytesFetched=mvindex(BytesFetched, 2), MutationBytes=mvindex(MutationBytes, 2), BytesInMemoryQueue=BytesInput-BytesDurable
+| timechart span=$ChartBinSizeToken$ limit=0 avg(BytesInMemoryQueue) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 15: Runtime Monitoring - StorageServer KVStore Free Space Ratio (storage_server_min_free_space)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="StorageMetrics" AND TrackLatestType="Original" 
+| eval KvstoreBytesFreeRatio=KvstoreBytesFree/KvstoreBytesTotal
+| timechart span=$ChartBinSizeToken$ limit=0 avg(KvstoreBytesFreeRatio) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 16: Runtime Monitoring - TLog Queue Free Space Ratio (log_server_min_free_space)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="TLogMetrics" AND TrackLatestType="Original" 
+| eval QueueBytesFreeRatio=QueueDiskBytesFree/QueueDiskBytesTotal
+| timechart span=$ChartBinSizeToken$ limit=0 avg(QueueBytesFreeRatio) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 17: Runtime Monitoring - TLog KVStore Free Space Ratio (log_server_min_free_space)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="TLogMetrics" AND TrackLatestType="Original" 
+| eval KvstoreBytesFreeRatio=KvstoreBytesFree/KvstoreBytesTotal
+| timechart span=$ChartBinSizeToken$ limit=0 avg(KvstoreBytesFreeRatio) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 18: Runtime Monitoring - TLogBytes (log_server_write_queue)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="TLogMetrics" AND TrackLatestType="Original" 
+| makemv delim=" " BytesInput 
+| makemv delim=" " BytesDurable 
+| eval BytesInput=mvindex(BytesInput, 2), BytesDurable=mvindex(BytesDurable, 2), BytesInMemoryQueue=BytesInput-BytesDurable | timechart span=$ChartBinSizeToken$ limit=0 avg(BytesInMemoryQueue) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 19: Runtime Monitoring - Proxy Throughput</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ (Type="ProxyMetrics" OR Type="GrvProxyMetrics") AND TrackLatestType="Original" 
+| timechart span=$ChartBinSizeToken$ limit=0 avg(TxnRequestIn) avg(TxnRequestOut) avg(TxnStartIn) avg(TxnStartOut) avg(TxnStartBatch) avg(TxnStartErrors) avg(TxnCommitIn) avg(TxnCommitVersionAssigned) avg(TxnCommitResolving) avg(TxnCommitResolved) avg(TxnCommitOut) avg(TxnCommitOutSuccess) avg(TxnCommitErrors) avg(TxnThrottled) avg(TxnConflicts) avg(CommitBatchIn) avg(CommitBatchOut) avg(TxnRejectedForQueuedTooLong) avg(Mutations)  $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 20: Runtime Monitoring - Proxy Queue Length</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ (Type="ProxyMetrics" OR Type="GrvProxyMetrics") AND TrackLatestType="Original" | timechart span=$ChartBinSizeToken$ limit=0 avg(*QueueSize*)  $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 21: Runtime Monitoring - TLog UnpoppedVersion</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="TLogMetrics" AND TrackLatestType="Original" 
+| eval UnpoppedVersion=PersistentDataDurableVersion-QueuePoppedVersion 
+| timechart span=$ChartBinSizeToken$ limit=0 avg(UnpoppedVersion) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 22: Runtime Monitoring - Storage Server Disk (AIODiskStall)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="ProcessMetrics" 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="Role" AND As="StorageServer" 
+    | stats first(Machine) by Machine 
+    | rename first(Machine) as Machine 
+    | table Machine] 
+| timechart span=$ChartBinSizeToken$ limit=0 avg(AIODiskStall) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 23: Runtime Monitoring - StorageServer Query Queue Length</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="StorageMetrics" AND TrackLatestType="Original" 
+| makemv QueryQueue | eval QueryQueue=mvindex(QueryQueue, 1) | table _time QueryQueue Machine
+| timechart span=$ChartBinSizeToken$ limit=0 avg(QueryQueue) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 24: Transaction Trace Stats - GRV Latency (only show CC transactions by default; it shows client transactions only when you manually open client transaction trace)</title>
+      <input type="dropdown" token="GRVByMachineStatsToken" searchWhenChanged="true">
+        <label>By Machine</label>
+        <choice value="Machine">Yes</choice>
+        <choice value="">No</choice>
+        <default></default>
+      </input>
+      <input type="text" token="StatsGRVSpanToken" searchWhenChanged="true">
+        <label>Span</label>
+        <default>500ms</default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="TransactionDebug" AND (*ProxyServer.masterProxyServerCore.Broadcast OR *ProxyServer.getLiveCommittedVersion.confirmEpochLive OR *ProxyServer.getLiveCommittedVersion.After) 
+| table Time Type ID Location Machine Roles 
+| append 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="TransactionDebug" AND (*ProxyServer.queueTransactionStartRequests.Before) 
+    | rename ID as ParentID 
+    | table Time Type ParentID Location Machine Roles 
+    | join ParentID 
+        [ search index=$Index$ LogGroup=$LogGroup$ Type="TransactionAttachID" 
+        | rename ID as ParentID 
+        | rename To as ID 
+        | table ParentID ID] 
+    | table Time Type ID Location Machine Roles] 
+| table Time Type ID Location Machine Roles 
+| sort 0 Time 
+| table Machine Location Time Roles Type ID 
+| stats list(*) by ID 
+| rename list(*) as * 
+| eval TBegin=mvindex(Time, 0), TEnd=mvindex(Time, -1), TimeSpan=TEnd-TBegin, _time=TBegin 
+| bin bins=20 span=$StatsGRVSpanToken$ TimeSpan 
+| chart limit=0 count by TimeSpan $GRVByMachineStatsToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">column</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 25: Transaction Trace Stats - GetValue Latency (only show CC transactions by default; it shows client transactions only when you manually open client transaction trace)</title>
+      <input type="dropdown" token="GetValueByMachineStatsToken" searchWhenChanged="true">
+        <label>By Machine</label>
+        <choice value="Machine">Yes</choice>
+        <choice value="">No</choice>
+        <default></default>
+      </input>
+      <input type="text" token="StatsReadSpanToken" searchWhenChanged="true">
+        <label>Span</label>
+        <default>500ms</default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (storageServer.received OR getValueQ.DoRead OR getValueQ.AfterVersion OR Reader.Before OR Reader.After OR getValueQ.AfterRead OR NativeAPI.getKeyLocation.Before OR NativeAPI.getKeyLocation.After) 
+| table Machine Location Time Roles ID Type 
+| eval Order=case(Location=="NativeAPI.getKeyLocation.Before", 0, Location=="NativeAPI.getKeyLocation.After", 1, Location=="NativeAPI.getValue.Before", 2, Location=="storageServer.received", 3, Location=="getValueQ.DoRead", 4, Location=="getValueQ.AfterVersion", 5, Location=="Reader.Before", 6, Location=="Reader.After", 7, Location=="getValueQ.AfterRead", 8, Location=="NativeAPI.getValue.After", 9, Location=="NativeAPI.getValue.Error", 10) 
+| sort 0 Time Order 
+| stats list(*) by ID 
+| rename list(*) as * 
+| table Machine Location Time Roles ID Type 
+| eval count = mvcount(Location)
+| search count&gt;2
+| eval TEnd=mvindex(Time, -1), TBegin=mvindex(Time, 0), TimeSpan=TEnd-TBegin, _time=TBegin
+| table _time ID TimeSpan Machine Location Time 
+| bin bins=20 span=$StatsReadSpanToken$ TimeSpan 
+| chart limit=0 count by TimeSpan $GetValueByMachineStatsToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">column</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 26: Transaction Trace Stats - Commit Latency (only show CC transactions by default; it shows client transactions only when you manually open client transaction trace)</title>
+      <input type="dropdown" token="CommitByMachineStatsToken">
+        <label>By Machine</label>
+        <choice value="Machine">Yes</choice>
+        <choice value="">No</choice>
+        <default>Machine</default>
+      </input>
+      <input type="text" token="StatsCommitSpanToken" searchWhenChanged="true">
+        <label>Span</label>
+        <default>500ms</default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="CommitDebug" AND (*ProxyServer.commitBatch.Before OR *ProxyServer.commitBatch.GettingCommitVersion OR *ProxyServer.commitBatch.GotCommitVersion OR *ProxyServer.commitBatch.ProcessingMutations OR *ProxyServer.commitBatch.AfterStoreCommits OR *ProxyServer.commitBatch.AfterLogPush OR *ProxyServer.commitBatch.AfterResolution) 
+| table Time Type ID Location Machine Roles 
+| sort 0 Time 
+| table Machine Location Time Roles Type ID
+| stats list(*) by ID
+| rename list(*) as *
+| eval Count=mvcount(Location)
+| search Count&gt;=2
+| eval TBegin=mvindex(Time, 0), TEnd=mvindex(Time, -1), TimeSpan=TEnd-TBegin, _time=T1
+| table _time TimeSpan Machine
+| bin bins=20 span=$StatsCommitSpanToken$ TimeSpan 
+| chart limit=0 count by TimeSpan $CommitByMachineStatsToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">column</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 27: Transaction Tracing - GRV Latency (only show CC transactions by default; it shows client transactions only when you manually open client transaction trace)</title>
+      <input type="dropdown" token="GRVLatencyByMachineToken" searchWhenChanged="true">
+        <label>By Machine</label>
+        <choice value="by Machine">Yes</choice>
+        <choice value="">No</choice>
+        <default></default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="TransactionDebug" AND (*ProxyServer.*ProxyServerCore.Broadcast OR *ProxyServer.getLiveCommittedVersion.confirmEpochLive OR *ProxyServer.getLiveCommittedVersion.After) 
+| table Time Type ID Location Machine Roles 
+| append 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="TransactionDebug" AND (*ProxyServer.queueTransactionStartRequests.Before) 
+    | rename ID as ParentID 
+    | table Time Type ParentID Location Machine Roles 
+    | join ParentID 
+        [ search index=$Index$ LogGroup=$LogGroup$ Type="TransactionAttachID" 
+        | rename ID as ParentID 
+        | rename To as ID 
+        | table ParentID ID] 
+    | table Time Type ID Location Machine Roles] 
+| table Time Type ID Location Machine Roles 
+| eval Order = case(Location=="NativeAPI.getConsistentReadVersion.Before", 0, Location like "%ProxyServer.queueTransactionStartRequests.Before", 1, Location="MasterProxyServer.masterProxyServerCore.Broadcast", 2, Location like "%ProxyServer.getLiveCommittedVersion.confirmEpochLive", 3, Location like "%ProxyServer.getLiveCommittedVersion.After", 5, Location=="NativeAPI.getConsistentReadVersion.After", 6) 
+| table Time Order Type ID Location Machine Roles 
+| sort 0 Order Time 
+| table Machine Location Time Roles Type ID 
+| stats list(*) by ID 
+| rename list(*) as * 
+| eval T1=mvindex(Time, 0), T2=mvindex(Time, 1), T3=mvindex(Time, 2), T4=mvindex(Time, 3), TimeInQueue = T2-T1, TimeGetVersionFromProxies = if(mvcount==4, T3-T2, -0.0000001), TimeConfirmLivenessFromTLogs = if(mvcount==4, T4-T3, T3-T2), TimeSpan=if(mvcount==4,T4-T1,T3-T1), _time=T1 
+| table _time TimeSpan TimeInQueue TimeGetVersionFromProxies TimeConfirmLivenessFromTLogs Machine 
+| timechart span=$ChartBinSizeToken$ limit=0 avg(TimeSpan), avg(TimeInQueue), avg(TimeGetVersionFromProxies), avg(TimeConfirmLivenessFromTLogs) $GRVLatencyByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 28: Transaction Tracing - GetValue Latency (only show CC transactions by default; it shows client transactions only when you manually open client transaction trace)</title>
+      <input type="dropdown" token="GetValueLatencyByMachineToken" searchWhenChanged="true">
+        <label>By Machine</label>
+        <choice value="by Machine">Yes</choice>
+        <choice value="">No</choice>
+        <default></default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (storageServer.received OR getValueQ.DoRead OR getValueQ.AfterVersion OR Reader.Before OR Reader.After OR getValueQ.AfterRead OR NativeAPI.getKeyLocation.Before OR NativeAPI.getKeyLocation.After) 
+| table Machine Location Time Roles ID Type 
+| eval Order=case(Location=="NativeAPI.getKeyLocation.Before", 0, Location=="NativeAPI.getKeyLocation.After", 1, Location=="NativeAPI.getValue.Before", 2, Location=="storageServer.received", 3, Location=="getValueQ.DoRead", 4, Location=="getValueQ.AfterVersion", 5, Location=="Reader.Before", 6, Location=="Reader.After", 7, Location=="getValueQ.AfterRead", 8, Location=="NativeAPI.getValue.After", 9, Location=="NativeAPI.getValue.Error", 10) 
+| sort 0 Time Order 
+| stats list(*) by ID 
+| rename list(*) as * 
+| table Machine Location Time Roles ID Type 
+| eval count = mvcount(Location)
+| search count&gt;2
+| eval TEnd=mvindex(Time, -1), TBegin=mvindex(Time, 0), TimeSpan=TEnd-TBegin, _time=TBegin
+| table _time TimeSpan  
+| timechart span=30s limit=0 avg(TimeSpan) $GetValueLatencyByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 29: Transaction Tracing - Commit Latency (only show CC transactions by default; it shows client transactions only when you manually open client transaction trace)</title>
+      <input type="dropdown" token="CommitByMachineToken" searchWhenChanged="true">
+        <label>By Machine</label>
+        <choice value="By Machine">Yes</choice>
+        <choice value="">No</choice>
+        <default></default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="CommitDebug" AND (*ProxyServer.commitBatch.Before OR *ProxyServer.commitBatch.GettingCommitVersion OR *ProxyServer.commitBatch.GotCommitVersion OR *ProxyServer.commitBatch.ProcessingMutations OR *ProxyServer.commitBatch.AfterStoreCommits OR *ProxyServer.commitBatch.AfterLogPush OR *ProxyServer.commitBatch.AfterResolution) 
+| table Time Type ID Location Machine Roles 
+| eval Order=case(Location=="NativeAPI.commit.Before", 0, Location like "%ProxyServer.batcher", 1, Location like "%ProxyServer.commitBatch.Before", 2, Location like "%ProxyServer.commitBatch.GettingCommitVersion", 3, Location like "%ProxyServer.commitBatch.GotCommitVersion", 4, Location=="Resolver.resolveBatch.Before", 5, Location=="Resolver.resolveBatch.AfterQueueSizeCheck", 6, Location=="Resolver.resolveBatch.AfterOrderer", 7, Location=="Resolver.resolveBatch.After", 8, Location like "%ProxyServer.commitBatch.AfterResolution", 8.5, Location like "%ProxyServer.commitBatch.ProcessingMutations", 9, Location like "%ProxyServer.commitBatch.AfterStoreCommits", 10, Location=="TLog.tLogCommit.BeforeWaitForVersion", 11, Location=="TLog.tLogCommit.Before", 12, Location=="TLog.tLogCommit.AfterTLogCommit", 13, Location=="TLog.tLogCommit.After", 14, Location like "%ProxyServer.commitBatch.AfterLogPush", 15, Location=="NativeAPI.commit.After", 16)
+| table Time Order Type ID Location Machine Roles 
+| sort 0 Time Order 
+| table Machine Location Time Roles Type ID
+| stats list(*) by ID
+| rename list(*) as *
+| eval Count=mvcount(Location)
+| search Count=7
+| eval T1=mvindex(Time, 0), T2=mvindex(Time, 1), T3=mvindex(Time, 2), T4=mvindex(Time, 3), T5=mvindex(Time, 4), T6=mvindex(Time, 5), T7=mvindex(Time, 6), TimeSpan=T7-T1, TimeResolution=T4-T3, TimePostResolution=T5-T4, TimeProcessingMutation=T6-T5, TimeTLogPush=T7-T6, _time=T1
+| table _time TimeSpan TimeResolution TimePostResolution TimeProcessingMutation TimeTLogPush Machine
+| timechart span=$ChartBinSizeToken$ limit=0 avg(TimeSpan), avg(TimeResolution), avg(TimePostResolution), avg(TimeProcessingMutation), avg(TimeTLogPush) $CommitByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 30: Transaction Tracing - Commit - TLogPush and Resolver Latency (only show CC transactions by default; it shows client transactions only when you manually open client transaction trace)</title>
+      <input type="dropdown" token="TLogResolverByMachineToken" searchWhenChanged="true">
+        <label>By Machine</label>
+        <choice value="MachineStep">Yes</choice>
+        <choice value="Step">No</choice>
+        <default>Step</default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="CommitDebug" AND (Resolver.resolveBatch.Before OR Resolver.resolveBatch.AfterQueueSizeCheck OR Resolver.resolveBatch.AfterOrderer OR Resolver.resolveBatch.After OR TLog.tLogCommit.BeforeWaitForVersion OR TLog.tLogCommit.Before OR TLog.tLogCommit.AfterTLogCommit OR TLog.tLogCommit.After) 
+| table Time Type ID Location Machine Roles 
+| eval Order=case(Location=="NativeAPI.commit.Before", 0, Location=="MasterProxyServer.batcher", 1, Location=="MasterProxyServer.commitBatch.Before", 2, Location=="MasterProxyServer.commitBatch.GettingCommitVersion", 3, Location=="MasterProxyServer.commitBatch.GotCommitVersion", 4, Location=="Resolver.resolveBatch.Before", 5, Location=="Resolver.resolveBatch.AfterQueueSizeCheck", 6, Location=="Resolver.resolveBatch.AfterOrderer", 7, Location=="Resolver.resolveBatch.After", 8, Location=="MasterProxyServer.commitBatch.AfterResolution", 8.5, Location=="MasterProxyServer.commitBatch.ProcessingMutations", 9, Location=="MasterProxyServer.commitBatch.AfterStoreCommits", 10, Location=="TLog.tLogCommit.BeforeWaitForVersion", 11, Location=="TLog.tLogCommit.Before", 12, Location=="TLog.tLogCommit.AfterTLogCommit", 13, Location=="TLog.tLogCommit.After", 14, Location=="MasterProxyServer.commitBatch.AfterLogPush", 15, Location=="NativeAPI.commit.After", 16)
+| table Time Order Type ID Location Machine Roles 
+| sort 0 Time Order 
+| table Machine Location Time Roles Type ID
+| stats list(*) by ID
+| rename list(*) as *
+| eval Count=mvcount(Location), Step=case(Count=4 and (mvindex(Location, 0) like "TLog%"), "TimeTLogCommit", Count=4 and (mvindex(Location, 0) like "Resolver%"), "TimeResolver", Count=10, "TimeSpan"), BeginTime=mvindex(Time, 0), EndTime=mvindex(Time, -1), Duration=EndTime-BeginTime, _time=BeginTime
+| search Count=4
+| eval Machinei=mvindex(Machine, 0), MachineStep = Step."-".Machinei
+| table _time Step Duration Machinei Location Machine MachineStep
+| timechart span=$ChartBinSizeToken$ limit=0 avg(Duration) by $TLogResolverByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 31: Machine Performance - CPU Utilization (CPU Time divided by Elapsed)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics AND TrackLatestType="Original" 
+| table _time Machine CPUSeconds DiskFreeBytes DiskIdleSeconds DiskQueueDepth DiskReadsCount DiskWriteSectors DiskTotalBytes DiskWritesCount FileReads MbpsReceived MbpsSent Memory ResidentMemory UnusedAllocatedMemory Elapsed
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="Role" AND As=$RolePerformanceChartToken$ 
+    | stats first(Machine) by Machine 
+    | rename first(Machine) as Machine 
+    | table Machine] 
+| eval Utilization=CPUSeconds/Elapsed
+| timechart span=$ChartBinSizeToken$ avg(Utilization) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 32: Machine Performance - Memory Utilization (ResidentMemory divided by Memory)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics AND TrackLatestType="Original" 
+| table _time Machine CPUSeconds DiskFreeBytes DiskIdleSeconds DiskQueueDepth DiskReadsCount DiskWriteSectors DiskTotalBytes DiskWritesCount FileReads MbpsReceived MbpsSent Memory ResidentMemory UnusedAllocatedMemory 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="Role" AND As=$RolePerformanceChartToken$ 
+    | stats first(Machine) by Machine 
+    | rename first(Machine) as Machine 
+    | table Machine] 
+| eval Utilization = ResidentMemory/Memory
+| timechart span=$ChartBinSizeToken$ avg(Utilization) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 33: Machine Performance - Disk Utilization ((DiskTotalBytes-DiskFreeBytes)/DiskTotalBytes)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics AND TrackLatestType="Original" 
+| table _time Machine CPUSeconds DiskFreeBytes DiskIdleSeconds DiskQueueDepth DiskReadsCount DiskWriteSectors DiskTotalBytes DiskWritesCount FileReads MbpsReceived MbpsSent Memory ResidentMemory UnusedAllocatedMemory 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="Role" AND As=$RolePerformanceChartToken$
+    | stats first(Machine) by Machine 
+    | rename first(Machine) as Machine 
+    | table Machine] 
+| eval Utilization = (DiskTotalBytes-DiskFreeBytes)/DiskTotalBytes
+| timechart span=$ChartBinSizeToken$ avg(Utilization) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 34: Machine Performance - Network (Mbps Received and Mbps Sent)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics AND TrackLatestType="Original" 
+| table _time Machine CPUSeconds DiskFreeBytes DiskIdleSeconds DiskQueueDepth DiskReadsCount DiskWriteSectors DiskTotalBytes DiskWritesCount FileReads MbpsReceived MbpsSent Memory ResidentMemory UnusedAllocatedMemory 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="Role" AND As=$RolePerformanceChartToken$ 
+    | stats first(Machine) by Machine 
+    | rename first(Machine) as Machine 
+    | table Machine] 
+| timechart span=$ChartBinSizeToken$ avg(MbpsReceived) avg(MbpsSent) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.axisY.scale">log</option>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 35: Machine Performance - Disk (Reads Count and Writes Count)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics AND TrackLatestType="Original" 
+| table _time Machine CPUSeconds DiskFreeBytes DiskIdleSeconds DiskQueueDepth DiskReadsCount DiskWriteSectors DiskTotalBytes DiskWritesCount FileReads MbpsReceived MbpsSent Memory ResidentMemory UnusedAllocatedMemory 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="Role" AND As=$RolePerformanceChartToken$ 
+    | stats first(Machine) by Machine 
+    | rename first(Machine) as Machine 
+    | table Machine] 
+| timechart span=$ChartBinSizeToken$ avg(DiskReadsCount) avg(DiskWritesCount) $ChartByMachineToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 36: Network Performance - Timeout</title>
+      <input type="dropdown" token="TimeoutByConnectionToken" searchWhenChanged="true">
+        <label>By Connection</label>
+        <choice value="By Connection">Yes</choice>
+        <choice value="">No</choice>
+        <default></default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type=ConnectionTimedOut OR Type=ConnectionTimeout) 
+| replace *:tls with * in PeerAddr 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($SourcePerfConnectionToken$)) 
+    | dedup ID] 
+| join PeerAddr 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($DestinationPerfConnectionToken$)) 
+    | dedup ID 
+    | rename Machine as PeerAddr] 
+| eval Connection=Machine."-".PeerAddr
+| timechart useother=0 span=$ChartBinSizeToken$ count $TimeoutByConnectionToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+    <panel>
+      <title>Chart 37: Network Performance - PingLatency</title>
+      <input type="dropdown" token="PingLatencyByConnectionToken" searchWhenChanged="true">
+        <label>By Connection</label>
+        <choice value="By Connection">Yes</choice>
+        <choice value="">No</choice>
+        <default></default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type=PingLatency) 
+| replace *:tls with * in PeerAddr 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($SourcePerfConnectionToken$)) 
+    | dedup ID] 
+| join PeerAddr 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($DestinationPerfConnectionToken$)) 
+    | dedup ID 
+    | rename Machine as PeerAddr] 
+| eval Connection=Machine."-".PeerAddr
+| timechart useother=0 span=$ChartBinSizeToken$ avg(MeanLatency) avg(MaxLatency)   $PingLatencyByConnectionToken$</query>
+          <earliest>$TimeSpan.earliest$</earliest>
+          <latest>$TimeSpan.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+</form>
\ No newline at end of file
diff --git a/contrib/observability_splunk_dashboard/recovery.xml b/contrib/observability_splunk_dashboard/recovery.xml
new file mode 100644
index 0000000000..6ba6b9a63b
--- /dev/null
+++ b/contrib/observability_splunk_dashboard/recovery.xml
@@ -0,0 +1,873 @@
+<form theme="dark">
+  <label>FoundationDB - Long Recovery (Dev)</label>
+  <fieldset submitButton="false" autoRun="false"></fieldset>
+  <row>
+    <panel>
+      <title>Table 1: Find long recovery (Input Index and LogGroup and Select a time span).</title>
+      <input type="text" token="IndexForOverview" searchWhenChanged="true">
+        <label>Index</label>
+        <default>*</default>
+      </input>
+      <input type="text" token="LogGroupForOverview" searchWhenChanged="true">
+        <label>LogGroup</label>
+        <default></default>
+      </input>
+      <input type="time" token="time_token_for_recoveryhistorytable" searchWhenChanged="true">
+        <label>Select a time span</label>
+        <default>
+          <earliest>-0s</earliest>
+          <latest>now</latest>
+        </default>
+      </input>
+      <table>
+        <search>
+          <query>index=$IndexForOverview$ LogGroup=$LogGroupForOverview$
+    ((Type="MasterRecoveryState" AND (Status="reading_coordinated_state" OR Status="fully_recovered" OR Status="accepting_commits")) OR (Type="Role" AND As="MasterServer" AND ("Transition"="Begin" OR "Transition"="End")) OR Type="MasterTerminated") AND (NOT TrackLatestType="Rolled") | eval DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)")
+| table ID Machine Type Transition As Status DateTime Time ErrorDescription LogGroup
+| search NOT ErrorDescription="Success"
+| eval EventType=case(Transition="Begin" AND As="MasterServer" AND Type="Role", "MasterStart", Type="MasterRecoveryState" AND Status="fully_recovered", "FullRecovery", Type="MasterRecoveryState" AND Status="reading_coordinated_state", "StartRecoveryAttempt", Transition="End" AND As="MasterServer" AND Type="Role", "MasterTerminated", Type="MasterTerminated", "MasterTerminated", Type="MasterRecoveryState" AND Status="accepting_commits", "AcceptingCommits") 
+| table ID Machine EventType DateTime Time ErrorDescription LogGroup
+| fillnull value="-" 
+| sort -Time 
+| eval ifMasterTerminatedEvent=if(EventType="MasterTerminated", 1, 0) 
+| stats list(*) by ID Machine ifMasterTerminatedEvent 
+| rename list(*) as * 
+| table ID Machine EventType DateTime Time ErrorDescription LogGroup
+| sort -Time 
+| eval LastTime=mvindex(Time, 0), FirstTime=mvindex(Time, -1), Duration=LastTime-FirstTime 
+| table ID Machine Duration EventType DateTime Time ErrorDescription LogGroup</query>
+          <earliest>$time_token_for_recoveryhistorytable.earliest$</earliest>
+          <latest>$time_token_for_recoveryhistorytable.latest$</latest>
+        </search>
+        <option name="count">15</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 2: Select timespan containing the long recovery and see all recovery attempts in the time span (The input Index and LogGroup and Timespan are for all following tables and charts)</title>
+      <input type="text" token="Index" searchWhenChanged="true">
+        <label>Index</label>
+        <default>*</default>
+      </input>
+      <input type="text" searchWhenChanged="true" token="LogGroup">
+        <label>LogGroup</label>
+      </input>
+      <input type="time" token="ReoveryTime" searchWhenChanged="true">
+        <label>ReoveryTimeSpan</label>
+        <default>
+          <earliest>-0s@s</earliest>
+          <latest>now</latest>
+        </default>
+      </input>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type="MasterRecoveryState" OR (Type="MasterTerminated") OR (Type="Role" AND As="MasterServer" AND "Transition"="End") OR Type="RecoveryInternal" OR Type="ProxyReplies" OR Type="CommitProxyReplies" OR Type="ResolverReplies" OR Type="MasterRecruitedInitialStorageServers") AND (NOT TrackLatestType="Rolled") 
+| rename ID as MasterID 
+| eval DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+| table MasterID Machine Status Step Type DateTime Time StatusCode MyRecoveryCount ErrorDescription Reason ErrorCode 
+| fillnull value="-" ErrorDescription Reason ErrorCode 
+| eval Status=case(Type=="MasterRecoveryState", Status, Type=="Role", "RoleEnd", Type=="MasterTerminated", "MasterTerminated", Type=="RecoveryInternal", Status."/".Step, Type=="ProxyReplies" OR Type=="CommitProxyReplies", "initializing_transaction_servers/ProxyReplies", Type="ResolverReplies", "initializing_transaction_servers/ResolverReplies", Type=="MasterRecruitedInitialStorageServers", "initializing_transaction_servers/MasterRecruitedInitialStorageServers"), StatusCode=case(Type=="ProxyReplies" OR Type=="CommitProxyReplies" OR Type=="ResolverReplies" OR Type=="MasterRecruitedInitialStorageServers", "8", Type!="ProxyReplies" AND Type!="CommitProxyReplies" AND Type!="ResolverReplies" AND Type!="MasterRecruitedInitialStorageServers", StatusCode)
+| fillnull value="-" StatusCode 
+| sort 0 -Time -StatusCode
+| stats list(*) by MasterID Machine 
+| rename list(*) as * 
+| eval FirstTime=mvindex(Time, -1), LastTime=mvindex(Time, 0), Duration=LastTime-FirstTime 
+| table MasterID Machine MyRecoveryCount Duration ErrorDescription Reason ErrorCode StatusCode Status DateTime Time 
+| sort -MyRecoveryCount 
+| fillnull value="-" MyRecoveryCount</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="count">3</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+        <option name="wrap">false</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 3: Why recovery is triggered? Using WaitFailureClient event. Machine A detects Machine B's failure. First column is the time when WaitFailureClient happens. Columns of 2,3,4,5 are for A. Columns of 6,7 are for B.</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="WaitFailureClient" 
+| table Type Time Machine FailedEndpoint 
+| replace *:tls with * in FailedEndpoint 
+| join Machine type=left 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="Role" AND Transition="End" 
+    | eval EndTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+    | rename As as Role 
+    | table ID EndTime Machine Role] 
+| join FailedEndpoint type=left 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type="Role" 
+    | stats latest(*) by ID | rename latest(*) as *
+    | rename Machine as FailedEndpoint 
+    | eval FailedEndpointLatestRoleEventInfo=As."/".ID."/".Type.Transition."/".strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+    | stats list(*) by FailedEndpoint 
+    | rename list(*) as * 
+    | table FailedEndpoint FailedEndpointLatestRoleEventInfo] 
+| eval FailureDetectedTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+| makemv delim=" " FailedEndpointLatestRoleEventInfo 
+| table FailureDetectedTime Machine ID Role EndTime FailedEndpoint FailedEndpointLatestRoleEventInfo</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="wrap">false</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 4: New Recruitment Configuration (using MasterRecoveredConfig event)</title>
+      <event>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="MasterRecoveredConfig" AND TrackLatestType="Original" 
+| eval Configuration=replace(Conf, "&amp;quot;", "\"") 
+| rename Configuration as _raw</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="list.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </event>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 5: Data Centers (using ProcessMetrics event)</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type=ProcessMetrics 
+| dedup DCID 
+| rename DCID as DataCenterID 
+| table DataCenterID pie_work_unit
+| fillnull value="-"</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+    <panel>
+      <title>Table 6: New Role (using Role event joined by ProcessMetrics event)</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type="Role" AND ((As="ClusterController") OR (As="MasterServer") OR (As="TLog") OR (As="Resolver") OR (As="MasterProxyServer") OR (As="CommitProxyServer") OR (As="GrvProxyServer") OR (As="LogRouter")) AND (NOT TrackLatestType="Rolled") AND (NOT Transition="Refresh")) 
+| eventstats count by ID 
+| rename As as Role 
+| search count=1 AND Transition="Begin" 
+| table ID Role Machine 
+| join type=left Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics 
+    | dedup Machine, DCID 
+    | rename DCID as DataCenter 
+    | table Machine DataCenter] 
+| table ID Role Machine DataCenter 
+| fillnull value="null" DataCenter 
+| stats count by Role DataCenter</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 7: Role Details</title>
+      <input type="multiselect" token="RolesToken" searchWhenChanged="true">
+        <label>Roles</label>
+        <choice value="MasterServer">MasterServer</choice>
+        <choice value="TLog">TLog</choice>
+        <choice value="Resolver">Resolver</choice>
+        <choice value="MasterProxyServer">MasterProxyServer (for &lt;7.0)</choice>
+        <choice value="LogRouter">LogRouter</choice>
+        <choice value="CommitProxyServer">CommitProxyServer (for 7.0+)</choice>
+        <choice value="GrvProxyServer">GrvProxyServer (for 7.0+)</choice>
+        <valuePrefix>As="</valuePrefix>
+        <valueSuffix>"</valueSuffix>
+        <delimiter> OR </delimiter>
+      </input>
+      <input type="dropdown" token="RoleDetailTableWhichRoleToken" searchWhenChanged="true">
+        <label>Begin/End</label>
+        <choice value="count=1 AND Transition=&quot;Begin&quot;">Begin</choice>
+        <choice value="count=1 AND Transition=&quot;End&quot;">End</choice>
+        <choice value="count=2">Begin-&gt;End</choice>
+        <default>count=1 AND Transition="Begin"</default>
+      </input>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type="Role" AND ($RolesToken$) AND (NOT TrackLatestType="Rolled") AND (NOT Transition="Refresh")) 
+| eventstats count by ID 
+| rename As as Role 
+| search $RoleDetailTableWhichRoleToken$
+| table ID Role Machine Time
+| join type=left Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics 
+    | dedup Machine, DCID 
+    | rename DCID as DataCenter 
+    | table Machine DataCenter] 
+| table ID Role Machine DataCenter Time
+| fillnull value="null" DataCenter 
+| eval DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+| table ID Role Machine DataCenter DateTime 
+| sort 0 -DateTime</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 8: CC Recruitment SevWarn OR SevError (use events in clusterRecruitFromConfiguration and clusterRecruitRemoteFromConfiguration)</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="RecruitFromConfigurationNotAvailable" OR Type="RecruitFromConfigurationRetry" OR Type="RecruitFromConfigurationError" OR Type="RecruitRemoteFromConfigurationNotAvailable" OR Type="RecruitRemoteFromConfigurationRetry" OR Type="RecruitRemoteFromConfigurationError"
+    | eval DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)"), GoodRecruitmentTimeReady=case(Type=="RecruitFromConfigurationNotAvailable" OR Type=="RecruitRemoteFromConfigurationNotAvailable", "True", Type=="RecruitFromConfigurationRetry" OR Type=="RecruitRemoteFromConfigurationRetry", GoodRecruitmentTimeReady, Type=="RecruitFromConfigurationError" OR Type=="RecruitRemoteFromConfigurationError", "-")
+    | table Type GoodRecruitmentTimeReady Time DateTime</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 9: RecoveryCount of the selected TLog (in Table 11)</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (ID=$row.TLogID$ AND Type="TLogStart") OR (LogId=$row.TLogID$ AND Type="TLogPersistentStateRestore") 
+| eval ID=if(Type="TLogStart", ID, LogId), DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)")  
+| table ID RecoveryCount Type DateTime | fillnull value="Not found. The fdb version is somewhat old."</query>
+          <earliest>-7d@h</earliest>
+          <latest>now</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+    <panel>
+      <title>Table 10: Which roles the selected TLog (in Table 11) talks to</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    ((Type="TLogRejoining" AND ID=$row.TLogID$) OR ((Type="TLogJoinedMe" OR Type="TLogJoinedMeUnknown" OR Type="TLogRejoinSlow") AND TLog=$row.TLogID$) OR ((Type="TLogLockStarted" OR Type="TLogLocked") AND TLog=$row.TLogID$) OR (Type="TLogStop" AND ID=$row.TLogID$) OR (Type="TLogStop2" AND LogId=$row.TLogID$) OR (Type="Role" AND As="TLog" AND NOT Transition="Refresh" AND ID=$row.TLogID$)) AND (NOT TrackLatestType="Rolled") 
+| sort -Time 
+| eval TLogID=case((Type="TLogRejoining"), ID, (Type="TLogJoinedMe") OR (Type="TLogJoinedMeUnknown") OR (Type="TLogRejoinSlow"), TLog, (Type="TLogLockStarted") OR (Type="TLogLocked"), TLog, (Type="TLogStop"), ID, (Type="TLogStop2"), LogId, Type="Role", ID), TLogEvents=case((Type="TLogRejoining"), Time." ".Type." ".Master, (Type="TLogJoinedMe") OR (Type="TLogJoinedMeUnknown") OR (Type="TLogRejoinSlow") OR (Type="TLogLockStarted") OR (Type="TLogLocked"), Time." ".Type." ".ID." "."Null", (Type="TLogStop") OR (Type="TLogStop2"), Time." ".Type." "."Null", (Type="Role" AND As="TLog" AND NOT Transition="Refresh" AND NOT TrackLatestType="Rolled"), Time." "."Role".Transition." "."Null") 
+| stats list(*) by TLogID 
+| rename list(*) As * 
+| table TLogID TLogEvents 
+| eval ignore = if(mvcount(TLogEvents)==1 AND like(mvindex(TLogEvents, 0), "% RoleEnd"), 1, 0) 
+| search ignore=0 
+| sort TLogID 
+| table TLogID TLogEvents 
+| mvexpand TLogEvents 
+| eval temp=split(TLogEvents," "), Time=mvindex(temp,0), Event=mvindex(temp,1), MasterID=mvindex(temp,2) 
+| fields - temp - TLogEvents 
+| sort 0 -Time 
+| search NOT MasterID="NULL" 
+| dedup MasterID 
+| rename MasterID as ID 
+| join type=left ID 
+    [ search index=$Index$ LogGroup=$LogGroup$ 
+        (Type="Role")
+    | sort 0 -Time 
+    | dedup ID 
+    | table ID Machine As] 
+| table ID Machine As | fillnull value="null" Machine As</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 11: TLog Events (Collecting all TLogs that produce interesting events during the time span)</title>
+      <input type="text" token="SeeLogEventDetailTableToken" searchWhenChanged="true">
+        <label>Input * to do search</label>
+      </input>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type="TLogRecover") OR (Type="TLogReady") OR (Type="TLogStart") OR
+    ((Type="TLogLockStarted") OR (Type="TLogLocked") OR (Type="TLogStop") OR (Type="TLogStop2")) OR (Type="Role" AND As="TLog" AND NOT Transition="Refresh") AND (NOT TrackLatestType="Rolled") AND $SeeLogEventDetailTableToken$
+| sort -Time 
+| eval TLogID=case((Type="TLogRecover"), LogId, (Type="TLogReady"), ID, (Type="TLogStart"), ID, (Type="TLogLockStarted") OR (Type="TLogLocked"), TLog, (Type="TLogStop"), ID, (Type="TLogStop2"), LogId, Type="Role", ID), TLogEvents=case((Type="TLogRecover"), Time." ".Type." "."null", (Type="TLogReady"), Time." ".Type." "."null", (Type="TLogStart"), Time." ".Type." "."null", (Type="TLogLockStarted") OR (Type="TLogLocked"), Time." ".Type." ".ID." "."null", (Type="TLogStop") OR (Type="TLogStop2"), Time." ".Type." "."null", (Type="Role" AND As="TLog" AND NOT Transition="Refresh" AND NOT TrackLatestType="Rolled"), Time." "."Role".Transition." "."null") 
+| stats list(TLogEvents) by TLogID 
+| rename list(TLogEvents) As TLogEvents 
+| eval EarliestEvent=mvindex(TLogEvents, -1) , LatestEvent=mvindex(TLogEvents, 0) 
+| table TLogID TLogEvents EarliestEvent LatestEvent 
+| eval ignore = if(mvcount(TLogEvents)==1 AND like(mvindex(TLogEvents, 0), "% RoleEnd"), 1, 0) 
+| search ignore=0 
+| sort TLogID 
+| join type=left TLogID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND As="TLog") 
+    | sort 0 -Time 
+    | dedup ID 
+    | rename ID as TLogID 
+    | table TLogID host LogGroup Machine] 
+| table TLogID Machine LogGroup host EarliestEvent LatestEvent 
+| fillnull value="null" Machine host LogGroup
+| eval temp=split(LatestEvent," "), LatestTime=mvindex(temp,0), LatestEvent=mvindex(temp,1), temp2=split(EarliestEvent," "), EarliestTime=mvindex(temp2,0), EarliestEvent=mvindex(temp2,1), Duration=LatestTime-EarliestTime 
+| table TLogID Machine EarliestTime Duration LogGroup host 
+| join type=left Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ 
+        Type=ProcessMetrics 
+    | dedup Machine, DCID 
+    | rename DCID as DataCenter 
+    | table Machine DataCenter] 
+| fillnull value="null" DataCenter 
+| table TLogID Machine DataCenter EarliestTime Duration host LogGroup 
+| join type=left TLogID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        ((Type="TLogRejoining") OR ((Type="TLogJoinedMe" OR Type="TLogJoinedMeUnknown" OR Type="TLogRejoinSlow")) OR ((Type="TLogLockStarted" OR Type="TLogLocked")) OR (Type="TLogStop") OR (Type="TLogStop2") OR (Type="Role" AND As="TLog" AND NOT Transition="Refresh")) AND (NOT TrackLatestType="Rolled") 
+    | sort -Time 
+    | eval TLogID=case((Type="TLogRejoining"), ID, (Type="TLogJoinedMe") OR (Type="TLogJoinedMeUnknown") OR (Type="TLogRejoinSlow"), TLog, (Type="TLogLockStarted") OR (Type="TLogLocked"), TLog, (Type="TLogStop"), ID, (Type="TLogStop2"), LogId, Type="Role", ID), TLogEvents=case((Type="TLogRejoining"), Time." ".Type." ".Master, (Type="TLogJoinedMe") OR (Type="TLogJoinedMeUnknown") OR (Type="TLogRejoinSlow") OR (Type="TLogLockStarted") OR (Type="TLogLocked"), Time." ".Type." ".ID." "."Null", (Type="TLogStop") OR (Type="TLogStop2"), Time." ".Type." "."Null", (Type="Role" AND As="TLog" AND NOT Transition="Refresh" AND NOT TrackLatestType="Rolled"), Time." "."Role".Transition." "."Null") 
+    | stats list(*) by TLogID 
+    | rename list(*) As * 
+    | table TLogID TLogEvents 
+    | eval ignore = if(mvcount(TLogEvents)==1 AND like(mvindex(TLogEvents, 0), "% RoleEnd"), 1, 0) 
+    | search ignore=0 
+    | sort TLogID 
+    | table TLogID TLogEvents 
+    | mvexpand TLogEvents 
+    | eval temp=split(TLogEvents," "), Time=mvindex(temp,0), Event=mvindex(temp,1), RoleID=mvindex(temp,2) 
+    | fields - temp - TLogEvents 
+    | sort 0 -Time 
+    | search NOT RoleID="NULL" 
+    | table TLogID RoleID MasterMachine 
+    | stats list(*) by TLogID 
+    | rename list(*) as * 
+    | streamstats count 
+    | mvexpand RoleID 
+    | dedup count RoleID 
+    | fields - count 
+    | stats count by TLogID 
+    | rename count as Roles 
+    | table TLogID Roles] 
+| table TLogID Machine DataCenter Roles EarliestTime Duration host LogGroup 
+| join type=left TLogID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="TLogRecover") OR (Type="TLogReady") OR (Type="TLogStart") OR
+        ((Type="TLogRejoinSlow") OR (Type="TLogLockStarted") OR (Type="TLogLocked") OR (Type="TLogStop") OR (Type="TLogStop2") OR (Type="Role" AND As="TLog" AND NOT Transition="Refresh") AND (NOT TrackLatestType="Rolled")) 
+    | sort -Time 
+    | eval TLogID=case((Type="TLogRecover"), LogId, (Type="TLogReady"), ID, (Type="TLogStart"), ID, (Type="TLogRejoinSlow"), TLog, (Type="TLogLockStarted") OR (Type="TLogLocked"), TLog, (Type="TLogStop"), ID, (Type="TLogStop2"), LogId, Type="Role", ID), TLogEvents=if(Type="Role", Type.Transition, Type) 
+    | sort 0 TLogEvents 
+    | stats list(TLogEvents) by TLogID 
+    | rename list(TLogEvents) As TLogEvents 
+    | table TLogID TLogEvents 
+    | eval ignore = if(mvcount(TLogEvents)==1 AND like(mvindex(TLogEvents, 0), "% RoleEnd"), 1, 0) 
+    | search ignore=0 
+    | mvcombine delim=" " TLogEvents 
+    | table TLogID TLogEvents] 
+| table TLogID Machine DataCenter Roles Duration TLogEvents EarliestTime host LogGroup 
+| eval EarliestDateTime=strftime(EarliestTime, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+| table TLogID Machine DataCenter Roles Duration TLogEvents EarliestDateTime host LogGroup
+| join type=left TLogID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="TLogStart") OR (Type="TLogPersistentStateRestore") 
+    | eval TLogID=if(Type="TLogStart", ID, LogId) 
+    | table TLogID RecoveryCount] 
+| table TLogID RecoveryCount Machine DataCenter Roles Duration TLogEvents EarliestDateTime host LogGroup 
+| fillnull value="TLog too old, click and see details" RecoveryCount</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">cell</option>
+        <option name="wrap">false</option>
+        <drilldown>
+          <set token="row.TLogID">$click.value$</set>
+        </drilldown>
+      </table>
+    </panel>
+    <panel>
+      <title>Table 12: Event Details (Including rejoining events) of the selected TLog (in Table 11)</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type="TLogRecover" AND LogId=$row.TLogID$) OR (Type="TLogReady" AND ID=$row.TLogID$) OR (Type="TLogStart" AND ID=$row.TLogID$) OR
+    ((Type="TLogRejoining" AND ID=$row.TLogID$) OR ((Type="TLogJoinedMe" OR Type="TLogJoinedMeUnknown" OR Type="TLogRejoinSlow") AND TLog=$row.TLogID$) OR ((Type="TLogLockStarted" OR Type="TLogLocked") AND TLog=$row.TLogID$) OR (Type="TLogStop" AND ID=$row.TLogID$) OR (Type="TLogStop2" AND LogId=$row.TLogID$) OR (Type="Role" AND As="TLog" AND NOT Transition="Refresh" AND ID=$row.TLogID$)) AND (NOT TrackLatestType="Rolled") 
+| sort -Time 
+| eval TLogID=case((Type="TLogRecover"), LogId, (Type="TLogReady"), ID, (Type="TLogStart"), ID, (Type="TLogRejoining"), ID, (Type="TLogJoinedMe") OR (Type="TLogJoinedMeUnknown") OR (Type="TLogRejoinSlow"), TLog, (Type="TLogLockStarted") OR (Type="TLogLocked"), TLog, (Type="TLogStop"), ID, (Type="TLogStop2"), LogId, Type="Role", ID), TLogEvents=case((Type="TLogRecover"), Time." ".Type." "."-"." "."-", (Type="TLogReady"), Time." ".Type." "."-"." "."-", (Type="TLogStart"), Time." ".Type." "."-"." "."-", (Type="TLogRejoining"), Time." ".Type." ".Master." "."-", (Type="TLogJoinedMe") OR (Type="TLogJoinedMeUnknown") OR (Type="TLogRejoinSlow") OR (Type="TLogLockStarted") OR (Type="TLogLocked"), Time." ".Type." ".ID." "."-", (Type="TLogStop") OR (Type="TLogStop2"), Time." ".Type." "."-"." "."-", (Type="Role" AND As="TLog" AND Transition="Begin" AND NOT TrackLatestType="Rolled"), Time." "."Role".Transition." "."-"." ".Origination, (Type="Role" AND As="TLog" AND Transition="End" AND NOT TrackLatestType="Rolled"), Time." "."Role".Transition." "."-"." "."-") 
+| stats list(*) by TLogID 
+| rename list(*) As * 
+| table TLogID TLogEvents 
+| eval ignore = if(mvcount(TLogEvents)==1 AND like(mvindex(TLogEvents, 0), "% RoleEnd"), 1, 0) 
+| search ignore=0 
+| sort TLogID 
+| join type=left TLogID 
+    [ search index=$Index$ LogGroup=$LogGroup$ (Type="Role" AND As="TLog" AND ID=$row.TLogID$) 
+    | dedup ID 
+    | rename ID as TLogID 
+    | table TLogID Machine] 
+| table TLogID Machine TLogEvents 
+| fillnull value="-" Machine 
+| mvexpand TLogEvents 
+| eval temp=split(TLogEvents," "), Time=mvindex(temp,0), Event=mvindex(temp,1), ToID=mvindex(temp,2), Origination= mvindex(temp,3) 
+| fields - temp - TLogEvents 
+| join type=left 
+    [ search index=$Index$ LogGroup=$LogGroup$ (Type="Role") 
+    | dedup ID 
+    | rename ID as ToID 
+    | rename As as ToRole 
+    | rename Machine as ToMachine 
+    | table ToID ToRole ToMachine] 
+| sort 0 -Time 
+| fillnull value="-" ToRole ToMachine 
+| eval DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+| table TLogID Machine Event DateTime ToID ToRole ToMachine Time DateTime</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="count">14</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+        <option name="wrap">false</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 13: All Tags of the selected TLog (in Table 11) that have been popped by SSes (using TLogPoppedTag event)</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (ID=$row.TLogID$ AND Type="TLogPoppedTag") 
+| eval DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+| rename ID as TLogID 
+| rename Tags as UnpoppedRecoveredTagCount 
+| rename Tag as TagPopped 
+| rename DurableKCVer as DurableKnownCommittedVersion 
+| search TagPopped!="-1:2" 
+| table TLogID DateTime UnpoppedRecoveredTagCount TagPopped DurableKnownCommittedVersion RecoveredAt 
+| sort 0 -UnpoppedRecoveredTagCount 
+| join TagPopped type=left 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="StorageMetrics") 
+    | stats latest(*) by Machine 
+    | rename latest(*) as * 
+    | rename Tag as TagPopped 
+    | table TagPopped ID Machine] 
+| table TLogID DateTime UnpoppedRecoveredTagCount TagPopped DurableKnownCommittedVersion RecoveredAt ID Machine
+| join type=left Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type=ProcessMetrics
+    | dedup Machine, DCID 
+    | rename DCID as DataCenter 
+    | table Machine DataCenter] 
+| rename ID as SSID 
+| rename Machine as SSMachine 
+| rename DataCenter as SSDataCenter 
+| table TLogID DateTime UnpoppedRecoveredTagCount TagPopped SSID SSMachine SSDataCenter DurableKnownCommittedVersion RecoveredAt 
+| fillnull value="-"</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+        <option name="wrap">false</option>
+      </table>
+    </panel>
+    <panel>
+      <title>Table 14: All Tags of the selected TLog (in Table 11) to be popped by SSes (using TLogReady event)</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (ID=$row.TLogID$ AND Type="TLogReady") 
+| eval DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+| rename ID as TLogID 
+| table TLogID Type AllTags Locality 
+| makemv delim="," AllTags 
+| mvexpand AllTags 
+| rename AllTags as Tag | sort 0 Tag
+| join Tag type=left 
+    [ search index=$Index$ LogGroup=$LogGroup$ 
+        (Type="StorageMetrics") 
+    | stats latest(*) by Machine 
+    | rename latest(*) as * 
+    | table Tag ID Machine] 
+| table TLogID Tag ID Machine
+| join type=left Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type=ProcessMetrics 
+    | dedup Machine, DCID 
+    | rename DCID as DataCenter 
+    | table Machine DataCenter] 
+| fillnull value="-"
+| table TLogID Tag ID Machine DataCenter 
+| rename ID as SSID | rename Machine as SSMachine | rename DataCenter as SSDataCenter
+| search Tag!="-1:2"</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 15: The Tags of the selected TLog (in Table 11) that are not popped by SSes (using set diff tags in Table 13 and Table 14) (if result contains "...", the result of Table 15 is wrong)</title>
+      <table>
+        <search>
+          <query>| set diff 
+    [ search index=$Index$ LogGroup=$LogGroup$ 
+        (ID=$row.TLogID$ AND Type="TLogReady") 
+    | table AllTags  
+    | makemv delim="," AllTags 
+    | mvexpand AllTags 
+    | rename AllTags as Tag 
+    | table Tag] 
+    [ search index=$Index$ LogGroup=$LogGroup$ 
+        (ID=$row.TLogID$ AND Type="TLogPoppedTag") 
+    | table Tag]</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+    <panel>
+      <title>Table 16: All Current Storage Servers (assume each machine has at most one SS)</title>
+      <input type="text" token="TriggerSSTableToken" searchWhenChanged="true">
+        <label>Input * to search</label>
+      </input>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type="StorageMetrics") AND $TriggerSSTableToken$ 
+| stats latest(*) by Machine 
+| rename latest(*) as * 
+| table Tag ID Machine 
+| join type=left Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type=ProcessMetrics 
+    | dedup Machine, DCID 
+    | rename DCID as DataCenter 
+    | table Machine DataCenter] 
+| table ID Machine DataCenter Tag 
+| join ID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ((As="StorageServer")) AND (NOT TrackLatestType="Rolled")) 
+    | stats latest(*) by Machine 
+    | rename latest(*) as * 
+    | rename As as Role 
+    | table ID Role Machine 
+    | join type=left Machine 
+        [ search index=$Index$ LogGroup=$LogGroup$
+            Type=ProcessMetrics 
+        | dedup Machine, DCID 
+        | rename DCID as DataCenter 
+        | table Machine DataCenter] 
+    | table ID Role Machine DataCenter 
+    | fillnull value="null" DataCenter] 
+| sort 0 DataCenter
+| table Tag ID Machine DataCenter | sort 0 Tag</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 1: Timeout/TimedOut event distribution grouped by source (Machine)</title>
+      <input type="text" token="TimeoutEventByMachineTableTimeSpanToken" searchWhenChanged="true">
+        <label>TimeSpan</label>
+        <default>5s</default>
+      </input>
+      <input type="multiselect" token="TimeoutbyMachineTableSourceRoleToken" searchWhenChanged="true">
+        <label>Select Source Roles</label>
+        <choice value="TLog">TLog</choice>
+        <choice value="MasterServer">MasterServer</choice>
+        <choice value="MasterProxyServer">MasterProxyServer (for version &lt; 7)</choice>
+        <choice value="Resolver">Resolver</choice>
+        <choice value="ClusterController">ClusterController</choice>
+        <choice value="SharedTLog">SharedTLog</choice>
+        <choice value="LogRouter">LogRouter</choice>
+        <choice value="Coordinator">Coordinator</choice>
+        <choice value="StorageServer">StorageServer</choice>
+        <choice value="CommitProxyServer">CommitProxyServer (for version 7+)</choice>
+        <choice value="GrvProxyServer">GrvProxyServer (for ver 7+)</choice>
+        <valuePrefix>As="</valuePrefix>
+        <valueSuffix>"</valueSuffix>
+        <delimiter> OR </delimiter>
+      </input>
+      <input type="multiselect" token="TimeoutbyMachineTableDestinationRoleToken" searchWhenChanged="true">
+        <label>Select Destination Roles</label>
+        <choice value="TLog">TLog</choice>
+        <choice value="MasterServer">MasterServer</choice>
+        <choice value="MasterProxyServer">MasterProxyServer (for version &lt;7)</choice>
+        <choice value="Resolver">Resolver</choice>
+        <choice value="ClusterController">ClusterController</choice>
+        <choice value="SharedTLog">SharedTLog</choice>
+        <choice value="LogRouter">LogRouter</choice>
+        <choice value="Coordinator">Coordinator</choice>
+        <choice value="StorageServer">StorageServer</choice>
+        <choice value="CommitProxyServer">CommitProxyServer (for version 7+)</choice>
+        <choice value="GrvProxyServer">GrvProxyServer (for version 7+)</choice>
+        <valuePrefix>As="</valuePrefix>
+        <valueSuffix>"</valueSuffix>
+        <delimiter> OR </delimiter>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type=ConnectionTimedOut OR Type=ConnectionTimeout) 
+| replace *:tls with * in PeerAddr 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($TimeoutbyMachineTableSourceRoleToken$)) 
+    | dedup ID] 
+| join PeerAddr 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($TimeoutbyMachineTableDestinationRoleToken$)) 
+    | dedup ID 
+    | rename Machine as PeerAddr] 
+| timechart useother=0 span=$TimeoutEventByMachineTableTimeSpanToken$ count by Machine</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="height">233</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 2: Timeout/TimedOut event distribution grouped by destination (PeerAddr)</title>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type=ConnectionTimedOut OR Type=ConnectionTimeout) 
+| replace *:tls with * in PeerAddr 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($TimeoutbyMachineTableSourceRoleToken$)) 
+    | dedup ID] 
+| join PeerAddr 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($TimeoutbyMachineTableDestinationRoleToken$)) 
+    | dedup ID 
+    | rename Machine as PeerAddr] 
+| timechart useother=0 span=$TimeoutEventByMachineTableTimeSpanToken$ count by PeerAddr</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="height">219</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 17: Check Type=ConnectionTimedOut OR Type=ConnectionTimeout events between transaction roles in the recovery (including the role that refresh/begin/end in the timespan)</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type=ConnectionTimedOut OR Type=ConnectionTimeout) 
+| replace *:tls with * in PeerAddr 
+| stats count as TotalTimeouts by Machine PeerAddr 
+| table Machine PeerAddr TotalTimeouts 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($TimeoutbyMachineTableSourceRoleToken$)) 
+    | stats latest(*) by ID 
+    | rename latest(*) as * 
+    | eval Role = As."/".ID."/".Type.Transition."/".strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+    | stats list(Role) AS MachineRoleLatestEvent BY Machine 
+        ] 
+| join PeerAddr 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="Role" AND ($TimeoutbyMachineTableDestinationRoleToken$)) 
+    | stats latest(*) by ID 
+    | rename latest(*) as * 
+    | eval Role = As."/".ID."/".Type.Transition."/".strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+    | stats list(Role) AS PeerRoleLatestEvent BY Machine 
+    | rename Machine AS PeerAddr
+        ] 
+| table Machine PeerAddr TotalTimeouts MachineRoleLatestEvent PeerRoleLatestEvent</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 18: Proxy 0</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Type="ProxyReplies" OR Type="CommitProxyReplies") AND FirstProxy="True" 
+| eval DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+| table WorkerID LogGroup FirstProxy Time DateTime 
+| sort 0 -Time 
+| join type=left WorkerID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type="Role" AND As="Worker" AND Transition="Refresh" 
+    | dedup ID 
+    | rename ID as WorkerID 
+    | stats list(*) by WorkerID 
+    | rename list(*) as * 
+    | table WorkerID Machine Roles] 
+| table WorkerID Machine Roles LogGroup FirstProxy Time DateTime 
+| join type=left Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type="Role" AND (As="MasterProxyServer" OR As="CommitProxyServer") AND Transition="Refresh" 
+    | dedup ID 
+    | rename ID as ProxyID 
+    | table Machine ProxyID] 
+| table ProxyID Machine LogGroup FirstProxy</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 19: Latest Role Events on the input Machine (Input Machine, like 172.27.113.121:4500)</title>
+      <input type="text" token="SearchMachineToken" searchWhenChanged="true">
+        <label>Machine (IP:PORT)</label>
+      </input>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="Role" AND Machine=$SearchMachineToken$ 
+| stats latest(*) by ID Transition 
+| rename latest(*) as * 
+| eval DateTime=strftime(Time, "%Y-%m-%d %H:%M:%S.%Q (%Z)") 
+| table DateTime Machine ID Transition As Roles LogGroup Error ErrorDescription Reason 
+| sort 0 -DateTime 
+| fillnull value="-"</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Chart 3: severity&gt;=20 event distribution (including roles that refresh/begin/end in the timespan)</title>
+      <input type="text" token="BadEvents" searchWhenChanged="true">
+        <label>Events</label>
+        <default>*</default>
+      </input>
+      <input type="multiselect" token="BadEventRoleToken" searchWhenChanged="true">
+        <label>Roles</label>
+        <choice value="TLog">TLog</choice>
+        <choice value="MasterServer">MasterServer</choice>
+        <choice value="MasterProxyServer">MasterProxyServer (for version &lt;7)</choice>
+        <choice value="Resolver">Resolver</choice>
+        <choice value="ClusterController">ClusterController</choice>
+        <choice value="SharedTLog">SharedTLog</choice>
+        <choice value="LogRouter">LogRouter</choice>
+        <choice value="Coordinator">Coordinator</choice>
+        <choice value="StorageServer">StorageServer</choice>
+        <choice value="CommitProxyServer">CommitProxyServer (for version 7+)</choice>
+        <choice value="GrvProxyServer">GrvProxyServer (for version 7+)</choice>
+        <valuePrefix>As="</valuePrefix>
+        <valueSuffix>"</valueSuffix>
+        <delimiter> OR </delimiter>
+      </input>
+      <input type="dropdown" token="BadEventChartBy" searchWhenChanged="true">
+        <label>By</label>
+        <choice value="Type">EventType</choice>
+        <choice value="Machine">Machine</choice>
+        <choice value="Severity">Severity</choice>
+        <default>Type</default>
+      </input>
+      <input type="text" token="BadEventChartTimeSpanToken" searchWhenChanged="true">
+        <label>TimeSpan</label>
+        <default>5s</default>
+      </input>
+      <chart>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Severity&gt;10 AND $BadEvents$
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type="Role" AND ($BadEventRoleToken$)
+    | dedup ID | table Machine] 
+| table Machine Type Severity _time
+| timechart useother=0 span=$BadEventChartTimeSpanToken$ count by $BadEventChartBy$</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="charting.chart">line</option>
+        <option name="charting.drilldown">none</option>
+        <option name="height">305</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Table 20: Check severity&gt;20 events of roles in the recovery (including the role that refresh/begin/end in the timespan)</title>
+      <table>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ 
+    Severity&gt;10 
+| stats count by Machine Type 
+| rename count as Count 
+| join Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ 
+        Type="Role" AND ($BadEventRoleToken$)
+    | dedup ID 
+    | eval Role=As."-".ID 
+    | stats list(Role) by Machine 
+    | rename list(Role) as Roles 
+    | table Machine Roles] 
+| table Type Count Roles Machine 
+| sort -Count</query>
+          <earliest>$ReoveryTime.earliest$</earliest>
+          <latest>$ReoveryTime.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
+        <option name="wrap">false</option>
+      </table>
+    </panel>
+  </row>
+</form>
\ No newline at end of file
diff --git a/contrib/observability_splunk_dashboard/transaction_latency.xml b/contrib/observability_splunk_dashboard/transaction_latency.xml
new file mode 100644
index 0000000000..99b551f2c9
--- /dev/null
+++ b/contrib/observability_splunk_dashboard/transaction_latency.xml
@@ -0,0 +1,247 @@
+<form theme="dark">
+  <label>FoundationDB - Tracing GRV and Commit Long Latency of CC Transactions (6.3 and 7.0+) (DEV)</label>
+  <description>Design for ClusterController issued transactions.</description>
+  <fieldset submitButton="false" autoRun="true">
+    <input type="text" token="Index" searchWhenChanged="true">
+      <label>Index</label>
+      <default></default>
+    </input>
+    <input type="text" token="LogGroup" searchWhenChanged="true">
+      <label>LogGroup</label>
+      <default>*</default>
+    </input>
+    <input type="text" token="transactionID">
+      <label>Hex Transaction ID (optional)</label>
+      <default>*</default>
+    </input>
+    <input type="time" token="time_token" searchWhenChanged="true">
+      <label>Time span</label>
+      <default>
+        <earliest>@d</earliest>
+        <latest>now</latest>
+      </default>
+    </input>
+  </fieldset>
+  <row>
+    <panel>
+      <title>All Transactions (Currently, this table also does not cover getrange operation and the operation which not do commit).</title>
+      <table>
+        <title>for FDB 6.3 and 7.0+</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ ID=$transactionID$
+    (Type="TransactionAttachID" OR Type="GetValueAttachID" OR Type="CommitAttachID") 
+| eval To=case(Type=="TransactionAttachID", "0"."-".To, Type="GetValueAttachID", "1"."-".To, Type=="CommitAttachID", "2"."-".To) 
+| stats list(To) by ID 
+| rename list(To) as ToList 
+| table ID ToList 
+| eval Count = mvcount(ToList) 
+| search Count=3 
+| eval To0=mvindex(ToList,0), To1=mvindex(ToList,1), To2=mvindex(ToList,2), To0=split(To0,"-"), To1=split(To1,"-"), To2=split(To2,"-"), GrvID=case(mvindex(To0, 0)=="0", mvindex(To0, 1), mvindex(To1, 0)=="0", mvindex(To1, 1), mvindex(To2, 0)=="0", mvindex(To2, 1)), ReadID=case(mvindex(To0, 0)=="1", mvindex(To0, 1), mvindex(To1, 0)=="1", mvindex(To1, 1), mvindex(To2, 0)=="1", mvindex(To2, 1)), CommitID=case(mvindex(To0, 0)=="2", mvindex(To0, 1), mvindex(To1, 0)=="2", mvindex(To1, 1), mvindex(To2, 0)=="2", mvindex(To2, 1)) 
+| table ID GrvID ReadID CommitID 
+| join GrvID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="TransactionDebug" AND Location="NativeAPI.getConsistentReadVersion.Before") 
+    | rename ID as GrvID 
+    | rename Time as BeginTime 
+    | table GrvID BeginTime
+        ] 
+| join GrvID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="TransactionDebug" AND Location="NativeAPI.getConsistentReadVersion.After") 
+    | rename ID as GrvID 
+    | rename Time as GRVDoneTime 
+    | table GrvID GRVDoneTime
+        ] 
+| join ReadID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="GetValueDebug" AND Location="NativeAPI.getValue.After") 
+    | rename ID as ReadID 
+    | rename Time as ReadDoneTime 
+    | table ReadID ReadDoneTime
+        ] 
+| join CommitID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        (Type="CommitDebug" AND Location="NativeAPI.commit.After") 
+    | rename ID as CommitID 
+    | rename Time as CommitDoneTime 
+    | table CommitID CommitDoneTime
+        ] 
+| rename ID as TransactionID 
+| eval BeginToGRVDone = GRVDoneTime-BeginTime, GRVDoneToReadDone = ReadDoneTime-GRVDoneTime, ReadDoneToCommitDone = CommitDoneTime-ReadDoneTime, Duration=CommitDoneTime-BeginTime, BeginTimeScope=BeginTime-1, EndTimeScope=CommitDoneTime+1, BeginDateTime=strftime(BeginTime, "%Y-%m-%d %H:%M:%S.%Q (%Z)")
+| table TransactionID Duration BeginDateTime BeginToGRVDone GRVDoneToReadDone ReadDoneToCommitDone Duration  GrvID ReadID CommitID BeginTimeScope EndTimeScope | sort -Duration</query>
+          <earliest>$time_token.earliest$</earliest>
+          <latest>$time_token.latest$</latest>
+        </search>
+        <option name="drilldown">cell</option>
+        <drilldown>
+          <set token="BeginTime">$row.BeginTimeScope$</set>
+          <set token="EndTime">$row.EndTimeScope$</set>
+          <set token="ReadID">$row.ReadID$</set>
+          <set token="GrvID">$row.GrvID$</set>
+          <set token="CommitID">$row.CommitID$</set>
+        </drilldown>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Step1: GRV</title>
+      <table>
+        <title>for FDB 6.3 and 7.0+</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ 
+    Type="TransactionDebug" AND (NOT MasterProxyServer.masterProxyServerCore.GetRawCommittedVersion) 
+AND (ID=$GrvID$ OR ID= 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type="TransactionAttachID" AND ID=$GrvID$
+    | return $To])
+| table Time Type ID Location Machine Roles
+| eventstats min(Time) as MinTime
+| eval Delta = Time - MinTime, Order = case(Location=="NativeAPI.getConsistentReadVersion.Before", 0, Location like "%ProxyServer.queueTransactionStartRequests.Before", 1, Location=="MasterProxyServer.masterProxyServerCore.Broadcast", 2, Location=="GrvProxyServer.transactionStarter.AskLiveCommittedVersionFromMaster", 2.1, Location like "%ProxyServer.getLiveCommittedVersion.confirmEpochLive", 3, Location=="MasterServer.serveLiveCommittedVersion.GetRawCommittedVersion", 4, Location like "%ProxyServer.getLiveCommittedVersion.After", 5, Location=="NativeAPI.getConsistentReadVersion.After", 6)
+| table Time Delta Order Type ID Location Machine Roles
+| sort 0 Order
+| table Machine Location Delta Time Roles ID Type</query>
+          <earliest>$BeginTime$</earliest>
+          <latest>$EndTime$</latest>
+        </search>
+        <option name="drilldown">none</option>
+      </table>
+    </panel>
+    <panel>
+      <title>Step1: (Only for FDB v6.3): GRV --- Get Committed Version (MasterProxyServer.masterProxyServerCore.GetRawCommittedVersion Events)</title>
+      <table>
+        <title>only for FDB 6.3</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="TransactionDebug" AND Location="MasterProxyServer.masterProxyServerCore.GetRawCommittedVersion" 
+    AND ID= 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type="TransactionAttachID" AND ID=$GrvID$ 
+    | return $To] 
+| table Time Type ID Location Machine Roles
+| eventstats min(Time) as MinTime
+| eval Delta = Time - MinTime
+| sort 0 -Time
+| table Machine Delta Time Roles ID Type</query>
+          <earliest>$BeginTime$</earliest>
+          <latest>$EndTime$</latest>
+        </search>
+        <option name="drilldown">none</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Step2: GetValue</title>
+      <table>
+        <title>for FDB 6.3 and 7.0+</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$ Type="GetValueDebug" AND ID=$ReadID$ 
+| eventstats min(Time) as MinTime 
+| eval Delta = Time-MinTime 
+| table Machine Location Delta Time Roles ID Type 
+| eval Order=case(Location=="NativeAPI.getKeyLocation.Before", 0, Location=="NativeAPI.getKeyLocation.After", 1, Location=="NativeAPI.getValue.Before", 2, Location=="storageServer.received", 3, Location=="getValueQ.DoRead", 4, Location=="getValueQ.AfterVersion", 5, Location=="Reader.Before", 6, Location=="Reader.After", 7, Location=="getValueQ.AfterRead", 8, Location=="NativeAPI.getValue.After", 9, Location=="NativeAPI.getValue.Error", 10) 
+| sort 0 Order
+| table Machine Location Delta Time Roles ID Type</query>
+          <earliest>$time_token.earliest$</earliest>
+          <latest>$time_token.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Step3: Commit</title>
+      <table>
+        <title>for FDB 6.3 and 7.0+</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    Type="CommitDebug" AND (ID=$CommitID$ OR ID= 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type="CommitAttachID" AND ID=$CommitID$ 
+    | return $To]) 
+
+| table Time Type ID Location Machine Roles 
+| eventstats min(Time) as MinTime
+| eval Delta = Time-MinTime
+| table Machine Location Delta Time Roles ID Type
+| eval Order=case(Location=="NativeAPI.commit.Before", 0, Location like "%ProxyServer.batcher", 1, Location like "%ProxyServer.commitBatch.Before", 2, Location like "%ProxyServer.commitBatch.GettingCommitVersion", 3, Location like "%ProxyServer.commitBatch.GotCommitVersion", 4, Location=="Resolver.resolveBatch.Before", 5, Location=="Resolver.resolveBatch.AfterQueueSizeCheck", 6, Location=="Resolver.resolveBatch.AfterOrderer", 7, Location=="Resolver.resolveBatch.After", 8, Location like "%ProxyServer.commitBatch.AfterResolution", 8.5, Location like "%ProxyServer.commitBatch.ProcessingMutations", 9, Location like "%ProxyServer.commitBatch.AfterStoreCommits", 10, Location=="TLogServer.tLogCommit.BeforeWaitForVersion", 11, Location=="TLogServer.tLogCommit.Before", 12, Location=="TLogServer.tLogCommit.AfterTLogCommit", 13, Location=="TLogServer.tLogCommit.After", 14, Location like "%ProxyServer.commitBatch.AfterLogPush", 15, Location=="NativeAPI.commit.After", 16)
+| sort 0 Order
+| table Machine Location Delta Time Roles ID Type</query>
+          <earliest>$BeginTime$</earliest>
+          <latest>$EndTime$</latest>
+        </search>
+        <option name="drilldown">none</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Step3: Commit --- Resolver</title>
+      <table>
+        <title>for FDB 6.3 and 7.0+</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Location="Resolver*") 
+| join ID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type="CommitAttachID" AND ID= 
+        [ search index=$Index$ LogGroup=$LogGroup$
+            Type="CommitAttachID" AND ID=$CommitID$ 
+        | return $To] 
+    | rename To as ID 
+    | table ID] 
+| eventstats min(Time) as MinTime 
+| eval Delta = Time-MinTime 
+| eval Order=case(Location=="Resolver.resolveBatch.Before", 5, Location=="Resolver.resolveBatch.AfterQueueSizeCheck", 6, Location=="Resolver.resolveBatch.AfterOrderer", 7, Location=="Resolver.resolveBatch.After", 8)
+| sort 0 Time Order
+| stats list(*) by Type ID Machine Roles
+| rename list(*) as *
+| eval T1=mvindex(Time, 0), T2=mvindex(Time, 3), Duration=T2-T1 | sort -Duration
+| table Machine Roles Duration Location Delta Time
+| join type=left Machine 
+    [ search index=$Index$ LogGroup=$LogGroup$ Type=ProcessMetrics 
+    | dedup Machine, DCID 
+    | rename DCID as DataCenter 
+    | table Machine DataCenter]
+| table Machine DataCenter Roles Duration Location Delta Time</query>
+          <earliest>$time_token.earliest$</earliest>
+          <latest>$time_token.latest$</latest>
+        </search>
+        <option name="drilldown">none</option>
+      </table>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>Step3: Commit --- Commit to TLogs (CommitDebug Events), grouped by Machine and sorted by Duration</title>
+      <table>
+        <title>for FDB 6.3 and 7.0+</title>
+        <search>
+          <query>index=$Index$ LogGroup=$LogGroup$
+    (Location="TLog*") 
+| join ID 
+    [ search index=$Index$ LogGroup=$LogGroup$
+        Type="CommitAttachID" AND ID= 
+        [ search index=$Index$ LogGroup=$LogGroup$
+            Type="CommitAttachID" AND ID=$CommitID$ 
+        | return $To] 
+    | rename To as ID 
+    | table ID] 
+| eventstats min(Time) as MinTime 
+| eval Delta = Time-MinTime 
+| sort 0 Time
+| stats list(*) by Type ID Machine Roles
+| rename list(*) as *
+| eval T1=mvindex(Time, 0), T2=mvindex(Time, 3), Duration=T2-T1 | sort -Duration
+| table Machine Roles Duration Location Delta Time</query>
+          <earliest>$BeginTime$</earliest>
+          <latest>$EndTime$</latest>
+        </search>
+        <option name="count">10</option>
+        <option name="drilldown">none</option>
+      </table>
+    </panel>
+  </row>
+</form>
\ No newline at end of file
diff --git a/contrib/pkg_tester/test_fdb_pkgs.py b/contrib/pkg_tester/test_fdb_pkgs.py
index 08ccd35aa6..178f84d93c 100644
--- a/contrib/pkg_tester/test_fdb_pkgs.py
+++ b/contrib/pkg_tester/test_fdb_pkgs.py
@@ -165,7 +165,6 @@ def centos_image_with_fdb_helper(versioned: bool) -> Iterator[Optional[Image]]:
         container = Container("centos:7", initd=True)
         for rpm in rpms:
             container.copy_to(rpm, "/opt")
-        container.run(["bash", "-c", "yum update -y"])
         container.run(
             ["bash", "-c", "yum install -y prelink"]
         )  # this is for testing libfdb_c execstack permissions
@@ -327,7 +326,7 @@ def test_execstack_permissions_libfdb_c(linux_container: Container, snapshot):
         [
             "bash",
             "-c",
-            "execstack -q $(ldconfig -p | grep libfdb_c | awk '{print $(NF)}')",
+            "execstack -q $(ldconfig -p | grep libfdb_c.so | awk '{print $(NF)}')",
         ]
     )
 
diff --git a/contrib/sqlite/sqlite3.amalgamation.c b/contrib/sqlite/sqlite3.amalgamation.c
index 2b0058be91..2992e521e2 100644
--- a/contrib/sqlite/sqlite3.amalgamation.c
+++ b/contrib/sqlite/sqlite3.amalgamation.c
@@ -87009,7 +87009,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
   }
   sqlite3_query_plan[nQPlan] = 0;
   nQPlan = 0;
-#endif /* SQLITE_TEST // Testing and debugging use only */
+#endif /* SQLITE_TEST // Testing and debugging use only */");
 
   /* Record the continuation address in the WhereInfo structure.  Then
   ** clean up and return.
diff --git a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py
index e704cacb72..79534596b5 100644
--- a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py
+++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py
@@ -49,7 +49,7 @@ PROTOCOL_VERSION_6_1 = 0x0FDB00B061060001
 PROTOCOL_VERSION_6_2 = 0x0FDB00B062010001
 PROTOCOL_VERSION_6_3 = 0x0FDB00B063010001
 PROTOCOL_VERSION_7_0 = 0x0FDB00B070010001
-PROTOCOL_VERSION_7_1 = 0x0FDB00B071010001
+PROTOCOL_VERSION_7_1 = 0x0FDB00B071010000
 PROTOCOL_VERSION_7_2 = 0x0FDB00B072000000
 supported_protocol_versions = frozenset([PROTOCOL_VERSION_5_2, PROTOCOL_VERSION_6_0, PROTOCOL_VERSION_6_1,
                                          PROTOCOL_VERSION_6_2, PROTOCOL_VERSION_6_3, PROTOCOL_VERSION_7_0,
@@ -194,7 +194,7 @@ class BaseInfo(object):
         if protocol_version >= PROTOCOL_VERSION_6_3:
             self.dc_id = bb.get_bytes_with_length()
         if protocol_version >= PROTOCOL_VERSION_7_1:
-            if bb.get_bytes(1):
+            if bb.get_bool():
                 self.tenant = bb.get_bytes_with_length()
 
 class GetVersionInfo(BaseInfo):
@@ -244,6 +244,11 @@ class CommitInfo(BaseInfo):
         self.read_snapshot_version = bb.get_long()
         if protocol_version >= PROTOCOL_VERSION_6_3:
             self.report_conflicting_keys = bb.get_bool()
+        
+        if protocol_version >= PROTOCOL_VERSION_7_1:
+            lock_aware = bb.get_bool()
+            if bb.get_bool():
+                spanId = bb.get_bytes(16)
 
 
 class ErrorGetInfo(BaseInfo):
@@ -279,6 +284,12 @@ class ErrorCommitInfo(BaseInfo):
         if protocol_version >= PROTOCOL_VERSION_6_3:
             self.report_conflicting_keys = bb.get_bool()
 
+        if protocol_version >= PROTOCOL_VERSION_7_1:
+            lock_aware = bb.get_bool()
+            if bb.get_bool():
+                spanId = bb.get_bytes(16)
+
+
 class UnsupportedProtocolVersionError(Exception):
     def __init__(self, protocol_version):
         super().__init__("Unsupported protocol version 0x%0.2X" % protocol_version)
diff --git a/contrib/tsan.suppressions b/contrib/tsan.suppressions
new file mode 100644
index 0000000000..2078f7e8c6
--- /dev/null
+++ b/contrib/tsan.suppressions
@@ -0,0 +1,5 @@
+# ThreadSanitizer suppressions file for FDB
+# https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions
+
+# FDB signal handler is not async-signal safe
+signal:crashHandler
diff --git a/design/LoadBalancing/LoadBalancing.md b/design/LoadBalancing/LoadBalancing.md
new file mode 100644
index 0000000000..5100437825
--- /dev/null
+++ b/design/LoadBalancing/LoadBalancing.md
@@ -0,0 +1,227 @@
+# Load Balancing in FoundationDB
+
+## Introduction
+
+FoundationDB is a distributed key-value database. A FoundationDB cluster is constituted by one or more processes over one or more physical machines, where each process is a *worker* and takes certain *role*s, such as coordinator, proxy, TLog, storage server, etc., in the system.
+
+The interpocess communications (IPC) between the processes are supported by the [`flow`](https://github.com/apple/foundationdb/tree/main/flow) infrastructure. In the `flow` context, each process will expose one or more *interface*(s). Each interface is able to accept given type of *request*s, and *reply* `Void`, requested data or error. The interfaces and the corresponding request/reply pairs forms the IPC protocol of FoundationDB.
+
+In many cases, the same request can be proceed by multiple processes, e.g. all commit proxies can accept commit requests, and multiple storage server processes can provide values for a given key in double/triple redundancy mode. A load balancer (LB) can be used to distribute the requests over the possible interfaces, preventing one or a few processes getting overloaded. The interface candidates are also referred as *alternative*s. The LB is also able to react when one or more interfaces are (temporarily) unavailable by retrying, or re-routing the request to other candidates. The interface candidates are also known as *alternative*s.
+
+Two LBs are provided in FoundationDB: `basicLoadBalance` and `loadBalance`, both defined in [`LoadBalance.actor.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/LoadBalance.actor.h). The `basicLoadBalance` is a simple load balancer which each interface is equally chosen; while the `loadBalance` accepts a model object, which provides [datacenter](https://apple.github.io/foundationdb/configuration.html#configuring-regions) (DC) awaring balancing algorithms, allowing requests being sent to interfaces in the same DC.
+
+In the following sections, the two LBs will be discussed in details.
+
+## `basicLoadBalance`
+
+`basicLoadBalance` implements a simple load balancing algorithm. It applies to
+
+* Commit proxy interface
+* GetReadVersion proxy interface
+* ConfigFollower interface
+
+Here, the interfaces are assumed to be always *fresh*, i.e. the list of the servers is fixed.
+
+```mermaid
+graph LR
+    H0{Has alternatives?}
+    H1[Pick an alternative]
+    H2[Backoff]
+    H3[Request]
+    H4([Reply])
+    H5([Error])
+    H6([Never])
+    H((Start)) --> H0
+    H0 --No--> H6
+    H0 --Yes--> H1
+    H1 --No healthy alternatives--> H2 --Retry--> H1
+    H1 --Has alternative--> H3 --Success--> H4
+    H3 --Exception--> H5
+    H3 --Broken Promise --> H2
+```
+
+### Alternative pick algorithm
+
+In `basicLoadBalance`, a *best* alternative is picked and used at the beginning. At this stage, this alternative is randomly picked among all alternatives. If the best alternative does not work, it will iteratively try other interfaces, see [here](#picking-up-an-alternative-in-basic-load-balancing-algorithm).
+
+## `loadBalance`
+
+`loadBalance` provides a more sophisticated implementation of load balancing. In addition of the basic load balancing, it also provides a variety of features:
+
+* Support for Test Storage Server ([TSS](https://github.com/apple/foundationdb/blob/main/documentation/sphinx/source/tss.rst))
+* Datacenter awaring alternative election
+* Recording the latency and penalty from interfaces, and [prioritize the interfaces based on previously stored data](#with-queuemodel).
+* Able to handle timeouts and SS exceptions with retries.
+
+Currently it is used for
+
+* Storage Server interface
+* BlobWorker interface
+
+
+
+```mermaid
+graph LR
+    H((Start))
+    H0{Has alternatives?}
+    H1[Choose initial candidates]
+    H4([Never])
+    H5[pick an alternative]
+    H6[Send request]
+    H7[Wait for available alternative]
+    H8([Response])
+    H9([All alternatives failed])
+
+    H --> H0 --No--> H4
+    H0 --Yes--> H1
+    H1 --> H5
+    H5 --Has alternative--> H6
+    H5 --No alternative-->H7
+    H6 --Success--> H8
+    H6 --Failure--> H5
+    H7 --At least one alternative--> H5
+    H7 --> H9
+```
+
+Note:
+
+* The response could be either a reply, or an `Error`, e.g. `process_behind` or `request_maybe_delivered`.
+
+### Choose initial candidates
+
+Two initial candidates will be picked before the requests start. They will be selected as the first two alternatives for the load balancer. If both of them failed, other alternatives are used in a round-robin way.
+
+#### No `QueueModel`
+
+If no `QueueModel` is provided, the initial candidates are picked randomly. The first candidate, or the *best* alternative, will be the one that in the same DC, if possible.
+
+#### With `QueueModel`
+
+`QueueModel` holds information about each candidate related to future version, latency and penalty.
+
+* If the storage server is returning a future version error, it is marked as not available until some certain time.
+* Penalty is reported by storage server in each response (see `storageserver.actor.cpp:StorageServer::getPenalty`). It is determined by the write queue length and the durability lagging.
+
+If `QueueModel` exists, the candidates will be picked base on the penalty. Workers with high penalties will be avoided when picking the first two candidates.
+
+### Pick an alternative
+
+The alternatives are chosen in the round-robin way when the first two candidates failed. If all alternatives failed, a flag is set, and if the next request fails with `process_behind`, the caller will receive the `process_behind` error.
+
+### Send requests to workers
+
+Here it is assumed that there are at least one alternative available. If no alternative is available, the LB will wait.
+
+```mermaid
+graph LR
+    H((start))
+    H0{Is first request}
+    H1[Send first request]
+   	H2([Response])
+   	H3[Pick up next alternative]
+   	H4[Send additional request]
+
+    H --> H3
+    H3 -->H0
+   	H0 --Yes--> H1
+   	H1 --Success--> H2
+   	H1 --Timeout--> H3
+   	H0 --No--> H4
+   	H4 --First request succeed--> H2
+   	H4 --Second request succeed--> H2
+   	H4 --Additional request failed--> H3
+```
+
+The first request has a timeout option. If the LB is not able to retrieve the response within the timout, more requests will be sent to secondary and other available interfaces. If the first request failed, it is reset and the next request will be considered as the first request. Certain types of errors can also be returned as response, e.g. `request_may_be_delivered` or `process_behind`, which may not trigger a load-balancer retry.
+
+### Wait for available alternative
+
+When there is no alternatives available, the load balancer may wait until at least one interface is up.
+
+```mermaid
+graph LR
+    H0((start))
+    H1{Is first request in-flight}
+    H2[Wait for the first request]
+    H3([Response])
+    H4([Retry])
+    H5[Wait for alternatives]
+    H6([all_alternatives_failed])
+    
+    H0 --> H1
+    H1 --Yes--> H2
+    H1 --No--> H5
+    H5 --Timeout-->H6
+    H5 --Success-->H4
+    H2 --Success-->H3
+    H2 --Failed-->H4
+```
+
+Note that "Wait for alternatives" will only timeout if the alternatives are always not fresh, i.e. this only happens when accessing storage servers. LB will throw `all_alternatives_failed` when timeout in this case.
+
+#### Requests
+
+Original requests in `loadBalancer` are wrapped by `LoadBalance.actor.h:RequestData`. It provides the following additional operations besides the original `flow` request:
+
+* TSS support if `QueueModel` is available
+* Translate some errors into `maybe_delivered`, `process_behind` or retries
+* Update the `QueueModel` information including latency, penalty, etc.
+
+## Appendix
+
+### Picking an alternative in basic load balancing algorithm
+
+The following script simulates the alternative picking up algorithm. The chosen alternatives will be printed out one-by-one. The `loadBalance` function uses a similar approach, though the interfaces in the same DC are used firstly.
+
+```python
+#! /usr/bin/env python3
+
+import random
+import time
+
+
+class Alternatives:
+
+    def __init__(self, num_alternatives):
+        self._size = num_alternatives
+    
+    def size(self):
+        return self._size
+
+    def get_best(self):
+        return random.randint(0, self._size - 1)
+
+
+# Entry
+NUM_ALTERNATIVES = 10
+alts = Alternatives(NUM_ALTERNATIVES)
+
+best_alt = alts.get_best()
+next_alt = random.randint(0, alts.size() - 2)
+if next_alt >= best_alt:
+    next_alt += 1
+start_alt = next_alt
+start_distance = (best_alt + alts.size() - start_alt) % alts.size()
+use_alt = None
+
+print("best_alt = {}".format(best_alt))
+print("start_alt = {}".format(start_alt))
+print("start_distance = {}".format(start_distance))
+
+while True:
+    for alt_num in range(0, alts.size()):
+        use_alt = next_alt
+        if next_alt == start_alt:
+            print("  Going back to the start_alt")
+            use_alt = best_alt
+        elif (next_alt + alts.size() - start_alt) % alts.size() <= start_distance:
+            print("  Entering start_distance")
+            use_alt = (next_alt + alts.size() - 1) % alts.size()
+        
+        print("Attempting alt: {}".format(use_alt))
+
+        # Next loop
+        next_alt = (next_alt + 1) % alts.size()
+        time.sleep(.2)
+```
+
diff --git a/design/LoadBalancing/LoadBalancing.pdf b/design/LoadBalancing/LoadBalancing.pdf
new file mode 100644
index 0000000000..a0ef91dbd7
Binary files /dev/null and b/design/LoadBalancing/LoadBalancing.pdf differ
diff --git a/design/backup-dataFormat.md b/design/backup-dataFormat.md
index 73942e41ef..f6f9a0338c 100644
--- a/design/backup-dataFormat.md
+++ b/design/backup-dataFormat.md
@@ -54,7 +54,7 @@ NOTE: All blocks except for the final block will have one last  value which will
 
 The code related to how a range file is written is in the `struct RangeFileWriter` in `namespace fileBackup`.
 
-The code that decodes a range block is in `ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset, int len)`.
+The code that decodes a range block is in `ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset, int len, Database cx)`.
 
 
 ### Data format in a log file
diff --git a/design/data-distributor-internals.md b/design/data-distributor-internals.md
index 4e67534dfa..ccaba537b6 100644
--- a/design/data-distributor-internals.md
+++ b/design/data-distributor-internals.md
@@ -20,7 +20,7 @@ Data distribution manages the lifetime of storage servers, decides which storage
 
 **RelocateShard (`struct RelocateShard`)**: A `RelocateShard` records the key range that need to be moved among servers and the data movement’s priority. DD always move shards with higher priorities first.
 
-**Data distribution queue (`struct DDQueueData`)**: It receives shards to be relocated (i.e., RelocateShards), decides which shard should be moved to which server team, prioritizes the data movement based on relocate shard’s priority, and controls the progress of data movement based on servers’ workload.
+**Data distribution queue (`struct DDQueue`)**: It receives shards to be relocated (i.e., RelocateShards), decides which shard should be moved to which server team, prioritizes the data movement based on relocate shard’s priority, and controls the progress of data movement based on servers’ workload.
 
 **Special keys in the system keyspace**: DD saves its state in the system keyspace to recover from failure and to ensure every process (e.g., commit proxies, tLogs and storage servers) has a consistent view of which storage server is responsible for which key range.
 
@@ -69,10 +69,11 @@ When a data distribution role is created, it recovers the states of the previous
 ### When to move keys?
 
 Keys can be moved from a server to another for several reasons:
-(1) DD moves keys from overutilized servers to underutilized servers, where a server’s utilization is defined as the server’s disk usage;
-(2) DD splits or merges shards in order to rebalance the disk usage of servers;
-(3) DD removes redundant teams when the team number is larger than the desired number;
-(4) DD repairs the replication factor by duplicate shards from a server to another when servers in a team fail.
+(1) DD moves keys from disk-overutilized servers to disk-underutilized servers, where a server’s disk-utilization is defined as the server’s disk space usage;
+(2) DD moves keys from read-busy servers to read-cold servers if read-aware data distribution is enabled;
+(3) DD splits or merges shards in order to rebalance the disk usage of servers;
+(4) DD removes redundant teams when the team number is larger than the desired number;
+(5) DD repairs the replication factor by duplicate shards from a server to another when servers in a team fail.
 
 Actors are created to monitor the reasons of key movement:
 (1) `MountainChopper` and `ValleyFiller` actors periodically measure a random server team’s utilization and rebalance the server’s keys among other servers;
@@ -93,3 +94,84 @@ The data movement from one server (called source server) to another (called dest
 (2) The destination server will issue transactions to read the shard range and write the key-value pairs back. The key-value will be routed to the destination server and saved in the server’s storage engine;
 (3) DD removes the source server from the shard’s ownership by modifying the system keyspace;
 (4) DD removes the shard’s information owned by the source server from the server’s team information (i.e., *shardsAffectedByTeamFailure*).
+
+# Read-aware Data Distribution
+
+## Motivation
+Before FDB 7.2, when the data distributor wants to rebalance shard, it only considers write bandwidth when choosing source and destination team, and the moved shard is chosen randomly. There are several cases where uneven read distribution from users causes a small subset of servers to be busy with read requests. This motivates the data distributor considering read busyness to minimize the read load unevenness.
+
+## When does read rebalance happen
+The data distributor will periodically check whether the read rebalance is needed. The conditions of rebalancing are 
+* the **worst CPU usage of source team >= 0.15** , which means the source team is somewhat busy;
+* the ongoing relocation is less than the parallelism budget. `queuedRelocation[ priority ] < countLimit (default 50)`;
+* the source team is not throttled to be a data movement source team. `( now() - The last time the source team was selected ) * time volumn (default 20) > read sample interval (2 min default)`;
+* the read load difference between source team and destination team is larger than 30% of the source team load;
+
+## Metrics definition
+* READ_LOAD = ceil(READ_BYTES_PER_KSECOND / PAGE_SIZE) 
+* READ_IMBALANCE = ( MAX READ_LOAD / AVG READ_LOAD )
+* MOVE_SCORE = READ_DENSITY = READ_BYTES_PER_KSECOND / SHARD_BYTE
+
+The aim for read-aware data distributor is to minimize the IMBALANCE while not harm the disk utilization balance.
+
+## Which shard to move
+Basically, the MountainChopper will handle read-hot shards distribution with following steps:
+1. The MountainChopper chooses **the source team** with the largest READ_LOAD while it satisfies HARD_CONSTRAINT, then check whether rebalance is needed; 
+    * Hard constraint:
+        * Team is healthy
+        * The last time this team was source team is larger than (READ_SAMPLE_INTERVAL / MOVEMENT_PER_SAMPLE)
+        * The worst CPU usage of source team >= 0.15
+2. Choose the destination team for moving
+    * Hard constraint:
+        * Team is healthy
+        * The team’s available space is larger than the median free space
+    * Goals
+        * The destination team has the least LOAD in a random team set while it satisfies HARD_CONSTRAINT;
+3. Select K shards on the source team of which 
+    a. `LOAD(shard) < (LOAD(src) - LOAD(dest)) * READ_REBALANCE_MAX_SHARD_FRAC `; 
+    b. `LOAD(shard) > AVG(SourceShardLoad)`; 
+    c. with the highest top-K `MOVE_SCORE`; 
+
+    We use 3.a and 3.b to set a eligible shard bandwidth for read rebalance moving. If the upper bound is too large, it’ll just make the hot shard shift to another team but not even the read load. If the upper bound is small, we’ll just move some cold shards to other servers, which is also not helpful. The default value of READ_REBALANCE_MAX_SHARD_FRAC is 0.2 (up to 0.5) which is decided based on skewed workload test.
+4. Issue relocation request to move a random shard in the top k set. If the maximum limit of read-balance movement is reached, give up this relocation.
+
+Note: The ValleyFiller chooses a source team from a random set with the largest LOAD, and a destination team with the least LOAD.
+
+## Performance Test and Summary
+### Metrics to measure
+1. StorageMetrics trace event report “FinishedQueries” which means the current storage server finishes how many read operations. The rate of FinishedQueries is what we measure first. The better the load balance is, the more similar the FinishedQueries rate across all storage servers.
+CPU utilization. This metric is in a positive relationship with “FinishedQueries rate”. A even “FinishedQueries” generally means even CPU utilization in the read-only scenario.
+2. Data movement size. We want to achieve load balance with as little movement as possible;
+3. StandardDeviation(FinishedQueries). It indicates how much difference read load each storage server has.
+
+### Typical Test Setup
+120GB data, key=32B, value=200B; Single replica; 8 SS (20%) serves 80% read; 8 SS servers 60% write; 4 servers are both read and write hot;  TPS=100000, 7 read/txn + 1 write/txn; 
+
+### Test Result Summary and Recommendation 
+* With intersected sets of read-hot and write-hot servers, read-aware DD even out the read + write load on the double-hot (be both read and write hot) server, which means the converged write load is similar to disk rebalance only algorithm.
+* Read-aware DD will balance the read workload under the read-skew scenario. Starting from an imbalance `STD(FinishedQueries per minute)=16k`,the best result it can achieve is `STD(FinishedQueries per minute) = 2k`.
+* The typical movement size under a read-skew scenario is 100M ~ 600M under default KNOB value `READ_REBALANCE_MAX_SHARD_FRAC=0.2, READ_REBALANCE_SRC_PARALLELISM = 20`. Increasing those knobs may accelerate the converge speed with the risk of data movement churn, which overwhelms the destination and over-cold the source.
+* The upper bound of `READ_REBALANCE_MAX_SHARD_FRAC` is 0.5. Any value larger than 0.5 can result in hot server switching.
+* When needing a deeper diagnosis of the read aware DD, `BgDDMountainChopper_New`, and `BgDDValleyFiller_New` trace events are where to go.
+
+## Data Distribution Diagnosis Q&A
+* Why Read-aware DD hasn't been triggered when there's a read imbalance? 
+  * Check `BgDDMountainChopper_New`, `BgDDValleyFiller_New` `SkipReason` field.
+* The Read-aware DD is triggered, and some data movement happened, but it doesn't help the read balance. Why? 
+  * Need to figure out which server is selected as the source and destination. The information is in `BgDDMountainChopper*`, `BgDDValleyFiller*`  `DestTeam` and `SourceTeam` field.
+  * Also, the `DDQueueServerCounter` event tells how many times a server being a source or destination (defined in 
+  ```c++
+  enum CountType : uint8_t { ProposedSource = 0, QueuedSource, LaunchedSource, LaunchedDest };
+  ```
+  ) for different relocation reason (`Other`, `RebalanceDisk` and so on) in different phase within `DD_QUEUE_COUNTER_REFRESH_INTERVAL` (default 60) seconds. For example, 
+  ```xml
+  <Event Severity="10" Time="1659974950.984176" DateTime="2022-08-08T16:09:10Z" Type="DDQueueServerCounter" ID="0000000000000000" ServerId="0000000000000004" OtherPQSD="0 1 3 2" RebalanceDiskPQSD="0 0 1 4" RebalanceReadPQSD="2 0 0 5" MergeShardPQSD="0 0 1 0" SizeSplitPQSD="0 0 5 0" WriteSplitPQSD="1 0 0 0" ThreadID="9733255463206053180" Machine="0.0.0.0:0" LogGroup="default" Roles="TS" />
+  ```
+  `RebalanceReadPQSD="2 0 0 5"` means server `0000000000000004` has been selected as for read balancing for twice, but it's not queued and executed yet. This server also has been a destination for read balancing for 5 times in the past 1 min. Note that the field will be skipped if all 4 numbers are 0. To avoid spammy traces, if is enabled with knob `DD_QUEUE_COUNTER_SUMMARIZE = true`, event `DDQueueServerCounterTooMany` will summarize the unreported servers that involved in launched relocations (aka. `LaunchedSource`, `LaunchedDest` count are non-zero):
+    ```xml
+    <Event Severity="10" Time="1660095057.995837" DateTime="2022-08-10T01:30:57Z" Type="DDQueueServerCounterTooMany" ID="0000000000000000" RemainedLaunchedSources="000000000000007f,00000000000000d9,00000000000000e8,000000000000014c,0000000000000028,00000000000000d6,0000000000000067,000000000000003e,000000000000007d,000000000000000a,00000000000000cb,0000000000000106,00000000000000c1,000000000000003c,000000000000016e,00000000000000e4,000000000000013c,0000000000000016,0000000000000179,0000000000000061,00000000000000c2,000000000000005a,0000000000000001,00000000000000c9,000000000000012a,00000000000000fb,0000000000000146," RemainedLaunchedDestinations="0000000000000079,0000000000000115,000000000000018e,0000000000000167,0000000000000135,0000000000000139,0000000000000077,0000000000000118,00000000000000bb,0000000000000177,00000000000000c0,000000000000014d,000000000000017f,00000000000000c3,000000000000015c,00000000000000fb,0000000000000186,0000000000000157,00000000000000b6,0000000000000072,0000000000000144," ThreadID="1322639651557440362" Machine="0.0.0.0:0" LogGroup="default" Roles="TS" />
+    ```
+* How to track the lifecycle of a relocation attempt for balancing?
+  * First find the TraceId fields in `BgDDMountainChopper*`, `BgDDValleyFiller*`, which indicates a relocation is triggered.
+  * (Only when enabled) Find the `QueuedRelocation` event with the same `BeginPair` and `EndPair` as the original `TraceId`. This means the relocation request is queued.
+  * Find the `RelocateShard` event whose `BeginPair`, `EndPair` field is the same as `TraceId`. This event means the relocation is ongoing.
diff --git a/design/dynamic-knobs.md b/design/dynamic-knobs.md
new file mode 100644
index 0000000000..00fe39e725
--- /dev/null
+++ b/design/dynamic-knobs.md
@@ -0,0 +1,420 @@
+# Dynamic Knobs
+
+This document is largely adapted from original design documents by Markus
+Pilman and Trevor Clinkenbeard.
+
+## Background
+
+FoundationDB parameters control the behavior of the database, including whether
+certain features are available and the value of internal constants. Parameters
+will be referred to as knobs for the remainder of this document. Currently,
+these knobs are configured through arguments passed to `fdbserver` processes,
+often controlled by `fdbmonitor`. This has a number of problems:
+
+1. Updating knobs involves updating `foundationdb.conf` files on each host in a
+   cluster. This has a lot of overhead and typically requires external tooling
+   for large scale changes.
+2. All knob changes require a process restart.
+3. We can't easily track the history of knob changes.
+
+## Overview
+
+The dynamic knobs project creates a strictly serializable quorum-based
+configuration database stored on the coordinators. Each `fdbserver` process
+specifies a configuration path and applies knob overrides from the
+configuration database for its specified classes.
+
+### Caveats
+
+The configuration database explicitly does not support the following:
+
+1. A high load. The update rate, while not specified, should be relatively low.
+2. A large amount of data. The database is meant to be relatively small (under
+   one megabyte). Data is not sharded and every coordinator stores a complete
+   copy.
+3. Concurrent writes. At most one write can succeed at a time, and clients must
+   retry their failed writes.
+
+## Design
+
+### Configuration Path
+
+Each `fdbserver` process can now include a `--config_path` argument specifying
+its configuration path. A configuration path is a hierarchical list of
+configuration classes specifying which knob overrides the `fdbserver` process
+should apply from the configuration database. For example:
+
+```bash
+$ fdbserver --config_path classA/classB/classC ...
+```
+
+Knob overrides follow descending priority:
+
+1. Manually specified command line knobs.
+2. Individual configuration class overrides.
+  * Subdirectories override parent directories. For example, if the
+    configuration path is `az-1/storage/gp3`, the `gp3` configuration takes
+    priority over the `storage` configuration, which takes priority over the
+    `az-1` configuration.
+3. Global configuration knobs.
+4. Default knob values.
+
+#### Example
+
+For example, imagine an `fdbserver` process run as follows:
+
+```bash
+$ fdbserver --datadir /mnt/fdb/storage/4500 --logdir /var/log/foundationdb --public_address auto:4500 --config_path az-1/storage/gp3 --knob_disable_asserts false
+```
+
+And the configuration database contains:
+
+| ConfigClass | KnobName            | KnobValue |
+|-------------|---------------------|-----------|
+| az-2        | page_cache_4k       | 8e9       |
+| storage     | min_trace_severity  | 20        |
+| az-1        | compaction_interval | 280       |
+| storage     | compaction_interval | 350       |
+| az-1        | disable_asserts     | true      |
+| \<global\>  | max_metric_size     | 5000      |
+| gp3         | max_metric_size     | 1000      |
+
+The final configuration for the process will be:
+
+| KnobName            |  KnobValue  | Explanation |
+|---------------------|-------------|-------------|
+| page_cache_4k       | \<default\> | The configuration database knob override for `az-2` is ignored, so the compiled default is used |
+| min_trace_severity  | 20          | Because the `storage` configuration class is part of the process’s configuration path, the corresponding knob override is applied from the configuration database |
+| compaction_interval | 350         | The `storage` knob override takes precedence over the `az-1` knob override |
+| disable_asserts     | false       | This knob is manually overridden, so all other overrides are ignored |
+| max_metric_size     | 1000        | Knob overrides for specific configuration classes take precedence over global knob overrides, so the global override is ignored |
+
+### Clients
+
+Clients can write to the configuration database using transactions.
+Configuration database transactions are differentiated from regular
+transactions through specification of the `USE_CONFIG_DATABASE` database
+option.
+
+In configuration transactions, the client uses the tuple layer to interact with
+the configuration database. Keys are tuples of size two, where the first item
+is the configuration class being written, and the second item is the knob name.
+The value should be specified as a string. It will be converted to the
+appropriate type based on the declared type of the knob being set.
+
+Below is a sample Python script to write to the configuration database.
+
+```python
+import fdb
+
+fdb.api_version(720)
+
+@fdb.transactional
+def set_knob(tr, knob_name, knob_value, config_class, description):
+        tr['\xff\xff/description'] = description
+        tr[fdb.tuple.pack((config_class, knob_name,))] = knob_value
+
+# This function performs two knob changes transactionally.
+@fdb.transactional
+def set_multiple_knobs(tr):
+        tr['\xff\xff/description'] = 'description'
+        tr[fdb.tuple.pack((None, 'min_trace_severity',))] = '10'
+        tr[fdb.tuple.pack(('az-1', 'min_trace_severity',))] = '20'
+
+db = fdb.open()
+db.options.set_use_config_database()
+
+set_knob(db, 'min_trace_severity', '10', None, 'description')
+set_knob(db, 'min_trace_severity', '20', 'az-1', 'description')
+```
+
+### Disable the Configuration Database
+
+The configuration database includes both client and server changes and is
+enabled by default. Thus, to disable the configuration database, changes must
+be made to both.
+
+#### Server
+
+The configuration database can be disabled by specifying the ``fdbserver``
+command line option ``--no-config-db``. Note that this option must be specified
+for *every* ``fdbserver`` process.
+
+#### Client
+
+The only client change from the configuration database is as part of the change
+coordinators command. The change coordinators command is not considered
+successful until the configuration database is readable on the new
+coordinators. This will cause the change coordinators command to hang if run
+against a database with dynamic knobs disabled. To disable the client side
+configuration database liveness check, specify the ``--no-config-db`` flag when
+changing coordinators. For example:
+
+```
+fdbcli> coordinators auto --no-config-db
+```
+
+## Status
+
+The current state of the configuration database is output as part of `status
+json`. The configuration path for each process can be determined from the
+``command_line`` key associated with each process.
+
+Sample from ``status json``:
+
+```
+"configuration_database" : {
+    "commits" : [
+        {
+            "description" : "set some knobs",
+            "timestamp" : 1659570000,
+            "version" : 1
+        },
+        {
+            "description" : "make some other changes",
+            "timestamp" : 1659570000,
+            "version" : 2
+        }
+    ],
+    "last_compacted_version" : 0,
+    "most_recent_version" : 2,
+    "mutations" : [
+        {
+            "config_class" : "<global>",
+            "knob_name" : "min_trace_severity",
+            "knob_value" : "int:5",
+            "type" : "set",
+            "version" : 1
+        },
+        {
+            "config_class" : "<global>",
+            "knob_name" : "compaction_interval",
+            "knob_value" : "double:30.000000",
+            "type" : "set",
+            "version" : 1
+        },
+        {
+            "config_class" : "az-1",
+            "knob_name" : "compaction_interval",
+            "knob_value" : "double:60.000000",
+            "type" : "set",
+            "version" : 1
+        },
+        {
+            "config_class" : "<global>",
+            "knob_name" : "compaction_interval",
+            "type" : "clear",
+            "version" : 2
+        },
+        {
+            "config_class" : "<global>",
+            "knob_name" : "update_node_timeout",
+            "knob_value" : "double:4.000000",
+            "type" : "set",
+            "version" : 2
+        }
+    ],
+    "snapshot" : {
+        "<global>" : {
+            "min_trace_severity" : "int:5",
+            "update_node_timeout" : "double:4.000000"
+        },
+        "az-1" : {
+            "compaction_interval" : "double:60.000000"
+        }
+    }
+}
+```
+
+After compaction, ``status json`` would show:
+
+```
+"configuration_database" : {
+    "commits" : [
+    ],
+    "last_compacted_version" : 2,
+    "most_recent_version" : 2,
+    "mutations" : [
+    ],
+    "snapshot" : {
+        "<global>" : {
+            "min_trace_severity" : "int:5",
+            "update_node_timeout" : "double:4.000000"
+        },
+        "az-1" : {
+            "compaction_interval" : "double:60.000000"
+        }
+    }
+}
+```
+
+## Detailed Implementation
+
+The configuration database is implemented as a replicated state machine living
+on the coordinators. This allows configuration database transactions to
+continue to function in the event of a catastrophic loss of the transaction
+subsystem.
+
+To commit a transaction, clients run the two phase Paxos protocol. First, the
+client asks for a live version from a quorum of coordinators. When a
+coordinator receives a request for its live version, it increments its local
+live version by one and returns it to the client. Then, the client submits its
+writes at the live version it received in the previous step. A coordinator will
+accept the commit if it is still on the same live version. If a majority of
+coordinators accept the commit, it is considered committed.
+
+### Coordinator
+
+Each coordinator runs a ``ConfigNode`` which serves as a replica storing one
+full copy of the configuration database. Coordinators never communicate with
+other coordinators while processing configuration database transactions.
+Instead, the client runs the transaction and determines when it has quorum
+agreement.
+
+Coordinators serve the following ``ConfigTransactionInterface`` to allow
+clients to read from and write to the configuration database.
+
+#### ``ConfigTransactionInterface``
+| Request          | Request fields                                                 | Reply fields                                                                                  | Explanation                                                                                                      |
+|------------------|----------------------------------------------------------------|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------|
+| GetGeneration    | (coordinatorsHash)                                             | (generation) or (coordinators_changed error)                                                  | Get a new read version. This read version is used for all future requests in the transaction                     |
+| Get              | (configuration class, knob name, coordinatorsHash, generation) | (knob value or empty) or (coordinators_changed error) or (transaction_too_old error)          | Returns the current value stored at the specified configuration class and knob name, or empty if no value exists |
+| GetConfigClasses | (coordinatorsHash, generation)                                 | (configuration classes) or (coordinators_changed error) or (transaction_too_old error)        | Returns a list of all configuration classes stored in the configuration database                                 |
+| GetKnobs         | (configuration class, coordinatorsHash, generation)            | (knob names) or (coordinators_changed error) or (transaction_too_old error)                   | Returns a list of all knob names stored for the provided configuration class                                     |
+| Commit           | (mutation list, coordinatorsHash, generation)                  | ack or (coordinators_changed error) or (commit_unknown_result error) or (not_committed error) | Commit mutations set by the transaction                                                                          |
+
+Coordinators also serve the following ``ConfigFollowerInterface`` to provide
+access to (and modification of) their current state. Most interaction through
+this interface is done by the cluster controller through its
+``IConfigConsumer`` implementation living on the ``ConfigBroadcaster``.
+
+#### ``ConfigFollowerInterface``
+| Request               | Request fields                                                       | Reply fields                                                                            | Explanation                                                                                                         |
+|-----------------------|----------------------------------------------------------------------|-----------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|
+| GetChanges            | (lastSeenVersion, mostRecentVersion)                                 | (mutation list, version) or (version_already_compacted error) or (process_behind error) | Request changes since the last seen version, receive a new most recent version, as well as recent mutations         |
+| GetSnapshotAndChanges | (mostRecentVersion)                                                  | (snapshot, snapshotVersion, changes)                                                    | Request the full configuration database, in the form of a base snapshot and changes to apply on top of the snapshot |
+| Compact               | (version)                                                            | ack                                                                                     | Compact mutations up to the provided version                                                                        |
+| Rollforward           | (rollbackTo, lastKnownCommitted, target, changes, specialZeroQuorum) | ack or (version_already_compacted error) or (transaction_too_old error)                 | Rollback/rollforward mutations on a node to catch it up with the majority                                           |
+| GetCommittedVersion   | ()                                                                   | (registered, lastCompacted, lastLive, lastCommitted)                                    | Request version information from a ``ConfigNode``                                                                   |
+| Lock                  | (coordinatorsHash)                                                   | ack                                                                                     | Lock a ``ConfigNode`` to prevent it from serving requests during a coordinator change                               |
+
+### Cluster Controller
+
+The cluster controller runs a singleton ``ConfigBroadcaster`` which is
+responsible for periodically polling the ``ConfigNode``s for updates, then
+broadcasting these updates to workers through the ``ConfigBroadcastInterface``.
+When workers join the cluster, they register themselves and their
+``ConfigBroadcastInterface`` with the broadcaster. The broadcaster then pushes
+new updates to registered workers.
+
+The ``ConfigBroadcastInterface`` is also used by ``ConfigNode``s to register
+with the ``ConfigBroadcaster``. ``ConfigNode``s need to register with the
+broadcaster because the broadcaster decides when the ``ConfigNode`` may begin
+serving requests, based on global information about status of other
+``ConfigNode``s. For example, if a system with three ``ConfigNode``s suffers a
+fault where one ``ConfigNode`` loses data, the faulty ``ConfigNode``  should
+not be allowed to begin serving requests again until it has been rolled forward
+and is up to date with the latest state of the configuration database.
+
+#### ``ConfigBroadcastInterface``
+
+| Request    | Request fields                                             | Reply fields                  | Explanation                                                                                 |
+|------------|------------------------------------------------------------|-------------------------------|---------------------------------------------------------------------------------------------|
+| Snapshot   | (snapshot, version, restartDelay)                          | ack                           | A snapshot of the configuration database sent by the broadcaster to workers                 |
+| Changes    | (changes, mostRecentVersion, restartDelay)                 | ack                           | A list of changes up to and including mostRecentVersion, sent by the broadcaster to workers |
+| Registered | ()                                                         | (registered, lastSeenVersion) | Sent by the broadcaster to new ``ConfigNode``s to determine their registration status       |
+| Ready      | (snapshot, snapshotVersion, liveVersion, coordinatorsHash) | ack                           | Sent by the broadcaster to new ``ConfigNode``s to allow them to start serving requests      |
+
+### Worker
+
+Each worker runs a ``LocalConfiguration`` instance which receives and applies
+knob updates from the ``ConfigBroadcaster``. The local configuration maintains
+a durable ``KeyValueStoreMemory`` containing the following:
+
+* The latest known configuration version
+* The most recently used configuration path
+* All knob overrides corresponding to the configuration path at the latest known version
+
+Once a worker starts, it will:
+
+* Apply manually set knobs
+* Read its local configuration file
+  * If the stored configuration path does not match the configuration path
+    specified on the command line, delete the local configuration file
+  * Otherwise, apply knob updates from the local configuration file. Manually
+    specified knobs will not be overridden
+  * Register with the broadcaster to receive new updates for its configuration
+    classes
+    * Persist these updates when received and restart if necessary
+
+### Knob Atomicity
+
+All knobs are classified as either atomic or non-atomic. Atomic knobs require a
+process restart when changed, while non-atomic knobs do not.
+
+### Compaction
+
+``ConfigNode``s store individual mutations in order to be able to update other,
+out of date ``ConfigNode``s without needing to send a full snapshot. Each
+configuration database commit also contains additional metadata such as a
+timestamp and a text description of the changes being made. To keep the size of
+the configuration database manageable, a compaction process runs periodically
+(defaulting to every five minutes) which compacts individual mutations into a
+simplified snapshot of key-value pairs. Compaction is controlled by the
+``ConfigBroadcaster``, using information it peridiodically requests from
+``ConfigNode``s. Compaction will only compact up to the minimum known version
+across *all* ``ConfigNode``s. This means that if one ``ConfigNode`` is
+permanently partitioned from the ``ConfigBroadcaster`` or from clients, no
+compaction will ever take place.
+
+### Rollback / Rollforward
+
+It is necessary to be able to roll ``ConfigNode``s backward and forward with
+respect to their committed versions due to the nature of quorum logic and
+unreliable networks.
+
+Consider a case where a client commit gets persisted durably on one out of
+three ``ConfigNode``s (assume commit messages to the other two nodes are lost).
+Since the value is not committed on a majority of ``ConfigNode``s, it cannot be
+considered committed. But it is also incorrect to have the value persist on one
+out of three nodes as future commits are made. In this case, the most common
+result is that the ``ConfigNode`` will be rolled back when the next commit from
+a different client is made, and then rolled forward to contain the data from
+the commit. ``PaxosConfigConsumer`` contains logic to recognize ``ConfigNode``
+minorities and update them to match the quorum.
+
+### Changing Coordinators
+
+Since the configuration database lives on the coordinators and the
+[coordinators can be
+changed](https://apple.github.io/foundationdb/configuration.html#configuration-changing-coordination-servers),
+it is necessary to copy the configuration database from the old to the new
+coordinators during such an event. A coordinator change performs the following
+steps in regards to the configuration database:
+
+1. Write ``\xff/coordinatorsKey`` with the new coordinators string. The key
+   ``\xff/previousCoordinators`` contains the current (old) set of
+   coordinators.
+2. Lock the old ``ConfigNode``s so they can no longer serve client requests.
+3. Start a recovery, causing a new cluster controller (and therefore
+   ``ConfigBroadcaster``) to be selected.
+4. Read ``\xff/previousCoordinators`` on the ``ConfigBroadcaster`` and, if
+   present, read an up-to-date snapshot of the configuration database on the
+   old coordinators.
+5. Determine if each registering ``ConfigNode`` needs an up-to-date snapshot of
+   the configuration database sent to it, based on its reported version and the
+   snapshot version of the database received from the old coordinators.
+   * Some new coordinators which were also coordinators in the previous
+     configuration may not need a snapshot.
+6. Send ready requests to new ``ConfigNode``s, including an up-to-date snapshot
+   if necessary. This allows the new coordinators to begin serving
+   configuration database requests from clients.
+
+## Testing
+
+The ``ConfigDatabaseUnitTests`` class unit test a number of different
+configuration database dimensions.
+
+The ``ConfigIncrement`` workload tests contention between clients attempting to
+write to the configuration database, paired with machine failure and
+coordinator changes.
diff --git a/design/global-tag-throttling.md b/design/global-tag-throttling.md
new file mode 100644
index 0000000000..ad7750a3cc
--- /dev/null
+++ b/design/global-tag-throttling.md
@@ -0,0 +1,131 @@
+## Global Tag Throttling
+
+When the `GLOBAL_TAG_THROTTLING` knob is enabled, the ratekeeper will use the [transaction tagging feature](https://apple.github.io/foundationdb/transaction-tagging.html) to throttle tags according to the global tag throttling algorithm. This page describes the implementation of this algorithm.
+
+### Tag Quotas
+The global tag throttler bases throttling decisions on "quotas" provided by clients through the tag quota API. Each tag quota has two components:
+
+* Reserved quota
+* Total quota
+
+The global tag throttler cannot throttle tags to a throughput below the reserved quota, and it cannot allow throughput to exceed the total quota.
+
+### Cost
+Internally, the units for these quotas are "page costs", computed as follows. The "page cost" of a read operation is computed as:
+
+```
+readCost = ceiling(bytesRead / CLIENT_KNOBS->READ_COST_BYTE_FACTOR);
+```
+
+The "page cost" of a write operation is computed as:
+
+```
+writeCost = SERVER_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO * ceiling(bytesWritten / CLIENT_KNOBS->WRITE_COST_BYTE_FACTOR);
+```
+
+Here `bytesWritten` includes cleared bytes. The size of range clears is estimated at commit time.
+
+### Tuple Layer
+Tag quotas are stored inside of the system keyspace (with prefix `\xff/tagQuota/`). They are stored using the tuple layer, in a tuple of form: `(reservedQuota, totalQuota)`. There is currently no custom code in the bindings for manipulating these system keys. However, in any language for which bindings are available, it is possible to use the tuple layer to manipulate tag quotas.
+
+### fdbcli
+The easiest way for an external client to interact with tag quotas is through `fdbcli`. To get the quota (in bytes/second) of a particular tag, run the following command:
+
+```
+fdbcli> quota get <tag> [reserved_throughput|total_throughput]
+```
+
+To set the quota through `fdbcli`, run:
+
+```
+fdbcli> quota set <tag> [reserved_throughput|total_throughput] <bytes_per_second>
+```
+
+Note that the quotas are specified in terms of bytes/second, and internally converted to page costs:
+
+```
+page_cost_quota = ceiling(byte_quota / CLIENT_KNOBS->READ_COST_BYTE_FACTOR)
+```
+
+### Limit Calculation
+The transaction budget that ratekeeper calculates and distributes to clients (via GRV proxies) for each tag is calculated based on several intermediate rate calculations, outlined in this section.
+
+* Reserved Rate: Based on reserved quota and the average transaction cost, a reserved TPS rate is computed for each tag.
+
+* Desired Rate: Based on total quota and the average transaction cost, a desired TPS rate is computed for each tag.
+
+* Limiting Rate: When a storage server is near saturation, tags contributing notably to the workload on this storage server will receive a limiting TPS rate, computed to relieve the workload on the storage server.
+
+* Target Rate: The target rate is the cluster-wide rate enforced by the global tag throttler. This rate is computed as:
+
+```
+targetTps = max(reservedTps, min(desiredTps, limitingTps));
+```
+
+* Per-Client Rate: While the target rate represents the cluster-wide desired throughput according to the global tag throttler, this budget must be shared across potentially many clients. Therefore, based on observed throughput from various clients, each client will receive an equal budget based on a per-client, per-tag rate computed by the global tag throttler. This rate is in the end what will be sent to clients to enforce throttling.
+
+## Implementation
+
+### Stat collection
+The transaction rates and costs of all transactions must be visible to the global tag throttler. Whenever a client tags a transaction, sampling is performed to determine whether to attach the tag to messages sent to storage servers and commit proxies.
+
+For read operations that are sampled (with probability `CLIENT_KNOBS->READ_TAG_SAMPLE_RATE`), read costs are aggregated on storage servers using the `TransactionTagCounter` class. This class tracks the busyness of the top-k tags affecting the storage server with read load (here `k` is determined by `SERVER_KNOBS->SS_THROTTLE_TAGS_TRACKED`). Storage servers periodically send per-tag read cost statistics to the ratekeeper through `StorageQueuingMetricsReply` messages.
+
+For write operations that are sampled (with probability `COMMIT_SAMPLE_COST`), write costs are aggregated on commit proxies in the `ProxyCommitData::ssTrTagCommitCost` object. Per-storage, per-tag write cost statistics are periodically sent from commit proxies to the ratekeeper through `ReportCommitCostEstimationRequest` messages.
+
+The ratekeeper tracks per-storage, per-tag cost statistics in the `GlobalTagThrottlerImpl::throughput` object.
+
+The ratekeeper must also track the rate of transactions performed with each tag. Each GRV proxy agreggates a per-tag counter of transactions started (without sampling). These are sent to the ratekeeper through `GetRateInfoRequest` messages. The global tag throttler then tracks per tag transaction rates in the `GlobalTagThrottlerImpl::tagStatistics` object.
+
+### Average Cost Calculation
+Quotas are expressed in terms of cost, but because throttling is enforced at the beginning of transactions, budgets need to be calculated in terms of transactions per second. To make this conversion, it is necessary to track the average cost of transactions (per-tag, and per-tag on a particular storage server).
+
+Both cost and transaction counters are smoothed using the `Smoother` class to provide stability over time. The "smoothing interval" can be modified through `SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME`.
+
+### Reserved Rate Calculation
+The global tag throttler periodically reads reserved quotas from the system keyspace. Using these reserved quotas and the average cost of transactions with the given tag, a reserved TPS rate is computed. Read and write rates are aggregated as follows:
+
+```
+reservedTps = max(reservedReadTps, reservedWriteTps);
+```
+
+### Desired Rate Calculation
+Similar to reserved rate calculation, the total quota is read from the system key space. Then, using the average cost of transactions with the given tag, a desired TPS rate is computed. Read and write rates are aggregated as follows:
+
+```
+desiredTps = min(desiredReadTps, desiredWriteTps);
+```
+
+### Limiting Rate Calculation
+In addition to tag busyness statistics, the `StorageQueuingMetricsReply` messages sent from storage servers to the ratekeeper also contain metrics on the health of storage servers. The ratekeeper uses these metrics as part of its calculation of a global transaction rate (independent of tag throttling).
+
+The global tag throttler also makes use of these metrics to compute a "throttling ratio" for each storage server. This throttling ratio is computed in `StorageQueueInfo::getThrottlingRatio`. The global tag throttler uses the throttling ratio for each tracked storage server to compute a "limiting transaction rate" for each combination of storage server and tag.
+
+In the "healthy" case where no metrics are near saturation, the throttling ratio will be an empty `Optional<double>`, indicating that the storage server is not near saturation. If, on the other hand, the metrics indicate approaching saturation, the throttling ratio will be a number between 0 and 2 indicating the ratio of current throughput the storage server can serve. In this case, the global tag throttler looks at the current cost being served by the storage server, multiplies it by the throttling ratio, and computes a limiting cost for the storage server. Among all tags using significant resources on this storage server, this limiting cost is divided up according to the relative total quotas allocated to these tags. Next, a transaction limit is determined for each tag, based on how much the average transaction for the given tag affects the given storage server.
+
+These per-tag, per-storage limiting transaction rates are aggregated to compute per-tag limiting transaction rates:
+
+```
+limitingTps(tag) = min{limitingTps(tag, storage) : all storage servers}
+```
+
+If the throttling ratio is empty for all storage servers affected by a tag, then the per-tag, per-storage limiting TPS rate is also empty. In this case the target rate for this tag is simply the desired rate.
+
+If an individual zone is unhealthy, it may cause the throttling ratio for storage servers in that zone to shoot up. This should not be misinterpretted as a workload issue that requires active throttling. Therefore, the zone with the worst throttling ratios is ignored when computing the limiting transaction rate for a tag (similar to the calculation of the global transaction limit in `Ratekeeper::updateRate`).
+
+### Client Rate Calculation
+The smoothed per-client rate for each tag is tracked within `GlobalTagThrottlerImpl::PerTagStatistics`. Once a target rate has been computed, this is passed to `GlobalTagThrotterImpl::PerTagStatistics::updateAndGetPerClientRate` which adjusts the per-client rate. The per-client rate is meant to limit the busiest clients, so that at equilibrium, the per-client rate will remain constant and the sum of throughput from all clients will match the target rate.
+
+## Testing
+The `GlobalTagThrottling.toml` test provides a simple end-to-end test using the global tag throttler. Quotas are set using the internal tag quota API in the `GlobalTagThrottling` workload. This is run in parallel with the `ReadWrite` workload, which tags transactions. The number of `transaction_tag_throttled` errors is reported, along with the throughput, which should be roughly predictable based on the quota parameters chosen. 
+
+In addition to this end-to-end test, there is a suite of unit tests with the `/GlobalTagThrottler/` prefix. These tests run in a mock environment, with mock storage servers providing simulated storage queue statistics and tag busyness reports. Mock clients simulate workload on these mock storage servers, and get throttling feedback directly from a global tag throttler which is monitoring the mock storage servers.
+
+In each test, the `GlobalTagThrottlerTesting::monitor` function is used to periodically check whether or not a desired equilibrium state has been reached. If the desired state is reached and maintained for a sufficient period of time, the test passes. If the unit test is unable to reach this desired equilibrium state before a timeout, the test will fail. Commonly, the desired state is for the global tag throttler to report a client rate sufficiently close to the desired rate specified as an input to the `GlobalTagThrottlerTesting::rateIsNear` function.
+
+## Visibility
+
+### Tracing
+On the ratekeeper, every `SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL` seconds, the ratekeeper will call `GlobalTagThrottler::getClientRates`. At the end of the rate calculation for each tag, a trace event of type `GlobalTagThrottler_GotClientRate` is produced. This trace event reports the relevant inputs that went in to the rate calculation, and can be used for debugging.
+
+On storage servers, every `SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL` seconds, there are `BusyReadTag` events for every tag that has sufficient read cost to be reported to the ratekeeper. Both cost and fractional busyness are reported.
diff --git a/design/special-key-space.md b/design/special-key-space.md
index 7cdcfe460d..be104915fe 100644
--- a/design/special-key-space.md
+++ b/design/special-key-space.md
@@ -32,10 +32,10 @@ public:
     explicit SKRExampleImpl(KeyRangeRef kr): SpecialKeyRangeReadImpl(kr) {
         // Our implementation is quite simple here, the key-value pairs are formatted as:
         // \xff\xff/example/<country_name> : <capital_city_name>
-        CountryToCapitalCity[LiteralStringRef("USA")] = LiteralStringRef("Washington, D.C.");
-        CountryToCapitalCity[LiteralStringRef("UK")] = LiteralStringRef("London");
-        CountryToCapitalCity[LiteralStringRef("Japan")] = LiteralStringRef("Tokyo");
-        CountryToCapitalCity[LiteralStringRef("China")] = LiteralStringRef("Beijing");
+        CountryToCapitalCity["USA"_sr] = "Washington, D.C."_sr;
+        CountryToCapitalCity["UK"_sr] = "London"_sr;
+        CountryToCapitalCity["Japan"_sr] = "Tokyo"_sr;
+        CountryToCapitalCity["China"_sr] = "Beijing"_sr;
     }
     // Implement the getRange interface
     Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
@@ -58,7 +58,7 @@ private:
 };
 // Instantiate the function object
 // In development, you should have a function object pointer in DatabaseContext(DatabaseContext.h) and initialize in DatabaseContext's constructor(NativeAPI.actor.cpp)
-const KeyRangeRef exampleRange(LiteralStringRef("\xff\xff/example/"), LiteralStringRef("\xff\xff/example/\xff"));
+const KeyRangeRef exampleRange("\xff\xff/example/"_sr, "\xff\xff/example/\xff"_sr);
 SKRExampleImpl exampleImpl(exampleRange);
 // Assuming the database handler is `cx`, register to special-key-space
 // In development, you should register all function objects in the constructor of DatabaseContext(NativeAPI.actor.cpp)
@@ -67,16 +67,16 @@ cx->specialKeySpace->registerKeyRange(exampleRange, &exampleImpl);
 state ReadYourWritesTransaction tr(cx);
 // get
 Optional<Value> res1 = wait(tr.get("\xff\xff/example/Japan"));
-ASSERT(res1.present() && res.getValue() == LiteralStringRef("Tokyo"));
+ASSERT(res1.present() && res.getValue() == "Tokyo"_sr);
 // getRange
 // Note: for getRange(key1, key2), both key1 and key2 should prefixed with \xff\xff
 // something like getRange("normal_key", "\xff\xff/...") is not supported yet
-RangeResult res2 = wait(tr.getRange(LiteralStringRef("\xff\xff/example/U"), LiteralStringRef("\xff\xff/example/U\xff")));
+RangeResult res2 = wait(tr.getRange("\xff\xff/example/U"_sr, "\xff\xff/example/U\xff"_sr));
 // res2 should contain USA and UK
 ASSERT(
     res2.size() == 2 &&
-    res2[0].value == LiteralStringRef("London") &&
-    res2[1].value == LiteralStringRef("Washington, D.C.")
+    res2[0].value == "London"_sr &&
+    res2[1].value == "Washington, D.C."_sr
 );
 ```
 
diff --git a/documentation/sphinx/conf.py b/documentation/sphinx/conf.py
index 031c7d6f67..04ed43d87b 100644
--- a/documentation/sphinx/conf.py
+++ b/documentation/sphinx/conf.py
@@ -69,7 +69,7 @@ release = root.find(".//{http://schemas.microsoft.com/developer/msbuild/2003}Ver
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -185,7 +185,7 @@ html_show_copyright = True
 htmlhelp_basename = 'FoundationDB'
 
 # Disable permalinks
-html_add_permalinks = ""
+html_permalinks = False
 
 
 # -- Options for LaTeX output --------------------------------------------------
diff --git a/documentation/sphinx/extensions/rubydomain.py b/documentation/sphinx/extensions/rubydomain.py
index 1e5fb0bce4..e6c849cc41 100755
--- a/documentation/sphinx/extensions/rubydomain.py
+++ b/documentation/sphinx/extensions/rubydomain.py
@@ -42,7 +42,7 @@ from docutils.parsers.rst import directives, Directive
 
 from sphinx import addnodes
 from sphinx.roles import XRefRole
-from sphinx.locale import l_, _
+from sphinx.locale import _
 from sphinx.domains import Domain, ObjType, Index
 from sphinx.directives import ObjectDescription
 from sphinx.util.nodes import make_refnode
@@ -83,18 +83,18 @@ class RubyObject(ObjectDescription):
     }
 
     doc_field_types = [
-        TypedField('parameter', label=l_('Parameters'),
+        TypedField('parameter', label=_('Parameters'),
                    names=('param', 'parameter', 'arg', 'argument'),
                    typerolename='obj', typenames=('paramtype', 'type')),
-        TypedField('variable', label=l_('Variables'), rolename='obj',
+        TypedField('variable', label=_('Variables'), rolename='obj',
                    names=('var', 'ivar', 'cvar'),
                    typerolename='obj', typenames=('vartype',)),
-        GroupedField('exceptions', label=l_('Raises'), rolename='exc',
+        GroupedField('exceptions', label=_('Raises'), rolename='exc',
                      names=('raises', 'raise', 'exception', 'except'),
                      can_collapse=True),
-        Field('returnvalue', label=l_('Returns'), has_arg=False,
+        Field('returnvalue', label=_('Returns'), has_arg=False,
               names=('returns', 'return')),
-        Field('returntype', label=l_('Return type'), has_arg=False,
+        Field('returntype', label=_('Return type'), has_arg=False,
               names=('rtype',)),
     ]
 
@@ -493,8 +493,8 @@ class RubyModuleIndex(Index):
     """
 
     name = 'modindex'
-    localname = l_('Ruby Module Index')
-    shortname = l_('modules')
+    localname = _('Ruby Module Index')
+    shortname = _('modules')
 
     def generate(self, docnames=None):
         content = {}
@@ -561,17 +561,17 @@ class RubyDomain(Domain):
     name = 'rb'
     label = 'Ruby'
     object_types = {
-        'function':        ObjType(l_('function'),         'func', 'obj'),
-        'global':          ObjType(l_('global variable'),  'global', 'obj'),
-        'method':          ObjType(l_('method'),           'meth', 'obj'),
-        'class':           ObjType(l_('class'),            'class', 'obj'),
-        'exception':       ObjType(l_('exception'),        'exc', 'obj'),
-        'classmethod':     ObjType(l_('class method'),     'meth', 'obj'),
-        'attr_reader':     ObjType(l_('attribute'),        'attr', 'obj'),
-        'attr_writer':     ObjType(l_('attribute'),        'attr', 'obj'),
-        'attr_accessor':   ObjType(l_('attribute'),        'attr', 'obj'),
-        'const':           ObjType(l_('const'),            'const', 'obj'),
-        'module':          ObjType(l_('module'),           'mod', 'obj'),
+        'function':        ObjType(_('function'),         'func', 'obj'),
+        'global':          ObjType(_('global variable'),  'global', 'obj'),
+        'method':          ObjType(_('method'),           'meth', 'obj'),
+        'class':           ObjType(_('class'),            'class', 'obj'),
+        'exception':       ObjType(_('exception'),        'exc', 'obj'),
+        'classmethod':     ObjType(_('class method'),     'meth', 'obj'),
+        'attr_reader':     ObjType(_('attribute'),        'attr', 'obj'),
+        'attr_writer':     ObjType(_('attribute'),        'attr', 'obj'),
+        'attr_accessor':   ObjType(_('attribute'),        'attr', 'obj'),
+        'const':           ObjType(_('const'),            'const', 'obj'),
+        'module':          ObjType(_('module'),           'mod', 'obj'),
     }
 
     directives = {
diff --git a/documentation/sphinx/requirements.txt b/documentation/sphinx/requirements.txt
index 06e23ea6d3..8e33b564f2 100644
--- a/documentation/sphinx/requirements.txt
+++ b/documentation/sphinx/requirements.txt
@@ -1,6 +1,6 @@
 --index-url https://pypi.python.org/simple
-setuptools>=20.10.0,<=57.4.0
-sphinx==1.5.6
-sphinx-bootstrap-theme==0.4.8
-docutils==0.16
-Jinja2==3.0.3
+setuptools==65.3.0
+sphinx==5.1.1
+sphinx-bootstrap-theme==0.8.1
+docutils==0.19
+Jinja2==3.1.2
diff --git a/documentation/sphinx/source/administration.rst b/documentation/sphinx/source/administration.rst
index 424d9a6f2a..018ffd66c5 100644
--- a/documentation/sphinx/source/administration.rst
+++ b/documentation/sphinx/source/administration.rst
@@ -12,6 +12,7 @@ Administration
    configuration
    moving-a-cluster
    tls
+   authorization
 
 This document covers the administration of an existing FoundationDB cluster. We recommend you read this document before setting up a cluster for performance testing or production use.
 
diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst
index 7e14d5c675..3ec3fca9bf 100644
--- a/documentation/sphinx/source/api-c.rst
+++ b/documentation/sphinx/source/api-c.rst
@@ -222,7 +222,7 @@ The FoundationDB client library performs most tasks on a singleton thread (which
 Future
 ======
 
-Most functions in the FoundationDB API are asynchronous, meaning that they may return to the caller before actually delivering their result. These functions always return :type:`FDBFuture*`. An :type:`FDBFuture` object represents a result value or error to be delivered at some future time. You can wait for a Future to be "ready" -- to have a value or error delivered -- by setting a callback function, or by blocking a thread, or by polling. Once a Future is ready, you can extract either an error code or a value of the appropriate type (the documentation for the original function will tell you which :func:`fdb_future_get_*()` function you should call).
+Most functions in the FoundationDB API are asynchronous, meaning that they may return to the caller before actually delivering their result. These functions always return ``FDBFuture*``. An :type:`FDBFuture` object represents a result value or error to be delivered at some future time. You can wait for a Future to be "ready" -- to have a value or error delivered -- by setting a callback function, or by blocking a thread, or by polling. Once a Future is ready, you can extract either an error code or a value of the appropriate type (the documentation for the original function will tell you which ``fdb_future_get_()`` function you should call).
 
 To use the API in a synchronous way, you would typically do something like this for each asynchronous call::
 
@@ -282,7 +282,7 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
 
 .. type:: FDBCallback
 
-   A pointer to a function which takes :type:`FDBFuture*` and ``void*`` and returns ``void``.
+   A pointer to a function which takes ``FDBFuture*`` and ``void*`` and returns ``void``.
 
 .. function:: void fdb_future_release_memory(FDBFuture* future)
 
@@ -298,13 +298,13 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
 
 .. function:: fdb_error_t fdb_future_get_int64(FDBFuture* future, int64_t* out)
 
-   Extracts a 64-bit integer from an :type:`FDBFuture*` into a caller-provided variable of type ``int64_t``. |future-warning|
+   Extracts a 64-bit integer from a pointer to :type:`FDBFuture` into a caller-provided variable of type ``int64_t``. |future-warning|
 
    |future-get-return1| |future-get-return2|.
 
 .. function:: fdb_error_t fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array, int* out_count)
 
-   Extracts an array of :type:`FDBKey` from an :type:`FDBFuture*` into a caller-provided variable of type ``FDBKey*``. The size of the array will also be extracted and passed back by a caller-provided variable of type ``int`` |future-warning|
+   Extracts an array of :type:`FDBKey` from an ``FDBFuture*`` into a caller-provided variable of type ``FDBKey*``. The size of the array will also be extracted and passed back by a caller-provided variable of type ``int`` |future-warning|
 
    |future-get-return1| |future-get-return2|.
 
@@ -547,13 +547,13 @@ Applications must provide error handling and an appropriate retry loop around th
 
 .. function:: void fdb_transaction_set_read_version(FDBTransaction* transaction, int64_t version)
 
-   Sets the snapshot read version used by a transaction. This is not needed in simple cases. If the given version is too old, subsequent reads will fail with error_code_transaction_too_old; if it is too new, subsequent reads may be delayed indefinitely and/or fail with error_code_future_version. If any of :func:`fdb_transaction_get_*()` have been called on this transaction already, the result is undefined.
+   Sets the snapshot read version used by a transaction. This is not needed in simple cases. If the given version is too old, subsequent reads will fail with error_code_transaction_too_old; if it is too new, subsequent reads may be delayed indefinitely and/or fail with error_code_future_version. If any of ``fdb_transaction_get_*()`` have been called on this transaction already, the result is undefined.
 
 .. function:: FDBFuture* fdb_transaction_get_read_version(FDBTransaction* transaction)
 
    |future-return0| the transaction snapshot read version. |future-return1| call :func:`fdb_future_get_int64()` to extract the version into an int64_t that you provide, |future-return2|
 
-   The transaction obtains a snapshot read version automatically at the time of the first call to :func:`fdb_transaction_get_*()` (including this one) and (unless causal consistency has been deliberately compromised by transaction options) is guaranteed to represent all transactions which were reported committed before that call.
+   The transaction obtains a snapshot read version automatically at the time of the first call to ``fdb_transaction_get_*()`` (including this one) and (unless causal consistency has been deliberately compromised by transaction options) is guaranteed to represent all transactions which were reported committed before that call.
 
 .. function:: FDBFuture* fdb_transaction_get(FDBTransaction* transaction, uint8_t const* key_name, int key_name_length, fdb_bool_t snapshot)
 
@@ -829,7 +829,7 @@ Applications must provide error handling and an appropriate retry loop around th
 
    |future-returnvoid|
 
-   Callers will usually want to retry a transaction if the commit or a prior :func:`fdb_transaction_get_*()` returns a retryable error (see :func:`fdb_transaction_on_error()`).
+   Callers will usually want to retry a transaction if the commit or a prior ``fdb_transaction_get_*()`` returns a retryable error (see :func:`fdb_transaction_on_error()`).
 
    |commit-unknown-result-blurb|
 
@@ -878,9 +878,9 @@ Applications must provide error handling and an appropriate retry loop around th
 
 .. function:: FDBFuture* fdb_transaction_on_error(FDBTransaction* transaction, fdb_error_t error)
 
-   Implements the recommended retry and backoff behavior for a transaction. This function knows which of the error codes generated by other :func:`fdb_transaction_*()` functions represent temporary error conditions and which represent application errors that should be handled by the application. It also implements an exponential backoff strategy to avoid swamping the database cluster with excessive retries when there is a high level of conflict between transactions.
+   Implements the recommended retry and backoff behavior for a transaction. This function knows which of the error codes generated by other ``fdb_transaction_*()`` functions represent temporary error conditions and which represent application errors that should be handled by the application. It also implements an exponential backoff strategy to avoid swamping the database cluster with excessive retries when there is a high level of conflict between transactions.
 
-   On receiving any type of error from an :func:`fdb_transaction_*()` function, the application should:
+   On receiving any type of error from an ``fdb_transaction_*()`` function, the application should:
 
    1. Call :func:`fdb_transaction_on_error()` with the returned :type:`fdb_error_t` code.
 
@@ -963,15 +963,15 @@ Key selectors
 
 In the FoundationDB C API, key selectors are not represented by a structure of any kind, but are instead expressed as sequential parameters to |get-key-func| and |get-range-func|. For convenience, the most common key selectors are available as C macros that expand to the appropriate parameters.
 
-.. function:: FDB_KEYSEL_LAST_LESS_THAN(key_name, key_name_length)
+.. type:: FDB_KEYSEL_LAST_LESS_THAN(key_name, key_name_length)
 
-.. function:: FDB_KEYSEL_LAST_LESS_OR_EQUAL(key_name, key_name_length)
+.. type:: FDB_KEYSEL_LAST_LESS_OR_EQUAL(key_name, key_name_length)
 
-.. function:: FDB_KEYSEL_FIRST_GREATER_THAN(key_name, key_name_length)
+.. type:: FDB_KEYSEL_FIRST_GREATER_THAN(key_name, key_name_length)
 
-.. function:: FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(key_name, key_name_length)
+.. type:: FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(key_name, key_name_length)
 
-To use one of these macros, simply replace the four parameters in the function with one of :func:`FDB_KEYSEL_*`::
+To use one of these macros, simply replace the four parameters in the function with one of ``FDB_KEYSEL_*``::
 
     future = fdb_transaction_get_key(transaction, "key", 3, 0, 2, 0);
 
diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst
index c024c7d2d5..2577bdfc5f 100644
--- a/documentation/sphinx/source/api-python.rst
+++ b/documentation/sphinx/source/api-python.rst
@@ -194,10 +194,6 @@ After importing the ``fdb`` module and selecting an API version, you probably wa
 
        |option-tls-key-bytes|
     
-    .. method :: fdb.options.set_tls_verify_peers(verification_pattern)
-
-       |option-tls-verify-peers|
-    
     .. method :: fdb.options.set_tls_ca_bytes(ca_bundle)
 
        |option-tls-ca-bytes|
@@ -210,10 +206,6 @@ After importing the ``fdb`` module and selecting an API version, you probably wa
 
        |option-tls-password|
 
-    .. method :: fdb.options.set_disable_multi_version_client_api()
-
-       |option-disable-multi-version-client-api|
-
     .. method :: fdb.options.set_disable_local_client()
 
       |option-set-disable-local-client|
@@ -761,10 +753,6 @@ In each of the methods below, ``param`` should be a string appropriately packed
 Committing
 ----------
 
-.. decorator:: transactional()
-
-    The ``transactional`` decorator makes it easy to write transactional functions which accept a :class:`Database`, :class`Tenant`, or :class:`Transaction` as a parameter and automatically commit. See :func:`@fdb.transactional <transactional>` for explanation and examples.
-
 .. method :: Transaction.commit()
 
     Attempt to commit the changes made in the transaction to the database.  Returns a :class:`FutureVoid` representing the asynchronous result of the commit. You **must** call the :meth:`Future.wait()` method on the returned :class:`FutureVoid`, which will raise an exception if the commit failed.
diff --git a/documentation/sphinx/source/architecture.rst b/documentation/sphinx/source/architecture.rst
index 7c28518d74..f693865430 100644
--- a/documentation/sphinx/source/architecture.rst
+++ b/documentation/sphinx/source/architecture.rst
@@ -14,8 +14,12 @@ Detailed FoundationDB Architecture
 
 The FoundationDB architecture chooses a decoupled design, where
 processes are assigned different heterogeneous roles (e.g.,
-Coordinators, Storage Servers, Master). Scaling the database is achieved
-by horizontally expanding the number of processes for separate roles:
+Coordinators, Storage Servers, Master). Cluster attempts to recruit
+different roles as separate processes, however, it is possible that
+multiple Stateless roles gets colocated (recruited) on a single
+process to meet the cluster recruitment goals. Scaling the database
+is achieved by horizontally expanding the number of processes for
+separate roles:
 
 Coordinators
 ~~~~~~~~~~~~
diff --git a/documentation/sphinx/source/authorization.rst b/documentation/sphinx/source/authorization.rst
new file mode 100644
index 0000000000..ce25df1057
--- /dev/null
+++ b/documentation/sphinx/source/authorization.rst
@@ -0,0 +1,124 @@
+#############
+Authorization
+#############
+
+.. warning :: Authorization is currently experimental and is not recommended for use in production.
+
+Introduction
+============
+
+:ref:`Multi-tenant <multi-tenancy>` database implies a couple of new concepts that did not previously exist in FoundationDB.
+The first is the concept of privilege levels: we have *data-plane clients* whose typical workload is limited to accessing a tenant keyspace.
+On the other hand, we have *control-plane clients* or *administrators* who may read or update cluster-wide configurations through system keyspace.
+These operations also include creation and deletion of tenants.
+The second is access control: with multiple tenant keyspaces, it comes naturally that we would want to restrict database access of a client to a subset of them.
+
+Privilege Levels
+----------------
+
+Authorization feature extends FoundationDB's existing TLS policy to distinguish administrators from data-plane clients,
+making TLS configuration a prerequisite for enabling authorization.
+There are only two privilege levels: *trusted* versus *untrusted* clients.
+Trusted clients are authorized to perform any operation that pre-authorization FoundationDB clients used to perform, including those accessing the system keyspace.
+Untrusted clients may only request what is necessary to access tenant keyspaces for which they are authorized.
+Untrusted clients are blocked from accessing anything in the system keyspace or issuing management operations that modify the cluster in any way.
+
+In order to be considered a trusted client, a client needs to be :ref:`configured with a valid chain of X.509 certificates and a private key <configuring-tls>`,
+and its certificate chain must be trusted by the server. In other words, a client must successfully complete a mutual TLS authentication.
+Additionally, if the server was configured with trusted IP subnets, i.e. run with one or more ``--trusted-subnet-SUBNET_NAME`` followed by a CIDR block describing the subnet,
+then the client's IP as seen from the server must belong to at least one of the subnets.
+
+Choosing to respond with an empty certificate chain during `client authentication <https://www.rfc-editor.org/rfc/rfc5246#section-7.4.6>`_ marks the client as untrusted.
+If the server specifies a list of trusted subnets and the client's server-facing IP is not part of any of the subnets,
+then the client is untrusted even if it successfully completes a mutual TLS authentication.
+
+.. note:: Presenting a bad or untrusted certificate chain causes the server to break the client connection and eventually throttle the client.
+          It does not let the client connect untrusted.
+
+Access Control
+--------------
+
+To restrict untrusted client's database access to a subset of tenant keyspaces, authorization feature allows database administrators
+to grant tenant-scoped access in the form of `JSON Web Tokens <https://www.rfc-editor.org/rfc/rfc7519>`_.
+Token verification is performed against a set of named public keys written in `JWK Set <https://www.rfc-editor.org/rfc/rfc7517#section-5>`_ format.
+A token's header part must contain the `key identifier <https://www.rfc-editor.org/rfc/rfc7515.html#section-4.1.4>`_ of the public key which shall be used to verify the token itself.
+Below is the list of token fields recognized by FoundationDB.
+Note that some of the fields are *recognized* by FoundationDB but not *actively used* in enforcing security, pending future implementation.
+Those fields are marked as **NOT required**.
+
+
+.. table:: JSON Web Token Fields supported by FoundationDB
+   :align: left
+   :widths: auto
+
+   =============== =========== ======== ==================================================== ================================================================================
+   Containing Part Field Name  Required Purpose                                              Reference
+   =============== =========== ======== ==================================================== ================================================================================
+   Header          ``typ``     Yes      Type of JSON Web Signature. Must be ``JWT``.         `RFC7519 Section 5.1 <https://www.rfc-editor.org/rfc/rfc7519#section-5.1>`_
+   Header          ``alg``     Yes      Algorithm used to generate the signature. Only       `RFC7515 Section 4.1.1 <https://www.rfc-editor.org/rfc/rfc7515#section-4.1.1>`_
+                                        ``ES256`` and ``RS256`` are supported.
+                                        Must match the ``alg`` attribute of public key.
+   Header          ``kid``     Yes      Name of public key with which to verify the token.   `RFC7515 Section 4.1.4 <https://www.rfc-editor.org/rfc/rfc7515#section-4.1.4>`_
+                                        Must match the ``kid`` attribute of public key.
+   Claim           ``exp``     Yes      Timestamp after which token is not accepted.         `RFC7519 Section 4.1.4 <https://www.rfc-editor.org/rfc/rfc7519#section-4.1.4>`_
+   Claim           ``nbf``     Yes      Timestamp before which token is not accepted.        `RFC7519 Section 4.1.5 <https://www.rfc-editor.org/rfc/rfc7519#section-4.1.5>`_
+   Claim           ``iat``     Yes      Timestamp at which token was issued.                 `RFC7519 Section 4.1.6 <https://www.rfc-editor.org/rfc/rfc7519#section-4.1.6>`_
+   Claim           ``tenants`` Yes      Tenants names for which token holder is authorized.  N/A
+                                        Must be an array.
+   Claim           ``iss``     No       Issuer of the token.                                 `RFC7519 Section 4.1.1 <https://www.rfc-editor.org/rfc/rfc7519#section-4.1.1>`_
+   Claim           ``sub``     No       Subject of the token.                                `RFC7519 Section 4.1.2 <https://www.rfc-editor.org/rfc/rfc7519#section-4.1.2>`_
+   Claim           ``aud``     No       Intended recipients of the token. Must be an array.  `RFC7519 Section 4.1.3 <https://www.rfc-editor.org/rfc/rfc7519#section-4.1.3>`_
+   Claim           ``jti``     No       String that uniquely identifies a token.             `RFC7519 Section 4.1.7 <https://www.rfc-editor.org/rfc/rfc7519#section-4.1.7>`_
+   =============== =========== ======== ==================================================== ================================================================================
+
+Public keys with which to verify the token must be serialized in `JWK Set <https://www.rfc-editor.org/rfc/rfc7517#section-5>`_ format and stored in a file.
+The location of the key set file must be passed as command line argument ``--authorization-public-key-file`` to the ``fdbserver`` executable.
+Public keys in the set must be either `RSA <https://datatracker.ietf.org/doc/html/rfc7518#section-6.3>`_ public keys
+containing ``n`` and ``e`` parameters, each containing `Base64urlUInt <https://www.rfc-editor.org/rfc/rfc7518#section-2>`_-encoded modulus and exponent,
+or `Elliptic Curve <https://datatracker.ietf.org/doc/html/rfc7518#section-6.2>`_ public keys on a ``P-256`` curve,
+where ``crv`` parameter is set to ``P-256`` and ``x`` and ``y`` parameters contain
+`base64url <https://datatracker.ietf.org/doc/html/rfc4648#section-5>`_-encoded affine coordinates.
+In addition, each public key JSON object in set must contain ``kty`` (set to either ``EC`` or ``RSA``) field to indicate public key algorithm,
+along with ``kid``, and ``alg`` fields to be compared against their token header counterparts.
+Private keys are strongly recommended against being included in the public key set and, if found, are excluded from consideration.
+
+.. note:: By design, FoundationDB authorization feature does not support revocation of outstanding tokens.
+          Use extra caution in signing tokens with long token durations.
+
+Enabling Clients to use Authorization Tokens
+============================================
+
+In order to use an untrusted client with an authorization token, a client must be configured to trust the server's CA,
+but must not be configured to use the client's own certificates and keys.
+More concretely, the client's ``TLS_CA_FILE`` must include the server's root CA certificate,
+but the client must not be configured with its own ``TLS_CERTIFICATE_FILE`` or ``TLS_KEY_FILE``, neither programmatically nor by environment variable.
+Before performing a tenant data read or update, a client must set ``AUTHORIZATION_TOKEN`` transaction option with the token string as argument.
+It is the client's responsibility to keep the token up-to-date, by timely assigning a new token to the transaction object.
+
+.. note:: The TLS authentication mode of an untrusted client is similar to how typical web browsers connect to TLS-enabled web services.
+          They authenticate the server using their bundle of trusted root CA certificates,
+          but they do not authenticate themselves to the server.
+
+Public Key Rotation
+===================
+
+FoundationDB's internal public key set automatically refreshes itself based on the key set file's latest content every ``PUBLIC_KEY_FILE_REFRESH_INTERVAL_SECONDS`` seconds.
+The in-memory set of public keys does not update unless the key file holds a correct `JWK Set`_.
+
+Token Caching
+=============
+
+In a single-threaded runtime environment such as FoundationDB, it is important not to let the main thread be overloaded with computationally expensive operations,
+such as token signature verification. FoundationDB internally caches the tokens that are considered valid at the time of verification in a fixed-size cache,
+whose size may be configured using ``TOKEN_CACHE_SIZE`` knob.
+
+.. note:: Token cache is independent of the active public key set. Once the token reaches the cache, it is valid until its expiration time,
+          regardless of any key rotation that takes place thereafter.
+
+Allowing Untrusted Clients to Access Tenant Data Without Tokens
+===============================================================
+
+Rolling out a public key distribution infrastructure and an authorization-enabled FoundationDB cluster in lockstep might not be feasible with large scale distributed systems.
+To support incremental rollout, authorization feature introduces ``ALLOW_TOKENLESS_TENANT_ACCESS`` boolean knob,
+which preserves the TLS-based privilege level policy without untrusted clients having to set authorization tokens to their transactions in order to access tenant data.
+With this knob active, any authorization token assigned to the client transaction is simply ignored.
diff --git a/documentation/sphinx/source/client-testing.rst b/documentation/sphinx/source/client-testing.rst
index 433a47ce7d..0eb159e8f4 100644
--- a/documentation/sphinx/source/client-testing.rst
+++ b/documentation/sphinx/source/client-testing.rst
@@ -373,3 +373,302 @@ with the ``multitest`` role:
    fdbserver -r multitest -f testfile.txt
 
 This command will block until all tests are completed.
+
+##########
+API Tester
+##########
+
+Introduction
+============
+
+API tester is a framework for implementing end-to-end tests of FDB C API, i.e. testing the API on a real
+FDB cluster through all layers of the FDB client. Its executable is ``fdb_c_api_tester``, and the source
+code is located in ``bindings/c/test/apitester``. The structure of API Tests is similar to that of the 
+Simulation Tests. The tests are implemented as workloads using FDB API, which are all built into the 
+``fdb_c_api_tester``. A concrete test configuration is defined as a TOML file, which specifies the
+combination of workloads to be executed by the test together with their parameters. The test can be then
+executed by passing the TOML file as a parameter to ``fdb_c_api_tester``. 
+
+Since simulation tests rely on the actor model to execute the tests deterministically in single-threaded
+mode, they are not suitable for testing various multi-threaded aspects of the FDB client. End-to-end API
+tests complement the simulation tests by testing the FDB Client layers above the single-threaded Native
+Client. 
+
+- The specific testing goals of the end-to-end tests are:
+- Check functional correctness of the Multi-Version Client (MVC) and Thread-Safe Client 
+- Detecting race conditions. They can be caused by accessing the state of the Native Client from wrong
+  threads or introducing other shared state without proper synchronization
+- Detecting memory management errors. Thread-safe reference counting must be used where necessary. MVC
+  works with multiple client libraries. Memory allocated by one client library must be also deallocated
+  by the same library.
+- Maintaining interoperability with other client versions.  The client functionality is made available
+  depending on the selected API version. The API changes are correctly adapted. 
+- Client API behaves correctly in case of cluster upgrades. Database and transaction state is correctly
+  migrated to the upgraded connections. Pending operations are canceled and successfully retried on the
+  upgraded connections.
+
+Implementing a Workload
+=======================
+
+Each workload is declared as a direct or indirect subclass of ``WorkloadBase`` implementing a constructor
+with ``WorkloadConfig`` as a parameter and the method ``start()``, which defines the entry point of the
+workload. 
+
+``WorkloadBase`` provides a set of methods that serve as building blocks for implementation of a workload:
+
+.. function:: execTransaction(start, cont, failOnError = true)
+
+   creates and executes an FDB transaction. Here ``start`` is a function that takes a transaction context
+   as parameter and implements the starting point of the transaction, and ``cont`` is a function implementing
+   a continuation to be executed after finishing the transaction execution. Transactions are automatically
+   retried on retryable errors. Transactions are retried by calling the ``start`` function again. In case
+   of a fatal error, the entire workload is considered as failed unless ``failOnError`` is set to ``false``.
+
+.. function:: schedule(task)
+
+   schedules a task for asynchronous execution. It is usually used in the continuations to schedule 
+   the next step of the workload.
+
+.. function:: info(msg) 
+              error(msg) 
+              
+   are used for logging a message with a tag identifying the workload. Issuing an error message marks
+   the workload as failed.
+
+The transaction context provides methods for implementation of the transaction logics:
+
+.. function:: tx()
+   
+   the reference to the FDB transaction object
+
+.. function:: continueAfter(future, cont, retryOnError = true)
+   
+   set a continuation to be executed when the future is ready. The ``retryOnError`` flag controls whether
+   the transaction should be automatically retried in case the future results in a retriable error.
+
+.. function:: continueAfterAll(futures, cont)
+   
+   takes a vector of futures and sets a continuation to be executed when all of the futures get ready.
+   The transaction is retried if at least one of the futures results in an error. This method is useful 
+   for handling multiple concurrent reads.
+
+.. function:: commit() 
+   
+   commit and finish the transaction. If the commit is successful, the execution proceeds to the
+   continuation of ``execTransaction()``. In case of a retriable error the transaction is
+   automatically retried. A fatal error results in a failure of the workoad.
+
+
+.. function:: done() 
+   
+   finish the transaction without committing. This method should be used to finish read transactions. 
+   The transaction gets destroyed and execution proceeds to the continuation of ``execTransaction()``.
+   Each transaction must be finished either by ``commit()`` or ``done()``, because otherwise
+   the framework considers that the transaction is still being executed, so it won't destroy it and
+   won't call the continuation.
+
+.. function:: onError(err) 
+   
+   Handle an error: restart the transaction in case of a retriable error, otherwise fail the workload.
+   This method is typically used in the continuation of ``continueAfter`` called with
+   ``retryOnError=false`` as a fallback to the default error handling.
+
+A workload execution ends automatically when it is marked as failed or its last continuation does not
+schedule any new task or transaction. 
+
+The workload class should be defined in the namespace FdbApiTester. The file name convention is
+``Tester{Name}Workload.cpp`` so that we distinguish them from the source files of simulation workloads.
+
+Basic Workload Example
+======================
+
+The code below implements a workload that consists of only two transactions. The first one sets a
+randomly generated key to a randomly generated value, and the second one reads the key and checks if
+the returned value matches the written one.
+
+.. literalinclude:: ../../../bindings/c/test/apitester/TesterExampleWorkload.cpp
+   :language: C++
+   :lines: 21-
+
+The workload is implemented in the method ``setAndGet``. It generates a random key and a random value
+and executes a transaction that writes that key-value pair and commits. In the continuation of the
+first ``execTransaction`` call, we execute the second transaction that reads the same key. The read
+operation returns a future. So we call ``continueAfter`` to set a continuation for that future. In the
+continuation we check if the returned value matches the written one and finish the transaction by
+calling ``ctx->done()``. After completing the second transaction we execute the continuation passed
+as parameter to the ``setAndGet`` method by the start method. In this case it is ``NO_OP_TASK``, which
+does nothing and so finishes the workload.
+
+Finally, we declare an instance ``WorkloadFactory`` to register this workload with the name ``SetAndGet``.
+
+Note that we use ``workloadId`` as a key prefix. This is necessary for isolating the key space of this
+workload, because the framework may be instructed to create multiple instances of the ``SetAndGet``
+workload. If we do not isolate the key space, another workload can write a different value for the
+same key and so break the assumption of the test.
+
+The workload is implemented using the internal C++ API, implemented in ``fdb_api.hpp``. It introduces
+a set of classes representing the FDB objects (transactions, futures, etc.). These classes provide C++-style 
+methods wrapping FDB C API calls and automate memory management by means of reference counting.
+
+Implementing Control Structures
+===============================
+
+Our basic workload executes just 2 transactions, but in practice we want to have workloads that generate
+multiple transactions. The following code demonstrates how we can modify our basic workload to generate
+multiple transactions in a loop. 
+
+.. code-block:: C++
+
+   class SetAndGetWorkload : public WorkloadBase {
+   public:
+      ...
+      int numIterations;
+      int iterationsLeft;
+
+      SetAndGetWorkload(const WorkloadConfig& config) : WorkloadBase(config) {
+         keyPrefix = fdb::toBytesRef(fmt::format("{}/", workloadId));
+         numIterations = config.getIntOption("numIterations", 1000);
+      }
+
+      void start() override {
+         iterationsLeft = numIterations;
+         setAndGetLoop();
+      }
+
+      void setAndGetLoop() {
+         if (iterationsLeft == 0) {
+            return;
+         }
+         iterationsLeft--;
+         setAndGet([this]() { setAndGetLoop(); });
+      }
+      ...
+   }
+
+We introduce a workload parameter ``numIterations`` to specify the number of iterations. If not specified
+in the test configuration it defaults to 1000.
+
+The method ``setAndGetLoop`` implements the loop that decrements iterationsLeft counter until it reaches 0
+and each iteration calls setAndGet with a continuation that returns the execution to the loop. As you
+can see we don't need any change in ``setAndGet``, just call it with another continuation. 
+
+The pattern of passing a continuation as a parameter also can be used to decompose the workload into a
+sequence of steps. For example,  we can introduce setup and cleanUp steps to our workload and modify the
+``setAndGetLoop`` to make it composable with an arbitrary continuation:
+
+.. code-block:: C++
+
+    void start() override {
+       setup([this](){
+           iterationsLeft = numIterations;
+           setAndGetLoop([this](){
+               cleanup(NO_OP_TASK);
+           });
+       });
+    }
+
+    void setAndGetLoop(TTaskFct cont) {
+       if (iterationsLeft == 0) {
+           schedule(cont);
+       }
+       iterationsLeft--;
+       setAndGet([this, cont]() { setAndGetLoop(cont); });
+   }
+
+   void setup(TTaskFct cont) { ... }
+
+   void cleanup(TTaskFct cont) {  ... }
+
+Note that we call ``schedule(cont)`` in ``setAndGetLoop`` instead of calling the continuation directly.
+In this way we avoid keeping ``setAndGetLoop`` in the call stack, when executing the next step.
+
+Subclassing ApiWorkload
+=======================
+
+``ApiWorkload`` is an abstract subclass of ``WorkloadBase`` that provides a framework for a typical
+implementation of API test workloads. It implements a workflow consisting of cleaning up the key space
+of the workload, populating it with newly generated data and then running a loop consisting of random
+database operations. The concrete subclasses of ``ApiWorkload`` are expected to override the method
+``randomOperation`` with an implementation of concrete random operations.
+
+The ``ApiWorkload`` maintains a local key-value store that mirrors the part of the database state
+relevant to the workload. A successful database write operation should be followed by a continuation
+that performs equivalent changes in the local store, and the results of a database read operation should
+be validated against the values from the local store. 
+
+Test Configuration
+==================
+
+A concrete test configuration is specified by a TOML file. The file must contain one ``[[test]]`` section
+specifying the general settings for test execution followed by one or more ``[[test.workload]]``
+configuration sessions, specifying the workloads to be executed and their parameters. The specified
+workloads are started all at once and executed concurrently.
+
+The ``[[test]]`` section can contain the following options:
+
+- ``title``: descriptive title of the test
+- ``multiThreaded``: enable multi-threading (default: false)
+- ``minFdbThreads`` and ``maxFdbThreads``: the number of FDB (network) threads to be randomly selected
+  from the given range (default: 1-1). Used only if ``multiThreaded=true``. It is also important to use
+  multiple database instances to make use of the multithreading.
+- ``minDatabases`` and ``maxDatabases``: the number of database instances to be randomly selected from
+  the given range (default 1-1). The transactions of all workloads are randomly load-balanced over the
+  pool of database instances.
+- ``minClients`` and ``maxClients``: the number of clients, i.e. instances of each workload, to be
+  randomly selected from the given range (default 1-8).
+- ``minClientThreads`` and ``maxClientThreads``: the number of client threads, i.e. the threads used
+  for execution of the workload, to be randomly selected from the given range (default 1-1).
+- ``blockOnFutures``: use blocking waits on futures instead of scheduling future callbacks asynchronously
+  (default: false)
+- ``buggify``: Enable client-side failure injection (default: false)
+- ``databasePerTransaction``: Create a separate database instance for each transaction (default: false).
+  It is a special mode useful for testing bugs related to creation and destruction of database instances. 
+- ``fdbCallbacksOnExternalThreads``: Enables the option ``FDB_NET_OPTION_CALLBACKS_ON_EXTERNAL_THREADS``
+  causting the callbacks of futures to be executed directly on the threads of the external FDB clients 
+  rather than on the thread of the local FDB client. 
+
+The workload section ``[[test.workload]]`` must contain the attribute name matching the registered name
+of the workload to be executed. Other options are workload-specific. 
+
+The subclasses of the ``ApiWorkload`` inherit the following configuration options:
+
+- ``minKeyLength`` and ``maxKeyLength``: the size range of randomly generated keys (default: 1-64)
+- ``minValueLength`` and ``maxValueLength``:  the size range of randomly generated values 
+  (default: 1-1000)
+- ``maxKeysPerTransaction``: the maximum number of keys per transaction (default: 50)
+- ``initialSize``: the number of key-value pairs in the initially populated database (default: 1000)
+- ``readExistingKeysRatio``: the probability of choosing an existing key for read operations 
+  (default: 0.9)
+- ``numRandomOperations``: the number of random operations to be executed per workload (default: 1000)
+- ``runUntilStop``: run the workload indefinitely until the stop command is received (default: false).
+   This execution mode in upgrade tests and other scripted tests, where the workload needs to
+   be generated continously until completion of the scripted test.
+- ``numOperationsForProgressCheck``: the number of operations to be performed to confirm a progress 
+   check (default: 10). This option is used in combination with ``runUntilStop``. Progress checks are
+   initiated by a test script to check if the client workload is successfully progressing after a
+   cluster change.
+
+Executing the Tests
+===================
+
+The ``fdb_c_api_tester`` executable takes a single TOML file as a parameter and executes the test
+according to its specification. Before that we must create a FDB cluster and pass its cluster file as
+a parameter to ``fdb_c_api_tester``. Note that multithreaded tests also need to be provided with an
+external client library. 
+
+For example, we can create a temporary cluster and use it for execution of one of the existing API tests:
+
+.. code-block:: bash
+
+   ${srcDir}/tests/TestRunner/tmp_cluster.py --build-dir ${buildDir} -- \
+      ${buildDir}/bin/fdb_c_api_tester \
+      --cluster-file @CLUSTER_FILE@ \
+      --external-client-library=${buildDir}/bindings/c/libfdb_c_external.so \
+      --test-file ${srcDir}/bindings/c/test/apitester/tests/CApiCorrectnessMultiThr.toml
+
+The test specifications added to the ``bindings/c/test/apitester/tests/`` directory are executed as a part
+of the regression test suite. They can be executed using the ``ctest`` target ``fdb_c_api_tests``:
+
+.. code-block:: bash
+   
+   ctest -R fdb_c_api_tests -VV
diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst
index 09969a0640..9c51196a35 100644
--- a/documentation/sphinx/source/command-line-interface.rst
+++ b/documentation/sphinx/source/command-line-interface.rst
@@ -64,7 +64,7 @@ The ``commit`` command commits the current transaction. Any sets or clears execu
 configure
 ---------
 
-The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=<N>] [commit_proxies=<N>] [resolvers=<N>] [logs=<N>] [count=<TSS_COUNT>] [perpetual_storage_wiggle=<WIGGLE_SPEED>] [perpetual_storage_wiggle_locality=<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>] [storage_migration_type={disabled|aggressive|gradual}] [tenant_mode={disabled|optional_experimental|required_experimental}]``.
+The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=<N>] [commit_proxies=<N>] [resolvers=<N>] [logs=<N>] [count=<TSS_COUNT>] [perpetual_storage_wiggle=<WIGGLE_SPEED>] [perpetual_storage_wiggle_locality=<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>] [storage_migration_type={disabled|aggressive|gradual}] [tenant_mode={disabled|optional_experimental|required_experimental}] [encryption_at_rest_mode={aes_256_ctr|disabled}]``.
 
 The ``new`` option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When ``new`` is used, both a redundancy mode and a storage engine must be specified.
 
@@ -153,13 +153,6 @@ If ``description=<DESC>`` is specified, the description field in the cluster fil
 
 For more information on setting the cluster description, see :ref:`configuration-setting-cluster-description`.
 
-createtenant
-------------
-
-The ``createtenant`` command is used to create new tenants in the cluster. Its syntax is ``createtenant <TENANT_NAME>``.
-
-The tenant name can be any byte string that does not begin with the ``\xff`` byte. If the tenant already exists, ``fdbcli`` will report an error.
-
 defaulttenant
 -------------
 
@@ -167,13 +160,6 @@ The ``defaulttenant`` command configures ``fdbcli`` to run its commands without
 
 The active tenant cannot be changed while a transaction (using ``begin``) is open.
 
-deletetenant
-------------
-
-The ``deletetenant`` command is used to delete tenants from the cluster. Its syntax is ``deletetenant <TENANT_NAME>``.
-
-In order to delete a tenant, it must be empty. To delete a tenant with data, first clear that data using the ``clear`` command. If the tenant does not exist, ``fdbcli`` will report an error.
-
 exclude
 -------
 
@@ -231,33 +217,8 @@ The ``getrangekeys`` command fetches keys in a range. Its syntax is ``getrangeke
 
 Note that :ref:`characters can be escaped <cli-escaping>` when specifying keys (or values) in ``fdbcli``.
 
-gettenant
----------
-
-The ``gettenant`` command fetches metadata for a given tenant and displays it. Its syntax is ``gettenant <TENANT_NAME> [JSON]``.
-
-Included in the output of this command are the ``id`` and ``prefix`` assigned to the tenant. If the tenant does not exist, ``fdbcli`` will report an error. If ``JSON`` is specified, then the output will be written as a JSON document::
-
-    {
-        "tenant": {
-            "id": 0,
-            "prefix": {
-              "base64": "AAAAAAAAAAU=",
-              "printable": "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05",
-            }
-        },
-        "type": "success"
-    }
-
-In the event of an error, the output will include an error message::
-
-    {
-        "error": "...",
-        "type": "error"
-    }
-
-    getversion
-    ----------
+getversion
+----------
 
 The ``getversion`` command fetches the current read version of the cluster or currently running transaction.
 
@@ -346,13 +307,6 @@ Attempts to kill all specified processes. Each address should include the IP and
 
 Attempts to kill all known processes in the cluster.
 
-listtenants
------------
-
-The ``listtenants`` command prints the names of tenants in the cluster. Its syntax is ``listtenants [BEGIN] [END] [LIMIT]``.
-
-By default, the ``listtenants`` command will print up to 100 entries from the entire range of tenants. A narrower sub-range can be printed using the optional ``[BEGIN]`` and ``[END]`` parameters, and the limit can be changed by specifying an integer ``[LIMIT]`` parameter.
-
 lock
 ----
 
@@ -417,13 +371,6 @@ heap
 
 Enables heap profiling for the specified process.
 
-renametenant
-------------
-
-The ``renametenant`` command can rename an existing tenant to a new name. Its syntax is ``renametenant <OLD_NAME> <NEW_NAME>``.
-
-This command requires that ``OLD_NAME`` is a tenant that already exists on the cluster, and that ``NEW_NAME`` is not already a name of a tenant in the cluster.
-
 reset
 -----
 
@@ -484,6 +431,141 @@ status json
 
 .. _cli-throttle:
 
+tenant
+------
+
+The ``tenant`` command is used to view and manage the tenants in a cluster. The ``tenant`` command has the following subcommands:
+
+create
+^^^^^^
+
+``tenant create <NAME> [tenant_group=<TENANT_GROUP>]``
+
+Creates a new tenant in the cluster.
+
+``NAME`` - The desired name of the tenant. The name can be any byte string that does not begin with the ``\xff`` byte. 
+
+``TENANT_GROUP`` - The tenant group the tenant will be placed in.
+
+delete
+^^^^^^
+
+``tenant delete <NAME>``
+
+Deletes a tenant from the cluster. The tenant must be empty.
+
+``NAME`` - the name of the tenant to delete.
+
+list
+^^^^
+
+``tenant list [BEGIN] [END] [LIMIT]``
+
+Lists the tenants present in the cluster.
+
+``BEGIN`` - the first tenant to list. Defaults to the empty tenant name ``""``.
+
+``END`` - the exclusive end tenant to list. Defaults to ``\xff\xff``.
+
+``LIMIT`` - the number of tenants to list. Defaults to 100.
+
+get
+^^^
+
+``tenant get <NAME> [JSON]``
+
+Prints the metadata for a tenant.
+
+``NAME`` - the name of the tenant to print.
+
+``JSON`` - if specified, the output of the command will be printed in the form of a JSON string::
+
+    {
+        "tenant": {
+            "id": 0,
+            "prefix": {
+              "base64": "AAAAAAAAAAU=",
+              "printable": "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05",
+            }
+        },
+        "type": "success"
+    }
+
+In the event of an error, the JSON output will include an error message::
+
+    {
+        "error": "...",
+        "type": "error"
+    }
+
+configure
+^^^^^^^^^
+
+``tenant configure <TENANT_NAME> <[unset] tenant_group[=GROUP_NAME]>``
+
+Changes the configuration of a tenant.
+
+``TENANT_NAME`` - the name of the tenant to reconfigure.
+
+The following tenant fields can be configured:
+
+``tenant_group`` - changes the tenant group a tenant is assigned to. If ``unset`` is specified, the tenant will be configured to not be in a group. Otherwise, ``GROUP_NAME`` must be specified to the new group that the tenant should be made a member of.
+
+rename
+^^^^^^
+
+``tenant rename <OLD_NAME> <NEW_NAME>``
+
+Changes the name of an existing tenant.
+
+``OLD_NAME`` - the name of the tenant being renamed.
+
+``NEW_NAME`` - the desired name of the tenant. This name must not already be in use.
+
+
+tenantgroup
+-----------
+
+The ``tenantgroup`` command is used to view details about the tenant groups in a cluster. The ``tenantgroup`` command has the following subcommands:
+
+list
+^^^^
+
+``tenantgroup list [BEGIN] [END] [LIMIT]``
+
+Lists the tenant groups present in the cluster.
+
+``BEGIN`` - the first tenant group to list. Defaults to the empty tenant group name ``""``.
+
+``END`` - the exclusive end tenant group to list. Defaults to ``\xff\xff``.
+
+``LIMIT`` - the number of tenant groups to list. Defaults to 100.
+
+get
+^^^
+
+``tenantgroup get <NAME> [JSON]``
+
+Prints the metadata for a tenant group.
+
+``NAME`` - the name of the tenant group to print.
+
+``JSON`` - if specified, the output of the command will be printed in the form of a JSON string::
+
+    {
+        "tenant_group": {
+            "assigned_cluster": "cluster1",
+        },
+        "type": "success"
+    }
+
+In the event of an error, the JSON output will include an error message::
+
+    {
+        "error": "...",
+        "type": "error"
+    }
+
 throttle
 --------
 
diff --git a/documentation/sphinx/source/configuration.rst b/documentation/sphinx/source/configuration.rst
index 699c811139..5d52d40910 100644
--- a/documentation/sphinx/source/configuration.rst
+++ b/documentation/sphinx/source/configuration.rst
@@ -416,6 +416,9 @@ FoundationDB will never use processes on the same machine for the replication of
 ``three_data_hall`` mode
     FoundationDB stores data in triplicate, with one copy on a storage server in each of three data halls. The transaction logs are replicated four times, with two data halls containing two replicas apiece. Four available machines (two in each of two data halls) are therefore required to make progress. This configuration enables the cluster to remain available after losing a single data hall and one machine in another data hall.
 
+``three_data_hall_fallback`` mode
+    FoundationDB stores data in duplicate, with one copy each on a storage server in two of three data halls. The transaction logs are replicated four times, with two data halls containing two replicas apiece. Four available machines (two in each of two data halls) are therefore required to make progress. This configuration is similar to ``three_data_hall``, differing only in that data is stored on two instead of three replicas. This configuration is useful to unblock data distribution when a data hall becomes temporarily unavailable. Because ``three_data_hall_fallback`` reduces the redundancy level to two, it should only be used as a temporary measure to restore cluster health during a datacenter outage.
+
 Datacenter-aware mode
 ---------------------
 
diff --git a/documentation/sphinx/source/data-modeling.rst b/documentation/sphinx/source/data-modeling.rst
index 146b006809..5972c5110a 100644
--- a/documentation/sphinx/source/data-modeling.rst
+++ b/documentation/sphinx/source/data-modeling.rst
@@ -1,7 +1,6 @@
 .. default-domain:: py
 .. default-domain:: py
 .. highlight:: python
-.. module:: fdb
 
 .. Required substitutions for api-common.rst.inc
 
diff --git a/documentation/sphinx/source/developer-guide.rst b/documentation/sphinx/source/developer-guide.rst
index 3bf5ec30c0..c889e4909b 100644
--- a/documentation/sphinx/source/developer-guide.rst
+++ b/documentation/sphinx/source/developer-guide.rst
@@ -1,7 +1,6 @@
 .. default-domain:: py
 .. default-domain:: py
 .. highlight:: python
-.. module:: fdb
 
 .. Required substitutions for api-common.rst.inc
 
diff --git a/documentation/sphinx/source/global-configuration.rst b/documentation/sphinx/source/global-configuration.rst
index aad4c31d93..663ad26eb4 100644
--- a/documentation/sphinx/source/global-configuration.rst
+++ b/documentation/sphinx/source/global-configuration.rst
@@ -82,10 +82,10 @@ Values must always be encoded according to the :ref:`api-python-tuple-layer`.
    // In GlobalConfig.actor.h
    extern const KeyRef myGlobalConfigKey;
    // In GlobalConfig.actor.cpp
-   const KeyRef myGlobalConfigKey = LiteralStringRef("config/key");
+   const KeyRef myGlobalConfigKey = "config/key"_sr;
    
    // When you want to set the value..
-   Tuple value = Tuple().appendDouble(1.5);
+   Tuple value = Tuple::makeTuple((double)1.5);
    
    FDBTransaction* tr = ...;
    tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
diff --git a/documentation/sphinx/source/index.rst b/documentation/sphinx/source/index.rst
index 40c7a76279..167bebda43 100644
--- a/documentation/sphinx/source/index.rst
+++ b/documentation/sphinx/source/index.rst
@@ -50,6 +50,7 @@ The latest changes are detailed in :ref:`release-notes`. The documentation has t
    :hidden:
 
    local-dev
+   internal-dev-tools
    why-foundationdb
    technical-overview
    client-design
diff --git a/documentation/sphinx/source/internal-dev-tools.rst b/documentation/sphinx/source/internal-dev-tools.rst
new file mode 100644
index 0000000000..ea80947312
--- /dev/null
+++ b/documentation/sphinx/source/internal-dev-tools.rst
@@ -0,0 +1,58 @@
+##################
+Internal Dev Tools
+##################
+
+Code Probes
+===========
+
+Code probes are a mechanism in FDB to prove that certain code-paths are being tested under the right conditions. They differ from code coverage in multiple ways (explained below).
+
+The general format of a code probe is:
+
+.. code-block:: C++
+
+   CODE_PROBE(<condition>, "Comment", [annotations...]);
+
+A simple example of a code probe could look as follows:
+
+.. code-block:: C++
+
+   CODE_PROBE(self->forceRecovery, "Resolver detects forced recovery", probe::context::sim2);
+
+On a very high level, the above code will indicate that whenever this line is executed and ``self->forceRecovery`` is ``true``, we ran into some interesting case. In addition this probe is also annotated with ``probe::context::sim2``. This indicates that we expect this code to be eventually hit in simulation.
+
+By default, FDB simply will write a trace-line when this code is hit and the condition is ``true``. If the code is never hit, the simulator will, at the end of the run, print the code probe but set the ``covered`` field to ``false``. This all happens in the context of a single simulation run (``fdbserver`` doesn't have a concept of ensembles). This information is written into the log file. ``TestHarness`` (see below) will then use this information to write code probe statistics to the ensemble in the Joshua cluster (if the test is run in Joshua).
+
+We expect that ALL code probes will be hit in a nightly run. In the future we can potentially use this feature for other things (like instructing the simulator to do an extensive search starting when one of these probes is being hit).
+
+In addition to ``context`` annotations, users can also define and pass assertions. For example:
+
+.. code-block:: C++
+
+   CODE_PROBE(condition, "Some comment", assert::simOnly);
+
+These will add an assertion to the code. In addition to that, the simulator will not print missed code probes that asserted that the probe won't be hit in simulation.
+
+Test Harness
+============
+
+TestHarness is our primary testing tool. It has multiple jobs:
+
+* *Running*: It can run a test in Joshua.
+* *Statistics*: It will choose a test to run based on previous runs (within the same ensemble) spent CPU time for each test. It does that by writing statistics about the test at the end of each run.
+* *Reporting*: After an ensemble has finished (or while it is running), ``TestHarness`` can be used to generate a report in ``xml`` or ``json``.
+
+Test Harness can be found in the FDB source repository under ``contrib/TestHarness2``. It has a weak dependency to `joshua <https://github.com/foundationDB/fdb-joshua>`_ (if Test Harness can find joshua it will report back about failed tests, otherwise it will just print out general statistics about the ensemble). Joshua will call Test Harness as follows:
+
+.. code-block:: shell
+
+   python3 -m test_harness.app -s ${JOSHUA_SEED} --old-binaries-path ${OLDBINDIR}
+
+Here the seed is a random number generated by joshua and ``OLDBINDIR`` is a directory path where the old fdb binaries can be found (this is needed for restart tests). If one wants to retry a test they can pass the previous joshua seed, a directory path that has *exactly* the same content as ``OLDBINARYDIR``, plus the reported statistics to the test harness app. This should then re-run the same code as before.
+
+In order to figure out what command line arguments ``test_harness.app`` (and ``test_harness.results``) accepts, one can check the contents of ``contrib/TestHarness2/test_harness/config.py``.
+
+Reporting
+---------
+
+After a joshua ensemble completed, ``test_harness.results`` can be used in order to get a report on the ensemble. This will include, by default, a list of all failed tests (similar to ``joshua tail --errors``, though in a more human readable file). For completed ensemble it will also print code probes that weren't hit often enough. An ensemble is considered to be successful if no simulation runs completed with an error AND all code probes have been hit sufficiently often.
diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
index 79cad80a8f..bd4f1388d9 100644
--- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc
+++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
@@ -379,7 +379,9 @@
                   "log_server_min_free_space",
                   "log_server_min_free_space_ratio",
                   "storage_server_durability_lag",
-                  "storage_server_list_fetch_failed"
+                  "storage_server_list_fetch_failed",
+                  "blob_worker_lag",
+                  "blob_worker_missing"
                ]
             },
             "description":"The database is not being saturated by the workload."
@@ -400,7 +402,9 @@
                   "log_server_min_free_space",
                   "log_server_min_free_space_ratio",
                   "storage_server_durability_lag",
-                  "storage_server_list_fetch_failed"
+                  "storage_server_list_fetch_failed",
+                  "blob_worker_lag",
+                  "blob_worker_missing"
                ]
             },
             "description":"The database is not being saturated by the workload."
@@ -523,7 +527,8 @@
                   "duplicate_mutation_streams",
                   "duplicate_mutation_fetch_timeout",
                   "primary_dc_missing",
-                  "fetch_primary_dc_timeout"
+                  "fetch_primary_dc_timeout",
+                  "fetch_storage_wiggler_stats_timeout"
                ]
             },
             "issues":[
@@ -598,7 +603,7 @@
                "counter":0,
                "roughness":0.0
             },
-            "memory_errors":{ // measures number of proxy_memory_limit_exceeded errors
+            "memory_errors":{ // measures number of (commit/grv)_proxy_memory_limit_exceeded errors
                "hz":0.0,
                "counter":0,
                "roughness":0.0
@@ -785,6 +790,11 @@
              "disabled",
              "optional_experimental",
              "required_experimental"
+         ]},
+         "encryption_at_rest_mode": {
+             "$enum":[
+             "disabled",
+             "aes_256_ctr"
          ]}
       },
       "data":{
diff --git a/documentation/sphinx/source/mr-status.rst b/documentation/sphinx/source/mr-status.rst
index 5eb404bbd4..ed550cbee7 100644
--- a/documentation/sphinx/source/mr-status.rst
+++ b/documentation/sphinx/source/mr-status.rst
@@ -131,6 +131,9 @@ min_free_space_ratio                Running out of space (approaching 5% limit).
 log_server_min_free_space           Log server running out of space (approaching 100MB limit).
 log_server_min_free_space_ratio     Log server running out of space (approaching 5% limit).
 storage_server_durability_lag       Storage server durable version falling behind.
+storage_server_list_fetch_failed    Unable to fetch storage server list.
+blob_worker_lag                     Blob worker granule version falling behind.
+blob_worker_missing                 No blob workers are reporting metrics.
 =================================== ====================================================
 
 The JSON path ``cluster.qos.throttled_tags``, when it exists, is an Object containing ``"auto"`` , ``"manual"`` and ``"recommended"``.  The possible fields for those object are in the following table:
diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index d9312f3781..9dad6e05af 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -2,6 +2,61 @@
 Release Notes
 #############
 
+7.1.23
+======
+* Same as 7.1.22 release with AVX enabled.
+
+7.1.22
+======
+* Released with AVX disabled.
+* Added new latency samples for GetValue, GetRange, QueueWait, and VersionWait in storage servers. `(PR #8215) <https://github.com/apple/foundationdb/pull/8215>`_
+* Fixed a rare partial data write for TLogs. `(PR #8210) <https://github.com/apple/foundationdb/pull/8210>`_
+* Added HTTP proxy support for backup agents. `(PR #8193) <https://github.com/apple/foundationdb/pull/8193>`_
+* Fixed a memory bug of secondary queries in index prefetch. `(PR #8195) <https://github.com/apple/foundationdb/pull/8195>`_, `(PR #8190) <https://github.com/apple/foundationdb/pull/8190>`_
+* Introduced STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT knob to recreate SS at io_timeout errors. `(PR #8123) <https://github.com/apple/foundationdb/pull/8123>`_
+* Fixed two TLog stopped bugs and a CC leader replacement bug. `(PR #8081) <https://github.com/apple/foundationdb/pull/8081>`_
+* Added back RecoveryAvailable trace event for status's seconds_since_last_recovered field. `(PR #8068) <https://github.com/apple/foundationdb/pull/8068>`_
+
+7.1.21
+======
+* Same as 7.1.20 release with AVX enabled.
+
+7.1.20
+======
+* Released with AVX disabled.
+* Fixed missing localities for fdbserver that can cause cross DC calls among storage servers. `(PR #7995) <https://github.com/apple/foundationdb/pull/7995>`_
+* Removed extremely spammy trace event in FetchKeys and fixed transaction_profiling_analyzer.py. `(PR #7934) <https://github.com/apple/foundationdb/pull/7934>`_
+* Fixed bugs when GRV proxy returns an error. `(PR #7860) <https://github.com/apple/foundationdb/pull/7860>`_
+
+7.1.19
+======
+* Same as 7.1.18 release with AVX enabled.
+
+7.1.18
+======
+* Released with AVX disabled.
+* Added knobs for the minimum and the maximum of the Ratekeeper's default priority. `(PR #7820) <https://github.com/apple/foundationdb/pull/7820>`_
+* Fixed bugs in ``getRange`` of the special key space. `(PR #7778) <https://github.com/apple/foundationdb/pull/7778>`_, `(PR #7720) <https://github.com/apple/foundationdb/pull/7720>`_
+* Added debug ID for secondary queries in index prefetching. `(PR #7755) <https://github.com/apple/foundationdb/pull/7755>`_
+* Changed hostname resolving to prefer IPv6 addresses. `(PR #7750) <https://github.com/apple/foundationdb/pull/7750>`_
+* Added more transaction debug events for prefetch queries. `(PR #7732) <https://github.com/apple/foundationdb/pull/7732>`_
+
+7.1.17
+======
+* Same as 7.1.16 release with AVX enabled.
+
+7.1.16
+======
+* Released with AVX disabled.
+* Fixed a crash bug when cluster controller shuts down. `(PR #7706) <https://github.com/apple/foundationdb/pull/7706>`_
+* Fixed a storage server failure when getReadVersion returns an error. `(PR #7688) <https://github.com/apple/foundationdb/pull/7688>`_
+* Fixed unbounded status json generation. `(PR #7680) <https://github.com/apple/foundationdb/pull/7680>`_
+* Fixed ScopeEventFieldTypeMismatch error for TLogMetrics. `(PR #7640) <https://github.com/apple/foundationdb/pull/7640>`_
+* Added getMappedRange latency metrics. `(PR #7632) <https://github.com/apple/foundationdb/pull/7632>`_
+* Fixed a version vector performance bug due to not updating client side tag cache. `(PR #7616) <https://github.com/apple/foundationdb/pull/7616>`_
+* Fixed DiskReadSeconds and DiskWriteSeconds calculation in ProcessMetrics. `(PR #7609) <https://github.com/apple/foundationdb/pull/7609>`_
+* Added Rocksdb compression and data size stats. `(PR #7596) <https://github.com/apple/foundationdb/pull/7596>`_
+
 7.1.15
 ======
 * Same as 7.1.14 release with AVX enabled.
@@ -58,7 +113,7 @@ Release Notes
 * Added support of the reboot command in go bindings. `(PR #7270) <https://github.com/apple/foundationdb/pull/7270>`_
 * Fixed several issues in profiling special keys using GlobalConfig. `(PR #7120) <https://github.com/apple/foundationdb/pull/7120>`_
 * Fixed a stuck transaction system bug due to inconsistent recovery transaction version. `(PR #7261) <https://github.com/apple/foundationdb/pull/7261>`_
-* Fixed a unknown_error crash due to not resolving hostnames. `(PR #7254) <https://github.com/apple/foundationdb/pull/7254>`_
+* Fixed an unknown_error crash due to not resolving hostnames. `(PR #7254) <https://github.com/apple/foundationdb/pull/7254>`_
 * Fixed a heap-use-after-free bug. `(PR #7250) <https://github.com/apple/foundationdb/pull/7250>`_
 * Fixed a performance issue that remote TLogs are sending too many pops to log routers. `(PR #7235) <https://github.com/apple/foundationdb/pull/7235>`_
 * Fixed an issue that SharedTLogs are not displaced and leaking disk space. `(PR #7246) <https://github.com/apple/foundationdb/pull/7246>`_
diff --git a/documentation/sphinx/source/release-notes/release-notes-720.rst b/documentation/sphinx/source/release-notes/release-notes-720.rst
index 1496b27c4f..c0093232b4 100644
--- a/documentation/sphinx/source/release-notes/release-notes-720.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-720.rst
@@ -20,6 +20,8 @@ Fixes
 -----
 
 * In ``fdbcli``, integer options are now expressed as integers rather than byte strings (e.g. ``option on TIMEOUT 1000``). `(PR #7571) <https://github.com/apple/foundationdb/pull/7571>`_
+* Fixed the bug in ``ConflictingKeysImpl::getRange`` which happens when an underlying conflicting range contains the read range. Added additional test coverage for validating random ``getRange`` results from special keys. `(PR #7597) <https://github.com/apple/foundationdb/pull/7597>`_
+* Fixed the bug in ``SpecialKeyRangeAsyncImpl::getRange`` that the local cache is updated incorrectly after a cross-module read range if it touched more than one ``SpecialKeyRangeAsyncImpl`` in resolving key selectors. Extended the ``SpecialKeySpaceCorrectness`` workload to catch the bug. `(PR #7671) <https://github.com/apple/foundationdb/pull/7671>`_
 
 Status
 ------
diff --git a/documentation/sphinx/source/special-keys.rst b/documentation/sphinx/source/special-keys.rst
index 1a278d19b4..75877d922b 100644
--- a/documentation/sphinx/source/special-keys.rst
+++ b/documentation/sphinx/source/special-keys.rst
@@ -22,16 +22,15 @@ Each special key that existed before api version 630 is its own module. These ar
 #. ``\xff\xff/cluster_file_path`` - See :ref:`cluster file client access <cluster-file-client-access>`
 #. ``\xff\xff/status/json`` - See :doc:`Machine-readable status <mr-status>`
 
+#. ``\xff\xff/worker_interfaces`` - key as the worker's network address and value as the serialized ClientWorkerInterface, not transactional
+
 Prior to api version 630, it was also possible to read a range starting at ``\xff\xff/worker_interfaces``. This is mostly an implementation detail of fdbcli,
 but it's available in api version 630 as a module with prefix ``\xff\xff/worker_interfaces/``.
 
-Api version 630 includes two new modules:
+Api version 630 includes three new modules:
 
 #. ``\xff\xff/transaction/`` - information about the current transaction
 #. ``\xff\xff/metrics/`` - various metrics, not transactional
-
-Api version 720 includes one new module:
-
 #. ``\xff\xff/clusterId`` - returns an immutable unique ID for a cluster
 
 Transaction module
@@ -209,6 +208,8 @@ that process, and wait for necessary data to be moved away.
 #. ``\xff\xff/management/options/excluded_locality/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/excluded_locality/<locality>``. Setting this key only has an effect in the current transaction and is not persisted on commit.
 #. ``\xff\xff/management/options/failed_locality/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/failed_locality/<locality>``. Setting this key only has an effect in the current transaction and is not persisted on commit.
 #. ``\xff\xff/management/tenant/map/<tenant>`` Read/write. Setting a key in this range to any value will result in a tenant being created with name ``<tenant>``. Clearing a key in this range will delete the tenant with name ``<tenant>``. Reading all or a portion of this range will return the list of tenants currently present in the cluster, excluding any changes in this transaction. Values read in this range will be JSON objects containing the metadata for the associated tenants.
+#. ``\xff\xff/management/tenant/rename/<tenant>`` Read/write. Setting a key in this range to an unused tenant name will result in the tenant with the name ``<tenant>`` to be renamed to the value provided. If the rename operation is a transaction retried in a loop, it is possible for the rename to be applied twice, in which case ``tenant_not_found`` or ``tenant_already_exists`` errors may be returned. This can be avoided by checking for the tenant's existence first.
+#. ``\xff\xff/management/options/worker_interfaces/verify`` Read/write. Setting this key will add a verification phase in reading ``\xff\xff/worker_interfaces``. Setting this key only has an effect in the current transaction and is not persisted on commit. Try to establish connections with every worker from the list returned by Cluster Controller and only return those workers that the client can connect to. This option is now only used in fdbcli commands ``kill``, ``suspend`` and ``expensive_data_check`` to populate the worker list.
 
 An exclusion is syntactically either an ip address (e.g. ``127.0.0.1``), or
 an ip address and port (e.g. ``127.0.0.1:4500``) or any locality (e.g ``locality_dcid:primary-satellite`` or
@@ -275,7 +276,6 @@ Deprecated Keys
 Listed below are the special keys that have been deprecated. Special key(s) will no longer be accessible when the client specifies an API version equal to or larger than the version where they were deprecated. Clients specifying older API versions will be able to continue using the deprecated key(s).
 
 #. ``\xff\xff/management/profiling/<client_txn_sample_rate|client_txn_size_limit>`` Deprecated as of API version 720. The corresponding functionalities are now covered by the global configuration module. For details, see :doc:`global-configuration`. Read/write. Changing these two keys will change the corresponding system keys ``\xff\x02/fdbClientInfo/<client_txn_sample_rate|client_txn_size_limit>``, respectively. The value of ``\xff\xff/management/client_txn_sample_rate`` is a literal text of ``double``, and the value of ``\xff\xff/management/client_txn_size_limit`` is a literal text of ``int64_t``. A special value ``default`` can be set to or read from these two keys, representing the client profiling is disabled. In addition, ``clear`` in this range is not allowed. For more details, see help text of ``fdbcli`` command ``profile client``.
-#. ``\xff\xff/management/tenant_map/<tenant>`` Removed as of API version 720 and renamed to ``\xff\xff/management/tenant/map/<tenant>``.
 
 Versioning
 ==========
diff --git a/documentation/sphinx/source/tenants.rst b/documentation/sphinx/source/tenants.rst
index d22603b20e..07bd7b2a42 100644
--- a/documentation/sphinx/source/tenants.rst
+++ b/documentation/sphinx/source/tenants.rst
@@ -2,6 +2,8 @@
 Tenants
 #######
 
+.. _multi-tenancy:
+
 .. warning :: Tenants are currently experimental and are not recommended for use in production.
 
 FoundationDB provides a feature called tenants that allow you to configure one or more named transaction domains in your cluster. A transaction domain is a key-space in which a transaction is allowed to operate, and no tenant operations are allowed to use keys outside the tenant key-space. Tenants can be useful for managing separate, unrelated use-cases and preventing them from interfering with each other. They can also be helpful for defining safe boundaries when moving a subset of data between clusters.
@@ -49,7 +51,7 @@ All operations performed within a tenant transaction will occur within the tenan
 Raw access
 ----------
 
-When operating in the tenant mode ``required_experimental``, transactions are not ordinarily permitted to run without using a tenant. In order to access the system keys or perform maintenance operations that span multiple tenants, it is required to use the ``RAW_ACCESS`` transaction option to access the global key-space. It is an error to specify ``RAW_ACCESS`` on a transaction that is configured to use a tenant.
+When operating in the tenant mode ``required_experimental`` or using a metacluster, transactions are not ordinarily permitted to run without using a tenant. In order to access the system keys or perform maintenance operations that span multiple tenants, it is required to use the ``RAW_ACCESS`` transaction option to access the global key-space. It is an error to specify ``RAW_ACCESS`` on a transaction that is configured to use a tenant.
 
 .. note :: Setting the ``READ_SYSTEM_KEYS`` or ``ACCESS_SYSTEM_KEYS`` options implies ``RAW_ACCESS`` for your transaction.
 
diff --git a/documentation/sphinx/source/tls.rst b/documentation/sphinx/source/tls.rst
index 3fb4a08d0c..dcb0c2c930 100644
--- a/documentation/sphinx/source/tls.rst
+++ b/documentation/sphinx/source/tls.rst
@@ -126,11 +126,11 @@ Default Values
 Certificate file default location
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The default behavior when the certificate or key file is not specified is to look for a file named ``fdb.pem`` in the current working directory. If this file is not present, an attempt is made to load a file from a system-dependent location as follows:
+The default behavior when the certificate or key file is not specified is to look for files named ``cert.pem`` or  ``key.pem`` respectively, in system-dependent locations as follows:
 
-* Linux: ``/etc/foundationdb/fdb.pem``
-* macOS: ``/usr/local/etc/foundationdb/fdb.pem``
-* Windows: ``C:\ProgramData\foundationdb\fdb.pem``
+* Linux: ``/etc/foundationdb/cert.pem`` and ``/etc/foundationdb/key.pem``
+* macOS: ``/usr/local/etc/foundationdb/cert.pem`` and ``/usr/local/etc/foundationdb/key.pem``
+* Windows: ``C:\ProgramData\foundationdb\cert.pem`` and ``C:\ProgramData\foundationdb\key.pem``
 
 Default Peer Verification
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -214,9 +214,12 @@ Certificate creation
 
 If your organization already makes use of certificates for access control and securing communications, you should ask your security expert for organizational procedure for obtaining and verifying certificates. If the goal of enabling TLS is to make sure that only known machines can join or access the FoundationDB cluster and for securing communications, then creating your own certificates can serve these purposes.
 
-The following set of commands uses the OpenSSL command-line tools to create a self-signed certificate and private key. The certificate is then joined with the private key in the output ``fdb.pem`` file::
+The following set of commands uses the OpenSSL command-line tools to create a self-signed certificate and private key::
+
+  user@host:> openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout key.pem -out cert.pem
+
+Optionally, the certificate can be joined with the private key as supplied as both certificate and key files::
 
-  user@host:> openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout private.key -out cert.crt
   user@host:> cat cert.crt private.key > fdb.pem
 
 Peer verification
diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp
index 9d980ff3d6..245e6d09e3 100644
--- a/documentation/tutorial/tutorial.actor.cpp
+++ b/documentation/tutorial/tutorial.actor.cpp
@@ -478,7 +478,7 @@ ACTOR Future<Void> fdbClient() {
 	state Transaction tx(db);
 	state std::string keyPrefix = "/tut/";
 	state Key startKey;
-	state KeyRef endKey = LiteralStringRef("/tut0");
+	state KeyRef endKey = "/tut0"_sr;
 	state int beginIdx = 0;
 	loop {
 		try {
@@ -494,7 +494,7 @@ ACTOR Future<Void> fdbClient() {
 			RangeResult range = wait(tx.getRange(KeyRangeRef(startKey, endKey), 100));
 			for (int i = 0; i < 10; ++i) {
 				Key k = Key(keyPrefix + std::to_string(beginIdx + deterministicRandom()->randomInt(0, 100)));
-				tx.set(k, LiteralStringRef("foo"));
+				tx.set(k, "foo"_sr);
 			}
 			wait(tx.commit());
 			std::cout << "Committed\n";
diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index ab4bf0fd3d..24601308e1 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "flow/ApiVersion.h"
 #include "fmt/format.h"
 #include "fdbbackup/BackupTLSConfig.h"
 #include "fdbclient/JsonBuilder.h"
@@ -36,6 +37,7 @@
 #include "flow/genericactors.actor.h"
 #include "flow/TLSConfig.actor.h"
 
+#include "fdbclient/DatabaseContext.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/Status.h"
@@ -903,12 +905,12 @@ CSimpleOpt::SOption g_rgDBPauseOptions[] = {
 	SO_END_OF_OPTIONS
 };
 
-const KeyRef exeAgent = LiteralStringRef("backup_agent");
-const KeyRef exeBackup = LiteralStringRef("fdbbackup");
-const KeyRef exeRestore = LiteralStringRef("fdbrestore");
-const KeyRef exeFastRestoreTool = LiteralStringRef("fastrestore_tool"); // must be lower case
-const KeyRef exeDatabaseAgent = LiteralStringRef("dr_agent");
-const KeyRef exeDatabaseBackup = LiteralStringRef("fdbdr");
+const KeyRef exeAgent = "backup_agent"_sr;
+const KeyRef exeBackup = "fdbbackup"_sr;
+const KeyRef exeRestore = "fdbrestore"_sr;
+const KeyRef exeFastRestoreTool = "fastrestore_tool"_sr; // must be lower case
+const KeyRef exeDatabaseAgent = "dr_agent"_sr;
+const KeyRef exeDatabaseBackup = "fdbdr"_sr;
 
 extern const char* getSourceVersion();
 
@@ -927,7 +929,7 @@ void parentWatcher(void* parentHandle) {
 static void printVersion() {
 	printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
 	printf("source version %s\n", getSourceVersion());
-	printf("protocol %llx\n", (long long)currentProtocolVersion.version());
+	printf("protocol %llx\n", (long long)currentProtocolVersion().version());
 }
 
 static void printBuildInformation() {
@@ -1349,7 +1351,7 @@ ProgramExe getProgramType(std::string programExe) {
 	}
 #endif
 	// For debugging convenience, remove .debug suffix if present.
-	if (StringRef(programExe).endsWith(LiteralStringRef(".debug")))
+	if (StringRef(programExe).endsWith(".debug"_sr))
 		programExe = programExe.substr(0, programExe.size() - 6);
 
 	// Check if backup agent
@@ -1854,11 +1856,7 @@ ACTOR Future<Void> submitDBBackup(Database src,
                                   std::string tagName) {
 	try {
 		state DatabaseBackupAgent backupAgent(src);
-
-		// Backup everything, if no ranges were specified
-		if (backupRanges.size() == 0) {
-			backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
-		}
+		ASSERT(!backupRanges.empty());
 
 		wait(backupAgent.submitBackup(
 		    dest, KeyRef(tagName), backupRanges, StopWhenDone::False, StringRef(), StringRef(), LockDB::True));
@@ -1904,6 +1902,7 @@ ACTOR Future<Void> submitBackup(Database db,
                                 int initialSnapshotIntervalSeconds,
                                 int snapshotIntervalSeconds,
                                 Standalone<VectorRef<KeyRangeRef>> backupRanges,
+                                bool encryptionEnabled,
                                 std::string tagName,
                                 bool dryRun,
                                 WaitForComplete waitForCompletion,
@@ -1912,11 +1911,7 @@ ACTOR Future<Void> submitBackup(Database db,
                                 IncrementalBackupOnly incrementalBackupOnly) {
 	try {
 		state FileBackupAgent backupAgent;
-
-		// Backup everything, if no ranges were specified
-		if (backupRanges.size() == 0) {
-			backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
-		}
+		ASSERT(!backupRanges.empty());
 
 		if (dryRun) {
 			state KeyBackedTag tag = makeBackupTag(tagName);
@@ -1963,6 +1958,7 @@ ACTOR Future<Void> submitBackup(Database db,
 			                              snapshotIntervalSeconds,
 			                              tagName,
 			                              backupRanges,
+			                              encryptionEnabled,
 			                              stopWhenDone,
 			                              usePartitionedLog,
 			                              incrementalBackupOnly));
@@ -2016,11 +2012,7 @@ ACTOR Future<Void> switchDBBackup(Database src,
                                   ForceAction forceAction) {
 	try {
 		state DatabaseBackupAgent backupAgent(src);
-
-		// Backup everything, if no ranges were specified
-		if (backupRanges.size() == 0) {
-			backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
-		}
+		ASSERT(!backupRanges.empty());
 
 		wait(backupAgent.atomicSwitchover(dest, KeyRef(tagName), backupRanges, StringRef(), StringRef(), forceAction));
 		printf("The DR on tag `%s' was successfully switched.\n", printable(StringRef(tagName)).c_str());
@@ -2287,9 +2279,7 @@ ACTOR Future<Void> runRestore(Database db,
                               OnlyApplyMutationLogs onlyApplyMutationLogs,
                               InconsistentSnapshotOnly inconsistentSnapshotOnly,
                               Optional<std::string> encryptionKeyFile) {
-	if (ranges.empty()) {
-		ranges.push_back_deep(ranges.arena(), normalKeys);
-	}
+	ASSERT(!ranges.empty());
 
 	if (targetVersion != invalidVersion && !targetTimestamp.empty()) {
 		fprintf(stderr, "Restore target version and target timestamp cannot both be specified\n");
@@ -2313,7 +2303,7 @@ ACTOR Future<Void> runRestore(Database db,
 			throw restore_error();
 		}
 
-		origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST);
+		origDb = Database::createDatabase(originalClusterFile, ApiVersion::LATEST_VERSION);
 		Version v = wait(timeKeeperVersionFromDatetime(targetTimestamp, origDb.get()));
 		fmt::print("Timestamp '{0}' resolves to version {1}\n", targetTimestamp, v);
 		targetVersion = v;
@@ -2370,7 +2360,7 @@ ACTOR Future<Void> runRestore(Database db,
 				fmt::print("Restored to version {}\n", restoredVersion);
 			}
 		} else {
-			state Optional<RestorableFileSet> rset = wait(bc->getRestoreSet(targetVersion, ranges));
+			state Optional<RestorableFileSet> rset = wait(bc->getRestoreSet(targetVersion, db, ranges));
 
 			if (!rset.present()) {
 				fmt::print(stderr,
@@ -2447,8 +2437,8 @@ ACTOR Future<Void> runFastRestoreTool(Database db,
 			                                       dbVersion,
 			                                       LockDB::True,
 			                                       randomUID,
-			                                       LiteralStringRef(""),
-			                                       LiteralStringRef("")));
+			                                       ""_sr,
+			                                       ""_sr));
 			// TODO: Support addPrefix and removePrefix
 			if (waitForDone) {
 				// Wait for parallel restore to finish and unlock DB after that
@@ -2480,7 +2470,7 @@ ACTOR Future<Void> runFastRestoreTool(Database db,
 				restoreVersion = dbVersion;
 			}
 
-			state Optional<RestorableFileSet> rset = wait(bc->getRestoreSet(restoreVersion));
+			state Optional<RestorableFileSet> rset = wait(bc->getRestoreSet(restoreVersion, db));
 			if (!rset.present()) {
 				fmt::print(stderr, "Insufficient data to restore to version {}\n", restoreVersion);
 				throw restore_invalid_version();
@@ -2685,7 +2675,8 @@ ACTOR Future<Void> queryBackup(const char* name,
                                Version restoreVersion,
                                std::string originalClusterFile,
                                std::string restoreTimestamp,
-                               Verbose verbose) {
+                               Verbose verbose,
+                               Optional<Database> cx) {
 	state UID operationId = deterministicRandom()->randomUniqueID();
 	state JsonBuilderObject result;
 	state std::string errorMessage;
@@ -2719,7 +2710,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 			return Void();
 		}
 
-		Database origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST);
+		Database origDb = Database::createDatabase(originalClusterFile, ApiVersion::LATEST_VERSION);
 		Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb));
 		result["restore_timestamp"] = restoreTimestamp;
 		result["restore_timestamp_resolved_version"] = v;
@@ -2750,7 +2741,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 			                           format("the specified restorable version %lld is not valid", restoreVersion));
 			return Void();
 		}
-		Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter));
+		Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(restoreVersion, cx, keyRangesFilter));
 		if (fileSet.present()) {
 			int64_t totalRangeFilesSize = 0, totalLogFilesSize = 0;
 			result["restore_version"] = fileSet.get().targetVersion;
@@ -3087,7 +3078,7 @@ static void addKeyRange(std::string optionValue, Standalone<VectorRef<KeyRangeRe
 Version parseVersion(const char* str) {
 	StringRef s((const uint8_t*)str, strlen(str));
 
-	if (s.endsWith(LiteralStringRef("days")) || s.endsWith(LiteralStringRef("d"))) {
+	if (s.endsWith("days"_sr) || s.endsWith("d"_sr)) {
 		float days;
 		if (sscanf(str, "%f", &days) != 1) {
 			fprintf(stderr, "Could not parse version: %s\n", str);
@@ -3126,7 +3117,7 @@ Optional<Database> connectToCluster(std::string const& clusterFile,
 	}
 
 	try {
-		db = Database::createDatabase(ccf, -1, IsInternal::True, localities);
+		db = Database::createDatabase(ccf, ApiVersion::LATEST_VERSION, IsInternal::True, localities);
 	} catch (Error& e) {
 		if (!quiet) {
 			fprintf(stderr, "ERROR: %s\n", e.what());
@@ -3376,6 +3367,8 @@ int main(int argc, char* argv[]) {
 		bool trace = false;
 		bool quietDisplay = false;
 		bool dryRun = false;
+		// TODO (Nim): Set this value when we add optional encrypt_files CLI argument to backup agent start
+		bool encryptionEnabled = true;
 		std::string traceDir = "";
 		std::string traceFormat = "";
 		std::string traceLogGroup;
@@ -3606,7 +3599,7 @@ int main(int argc, char* argv[]) {
 			case OPT_DESTCONTAINER:
 				destinationContainer = args->OptionArg();
 				// If the url starts with '/' then prepend "file://" for backwards compatibility
-				if (StringRef(destinationContainer).startsWith(LiteralStringRef("/")))
+				if (StringRef(destinationContainer).startsWith("/"_sr))
 					destinationContainer = std::string("file://") + destinationContainer;
 				modifyOptions.destURL = destinationContainer;
 				break;
@@ -3652,7 +3645,7 @@ int main(int argc, char* argv[]) {
 			case OPT_RESTORECONTAINER:
 				restoreContainer = args->OptionArg();
 				// If the url starts with '/' then prepend "file://" for backwards compatibility
-				if (StringRef(restoreContainer).startsWith(LiteralStringRef("/")))
+				if (StringRef(restoreContainer).startsWith("/"_sr))
 					restoreContainer = std::string("file://") + restoreContainer;
 				break;
 			case OPT_DESCRIBE_DEEP:
@@ -3943,6 +3936,12 @@ int main(int argc, char* argv[]) {
 			return result.present();
 		};
 
+		// The fastrestore tool does not yet support multiple ranges and is incompatible with tenants
+		// or other features that back up data in the system keys
+		if (backupKeys.empty() && programExe != ProgramExe::FASTRESTORE_TOOL) {
+			addDefaultBackupRanges(backupKeys);
+		}
+
 		switch (programExe) {
 		case ProgramExe::AGENT:
 			if (!initCluster())
@@ -3962,6 +3961,7 @@ int main(int argc, char* argv[]) {
 				                           initialSnapshotIntervalSeconds,
 				                           snapshotIntervalSeconds,
 				                           backupKeys,
+				                           encryptionEnabled,
 				                           tagName,
 				                           dryRun,
 				                           waitForDone,
@@ -4082,7 +4082,8 @@ int main(int argc, char* argv[]) {
 				                          restoreVersion,
 				                          restoreClusterFileOrig,
 				                          restoreTimestamp,
-				                          Verbose{ !quietDisplay }));
+				                          Verbose{ !quietDisplay },
+				                          db));
 				break;
 
 			case BackupType::DUMP:
@@ -4122,7 +4123,7 @@ int main(int argc, char* argv[]) {
 				}
 
 				try {
-					db = Database::createDatabase(restoreClusterFileDest, Database::API_VERSION_LATEST);
+					db = Database::createDatabase(restoreClusterFileDest, ApiVersion::LATEST_VERSION);
 				} catch (Error& e) {
 					fprintf(stderr,
 					        "Restore destination cluster file '%s' invalid: %s\n",
@@ -4201,7 +4202,7 @@ int main(int argc, char* argv[]) {
 				}
 
 				try {
-					db = Database::createDatabase(restoreClusterFileDest, Database::API_VERSION_LATEST);
+					db = Database::createDatabase(restoreClusterFileDest, ApiVersion::LATEST_VERSION);
 				} catch (Error& e) {
 					fprintf(stderr,
 					        "Restore destination cluster file '%s' invalid: %s\n",
@@ -4321,19 +4322,19 @@ int main(int argc, char* argv[]) {
 				char* demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL);
 				if (demangled) {
 					s = demangled;
-					if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::")))
-						s = s.substr(LiteralStringRef("(anonymous namespace)::").size());
+					if (StringRef(s).startsWith("(anonymous namespace)::"_sr))
+						s = s.substr("(anonymous namespace)::"_sr.size());
 					free(demangled);
 				} else
 					s = i->first;
 #else
 				s = i->first;
-				if (StringRef(s).startsWith(LiteralStringRef("class `anonymous namespace'::")))
-					s = s.substr(LiteralStringRef("class `anonymous namespace'::").size());
-				else if (StringRef(s).startsWith(LiteralStringRef("class ")))
-					s = s.substr(LiteralStringRef("class ").size());
-				else if (StringRef(s).startsWith(LiteralStringRef("struct ")))
-					s = s.substr(LiteralStringRef("struct ").size());
+				if (StringRef(s).startsWith("class `anonymous namespace'::"_sr))
+					s = s.substr("class `anonymous namespace'::"_sr.size());
+				else if (StringRef(s).startsWith("class "_sr))
+					s = s.substr("class "_sr.size());
+				else if (StringRef(s).startsWith("struct "_sr))
+					s = s.substr("struct "_sr.size());
 #endif
 
 				typeNames.emplace_back(s, i->first);
diff --git a/fdbcli/AdvanceVersionCommand.actor.cpp b/fdbcli/AdvanceVersionCommand.actor.cpp
index 223af2d8e5..d3ba08d675 100644
--- a/fdbcli/AdvanceVersionCommand.actor.cpp
+++ b/fdbcli/AdvanceVersionCommand.actor.cpp
@@ -31,7 +31,7 @@
 
 namespace fdb_cli {
 
-const KeyRef advanceVersionSpecialKey = LiteralStringRef("\xff\xff/management/min_required_commit_version");
+const KeyRef advanceVersionSpecialKey = "\xff\xff/management/min_required_commit_version"_sr;
 
 ACTOR Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() != 2) {
diff --git a/fdbcli/BlobKeyCommand.actor.cpp b/fdbcli/BlobKeyCommand.actor.cpp
new file mode 100644
index 0000000000..34d5b98720
--- /dev/null
+++ b/fdbcli/BlobKeyCommand.actor.cpp
@@ -0,0 +1,188 @@
+/*
+ * BlobKeyCommand.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/NativeAPI.actor.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+
+ACTOR Future<bool> printBlobHistory(Database db, Key key, Optional<Version> version) {
+	fmt::print("Printing blob history for {0}", key.printable());
+	if (version.present()) {
+		fmt::print(" @ {0}", version.get());
+	}
+	fmt::print("\n");
+
+	state Transaction tr(db);
+	state KeyRange activeGranule;
+	state KeyRange queryRange(KeyRangeRef(key, keyAfter(key)));
+	loop {
+		try {
+			Standalone<VectorRef<KeyRangeRef>> granules = wait(tr.getBlobGranuleRanges(queryRange, 2));
+			if (granules.empty()) {
+				fmt::print("No active granule for {0}\n", key.printable());
+				return false;
+			}
+			ASSERT(granules.size() == 1);
+			activeGranule = granules[0];
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+	fmt::print("Active granule: [{0} - {1})\n", activeGranule.begin.printable(), activeGranule.end.printable());
+
+	// get latest history entry for range
+	state GranuleHistory history;
+	loop {
+		try {
+			RangeResult result =
+			    wait(tr.getRange(blobGranuleHistoryKeyRangeFor(activeGranule), 1, Snapshot::False, Reverse::True));
+			ASSERT(result.size() <= 1);
+
+			if (result.empty()) {
+				fmt::print("No history entry found\n");
+				return true;
+			}
+
+			std::pair<KeyRange, Version> decodedKey = decodeBlobGranuleHistoryKey(result[0].key);
+			ASSERT(activeGranule == decodedKey.first);
+			history = GranuleHistory(activeGranule, decodedKey.second, decodeBlobGranuleHistoryValue(result[0].value));
+
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+
+	fmt::print("History:\n\n");
+	loop {
+		// print history
+		std::string boundaryChangeAction;
+		if (history.value.parentVersions.empty()) {
+			boundaryChangeAction = "root";
+		} else if (history.value.parentVersions.size() == 1) {
+			boundaryChangeAction = "split";
+		} else {
+			boundaryChangeAction = "merge";
+		}
+		fmt::print("{0}) {1}\n\t{2}\n\t{3}\n({4})\n\n",
+		           history.version,
+		           history.value.granuleID.toString(),
+		           history.range.begin.printable(),
+		           history.range.end.printable(),
+		           boundaryChangeAction);
+		// traverse back
+
+		if (history.value.parentVersions.empty() || (version.present() && history.version <= version.get())) {
+			break;
+		}
+
+		int i;
+		for (i = 0; i < history.value.parentBoundaries.size(); i++) {
+			if (history.value.parentBoundaries[i] <= key) {
+				break;
+			}
+		}
+		// key should fall between boundaries
+		ASSERT(i < history.value.parentBoundaries.size());
+		KeyRangeRef parentRange(history.value.parentBoundaries[i], history.value.parentBoundaries[i + 1]);
+		Version parentVersion = history.value.parentVersions[i];
+		state Key parentHistoryKey = blobGranuleHistoryKeyFor(parentRange, parentVersion);
+		state bool foundParent;
+
+		loop {
+			try {
+				Optional<Value> parentHistoryValue = wait(tr.get(parentHistoryKey));
+				foundParent = parentHistoryValue.present();
+				if (foundParent) {
+					std::pair<KeyRange, Version> decodedKey = decodeBlobGranuleHistoryKey(parentHistoryKey);
+					history = GranuleHistory(
+					    decodedKey.first, decodedKey.second, decodeBlobGranuleHistoryValue(parentHistoryValue.get()));
+				}
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+		if (!foundParent) {
+			break;
+		}
+	}
+
+	fmt::print("Done\n");
+	return true;
+}
+
+} // namespace
+
+namespace fdb_cli {
+
+ACTOR Future<bool> blobKeyCommandActor(Database localDb,
+                                       Optional<TenantMapEntry> tenantEntry,
+                                       std::vector<StringRef> tokens) {
+	// enables blob writing for the given range
+	if (tokens.size() != 3 && tokens.size() != 4) {
+		printUsage(tokens[0]);
+		return false;
+	}
+
+	ASSERT(tokens[1] == "history"_sr);
+
+	Key key;
+	Optional<Version> version;
+
+	if (tenantEntry.present()) {
+		key = tokens[2].withPrefix(tenantEntry.get().prefix);
+	} else {
+		key = tokens[2];
+	}
+
+	if (tokens.size() > 3) {
+		Version v;
+		int n = 0;
+		if (sscanf(tokens[3].toString().c_str(), "%" PRId64 "%n", &v, &n) != 1 || n != tokens[3].size()) {
+			printUsage(tokens[0]);
+			return false;
+		}
+		version = v;
+	}
+
+	if (key >= "\xff"_sr) {
+		fmt::print("No blob history for system keyspace\n", key.printable());
+		return false;
+	} else {
+		bool result = wait(printBlobHistory(localDb, key, version));
+		return result;
+	}
+}
+
+// can extend to other blobkey commands later
+CommandFactory blobKeyFactory("blobkey", CommandHelp("blobkey history <key> [version]", "", ""));
+} // namespace fdb_cli
diff --git a/fdbcli/BlobRangeCommand.actor.cpp b/fdbcli/BlobRangeCommand.actor.cpp
index 02b922c4a8..38edaa8568 100644
--- a/fdbcli/BlobRangeCommand.actor.cpp
+++ b/fdbcli/BlobRangeCommand.actor.cpp
@@ -23,6 +23,7 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
 #include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/NativeAPI.actor.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
@@ -31,33 +32,62 @@
 
 namespace {
 
-// copy to standalones for krm
-ACTOR Future<Void> setBlobRange(Database db, Key startKey, Key endKey, Value value) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
-
+ACTOR Future<Version> getLatestReadVersion(Database db) {
+	state Transaction tr(db);
 	loop {
 		try {
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-
-			// FIXME: check that the set range is currently inactive, and that a revoked range is currently its own
-			// range in the map and fully set.
-
-			tr->set(blobRangeChangeKey, deterministicRandom()->randomUniqueID().toString());
-			// This is not coalescing because we want to keep each range logically separate.
-			wait(krmSetRange(tr, blobRangeKeys.begin, KeyRange(KeyRangeRef(startKey, endKey)), value));
-			wait(tr->commit());
-			printf("Successfully updated blob range [%s - %s) to %s\n",
-			       startKey.printable().c_str(),
-			       endKey.printable().c_str(),
-			       value.printable().c_str());
-			return Void();
+			Version rv = wait(tr.getReadVersion());
+			fmt::print("Resolved latest read version as {0}\n", rv);
+			return rv;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			wait(tr.onError(e));
 		}
 	}
 }
 
+// print after delay if not cancelled
+ACTOR Future<Void> printAfterDelay(double delaySeconds, std::string message) {
+	wait(delay(delaySeconds));
+	fmt::print("{}\n", message);
+	return Void();
+}
+
+ACTOR Future<Void> doBlobPurge(Database db, Key startKey, Key endKey, Optional<Version> version, bool force) {
+	state Version purgeVersion;
+	if (version.present()) {
+		purgeVersion = version.get();
+	} else {
+		wait(store(purgeVersion, getLatestReadVersion(db)));
+	}
+
+	state Key purgeKey = wait(db->purgeBlobGranules(KeyRange(KeyRangeRef(startKey, endKey)), purgeVersion, {}, force));
+
+	fmt::print("Blob purge registered for [{0} - {1}) @ {2}\n", startKey.printable(), endKey.printable(), purgeVersion);
+
+	state Future<Void> printWarningActor = printAfterDelay(
+	    5.0, "Waiting for purge to complete. (interrupting this wait with CTRL+C will not cancel the purge)");
+	wait(db->waitPurgeGranulesComplete(purgeKey));
+
+	fmt::print("Blob purge complete for [{0} - {1}) @ {2}\n", startKey.printable(), endKey.printable(), purgeVersion);
+
+	return Void();
+}
+
+ACTOR Future<Void> doBlobCheck(Database db, Key startKey, Key endKey, Optional<Version> version) {
+	state double elapsed = -timer_monotonic();
+
+	state Version readVersionOut = wait(db->verifyBlobRange(KeyRangeRef(startKey, endKey), version));
+
+	elapsed += timer_monotonic();
+
+	fmt::print("Blob check complete for [{0} - {1}) @ {2} in {3:.6f} seconds\n",
+	           startKey.printable(),
+	           endKey.printable(),
+	           readVersionOut,
+	           elapsed);
+	return Void();
+}
+
 } // namespace
 
 namespace fdb_cli {
@@ -66,7 +96,7 @@ ACTOR Future<bool> blobRangeCommandActor(Database localDb,
                                          Optional<TenantMapEntry> tenantEntry,
                                          std::vector<StringRef> tokens) {
 	// enables blob writing for the given range
-	if (tokens.size() != 4) {
+	if (tokens.size() != 4 && tokens.size() != 5) {
 		printUsage(tokens[0]);
 		return false;
 	}
@@ -82,31 +112,80 @@ ACTOR Future<bool> blobRangeCommandActor(Database localDb,
 		end = tokens[3];
 	}
 
-	if (end > LiteralStringRef("\xff")) {
+	if (end > "\xff"_sr) {
 		// TODO is this something we want?
-		printf("Cannot blobbify system keyspace! Problematic End Key: %s\n", tokens[3].printable().c_str());
+		fmt::print("Cannot blobbify system keyspace! Problematic End Key: {0}\n", tokens[3].printable());
 		return false;
 	} else if (tokens[2] >= tokens[3]) {
-		printf("Invalid blob range [%s - %s)\n", tokens[2].printable().c_str(), tokens[3].printable().c_str());
+		fmt::print("Invalid blob range [{0} - {1})\n", tokens[2].printable(), tokens[3].printable());
 	} else {
-		if (tokencmp(tokens[1], "start")) {
-			printf("Starting blobbify range for [%s - %s)\n",
-			       tokens[2].printable().c_str(),
-			       tokens[3].printable().c_str());
-			wait(setBlobRange(localDb, begin, end, LiteralStringRef("1")));
-		} else if (tokencmp(tokens[1], "stop")) {
-			printf("Stopping blobbify range for [%s - %s)\n",
-			       tokens[2].printable().c_str(),
-			       tokens[3].printable().c_str());
-			wait(setBlobRange(localDb, begin, end, StringRef()));
+		if (tokencmp(tokens[1], "start") || tokencmp(tokens[1], "stop")) {
+			state bool starting = tokencmp(tokens[1], "start");
+			if (tokens.size() > 4) {
+				printUsage(tokens[0]);
+				return false;
+			}
+			fmt::print("{0} blobbify range for [{1} - {2})\n",
+			           starting ? "Starting" : "Stopping",
+			           tokens[2].printable(),
+			           tokens[3].printable());
+			state bool success = false;
+			if (starting) {
+				wait(store(success, localDb->blobbifyRange(KeyRangeRef(begin, end))));
+			} else {
+				wait(store(success, localDb->unblobbifyRange(KeyRangeRef(begin, end))));
+			}
+			if (success) {
+				fmt::print("{0} updated blob range [{1} - {2}) succeeded\n",
+				           starting ? "Starting" : "Stopping",
+				           tokens[2].printable(),
+				           tokens[3].printable());
+			} else {
+				fmt::print("{0} blobbify range for [{1} - {2}) failed\n",
+				           starting ? "Starting" : "Stopping",
+				           tokens[2].printable(),
+				           tokens[3].printable());
+			}
+			return success;
+		} else if (tokencmp(tokens[1], "purge") || tokencmp(tokens[1], "forcepurge") || tokencmp(tokens[1], "check")) {
+			bool purge = tokencmp(tokens[1], "purge") || tokencmp(tokens[1], "forcepurge");
+			bool forcePurge = tokencmp(tokens[1], "forcepurge");
+
+			Optional<Version> version;
+			if (tokens.size() > 4) {
+				Version v;
+				int n = 0;
+				if (sscanf(tokens[4].toString().c_str(), "%" PRId64 "%n", &v, &n) != 1 || n != tokens[4].size()) {
+					printUsage(tokens[0]);
+					return false;
+				}
+				version = v;
+			}
+
+			fmt::print("{0} blob range [{1} - {2}){3}",
+			           purge ? "Purging" : "Checking",
+			           tokens[2].printable(),
+			           tokens[3].printable(),
+			           forcePurge ? " (force)" : "");
+			if (version.present()) {
+				fmt::print(" @ {0}", version.get());
+			}
+			fmt::print("\n");
+
+			if (purge) {
+				wait(doBlobPurge(localDb, begin, end, version, forcePurge));
+			} else {
+				wait(doBlobCheck(localDb, begin, end, version));
+			}
 		} else {
 			printUsage(tokens[0]);
-			printf("Usage: blobrange <start|stop> <startkey> <endkey>");
 			return false;
 		}
 	}
 	return true;
 }
 
-CommandFactory blobRangeFactory("blobrange", CommandHelp("blobrange <start|stop> <startkey> <endkey>", "", ""));
+CommandFactory blobRangeFactory(
+    "blobrange",
+    CommandHelp("blobrange <start|stop|check|purge|forcepurge> <startkey> <endkey> [version]", "", ""));
 } // namespace fdb_cli
diff --git a/fdbcli/CMakeLists.txt b/fdbcli/CMakeLists.txt
index aff53b1f63..c25c335a15 100644
--- a/fdbcli/CMakeLists.txt
+++ b/fdbcli/CMakeLists.txt
@@ -1,8 +1,16 @@
+include(AddFdbTest)
 fdb_find_sources(FDBCLI_SRCS)
 
 add_flow_target(EXECUTABLE NAME fdbcli SRCS ${FDBCLI_SRCS})
 target_include_directories(fdbcli PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 target_link_libraries(fdbcli PRIVATE fdbclient SimpleOpt)
+if (USE_UBSAN)
+  # The intent is to put typeinfo symbols in the dynamic symbol table so that
+  # the types in fdbcli and external libfdb_c clients agree for ubsan's vptr
+  # check. This would not be a good idea for the normal build, or if we ever
+  # start testing old libfdb_c's that are ubsan-instrumented.
+  target_link_options(fdbcli PRIVATE "-rdynamic")
+endif()
 
 if(NOT WIN32)
   target_link_libraries(fdbcli PRIVATE linenoise)
@@ -16,3 +24,38 @@ if(NOT OPEN_FOR_IDE)
     fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/packages/bin/fdbcli DESTINATION bin COMPONENT clients)
   endif()
 endif()
+
+if (NOT WIN32 AND NOT OPEN_FOR_IDE)
+  add_dependencies(fdbcli external_client)
+
+  add_fdbclient_test(
+    NAME single_process_fdbcli_tests
+    COMMAND ${CMAKE_SOURCE_DIR}/fdbcli/tests/fdbcli_tests.py
+            ${CMAKE_BINARY_DIR}
+            @CLUSTER_FILE@
+            )
+  add_fdbclient_test(
+    NAME multi_process_fdbcli_tests
+    PROCESS_NUMBER 5
+    COMMAND ${CMAKE_SOURCE_DIR}/fdbcli/tests/fdbcli_tests.py
+            ${CMAKE_BINARY_DIR}
+            @CLUSTER_FILE@
+            5
+            )
+  add_fdbclient_test(
+  NAME single_process_external_client_fdbcli_tests
+  COMMAND ${CMAKE_SOURCE_DIR}/fdbcli/tests/fdbcli_tests.py
+          ${CMAKE_BINARY_DIR}
+          @CLUSTER_FILE@
+          --external-client-library ${CMAKE_BINARY_DIR}/bindings/c/libfdb_c_external.so
+          )
+  add_fdbclient_test(
+  NAME multi_process_external_client_fdbcli_tests
+  PROCESS_NUMBER 5
+  COMMAND ${CMAKE_SOURCE_DIR}/fdbcli/tests/fdbcli_tests.py
+          ${CMAKE_BINARY_DIR}
+          @CLUSTER_FILE@
+          5
+          --external-client-library ${CMAKE_BINARY_DIR}/bindings/c/libfdb_c_external.so
+          )
+endif()
diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.actor.cpp
index 37474242e1..26a3da9876 100644
--- a/fdbcli/ConfigureCommand.actor.cpp
+++ b/fdbcli/ConfigureCommand.actor.cpp
@@ -44,20 +44,20 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 	if (tokens.size() < 2)
 		result = ConfigurationResult::NO_OPTIONS_PROVIDED;
 	else {
-		if (tokens[startToken] == LiteralStringRef("FORCE")) {
+		if (tokens[startToken] == "FORCE"_sr) {
 			force = true;
 			startToken = 2;
 		}
 
 		state Optional<ConfigureAutoResult> conf;
-		if (tokens[startToken] == LiteralStringRef("auto")) {
+		if (tokens[startToken] == "auto"_sr) {
 			// get cluster status
 			state Reference<ITransaction> tr = db->createTransaction();
 			if (!tr->isValid()) {
 				StatusObject _s = wait(StatusClient::statusFetcher(localDb));
 				s = _s;
 			} else {
-				state ThreadFuture<Optional<Value>> statusValueF = tr->get(LiteralStringRef("\xff\xff/status/json"));
+				state ThreadFuture<Optional<Value>> statusValueF = tr->get("\xff\xff/status/json"_sr);
 				Optional<Value> statusValue = wait(safeThreadFutureToFuture(statusValueF));
 				if (!statusValue.present()) {
 					fprintf(stderr, "ERROR: Failed to get status json from the cluster\n");
@@ -166,7 +166,7 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 	case ConfigurationResult::CONFLICTING_OPTIONS:
 	case ConfigurationResult::UNKNOWN_OPTION:
 	case ConfigurationResult::INCOMPLETE_CONFIGURATION:
-		printUsage(LiteralStringRef("configure"));
+		printUsage("configure"_sr);
 		ret = false;
 		break;
 	case ConfigurationResult::INVALID_CONFIGURATION:
@@ -259,7 +259,6 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 		fprintf(stderr,
 		        "Type `configure perpetual_storage_wiggle=1' to enable the perpetual wiggle, or `configure "
 		        "storage_migration_type=gradual' to set the gradual migration type.\n");
-		ret = false;
 		break;
 	case ConfigurationResult::SUCCESS_WARN_ROCKSDB_EXPERIMENTAL:
 		printf("Configuration changed\n");
@@ -272,6 +271,14 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 		    stderr,
 		    "WARN: Sharded RocksDB storage engine type is still in experimental stage, not yet production tested.\n");
 		break;
+	case ConfigurationResult::DATABASE_IS_REGISTERED:
+		fprintf(stderr, "ERROR: A cluster cannot change its tenant mode while part of a metacluster.\n");
+		ret = false;
+		break;
+	case ConfigurationResult::ENCRYPTION_AT_REST_MODE_ALREADY_SET:
+		fprintf(stderr, "ERROR: A cluster cannot change its encryption_at_rest state after database creation.\n");
+		ret = false;
+		break;
 	default:
 		ASSERT(false);
 		ret = false;
@@ -305,6 +312,7 @@ void configureGenerator(const char* text,
 		                   "storage_migration_type=",
 		                   "tenant_mode=",
 		                   "blob_granules_enabled=",
+		                   "encryption_at_rest_mode=",
 		                   nullptr };
 	arrayGenerator(text, line, opts, lc);
 }
@@ -317,7 +325,8 @@ CommandFactory configureFactory(
         "commit_proxies=<COMMIT_PROXIES>|grv_proxies=<GRV_PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*|"
         "count=<TSS_COUNT>|perpetual_storage_wiggle=<WIGGLE_SPEED>|perpetual_storage_wiggle_locality="
         "<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>|storage_migration_type={disabled|gradual|aggressive}"
-        "|tenant_mode={disabled|optional_experimental|required_experimental}|blob_granules_enabled={0|1}",
+        "|tenant_mode={disabled|optional_experimental|required_experimental}|blob_granules_enabled={0|1}"
+        "|encryption_at_rest_mode={disabled|aes_256_ctr}",
         "change the database configuration",
         "The `new' option, if present, initializes a new database with the given configuration rather than changing "
         "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be "
@@ -351,6 +360,9 @@ CommandFactory configureFactory(
         "tenant_mode=<disabled|optional_experimental|required_experimental>: Sets the tenant mode for the cluster. If "
         "optional, then transactions can be run with or without specifying tenants. If required, all data must be "
         "accessed using tenants.\n\n"
+        "encryption_at_rest_mode=<disabled|aes_256_ctr>: Sets the cluster encryption data at-rest support for the "
+        "database. The configuration can be updated ONLY at the time of database creation and once set can't be "
+        "updated for the lifetime of the database.\n\n"
 
         "See the FoundationDB Administration Guide for more information."),
     &configureGenerator);
diff --git a/fdbcli/ConsistencyCheckCommand.actor.cpp b/fdbcli/ConsistencyCheckCommand.actor.cpp
index 2e14e71fcc..1f225d1dfe 100644
--- a/fdbcli/ConsistencyCheckCommand.actor.cpp
+++ b/fdbcli/ConsistencyCheckCommand.actor.cpp
@@ -30,7 +30,7 @@
 
 namespace fdb_cli {
 
-const KeyRef consistencyCheckSpecialKey = LiteralStringRef("\xff\xff/management/consistency_check_suspended");
+const KeyRef consistencyCheckSpecialKey = "\xff\xff/management/consistency_check_suspended"_sr;
 
 ACTOR Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr,
                                                 std::vector<StringRef> tokens,
diff --git a/fdbcli/ConsistencyScanCommand.actor.cpp b/fdbcli/ConsistencyScanCommand.actor.cpp
new file mode 100644
index 0000000000..532e43119a
--- /dev/null
+++ b/fdbcli/ConsistencyScanCommand.actor.cpp
@@ -0,0 +1,122 @@
+/*
+ * ConsistencyScanCommand.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/IClientApi.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "fdbclient/ConsistencyScanInterface.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace fdb_cli {
+
+ACTOR Future<bool> consistencyScanCommandActor(Database db, std::vector<StringRef> tokens) {
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
+	// Here we do not proceed in a try-catch loop since the transaction is always supposed to succeed.
+	// If not, the outer loop catch block(fdbcli.actor.cpp) will handle the error and print out the error message
+	state int usageError = 0;
+	state ConsistencyScanInfo csInfo = ConsistencyScanInfo();
+	tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+
+	// Get the exisiting consistencyScanInfo object if present
+	state Optional<Value> consistencyScanInfo = wait(ConsistencyScanInfo::getInfo(tr));
+	wait(tr->commit());
+	if (consistencyScanInfo.present())
+		csInfo = ObjectReader::fromStringRef<ConsistencyScanInfo>(consistencyScanInfo.get(), IncludeVersion());
+	tr->reset();
+
+	if (tokens.size() == 1) {
+		printf("Consistency Scan Info: %s\n", csInfo.toString().c_str());
+	} else if ((tokens.size() == 2) && tokencmp(tokens[1], "off")) {
+		csInfo.consistency_scan_enabled = false;
+		wait(ConsistencyScanInfo::setInfo(tr, csInfo));
+		wait(tr->commit());
+	} else if ((tokencmp(tokens[1], "on") && tokens.size() > 2)) {
+		csInfo.consistency_scan_enabled = true;
+		state std::vector<StringRef>::iterator t;
+		for (t = tokens.begin() + 2; t != tokens.end(); ++t) {
+			if (tokencmp(t->toString(), "restart")) {
+				if (++t != tokens.end()) {
+					if (tokencmp(t->toString(), "0")) {
+						csInfo.restart = false;
+					} else if (tokencmp(t->toString(), "1")) {
+						csInfo.restart = true;
+					} else {
+						usageError = 1;
+					}
+				} else {
+					usageError = 1;
+				}
+			} else if (tokencmp(t->toString(), "maxRate")) {
+				if (++t != tokens.end()) {
+					char* end;
+					csInfo.max_rate = std::strtod(t->toString().data(), &end);
+					if (!std::isspace(*end) && (*end != '\0')) {
+						fprintf(stderr, "ERROR: %s failed to parse.\n", t->toString().c_str());
+						return false;
+					}
+				} else {
+					usageError = 1;
+				}
+			} else if (tokencmp(t->toString(), "targetInterval")) {
+				if (++t != tokens.end()) {
+					char* end;
+					csInfo.target_interval = std::strtod(t->toString().data(), &end);
+					if (!std::isspace(*end) && (*end != '\0')) {
+						fprintf(stderr, "ERROR: %s failed to parse.\n", t->toString().c_str());
+						return false;
+					}
+				} else {
+					usageError = 1;
+				}
+			} else {
+				usageError = 1;
+			}
+		}
+
+		if (!usageError) {
+			wait(ConsistencyScanInfo::setInfo(tr, csInfo));
+			wait(tr->commit());
+		}
+	} else {
+		usageError = 1;
+	}
+
+	if (usageError) {
+		printUsage(tokens[0]);
+		return false;
+	}
+	return true;
+}
+
+CommandFactory consistencyScanFactory(
+    "consistencyscan",
+    CommandHelp("consistencyscan <on|off> <restart 0|1> <maxRate val> <targetInterval val>",
+                "enables or disables consistency scan",
+                "Calling this command with `on' enables the consistency scan process to run the scan with given "
+                "arguments and `off' will halt the scan. "
+                "Calling this command with no arguments will display if consistency scan is currently enabled.\n"));
+
+} // namespace fdb_cli
\ No newline at end of file
diff --git a/fdbcli/CoordinatorsCommand.actor.cpp b/fdbcli/CoordinatorsCommand.actor.cpp
index b68d5ab3d3..4680c5393a 100644
--- a/fdbcli/CoordinatorsCommand.actor.cpp
+++ b/fdbcli/CoordinatorsCommand.actor.cpp
@@ -64,17 +64,26 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 	state int notEnoughMachineResults = 0;
 	state StringRef new_cluster_description;
 	state std::string auto_coordinators_str;
-	StringRef nameTokenBegin = LiteralStringRef("description=");
+	state bool disableConfigDB = false;
+	StringRef nameTokenBegin = "description="_sr;
+	StringRef noConfigDB = "--no-config-db"_sr;
 	for (auto tok = tokens.begin() + 1; tok != tokens.end(); ++tok) {
-		if (tok->startsWith(nameTokenBegin)) {
+		if (tok->startsWith(nameTokenBegin) && new_cluster_description.empty()) {
 			new_cluster_description = tok->substr(nameTokenBegin.size());
+			auto next = tok - 1;
 			std::copy(tok + 1, tokens.end(), tok);
 			tokens.resize(tokens.size() - 1);
-			break;
+			tok = next;
+		} else if (tok->startsWith(noConfigDB)) {
+			disableConfigDB = true;
+			auto next = tok - 1;
+			std::copy(tok + 1, tokens.end(), tok);
+			tokens.resize(tokens.size() - 1);
+			tok = next;
 		}
 	}
 
-	state bool automatic = tokens.size() == 2 && tokens[1] == LiteralStringRef("auto");
+	state bool automatic = tokens.size() == 2 && tokens[1] == "auto"_sr;
 	state Reference<ITransaction> tr = db->createTransaction();
 	loop {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -83,6 +92,10 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 			if (new_cluster_description.size()) {
 				tr->set(fdb_cli::clusterDescriptionSpecialKey, new_cluster_description);
 			}
+			if (disableConfigDB) {
+				// All that matters is the key is set.
+				tr->set(fdb_cli::configDBSpecialKey, ""_sr);
+			}
 			// if auto change, read the special key to retrieve the recommended config
 			if (automatic) {
 				// if previous read failed, retry, otherwise, use the same recommened config
@@ -173,9 +186,10 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 
 namespace fdb_cli {
 
-const KeyRef clusterDescriptionSpecialKey = LiteralStringRef("\xff\xff/configuration/coordinators/cluster_description");
-const KeyRef coordinatorsAutoSpecialKey = LiteralStringRef("\xff\xff/management/auto_coordinators");
-const KeyRef coordinatorsProcessSpecialKey = LiteralStringRef("\xff\xff/configuration/coordinators/processes");
+const KeyRef clusterDescriptionSpecialKey = "\xff\xff/configuration/coordinators/cluster_description"_sr;
+const KeyRef configDBSpecialKey = "\xff\xff/configuration/coordinators/config_db"_sr;
+const KeyRef coordinatorsAutoSpecialKey = "\xff\xff/management/auto_coordinators"_sr;
+const KeyRef coordinatorsProcessSpecialKey = "\xff\xff/configuration/coordinators/processes"_sr;
 
 ACTOR Future<bool> coordinatorsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() < 2) {
diff --git a/fdbcli/DataDistributionCommand.actor.cpp b/fdbcli/DataDistributionCommand.actor.cpp
index 7000bdf5c7..8b7690b009 100644
--- a/fdbcli/DataDistributionCommand.actor.cpp
+++ b/fdbcli/DataDistributionCommand.actor.cpp
@@ -108,8 +108,8 @@ Future<Void> setDDIgnoreRebalanceOff(Reference<IDatabase> db, uint8_t DDIgnoreOp
 
 namespace fdb_cli {
 
-const KeyRef ddModeSpecialKey = LiteralStringRef("\xff\xff/management/data_distribution/mode");
-const KeyRef ddIgnoreRebalanceSpecialKey = LiteralStringRef("\xff\xff/management/data_distribution/rebalance_ignored");
+const KeyRef ddModeSpecialKey = "\xff\xff/management/data_distribution/mode"_sr;
+const KeyRef ddIgnoreRebalanceSpecialKey = "\xff\xff/management/data_distribution/rebalance_ignored"_sr;
 constexpr auto usage =
     "Usage: datadistribution <on|off|disable <ssfailure|rebalance|rebalance_disk|rebalance_read>|enable "
     "<ssfailure|rebalance|rebalance_disk|rebalance_read>>\n";
@@ -127,7 +127,7 @@ ACTOR Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::ve
 			printf("Data distribution is turned off.\n");
 		} else if (tokencmp(tokens[1], "disable")) {
 			if (tokencmp(tokens[2], "ssfailure")) {
-				wait(success((setHealthyZone(db, LiteralStringRef("IgnoreSSFailures"), 0))));
+				wait(success((setHealthyZone(db, "IgnoreSSFailures"_sr, 0))));
 				printf("Data distribution is disabled for storage server failures.\n");
 			} else if (tokencmp(tokens[2], "rebalance")) {
 				wait(setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_DISK | DDIgnore::REBALANCE_READ));
diff --git a/fdbcli/ExcludeCommand.actor.cpp b/fdbcli/ExcludeCommand.actor.cpp
index db67bd8a6e..7c8b7217e0 100644
--- a/fdbcli/ExcludeCommand.actor.cpp
+++ b/fdbcli/ExcludeCommand.actor.cpp
@@ -227,22 +227,19 @@ ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::vector<Add
 
 namespace fdb_cli {
 
-const KeyRangeRef excludedServersSpecialKeyRange(LiteralStringRef("\xff\xff/management/excluded/"),
-                                                 LiteralStringRef("\xff\xff/management/excluded0"));
-const KeyRangeRef failedServersSpecialKeyRange(LiteralStringRef("\xff\xff/management/failed/"),
-                                               LiteralStringRef("\xff\xff/management/failed0"));
-const KeyRangeRef excludedLocalitySpecialKeyRange(LiteralStringRef("\xff\xff/management/excluded_locality/"),
-                                                  LiteralStringRef("\xff\xff/management/excluded_locality0"));
-const KeyRangeRef failedLocalitySpecialKeyRange(LiteralStringRef("\xff\xff/management/failed_locality/"),
-                                                LiteralStringRef("\xff\xff/management/failed_locality0"));
-const KeyRef excludedForceOptionSpecialKey = LiteralStringRef("\xff\xff/management/options/excluded/force");
-const KeyRef failedForceOptionSpecialKey = LiteralStringRef("\xff\xff/management/options/failed/force");
-const KeyRef excludedLocalityForceOptionSpecialKey =
-    LiteralStringRef("\xff\xff/management/options/excluded_locality/force");
-const KeyRef failedLocalityForceOptionSpecialKey =
-    LiteralStringRef("\xff\xff/management/options/failed_locality/force");
-const KeyRangeRef exclusionInProgressSpecialKeyRange(LiteralStringRef("\xff\xff/management/in_progress_exclusion/"),
-                                                     LiteralStringRef("\xff\xff/management/in_progress_exclusion0"));
+const KeyRangeRef excludedServersSpecialKeyRange("\xff\xff/management/excluded/"_sr,
+                                                 "\xff\xff/management/excluded0"_sr);
+const KeyRangeRef failedServersSpecialKeyRange("\xff\xff/management/failed/"_sr, "\xff\xff/management/failed0"_sr);
+const KeyRangeRef excludedLocalitySpecialKeyRange("\xff\xff/management/excluded_locality/"_sr,
+                                                  "\xff\xff/management/excluded_locality0"_sr);
+const KeyRangeRef failedLocalitySpecialKeyRange("\xff\xff/management/failed_locality/"_sr,
+                                                "\xff\xff/management/failed_locality0"_sr);
+const KeyRef excludedForceOptionSpecialKey = "\xff\xff/management/options/excluded/force"_sr;
+const KeyRef failedForceOptionSpecialKey = "\xff\xff/management/options/failed/force"_sr;
+const KeyRef excludedLocalityForceOptionSpecialKey = "\xff\xff/management/options/excluded_locality/force"_sr;
+const KeyRef failedLocalityForceOptionSpecialKey = "\xff\xff/management/options/failed_locality/force"_sr;
+const KeyRangeRef exclusionInProgressSpecialKeyRange("\xff\xff/management/in_progress_exclusion/"_sr,
+                                                     "\xff\xff/management/in_progress_exclusion0"_sr);
 
 ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, Future<Void> warn) {
 	if (tokens.size() <= 1) {
@@ -281,11 +278,11 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 		if (!result)
 			return false;
 		for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) {
-			if (*t == LiteralStringRef("FORCE")) {
+			if (*t == "FORCE"_sr) {
 				force = true;
-			} else if (*t == LiteralStringRef("no_wait")) {
+			} else if (*t == "no_wait"_sr) {
 				waitForAllExcluded = false;
-			} else if (*t == LiteralStringRef("failed")) {
+			} else if (*t == "failed"_sr) {
 				markFailed = true;
 			} else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) &&
 			           t->toString().find(':') != std::string::npos) {
diff --git a/fdbcli/ExpensiveDataCheckCommand.actor.cpp b/fdbcli/ExpensiveDataCheckCommand.actor.cpp
index e9d5c5b989..3be572d3d1 100644
--- a/fdbcli/ExpensiveDataCheckCommand.actor.cpp
+++ b/fdbcli/ExpensiveDataCheckCommand.actor.cpp
@@ -46,7 +46,7 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
 	if (tokens.size() == 1) {
 		// initialize worker interfaces
 		address_interface->clear();
-		wait(getWorkerInterfaces(tr, address_interface));
+		wait(getWorkerInterfaces(tr, address_interface, true));
 	}
 	if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
 		if (address_interface->size() == 0) {
diff --git a/fdbcli/FileConfigureCommand.actor.cpp b/fdbcli/FileConfigureCommand.actor.cpp
index e35114c429..8cce2ec543 100644
--- a/fdbcli/FileConfigureCommand.actor.cpp
+++ b/fdbcli/FileConfigureCommand.actor.cpp
@@ -78,7 +78,7 @@ ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
 			    name + "=" +
 			    json_spirit::write_string(json_spirit::mValue(value.get_array()), json_spirit::Output_options::none);
 		} else {
-			printUsage(LiteralStringRef("fileconfigure"));
+			printUsage("fileconfigure"_sr);
 			return false;
 		}
 	}
diff --git a/fdbcli/IncludeCommand.actor.cpp b/fdbcli/IncludeCommand.actor.cpp
index a463772960..be55ac8476 100644
--- a/fdbcli/IncludeCommand.actor.cpp
+++ b/fdbcli/IncludeCommand.actor.cpp
@@ -92,8 +92,7 @@ ACTOR Future<Void> includeServers(Reference<IDatabase> db, std::vector<AddressEx
 					// This is why we now make two clears: first only of the ip
 					// address, the second will delete all ports.
 					if (s.isWholeMachine())
-						tr->clear(KeyRangeRef(addr.withSuffix(LiteralStringRef(":")),
-						                      addr.withSuffix(LiteralStringRef(";"))));
+						tr->clear(KeyRangeRef(addr.withSuffix(":"_sr), addr.withSuffix(";"_sr)));
 				}
 			}
 			wait(safeThreadFutureToFuture(tr->commit()));
@@ -112,9 +111,9 @@ ACTOR Future<bool> include(Reference<IDatabase> db, std::vector<StringRef> token
 	state bool failed = false;
 	state bool all = false;
 	for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) {
-		if (*t == LiteralStringRef("all")) {
+		if (*t == "all"_sr) {
 			all = true;
-		} else if (*t == LiteralStringRef("failed")) {
+		} else if (*t == "failed"_sr) {
 			failed = true;
 		} else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) && t->toString().find(':') != std::string::npos) {
 			// if the token starts with 'locality_' prefix.
diff --git a/fdbcli/KillCommand.actor.cpp b/fdbcli/KillCommand.actor.cpp
index 391bdb3064..c8fa75bb1c 100644
--- a/fdbcli/KillCommand.actor.cpp
+++ b/fdbcli/KillCommand.actor.cpp
@@ -29,6 +29,7 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
+#include "flow/CodeProbe.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
@@ -43,7 +44,7 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
 	if (tokens.size() == 1) {
 		// initialize worker interfaces
 		address_interface->clear();
-		wait(getWorkerInterfaces(tr, address_interface));
+		wait(getWorkerInterfaces(tr, address_interface, true));
 	}
 	if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
 		if (address_interface->size() == 0) {
diff --git a/fdbcli/LockCommand.actor.cpp b/fdbcli/LockCommand.actor.cpp
index 1ed988ee34..a2ac2c05cd 100644
--- a/fdbcli/LockCommand.actor.cpp
+++ b/fdbcli/LockCommand.actor.cpp
@@ -59,7 +59,7 @@ ACTOR Future<bool> lockDatabase(Reference<IDatabase> db, UID id) {
 
 namespace fdb_cli {
 
-const KeyRef lockSpecialKey = LiteralStringRef("\xff\xff/management/db_locked");
+const KeyRef lockSpecialKey = "\xff\xff/management/db_locked"_sr;
 
 ACTOR Future<bool> lockCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() != 1) {
diff --git a/fdbcli/MaintenanceCommand.actor.cpp b/fdbcli/MaintenanceCommand.actor.cpp
index 487490e09f..b6dd8cc139 100644
--- a/fdbcli/MaintenanceCommand.actor.cpp
+++ b/fdbcli/MaintenanceCommand.actor.cpp
@@ -69,10 +69,10 @@ ACTOR Future<Void> printHealthyZone(Reference<IDatabase> db) {
 
 namespace fdb_cli {
 
-const KeyRangeRef maintenanceSpecialKeyRange = KeyRangeRef(LiteralStringRef("\xff\xff/management/maintenance/"),
-                                                           LiteralStringRef("\xff\xff/management/maintenance0"));
+const KeyRangeRef maintenanceSpecialKeyRange =
+    KeyRangeRef("\xff\xff/management/maintenance/"_sr, "\xff\xff/management/maintenance0"_sr);
 // The special key, if present, means data distribution is disabled for storage failures;
-const KeyRef ignoreSSFailureSpecialKey = LiteralStringRef("\xff\xff/management/maintenance/IgnoreSSFailures");
+const KeyRef ignoreSSFailureSpecialKey = "\xff\xff/management/maintenance/IgnoreSSFailures"_sr;
 
 // add a zone to maintenance and specify the maintenance duration
 ACTOR Future<bool> setHealthyZone(Reference<IDatabase> db, StringRef zoneId, double seconds, bool printWarning) {
diff --git a/fdbcli/MetaclusterCommands.actor.cpp b/fdbcli/MetaclusterCommands.actor.cpp
new file mode 100644
index 0000000000..da7c0f79fd
--- /dev/null
+++ b/fdbcli/MetaclusterCommands.actor.cpp
@@ -0,0 +1,432 @@
+/*
+ * MetaclusterCommands.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/Knobs.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
+#include "fdbclient/Schemas.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace fdb_cli {
+
+Optional<std::pair<Optional<ClusterConnectionString>, Optional<DataClusterEntry>>>
+parseClusterConfiguration(std::vector<StringRef> const& tokens, DataClusterEntry const& defaults, int startIndex) {
+	Optional<DataClusterEntry> entry;
+	Optional<ClusterConnectionString> connectionString;
+
+	std::set<std::string> usedParams;
+	for (int tokenNum = startIndex; tokenNum < tokens.size(); ++tokenNum) {
+		StringRef token = tokens[tokenNum];
+		bool foundEquals;
+		StringRef param = token.eat("=", &foundEquals);
+		if (!foundEquals) {
+			fmt::print(stderr,
+			           "ERROR: invalid configuration string `{}'. String must specify a value using `='.\n",
+			           param.toString().c_str());
+			return {};
+		}
+		std::string value = token.toString();
+		if (!usedParams.insert(value).second) {
+			fmt::print(
+			    stderr, "ERROR: configuration parameter `{}' specified more than once.\n", param.toString().c_str());
+			return {};
+		}
+		if (tokencmp(param, "max_tenant_groups")) {
+			entry = defaults;
+
+			int n;
+			if (sscanf(value.c_str(), "%d%n", &entry.get().capacity.numTenantGroups, &n) != 1 || n != value.size() ||
+			    entry.get().capacity.numTenantGroups < 0) {
+				fmt::print(stderr, "ERROR: invalid number of tenant groups `{}'.\n", value.c_str());
+				return {};
+			}
+		} else if (tokencmp(param, "connection_string")) {
+			connectionString = ClusterConnectionString(value);
+		} else {
+			fmt::print(stderr, "ERROR: unrecognized configuration parameter `{}'.\n", param.toString().c_str());
+			return {};
+		}
+	}
+
+	return std::make_pair(connectionString, entry);
+}
+
+void printMetaclusterConfigureOptionsUsage() {
+	fmt::print("max_tenant_groups sets the maximum number of tenant groups that can be assigned\n"
+	           "to the named data cluster.\n");
+	fmt::print("connection_string sets the connection string for the named data cluster.\n");
+}
+
+// metacluster create command
+ACTOR Future<bool> metaclusterCreateCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() != 3) {
+		fmt::print("Usage: metacluster create_experimental <NAME>\n\n");
+		fmt::print("Configures the cluster to be a management cluster in a metacluster.\n");
+		fmt::print("NAME is an identifier used to distinguish this metacluster from other metaclusters.\n");
+		return false;
+	}
+
+	Optional<std::string> errorStr = wait(MetaclusterAPI::createMetacluster(db, tokens[2]));
+	if (errorStr.present()) {
+		fmt::print("ERROR: {}.\n", errorStr.get());
+	} else {
+		fmt::print("The cluster has been configured as a metacluster.\n");
+	}
+	return true;
+}
+
+// metacluster decommission command
+ACTOR Future<bool> metaclusterDecommissionCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() != 2) {
+		fmt::print("Usage: metacluster decommission\n\n");
+		fmt::print("Converts the current cluster from a metacluster management cluster back into an\n");
+		fmt::print("ordinary cluster. It must be called on a cluster with no registered data clusters.\n");
+		return false;
+	}
+
+	wait(MetaclusterAPI::decommissionMetacluster(db));
+
+	fmt::print("The cluster is no longer a metacluster.\n");
+	return true;
+}
+
+// metacluster register command
+ACTOR Future<bool> metaclusterRegisterCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() < 4) {
+		fmt::print("Usage: metacluster register <NAME> connection_string=<CONNECTION_STRING>\n"
+		           "[max_tenant_groups=<NUM_GROUPS>]\n\n");
+		fmt::print("Adds a data cluster to a metacluster.\n");
+		fmt::print("NAME is used to identify the cluster in future commands.\n");
+		printMetaclusterConfigureOptionsUsage();
+		return false;
+	}
+
+	DataClusterEntry defaultEntry;
+	auto config = parseClusterConfiguration(tokens, defaultEntry, 3);
+	if (!config.present()) {
+		return false;
+	} else if (!config.get().first.present()) {
+		fmt::print(stderr, "ERROR: connection_string must be configured when registering a cluster.\n");
+		return false;
+	}
+
+	wait(MetaclusterAPI::registerCluster(
+	    db, tokens[2], config.get().first.get(), config.get().second.orDefault(defaultEntry)));
+
+	fmt::print("The cluster `{}' has been added\n", printable(tokens[2]).c_str());
+	return true;
+}
+
+// metacluster remove command
+ACTOR Future<bool> metaclusterRemoveCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() < 3 || tokens.size() > 4 || (tokens.size() == 4 && tokens[2] != "FORCE"_sr)) {
+		fmt::print("Usage: metacluster remove [FORCE] <NAME> \n\n");
+		fmt::print("Removes the specified data cluster from a metacluster.\n");
+		fmt::print("If FORCE is specified, then the cluster will be detached even if it has\n"
+		           "tenants assigned to it.\n");
+		return false;
+	}
+
+	state ClusterNameRef clusterName = tokens[tokens.size() - 1];
+	wait(MetaclusterAPI::removeCluster(db, clusterName, tokens.size() == 4));
+
+	fmt::print("The cluster `{}' has been removed\n", printable(clusterName).c_str());
+	return true;
+}
+
+// metacluster configure command
+ACTOR Future<bool> metaclusterConfigureCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() < 4) {
+		fmt::print("Usage: metacluster configure <NAME> <max_tenant_groups=<NUM_GROUPS>|\n"
+		           "connection_string=<CONNECTION_STRING>> ...\n\n");
+		fmt::print("Updates the configuration of the metacluster.\n");
+		printMetaclusterConfigureOptionsUsage();
+		return false;
+	}
+
+	state Reference<ITransaction> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			Optional<DataClusterMetadata> metadata = wait(MetaclusterAPI::tryGetClusterTransaction(tr, tokens[2]));
+			if (!metadata.present()) {
+				throw cluster_not_found();
+			}
+
+			auto config = parseClusterConfiguration(tokens, metadata.get().entry, 3);
+			if (!config.present()) {
+				return false;
+			}
+
+			MetaclusterAPI::updateClusterMetadata(
+			    tr, tokens[2], metadata.get(), config.get().first, config.get().second);
+
+			wait(safeThreadFutureToFuture(tr->commit()));
+			break;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+
+	return true;
+}
+
+// metacluster list command
+ACTOR Future<bool> metaclusterListCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() > 5) {
+		fmt::print("Usage: metacluster list [BEGIN] [END] [LIMIT]\n\n");
+		fmt::print("Lists the data clusters in a metacluster.\n");
+		fmt::print("Only cluster names in the range BEGIN - END will be printed.\n");
+		fmt::print("An optional LIMIT can be specified to limit the number of results (default 100).\n");
+		return false;
+	}
+
+	state ClusterNameRef begin = tokens.size() > 2 ? tokens[2] : ""_sr;
+	state ClusterNameRef end = tokens.size() > 3 ? tokens[3] : "\xff"_sr;
+	int limit = 100;
+
+	if (tokens.size() > 4) {
+		int n = 0;
+		if (sscanf(tokens[3].toString().c_str(), "%d%n", &limit, &n) != 1 || n != tokens[3].size() || limit < 0) {
+			fmt::print(stderr, "ERROR: invalid limit {}\n", tokens[3].toString().c_str());
+			return false;
+		}
+	}
+
+	std::map<ClusterName, DataClusterMetadata> clusters = wait(MetaclusterAPI::listClusters(db, begin, end, limit));
+	if (clusters.empty()) {
+		if (tokens.size() == 2) {
+			fmt::print("The metacluster has no registered data clusters\n");
+		} else {
+			fmt::print("The metacluster has no registered data clusters in the specified range\n");
+		}
+	}
+
+	int index = 0;
+	for (auto cluster : clusters) {
+		fmt::print("  {}. {}\n", ++index, printable(cluster.first).c_str());
+	}
+
+	return true;
+}
+
+// metacluster get command
+ACTOR Future<bool> metaclusterGetCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() > 4 || (tokens.size() == 4 && tokens[3] != "JSON"_sr)) {
+		fmt::print("Usage: metacluster get <NAME> [JSON]\n\n");
+		fmt::print("Prints metadata associated with the given data cluster.\n");
+		fmt::print("If JSON is specified, then the output will be in JSON format.\n");
+		return false;
+	}
+
+	state bool useJson = tokens.size() == 4;
+
+	try {
+		DataClusterMetadata metadata = wait(MetaclusterAPI::getCluster(db, tokens[2]));
+
+		if (useJson) {
+			json_spirit::mObject obj;
+			obj["type"] = "success";
+			obj["cluster"] = metadata.toJson();
+			fmt::print("{}\n", json_spirit::write_string(json_spirit::mValue(obj), json_spirit::pretty_print).c_str());
+		} else {
+			fmt::print("  connection string: {}\n", metadata.connectionString.toString().c_str());
+			fmt::print("  cluster state: {}\n", DataClusterEntry::clusterStateToString(metadata.entry.clusterState));
+			fmt::print("  tenant group capacity: {}\n", metadata.entry.capacity.numTenantGroups);
+			fmt::print("  allocated tenant groups: {}\n", metadata.entry.allocated.numTenantGroups);
+		}
+	} catch (Error& e) {
+		if (useJson) {
+			json_spirit::mObject obj;
+			obj["type"] = "error";
+			obj["error"] = e.what();
+			fmt::print("{}\n", json_spirit::write_string(json_spirit::mValue(obj), json_spirit::pretty_print).c_str());
+			return false;
+		} else {
+			throw;
+		}
+	}
+
+	return true;
+}
+
+// metacluster status command
+ACTOR Future<bool> metaclusterStatusCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() < 2 || tokens.size() > 3) {
+		fmt::print("Usage: metacluster status [JSON]\n\n");
+		fmt::print("Prints metacluster metadata.\n");
+		fmt::print("If JSON is specified, then the output will be in JSON format.\n");
+		return false;
+	}
+
+	state bool useJson = tokens.size() == 3;
+
+	try {
+		std::map<ClusterName, DataClusterMetadata> clusters =
+		    wait(MetaclusterAPI::listClusters(db, ""_sr, "\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS));
+
+		ClusterUsage totalCapacity;
+		ClusterUsage totalAllocated;
+		for (auto cluster : clusters) {
+			totalCapacity.numTenantGroups +=
+			    std::max(cluster.second.entry.capacity.numTenantGroups, cluster.second.entry.allocated.numTenantGroups);
+			totalAllocated.numTenantGroups += cluster.second.entry.allocated.numTenantGroups;
+		}
+
+		if (useJson) {
+			json_spirit::mObject obj;
+			obj["type"] = "success";
+
+			json_spirit::mObject metaclusterObj;
+			metaclusterObj["data_clusters"] = (int)clusters.size();
+			metaclusterObj["capacity"] = totalCapacity.toJson();
+			metaclusterObj["allocated"] = totalAllocated.toJson();
+
+			obj["metacluster"] = metaclusterObj;
+			fmt::print("{}\n", json_spirit::write_string(json_spirit::mValue(obj), json_spirit::pretty_print).c_str());
+		} else {
+			fmt::print("  number of data clusters: {}\n", clusters.size());
+			fmt::print("  tenant group capacity: {}\n", totalCapacity.numTenantGroups);
+			fmt::print("  allocated tenant groups: {}\n", totalAllocated.numTenantGroups);
+		}
+
+		return true;
+	} catch (Error& e) {
+		if (useJson) {
+			json_spirit::mObject obj;
+			obj["type"] = "error";
+			obj["error"] = e.what();
+			fmt::print("{}\n", json_spirit::write_string(json_spirit::mValue(obj), json_spirit::pretty_print).c_str());
+			return false;
+		} else {
+			throw;
+		}
+	}
+}
+
+// metacluster command
+Future<bool> metaclusterCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() == 1) {
+		printUsage(tokens[0]);
+		return true;
+	} else if (tokencmp(tokens[1], "create_experimental")) {
+		return metaclusterCreateCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "decommission")) {
+		return metaclusterDecommissionCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "register")) {
+		return metaclusterRegisterCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "remove")) {
+		return metaclusterRemoveCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "configure")) {
+		return metaclusterConfigureCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "list")) {
+		return metaclusterListCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "get")) {
+		return metaclusterGetCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "status")) {
+		return metaclusterStatusCommand(db, tokens);
+	} else {
+		printUsage(tokens[0]);
+		return true;
+	}
+}
+
+void metaclusterGenerator(const char* text,
+                          const char* line,
+                          std::vector<std::string>& lc,
+                          std::vector<StringRef> const& tokens) {
+	if (tokens.size() == 1) {
+		const char* opts[] = {
+			"create_experimental", "decommission", "register", "remove", "configure", "list", "get", "status", nullptr
+		};
+		arrayGenerator(text, line, opts, lc);
+	} else if (tokens.size() > 1 && (tokencmp(tokens[1], "register") || tokencmp(tokens[1], "configure"))) {
+		const char* opts[] = { "max_tenant_groups=", "connection_string=", nullptr };
+		arrayGenerator(text, line, opts, lc);
+	} else if ((tokens.size() == 2 && tokencmp(tokens[1], "status")) ||
+	           (tokens.size() == 3 && tokencmp(tokens[1], "get"))) {
+		const char* opts[] = { "JSON", nullptr };
+		arrayGenerator(text, line, opts, lc);
+	}
+}
+
+std::vector<const char*> metaclusterHintGenerator(std::vector<StringRef> const& tokens, bool inArgument) {
+	if (tokens.size() == 1) {
+		return { "<create_experimental|decommission|register|remove|configure|list|get|status>", "[ARGS]" };
+	} else if (tokencmp(tokens[1], "create_experimental")) {
+		return { "<NAME>" };
+	} else if (tokencmp(tokens[1], "decommission")) {
+		return {};
+	} else if (tokencmp(tokens[1], "register") && tokens.size() < 5) {
+		static std::vector<const char*> opts = { "<NAME>",
+			                                     "connection_string=<CONNECTION_STRING>",
+			                                     "[max_tenant_groups=<NUM_GROUPS>]" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else if (tokencmp(tokens[1], "remove") && tokens.size() < 4) {
+		static std::vector<const char*> opts = { "[FORCE]", "<NAME>" };
+		if (tokens.size() == 2) {
+			return opts;
+		} else if (tokens.size() == 3 && (inArgument || tokens[2].size() == "FORCE"_sr.size()) &&
+		           "FORCE"_sr.startsWith(tokens[2])) {
+			return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+		} else {
+			return {};
+		}
+	} else if (tokencmp(tokens[1], "configure")) {
+		static std::vector<const char*> opts = {
+			"<NAME>", "<max_tenant_groups=<NUM_GROUPS>|connection_string=<CONNECTION_STRING>>"
+		};
+		return std::vector<const char*>(opts.begin() + std::min<int>(1, tokens.size() - 2), opts.end());
+	} else if (tokencmp(tokens[1], "list") && tokens.size() < 5) {
+		static std::vector<const char*> opts = { "[BEGIN]", "[END]", "[LIMIT]" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else if (tokencmp(tokens[1], "get") && tokens.size() < 4) {
+		static std::vector<const char*> opts = { "<NAME>", "[JSON]" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else if (tokencmp(tokens[1], "status") && tokens.size() == 2) {
+		return { "[JSON]" };
+	} else {
+		return {};
+	}
+}
+
+CommandFactory metaclusterRegisterFactory(
+    "metacluster",
+    CommandHelp("metacluster <create_experimental|decommission|register|remove|configure|list|get|status> [ARGS]",
+                "view and manage a metacluster",
+                "`create_experimental' and `decommission' set up or deconfigure a metacluster.\n"
+                "`register' and `remove' add and remove data clusters from the metacluster.\n"
+                "`configure' updates the configuration of a data cluster.\n"
+                "`list' prints a list of data clusters in the metacluster.\n"
+                "`get' prints the metadata for a particular data cluster.\n"
+                "`status' prints metacluster metadata.\n"),
+    &metaclusterGenerator,
+    &metaclusterHintGenerator);
+
+} // namespace fdb_cli
diff --git a/fdbcli/ProfileCommand.actor.cpp b/fdbcli/ProfileCommand.actor.cpp
index d7f66ce080..52325d3de8 100644
--- a/fdbcli/ProfileCommand.actor.cpp
+++ b/fdbcli/ProfileCommand.actor.cpp
@@ -97,8 +97,8 @@ ACTOR Future<bool> profileCommandActor(Database db,
 				}
 			}
 
-			Tuple rate = Tuple().appendDouble(sampleRate);
-			Tuple size = Tuple().append(sizeLimit);
+			Tuple rate = Tuple::makeTuple(sampleRate);
+			Tuple size = Tuple::makeTuple(sizeLimit);
 			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack());
 			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack());
@@ -115,17 +115,13 @@ ACTOR Future<bool> profileCommandActor(Database db,
 			return false;
 		}
 		// Hold the reference to the standalone's memory
-		state ThreadFuture<RangeResult> kvsFuture =
-		    tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
-		                             LiteralStringRef("\xff\xff/worker_interfaces0")),
-		                 CLIENT_KNOBS->TOO_MANY);
+		state ThreadFuture<RangeResult> kvsFuture = tr->getRange(
+		    KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr), CLIENT_KNOBS->TOO_MANY);
 		RangeResult kvs = wait(safeThreadFutureToFuture(kvsFuture));
 		ASSERT(!kvs.more);
 		for (const auto& pair : kvs) {
-			auto ip_port =
-			    (pair.key.endsWith(LiteralStringRef(":tls")) ? pair.key.removeSuffix(LiteralStringRef(":tls"))
-			                                                 : pair.key)
-			        .removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
+			auto ip_port = (pair.key.endsWith(":tls"_sr) ? pair.key.removeSuffix(":tls"_sr) : pair.key)
+			                   .removePrefix("\xff\xff/worker_interfaces/"_sr);
 			printf("%s\n", printable(ip_port).c_str());
 		}
 	} else {
diff --git a/fdbcli/QuotaCommand.actor.cpp b/fdbcli/QuotaCommand.actor.cpp
index ba8546fa15..e6a86e9b51 100644
--- a/fdbcli/QuotaCommand.actor.cpp
+++ b/fdbcli/QuotaCommand.actor.cpp
@@ -25,8 +25,6 @@ namespace {
 
 enum class LimitType { RESERVED, TOTAL };
 
-enum class OpType { READ, WRITE };
-
 Optional<TransactionTag> parseTag(StringRef token) {
 	if (token.size() > CLIENT_KNOBS->MAX_TRANSACTION_TAG_LENGTH) {
 		return {};
@@ -36,25 +34,15 @@ Optional<TransactionTag> parseTag(StringRef token) {
 }
 
 Optional<LimitType> parseLimitType(StringRef token) {
-	if (token == "reserved"_sr) {
+	if (token == "reserved_throughput"_sr) {
 		return LimitType::RESERVED;
-	} else if (token == "total"_sr) {
+	} else if (token == "total_throughput"_sr) {
 		return LimitType::TOTAL;
 	} else {
 		return {};
 	}
 }
 
-Optional<OpType> parseOpType(StringRef token) {
-	if (token == "read"_sr) {
-		return OpType::READ;
-	} else if (token == "write"_sr) {
-		return OpType::WRITE;
-	} else {
-		return {};
-	}
-}
-
 Optional<double> parseLimitValue(StringRef token) {
 	try {
 		return std::stod(token.toString());
@@ -63,7 +51,7 @@ Optional<double> parseLimitValue(StringRef token) {
 	}
 }
 
-ACTOR Future<Void> getQuota(Reference<IDatabase> db, TransactionTag tag, LimitType limitType, OpType opType) {
+ACTOR Future<Void> getQuota(Reference<IDatabase> db, TransactionTag tag, LimitType limitType) {
 	state Reference<ITransaction> tr = db->createTransaction();
 	loop {
 		tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@@ -74,14 +62,10 @@ ACTOR Future<Void> getQuota(Reference<IDatabase> db, TransactionTag tag, LimitTy
 				fmt::print("<empty>\n");
 			} else {
 				auto const quota = ThrottleApi::TagQuotaValue::fromValue(v.get());
-				if (limitType == LimitType::TOTAL && opType == OpType::READ) {
-					fmt::print("{}\n", quota.totalReadQuota);
-				} else if (limitType == LimitType::TOTAL && opType == OpType::WRITE) {
-					fmt::print("{}\n", quota.totalWriteQuota);
-				} else if (limitType == LimitType::RESERVED && opType == OpType::READ) {
-					fmt::print("{}\n", quota.reservedReadQuota);
-				} else if (limitType == LimitType::RESERVED && opType == OpType::WRITE) {
-					fmt::print("{}\n", quota.reservedWriteQuota);
+				if (limitType == LimitType::TOTAL) {
+					fmt::print("{}\n", quota.totalQuota * CLIENT_KNOBS->READ_COST_BYTE_FACTOR);
+				} else if (limitType == LimitType::RESERVED) {
+					fmt::print("{}\n", quota.reservedQuota * CLIENT_KNOBS->READ_COST_BYTE_FACTOR);
 				}
 			}
 			return Void();
@@ -91,11 +75,7 @@ ACTOR Future<Void> getQuota(Reference<IDatabase> db, TransactionTag tag, LimitTy
 	}
 }
 
-ACTOR Future<Void> setQuota(Reference<IDatabase> db,
-                            TransactionTag tag,
-                            LimitType limitType,
-                            OpType opType,
-                            double value) {
+ACTOR Future<Void> setQuota(Reference<IDatabase> db, TransactionTag tag, LimitType limitType, double value) {
 	state Reference<ITransaction> tr = db->createTransaction();
 	state Key key = tag.withPrefix(tagQuotaPrefix);
 	loop {
@@ -107,21 +87,14 @@ ACTOR Future<Void> setQuota(Reference<IDatabase> db,
 			if (v.present()) {
 				quota = ThrottleApi::TagQuotaValue::fromValue(v.get());
 			}
-			if (limitType == LimitType::TOTAL && opType == OpType::READ) {
-				quota.totalReadQuota = value;
-			} else if (limitType == LimitType::TOTAL && opType == OpType::WRITE) {
-				quota.totalWriteQuota = value;
-			} else if (limitType == LimitType::RESERVED && opType == OpType::READ) {
-				quota.reservedReadQuota = value;
-			} else if (limitType == LimitType::RESERVED && opType == OpType::WRITE) {
-				quota.reservedWriteQuota = value;
+			// Internally, costs are stored in terms of pages, but in the API,
+			// costs are specified in terms of bytes
+			if (limitType == LimitType::TOTAL) {
+				quota.totalQuota = (value - 1) / CLIENT_KNOBS->READ_COST_BYTE_FACTOR + 1;
+			} else if (limitType == LimitType::RESERVED) {
+				quota.reservedQuota = (value - 1) / CLIENT_KNOBS->READ_COST_BYTE_FACTOR + 1;
 			}
-			ThrottleApi::setTagQuota(tr,
-			                         tag,
-			                         quota.reservedReadQuota,
-			                         quota.totalReadQuota,
-			                         quota.reservedWriteQuota,
-			                         quota.totalWriteQuota);
+			ThrottleApi::setTagQuota(tr, tag, quota.reservedQuota, quota.totalQuota);
 			wait(safeThreadFutureToFuture(tr->commit()));
 			return Void();
 		} catch (Error& e) {
@@ -130,8 +103,8 @@ ACTOR Future<Void> setQuota(Reference<IDatabase> db,
 	}
 }
 
-constexpr auto usage =
-    "quota [get <tag> [reserved|total] [read|write]|set <tag> [reserved|total] [read|write] <value>]";
+constexpr auto usage = "quota [get <tag> [reserved_throughput|total_throughput] | set <tag> "
+                       "[reserved_throughput|total_throughput] <value>]";
 
 bool exitFailure() {
 	fmt::print(usage);
@@ -149,25 +122,24 @@ ACTOR Future<bool> quotaCommandActor(Reference<IDatabase> db, std::vector<String
 	} else {
 		auto tag = parseTag(tokens[2]);
 		auto limitType = parseLimitType(tokens[3]);
-		auto opType = parseOpType(tokens[4]);
-		if (!tag.present() || !limitType.present() || !opType.present()) {
+		if (!tag.present() || !limitType.present()) {
 			return exitFailure();
 		}
 		if (tokens[1] == "get"_sr) {
+			if (tokens.size() != 4) {
+				return exitFailure();
+			}
+			wait(getQuota(db, tag.get(), limitType.get()));
+			return true;
+		} else if (tokens[1] == "set"_sr) {
 			if (tokens.size() != 5) {
 				return exitFailure();
 			}
-			wait(getQuota(db, tag.get(), limitType.get(), opType.get()));
-			return true;
-		} else if (tokens[1] == "set"_sr) {
-			if (tokens.size() != 6) {
-				return exitFailure();
-			}
-			auto const limitValue = parseLimitValue(tokens[5]);
+			auto const limitValue = parseLimitValue(tokens[4]);
 			if (!limitValue.present()) {
 				return exitFailure();
 			}
-			wait(setQuota(db, tag.get(), limitType.get(), opType.get(), limitValue.get()));
+			wait(setQuota(db, tag.get(), limitType.get(), limitValue.get()));
 			return true;
 		} else {
 			return exitFailure();
diff --git a/fdbcli/SetClassCommand.actor.cpp b/fdbcli/SetClassCommand.actor.cpp
index cd47e22f86..dc1f997e31 100644
--- a/fdbcli/SetClassCommand.actor.cpp
+++ b/fdbcli/SetClassCommand.actor.cpp
@@ -105,12 +105,10 @@ ACTOR Future<bool> setProcessClass(Reference<IDatabase> db, KeyRef network_addre
 namespace fdb_cli {
 
 const KeyRangeRef processClassSourceSpecialKeyRange =
-    KeyRangeRef(LiteralStringRef("\xff\xff/configuration/process/class_source/"),
-                LiteralStringRef("\xff\xff/configuration/process/class_source0"));
+    KeyRangeRef("\xff\xff/configuration/process/class_source/"_sr, "\xff\xff/configuration/process/class_source0"_sr);
 
 const KeyRangeRef processClassTypeSpecialKeyRange =
-    KeyRangeRef(LiteralStringRef("\xff\xff/configuration/process/class_type/"),
-                LiteralStringRef("\xff\xff/configuration/process/class_type0"));
+    KeyRangeRef("\xff\xff/configuration/process/class_type/"_sr, "\xff\xff/configuration/process/class_type0"_sr);
 
 ACTOR Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() != 3 && tokens.size() != 1) {
diff --git a/fdbcli/SnapshotCommand.actor.cpp b/fdbcli/SnapshotCommand.actor.cpp
index 5bc7302f0c..7606101bba 100644
--- a/fdbcli/SnapshotCommand.actor.cpp
+++ b/fdbcli/SnapshotCommand.actor.cpp
@@ -40,7 +40,7 @@ ACTOR Future<bool> snapshotCommandActor(Reference<IDatabase> db, std::vector<Str
 		for (int i = 1; i < tokens.size(); i++) {
 			snap_cmd = snap_cmd.withSuffix(tokens[i]);
 			if (i != tokens.size() - 1) {
-				snap_cmd = snap_cmd.withSuffix(LiteralStringRef(" "));
+				snap_cmd = snap_cmd.withSuffix(" "_sr);
 			}
 		}
 		try {
diff --git a/fdbcli/StatusCommand.actor.cpp b/fdbcli/StatusCommand.actor.cpp
index 47bbc6e4f8..9da5e768a3 100644
--- a/fdbcli/StatusCommand.actor.cpp
+++ b/fdbcli/StatusCommand.actor.cpp
@@ -411,6 +411,7 @@ void printStatus(StatusObjectReader statusObj,
 			outputString += "\nConfiguration:";
 			std::string outputStringCache = outputString;
 			bool isOldMemory = false;
+			bool blobGranuleEnabled{ false };
 			try {
 				// Configuration section
 				// FIXME: Should we suppress this if there are cluster messages implying that the database has no
@@ -433,7 +434,21 @@ void printStatus(StatusObjectReader statusObj,
 				} else
 					outputString += "unknown";
 
-				int intVal;
+				int intVal = 0;
+				if (statusObjConfig.get("blob_granules_enabled", intVal) && intVal) {
+					blobGranuleEnabled = true;
+				}
+				if (blobGranuleEnabled) {
+					outputString += "\n  Blob granules          - enabled";
+				}
+
+				outputString += "\n  Encryption at-rest    - ";
+				if (statusObjConfig.get("encryption_at_rest_mode", strVal)) {
+					outputString += strVal;
+				} else {
+					outputString += "disabled";
+				}
+
 				outputString += "\n  Coordinators           - ";
 				if (statusObjConfig.get("coordinators_count", intVal)) {
 					outputString += std::to_string(intVal);
@@ -1102,6 +1117,15 @@ void printStatus(StatusObjectReader statusObj,
 					outputString += "\n\nCoordination servers:";
 					outputString += getCoordinatorsInfoString(statusObj);
 				}
+
+				if (blobGranuleEnabled) {
+					outputString += "\n\nBlob Granules:";
+					StatusObjectReader statusObjBlobGranules = statusObjCluster["blob_granules"];
+					auto numWorkers = statusObjBlobGranules["number_of_blob_workers"].get_int();
+					outputString += "\n  Number of Workers      - " + format("%d", numWorkers);
+					auto numKeyRanges = statusObjBlobGranules["number_of_key_ranges"].get_int();
+					outputString += "\n  Number of Key Ranges   - " + format("%d", numKeyRanges);
+				}
 			}
 
 			// client time
@@ -1232,7 +1256,7 @@ ACTOR Future<bool> statusCommandActor(Reference<IDatabase> db,
 		StatusObject _s = wait(StatusClient::statusFetcher(localDb));
 		s = _s;
 	} else {
-		state ThreadFuture<Optional<Value>> statusValueF = tr->get(LiteralStringRef("\xff\xff/status/json"));
+		state ThreadFuture<Optional<Value>> statusValueF = tr->get("\xff\xff/status/json"_sr);
 		Optional<Value> statusValue = wait(safeThreadFutureToFuture(statusValueF));
 		if (!statusValue.present()) {
 			fprintf(stderr, "ERROR: Failed to get status json from the cluster\n");
diff --git a/fdbcli/SuspendCommand.actor.cpp b/fdbcli/SuspendCommand.actor.cpp
index 78a7fa1ed9..483ad4e445 100644
--- a/fdbcli/SuspendCommand.actor.cpp
+++ b/fdbcli/SuspendCommand.actor.cpp
@@ -43,7 +43,7 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
 	if (tokens.size() == 1) {
 		// initialize worker interfaces
 		address_interface->clear();
-		wait(getWorkerInterfaces(tr, address_interface));
+		wait(getWorkerInterfaces(tr, address_interface, true));
 		if (address_interface->size() == 0) {
 			printf("\nNo addresses can be suspended.\n");
 		} else if (address_interface->size() == 1) {
diff --git a/fdbcli/TenantCommands.actor.cpp b/fdbcli/TenantCommands.actor.cpp
index 49ea0c3443..8b1ce93e37 100644
--- a/fdbcli/TenantCommands.actor.cpp
+++ b/fdbcli/TenantCommands.actor.cpp
@@ -25,6 +25,7 @@
 #include "fdbclient/IClientApi.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
 #include "fdbclient/TenantManagement.actor.h"
 #include "fdbclient/Schemas.h"
 
@@ -35,166 +36,269 @@
 
 namespace fdb_cli {
 
-const KeyRangeRef tenantSpecialKeyRange(LiteralStringRef("\xff\xff/management/tenant/map/"),
-                                        LiteralStringRef("\xff\xff/management/tenant/map0"));
+const KeyRangeRef tenantMapSpecialKeyRange("\xff\xff/management/tenant/map/"_sr, "\xff\xff/management/tenant/map0"_sr);
+const KeyRangeRef tenantConfigSpecialKeyRange("\xff\xff/management/tenant/configure/"_sr,
+                                              "\xff\xff/management/tenant/configure0"_sr);
+const KeyRangeRef tenantRenameSpecialKeyRange("\xff\xff/management/tenant/rename/"_sr,
+                                              "\xff\xff/management/tenant/rename0"_sr);
 
-// createtenant command
-ACTOR Future<bool> createTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	if (tokens.size() != 2) {
-		printUsage(tokens[0]);
+Optional<std::map<Standalone<StringRef>, Optional<Value>>>
+parseTenantConfiguration(std::vector<StringRef> const& tokens, int startIndex, bool allowUnset) {
+	std::map<Standalone<StringRef>, Optional<Value>> configParams;
+	for (int tokenNum = startIndex; tokenNum < tokens.size(); ++tokenNum) {
+		Optional<Value> value;
+
+		StringRef token = tokens[tokenNum];
+		StringRef param;
+		if (allowUnset && token == "unset"_sr) {
+			if (++tokenNum == tokens.size()) {
+				fmt::print(stderr, "ERROR: `unset' specified without a configuration parameter.\n");
+				return {};
+			}
+			param = tokens[tokenNum];
+		} else {
+			bool foundEquals;
+			param = token.eat("=", &foundEquals);
+			if (!foundEquals) {
+				fmt::print(stderr,
+				           "ERROR: invalid configuration string `{}'. String must specify a value using `='.\n",
+				           param.toString().c_str());
+				return {};
+			}
+			value = token;
+		}
+
+		if (configParams.count(param)) {
+			fmt::print(
+			    stderr, "ERROR: configuration parameter `{}' specified more than once.\n", param.toString().c_str());
+			return {};
+		}
+
+		if (tokencmp(param, "tenant_group")) {
+			configParams[param] = value;
+		} else if (tokencmp(param, "assigned_cluster")) {
+			configParams[param] = value;
+		} else {
+			fmt::print(stderr, "ERROR: unrecognized configuration parameter `{}'.\n", param.toString().c_str());
+			return {};
+		}
+	}
+
+	return configParams;
+}
+
+Key makeConfigKey(TenantNameRef tenantName, StringRef configName) {
+	return tenantConfigSpecialKeyRange.begin.withSuffix(Tuple().append(tenantName).append(configName).pack());
+}
+
+void applyConfigurationToSpecialKeys(Reference<ITransaction> tr,
+                                     TenantNameRef tenantName,
+                                     std::map<Standalone<StringRef>, Optional<Value>> configuration) {
+	for (auto [configName, value] : configuration) {
+		if (configName == "assigned_cluster"_sr) {
+			fmt::print(stderr, "ERROR: assigned_cluster is only valid in metacluster configuration.\n");
+			throw invalid_tenant_configuration();
+		}
+		if (value.present()) {
+			tr->set(makeConfigKey(tenantName, configName), value.get());
+		} else {
+			tr->clear(makeConfigKey(tenantName, configName));
+		}
+	}
+}
+
+// tenant create command
+ACTOR Future<bool> tenantCreateCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() < 3 || tokens.size() > 5) {
+		fmt::print("Usage: tenant create <NAME> [tenant_group=<TENANT_GROUP>] [assigned_cluster=<CLUSTER_NAME>]\n\n");
+		fmt::print("Creates a new tenant in the cluster with the specified name.\n");
+		fmt::print("An optional group can be specified that will require this tenant\n");
+		fmt::print("to be placed on the same cluster as other tenants in the same group.\n");
+		fmt::print("An optional cluster name can be specified that this tenant will be placed in.\n");
 		return false;
 	}
 
-	state Key tenantNameKey = fdb_cli::tenantSpecialKeyRange.begin.withSuffix(tokens[1]);
+	state Key tenantNameKey = tenantMapSpecialKeyRange.begin.withSuffix(tokens[2]);
 	state Reference<ITransaction> tr = db->createTransaction();
 	state bool doneExistenceCheck = false;
 
+	state Optional<std::map<Standalone<StringRef>, Optional<Value>>> configuration =
+	    parseTenantConfiguration(tokens, 3, false);
+
+	if (!configuration.present()) {
+		return false;
+	}
+
 	loop {
-		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 		try {
-			if (!doneExistenceCheck) {
-				// Hold the reference to the standalone's memory
-				state ThreadFuture<Optional<Value>> existingTenantFuture = tr->get(tenantNameKey);
-				Optional<Value> existingTenant = wait(safeThreadFutureToFuture(existingTenantFuture));
-				if (existingTenant.present()) {
-					throw tenant_already_exists();
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
+			if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+				TenantMapEntry tenantEntry;
+				for (auto const& [name, value] : configuration.get()) {
+					tenantEntry.configure(name, value);
 				}
-				doneExistenceCheck = true;
+				wait(MetaclusterAPI::createTenant(db, tokens[2], tenantEntry));
+			} else {
+				if (!doneExistenceCheck) {
+					// Hold the reference to the standalone's memory
+					state ThreadFuture<Optional<Value>> existingTenantFuture = tr->get(tenantNameKey);
+					Optional<Value> existingTenant = wait(safeThreadFutureToFuture(existingTenantFuture));
+					if (existingTenant.present()) {
+						throw tenant_already_exists();
+					}
+					doneExistenceCheck = true;
+				}
+
+				tr->set(tenantNameKey, ValueRef());
+				applyConfigurationToSpecialKeys(tr, tokens[2], configuration.get());
+				wait(safeThreadFutureToFuture(tr->commit()));
 			}
 
-			tr->set(tenantNameKey, ValueRef());
-			wait(safeThreadFutureToFuture(tr->commit()));
 			break;
 		} catch (Error& e) {
 			state Error err(e);
 			if (e.code() == error_code_special_keys_api_failure) {
-				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
-				fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str());
+				std::string errorMsgStr = wait(getSpecialKeysFailureErrorMessage(tr));
+				fmt::print(stderr, "ERROR: {}\n", errorMsgStr.c_str());
 				return false;
 			}
 			wait(safeThreadFutureToFuture(tr->onError(err)));
 		}
 	}
 
-	printf("The tenant `%s' has been created\n", printable(tokens[1]).c_str());
+	fmt::print("The tenant `{}' has been created\n", printable(tokens[2]).c_str());
 	return true;
 }
 
-CommandFactory createTenantFactory("createtenant",
-                                   CommandHelp("createtenant <TENANT_NAME>",
-                                               "creates a new tenant in the cluster",
-                                               "Creates a new tenant in the cluster with the specified name."));
-
-// deletetenant command
-ACTOR Future<bool> deleteTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	if (tokens.size() != 2) {
-		printUsage(tokens[0]);
+// tenant delete command
+ACTOR Future<bool> tenantDeleteCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() != 3) {
+		fmt::print("Usage: tenant delete <NAME>\n\n");
+		fmt::print("Deletes a tenant from the cluster.\n");
+		fmt::print("Deletion will be allowed only if the specified tenant contains no data.\n");
 		return false;
 	}
 
-	state Key tenantNameKey = fdb_cli::tenantSpecialKeyRange.begin.withSuffix(tokens[1]);
+	state Key tenantNameKey = tenantMapSpecialKeyRange.begin.withSuffix(tokens[2]);
 	state Reference<ITransaction> tr = db->createTransaction();
 	state bool doneExistenceCheck = false;
 
 	loop {
-		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 		try {
-			if (!doneExistenceCheck) {
-				// Hold the reference to the standalone's memory
-				state ThreadFuture<Optional<Value>> existingTenantFuture = tr->get(tenantNameKey);
-				Optional<Value> existingTenant = wait(safeThreadFutureToFuture(existingTenantFuture));
-				if (!existingTenant.present()) {
-					throw tenant_not_found();
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
+			if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+				wait(MetaclusterAPI::deleteTenant(db, tokens[2]));
+			} else {
+				if (!doneExistenceCheck) {
+					// Hold the reference to the standalone's memory
+					state ThreadFuture<Optional<Value>> existingTenantFuture = tr->get(tenantNameKey);
+					Optional<Value> existingTenant = wait(safeThreadFutureToFuture(existingTenantFuture));
+					if (!existingTenant.present()) {
+						throw tenant_not_found();
+					}
+					doneExistenceCheck = true;
 				}
-				doneExistenceCheck = true;
+
+				tr->clear(tenantNameKey);
+				wait(safeThreadFutureToFuture(tr->commit()));
 			}
 
-			tr->clear(tenantNameKey);
-			wait(safeThreadFutureToFuture(tr->commit()));
 			break;
 		} catch (Error& e) {
 			state Error err(e);
 			if (e.code() == error_code_special_keys_api_failure) {
-				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
-				fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str());
+				std::string errorMsgStr = wait(getSpecialKeysFailureErrorMessage(tr));
+				fmt::print(stderr, "ERROR: {}\n", errorMsgStr.c_str());
 				return false;
 			}
 			wait(safeThreadFutureToFuture(tr->onError(err)));
 		}
 	}
 
-	printf("The tenant `%s' has been deleted\n", printable(tokens[1]).c_str());
+	fmt::print("The tenant `{}' has been deleted\n", printable(tokens[2]).c_str());
 	return true;
 }
 
-CommandFactory deleteTenantFactory(
-    "deletetenant",
-    CommandHelp(
-        "deletetenant <TENANT_NAME>",
-        "deletes a tenant from the cluster",
-        "Deletes a tenant from the cluster. Deletion will be allowed only if the specified tenant contains no data."));
-
-// listtenants command
-ACTOR Future<bool> listTenantsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	if (tokens.size() > 4) {
-		printUsage(tokens[0]);
+// tenant list command
+ACTOR Future<bool> tenantListCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() > 5) {
+		fmt::print("Usage: tenant list [BEGIN] [END] [LIMIT]\n\n");
+		fmt::print("Lists the tenants in a cluster.\n");
+		fmt::print("Only tenants in the range BEGIN - END will be printed.\n");
+		fmt::print("An optional LIMIT can be specified to limit the number of results (default 100).\n");
 		return false;
 	}
 
-	StringRef beginTenant = ""_sr;
-	StringRef endTenant = "\xff\xff"_sr;
+	state StringRef beginTenant = ""_sr;
+	state StringRef endTenant = "\xff\xff"_sr;
 	state int limit = 100;
 
-	if (tokens.size() >= 2) {
-		beginTenant = tokens[1];
-	}
 	if (tokens.size() >= 3) {
-		endTenant = tokens[2];
+		beginTenant = tokens[2];
+	}
+	if (tokens.size() >= 4) {
+		endTenant = tokens[3];
 		if (endTenant <= beginTenant) {
-			fprintf(stderr, "ERROR: end must be larger than begin");
+			fmt::print(stderr, "ERROR: end must be larger than begin");
 			return false;
 		}
 	}
-	if (tokens.size() == 4) {
+	if (tokens.size() == 5) {
 		int n = 0;
-		if (sscanf(tokens[3].toString().c_str(), "%d%n", &limit, &n) != 1 || n != tokens[3].size()) {
-			fprintf(stderr, "ERROR: invalid limit %s\n", tokens[3].toString().c_str());
+		if (sscanf(tokens[4].toString().c_str(), "%d%n", &limit, &n) != 1 || n != tokens[4].size() || limit <= 0) {
+			fmt::print(stderr, "ERROR: invalid limit `{}'\n", tokens[4].toString().c_str());
 			return false;
 		}
 	}
 
-	state Key beginTenantKey = fdb_cli::tenantSpecialKeyRange.begin.withSuffix(beginTenant);
-	state Key endTenantKey = fdb_cli::tenantSpecialKeyRange.begin.withSuffix(endTenant);
+	state Key beginTenantKey = tenantMapSpecialKeyRange.begin.withSuffix(beginTenant);
+	state Key endTenantKey = tenantMapSpecialKeyRange.begin.withSuffix(endTenant);
 	state Reference<ITransaction> tr = db->createTransaction();
 
 	loop {
 		try {
-			// Hold the reference to the standalone's memory
-			state ThreadFuture<RangeResult> kvsFuture =
-			    tr->getRange(firstGreaterOrEqual(beginTenantKey), firstGreaterOrEqual(endTenantKey), limit);
-			RangeResult tenants = wait(safeThreadFutureToFuture(kvsFuture));
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
+			state std::vector<TenantName> tenantNames;
+			if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+				std::vector<std::pair<TenantName, TenantMapEntry>> tenants =
+				    wait(MetaclusterAPI::listTenantsTransaction(tr, beginTenant, endTenant, limit));
+				for (auto tenant : tenants) {
+					tenantNames.push_back(tenant.first);
+				}
+			} else {
+				// Hold the reference to the standalone's memory
+				state ThreadFuture<RangeResult> kvsFuture =
+				    tr->getRange(firstGreaterOrEqual(beginTenantKey), firstGreaterOrEqual(endTenantKey), limit);
+				RangeResult tenants = wait(safeThreadFutureToFuture(kvsFuture));
+				for (auto tenant : tenants) {
+					tenantNames.push_back(tenant.key.removePrefix(tenantMapSpecialKeyRange.begin));
+				}
+			}
 
-			if (tenants.empty()) {
-				if (tokens.size() == 1) {
-					printf("The cluster has no tenants\n");
+			if (tenantNames.empty()) {
+				if (tokens.size() == 2) {
+					fmt::print("The cluster has no tenants\n");
 				} else {
-					printf("The cluster has no tenants in the specified range\n");
+					fmt::print("The cluster has no tenants in the specified range\n");
 				}
 			}
 
 			int index = 0;
-			for (auto tenant : tenants) {
-				printf("  %d. %s\n",
-				       ++index,
-				       printable(tenant.key.removePrefix(fdb_cli::tenantSpecialKeyRange.begin)).c_str());
+			for (auto tenantName : tenantNames) {
+				fmt::print("  {}. {}\n", ++index, printable(tenantName).c_str());
 			}
 
 			return true;
 		} catch (Error& e) {
 			state Error err(e);
 			if (e.code() == error_code_special_keys_api_failure) {
-				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
-				fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str());
+				std::string errorMsgStr = wait(getSpecialKeysFailureErrorMessage(tr));
+				fmt::print(stderr, "ERROR: {}\n", errorMsgStr.c_str());
 				return false;
 			}
 			wait(safeThreadFutureToFuture(tr->onError(err)));
@@ -202,59 +306,73 @@ ACTOR Future<bool> listTenantsCommandActor(Reference<IDatabase> db, std::vector<
 	}
 }
 
-CommandFactory listTenantsFactory(
-    "listtenants",
-    CommandHelp("listtenants [BEGIN] [END] [LIMIT]",
-                "print a list of tenants in the cluster",
-                "Print a list of tenants in the cluster. Only tenants in the range [BEGIN] - [END] will be printed. "
-                "The number of tenants to print can be specified using the [LIMIT] parameter, which defaults to 100."));
-
-// gettenant command
-ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, int apiVersion) {
-	if (tokens.size() < 2 || tokens.size() > 3 || (tokens.size() == 3 && tokens[2] != "JSON"_sr)) {
-		printUsage(tokens[0]);
+// tenant get command
+ACTOR Future<bool> tenantGetCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() < 3 || tokens.size() > 4 || (tokens.size() == 4 && tokens[3] != "JSON"_sr)) {
+		fmt::print("Usage: tenant get <NAME> [JSON]\n\n");
+		fmt::print("Prints metadata associated with the given tenant.\n");
+		fmt::print("If JSON is specified, then the output will be in JSON format.\n");
 		return false;
 	}
 
-	state bool useJson = tokens.size() == 3;
-	state Key tenantNameKey = fdb_cli::tenantSpecialKeyRange.begin.withSuffix(tokens[1]);
+	state bool useJson = tokens.size() == 4;
+	state Key tenantNameKey = tenantMapSpecialKeyRange.begin.withSuffix(tokens[2]);
 	state Reference<ITransaction> tr = db->createTransaction();
 
 	loop {
 		try {
-			// Hold the reference to the standalone's memory
-			state ThreadFuture<Optional<Value>> tenantFuture = tr->get(tenantNameKey);
-			Optional<Value> tenant = wait(safeThreadFutureToFuture(tenantFuture));
-			if (!tenant.present()) {
-				throw tenant_not_found();
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
+			state std::string tenantJson;
+			if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+				TenantMapEntry entry = wait(MetaclusterAPI::getTenantTransaction(tr, tokens[2]));
+				tenantJson = entry.toJson();
+			} else {
+				// Hold the reference to the standalone's memory
+				state ThreadFuture<Optional<Value>> tenantFuture = tr->get(tenantNameKey);
+				Optional<Value> tenant = wait(safeThreadFutureToFuture(tenantFuture));
+				if (!tenant.present()) {
+					throw tenant_not_found();
+				}
+				tenantJson = tenant.get().toString();
 			}
 
 			json_spirit::mValue jsonObject;
-			json_spirit::read_string(tenant.get().toString(), jsonObject);
+			json_spirit::read_string(tenantJson, jsonObject);
 
 			if (useJson) {
 				json_spirit::mObject resultObj;
 				resultObj["tenant"] = jsonObject;
 				resultObj["type"] = "success";
-				printf("%s\n",
-				       json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print).c_str());
+				fmt::print(
+				    "{}\n",
+				    json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print).c_str());
 			} else {
 				JSONDoc doc(jsonObject);
 
 				int64_t id;
 				std::string prefix;
+				std::string tenantState;
+				std::string tenantGroup;
+				std::string assignedCluster;
 
 				doc.get("id", id);
-				if (apiVersion >= 720) {
-					doc.get("prefix.printable", prefix);
-				} else {
-					doc.get("prefix", prefix);
+				doc.get("prefix.printable", prefix);
+
+				doc.get("tenant_state", tenantState);
+				bool hasTenantGroup = doc.tryGet("tenant_group.printable", tenantGroup);
+				bool hasAssignedCluster = doc.tryGet("assigned_cluster", assignedCluster);
+
+				fmt::print("  id: {}\n", id);
+				fmt::print("  prefix: {}\n", printable(prefix).c_str());
+				fmt::print("  tenant state: {}\n", printable(tenantState).c_str());
+				if (hasTenantGroup) {
+					fmt::print("  tenant group: {}\n", tenantGroup.c_str());
+				}
+				if (hasAssignedCluster) {
+					fmt::print("  assigned cluster: {}\n", printable(assignedCluster).c_str());
 				}
-
-				printf("  id: %" PRId64 "\n", id);
-				printf("  prefix: %s\n", prefix.c_str());
 			}
-
 			return true;
 		} catch (Error& e) {
 			try {
@@ -274,11 +392,11 @@ ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<St
 					json_spirit::mObject resultObj;
 					resultObj["type"] = "error";
 					resultObj["error"] = errorStr;
-					printf(
-					    "%s\n",
+					fmt::print(
+					    "{}\n",
 					    json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print).c_str());
 				} else {
-					fprintf(stderr, "ERROR: %s\n", errorStr.c_str());
+					fmt::print(stderr, "ERROR: {}\n", errorStr.c_str());
 				}
 
 				return false;
@@ -287,28 +405,252 @@ ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<St
 	}
 }
 
-CommandFactory getTenantFactory(
-    "gettenant",
-    CommandHelp("gettenant <TENANT_NAME> [JSON]",
-                "prints the metadata for a tenant",
-                "Prints the metadata for a tenant. If JSON is specified, then the output will be in JSON format."));
-
-// renametenant command
-ACTOR Future<bool> renameTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	if (tokens.size() != 3) {
-		printUsage(tokens[0]);
+// tenant configure command
+ACTOR Future<bool> tenantConfigureCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() < 4) {
+		fmt::print("Usage: tenant configure <TENANT_NAME> <[unset] tenant_group[=<GROUP_NAME>]> ...\n\n");
+		fmt::print("Updates the configuration for a tenant.\n");
+		fmt::print("Use `tenant_group=<GROUP_NAME>' to change the tenant group that a\n");
+		fmt::print("tenant is assigned to or `unset tenant_group' to remove a tenant from\n");
+		fmt::print("its tenant group.");
 		return false;
 	}
-	wait(safeThreadFutureToFuture(TenantAPI::renameTenant(db, tokens[1], tokens[2])));
 
-	printf("The tenant `%s' has been renamed to `%s'\n", printable(tokens[1]).c_str(), printable(tokens[2]).c_str());
+	state Optional<std::map<Standalone<StringRef>, Optional<Value>>> configuration =
+	    parseTenantConfiguration(tokens, 3, true);
+
+	if (!configuration.present()) {
+		return false;
+	}
+
+	state Reference<ITransaction> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
+			if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+				TenantMapEntry tenantEntry;
+				wait(MetaclusterAPI::configureTenant(db, tokens[2], configuration.get()));
+			} else {
+				applyConfigurationToSpecialKeys(tr, tokens[2], configuration.get());
+				wait(safeThreadFutureToFuture(tr->commit()));
+			}
+			break;
+		} catch (Error& e) {
+			state Error err(e);
+			if (e.code() == error_code_special_keys_api_failure) {
+				std::string errorMsgStr = wait(getSpecialKeysFailureErrorMessage(tr));
+				fmt::print(stderr, "ERROR: {}\n", errorMsgStr.c_str());
+				return false;
+			}
+			wait(safeThreadFutureToFuture(tr->onError(err)));
+		}
+	}
+
+	fmt::print("The configuration for tenant `{}' has been updated\n", printable(tokens[2]).c_str());
 	return true;
 }
 
-CommandFactory renameTenantFactory(
-    "renametenant",
-    CommandHelp(
-        "renametenant <OLD_NAME> <NEW_NAME>",
-        "renames a tenant in the cluster.",
-        "Renames a tenant in the cluster. The old name must exist and the new name must not exist in the cluster."));
+// Helper function to extract tenant ID from json metadata string
+int64_t getTenantId(Value metadata) {
+	json_spirit::mValue jsonObject;
+	json_spirit::read_string(metadata.toString(), jsonObject);
+	JSONDoc doc(jsonObject);
+	int64_t id;
+	doc.get("id", id);
+	return id;
+}
+
+// tenant rename command
+ACTOR Future<bool> tenantRenameCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() != 4) {
+		fmt::print("Usage: tenant rename <OLD_NAME> <NEW_NAME>\n\n");
+		fmt::print("Renames a tenant in the cluster. The old name must exist and the new\n");
+		fmt::print("name must not exist in the cluster.\n");
+		return false;
+	}
+	state Reference<ITransaction> tr = db->createTransaction();
+	state Key tenantRenameKey = tenantRenameSpecialKeyRange.begin.withSuffix(tokens[2]);
+	state Key tenantOldNameKey = tenantMapSpecialKeyRange.begin.withSuffix(tokens[2]);
+	state Key tenantNewNameKey = tenantMapSpecialKeyRange.begin.withSuffix(tokens[3]);
+	state bool firstTry = true;
+	state int64_t id = -1;
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
+			if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+				wait(MetaclusterAPI::renameTenant(db, tokens[2], tokens[3]));
+			} else {
+				// Hold the reference to the standalone's memory
+				state ThreadFuture<Optional<Value>> oldEntryFuture = tr->get(tenantOldNameKey);
+				state ThreadFuture<Optional<Value>> newEntryFuture = tr->get(tenantNewNameKey);
+				state Optional<Value> oldEntry = wait(safeThreadFutureToFuture(oldEntryFuture));
+				state Optional<Value> newEntry = wait(safeThreadFutureToFuture(newEntryFuture));
+				if (firstTry) {
+					if (!oldEntry.present()) {
+						throw tenant_not_found();
+					}
+					if (newEntry.present()) {
+						throw tenant_already_exists();
+					}
+					// Store the id we see when first reading this key
+					id = getTenantId(oldEntry.get());
+
+					firstTry = false;
+				} else {
+					// If we got commit_unknown_result, the rename may have already occurred.
+					if (newEntry.present()) {
+						int64_t checkId = getTenantId(newEntry.get());
+						if (id == checkId) {
+							ASSERT(!oldEntry.present() || getTenantId(oldEntry.get()) != id);
+							return true;
+						}
+						// If the new entry is present but does not match, then
+						// the rename should fail, so we throw an error.
+						throw tenant_already_exists();
+					}
+					if (!oldEntry.present()) {
+						throw tenant_not_found();
+					}
+					int64_t checkId = getTenantId(oldEntry.get());
+					// If the id has changed since we made our first attempt,
+					// then it's possible we've already moved the tenant. Don't move it again.
+					if (id != checkId) {
+						throw tenant_not_found();
+					}
+				}
+				tr->set(tenantRenameKey, tokens[3]);
+				wait(safeThreadFutureToFuture(tr->commit()));
+			}
+			break;
+		} catch (Error& e) {
+			state Error err(e);
+			if (e.code() == error_code_special_keys_api_failure) {
+				std::string errorMsgStr = wait(getSpecialKeysFailureErrorMessage(tr));
+				fmt::print(stderr, "ERROR: {}\n", errorMsgStr.c_str());
+				return false;
+			}
+			wait(safeThreadFutureToFuture(tr->onError(err)));
+		}
+	}
+
+	fmt::print(
+	    "The tenant `{}' has been renamed to `{}'\n", printable(tokens[2]).c_str(), printable(tokens[3]).c_str());
+	return true;
+}
+
+// tenant command
+Future<bool> tenantCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() == 1) {
+		printUsage(tokens[0]);
+		return true;
+	} else if (tokencmp(tokens[1], "create")) {
+		return tenantCreateCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "delete")) {
+		return tenantDeleteCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "list")) {
+		return tenantListCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "get")) {
+		return tenantGetCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "configure")) {
+		return tenantConfigureCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "rename")) {
+		return tenantRenameCommand(db, tokens);
+	} else {
+		printUsage(tokens[0]);
+		return true;
+	}
+}
+
+Future<bool> tenantCommandForwarder(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	ASSERT(!tokens.empty() && (tokens[0].endsWith("tenant"_sr) || tokens[0].endsWith("tenants"_sr)));
+	std::vector<StringRef> forwardedTokens = { "tenant"_sr,
+		                                       tokens[0].endsWith("tenant"_sr) ? tokens[0].removeSuffix("tenant"_sr)
+		                                                                       : tokens[0].removeSuffix("tenants"_sr) };
+	for (int i = 1; i < tokens.size(); ++i) {
+		forwardedTokens.push_back(tokens[i]);
+	}
+
+	return tenantCommand(db, forwardedTokens);
+} // namespace fdb_cli
+
+void tenantGenerator(const char* text,
+                     const char* line,
+                     std::vector<std::string>& lc,
+                     std::vector<StringRef> const& tokens) {
+	if (tokens.size() == 1) {
+		const char* opts[] = { "create", "delete", "list", "get", "configure", "rename", nullptr };
+		arrayGenerator(text, line, opts, lc);
+	} else if (tokens.size() == 3 && tokencmp(tokens[1], "create")) {
+		const char* opts[] = { "tenant_group=", nullptr };
+		arrayGenerator(text, line, opts, lc);
+	} else if (tokens.size() == 3 && tokencmp(tokens[1], "get")) {
+		const char* opts[] = { "JSON", nullptr };
+		arrayGenerator(text, line, opts, lc);
+	} else if (tokencmp(tokens[1], "configure")) {
+		if (tokens.size() == 3) {
+			const char* opts[] = { "tenant_group=", "unset", nullptr };
+			arrayGenerator(text, line, opts, lc);
+		} else if (tokens.size() == 4 && tokencmp(tokens[3], "unset")) {
+			const char* opts[] = { "tenant_group", nullptr };
+			arrayGenerator(text, line, opts, lc);
+		}
+	}
+}
+
+std::vector<const char*> tenantHintGenerator(std::vector<StringRef> const& tokens, bool inArgument) {
+	if (tokens.size() == 1) {
+		return { "<create|delete|list|get|configure|rename>", "[ARGS]" };
+	} else if (tokencmp(tokens[1], "create") && tokens.size() < 4) {
+		static std::vector<const char*> opts = { "<NAME> [tenant_group=<TENANT_GROUP>]" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else if (tokencmp(tokens[1], "delete") && tokens.size() < 3) {
+		static std::vector<const char*> opts = { "<NAME>" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else if (tokencmp(tokens[1], "list") && tokens.size() < 5) {
+		static std::vector<const char*> opts = { "[BEGIN]", "[END]", "[LIMIT]" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else if (tokencmp(tokens[1], "get") && tokens.size() < 4) {
+		static std::vector<const char*> opts = { "<NAME>", "[JSON]" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else if (tokencmp(tokens[1], "configure")) {
+		if (tokens.size() < 4) {
+			static std::vector<const char*> opts = { "<TENANT_NAME>", "<[unset] tenant_group[=<GROUP_NAME>]>" };
+			return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+		} else if (tokens.size() == 4 && tokencmp(tokens[3], "unset")) {
+			static std::vector<const char*> opts = { "<tenant_group[=<GROUP_NAME>]>" };
+			return std::vector<const char*>(opts.begin() + tokens.size() - 4, opts.end());
+		}
+		return {};
+	} else if (tokencmp(tokens[1], "rename") && tokens.size() < 4) {
+		static std::vector<const char*> opts = { "<OLD_NAME>", "<NEW_NAME>" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else {
+		return {};
+	}
+}
+
+CommandFactory tenantRegisterFactory("tenant",
+                                     CommandHelp("tenant <create|delete|list|get|configure|rename> [ARGS]",
+                                                 "view and manage tenants in a cluster or metacluster",
+                                                 "`create' and `delete' add and remove tenants from the cluster.\n"
+                                                 "`list' prints a list of tenants in the cluster.\n"
+                                                 "`get' prints the metadata for a particular tenant.\n"
+                                                 "`configure' modifies the configuration for a tenant.\n"
+                                                 "`rename' changes the name of a tenant.\n"),
+                                     &tenantGenerator,
+                                     &tenantHintGenerator);
+
+// Generate hidden commands for the old versions of the tenant commands
+CommandFactory createTenantFactory("createtenant");
+CommandFactory deleteTenantFactory("deletetenant");
+CommandFactory listTenantsFactory("listtenants");
+CommandFactory getTenantFactory("gettenant");
+CommandFactory configureTenantFactory("configuretenant");
+CommandFactory renameTenantFactory("renametenant");
+
 } // namespace fdb_cli
diff --git a/fdbcli/TenantGroupCommands.actor.cpp b/fdbcli/TenantGroupCommands.actor.cpp
new file mode 100644
index 0000000000..6a89360aeb
--- /dev/null
+++ b/fdbcli/TenantGroupCommands.actor.cpp
@@ -0,0 +1,240 @@
+/*
+ * TenantGroupCommands.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/Knobs.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
+#include "fdbclient/TenantManagement.actor.h"
+#include "fdbclient/Schemas.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace fdb_cli {
+
+// tenantgroup list command
+ACTOR Future<bool> tenantGroupListCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() > 5) {
+		fmt::print("Usage: tenantgroup list [BEGIN] [END] [LIMIT]\n\n");
+		fmt::print("Lists the tenant groups in a cluster.\n");
+		fmt::print("Only tenant groups in the range BEGIN - END will be printed.\n");
+		fmt::print("An optional LIMIT can be specified to limit the number of results (default 100).\n");
+		return false;
+	}
+
+	state StringRef beginTenantGroup = ""_sr;
+	state StringRef endTenantGroup = "\xff\xff"_sr;
+	state int limit = 100;
+
+	if (tokens.size() >= 3) {
+		beginTenantGroup = tokens[2];
+	}
+	if (tokens.size() >= 4) {
+		endTenantGroup = tokens[3];
+		if (endTenantGroup <= beginTenantGroup) {
+			fmt::print(stderr, "ERROR: end must be larger than begin");
+			return false;
+		}
+	}
+	if (tokens.size() == 5) {
+		int n = 0;
+		if (sscanf(tokens[4].toString().c_str(), "%d%n", &limit, &n) != 1 || n != tokens[4].size() || limit <= 0) {
+			fmt::print(stderr, "ERROR: invalid limit `{}'\n", tokens[4].toString());
+			return false;
+		}
+	}
+
+	state Reference<ITransaction> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
+			state std::vector<TenantGroupName> tenantGroupNames;
+			state std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups;
+			if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+				wait(store(tenantGroups,
+				           MetaclusterAPI::listTenantGroupsTransaction(tr, beginTenantGroup, endTenantGroup, limit)));
+			} else {
+				wait(store(tenantGroups,
+				           TenantAPI::listTenantGroupsTransaction(tr, beginTenantGroup, endTenantGroup, limit)));
+			}
+
+			if (tenantGroups.empty()) {
+				if (tokens.size() == 2) {
+					fmt::print("The cluster has no tenant groups\n");
+				} else {
+					fmt::print("The cluster has no tenant groups in the specified range\n");
+				}
+			}
+
+			int index = 0;
+			for (auto tenantGroup : tenantGroups) {
+				fmt::print("  {}. {}\n", ++index, printable(tenantGroup.first));
+			}
+
+			return true;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+// tenantgroup get command
+ACTOR Future<bool> tenantGroupGetCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() > 4 || (tokens.size() == 4 && tokens[3] != "JSON"_sr)) {
+		fmt::print("Usage: tenantgroup get <NAME> [JSON]\n\n");
+		fmt::print("Prints metadata associated with the given tenant group.\n");
+		fmt::print("If JSON is specified, then the output will be in JSON format.\n");
+		return false;
+	}
+
+	state bool useJson = tokens.size() == 4;
+	state Reference<ITransaction> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
+			state std::string tenantJson;
+			state Optional<TenantGroupEntry> entry;
+			if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+				wait(store(entry, MetaclusterAPI::tryGetTenantGroupTransaction(tr, tokens[2])));
+			} else {
+				wait(store(entry, TenantAPI::tryGetTenantGroupTransaction(tr, tokens[2])));
+				Optional<MetaclusterRegistrationEntry> metaclusterRegistration =
+				    wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
+
+				// We don't store assigned clusters in the tenant group entry on data clusters, so we can instead
+				// populate it from the metacluster registration
+				if (entry.present() && metaclusterRegistration.present() &&
+				    metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_DATA &&
+				    !entry.get().assignedCluster.present()) {
+					entry.get().assignedCluster = metaclusterRegistration.get().name;
+				}
+			}
+
+			if (!entry.present()) {
+				throw tenant_not_found();
+			}
+
+			if (useJson) {
+				json_spirit::mObject resultObj;
+				resultObj["tenant_group"] = entry.get().toJson();
+				resultObj["type"] = "success";
+				fmt::print("{}\n",
+				           json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print));
+			} else {
+				if (entry.get().assignedCluster.present()) {
+					fmt::print("  assigned cluster: {}\n", printable(entry.get().assignedCluster));
+				} else {
+					// This is a placeholder output for when a tenant group is read in a non-metacluster, where
+					// it currently has no metadata. When metadata is eventually added, we can print that instead.
+					fmt::print("The tenant group is present in the cluster\n");
+				}
+			}
+			return true;
+		} catch (Error& e) {
+			try {
+				wait(safeThreadFutureToFuture(tr->onError(e)));
+			} catch (Error& finalErr) {
+				state std::string errorStr;
+				if (finalErr.code() == error_code_tenant_not_found) {
+					errorStr = "tenant group not found";
+				} else if (useJson) {
+					errorStr = finalErr.what();
+				} else {
+					throw finalErr;
+				}
+
+				if (useJson) {
+					json_spirit::mObject resultObj;
+					resultObj["type"] = "error";
+					resultObj["error"] = errorStr;
+					fmt::print("{}\n",
+					           json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print));
+				} else {
+					fmt::print(stderr, "ERROR: {}\n", errorStr);
+				}
+
+				return false;
+			}
+		}
+	}
+}
+
+// tenantgroup command
+Future<bool> tenantGroupCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() == 1) {
+		printUsage(tokens[0]);
+		return true;
+	} else if (tokencmp(tokens[1], "list")) {
+		return tenantGroupListCommand(db, tokens);
+	} else if (tokencmp(tokens[1], "get")) {
+		return tenantGroupGetCommand(db, tokens);
+	} else {
+		printUsage(tokens[0]);
+		return true;
+	}
+}
+
+void tenantGroupGenerator(const char* text,
+                          const char* line,
+                          std::vector<std::string>& lc,
+                          std::vector<StringRef> const& tokens) {
+	if (tokens.size() == 1) {
+		const char* opts[] = { "list", "get", nullptr };
+		arrayGenerator(text, line, opts, lc);
+	} else if (tokens.size() == 3 && tokencmp(tokens[1], "get")) {
+		const char* opts[] = { "JSON", nullptr };
+		arrayGenerator(text, line, opts, lc);
+	}
+}
+
+std::vector<const char*> tenantGroupHintGenerator(std::vector<StringRef> const& tokens, bool inArgument) {
+	if (tokens.size() == 1) {
+		return { "<list|get>", "[ARGS]" };
+	} else if (tokencmp(tokens[1], "list") && tokens.size() < 5) {
+		static std::vector<const char*> opts = { "[BEGIN]", "[END]", "[LIMIT]" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else if (tokencmp(tokens[1], "get") && tokens.size() < 4) {
+		static std::vector<const char*> opts = { "<NAME>", "[JSON]" };
+		return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
+	} else {
+		return {};
+	}
+}
+
+CommandFactory tenantGroupRegisterFactory("tenantgroup",
+                                          CommandHelp("tenantgroup <list|get> [ARGS]",
+                                                      "view tenant group information",
+                                                      "`list' prints a list of tenant groups in the cluster.\n"
+                                                      "`get' prints the metadata for a particular tenant group.\n"),
+                                          &tenantGroupGenerator,
+                                          &tenantGroupHintGenerator);
+
+} // namespace fdb_cli
diff --git a/fdbcli/ThrottleCommand.actor.cpp b/fdbcli/ThrottleCommand.actor.cpp
index abff0e0475..057a83c78f 100644
--- a/fdbcli/ThrottleCommand.actor.cpp
+++ b/fdbcli/ThrottleCommand.actor.cpp
@@ -163,11 +163,11 @@ ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<Str
 			}
 		}
 		if (tokens.size() == 7) {
-			if (tokens[6] == LiteralStringRef("default")) {
+			if (tokens[6] == "default"_sr) {
 				priority = TransactionPriority::DEFAULT;
-			} else if (tokens[6] == LiteralStringRef("immediate")) {
+			} else if (tokens[6] == "immediate"_sr) {
 				priority = TransactionPriority::IMMEDIATE;
-			} else if (tokens[6] == LiteralStringRef("batch")) {
+			} else if (tokens[6] == "batch"_sr) {
 				priority = TransactionPriority::BATCH;
 			} else {
 				fprintf(stderr,
diff --git a/fdbcli/TssqCommand.actor.cpp b/fdbcli/TssqCommand.actor.cpp
index 654cbe73c5..710aac95a2 100644
--- a/fdbcli/TssqCommand.actor.cpp
+++ b/fdbcli/TssqCommand.actor.cpp
@@ -89,7 +89,7 @@ ACTOR Future<bool> tssQuarantine(Reference<IDatabase> db, bool enable, UID tssId
 			}
 
 			if (enable) {
-				tr->set(tssQuarantineKeyFor(tssId), LiteralStringRef(""));
+				tr->set(tssQuarantineKeyFor(tssId), ""_sr);
 				// remove server from TSS mapping when quarantine is enabled
 				tssMapDB.erase(tr, ssi.tssPairID.get());
 			} else {
@@ -112,19 +112,19 @@ namespace fdb_cli {
 
 ACTOR Future<bool> tssqCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() == 2) {
-		if (tokens[1] != LiteralStringRef("list")) {
+		if (tokens[1] != "list"_sr) {
 			printUsage(tokens[0]);
 			return false;
 		} else {
 			wait(tssQuarantineList(db));
 		}
 	} else if (tokens.size() == 3) {
-		if ((tokens[1] != LiteralStringRef("start") && tokens[1] != LiteralStringRef("stop")) ||
-		    (tokens[2].size() != 32) || !std::all_of(tokens[2].begin(), tokens[2].end(), &isxdigit)) {
+		if ((tokens[1] != "start"_sr && tokens[1] != "stop"_sr) || (tokens[2].size() != 32) ||
+		    !std::all_of(tokens[2].begin(), tokens[2].end(), &isxdigit)) {
 			printUsage(tokens[0]);
 			return false;
 		} else {
-			bool enable = tokens[1] == LiteralStringRef("start");
+			bool enable = tokens[1] == "start"_sr;
 			UID tssId = UID::fromString(tokens[2].toString());
 			bool success = wait(tssQuarantine(db, enable, tssId));
 			return success;
diff --git a/fdbcli/Util.actor.cpp b/fdbcli/Util.actor.cpp
index d40a5dcaeb..aed1133047 100644
--- a/fdbcli/Util.actor.cpp
+++ b/fdbcli/Util.actor.cpp
@@ -62,56 +62,49 @@ ACTOR Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransacti
 	return valueObj["message"].get_str();
 }
 
-ACTOR Future<Void> verifyAndAddInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
-                                         Reference<FlowLock> connectLock,
-                                         KeyValue kv) {
-	wait(connectLock->take());
-	state FlowLock::Releaser releaser(*connectLock);
-	state ClientWorkerInterface workerInterf;
-	try {
-		// the interface is back-ward compatible, thus if parsing failed, it needs to upgrade cli version
-		workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
-	} catch (Error& e) {
-		fprintf(stderr, "Error: %s; CLI version is too old, please update to use a newer version\n", e.what());
-		return Void();
-	}
-	state ClientLeaderRegInterface leaderInterf(workerInterf.address());
-	choose {
-		when(Optional<LeaderInfo> rep =
-		         wait(brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())))) {
-			StringRef ip_port =
-			    (kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key)
-			        .removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
-			(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
-
-			if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
-				Key full_ip_port2 =
-				    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
-				StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls"))
-				                         ? full_ip_port2.removeSuffix(LiteralStringRef(":tls"))
-				                         : full_ip_port2;
-				(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
-			}
+void addInterfacesFromKVs(RangeResult& kvs,
+                          std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
+	for (const auto& kv : kvs) {
+		ClientWorkerInterface workerInterf;
+		try {
+			// the interface is back-ward compatible, thus if parsing failed, it needs to upgrade cli version
+			workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
+		} catch (Error& e) {
+			fprintf(stderr, "Error: %s; CLI version is too old, please update to use a newer version\n", e.what());
+			return;
+		}
+		ClientLeaderRegInterface leaderInterf(workerInterf.address());
+		StringRef ip_port = (kv.key.endsWith(":tls"_sr) ? kv.key.removeSuffix(":tls"_sr) : kv.key)
+		                        .removePrefix("\xff\xff/worker_interfaces/"_sr);
+		(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
+
+		if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
+			Key full_ip_port2 =
+			    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
+			StringRef ip_port2 =
+			    full_ip_port2.endsWith(":tls"_sr) ? full_ip_port2.removeSuffix(":tls"_sr) : full_ip_port2;
+			(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
 		}
-		when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {}
 	}
-	return Void();
 }
 
 ACTOR Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
-                                       std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
+                                       std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
+                                       bool verify) {
+	if (verify) {
+		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		tr->set(workerInterfacesVerifyOptionSpecialKey, ValueRef());
+	}
 	// Hold the reference to the standalone's memory
 	state ThreadFuture<RangeResult> kvsFuture = tr->getRange(
-	    KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"), LiteralStringRef("\xff\xff/worker_interfaces0")),
-	    CLIENT_KNOBS->TOO_MANY);
-	RangeResult kvs = wait(safeThreadFutureToFuture(kvsFuture));
+	    KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr), CLIENT_KNOBS->TOO_MANY);
+	state RangeResult kvs = wait(safeThreadFutureToFuture(kvsFuture));
 	ASSERT(!kvs.more);
-	auto connectLock = makeReference<FlowLock>(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM);
-	std::vector<Future<Void>> addInterfs;
-	for (auto it : kvs) {
-		addInterfs.push_back(verifyAndAddInterface(address_interface, connectLock, it));
+	if (verify) {
+		// remove the option if set
+		tr->clear(workerInterfacesVerifyOptionSpecialKey);
 	}
-	wait(waitForAll(addInterfs));
+	addInterfacesFromKVs(kvs, address_interface);
 	return Void();
 }
 
diff --git a/fdbcli/VersionEpochCommand.actor.cpp b/fdbcli/VersionEpochCommand.actor.cpp
index 7d073e590d..a9dcd7e198 100644
--- a/fdbcli/VersionEpochCommand.actor.cpp
+++ b/fdbcli/VersionEpochCommand.actor.cpp
@@ -32,7 +32,7 @@
 
 namespace fdb_cli {
 
-const KeyRef versionEpochSpecialKey = LiteralStringRef("\xff\xff/management/version_epoch");
+const KeyRef versionEpochSpecialKey = "\xff\xff/management/version_epoch"_sr;
 
 struct VersionInfo {
 	int64_t version;
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index ba01fa3c22..a5c2e2e75a 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -44,11 +44,13 @@
 
 #include "fdbclient/ThreadSafeTransaction.h"
 #include "flow/flow.h"
+#include "flow/ApiVersion.h"
 #include "flow/ArgParseUtil.h"
 #include "flow/DeterministicRandom.h"
 #include "flow/FastRef.h"
 #include "flow/Platform.h"
 #include "flow/SystemMonitor.h"
+#include "flow/CodeProbe.h"
 
 #include "flow/TLSConfig.actor.h"
 #include "flow/ThreadHelper.actor.h"
@@ -72,7 +74,6 @@
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-#define FDB_API_VERSION 720
 /*
  * While we could just use the MultiVersionApi instance directly, this #define allows us to swap in any other IClientApi
  * instance (e.g. from ThreadSafeApi)
@@ -102,6 +103,7 @@ enum {
 	OPT_DEBUG_TLS,
 	OPT_API_VERSION,
 	OPT_MEMORY,
+	OPT_USE_FUTURE_PROTOCOL_VERSION
 };
 
 CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
@@ -126,6 +128,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
 	                                  { OPT_DEBUG_TLS, "--debug-tls", SO_NONE },
 	                                  { OPT_API_VERSION, "--api-version", SO_REQ_SEP },
 	                                  { OPT_MEMORY, "--memory", SO_REQ_SEP },
+	                                  { OPT_USE_FUTURE_PROTOCOL_VERSION, "--use-future-protocol-version", SO_NONE },
 	                                  TLS_OPTION_FLAGS,
 	                                  SO_END_OF_OPTIONS };
 
@@ -474,6 +477,9 @@ static void printProgramUsage(const char* name) {
 	       "                 Useful in reporting and diagnosing TLS issues.\n"
 	       "  --build-flags  Print build information and exit.\n"
 	       "  --memory       Resident memory limit of the CLI (defaults to 8GiB).\n"
+	       "  --use-future-protocol-version\n"
+	       "                 Use the simulated future protocol version to connect to the cluster.\n"
+	       "                 This option can be used testing purposes only!\n"
 	       "  -v, --version  Print FoundationDB CLI version information and exit.\n"
 	       "  -h, --help     Display this help and exit.\n");
 }
@@ -531,10 +537,10 @@ void initHelp() {
 	    CommandHelp("getversion",
 	                "Fetch the current read version",
 	                "Displays the current read version of the database or currently running transaction.");
-	helpMap["quota"] =
-	    CommandHelp("quota",
-	                "quota [get <tag> [reserved|total] [read|write]|set <tag> [reserved|total] [read|write] <value>]",
-	                "Get or modify the throughput quota for the specified tag.");
+	helpMap["quota"] = CommandHelp("quota",
+	                               "quota [get <tag> [reserved_throughput|total_throughput] | set <tag> "
+	                               "[reserved_throughput|total_throughput] <value>]",
+	                               "Get or modify the throughput quota for the specified tag.");
 	helpMap["reset"] =
 	    CommandHelp("reset",
 	                "reset the current transaction",
@@ -577,7 +583,7 @@ void initHelp() {
 void printVersion() {
 	printf("FoundationDB CLI " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
 	printf("source version %s\n", getSourceVersion());
-	printf("protocol %" PRIx64 "\n", currentProtocolVersion.version());
+	printf("protocol %" PRIx64 "\n", currentProtocolVersion().version());
 }
 
 void printBuildInformation() {
@@ -648,7 +654,7 @@ ACTOR Future<Void> checkStatus(Future<Void> f,
 		StatusObject _s = wait(StatusClient::statusFetcher(localDb));
 		s = _s;
 	} else {
-		state ThreadFuture<Optional<Value>> statusValueF = tr->get(LiteralStringRef("\xff\xff/status/json"));
+		state ThreadFuture<Optional<Value>> statusValueF = tr->get("\xff\xff/status/json"_sr);
 		Optional<Value> statusValue = wait(safeThreadFutureToFuture(statusValueF));
 		if (!statusValue.present()) {
 			fprintf(stderr, "ERROR: Failed to get status json from the cluster\n");
@@ -692,7 +698,7 @@ ACTOR Future<bool> createSnapshot(Database db, std::vector<StringRef> tokens) {
 	for (int i = 1; i < tokens.size(); i++) {
 		snapCmd = snapCmd.withSuffix(tokens[i]);
 		if (i != tokens.size() - 1) {
-			snapCmd = snapCmd.withSuffix(LiteralStringRef(" "));
+			snapCmd = snapCmd.withSuffix(" "_sr);
 		}
 	}
 	try {
@@ -871,6 +877,7 @@ struct CLIOptions {
 	Optional<std::string> exec;
 	bool initialStatusCheck = true;
 	bool cliHints = true;
+	bool useFutureProtocolVersion = false;
 	bool debugTLS = false;
 	std::string tlsCertPath;
 	std::string tlsKeyPath;
@@ -882,7 +889,7 @@ struct CLIOptions {
 	std::vector<std::pair<std::string, std::string>> knobs;
 
 	// api version, using the latest version by default
-	int apiVersion = FDB_API_VERSION;
+	int apiVersion = ApiVersion::LATEST_VERSION;
 
 	CLIOptions(int argc, char* argv[]) {
 		program_name = argv[0];
@@ -931,12 +938,12 @@ struct CLIOptions {
 			if (*endptr != '\0') {
 				fprintf(stderr, "ERROR: invalid client version %s\n", args.OptionArg());
 				return 1;
-			} else if (apiVersion < 700 || apiVersion > FDB_API_VERSION) {
+			} else if (apiVersion < 700 || apiVersion > ApiVersion::LATEST_VERSION) {
 				// multi-version fdbcli only available after 7.0
 				fprintf(stderr,
 				        "ERROR: api version %s is not supported. (Min: 700, Max: %d)\n",
 				        args.OptionArg(),
-				        FDB_API_VERSION);
+				        ApiVersion::LATEST_VERSION);
 				return 1;
 			}
 			break;
@@ -972,6 +979,10 @@ struct CLIOptions {
 			break;
 		case OPT_NO_HINTS:
 			cliHints = false;
+			break;
+		case OPT_USE_FUTURE_PROTOCOL_VERSION:
+			useFutureProtocolVersion = true;
+			break;
 
 		// TLS Options
 		case TLSConfig::OPT_TLS_PLUGIN:
@@ -1039,37 +1050,7 @@ Future<T> stopNetworkAfter(Future<T> what) {
 	}
 }
 
-ACTOR Future<Void> addInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
-                                Reference<FlowLock> connectLock,
-                                KeyValue kv) {
-	wait(connectLock->take());
-	state FlowLock::Releaser releaser(*connectLock);
-	state ClientWorkerInterface workerInterf =
-	    BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
-	state ClientLeaderRegInterface leaderInterf(workerInterf.address());
-	choose {
-		when(Optional<LeaderInfo> rep =
-		         wait(brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())))) {
-			StringRef ip_port =
-			    (kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key)
-			        .removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
-			(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
-
-			if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
-				Key full_ip_port2 =
-				    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
-				StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls"))
-				                         ? full_ip_port2.removeSuffix(LiteralStringRef(":tls"))
-				                         : full_ip_port2;
-				(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
-			}
-		}
-		when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {}
-	}
-	return Void();
-}
-
-ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
+ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterConnectionFile> ccf) {
 	state LineNoise& linenoise = *plinenoise;
 	state bool intrans = false;
 
@@ -1094,20 +1075,6 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 
 	state FdbOptions* options = &globalOptions;
 
-	state Reference<ClusterConnectionFile> ccf;
-
-	state std::pair<std::string, bool> resolvedClusterFile =
-	    ClusterConnectionFile::lookupClusterFileName(opt.clusterFile);
-	try {
-		ccf = makeReference<ClusterConnectionFile>(resolvedClusterFile.first);
-	} catch (Error& e) {
-		if (e.code() == error_code_operation_cancelled) {
-			throw;
-		}
-		fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str());
-		return 1;
-	}
-
 	// Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are
 	// logged. This thread will eventually run the network, so call it now.
 	TraceEvent::setNetworkThread();
@@ -1361,13 +1328,10 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				}
 
 				if (tokencmp(tokens[0], "fileconfigure")) {
-					if (tokens.size() == 2 || (tokens.size() == 3 && (tokens[1] == LiteralStringRef("new") ||
-					                                                  tokens[1] == LiteralStringRef("FORCE")))) {
-						bool _result =
-						    wait(makeInterruptable(fileConfigureCommandActor(db,
-						                                                     tokens.back().toString(),
-						                                                     tokens[1] == LiteralStringRef("new"),
-						                                                     tokens[1] == LiteralStringRef("FORCE"))));
+					if (tokens.size() == 2 ||
+					    (tokens.size() == 3 && (tokens[1] == "new"_sr || tokens[1] == "FORCE"_sr))) {
+						bool _result = wait(makeInterruptable(fileConfigureCommandActor(
+						    db, tokens.back().toString(), tokens[1] == "new"_sr, tokens[1] == "FORCE"_sr)));
 						if (!_result)
 							is_error = true;
 					} else {
@@ -1426,6 +1390,13 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "blobkey")) {
+					bool _result = wait(makeInterruptable(blobKeyCommandActor(localDb, tenantEntry, tokens)));
+					if (!_result)
+						is_error = true;
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "unlock")) {
 					if ((tokens.size() != 2) || (tokens[1].size() != 32) ||
 					    !std::all_of(tokens[1].begin(), tokens[1].end(), &isxdigit)) {
@@ -1615,6 +1586,13 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "consistencyscan")) {
+					bool _result = wait(makeInterruptable(consistencyScanCommandActor(localDb, tokens)));
+					if (!_result)
+						is_error = true;
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "profile")) {
 					getTransaction(db, managementTenant, tr, options, intrans);
 					bool _result = wait(makeInterruptable(profileCommandActor(localDb, tr, tokens, intrans)));
@@ -1907,41 +1885,39 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
-				if (tokencmp(tokens[0], "createtenant")) {
-					bool _result = wait(makeInterruptable(createTenantCommandActor(db, tokens)));
-					if (!_result)
+				if (tokencmp(tokens[0], "tenant")) {
+					bool _result = wait(makeInterruptable(tenantCommand(db, tokens)));
+					if (!_result) {
 						is_error = true;
-					continue;
-				}
-
-				if (tokencmp(tokens[0], "deletetenant")) {
-					bool _result = wait(makeInterruptable(deleteTenantCommandActor(db, tokens)));
-					if (!_result)
-						is_error = true;
-					else if (tenantName.present() && tokens[1] == tenantName.get()) {
+					} else if (tokens.size() >= 3 && tenantName.present() && tokencmp(tokens[1], "delete") &&
+					           tokens[2] == tenantName.get()) {
 						printAtCol("WARNING: the active tenant was deleted. Use the `usetenant' or `defaulttenant' "
 						           "command to choose a new tenant.\n",
 						           80);
 					}
+
 					continue;
 				}
 
-				if (tokencmp(tokens[0], "listtenants")) {
-					bool _result = wait(makeInterruptable(listTenantsCommandActor(db, tokens)));
+				if (tokencmp(tokens[0], "createtenant") || tokencmp(tokens[0], "deletetenant") ||
+				    tokencmp(tokens[0], "listtenants") || tokencmp(tokens[0], "gettenant") ||
+				    tokencmp(tokens[0], "configuretenant") || tokencmp(tokens[0], "renametenant")) {
+					bool _result = wait(makeInterruptable(tenantCommandForwarder(db, tokens)));
+					if (!_result) {
+						is_error = true;
+					}
+					continue;
+				}
+
+				if (tokencmp(tokens[0], "tenantgroup")) {
+					bool _result = wait(makeInterruptable(tenantGroupCommand(db, tokens)));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
-				if (tokencmp(tokens[0], "gettenant")) {
-					bool _result = wait(makeInterruptable(getTenantCommandActor(db, tokens, opt.apiVersion)));
-					if (!_result)
-						is_error = true;
-					continue;
-				}
-
-				if (tokencmp(tokens[0], "renametenant")) {
-					bool _result = wait(makeInterruptable(renameTenantCommandActor(db, tokens)));
+				if (tokencmp(tokens[0], "metacluster")) {
+					bool _result = wait(makeInterruptable(metaclusterCommand(db, tokens)));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1980,7 +1956,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	}
 }
 
-ACTOR Future<int> runCli(CLIOptions opt) {
+ACTOR Future<int> runCli(CLIOptions opt, Reference<ClusterConnectionFile> ccf) {
 	state LineNoise linenoise(
 	    [](std::string const& line, std::vector<std::string>& completions) { fdbcliCompCmd(line, completions); },
 	    [enabled = opt.cliHints](std::string const& line) -> LineNoise::Hint {
@@ -2044,7 +2020,7 @@ ACTOR Future<int> runCli(CLIOptions opt) {
 		    .GetLastError();
 	}
 
-	state int result = wait(cli(opt, &linenoise));
+	state int result = wait(cli(opt, &linenoise, ccf));
 
 	if (!historyFilename.empty()) {
 		try {
@@ -2066,6 +2042,31 @@ ACTOR Future<Void> timeExit(double duration) {
 	return Void();
 }
 
+const char* checkTlsConfigAgainstCoordAddrs(const ClusterConnectionString& ccs) {
+	// Resolve TLS config and inspect whether any of the certificate, key, ca bytes has been set
+	extern TLSConfig tlsConfig;
+	auto const loaded = tlsConfig.loadSync();
+	const bool tlsConfigured =
+	    !loaded.getCertificateBytes().empty() || !loaded.getKeyBytes().empty() || !loaded.getCABytes().empty();
+	int tlsAddrs = 0;
+	int totalAddrs = 0;
+	for (const auto& addr : ccs.coords) {
+		if (addr.isTLS())
+			tlsAddrs++;
+		totalAddrs++;
+	}
+	for (const auto& host : ccs.hostnames) {
+		if (host.isTLS)
+			tlsAddrs++;
+		totalAddrs++;
+	}
+	if (!tlsConfigured && tlsAddrs == totalAddrs) {
+		return "fdbcli is not configured with TLS, but all of the coordinators have TLS addresses.";
+	} else {
+		return nullptr;
+	}
+}
+
 int main(int argc, char** argv) {
 	platformInit();
 	Error::init();
@@ -2170,15 +2171,37 @@ int main(int argc, char** argv) {
 		return 0;
 	}
 
+	Reference<ClusterConnectionFile> ccf;
+	std::pair<std::string, bool> resolvedClusterFile = ClusterConnectionFile::lookupClusterFileName(opt.clusterFile);
+
+	try {
+		ccf = makeReference<ClusterConnectionFile>(resolvedClusterFile.first);
+	} catch (Error& e) {
+		if (e.code() == error_code_operation_cancelled) {
+			throw;
+		}
+		fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str());
+		return 1;
+	}
+
+	// Make sure that TLS configuration lines up with ":tls" prefix on coordinator addresses
+	if (auto errorMsg = checkTlsConfigAgainstCoordAddrs(ccf->getConnectionString())) {
+		fprintf(stderr, "ERROR: %s\n", errorMsg);
+		return 1;
+	}
+
 	try {
 		API->selectApiVersion(opt.apiVersion);
+		if (opt.useFutureProtocolVersion) {
+			API->useFutureProtocolVersion();
+		}
 		API->setupNetwork();
 		opt.setupKnobs();
 		if (opt.exit_code != -1) {
 			return opt.exit_code;
 		}
 		Future<Void> memoryUsageMonitor = startMemoryUsageMonitor(opt.memLimit);
-		Future<int> cliFuture = runCli(opt);
+		Future<int> cliFuture = runCli(opt, ccf);
 		Future<Void> timeoutFuture = opt.exit_timeout ? timeExit(opt.exit_timeout) : Never();
 		auto f = stopNetworkAfter(success(cliFuture) || timeoutFuture);
 		API->runNetwork();
diff --git a/fdbcli/include/fdbcli/fdbcli.actor.h b/fdbcli/include/fdbcli/fdbcli.actor.h
index ba754279e8..dce68eb10b 100644
--- a/fdbcli/include/fdbcli/fdbcli.actor.h
+++ b/fdbcli/include/fdbcli/fdbcli.actor.h
@@ -95,6 +95,7 @@ extern const KeyRef advanceVersionSpecialKey;
 extern const KeyRef consistencyCheckSpecialKey;
 // coordinators
 extern const KeyRef clusterDescriptionSpecialKey;
+extern const KeyRef configDBSpecialKey;
 extern const KeyRef coordinatorsAutoSpecialKey;
 extern const KeyRef coordinatorsProcessSpecialKey;
 // datadistribution
@@ -119,7 +120,8 @@ extern const KeyRef ignoreSSFailureSpecialKey;
 extern const KeyRangeRef processClassSourceSpecialKeyRange;
 extern const KeyRangeRef processClassTypeSpecialKeyRange;
 // Other special keys
-inline const KeyRef errorMsgSpecialKey = LiteralStringRef("\xff\xff/error_message");
+inline const KeyRef errorMsgSpecialKey = "\xff\xff/error_message"_sr;
+inline const KeyRef workerInterfacesVerifyOptionSpecialKey = "\xff\xff/management/options/worker_interfaces/verify"_sr;
 // help functions (Copied from fdbcli.actor.cpp)
 
 // get all workers' info
@@ -132,13 +134,14 @@ void printUsage(StringRef command);
 // Pre: tr failed with special_keys_api_failure error
 // Read the error message special key and return the message
 ACTOR Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransaction> tr);
-// Using \xff\xff/worker_interfaces/ special key, get all worker interfaces
+// Using \xff\xff/worker_interfaces/ special key, get all worker interfaces.
+// A worker list will be returned from CC.
+// If verify, we will try to establish connections to all workers returned.
+// In particular, it will deserialize \xff\xff/worker_interfaces/<address>:=<ClientInterface> kv pairs and issue RPC
+// calls, then only return interfaces(kv pairs) the client can talk to
 ACTOR Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
-                                       std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
-// Deserialize \xff\xff/worker_interfaces/<address>:=<ClientInterface> k-v pair and verify by a RPC call
-ACTOR Future<Void> verifyAndAddInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
-                                         Reference<FlowLock> connectLock,
-                                         KeyValue kv);
+                                       std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
+                                       bool verify = false);
 // print cluster status info
 void printStatus(StatusObjectReader statusObj,
                  StatusClient::StatusLevel level,
@@ -161,14 +164,12 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 ACTOR Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr,
                                                 std::vector<StringRef> tokens,
                                                 bool intrans);
+// consistency scan command
+ACTOR Future<bool> consistencyScanCommandActor(Database localDb, std::vector<StringRef> tokens);
 // coordinators command
 ACTOR Future<bool> coordinatorsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// createtenant command
-ACTOR Future<bool> createTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // datadistribution command
 ACTOR Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// deletetenant command
-ACTOR Future<bool> deleteTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // exclude command
 ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, Future<Void> warn);
 // expensive_data_check command
@@ -184,8 +185,6 @@ ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
                                              bool force);
 // force_recovery_with_data_loss command
 ACTOR Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// gettenant command
-ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, int apiVersion);
 // include command
 ACTOR Future<bool> includeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // kill command
@@ -193,11 +192,13 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
                                     Reference<ITransaction> tr,
                                     std::vector<StringRef> tokens,
                                     std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
-// listtenants command
-ACTOR Future<bool> listTenantsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // lock/unlock command
 ACTOR Future<bool> lockCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 ACTOR Future<bool> unlockDatabaseActor(Reference<IDatabase> db, UID uid);
+
+// metacluster command
+Future<bool> metaclusterCommand(Reference<IDatabase> db, std::vector<StringRef> tokens);
+
 // changefeed command
 ACTOR Future<bool> changeFeedCommandActor(Database localDb,
                                           Optional<TenantMapEntry> tenantEntry,
@@ -207,6 +208,11 @@ ACTOR Future<bool> changeFeedCommandActor(Database localDb,
 ACTOR Future<bool> blobRangeCommandActor(Database localDb,
                                          Optional<TenantMapEntry> tenantEntry,
                                          std::vector<StringRef> tokens);
+
+// blobkey command
+ACTOR Future<bool> blobKeyCommandActor(Database localDb,
+                                       Optional<TenantMapEntry> tenantEntry,
+                                       std::vector<StringRef> tokens);
 // maintenance command
 ACTOR Future<bool> setHealthyZone(Reference<IDatabase> db, StringRef zoneId, double seconds, bool printWarning = false);
 ACTOR Future<bool> clearHealthyZone(Reference<IDatabase> db,
@@ -218,8 +224,6 @@ ACTOR Future<bool> profileCommandActor(Database db,
                                        Reference<ITransaction> tr,
                                        std::vector<StringRef> tokens,
                                        bool intrans);
-// renametenant command
-ACTOR Future<bool> renameTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // quota command
 ACTOR Future<bool> quotaCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // setclass command
@@ -236,6 +240,12 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
                                        Reference<ITransaction> tr,
                                        std::vector<StringRef> tokens,
                                        std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
+// tenant command
+Future<bool> tenantCommand(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// tenant command compatibility layer
+Future<bool> tenantCommandForwarder(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// tenantgroup command
+Future<bool> tenantGroupCommand(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // throttle command
 ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // triggerteaminfolog command
diff --git a/bindings/python/tests/fdbcli_tests.py b/fdbcli/tests/fdbcli_tests.py
similarity index 74%
rename from bindings/python/tests/fdbcli_tests.py
rename to fdbcli/tests/fdbcli_tests.py
index 386916f5c6..530c80f865 100755
--- a/bindings/python/tests/fdbcli_tests.py
+++ b/fdbcli/tests/fdbcli_tests.py
@@ -7,6 +7,7 @@ import subprocess
 import logging
 import functools
 import json
+import tempfile
 import time
 import random
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
@@ -561,6 +562,7 @@ def profile(logger):
     assert output2 == default_profile_client_get_output
     # set rate and size limit
     run_fdbcli_command('profile', 'client', 'set', '0.5', '1GB')
+    time.sleep(1) # global config can take some time to sync
     output3 = run_fdbcli_command('profile', 'client', 'get')
     logger.debug(output3)
     output3_list = output3.split(' ')
@@ -569,6 +571,7 @@ def profile(logger):
     assert output3_list[-1] == '1000000000.'
     # change back to default value and check
     run_fdbcli_command('profile', 'client', 'set', 'default', 'default')
+    time.sleep(1) # global config can take some time to sync
     assert run_fdbcli_command('profile', 'client', 'get') == default_profile_client_get_output
 
 
@@ -590,47 +593,191 @@ def triggerddteaminfolog(logger):
     output = run_fdbcli_command('triggerddteaminfolog')
     assert output == 'Triggered team info logging in data distribution.'
 
+def setup_tenants(tenants):
+    command = '; '.join(['tenant create %s' % t for t in tenants])
+    run_fdbcli_command(command)
+
+def clear_database_and_tenants():
+    run_fdbcli_command('writemode on; option on SPECIAL_KEY_SPACE_ENABLE_WRITES; clearrange "" \\xff; clearrange \\xff\\xff/management/tenant/map/ \\xff\\xff/management/tenant/map0')
+
+def run_tenant_test(test_func):
+    test_func()
+    clear_database_and_tenants()
 
 @enable_logging()
-def tenants(logger):
-    output = run_fdbcli_command('listtenants')
-    assert output == 'The cluster has no tenants'
+def tenant_create(logger):
+    output1 = run_fdbcli_command('tenant create tenant')
+    assert output1 == 'The tenant `tenant\' has been created'
 
-    output = run_fdbcli_command('createtenant tenant')
-    assert output == 'The tenant `tenant\' has been created'
-
-    output = run_fdbcli_command('createtenant tenant2')
+    output = run_fdbcli_command('tenant create tenant2 tenant_group=tenant_group2')
     assert output == 'The tenant `tenant2\' has been created'
 
-    output = run_fdbcli_command('listtenants')
+    output = run_fdbcli_command_and_get_error('tenant create tenant')
+    assert output == 'ERROR: A tenant with the given name already exists (2132)'
+
+@enable_logging()
+def tenant_delete(logger):
+    setup_tenants(['tenant', 'tenant2'])
+    run_fdbcli_command('writemode on; usetenant tenant2; set tenant_test value')
+
+    # delete a tenant while the fdbcli is using that tenant
+    process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env)
+    cmd_sequence = ['writemode on', 'usetenant tenant', 'tenant delete tenant', 'get tenant_test', 'defaulttenant', 'usetenant tenant']
+    output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode())
+
+    lines = output.decode().strip().split('\n')[-6:]
+    error_lines = error_output.decode().strip().split('\n')[-2:]
+    assert lines[0] == 'Using tenant `tenant\''
+    assert lines[1] == 'The tenant `tenant\' has been deleted'
+    assert lines[2] == 'WARNING: the active tenant was deleted. Use the `usetenant\' or `defaulttenant\''
+    assert lines[3] == 'command to choose a new tenant.'
+    assert error_lines[0] == 'ERROR: Tenant does not exist (2131)'
+    assert lines[5] == 'Using the default tenant'
+    assert error_lines[1] == 'ERROR: Tenant `tenant\' does not exist'
+
+    # delete a non-empty tenant
+    process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env)
+    cmd_sequence = ['writemode on', 'tenant delete tenant2', 'usetenant tenant2', 'clear tenant_test', 'defaulttenant', 'tenant delete tenant2']
+    output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode())
+
+    lines = output.decode().strip().split('\n')[-4:]
+    error_lines = error_output.decode().strip().split('\n')[-1:]
+    assert error_lines[0] == 'ERROR: Cannot delete a non-empty tenant (2133)'
+    assert lines[0] == 'Using tenant `tenant2\''
+    assert lines[1].startswith('Committed')
+    assert lines[2] == 'Using the default tenant'
+    assert lines[3] == 'The tenant `tenant2\' has been deleted'
+
+    # delete a non-existing tenant
+    output = run_fdbcli_command_and_get_error('tenant delete tenant')
+    assert output == 'ERROR: Tenant does not exist (2131)'
+
+@enable_logging()
+def tenant_list(logger):
+    output = run_fdbcli_command('tenant list')
+    assert output == 'The cluster has no tenants'
+
+    setup_tenants(['tenant', 'tenant2'])
+
+    output = run_fdbcli_command('tenant list')
     assert output == '1. tenant\n  2. tenant2'
 
-    output = run_fdbcli_command('listtenants a z 1')
+    output = run_fdbcli_command('tenant list a z 1')
     assert output == '1. tenant'
 
-    output = run_fdbcli_command('listtenants a tenant2')
+    output = run_fdbcli_command('tenant list a tenant2')
     assert output == '1. tenant'
 
-    output = run_fdbcli_command('listtenants tenant2 z')
+    output = run_fdbcli_command('tenant list tenant2 z')
     assert output == '1. tenant2'
 
-    output = run_fdbcli_command('gettenant tenant')
+    output = run_fdbcli_command('tenant list a b')
+    assert output == 'The cluster has no tenants in the specified range'
+
+    output = run_fdbcli_command_and_get_error('tenant list b a')
+    assert output == 'ERROR: end must be larger than begin'
+
+    output = run_fdbcli_command_and_get_error('tenant list a b 12x')
+    assert output == 'ERROR: invalid limit `12x\''
+
+@enable_logging()
+def tenant_get(logger):
+    setup_tenants(['tenant', 'tenant2 tenant_group=tenant_group2'])
+
+    output = run_fdbcli_command('tenant get tenant')
     lines = output.split('\n')
-    assert len(lines) == 2
+    assert len(lines) == 3
     assert lines[0].strip().startswith('id: ')
     assert lines[1].strip().startswith('prefix: ')
+    assert lines[2].strip() == 'tenant state: ready'
 
-    output = run_fdbcli_command('gettenant tenant JSON')
+    output = run_fdbcli_command('tenant get tenant JSON')
     json_output = json.loads(output, strict=False)
     assert(len(json_output) == 2)
     assert('tenant' in json_output)
     assert(json_output['type'] == 'success')
-    assert(len(json_output['tenant']) == 2)
+    assert(len(json_output['tenant']) == 4)
     assert('id' in json_output['tenant'])
+    assert('encrypted' in json_output['tenant'])
     assert('prefix' in json_output['tenant'])
     assert(len(json_output['tenant']['prefix']) == 2)
     assert('base64' in json_output['tenant']['prefix'])
     assert('printable' in json_output['tenant']['prefix'])
+    assert(json_output['tenant']['tenant_state'] == 'ready')
+
+    output = run_fdbcli_command('tenant get tenant2')
+    lines = output.split('\n')
+    assert len(lines) == 4
+    assert lines[0].strip().startswith('id: ')
+    assert lines[1].strip().startswith('prefix: ')
+    assert lines[2].strip() == 'tenant state: ready'
+    assert lines[3].strip() == 'tenant group: tenant_group2'
+
+    output = run_fdbcli_command('tenant get tenant2 JSON')
+    json_output = json.loads(output, strict=False)
+    assert(len(json_output) == 2)
+    assert('tenant' in json_output)
+    assert(json_output['type'] == 'success')
+    assert(len(json_output['tenant']) == 5)
+    assert('id' in json_output['tenant'])
+    assert('encrypted' in json_output['tenant'])
+    assert('prefix' in json_output['tenant'])
+    assert(json_output['tenant']['tenant_state'] == 'ready')
+    assert('tenant_group' in json_output['tenant'])
+    assert(len(json_output['tenant']['tenant_group']) == 2)
+    assert('base64' in json_output['tenant']['tenant_group'])
+    assert(json_output['tenant']['tenant_group']['printable'] == 'tenant_group2')
+
+@enable_logging()
+def tenant_configure(logger):
+    setup_tenants(['tenant'])
+
+    output = run_fdbcli_command('tenant configure tenant tenant_group=tenant_group1')
+    assert output == 'The configuration for tenant `tenant\' has been updated'
+
+    output = run_fdbcli_command('tenant get tenant')
+    lines = output.split('\n')
+    assert len(lines) == 4
+    assert lines[3].strip() == 'tenant group: tenant_group1'
+
+    output = run_fdbcli_command('tenant configure tenant unset tenant_group')
+    assert output == 'The configuration for tenant `tenant\' has been updated'
+
+    output = run_fdbcli_command('tenant get tenant')
+    lines = output.split('\n')
+    assert len(lines) == 3
+
+    output = run_fdbcli_command_and_get_error('tenant configure tenant tenant_group=tenant_group1 tenant_group=tenant_group2')
+    assert output == 'ERROR: configuration parameter `tenant_group\' specified more than once.'
+
+    output = run_fdbcli_command_and_get_error('tenant configure tenant unset')
+    assert output == 'ERROR: `unset\' specified without a configuration parameter.'
+
+    output = run_fdbcli_command_and_get_error('tenant configure tenant unset tenant_group=tenant_group1')
+    assert output == 'ERROR: unrecognized configuration parameter `tenant_group=tenant_group1\'.'
+
+    output = run_fdbcli_command_and_get_error('tenant configure tenant tenant_group')
+    assert output == 'ERROR: invalid configuration string `tenant_group\'. String must specify a value using `=\'.'
+
+    output = run_fdbcli_command_and_get_error('tenant configure tenant3 tenant_group=tenant_group1')
+    assert output == 'ERROR: Tenant does not exist (2131)'
+
+@enable_logging()
+def tenant_rename(logger):
+    setup_tenants(['tenant', 'tenant2'])
+
+    output = run_fdbcli_command('tenant rename tenant tenant3')
+    assert output == 'The tenant `tenant\' has been renamed to `tenant3\''
+
+    output = run_fdbcli_command_and_get_error('tenant rename tenant tenant4')
+    assert output == 'ERROR: Tenant does not exist (2131)'
+
+    output = run_fdbcli_command_and_get_error('tenant rename tenant2 tenant3')
+    assert output == 'ERROR: A tenant with the given name already exists (2132)'
+
+@enable_logging()
+def tenant_usetenant(logger):
+    setup_tenants(['tenant', 'tenant2'])
 
     output = run_fdbcli_command('usetenant')
     assert output == 'Using the default tenant'
@@ -663,44 +810,103 @@ def tenants(logger):
     assert lines[3] == '`tenant_test\' is `tenant2\''
 
     process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env)
-    cmd_sequence = ['usetenant tenant', 'get tenant_test', 'defaulttenant', 'get tenant_test']
+    cmd_sequence = ['usetenant tenant', 'get tenant_test', 'usetenant tenant2', 'get tenant_test', 'defaulttenant', 'get tenant_test']
     output, _ = process.communicate(input='\n'.join(cmd_sequence).encode())
 
-    lines = output.decode().strip().split('\n')[-4:]
+    lines = output.decode().strip().split('\n')[-6:]
     assert lines[0] == 'Using tenant `tenant\''
     assert lines[1] == '`tenant_test\' is `tenant\''
-    assert lines[2] == 'Using the default tenant'
-    assert lines[3] == '`tenant_test\' is `default_tenant\''
+    assert lines[2] == 'Using tenant `tenant2\''
+    assert lines[3] == '`tenant_test\' is `tenant2\''
+    assert lines[4] == 'Using the default tenant'
+    assert lines[5] == '`tenant_test\' is `default_tenant\''
 
-    process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env)
-    cmd_sequence = ['writemode on', 'usetenant tenant', 'clear tenant_test',
-                    'deletetenant tenant', 'get tenant_test', 'defaulttenant', 'usetenant tenant']
-    output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode())
+@enable_logging()
+def tenant_old_commands(logger):
+    create_output = run_fdbcli_command('tenant create tenant')
+    list_output = run_fdbcli_command('tenant list')
+    get_output = run_fdbcli_command('tenant get tenant')
+    # Run the gettenant command here because the ID will be different in the second block
+    get_output_old = run_fdbcli_command('gettenant tenant')
+    configure_output = run_fdbcli_command('tenant configure tenant tenant_group=tenant_group1')
+    rename_output = run_fdbcli_command('tenant rename tenant tenant2')
+    delete_output = run_fdbcli_command('tenant delete tenant2')
 
-    lines = output.decode().strip().split('\n')[-7:]
-    error_lines = error_output.decode().strip().split('\n')[-2:]
-    assert lines[0] == 'Using tenant `tenant\''
-    assert lines[1].startswith('Committed')
-    assert lines[2] == 'The tenant `tenant\' has been deleted'
-    assert lines[3] == 'WARNING: the active tenant was deleted. Use the `usetenant\' or `defaulttenant\''
-    assert lines[4] == 'command to choose a new tenant.'
-    assert error_lines[0] == 'ERROR: Tenant does not exist (2131)'
-    assert lines[6] == 'Using the default tenant'
-    assert error_lines[1] == 'ERROR: Tenant `tenant\' does not exist'
+    create_output_old = run_fdbcli_command('createtenant tenant')
+    list_output_old = run_fdbcli_command('listtenants')
+    configure_output_old = run_fdbcli_command('configuretenant tenant tenant_group=tenant_group1')
+    rename_output_old = run_fdbcli_command('renametenant tenant tenant2')
+    delete_output_old = run_fdbcli_command('deletetenant tenant2')
 
-    process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env)
-    cmd_sequence = ['writemode on', 'deletetenant tenant2', 'usetenant tenant2', 'clear tenant_test', 'defaulttenant', 'deletetenant tenant2']
-    output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode())
+    assert create_output == create_output_old
+    assert list_output == list_output_old
+    assert get_output == get_output_old
+    assert configure_output == configure_output_old
+    assert rename_output == rename_output_old
+    assert delete_output == delete_output_old
 
-    lines = output.decode().strip().split('\n')[-4:]
-    error_lines = error_output.decode().strip().split('\n')[-1:]
-    assert error_lines[0] == 'ERROR: Cannot delete a non-empty tenant (2133)'
-    assert lines[0] == 'Using tenant `tenant2\''
-    assert lines[1].startswith('Committed')
-    assert lines[2] == 'Using the default tenant'
-    assert lines[3] == 'The tenant `tenant2\' has been deleted'
+@enable_logging()
+def tenant_group_list(logger):
+    output = run_fdbcli_command('tenantgroup list')
+    assert output == 'The cluster has no tenant groups'
 
-    run_fdbcli_command('writemode on; clear tenant_test')
+    setup_tenants(['tenant', 'tenant2 tenant_group=tenant_group2', 'tenant3 tenant_group=tenant_group3'])
+
+    output = run_fdbcli_command('tenantgroup list')
+    assert output == '1. tenant_group2\n  2. tenant_group3'
+
+    output = run_fdbcli_command('tenantgroup list a z 1')
+    assert output == '1. tenant_group2'
+
+    output = run_fdbcli_command('tenantgroup list a tenant_group3')
+    assert output == '1. tenant_group2'
+
+    output = run_fdbcli_command('tenantgroup list tenant_group3 z')
+    assert output == '1. tenant_group3'
+
+    output = run_fdbcli_command('tenantgroup list a b')
+    assert output == 'The cluster has no tenant groups in the specified range'
+
+    output = run_fdbcli_command_and_get_error('tenantgroup list b a')
+    assert output == 'ERROR: end must be larger than begin'
+
+    output = run_fdbcli_command_and_get_error('tenantgroup list a b 12x')
+    assert output == 'ERROR: invalid limit `12x\''
+
+@enable_logging()
+def tenant_group_get(logger):
+    setup_tenants(['tenant tenant_group=tenant_group'])
+
+    output = run_fdbcli_command('tenantgroup get tenant_group')
+    assert output == 'The tenant group is present in the cluster'
+
+    output = run_fdbcli_command('tenantgroup get tenant_group JSON')
+    json_output = json.loads(output, strict=False)
+    assert(len(json_output) == 2)
+    assert('tenant_group' in json_output)
+    assert(json_output['type'] == 'success')
+    assert(len(json_output['tenant_group']) == 0)
+
+    output = run_fdbcli_command_and_get_error('tenantgroup get tenant_group2')
+    assert output == 'ERROR: tenant group not found'
+
+    output = run_fdbcli_command('tenantgroup get tenant_group2 JSON')
+    json_output = json.loads(output, strict=False)
+    assert(len(json_output) == 2)
+    assert(json_output['type'] == 'error')
+    assert(json_output['error'] == 'tenant group not found')
+
+def tenants():
+    run_tenant_test(tenant_create)
+    run_tenant_test(tenant_delete)
+    run_tenant_test(tenant_list)
+    run_tenant_test(tenant_get)
+    run_tenant_test(tenant_configure)
+    run_tenant_test(tenant_rename)
+    run_tenant_test(tenant_usetenant)
+    run_tenant_test(tenant_old_commands)
+    run_tenant_test(tenant_group_list)
+    run_tenant_test(tenant_group_get)
 
 def integer_options():
     process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env)
@@ -712,6 +918,24 @@ def integer_options():
     assert lines[1].startswith('Committed')
     assert error_output == b''
 
+def tls_address_suffix():
+    # fdbcli shall prevent a non-TLS fdbcli run from connecting to an all-TLS cluster
+    preamble = 'eNW1yf1M:eNW1yf1M@'
+    num_server_addrs = [1, 2, 5]
+    err_output_server_tls = "ERROR: fdbcli is not configured with TLS, but all of the coordinators have TLS addresses."
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        cluster_fn = tmpdir + "/fdb.cluster"
+        for num_server_addr in num_server_addrs:
+            with open(cluster_fn, "w") as fp:
+                fp.write(preamble + ",".join(
+                    ["127.0.0.1:{}:tls".format(4000 + addr_idx) for addr_idx in range(num_server_addr)]))
+                fp.close()
+                fdbcli_process = subprocess.run(command_template[:2] + [cluster_fn], capture_output=True)
+                assert fdbcli_process.returncode != 0
+                err_out = fdbcli_process.stderr.decode("utf8").strip()
+                assert err_out == err_output_server_tls, f"unexpected output: {err_out}"
+
 if __name__ == '__main__':
     parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
                             description="""
@@ -758,6 +982,7 @@ if __name__ == '__main__':
         tenants()
         versionepoch()
         integer_options()
+        tls_address_suffix()
     else:
         assert args.process_number > 1, "Process number should be positive"
         coordinators()
diff --git a/fdbclient/Atomic.cpp b/fdbclient/Atomic.cpp
new file mode 100644
index 0000000000..f2614e3881
--- /dev/null
+++ b/fdbclient/Atomic.cpp
@@ -0,0 +1,47 @@
+/*
+ * Atomic.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/Atomic.h"
+#include "flow/Arena.h"
+#include "flow/UnitTest.h"
+
+void forceLinkAtomicTests() {}
+
+TEST_CASE("/Atomic/DoAppendIfFits") {
+	Arena arena;
+	{
+		Value existingValue = ValueRef(arena, "existing"_sr);
+		Value otherOperand = ValueRef(arena, "other"_sr);
+		auto result = doAppendIfFits(existingValue, otherOperand, arena);
+		ASSERT(compare("existingother"_sr, result) == 0);
+	}
+	{
+		Value existingValue = makeString(CLIENT_KNOBS->VALUE_SIZE_LIMIT - 1, arena);
+		Value otherOperand = makeString(2, arena);
+		deterministicRandom()->randomBytes(mutateString(existingValue), existingValue.size());
+		deterministicRandom()->randomBytes(mutateString(otherOperand), otherOperand.size());
+		// Appended values cannot fit in result, should return existingValue
+		auto result = doAppendIfFits(existingValue, otherOperand, arena);
+		ASSERT(compare(existingValue, result) == 0);
+	}
+	return Void();
+}
+
+// TODO: Add more unit tests for atomic operations defined in Atomic.h
diff --git a/fdbclient/BackupAgentBase.actor.cpp b/fdbclient/BackupAgentBase.actor.cpp
index cb999a6e12..ab136878c4 100644
--- a/fdbclient/BackupAgentBase.actor.cpp
+++ b/fdbclient/BackupAgentBase.actor.cpp
@@ -22,6 +22,10 @@
 #include <time.h>
 
 #include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BlobCipher.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/Metacluster.h"
 #include "fdbrpc/simulator.h"
 #include "flow/ActorCollection.h"
 #include "flow/actorcompiler.h" // has to be last include
@@ -253,16 +257,18 @@ std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key) {
 	    bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
 }
 
-void decodeBackupLogValue(Arena& arena,
-                          VectorRef<MutationRef>& result,
-                          int& mutationSize,
-                          StringRef value,
-                          StringRef addPrefix,
-                          StringRef removePrefix,
-                          Version version,
-                          Reference<KeyRangeMap<Version>> key_version) {
+ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
+                                               VectorRef<MutationRef>* result,
+                                               VectorRef<Optional<MutationRef>>* encryptedResult,
+                                               int* mutationSize,
+                                               Standalone<StringRef> value,
+                                               Key addPrefix,
+                                               Key removePrefix,
+                                               Version version,
+                                               Reference<KeyRangeMap<Version>> key_version,
+                                               Database cx) {
 	try {
-		uint64_t offset(0);
+		state uint64_t offset(0);
 		uint64_t protocolVersion = 0;
 		memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
 		offset += sizeof(uint64_t);
@@ -274,36 +280,48 @@ void decodeBackupLogValue(Arena& arena,
 			throw incompatible_protocol_version();
 		}
 
-		uint32_t totalBytes = 0;
+		state uint32_t totalBytes = 0;
 		memcpy(&totalBytes, value.begin() + offset, sizeof(uint32_t));
 		offset += sizeof(uint32_t);
-		uint32_t consumed = 0;
+		state uint32_t consumed = 0;
 
 		if (totalBytes + offset > value.size())
 			throw restore_missing_data();
 
-		int originalOffset = offset;
+		state int originalOffset = offset;
 
 		while (consumed < totalBytes) {
 			uint32_t type = 0;
 			memcpy(&type, value.begin() + offset, sizeof(uint32_t));
 			offset += sizeof(uint32_t);
-			uint32_t len1 = 0;
+			state uint32_t len1 = 0;
 			memcpy(&len1, value.begin() + offset, sizeof(uint32_t));
 			offset += sizeof(uint32_t);
-			uint32_t len2 = 0;
+			state uint32_t len2 = 0;
 			memcpy(&len2, value.begin() + offset, sizeof(uint32_t));
 			offset += sizeof(uint32_t);
 
 			ASSERT(offset + len1 + len2 <= value.size() && isValidMutationType(type));
 
-			MutationRef logValue;
-			Arena tempArena;
+			state MutationRef logValue;
+			state Arena tempArena;
 			logValue.type = type;
 			logValue.param1 = value.substr(offset, len1);
 			offset += len1;
 			logValue.param2 = value.substr(offset, len2);
 			offset += len2;
+			state Optional<MutationRef> encryptedLogValue = Optional<MutationRef>();
+
+			// Decrypt mutation ref if encrypted
+			if (logValue.isEncrypted()) {
+				encryptedLogValue = logValue;
+				Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
+				TextAndHeaderCipherKeys cipherKeys =
+				    wait(getEncryptCipherKeys(dbInfo, *logValue.encryptionHeader(), BlobCipherMetrics::BACKUP));
+				logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::BACKUP);
+			}
+			ASSERT(!logValue.isEncrypted());
+			MutationRef originalLogValue = logValue;
 
 			if (logValue.type == MutationRef::ClearRange) {
 				KeyRangeRef range(logValue.param1, logValue.param2);
@@ -311,7 +329,7 @@ void decodeBackupLogValue(Arena& arena,
 				for (auto r : ranges) {
 					if (version > r.value() && r.value() != invalidVersion) {
 						KeyRef minKey = std::min(r.range().end, range.end);
-						if (minKey == (removePrefix == StringRef() ? normalKeys.end : strinc(removePrefix))) {
+						if (minKey == (removePrefix == StringRef() ? allKeys.end : strinc(removePrefix))) {
 							logValue.param1 = std::max(r.range().begin, range.begin);
 							if (removePrefix.size()) {
 								logValue.param1 = logValue.param1.removePrefix(removePrefix);
@@ -319,9 +337,9 @@ void decodeBackupLogValue(Arena& arena,
 							if (addPrefix.size()) {
 								logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
 							}
-							logValue.param2 = addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix, tempArena);
-							result.push_back_deep(arena, logValue);
-							mutationSize += logValue.expectedSize();
+							logValue.param2 = addPrefix == StringRef() ? allKeys.end : strinc(addPrefix, tempArena);
+							result->push_back_deep(*arena, logValue);
+							*mutationSize += logValue.expectedSize();
 						} else {
 							logValue.param1 = std::max(r.range().begin, range.begin);
 							logValue.param2 = minKey;
@@ -333,8 +351,13 @@ void decodeBackupLogValue(Arena& arena,
 								logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
 								logValue.param2 = logValue.param2.withPrefix(addPrefix, tempArena);
 							}
-							result.push_back_deep(arena, logValue);
-							mutationSize += logValue.expectedSize();
+							result->push_back_deep(*arena, logValue);
+							*mutationSize += logValue.expectedSize();
+						}
+						if (originalLogValue.param1 == logValue.param1 && originalLogValue.param2 == logValue.param2) {
+							encryptedResult->push_back_deep(*arena, encryptedLogValue);
+						} else {
+							encryptedResult->push_back_deep(*arena, Optional<MutationRef>());
 						}
 					}
 				}
@@ -348,8 +371,15 @@ void decodeBackupLogValue(Arena& arena,
 					if (addPrefix.size()) {
 						logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
 					}
-					result.push_back_deep(arena, logValue);
-					mutationSize += logValue.expectedSize();
+					result->push_back_deep(*arena, logValue);
+					*mutationSize += logValue.expectedSize();
+					// If we did not remove/add prefixes to the mutation then keep the original encrypted mutation so we
+					// do not have to re-encrypt unnecessarily
+					if (originalLogValue.param1 == logValue.param1 && originalLogValue.param2 == logValue.param2) {
+						encryptedResult->push_back_deep(*arena, encryptedLogValue);
+					} else {
+						encryptedResult->push_back_deep(*arena, Optional<MutationRef>());
+					}
 				}
 			}
 
@@ -374,6 +404,7 @@ void decodeBackupLogValue(Arena& arena,
 		    .detail("Value", value);
 		throw;
 	}
+	return Void();
 }
 
 static double lastErrorTime = 0;
@@ -414,7 +445,7 @@ ACTOR Future<Void> readCommitted(Database cx,
 	loop {
 		try {
 			state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
-			                            (g_network->isSimulated() && !g_simulator.speedUpSimulation)
+			                            (g_network->isSimulated() && !g_simulator->speedUpSimulation)
 			                                ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
 			                                : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
 
@@ -493,7 +524,7 @@ ACTOR Future<Void> readCommitted(Database cx,
 	loop {
 		try {
 			state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
-			                            (g_network->isSimulated() && !g_simulator.speedUpSimulation)
+			                            (g_network->isSimulated() && !g_simulator->speedUpSimulation)
 			                                ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
 			                                : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
 
@@ -614,21 +645,24 @@ ACTOR Future<int> dumpData(Database cx,
 		state int mutationSize = 0;
 		loop {
 			try {
-				RCGroup group = waitNext(results.getFuture());
+				state RCGroup group = waitNext(results.getFuture());
 				lock->release(group.items.expectedSize());
 
 				BinaryWriter bw(Unversioned());
 				for (int i = 0; i < group.items.size(); ++i) {
 					bw.serializeBytes(group.items[i].value);
 				}
-				decodeBackupLogValue(req.arena,
-				                     req.transaction.mutations,
-				                     mutationSize,
-				                     bw.toValue(),
-				                     addPrefix,
-				                     removePrefix,
-				                     group.groupKey,
-				                     keyVersion);
+				Standalone<StringRef> value = bw.toValue();
+				wait(decodeBackupLogValue(&req.arena,
+				                          &req.transaction.mutations,
+				                          &req.transaction.encryptedMutations,
+				                          &mutationSize,
+				                          value,
+				                          addPrefix,
+				                          removePrefix,
+				                          group.groupKey,
+				                          keyVersion,
+				                          cx));
 				newBeginVersion = group.groupKey + 1;
 				if (mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
 					break;
@@ -652,8 +686,10 @@ ACTOR Future<int> dumpData(Database cx,
 		Key rangeEnd = getApplyKey(newBeginVersion, uid);
 
 		req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::SetValue, applyBegin, versionKey));
+		req.transaction.encryptedMutations.push_back_deep(req.arena, Optional<MutationRef>());
 		req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(applyBegin));
 		req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::ClearRange, rangeBegin, rangeEnd));
+		req.transaction.encryptedMutations.push_back_deep(req.arena, Optional<MutationRef>());
 		req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(rangeBegin));
 
 		// The commit request contains no read conflict ranges, so regardless of what read version we
@@ -968,10 +1004,9 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
 					                                                       .get(BackupAgentBase::keySourceStates)
 					                                                       .get(currLogUid)
 					                                                       .pack(DatabaseBackupAgent::keyStateStatus));
-					state Future<Optional<Value>> foundBackupKey =
-					    tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/"))
-					                         .withPrefix(fileBackupPrefixRange.begin))
-					                .pack(LiteralStringRef("stateEnum")));
+					state Future<Optional<Value>> foundBackupKey = tr->get(
+					    Subspace(currLogUid.withPrefix("uid->config/"_sr).withPrefix(fileBackupPrefixRange.begin))
+					        .pack("stateEnum"_sr));
 					wait(success(foundDRKey) && success(foundBackupKey));
 
 					if (foundDRKey.get().present() && foundBackupKey.get().present()) {
@@ -1165,3 +1200,38 @@ Standalone<StringRef> BackupAgentBase::getCurrentTime() {
 }
 
 std::string const BackupAgentBase::defaultTagName = "default";
+
+void addDefaultBackupRanges(Standalone<VectorRef<KeyRangeRef>>& backupKeys) {
+	backupKeys.push_back_deep(backupKeys.arena(), normalKeys);
+
+	for (auto& r : getSystemBackupRanges()) {
+		backupKeys.push_back_deep(backupKeys.arena(), r);
+	}
+}
+
+VectorRef<KeyRangeRef> const& getSystemBackupRanges() {
+	static Standalone<VectorRef<KeyRangeRef>> systemBackupRanges;
+	if (systemBackupRanges.empty()) {
+		systemBackupRanges.push_back_deep(systemBackupRanges.arena(), prefixRange(TenantMetadata::subspace()));
+		systemBackupRanges.push_back_deep(systemBackupRanges.arena(),
+		                                  singleKeyRange(MetaclusterMetadata::metaclusterRegistration().key));
+	}
+
+	return systemBackupRanges;
+}
+
+KeyRangeMap<bool> const& systemBackupMutationMask() {
+	static KeyRangeMap<bool> mask;
+	if (mask.size() == 1) {
+		for (auto r : getSystemBackupRanges()) {
+			mask.insert(r, true);
+		}
+	}
+
+	return mask;
+}
+
+KeyRangeRef const& getDefaultBackupSharedRange() {
+	static KeyRangeRef defaultSharedRange(""_sr, ""_sr);
+	return defaultSharedRange;
+}
diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 9cf2d0c0c3..c318a6591d 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -288,11 +288,46 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 #ifdef BUILD_AZURE_BACKUP
 		else if (u.startsWith("azure://"_sr)) {
 			u.eat("azure://"_sr);
-			auto accountName = u.eat("@"_sr).toString();
-			auto endpoint = u.eat("/"_sr).toString();
-			auto containerName = u.eat("/"_sr).toString();
-			r = makeReference<BackupContainerAzureBlobStore>(
-			    endpoint, accountName, containerName, encryptionKeyFileName);
+			auto address = u.eat("/"_sr);
+			if (address.endsWith(std::string(azure::storage_lite::constants::default_endpoint_suffix))) {
+				CODE_PROBE(true, "Azure backup url with standard azure storage account endpoint");
+				// <account>.<service>.core.windows.net/<resource_path>
+				auto endPoint = address.toString();
+				auto accountName = address.eat("."_sr).toString();
+				auto containerName = u.eat("/"_sr).toString();
+				r = makeReference<BackupContainerAzureBlobStore>(
+				    endPoint, accountName, containerName, encryptionKeyFileName);
+			} else {
+				// resolve the network address if necessary
+				std::string endpoint(address.toString());
+				Optional<NetworkAddress> parsedAddress = NetworkAddress::parseOptional(endpoint);
+				if (!parsedAddress.present()) {
+					try {
+						auto hostname = Hostname::parse(endpoint);
+						auto resolvedAddress = hostname.resolveBlocking();
+						if (resolvedAddress.present()) {
+							CODE_PROBE(true, "Azure backup url with hostname in the endpoint");
+							parsedAddress = resolvedAddress.get();
+						}
+					} catch (Error& e) {
+						TraceEvent(SevError, "InvalidAzureBackupUrl").error(e).detail("Endpoint", endpoint);
+						throw backup_invalid_url();
+					}
+				}
+				if (!parsedAddress.present()) {
+					TraceEvent(SevError, "InvalidAzureBackupUrl").detail("Endpoint", endpoint);
+					throw backup_invalid_url();
+				}
+				auto accountName = u.eat("/"_sr).toString();
+				// Avoid including ":tls" and "(fromHostname)"
+				// note: the endpoint needs to contain the account name
+				// so either "<account_name>.blob.core.windows.net" or "<ip>:<port>/<account_name>"
+				endpoint =
+				    fmt::format("{}/{}", formatIpPort(parsedAddress.get().ip, parsedAddress.get().port), accountName);
+				auto containerName = u.eat("/"_sr).toString();
+				r = makeReference<BackupContainerAzureBlobStore>(
+				    endpoint, accountName, containerName, encryptionKeyFileName);
+			}
 		}
 #endif
 		else {
@@ -393,21 +428,21 @@ ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Databa
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			state std::vector<std::pair<int64_t, Version>> results =
+			state KeyBackedRangeResult<std::pair<int64_t, Version>> rangeResult =
 			    wait(versionMap.getRange(tr, 0, time, 1, Snapshot::False, Reverse::True));
-			if (results.size() != 1) {
+			if (rangeResult.results.size() != 1) {
 				// No key less than time was found in the database
 				// Look for a key >= time.
-				wait(store(results, versionMap.getRange(tr, time, std::numeric_limits<int64_t>::max(), 1)));
+				wait(store(rangeResult, versionMap.getRange(tr, time, std::numeric_limits<int64_t>::max(), 1)));
 
-				if (results.size() != 1) {
+				if (rangeResult.results.size() != 1) {
 					fprintf(stderr, "ERROR: Unable to calculate a version for given date/time.\n");
 					throw backup_error();
 				}
 			}
 
 			// Adjust version found by the delta between time and the time found and min with 0.
-			auto& result = results[0];
+			auto& result = rangeResult.results[0];
 			return std::max<Version>(0, result.second + (time - result.first) * CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
 
 		} catch (Error& e) {
@@ -432,21 +467,21 @@ ACTOR Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version v, Reference
 		mid = (min + max + 1) / 2; // ceiling
 
 		// Find the highest time < mid
-		state std::vector<std::pair<int64_t, Version>> results =
+		state KeyBackedRangeResult<std::pair<int64_t, Version>> rangeResult =
 		    wait(versionMap.getRange(tr, min, mid, 1, Snapshot::False, Reverse::True));
 
-		if (results.size() != 1) {
+		if (rangeResult.results.size() != 1) {
 			if (mid == min) {
 				// There aren't any records having a version < v, so just look for any record having a time < now
 				// and base a result on it
-				wait(store(results, versionMap.getRange(tr, 0, (int64_t)now(), 1)));
+				wait(store(rangeResult, versionMap.getRange(tr, 0, (int64_t)now(), 1)));
 
-				if (results.size() != 1) {
+				if (rangeResult.results.size() != 1) {
 					// There aren't any timekeeper records to base a result on so return nothing
 					return Optional<int64_t>();
 				}
 
-				found = results[0];
+				found = rangeResult.results[0];
 				break;
 			}
 
@@ -454,7 +489,7 @@ ACTOR Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version v, Reference
 			continue;
 		}
 
-		found = results[0];
+		found = rangeResult.results[0];
 
 		if (v < found.second) {
 			max = found.first;
diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp
index b413bae0c8..5ad037d993 100644
--- a/fdbclient/BackupContainerFileSystem.actor.cpp
+++ b/fdbclient/BackupContainerFileSystem.actor.cpp
@@ -906,6 +906,7 @@ public:
 	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet(Reference<BackupContainerFileSystem> bc,
 	                                                               Version targetVersion,
 	                                                               VectorRef<KeyRangeRef> keyRangesFilter,
+	                                                               Optional<Database> cx,
 	                                                               bool logsOnly = false,
 	                                                               Version beginVersion = invalidVersion) {
 		for (const auto& range : keyRangesFilter) {
@@ -982,7 +983,7 @@ public:
 					                       restorable.ranges.end(),
 					                       [file = rit->first](const RangeFile f) { return f.fileName == file; });
 					ASSERT(it != restorable.ranges.end());
-					KeyRange result = wait(bc->getSnapshotFileKeyRange(*it));
+					KeyRange result = wait(bc->getSnapshotFileKeyRange(*it, cx));
 					ASSERT(rit->second.begin <= result.begin && rit->second.end >= result.end);
 				}
 			}
@@ -1349,7 +1350,9 @@ Future<Void> BackupContainerFileSystem::expireData(Version expireEndVersion,
 	    Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
 }
 
-ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem> bc, RangeFile file) {
+ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem> bc,
+                                                           RangeFile file,
+                                                           Optional<Database> cx) {
 	state int readFileRetries = 0;
 	state bool beginKeySet = false;
 	state Key beginKey;
@@ -1361,7 +1364,8 @@ ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupConta
 			state int64_t j = 0;
 			for (; j < file.fileSize; j += file.blockSize) {
 				int64_t len = std::min<int64_t>(file.blockSize, file.fileSize - j);
-				Standalone<VectorRef<KeyValueRef>> blockData = wait(fileBackup::decodeRangeFileBlock(inFile, j, len));
+				Standalone<VectorRef<KeyValueRef>> blockData =
+				    wait(fileBackup::decodeRangeFileBlock(inFile, j, len, cx));
 				if (!beginKeySet) {
 					beginKey = blockData.front().key;
 					beginKeySet = true;
@@ -1434,17 +1438,18 @@ ACTOR static Future<Optional<Version>> readVersionProperty(Reference<BackupConta
 	}
 }
 
-Future<KeyRange> BackupContainerFileSystem::getSnapshotFileKeyRange(const RangeFile& file) {
+Future<KeyRange> BackupContainerFileSystem::getSnapshotFileKeyRange(const RangeFile& file, Optional<Database> cx) {
 	ASSERT(g_network->isSimulated());
-	return getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem>::addRef(this), file);
+	return getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem>::addRef(this), file, cx);
 }
 
 Future<Optional<RestorableFileSet>> BackupContainerFileSystem::getRestoreSet(Version targetVersion,
+                                                                             Optional<Database> cx,
                                                                              VectorRef<KeyRangeRef> keyRangesFilter,
                                                                              bool logsOnly,
                                                                              Version beginVersion) {
 	return BackupContainerFileSystemImpl::getRestoreSet(
-	    Reference<BackupContainerFileSystem>::addRef(this), targetVersion, keyRangesFilter, logsOnly, beginVersion);
+	    Reference<BackupContainerFileSystem>::addRef(this), targetVersion, keyRangesFilter, cx, logsOnly, beginVersion);
 }
 
 Future<Optional<Version>> BackupContainerFileSystem::VersionProperty::get() {
@@ -1523,11 +1528,46 @@ Reference<BackupContainerFileSystem> BackupContainerFileSystem::openContainerFS(
 #ifdef BUILD_AZURE_BACKUP
 		else if (u.startsWith("azure://"_sr)) {
 			u.eat("azure://"_sr);
-			auto accountName = u.eat("@"_sr).toString();
-			auto endpoint = u.eat("/"_sr).toString();
-			auto containerName = u.eat("/"_sr).toString();
-			r = makeReference<BackupContainerAzureBlobStore>(
-			    endpoint, accountName, containerName, encryptionKeyFileName);
+			auto address = u.eat("/"_sr);
+			if (address.endsWith(std::string(azure::storage_lite::constants::default_endpoint_suffix))) {
+				CODE_PROBE(true, "Azure backup url with standard azure storage account endpoint");
+				// <account>.<service>.core.windows.net/<resource_path>
+				auto endPoint = address.toString();
+				auto accountName = address.eat("."_sr).toString();
+				auto containerName = u.eat("/"_sr).toString();
+				r = makeReference<BackupContainerAzureBlobStore>(
+				    endPoint, accountName, containerName, encryptionKeyFileName);
+			} else {
+				// resolve the network address if necessary
+				std::string endpoint(address.toString());
+				Optional<NetworkAddress> parsedAddress = NetworkAddress::parseOptional(endpoint);
+				if (!parsedAddress.present()) {
+					try {
+						auto hostname = Hostname::parse(endpoint);
+						auto resolvedAddress = hostname.resolveBlocking();
+						if (resolvedAddress.present()) {
+							CODE_PROBE(true, "Azure backup url with hostname in the endpoint");
+							parsedAddress = resolvedAddress.get();
+						}
+					} catch (Error& e) {
+						TraceEvent(SevError, "InvalidAzureBackupUrl").error(e).detail("Endpoint", endpoint);
+						throw backup_invalid_url();
+					}
+				}
+				if (!parsedAddress.present()) {
+					TraceEvent(SevError, "InvalidAzureBackupUrl").detail("Endpoint", endpoint);
+					throw backup_invalid_url();
+				}
+				auto accountName = u.eat("/"_sr).toString();
+				// Avoid including ":tls" and "(fromHostname)"
+				// note: the endpoint needs to contain the account name
+				// so either "<account_name>.blob.core.windows.net" or "<ip>:<port>/<account_name>"
+				endpoint =
+				    fmt::format("{}/{}", formatIpPort(parsedAddress.get().ip, parsedAddress.get().port), accountName);
+				auto containerName = u.eat("/"_sr).toString();
+				r = makeReference<BackupContainerAzureBlobStore>(
+				    endpoint, accountName, containerName, encryptionKeyFileName);
+			}
 		}
 #endif
 		else {
@@ -1631,7 +1671,8 @@ ACTOR static Future<Void> testWriteSnapshotFile(Reference<IBackupFile> file, Key
 
 ACTOR Future<Void> testBackupContainer(std::string url,
                                        Optional<std::string> proxy,
-                                       Optional<std::string> encryptionKeyFileName) {
+                                       Optional<std::string> encryptionKeyFileName,
+                                       Optional<Database> cx) {
 	state FlowLock lock(100e6);
 
 	if (encryptionKeyFileName.present()) {
@@ -1662,7 +1703,7 @@ ACTOR Future<Void> testBackupContainer(std::string url,
 
 	// List of sizes to use to test edge cases on underlying file implementations
 	state std::vector<int> fileSizes = { 0 };
-	if (StringRef(url).startsWith(LiteralStringRef("blob"))) {
+	if (StringRef(url).startsWith("blob"_sr)) {
 		fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE);
 		fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE + 10);
 	}
@@ -1670,8 +1711,8 @@ ACTOR Future<Void> testBackupContainer(std::string url,
 	loop {
 		state Version logStart = v;
 		state int kvfiles = deterministicRandom()->randomInt(0, 3);
-		state Key begin = LiteralStringRef("");
-		state Key end = LiteralStringRef("");
+		state Key begin = ""_sr;
+		state Key end = ""_sr;
 		state int blockSize = 3 * sizeof(uint32_t) + begin.size() + end.size() + 8;
 
 		while (kvfiles > 0) {
@@ -1738,13 +1779,13 @@ ACTOR Future<Void> testBackupContainer(std::string url,
 	for (; i < listing.snapshots.size(); ++i) {
 		{
 			// Ensure we can still restore to the latest version
-			Optional<RestorableFileSet> rest = wait(c->getRestoreSet(desc.maxRestorableVersion.get()));
+			Optional<RestorableFileSet> rest = wait(c->getRestoreSet(desc.maxRestorableVersion.get(), cx));
 			ASSERT(rest.present());
 		}
 
 		{
 			// Ensure we can restore to the end version of snapshot i
-			Optional<RestorableFileSet> rest = wait(c->getRestoreSet(listing.snapshots[i].endVersion));
+			Optional<RestorableFileSet> rest = wait(c->getRestoreSet(listing.snapshots[i].endVersion, cx));
 			ASSERT(rest.present());
 		}
 
@@ -1785,14 +1826,16 @@ ACTOR Future<Void> testBackupContainer(std::string url,
 }
 
 TEST_CASE("/backup/containers/localdir/unencrypted") {
-	wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {}, {}));
+	wait(testBackupContainer(
+	    format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {}, {}, {}));
 	return Void();
 }
 
 TEST_CASE("/backup/containers/localdir/encrypted") {
 	wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()),
 	                         {},
-	                         format("%s/test_encryption_key", params.getDataDir().c_str())));
+	                         format("%s/test_encryption_key", params.getDataDir().c_str()),
+	                         {}));
 	return Void();
 }
 
@@ -1800,7 +1843,7 @@ TEST_CASE("/backup/containers/url") {
 	if (!g_network->isSimulated()) {
 		const char* url = getenv("FDB_TEST_BACKUP_URL");
 		ASSERT(url != nullptr);
-		wait(testBackupContainer(url, {}, {}));
+		wait(testBackupContainer(url, {}, {}, {}));
 	}
 	return Void();
 }
diff --git a/fdbclient/BackupContainerLocalDirectory.actor.cpp b/fdbclient/BackupContainerLocalDirectory.actor.cpp
index 528910dabc..51abc24678 100644
--- a/fdbclient/BackupContainerLocalDirectory.actor.cpp
+++ b/fdbclient/BackupContainerLocalDirectory.actor.cpp
@@ -103,16 +103,15 @@ ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles_impl(st
 	// Remove .lnk files from results, they are a side effect of a backup that was *read* during simulation.  See
 	// openFile() above for more info on why they are created.
 	if (g_network->isSimulated())
-		files.erase(
-		    std::remove_if(files.begin(),
-		                   files.end(),
-		                   [](std::string const& f) { return StringRef(f).endsWith(LiteralStringRef(".lnk")); }),
-		    files.end());
+		files.erase(std::remove_if(files.begin(),
+		                           files.end(),
+		                           [](std::string const& f) { return StringRef(f).endsWith(".lnk"_sr); }),
+		            files.end());
 
 	for (const auto& f : files) {
 		// Hide .part or .temp files.
 		StringRef s(f);
-		if (!s.endsWith(LiteralStringRef(".part")) && !s.endsWith(LiteralStringRef(".temp")))
+		if (!s.endsWith(".part"_sr) && !s.endsWith(".temp"_sr))
 			results.push_back({ f.substr(m_path.size() + 1), ::fileSize(f) });
 	}
 
@@ -227,10 +226,10 @@ Future<Reference<IAsyncFile>> BackupContainerLocalDirectory::readFile(const std:
 			throw file_not_found();
 		}
 
-		if (g_simulator.getCurrentProcess()->uid == UID()) {
+		if (g_simulator->getCurrentProcess()->uid == UID()) {
 			TraceEvent(SevError, "BackupContainerReadFileOnUnsetProcessID").log();
 		}
-		std::string uniquePath = fullPath + "." + g_simulator.getCurrentProcess()->uid.toString() + ".lnk";
+		std::string uniquePath = fullPath + "." + g_simulator->getCurrentProcess()->uid.toString() + ".lnk";
 		unlink(uniquePath.c_str());
 		ASSERT(symlink(basename(path).c_str(), uniquePath.c_str()) == 0);
 		fullPath = uniquePath;
diff --git a/fdbclient/BlobCipher.cpp b/fdbclient/BlobCipher.cpp
new file mode 100644
index 0000000000..09a8bbd5ee
--- /dev/null
+++ b/fdbclient/BlobCipher.cpp
@@ -0,0 +1,1782 @@
+/*
+ * BlobCipher.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/BlobCipher.h"
+
+#include "fdbclient/Knobs.h"
+#include "flow/Arena.h"
+#include "flow/EncryptUtils.h"
+#include "flow/Knobs.h"
+#include "flow/Error.h"
+#include "flow/FastRef.h"
+#include "flow/IRandom.h"
+#include "flow/ITrace.h"
+#include "flow/Platform.h"
+#include "flow/flow.h"
+#include "flow/network.h"
+#include "flow/Trace.h"
+#include "flow/UnitTest.h"
+
+#include <chrono>
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <string>
+#include <thread>
+#include <utility>
+
+#ifndef _WIN32
+#include <unistd.h>
+#else
+#include <io.h>
+#endif
+
+#define BLOB_CIPHER_DEBUG false
+
+// BlobCipherMetrics methods
+
+BlobCipherMetrics::CounterSet::CounterSet(CounterCollection& cc, std::string name)
+  : encryptCPUTimeNS(name + "EncryptCPUTimeNS", cc), decryptCPUTimeNS(name + "DecryptCPUTimeNS", cc),
+    getCipherKeysLatency(name + "GetCipherKeysLatency",
+                         UID(),
+                         FLOW_KNOBS->ENCRYPT_KEY_CACHE_LOGGING_INTERVAL,
+                         FLOW_KNOBS->ENCRYPT_KEY_CACHE_LOGGING_SAMPLE_SIZE),
+    getLatestCipherKeysLatency(name + "GetLatestCipherKeysLatency",
+                               UID(),
+                               FLOW_KNOBS->ENCRYPT_KEY_CACHE_LOGGING_INTERVAL,
+                               FLOW_KNOBS->ENCRYPT_KEY_CACHE_LOGGING_SAMPLE_SIZE) {}
+
+BlobCipherMetrics::BlobCipherMetrics()
+  : cc("BlobCipher"), cipherKeyCacheHit("CipherKeyCacheHit", cc), cipherKeyCacheMiss("CipherKeyCacheMiss", cc),
+    cipherKeyCacheExpired("CipherKeyCacheExpired", cc), latestCipherKeyCacheHit("LatestCipherKeyCacheHit", cc),
+    latestCipherKeyCacheMiss("LatestCipherKeyCacheMiss", cc),
+    latestCipherKeyCacheNeedsRefresh("LatestCipherKeyCacheNeedsRefresh", cc),
+    getCipherKeysLatency("GetCipherKeysLatency",
+                         UID(),
+                         FLOW_KNOBS->ENCRYPT_KEY_CACHE_LOGGING_INTERVAL,
+                         FLOW_KNOBS->ENCRYPT_KEY_CACHE_LOGGING_SAMPLE_SIZE),
+    getLatestCipherKeysLatency("GetLatestCipherKeysLatency",
+                               UID(),
+                               FLOW_KNOBS->ENCRYPT_KEY_CACHE_LOGGING_INTERVAL,
+                               FLOW_KNOBS->ENCRYPT_KEY_CACHE_LOGGING_SAMPLE_SIZE),
+    counterSets({ CounterSet(cc, "TLog"),
+                  CounterSet(cc, "KVMemory"),
+                  CounterSet(cc, "KVRedwood"),
+                  CounterSet(cc, "BlobGranule"),
+                  CounterSet(cc, "Backup"),
+                  CounterSet(cc, "Test") }) {
+	specialCounter(cc, "CacheSize", []() { return BlobCipherKeyCache::getInstance()->getSize(); });
+	traceFuture = traceCounters("BlobCipherMetrics", UID(), FLOW_KNOBS->ENCRYPT_KEY_CACHE_LOGGING_INTERVAL, &cc);
+}
+
+// BlobCipherKey class methods
+
+BlobCipherKey::BlobCipherKey(const EncryptCipherDomainId& domainId,
+                             const EncryptCipherBaseKeyId& baseCiphId,
+                             const uint8_t* baseCiph,
+                             int baseCiphLen,
+                             const int64_t refreshAt,
+                             const int64_t expireAt) {
+	// Salt generated is used while applying HMAC Key derivation, hence, not using crypto-secure hash algorithm is ok.
+	// Further, 'deterministic' salt generation is used to preserve simulation determinism properties.
+	EncryptCipherRandomSalt salt;
+	if (g_network->isSimulated()) {
+		salt = deterministicRandom()->randomUInt64();
+	} else {
+		salt = nondeterministicRandom()->randomUInt64();
+	}
+
+	// Support two type of CipherKeys: 'revocable' & 'non-revocable' ciphers.
+	// In all cases, either cipherKey never expires i.e. refreshAt == infinite, or, refreshAt needs <= expireAt
+	// timestamp.
+	ASSERT(refreshAt == std::numeric_limits<int64_t>::max() || (refreshAt <= expireAt));
+
+	initKey(domainId, baseCiph, baseCiphLen, baseCiphId, salt, refreshAt, expireAt);
+}
+
+BlobCipherKey::BlobCipherKey(const EncryptCipherDomainId& domainId,
+                             const EncryptCipherBaseKeyId& baseCiphId,
+                             const uint8_t* baseCiph,
+                             int baseCiphLen,
+                             const EncryptCipherRandomSalt& salt,
+                             const int64_t refreshAt,
+                             const int64_t expireAt) {
+	initKey(domainId, baseCiph, baseCiphLen, baseCiphId, salt, refreshAt, expireAt);
+}
+
+void BlobCipherKey::initKey(const EncryptCipherDomainId& domainId,
+                            const uint8_t* baseCiph,
+                            int baseCiphLen,
+                            const EncryptCipherBaseKeyId& baseCiphId,
+                            const EncryptCipherRandomSalt& salt,
+                            const int64_t refreshAt,
+                            const int64_t expireAt) {
+	// Set the base encryption key properties
+	baseCipher = std::make_unique<uint8_t[]>(AES_256_KEY_LENGTH);
+	memset(baseCipher.get(), 0, AES_256_KEY_LENGTH);
+	memcpy(baseCipher.get(), baseCiph, std::min<int>(baseCiphLen, AES_256_KEY_LENGTH));
+	baseCipherLen = baseCiphLen;
+	baseCipherId = baseCiphId;
+	// Set the encryption domain for the base encryption key
+	encryptDomainId = domainId;
+	randomSalt = salt;
+	// derive the encryption key
+	cipher = std::make_unique<uint8_t[]>(AES_256_KEY_LENGTH);
+	memset(cipher.get(), 0, AES_256_KEY_LENGTH);
+	applyHmacSha256Derivation();
+	// update cipher 'refresh' and 'expire' TS
+	refreshAtTS = refreshAt;
+	expireAtTS = expireAt;
+
+#if BLOB_CIPHER_DEBUG
+	TraceEvent(SevDebug, "BlobCipherKeyInit")
+	    .detail("DomainId", domainId)
+	    .detail("BaseCipherId", baseCipherId)
+	    .detail("BaseCipherLen", baseCipherLen)
+	    .detail("RandomSalt", randomSalt)
+	    .detail("RefreshAt", refreshAtTS)
+	    .detail("ExpireAtTS", expireAtTS);
+#endif
+}
+
+void BlobCipherKey::applyHmacSha256Derivation() {
+	Arena arena;
+	uint8_t buf[baseCipherLen + sizeof(EncryptCipherRandomSalt)];
+	memcpy(&buf[0], baseCipher.get(), baseCipherLen);
+	memcpy(&buf[0] + baseCipherLen, &randomSalt, sizeof(EncryptCipherRandomSalt));
+	HmacSha256DigestGen hmacGen(baseCipher.get(), baseCipherLen);
+	unsigned int digestLen = hmacGen.digest(
+	    { { &buf[0], baseCipherLen + sizeof(EncryptCipherRandomSalt) } }, cipher.get(), AUTH_TOKEN_HMAC_SHA_SIZE);
+	if (digestLen < AES_256_KEY_LENGTH) {
+		memcpy(cipher.get() + digestLen, buf, AES_256_KEY_LENGTH - digestLen);
+	}
+}
+
+void BlobCipherKey::reset() {
+	memset(baseCipher.get(), 0, baseCipherLen);
+	memset(cipher.get(), 0, AES_256_KEY_LENGTH);
+}
+
+// BlobKeyIdCache class methods
+
+BlobCipherKeyIdCache::BlobCipherKeyIdCache(EncryptCipherDomainId dId, size_t* sizeStat)
+  : domainId(dId), latestBaseCipherKeyId(), latestRandomSalt(), sizeStat(sizeStat) {
+	ASSERT(sizeStat != nullptr);
+	TraceEvent(SevInfo, "BlobCipherKeyIdCacheInit").detail("DomainId", domainId);
+}
+
+BlobCipherKeyIdCacheKey BlobCipherKeyIdCache::getCacheKey(const EncryptCipherBaseKeyId& baseCipherKeyId,
+                                                          const EncryptCipherRandomSalt& salt) {
+	if (baseCipherKeyId == INVALID_ENCRYPT_CIPHER_KEY_ID || salt == INVALID_ENCRYPT_RANDOM_SALT) {
+		throw encrypt_invalid_id();
+	}
+	return std::make_pair(baseCipherKeyId, salt);
+}
+
+Reference<BlobCipherKey> BlobCipherKeyIdCache::getLatestCipherKey() {
+	if (!latestBaseCipherKeyId.present()) {
+		return Reference<BlobCipherKey>();
+	}
+	ASSERT_NE(latestBaseCipherKeyId.get(), INVALID_ENCRYPT_CIPHER_KEY_ID);
+	ASSERT(latestRandomSalt.present());
+	ASSERT_NE(latestRandomSalt.get(), INVALID_ENCRYPT_RANDOM_SALT);
+
+	return getCipherByBaseCipherId(latestBaseCipherKeyId.get(), latestRandomSalt.get());
+}
+
+Reference<BlobCipherKey> BlobCipherKeyIdCache::getCipherByBaseCipherId(const EncryptCipherBaseKeyId& baseCipherKeyId,
+                                                                       const EncryptCipherRandomSalt& salt) {
+	BlobCipherKeyIdCacheMapCItr itr = keyIdCache.find(getCacheKey(baseCipherKeyId, salt));
+	if (itr == keyIdCache.end()) {
+		return Reference<BlobCipherKey>();
+	}
+	return itr->second;
+}
+
+Reference<BlobCipherKey> BlobCipherKeyIdCache::insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId,
+                                                                   const uint8_t* baseCipher,
+                                                                   int baseCipherLen,
+                                                                   const int64_t refreshAt,
+                                                                   const int64_t expireAt) {
+	ASSERT_GT(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID);
+
+	// BaseCipherKeys are immutable, given the routine invocation updates 'latestCipher',
+	// ensure no key-tampering is done
+	Reference<BlobCipherKey> latestCipherKey = getLatestCipherKey();
+	if (latestCipherKey.isValid() && latestCipherKey->getBaseCipherId() == baseCipherId) {
+		if (memcmp(latestCipherKey->rawBaseCipher(), baseCipher, baseCipherLen) == 0) {
+#if BLOB_CIPHER_DEBUG
+			TraceEvent(SevDebug, "InsertBaseCipherKeyAlreadyPresent")
+			    .detail("BaseCipherKeyId", baseCipherId)
+			    .detail("DomainId", domainId);
+#endif
+
+			// Key is already present; nothing more to do.
+			return latestCipherKey;
+		} else {
+			TraceEvent(SevInfo, "BlobCipherUpdatetBaseCipherKey")
+			    .detail("BaseCipherKeyId", baseCipherId)
+			    .detail("DomainId", domainId);
+			throw encrypt_update_cipher();
+		}
+	}
+
+	TraceEvent(SevInfo, "BlobCipherKeyInsertBaseCipherKeyLatest")
+	    .detail("DomainId", domainId)
+	    .detail("BaseCipherId", baseCipherId)
+	    .detail("RefreshAt", refreshAt)
+	    .detail("ExpireAt", expireAt);
+
+	Reference<BlobCipherKey> cipherKey =
+	    makeReference<BlobCipherKey>(domainId, baseCipherId, baseCipher, baseCipherLen, refreshAt, expireAt);
+	BlobCipherKeyIdCacheKey cacheKey = getCacheKey(cipherKey->getBaseCipherId(), cipherKey->getSalt());
+	keyIdCache.emplace(cacheKey, cipherKey);
+
+	// Update the latest BaseCipherKeyId for the given encryption domain
+	latestBaseCipherKeyId = baseCipherId;
+	latestRandomSalt = cipherKey->getSalt();
+
+	(*sizeStat)++;
+	return cipherKey;
+}
+
+Reference<BlobCipherKey> BlobCipherKeyIdCache::insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId,
+                                                                   const uint8_t* baseCipher,
+                                                                   int baseCipherLen,
+                                                                   const EncryptCipherRandomSalt& salt,
+                                                                   const int64_t refreshAt,
+                                                                   const int64_t expireAt) {
+	ASSERT_NE(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID);
+	ASSERT_NE(salt, INVALID_ENCRYPT_RANDOM_SALT);
+
+	BlobCipherKeyIdCacheKey cacheKey = getCacheKey(baseCipherId, salt);
+
+	// BaseCipherKeys are immutable, ensure that cached value doesn't get updated.
+	BlobCipherKeyIdCacheMapCItr itr = keyIdCache.find(cacheKey);
+	if (itr != keyIdCache.end()) {
+		if (memcmp(itr->second->rawBaseCipher(), baseCipher, baseCipherLen) == 0) {
+#if BLOB_CIPHER_DEBUG
+			TraceEvent(SevDebug, "InsertBaseCipherKeyAlreadyPresent")
+			    .detail("BaseCipherKeyId", baseCipherId)
+			    .detail("DomainId", domainId);
+#endif
+
+			// Key is already present; nothing more to do.
+			return itr->second;
+		} else {
+			TraceEvent(SevInfo, "BlobCipherUpdateBaseCipherKey")
+			    .detail("BaseCipherKeyId", baseCipherId)
+			    .detail("DomainId", domainId);
+			throw encrypt_update_cipher();
+		}
+	}
+
+	TraceEvent(SevInfo, "BlobCipherKeyInsertBaseCipherKey")
+	    .detail("DomainId", domainId)
+	    .detail("BaseCipherId", baseCipherId)
+	    .detail("Salt", salt)
+	    .detail("RefreshAt", refreshAt)
+	    .detail("ExpireAt", expireAt);
+
+	Reference<BlobCipherKey> cipherKey =
+	    makeReference<BlobCipherKey>(domainId, baseCipherId, baseCipher, baseCipherLen, salt, refreshAt, expireAt);
+	keyIdCache.emplace(cacheKey, cipherKey);
+	(*sizeStat)++;
+	return cipherKey;
+}
+
+void BlobCipherKeyIdCache::cleanup() {
+	for (auto& keyItr : keyIdCache) {
+		keyItr.second->reset();
+	}
+
+	keyIdCache.clear();
+}
+
+std::vector<Reference<BlobCipherKey>> BlobCipherKeyIdCache::getAllCipherKeys() {
+	std::vector<Reference<BlobCipherKey>> cipherKeys;
+	for (auto& keyItr : keyIdCache) {
+		cipherKeys.push_back(keyItr.second);
+	}
+	return cipherKeys;
+}
+
+// BlobCipherKeyCache class methods
+
+Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipherDomainId& domainId,
+                                                             const EncryptCipherBaseKeyId& baseCipherId,
+                                                             const uint8_t* baseCipher,
+                                                             int baseCipherLen,
+                                                             const int64_t refreshAt,
+                                                             const int64_t expireAt) {
+	if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID) {
+		throw encrypt_invalid_id();
+	}
+
+	Reference<BlobCipherKey> cipherKey;
+
+	try {
+		auto domainItr = domainCacheMap.find(domainId);
+		if (domainItr == domainCacheMap.end()) {
+			// Add mapping to track new encryption domain
+			Reference<BlobCipherKeyIdCache> keyIdCache = makeReference<BlobCipherKeyIdCache>(domainId, &size);
+			cipherKey = keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen, refreshAt, expireAt);
+			domainCacheMap.emplace(domainId, keyIdCache);
+		} else {
+			// Track new baseCipher keys
+			Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
+			cipherKey = keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen, refreshAt, expireAt);
+		}
+	} catch (Error& e) {
+		TraceEvent(SevWarn, "BlobCipherInsertCipherKeyFailed")
+		    .detail("BaseCipherKeyId", baseCipherId)
+		    .detail("DomainId", domainId);
+		throw;
+	}
+	return cipherKey;
+}
+
+Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipherDomainId& domainId,
+                                                             const EncryptCipherBaseKeyId& baseCipherId,
+                                                             const uint8_t* baseCipher,
+                                                             int baseCipherLen,
+                                                             const EncryptCipherRandomSalt& salt,
+                                                             const int64_t refreshAt,
+                                                             const int64_t expireAt) {
+	if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID ||
+	    salt == INVALID_ENCRYPT_RANDOM_SALT) {
+		throw encrypt_invalid_id();
+	}
+
+	Reference<BlobCipherKey> cipherKey;
+	try {
+		auto domainItr = domainCacheMap.find(domainId);
+		if (domainItr == domainCacheMap.end()) {
+			// Add mapping to track new encryption domain
+			Reference<BlobCipherKeyIdCache> keyIdCache = makeReference<BlobCipherKeyIdCache>(domainId, &size);
+			cipherKey =
+			    keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen, salt, refreshAt, expireAt);
+			domainCacheMap.emplace(domainId, keyIdCache);
+		} else {
+			// Track new baseCipher keys
+			Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
+			cipherKey =
+			    keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen, salt, refreshAt, expireAt);
+		}
+	} catch (Error& e) {
+		TraceEvent(SevWarn, "BlobCipherInsertCipherKey_Failed")
+		    .detail("BaseCipherKeyId", baseCipherId)
+		    .detail("DomainId", domainId)
+		    .detail("Salt", salt);
+		throw;
+	}
+	return cipherKey;
+}
+
+Reference<BlobCipherKey> BlobCipherKeyCache::getLatestCipherKey(const EncryptCipherDomainId& domainId) {
+	if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
+		TraceEvent(SevWarn, "BlobCipherGetLatestCipherKeyInvalidID").detail("DomainId", domainId);
+		throw encrypt_invalid_id();
+	}
+	auto domainItr = domainCacheMap.find(domainId);
+	if (domainItr == domainCacheMap.end()) {
+		TraceEvent(SevInfo, "BlobCipherGetLatestCipherKeyDomainNotFound").detail("DomainId", domainId);
+		return Reference<BlobCipherKey>();
+	}
+
+	Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
+	Reference<BlobCipherKey> cipherKey = keyIdCache->getLatestCipherKey();
+
+	// Ensure 'freshness' guarantees for the latestCipher
+	if (cipherKey.isValid()) {
+		if (cipherKey->needsRefresh()) {
+#if BLOB_CIPHER_DEBUG
+			TraceEvent("SevDebug, BlobCipherGetLatestNeedsRefresh")
+			    .detail("DomainId", domainId)
+			    .detail("Now", now())
+			    .detail("RefreshAt", cipherKey->getRefreshAtTS());
+#endif
+			++BlobCipherMetrics::getInstance()->latestCipherKeyCacheNeedsRefresh;
+			return Reference<BlobCipherKey>();
+		}
+		++BlobCipherMetrics::getInstance()->latestCipherKeyCacheHit;
+	} else {
+		++BlobCipherMetrics::getInstance()->latestCipherKeyCacheMiss;
+	}
+
+	return cipherKey;
+}
+
+Reference<BlobCipherKey> BlobCipherKeyCache::getCipherKey(const EncryptCipherDomainId& domainId,
+                                                          const EncryptCipherBaseKeyId& baseCipherId,
+                                                          const EncryptCipherRandomSalt& salt) {
+	auto domainItr = domainCacheMap.find(domainId);
+	if (domainItr == domainCacheMap.end()) {
+		return Reference<BlobCipherKey>();
+	}
+
+	Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
+	Reference<BlobCipherKey> cipherKey = keyIdCache->getCipherByBaseCipherId(baseCipherId, salt);
+
+	// Ensure 'liveness' guarantees for the cipher
+	if (cipherKey.isValid()) {
+		if (cipherKey->isExpired()) {
+#if BLOB_CIPHER_DEBUG
+			TraceEvent(SevDebug, "BlobCipherGetCipherExpired")
+			    .detail("DomainId", domainId)
+			    .detail("BaseCipherId", baseCipherId)
+			    .detail("Now", now())
+			    .detail("ExpireAt", cipherKey->getExpireAtTS());
+#endif
+			++BlobCipherMetrics::getInstance()->cipherKeyCacheExpired;
+			return Reference<BlobCipherKey>();
+		}
+		++BlobCipherMetrics::getInstance()->cipherKeyCacheHit;
+	} else {
+		++BlobCipherMetrics::getInstance()->cipherKeyCacheMiss;
+	}
+
+	return cipherKey;
+}
+
+void BlobCipherKeyCache::resetEncryptDomainId(const EncryptCipherDomainId domainId) {
+	auto domainItr = domainCacheMap.find(domainId);
+	if (domainItr == domainCacheMap.end()) {
+		return;
+	}
+
+	Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
+	ASSERT(keyIdCache->getSize() <= size);
+	size -= keyIdCache->getSize();
+	keyIdCache->cleanup();
+	TraceEvent(SevInfo, "BlobCipherResetEncryptDomainId").detail("DomainId", domainId);
+}
+
+void BlobCipherKeyCache::cleanup() noexcept {
+	Reference<BlobCipherKeyCache> instance = BlobCipherKeyCache::getInstance();
+
+	TraceEvent(SevInfo, "BlobCipherKeyCacheCleanup").log();
+
+	for (auto& domainItr : instance->domainCacheMap) {
+		Reference<BlobCipherKeyIdCache> keyIdCache = domainItr.second;
+		keyIdCache->cleanup();
+		TraceEvent(SevInfo, "BlobCipherKeyCacheCleanup").detail("DomainId", domainItr.first);
+	}
+
+	instance->domainCacheMap.clear();
+	instance->size = 0;
+}
+
+std::vector<Reference<BlobCipherKey>> BlobCipherKeyCache::getAllCiphers(const EncryptCipherDomainId& domainId) {
+	auto domainItr = domainCacheMap.find(domainId);
+	if (domainItr == domainCacheMap.end()) {
+		return {};
+	}
+
+	Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
+	return keyIdCache->getAllCipherKeys();
+}
+
+// EncryptBlobCipherAes265Ctr class methods
+
+EncryptBlobCipherAes265Ctr::EncryptBlobCipherAes265Ctr(Reference<BlobCipherKey> tCipherKey,
+                                                       Reference<BlobCipherKey> hCipherKey,
+                                                       const uint8_t* cipherIV,
+                                                       const int ivLen,
+                                                       const EncryptAuthTokenMode mode,
+                                                       BlobCipherMetrics::UsageType usageType)
+  : ctx(EVP_CIPHER_CTX_new()), textCipherKey(tCipherKey), headerCipherKey(hCipherKey), authTokenMode(mode),
+    usageType(usageType) {
+	ASSERT_EQ(ivLen, AES_256_IV_LENGTH);
+	authTokenAlgo = getAuthTokenAlgoFromMode(authTokenMode);
+	memcpy(&iv[0], cipherIV, ivLen);
+	init();
+}
+
+EncryptBlobCipherAes265Ctr::EncryptBlobCipherAes265Ctr(Reference<BlobCipherKey> tCipherKey,
+                                                       Reference<BlobCipherKey> hCipherKey,
+                                                       const uint8_t* cipherIV,
+                                                       const int ivLen,
+                                                       const EncryptAuthTokenMode mode,
+                                                       const EncryptAuthTokenAlgo algo,
+                                                       BlobCipherMetrics::UsageType usageType)
+  : ctx(EVP_CIPHER_CTX_new()), textCipherKey(tCipherKey), headerCipherKey(hCipherKey), authTokenMode(mode),
+    authTokenAlgo(algo), usageType(usageType) {
+	ASSERT_EQ(ivLen, AES_256_IV_LENGTH);
+	memcpy(&iv[0], cipherIV, ivLen);
+	init();
+}
+
+EncryptBlobCipherAes265Ctr::EncryptBlobCipherAes265Ctr(Reference<BlobCipherKey> tCipherKey,
+                                                       Reference<BlobCipherKey> hCipherKey,
+                                                       const EncryptAuthTokenMode mode,
+                                                       BlobCipherMetrics::UsageType usageType)
+  : ctx(EVP_CIPHER_CTX_new()), textCipherKey(tCipherKey), headerCipherKey(hCipherKey), authTokenMode(mode),
+    usageType(usageType) {
+	authTokenAlgo = getAuthTokenAlgoFromMode(authTokenMode);
+	deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH);
+	init();
+}
+
+EncryptBlobCipherAes265Ctr::EncryptBlobCipherAes265Ctr(Reference<BlobCipherKey> tCipherKey,
+                                                       Reference<BlobCipherKey> hCipherKey,
+                                                       const EncryptAuthTokenMode mode,
+                                                       const EncryptAuthTokenAlgo algo,
+                                                       BlobCipherMetrics::UsageType usageType)
+  : ctx(EVP_CIPHER_CTX_new()), textCipherKey(tCipherKey), headerCipherKey(hCipherKey), authTokenMode(mode),
+    usageType(usageType) {
+	deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH);
+	init();
+}
+
+void EncryptBlobCipherAes265Ctr::init() {
+	ASSERT(textCipherKey.isValid());
+	ASSERT(headerCipherKey.isValid());
+
+	if (!isEncryptHeaderAuthTokenDetailsValid(authTokenMode, authTokenAlgo)) {
+		TraceEvent(SevWarn, "InvalidAuthTokenDetails")
+		    .detail("TokenMode", authTokenMode)
+		    .detail("TokenAlgo", authTokenAlgo);
+		throw internal_error();
+	}
+
+	if (ctx == nullptr) {
+		throw encrypt_ops_error();
+	}
+	if (EVP_EncryptInit_ex(ctx, EVP_aes_256_ctr(), nullptr, nullptr, nullptr) != 1) {
+		throw encrypt_ops_error();
+	}
+	if (EVP_EncryptInit_ex(ctx, nullptr, nullptr, textCipherKey.getPtr()->data(), iv) != 1) {
+		throw encrypt_ops_error();
+	}
+}
+
+Reference<EncryptBuf> EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plaintext,
+                                                          const int plaintextLen,
+                                                          BlobCipherEncryptHeader* header,
+                                                          Arena& arena) {
+	double startTime = 0.0;
+	if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING) {
+		startTime = timer_monotonic();
+	}
+
+	memset(reinterpret_cast<uint8_t*>(header), 0, sizeof(BlobCipherEncryptHeader));
+
+	// Alloc buffer computation accounts for 'header authentication' generation scheme. If single-auth-token needs
+	// to be generated, allocate buffer sufficient to append header to the cipherText optimizing memcpy cost.
+
+	const int allocSize = plaintextLen + AES_BLOCK_SIZE;
+	Reference<EncryptBuf> encryptBuf = makeReference<EncryptBuf>(allocSize, arena);
+	uint8_t* ciphertext = encryptBuf->begin();
+
+	int bytes{ 0 };
+	if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
+		TraceEvent(SevWarn, "BlobCipherEncryptUpdateFailed")
+		    .detail("BaseCipherId", textCipherKey->getBaseCipherId())
+		    .detail("EncryptDomainId", textCipherKey->getDomainId());
+		throw encrypt_ops_error();
+	}
+
+	int finalBytes{ 0 };
+	if (EVP_EncryptFinal_ex(ctx, ciphertext + bytes, &finalBytes) != 1) {
+		TraceEvent(SevWarn, "BlobCipherEncryptFinalFailed")
+		    .detail("BaseCipherId", textCipherKey->getBaseCipherId())
+		    .detail("EncryptDomainId", textCipherKey->getDomainId());
+		throw encrypt_ops_error();
+	}
+
+	if ((bytes + finalBytes) != plaintextLen) {
+		TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedCipherLen")
+		    .detail("PlaintextLen", plaintextLen)
+		    .detail("EncryptedBufLen", bytes + finalBytes);
+		throw encrypt_ops_error();
+	}
+
+	// Populate encryption header flags details
+	header->flags.size = sizeof(BlobCipherEncryptHeader);
+	header->flags.headerVersion = EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION;
+	header->flags.encryptMode = ENCRYPT_CIPHER_MODE_AES_256_CTR;
+	header->flags.authTokenMode = authTokenMode;
+	header->flags.authTokenAlgo = authTokenAlgo;
+
+	// Ensure encryption header authToken details sanity
+	ASSERT(isEncryptHeaderAuthTokenDetailsValid(authTokenMode, authTokenAlgo));
+
+	// Populate cipherText encryption-key details
+	header->cipherTextDetails.baseCipherId = textCipherKey->getBaseCipherId();
+	header->cipherTextDetails.encryptDomainId = textCipherKey->getDomainId();
+	header->cipherTextDetails.salt = textCipherKey->getSalt();
+	// Populate header encryption-key details
+	// TODO: HeaderCipherKey is not necessary if AuthTokenMode == NONE
+	header->cipherHeaderDetails.encryptDomainId = headerCipherKey->getDomainId();
+	header->cipherHeaderDetails.baseCipherId = headerCipherKey->getBaseCipherId();
+	header->cipherHeaderDetails.salt = headerCipherKey->getSalt();
+
+	memcpy(&header->iv[0], &iv[0], AES_256_IV_LENGTH);
+
+	if (authTokenMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
+		// No header 'authToken' generation needed.
+	} else {
+
+		// Populate header authToken details
+		if (header->flags.authTokenMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE) {
+			ASSERT_GE(allocSize, (bytes + finalBytes));
+			ASSERT_GE(encryptBuf->getLogicalSize(), (bytes + finalBytes));
+
+			computeAuthToken({ { ciphertext, bytes + finalBytes },
+			                   { reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader) } },
+			                 headerCipherKey->rawCipher(),
+			                 AES_256_KEY_LENGTH,
+			                 &header->singleAuthToken.authToken[0],
+			                 (EncryptAuthTokenAlgo)header->flags.authTokenAlgo,
+			                 AUTH_TOKEN_MAX_SIZE);
+		} else {
+			ASSERT_EQ(header->flags.authTokenMode, EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
+
+			// TOOD: Use HMAC_SHA encyrption authentication scheme as AES_CMAC needs minimum 16 bytes cipher key
+			computeAuthToken({ { ciphertext, bytes + finalBytes } },
+			                 reinterpret_cast<const uint8_t*>(&header->cipherTextDetails.salt),
+			                 sizeof(EncryptCipherRandomSalt),
+			                 &header->multiAuthTokens.cipherTextAuthToken[0],
+			                 EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
+			                 AUTH_TOKEN_MAX_SIZE);
+			computeAuthToken({ { reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader) } },
+			                 headerCipherKey->rawCipher(),
+			                 AES_256_KEY_LENGTH,
+			                 &header->multiAuthTokens.headerAuthToken[0],
+			                 (EncryptAuthTokenAlgo)header->flags.authTokenAlgo,
+			                 AUTH_TOKEN_MAX_SIZE);
+		}
+	}
+
+	encryptBuf->setLogicalSize(plaintextLen);
+
+	if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING) {
+		BlobCipherMetrics::counters(usageType).encryptCPUTimeNS += int64_t((timer_monotonic() - startTime) * 1e9);
+	}
+
+	CODE_PROBE(true, "BlobCipher data encryption");
+	CODE_PROBE(header->flags.authTokenAlgo == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE,
+	           "Encryption authentication disabled");
+	CODE_PROBE(header->flags.authTokenAlgo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
+	           "HMAC_SHA Auth token generation");
+	CODE_PROBE(header->flags.authTokenAlgo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC,
+	           "AES_CMAC Auth token generation");
+
+	return encryptBuf;
+}
+
+EncryptBlobCipherAes265Ctr::~EncryptBlobCipherAes265Ctr() {
+	if (ctx != nullptr) {
+		EVP_CIPHER_CTX_free(ctx);
+	}
+}
+
+// DecryptBlobCipherAes256Ctr class methods
+
+DecryptBlobCipherAes256Ctr::DecryptBlobCipherAes256Ctr(Reference<BlobCipherKey> tCipherKey,
+                                                       Reference<BlobCipherKey> hCipherKey,
+                                                       const uint8_t* iv,
+                                                       BlobCipherMetrics::UsageType usageType)
+  : ctx(EVP_CIPHER_CTX_new()), textCipherKey(tCipherKey), headerCipherKey(hCipherKey),
+    headerAuthTokenValidationDone(false), authTokensValidationDone(false), usageType(usageType) {
+	if (ctx == nullptr) {
+		throw encrypt_ops_error();
+	}
+	if (!EVP_DecryptInit_ex(ctx, EVP_aes_256_ctr(), nullptr, nullptr, nullptr)) {
+		throw encrypt_ops_error();
+	}
+	if (!EVP_DecryptInit_ex(ctx, nullptr, nullptr, tCipherKey.getPtr()->data(), iv)) {
+		throw encrypt_ops_error();
+	}
+}
+
+void DecryptBlobCipherAes256Ctr::verifyHeaderAuthToken(const BlobCipherEncryptHeader& header, Arena& arena) {
+	if (header.flags.authTokenMode != ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI) {
+		// NoneAuthToken mode; no authToken is generated; nothing to do
+		// SingleAuthToken mode; verification will happen as part of decryption.
+		return;
+	}
+
+	ASSERT_EQ(header.flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
+	ASSERT(isEncryptHeaderAuthTokenAlgoValid((EncryptAuthTokenAlgo)header.flags.authTokenAlgo));
+
+	BlobCipherEncryptHeader headerCopy;
+	memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+	       reinterpret_cast<const uint8_t*>(&header),
+	       sizeof(BlobCipherEncryptHeader));
+	memset(reinterpret_cast<uint8_t*>(&headerCopy.multiAuthTokens.headerAuthToken), 0, AUTH_TOKEN_MAX_SIZE);
+	uint8_t computedHeaderAuthToken[AUTH_TOKEN_MAX_SIZE]{};
+	computeAuthToken({ { reinterpret_cast<const uint8_t*>(&headerCopy), sizeof(BlobCipherEncryptHeader) } },
+	                 headerCipherKey->rawCipher(),
+	                 AES_256_KEY_LENGTH,
+	                 &computedHeaderAuthToken[0],
+	                 (EncryptAuthTokenAlgo)header.flags.authTokenAlgo,
+	                 AUTH_TOKEN_MAX_SIZE);
+
+	int authTokenSize = getEncryptHeaderAuthTokenSize(header.flags.authTokenAlgo);
+	ASSERT_LE(authTokenSize, AUTH_TOKEN_MAX_SIZE);
+	if (memcmp(&header.multiAuthTokens.headerAuthToken[0], &computedHeaderAuthToken[0], authTokenSize) != 0) {
+		TraceEvent(SevWarn, "BlobCipherVerifyEncryptBlobHeaderAuthTokenMismatch")
+		    .detail("HeaderVersion", header.flags.headerVersion)
+		    .detail("HeaderMode", header.flags.encryptMode)
+		    .detail("MultiAuthHeaderAuthToken",
+		            StringRef(arena, &header.multiAuthTokens.headerAuthToken[0], AUTH_TOKEN_MAX_SIZE).toString())
+		    .detail("ComputedHeaderAuthToken", StringRef(computedHeaderAuthToken, AUTH_TOKEN_MAX_SIZE));
+		throw encrypt_header_authtoken_mismatch();
+	}
+
+	headerAuthTokenValidationDone = true;
+}
+
+void DecryptBlobCipherAes256Ctr::verifyHeaderSingleAuthToken(const uint8_t* ciphertext,
+                                                             const int ciphertextLen,
+                                                             const BlobCipherEncryptHeader& header,
+                                                             Arena& arena) {
+	// Header authToken not set for single auth-token mode.
+	ASSERT(!headerAuthTokenValidationDone);
+
+	// prepare the payload {cipherText + encryptionHeader}
+	// ensure the 'authToken' is reset before computing the 'authentication token'
+	BlobCipherEncryptHeader headerCopy;
+	memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+	       reinterpret_cast<const uint8_t*>(&header),
+	       sizeof(BlobCipherEncryptHeader));
+	memset(reinterpret_cast<uint8_t*>(&headerCopy.singleAuthToken), 0, 2 * AUTH_TOKEN_MAX_SIZE);
+	uint8_t computed[AUTH_TOKEN_MAX_SIZE];
+	computeAuthToken({ { ciphertext, ciphertextLen },
+	                   { reinterpret_cast<const uint8_t*>(&headerCopy), sizeof(BlobCipherEncryptHeader) } },
+	                 headerCipherKey->rawCipher(),
+	                 AES_256_KEY_LENGTH,
+	                 &computed[0],
+	                 (EncryptAuthTokenAlgo)header.flags.authTokenAlgo,
+	                 AUTH_TOKEN_MAX_SIZE);
+
+	int authTokenSize = getEncryptHeaderAuthTokenSize(header.flags.authTokenAlgo);
+	ASSERT_LE(authTokenSize, AUTH_TOKEN_MAX_SIZE);
+	if (memcmp(&header.singleAuthToken.authToken[0], &computed[0], authTokenSize) != 0) {
+		TraceEvent(SevWarn, "BlobCipherVerifyEncryptBlobHeaderAuthTokenMismatch")
+		    .detail("HeaderVersion", header.flags.headerVersion)
+		    .detail("HeaderMode", header.flags.encryptMode)
+		    .detail("SingleAuthToken",
+		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_MAX_SIZE).toString())
+		    .detail("ComputedSingleAuthToken", StringRef(computed, AUTH_TOKEN_MAX_SIZE));
+		throw encrypt_header_authtoken_mismatch();
+	}
+}
+
+void DecryptBlobCipherAes256Ctr::verifyHeaderMultiAuthToken(const uint8_t* ciphertext,
+                                                            const int ciphertextLen,
+                                                            const BlobCipherEncryptHeader& header,
+                                                            Arena& arena) {
+	if (!headerAuthTokenValidationDone) {
+		verifyHeaderAuthToken(header, arena);
+	}
+	uint8_t computedCipherTextAuthToken[AUTH_TOKEN_MAX_SIZE];
+	// TOOD: Use HMAC_SHA encyrption authentication scheme as AES_CMAC needs minimum 16 bytes cipher key
+	computeAuthToken({ { ciphertext, ciphertextLen } },
+	                 reinterpret_cast<const uint8_t*>(&header.cipherTextDetails.salt),
+	                 sizeof(EncryptCipherRandomSalt),
+	                 &computedCipherTextAuthToken[0],
+	                 EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
+	                 AUTH_TOKEN_MAX_SIZE);
+	if (memcmp(&header.multiAuthTokens.cipherTextAuthToken[0], &computedCipherTextAuthToken[0], AUTH_TOKEN_MAX_SIZE) !=
+	    0) {
+		TraceEvent(SevWarn, "BlobCipherVerifyEncryptBlobHeaderAuthTokenMismatch")
+		    .detail("HeaderVersion", header.flags.headerVersion)
+		    .detail("HeaderMode", header.flags.encryptMode)
+		    .detail("MultiAuthCipherTextAuthToken",
+		            StringRef(arena, &header.multiAuthTokens.cipherTextAuthToken[0], AUTH_TOKEN_MAX_SIZE).toString())
+		    .detail("ComputedCipherTextAuthToken", StringRef(computedCipherTextAuthToken, AUTH_TOKEN_MAX_SIZE));
+		throw encrypt_header_authtoken_mismatch();
+	}
+}
+
+void DecryptBlobCipherAes256Ctr::verifyAuthTokens(const uint8_t* ciphertext,
+                                                  const int ciphertextLen,
+                                                  const BlobCipherEncryptHeader& header,
+                                                  Arena& arena) {
+	if (header.flags.authTokenMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE) {
+		verifyHeaderSingleAuthToken(ciphertext, ciphertextLen, header, arena);
+	} else {
+		ASSERT_EQ(header.flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
+		verifyHeaderMultiAuthToken(ciphertext, ciphertextLen, header, arena);
+	}
+
+	authTokensValidationDone = true;
+}
+
+void DecryptBlobCipherAes256Ctr::verifyEncryptHeaderMetadata(const BlobCipherEncryptHeader& header) {
+	// validate header flag sanity
+	if (header.flags.headerVersion != EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION ||
+	    header.flags.encryptMode != EncryptCipherMode::ENCRYPT_CIPHER_MODE_AES_256_CTR ||
+	    !isEncryptHeaderAuthTokenModeValid((EncryptAuthTokenMode)header.flags.authTokenMode)) {
+		TraceEvent(SevWarn, "BlobCipherVerifyEncryptBlobHeader")
+		    .detail("HeaderVersion", header.flags.headerVersion)
+		    .detail("ExpectedVersion", EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION)
+		    .detail("EncryptCipherMode", header.flags.encryptMode)
+		    .detail("ExpectedCipherMode", EncryptCipherMode::ENCRYPT_CIPHER_MODE_AES_256_CTR)
+		    .detail("EncryptHeaderAuthTokenMode", header.flags.authTokenMode);
+		throw encrypt_header_metadata_mismatch();
+	}
+}
+
+Reference<EncryptBuf> DecryptBlobCipherAes256Ctr::decrypt(const uint8_t* ciphertext,
+                                                          const int ciphertextLen,
+                                                          const BlobCipherEncryptHeader& header,
+                                                          Arena& arena) {
+	double startTime = 0.0;
+	if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING) {
+		startTime = timer_monotonic();
+	}
+
+	verifyEncryptHeaderMetadata(header);
+
+	if (header.flags.authTokenMode != EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE &&
+	    !headerCipherKey.isValid()) {
+		TraceEvent(SevWarn, "BlobCipherDecryptInvalidHeaderCipherKey")
+		    .detail("AuthTokenMode", header.flags.authTokenMode);
+		throw encrypt_ops_error();
+	}
+
+	const int allocSize = ciphertextLen + AES_BLOCK_SIZE;
+	Reference<EncryptBuf> decrypted = makeReference<EncryptBuf>(allocSize, arena);
+
+	if (header.flags.authTokenMode != EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
+		verifyAuthTokens(ciphertext, ciphertextLen, header, arena);
+		ASSERT(authTokensValidationDone);
+	}
+
+	uint8_t* plaintext = decrypted->begin();
+	int bytesDecrypted{ 0 };
+	if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
+		TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
+		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
+		    .detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
+		throw encrypt_ops_error();
+	}
+
+	int finalBlobBytes{ 0 };
+	if (EVP_DecryptFinal_ex(ctx, plaintext + bytesDecrypted, &finalBlobBytes) <= 0) {
+		TraceEvent(SevWarn, "BlobCipherDecryptFinalFailed")
+		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
+		    .detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
+		throw encrypt_ops_error();
+	}
+
+	if ((bytesDecrypted + finalBlobBytes) != ciphertextLen) {
+		TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedPlaintextLen")
+		    .detail("CiphertextLen", ciphertextLen)
+		    .detail("DecryptedBufLen", bytesDecrypted + finalBlobBytes);
+		throw encrypt_ops_error();
+	}
+
+	decrypted->setLogicalSize(ciphertextLen);
+
+	if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING) {
+		BlobCipherMetrics::counters(usageType).decryptCPUTimeNS += int64_t((timer_monotonic() - startTime) * 1e9);
+	}
+
+	CODE_PROBE(true, "BlobCipher data decryption");
+	CODE_PROBE(header.flags.authTokenAlgo == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE,
+	           "Decryption authentication disabled");
+	CODE_PROBE(header.flags.authTokenAlgo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
+	           "Decryption HMAC_SHA Auth token verification");
+	CODE_PROBE(header.flags.authTokenAlgo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC,
+	           "Decryption AES_CMAC Auth token verification");
+
+	return decrypted;
+}
+
+DecryptBlobCipherAes256Ctr::~DecryptBlobCipherAes256Ctr() {
+	if (ctx != nullptr) {
+		EVP_CIPHER_CTX_free(ctx);
+	}
+}
+
+// HmacSha256DigestGen class methods
+
+HmacSha256DigestGen::HmacSha256DigestGen(const unsigned char* key, size_t len) : ctx(HMAC_CTX_new()) {
+	if (!HMAC_Init_ex(ctx, key, len, EVP_sha256(), nullptr)) {
+		throw encrypt_ops_error();
+	}
+}
+
+HmacSha256DigestGen::~HmacSha256DigestGen() {
+	if (ctx != nullptr) {
+		HMAC_CTX_free(ctx);
+	}
+}
+
+unsigned int HmacSha256DigestGen::digest(const std::vector<std::pair<const uint8_t*, size_t>>& payload,
+                                         unsigned char* buf,
+                                         unsigned int bufLen) {
+	ASSERT_EQ(bufLen, HMAC_size(ctx));
+
+	for (const auto& p : payload) {
+		if (HMAC_Update(ctx, p.first, p.second) != 1) {
+			throw encrypt_ops_error();
+		}
+	}
+
+	unsigned int digestLen = 0;
+	if (HMAC_Final(ctx, buf, &digestLen) != 1) {
+		throw encrypt_ops_error();
+	}
+
+	CODE_PROBE(true, "HMAC_SHA Digest generation");
+
+	return digestLen;
+}
+
+// Aes256CtrCmacDigestGen methods
+Aes256CmacDigestGen::Aes256CmacDigestGen(const unsigned char* key, size_t keylen) : ctx(CMAC_CTX_new()) {
+	ASSERT_EQ(keylen, AES_256_KEY_LENGTH);
+
+	if (ctx == nullptr) {
+		throw encrypt_ops_error();
+	}
+	if (!CMAC_Init(ctx, key, keylen, EVP_aes_256_cbc(), NULL)) {
+		throw encrypt_ops_error();
+	}
+}
+
+size_t Aes256CmacDigestGen::digest(const std::vector<std::pair<const uint8_t*, size_t>>& payload,
+                                   uint8_t* digest,
+                                   int digestlen) {
+	ASSERT(ctx != nullptr);
+	ASSERT_GE(digestlen, AUTH_TOKEN_AES_CMAC_SIZE);
+
+	for (const auto& p : payload) {
+		if (!CMAC_Update(ctx, p.first, p.second)) {
+			throw encrypt_ops_error();
+		}
+	}
+	size_t ret;
+	if (!CMAC_Final(ctx, digest, &ret)) {
+		throw encrypt_ops_error();
+	}
+
+	return ret;
+}
+
+Aes256CmacDigestGen::~Aes256CmacDigestGen() {
+	if (ctx != nullptr) {
+		CMAC_CTX_free(ctx);
+	}
+}
+
+void computeAuthToken(const std::vector<std::pair<const uint8_t*, size_t>>& payload,
+                      const uint8_t* key,
+                      const int keyLen,
+                      unsigned char* digestBuf,
+                      const EncryptAuthTokenAlgo algo,
+                      unsigned int digestBufMaxSz) {
+	ASSERT_EQ(digestBufMaxSz, AUTH_TOKEN_MAX_SIZE);
+	ASSERT(isEncryptHeaderAuthTokenAlgoValid(algo));
+
+	int authTokenSz = getEncryptHeaderAuthTokenSize(algo);
+	ASSERT_LE(authTokenSz, AUTH_TOKEN_MAX_SIZE);
+
+	if (algo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA) {
+		ASSERT_EQ(authTokenSz, AUTH_TOKEN_HMAC_SHA_SIZE);
+
+		HmacSha256DigestGen hmacGenerator(key, keyLen);
+		unsigned int digestLen = hmacGenerator.digest(payload, digestBuf, authTokenSz);
+
+		ASSERT_EQ(digestLen, authTokenSz);
+	} else if (algo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC) {
+		ASSERT_EQ(authTokenSz, AUTH_TOKEN_AES_CMAC_SIZE);
+		ASSERT_EQ(keyLen, AES_256_KEY_LENGTH);
+
+		Aes256CmacDigestGen cmacGenerator(key, keyLen);
+		size_t digestLen = cmacGenerator.digest(payload, digestBuf, authTokenSz);
+
+		ASSERT_EQ(digestLen, authTokenSz);
+	} else {
+		throw not_implemented();
+	}
+}
+
+EncryptAuthTokenMode getEncryptAuthTokenMode(const EncryptAuthTokenMode mode) {
+	// Override mode if authToken isn't enabled
+	return FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED ? mode
+	                                                     : EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE;
+}
+
+// Only used to link unit tests
+void forceLinkBlobCipherTests() {}
+
+// Tests cases includes:
+// 1. Populate cache by inserting 'baseCipher' details for new encryptionDomainIds
+// 2. Random lookup for cipherKeys and content validation
+// 3. Inserting of 'identical' cipherKey (already cached) more than once works as desired.
+// 4. Inserting of 'non-identical' cipherKey (already cached) more than once works as desired.
+// 5. Validation encryption ops (correctness):
+//  5.1. Encrypt a buffer followed by decryption of the buffer, validate the contents.
+//  5.2. Simulate anomalies such as: EncryptionHeader corruption, authToken mismatch / encryptionMode mismatch etc.
+// 6. Cache cleanup
+//  6.1  cleanup cipherKeys by given encryptDomainId
+//  6.2. Cleanup all cached cipherKeys
+TEST_CASE("flow/BlobCipher") {
+	TraceEvent("BlobCipherTestStart").log();
+
+	// Construct a dummy External Key Manager representation and populate with some keys
+	class BaseCipher : public ReferenceCounted<BaseCipher>, NonCopyable {
+	public:
+		EncryptCipherDomainId domainId;
+		int len;
+		EncryptCipherBaseKeyId keyId;
+		std::unique_ptr<uint8_t[]> key;
+		int64_t refreshAt;
+		int64_t expireAt;
+		EncryptCipherRandomSalt generatedSalt;
+
+		BaseCipher(const EncryptCipherDomainId& dId,
+		           const EncryptCipherBaseKeyId& kId,
+		           const int64_t rAt,
+		           const int64_t eAt)
+		  : domainId(dId), len(deterministicRandom()->randomInt(AES_256_KEY_LENGTH / 2, AES_256_KEY_LENGTH + 1)),
+		    keyId(kId), key(std::make_unique<uint8_t[]>(len)), refreshAt(rAt), expireAt(eAt) {
+			deterministicRandom()->randomBytes(key.get(), len);
+		}
+	};
+
+	using BaseKeyMap = std::unordered_map<EncryptCipherBaseKeyId, Reference<BaseCipher>>;
+	using DomainKeyMap = std::unordered_map<EncryptCipherDomainId, BaseKeyMap>;
+	DomainKeyMap domainKeyMap;
+	const EncryptCipherDomainId minDomainId = 1;
+	const EncryptCipherDomainId maxDomainId = deterministicRandom()->randomInt(minDomainId, minDomainId + 10) + 5;
+	const EncryptCipherBaseKeyId minBaseCipherKeyId = 100;
+	const EncryptCipherBaseKeyId maxBaseCipherKeyId =
+	    deterministicRandom()->randomInt(minBaseCipherKeyId, minBaseCipherKeyId + 50) + 15;
+	for (int dId = minDomainId; dId <= maxDomainId; dId++) {
+		for (int kId = minBaseCipherKeyId; kId <= maxBaseCipherKeyId; kId++) {
+			domainKeyMap[dId].emplace(
+			    kId,
+			    makeReference<BaseCipher>(
+			        dId, kId, std::numeric_limits<int64_t>::max(), std::numeric_limits<int64_t>::max()));
+		}
+	}
+	ASSERT_EQ(domainKeyMap.size(), maxDomainId);
+
+	Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
+
+	// validate getLatestCipherKey return empty when there's no cipher key
+	TraceEvent("BlobCipherTestLatestKeyNotExists").log();
+	Reference<BlobCipherKey> latestKeyNonexists =
+	    cipherKeyCache->getLatestCipherKey(deterministicRandom()->randomInt(minDomainId, maxDomainId));
+	ASSERT(!latestKeyNonexists.isValid());
+	try {
+		cipherKeyCache->getLatestCipherKey(INVALID_ENCRYPT_DOMAIN_ID);
+		ASSERT(false); // shouldn't get here
+	} catch (Error& e) {
+		ASSERT_EQ(e.code(), error_code_encrypt_invalid_id);
+	}
+
+	// insert BlobCipher keys into BlobCipherKeyCache map and validate
+	TraceEvent("BlobCipherTestInsertKeys").log();
+	for (auto& domainItr : domainKeyMap) {
+		for (auto& baseKeyItr : domainItr.second) {
+			Reference<BaseCipher> baseCipher = baseKeyItr.second;
+
+			cipherKeyCache->insertCipherKey(baseCipher->domainId,
+			                                baseCipher->keyId,
+			                                baseCipher->key.get(),
+			                                baseCipher->len,
+			                                baseCipher->refreshAt,
+			                                baseCipher->expireAt);
+			Reference<BlobCipherKey> fetchedKey = cipherKeyCache->getLatestCipherKey(baseCipher->domainId);
+			baseCipher->generatedSalt = fetchedKey->getSalt();
+		}
+	}
+	// insert EncryptHeader BlobCipher key
+	Reference<BaseCipher> headerBaseCipher = makeReference<BaseCipher>(
+	    ENCRYPT_HEADER_DOMAIN_ID, 1, std::numeric_limits<int64_t>::max(), std::numeric_limits<int64_t>::max());
+	cipherKeyCache->insertCipherKey(headerBaseCipher->domainId,
+	                                headerBaseCipher->keyId,
+	                                headerBaseCipher->key.get(),
+	                                headerBaseCipher->len,
+	                                headerBaseCipher->refreshAt,
+	                                headerBaseCipher->expireAt);
+
+	TraceEvent("BlobCipherTestInsertKeysDone").log();
+
+	// validate the cipherKey lookups work as desired
+	for (auto& domainItr : domainKeyMap) {
+		for (auto& baseKeyItr : domainItr.second) {
+			Reference<BaseCipher> baseCipher = baseKeyItr.second;
+			Reference<BlobCipherKey> cipherKey =
+			    cipherKeyCache->getCipherKey(baseCipher->domainId, baseCipher->keyId, baseCipher->generatedSalt);
+			ASSERT(cipherKey.isValid());
+			// validate common cipher properties - domainId, baseCipherId, baseCipherLen, rawBaseCipher
+			ASSERT_EQ(cipherKey->getBaseCipherId(), baseCipher->keyId);
+			ASSERT_EQ(cipherKey->getDomainId(), baseCipher->domainId);
+			ASSERT_EQ(cipherKey->getBaseCipherLen(), baseCipher->len);
+			// ensure that baseCipher matches with the cached information
+			ASSERT_EQ(std::memcmp(cipherKey->rawBaseCipher(), baseCipher->key.get(), cipherKey->getBaseCipherLen()), 0);
+			// validate the encryption derivation
+			ASSERT_NE(std::memcmp(cipherKey->rawCipher(), baseCipher->key.get(), cipherKey->getBaseCipherLen()), 0);
+		}
+	}
+	TraceEvent("BlobCipherTestLooksupDone").log();
+
+	// Ensure attemtping to insert existing cipherKey (identical) more than once is treated as a NOP
+	try {
+		Reference<BaseCipher> baseCipher = domainKeyMap[minDomainId][minBaseCipherKeyId];
+		cipherKeyCache->insertCipherKey(baseCipher->domainId,
+		                                baseCipher->keyId,
+		                                baseCipher->key.get(),
+		                                baseCipher->len,
+		                                std::numeric_limits<int64_t>::max(),
+		                                std::numeric_limits<int64_t>::max());
+	} catch (Error& e) {
+		throw;
+	}
+	TraceEvent("BlobCipherTestReinsertIdempotentKeyDone").log();
+
+	// Ensure attemtping to insert an existing cipherKey (modified) fails with appropriate error
+	try {
+		Reference<BaseCipher> baseCipher = domainKeyMap[minDomainId][minBaseCipherKeyId];
+		uint8_t rawCipher[baseCipher->len];
+		memcpy(rawCipher, baseCipher->key.get(), baseCipher->len);
+		// modify few bytes in the cipherKey
+		for (int i = 2; i < 5; i++) {
+			rawCipher[i]++;
+		}
+		cipherKeyCache->insertCipherKey(baseCipher->domainId,
+		                                baseCipher->keyId,
+		                                &rawCipher[0],
+		                                baseCipher->len,
+		                                std::numeric_limits<int64_t>::max(),
+		                                std::numeric_limits<int64_t>::max());
+	} catch (Error& e) {
+		if (e.code() != error_code_encrypt_update_cipher) {
+			throw;
+		}
+	}
+	TraceEvent("BlobCipherTestReinsertNonIdempotentKeyDone").log();
+
+	// Validate Encryption ops
+	Reference<BlobCipherKey> cipherKey = cipherKeyCache->getLatestCipherKey(minDomainId);
+	Reference<BlobCipherKey> headerCipherKey = cipherKeyCache->getLatestCipherKey(ENCRYPT_HEADER_DOMAIN_ID);
+	const int bufLen = deterministicRandom()->randomInt(786, 2127) + 512;
+	uint8_t orgData[bufLen];
+	deterministicRandom()->randomBytes(&orgData[0], bufLen);
+
+	Arena arena;
+	uint8_t iv[AES_256_IV_LENGTH];
+	deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH);
+
+	BlobCipherEncryptHeader headerCopy;
+	// validate basic encrypt followed by decrypt operation for AUTH_MODE_NONE
+	{
+		TraceEvent("NoneAuthModeStart");
+
+		EncryptBlobCipherAes265Ctr encryptor(cipherKey,
+		                                     headerCipherKey,
+		                                     iv,
+		                                     AES_256_IV_LENGTH,
+		                                     EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE,
+		                                     BlobCipherMetrics::TEST);
+		BlobCipherEncryptHeader header;
+		Reference<EncryptBuf> encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+
+		ASSERT_EQ(encrypted->getLogicalSize(), bufLen);
+		ASSERT_NE(memcmp(&orgData[0], encrypted->begin(), bufLen), 0);
+		ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
+		ASSERT_EQ(header.flags.encryptMode, EncryptCipherMode::ENCRYPT_CIPHER_MODE_AES_256_CTR);
+		ASSERT_EQ(header.flags.authTokenMode, EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE);
+
+		TraceEvent("BlobCipherTestEncryptDone")
+		    .detail("HeaderVersion", header.flags.headerVersion)
+		    .detail("HeaderEncryptMode", header.flags.encryptMode)
+		    .detail("HeaderEncryptAuthTokenMode", header.flags.authTokenMode)
+		    .detail("HeaderEncryptAuthTokenAlgo", header.flags.authTokenAlgo)
+		    .detail("DomainId", header.cipherTextDetails.encryptDomainId)
+		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId);
+
+		Reference<BlobCipherKey> tCipherKeyKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
+		                                                                      header.cipherTextDetails.baseCipherId,
+		                                                                      header.cipherTextDetails.salt);
+		ASSERT(tCipherKeyKey->isEqual(cipherKey));
+		DecryptBlobCipherAes256Ctr decryptor(
+		    tCipherKeyKey, Reference<BlobCipherKey>(), &header.iv[0], BlobCipherMetrics::TEST);
+		Reference<EncryptBuf> decrypted = decryptor.decrypt(encrypted->begin(), bufLen, header, arena);
+
+		ASSERT_EQ(decrypted->getLogicalSize(), bufLen);
+		ASSERT_EQ(memcmp(decrypted->begin(), &orgData[0], bufLen), 0);
+
+		TraceEvent("BlobCipherTestDecryptDone");
+
+		// induce encryption header corruption - headerVersion corrupted
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.headerVersion += 1;
+		try {
+			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+			DecryptBlobCipherAes256Ctr decryptor(
+			    tCipherKeyKey, Reference<BlobCipherKey>(), header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - encryptionMode corrupted
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.encryptMode += 1;
+		try {
+			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+			DecryptBlobCipherAes256Ctr decryptor(
+			    tCipherKeyKey, Reference<BlobCipherKey>(), header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encrypted buffer payload corruption
+		try {
+			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+			uint8_t temp[bufLen];
+			memcpy(encrypted->begin(), &temp[0], bufLen);
+			int tIdx = deterministicRandom()->randomInt(0, bufLen - 1);
+			temp[tIdx] += 1;
+			DecryptBlobCipherAes256Ctr decryptor(
+			    tCipherKeyKey, Reference<BlobCipherKey>(), header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(&temp[0], bufLen, header, arena);
+		} catch (Error& e) {
+			// No authToken, hence, no corruption detection supported
+			ASSERT(false);
+		}
+
+		TraceEvent("NoneAuthModeDone");
+	}
+
+	// validate basic encrypt followed by decrypt operation for AUTH_TOKEN_MODE_SINGLE
+	// HMAC_SHA authToken algorithm
+	{
+		TraceEvent("SingleAuthModeHmacShaStart").log();
+
+		EncryptBlobCipherAes265Ctr encryptor(cipherKey,
+		                                     headerCipherKey,
+		                                     iv,
+		                                     AES_256_IV_LENGTH,
+		                                     EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE,
+		                                     EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
+		                                     BlobCipherMetrics::TEST);
+		BlobCipherEncryptHeader header;
+		Reference<EncryptBuf> encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+
+		ASSERT_EQ(encrypted->getLogicalSize(), bufLen);
+		ASSERT_NE(memcmp(&orgData[0], encrypted->begin(), bufLen), 0);
+		ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
+		ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
+		ASSERT_EQ(header.flags.authTokenMode, EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
+		ASSERT_EQ(header.flags.authTokenAlgo, EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA);
+
+		TraceEvent("BlobCipherTestEncryptDone")
+		    .detail("HeaderVersion", header.flags.headerVersion)
+		    .detail("HeaderEncryptMode", header.flags.encryptMode)
+		    .detail("HeaderEncryptAuthTokenMode", header.flags.authTokenMode)
+		    .detail("HeaderEncryptAuthTokenAlgo", header.flags.authTokenAlgo)
+		    .detail("DomainId", header.cipherTextDetails.encryptDomainId)
+		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
+		    .detail("HeaderAuthToken",
+		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_HMAC_SHA_SIZE).toString());
+
+		Reference<BlobCipherKey> tCipherKeyKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
+		                                                                      header.cipherTextDetails.baseCipherId,
+		                                                                      header.cipherTextDetails.salt);
+		Reference<BlobCipherKey> hCipherKey = cipherKeyCache->getCipherKey(header.cipherHeaderDetails.encryptDomainId,
+		                                                                   header.cipherHeaderDetails.baseCipherId,
+		                                                                   header.cipherHeaderDetails.salt);
+		ASSERT(tCipherKeyKey->isEqual(cipherKey));
+		DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+		Reference<EncryptBuf> decrypted = decryptor.decrypt(encrypted->begin(), bufLen, header, arena);
+
+		ASSERT_EQ(decrypted->getLogicalSize(), bufLen);
+		ASSERT_EQ(memcmp(decrypted->begin(), &orgData[0], bufLen), 0);
+
+		TraceEvent("BlobCipherTestDecryptDone");
+
+		// induce encryption header corruption - headerVersion corrupted
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.headerVersion += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - encryptionMode corrupted
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.encryptMode += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - authToken mismatch
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		int hIdx = deterministicRandom()->randomInt(0, AUTH_TOKEN_HMAC_SHA_SIZE - 1);
+		headerCopy.singleAuthToken.authToken[hIdx] += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encrypted buffer payload corruption
+		try {
+			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+			uint8_t temp[bufLen];
+			memcpy(encrypted->begin(), &temp[0], bufLen);
+			int tIdx = deterministicRandom()->randomInt(0, bufLen - 1);
+			temp[tIdx] += 1;
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(&temp[0], bufLen, header, arena);
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		TraceEvent("SingleAuthModeHmacShaDone");
+	}
+	// AES_CMAC authToken algorithm
+	{
+		TraceEvent("SingleAuthModeAesCMacStart").log();
+
+		EncryptBlobCipherAes265Ctr encryptor(cipherKey,
+		                                     headerCipherKey,
+		                                     iv,
+		                                     AES_256_IV_LENGTH,
+		                                     EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE,
+		                                     EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC,
+		                                     BlobCipherMetrics::TEST);
+		BlobCipherEncryptHeader header;
+		Reference<EncryptBuf> encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+
+		ASSERT_EQ(encrypted->getLogicalSize(), bufLen);
+		ASSERT_NE(memcmp(&orgData[0], encrypted->begin(), bufLen), 0);
+		ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
+		ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
+		ASSERT_EQ(header.flags.authTokenMode, EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
+		ASSERT_EQ(header.flags.authTokenAlgo, EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC);
+
+		TraceEvent("BlobCipherTestEncryptDone")
+		    .detail("HeaderVersion", header.flags.headerVersion)
+		    .detail("HeaderEncryptMode", header.flags.encryptMode)
+		    .detail("HeaderEncryptAuthTokenMode", header.flags.authTokenMode)
+		    .detail("HeaderEncryptAuthTokenAlgo", header.flags.authTokenAlgo)
+		    .detail("DomainId", header.cipherTextDetails.encryptDomainId)
+		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
+		    .detail("HeaderAuthToken",
+		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_AES_CMAC_SIZE).toString());
+
+		Reference<BlobCipherKey> tCipherKeyKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
+		                                                                      header.cipherTextDetails.baseCipherId,
+		                                                                      header.cipherTextDetails.salt);
+		Reference<BlobCipherKey> hCipherKey = cipherKeyCache->getCipherKey(header.cipherHeaderDetails.encryptDomainId,
+		                                                                   header.cipherHeaderDetails.baseCipherId,
+		                                                                   header.cipherHeaderDetails.salt);
+		ASSERT(tCipherKeyKey->isEqual(cipherKey));
+		DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+		Reference<EncryptBuf> decrypted = decryptor.decrypt(encrypted->begin(), bufLen, header, arena);
+
+		ASSERT_EQ(decrypted->getLogicalSize(), bufLen);
+		ASSERT_EQ(memcmp(decrypted->begin(), &orgData[0], bufLen), 0);
+
+		TraceEvent("BlobCipherTestDecryptDone").log();
+
+		// induce encryption header corruption - headerVersion corrupted
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.headerVersion += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - encryptionMode corrupted
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.encryptMode += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - authToken mismatch
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		int hIdx = deterministicRandom()->randomInt(0, AUTH_TOKEN_AES_CMAC_SIZE - 1);
+		headerCopy.singleAuthToken.authToken[hIdx] += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encrypted buffer payload corruption
+		try {
+			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+			uint8_t temp[bufLen];
+			memcpy(encrypted->begin(), &temp[0], bufLen);
+			int tIdx = deterministicRandom()->randomInt(0, bufLen - 1);
+			temp[tIdx] += 1;
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(&temp[0], bufLen, header, arena);
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		TraceEvent("SingleAuthModeAesCmacDone");
+	}
+
+	// validate basic encrypt followed by decrypt operation for AUTH_TOKEN_MODE_MULTI
+	// HMAC_SHA authToken algorithm
+	{
+		TraceEvent("MultiAuthModeHmacShaStart").log();
+
+		EncryptBlobCipherAes265Ctr encryptor(cipherKey,
+		                                     headerCipherKey,
+		                                     iv,
+		                                     AES_256_IV_LENGTH,
+		                                     EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI,
+		                                     EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
+		                                     BlobCipherMetrics::TEST);
+		BlobCipherEncryptHeader header;
+		Reference<EncryptBuf> encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+
+		ASSERT_EQ(encrypted->getLogicalSize(), bufLen);
+		ASSERT_NE(memcmp(&orgData[0], encrypted->begin(), bufLen), 0);
+		ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
+		ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
+		ASSERT_EQ(header.flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
+		ASSERT_EQ(header.flags.authTokenAlgo, EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA);
+
+		TraceEvent("BlobCipherTestEncryptDone")
+		    .detail("HeaderVersion", header.flags.headerVersion)
+		    .detail("HeaderEncryptMode", header.flags.encryptMode)
+		    .detail("HeaderEncryptAuthTokenMode", header.flags.authTokenMode)
+		    .detail("HeaderEncryptAuthTokenAlgo", header.flags.authTokenAlgo)
+		    .detail("DomainId", header.cipherTextDetails.encryptDomainId)
+		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
+		    .detail("HeaderAuthToken",
+		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_HMAC_SHA_SIZE).toString());
+
+		Reference<BlobCipherKey> tCipherKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
+		                                                                   header.cipherTextDetails.baseCipherId,
+		                                                                   header.cipherTextDetails.salt);
+		Reference<BlobCipherKey> hCipherKey = cipherKeyCache->getCipherKey(header.cipherHeaderDetails.encryptDomainId,
+		                                                                   header.cipherHeaderDetails.baseCipherId,
+		                                                                   header.cipherHeaderDetails.salt);
+
+		ASSERT(tCipherKey->isEqual(cipherKey));
+		DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+		Reference<EncryptBuf> decrypted = decryptor.decrypt(encrypted->begin(), bufLen, header, arena);
+
+		ASSERT_EQ(decrypted->getLogicalSize(), bufLen);
+		ASSERT_EQ(memcmp(decrypted->begin(), &orgData[0], bufLen), 0);
+
+		TraceEvent("BlobCipherTestDecryptDone").log();
+
+		// induce encryption header corruption - headerVersion corrupted
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.headerVersion += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - encryptionMode corrupted
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.encryptMode += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - cipherText authToken mismatch
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		int hIdx = deterministicRandom()->randomInt(0, AUTH_TOKEN_HMAC_SHA_SIZE - 1);
+		headerCopy.multiAuthTokens.cipherTextAuthToken[hIdx] += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - header authToken mismatch
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		hIdx = deterministicRandom()->randomInt(0, AUTH_TOKEN_HMAC_SHA_SIZE - 1);
+		headerCopy.multiAuthTokens.headerAuthToken[hIdx] += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		try {
+			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+			uint8_t temp[bufLen];
+			memcpy(encrypted->begin(), &temp[0], bufLen);
+			int tIdx = deterministicRandom()->randomInt(0, bufLen - 1);
+			temp[tIdx] += 1;
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(&temp[0], bufLen, header, arena);
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		TraceEvent("MultiAuthModeHmacShaDone");
+	}
+	// AES_CMAC authToken algorithm
+	{
+		TraceEvent("MultiAuthModeAesCmacStart");
+
+		EncryptBlobCipherAes265Ctr encryptor(cipherKey,
+		                                     headerCipherKey,
+		                                     iv,
+		                                     AES_256_IV_LENGTH,
+		                                     EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI,
+		                                     EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC,
+		                                     BlobCipherMetrics::TEST);
+		BlobCipherEncryptHeader header;
+		Reference<EncryptBuf> encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+
+		ASSERT_EQ(encrypted->getLogicalSize(), bufLen);
+		ASSERT_NE(memcmp(&orgData[0], encrypted->begin(), bufLen), 0);
+		ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
+		ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
+		ASSERT_EQ(header.flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
+		ASSERT_EQ(header.flags.authTokenAlgo, EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC);
+
+		TraceEvent("BlobCipherTestEncryptDone")
+		    .detail("HeaderVersion", header.flags.headerVersion)
+		    .detail("HeaderEncryptMode", header.flags.encryptMode)
+		    .detail("HeaderEncryptAuthTokenMode", header.flags.authTokenMode)
+		    .detail("HeaderEncryptAuthTokenAlgo", header.flags.authTokenAlgo)
+		    .detail("DomainId", header.cipherTextDetails.encryptDomainId)
+		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
+		    .detail("HeaderAuthToken",
+		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_AES_CMAC_SIZE).toString());
+
+		Reference<BlobCipherKey> tCipherKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
+		                                                                   header.cipherTextDetails.baseCipherId,
+		                                                                   header.cipherTextDetails.salt);
+		Reference<BlobCipherKey> hCipherKey = cipherKeyCache->getCipherKey(header.cipherHeaderDetails.encryptDomainId,
+		                                                                   header.cipherHeaderDetails.baseCipherId,
+		                                                                   header.cipherHeaderDetails.salt);
+
+		ASSERT(tCipherKey->isEqual(cipherKey));
+		DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+		Reference<EncryptBuf> decrypted = decryptor.decrypt(encrypted->begin(), bufLen, header, arena);
+
+		ASSERT_EQ(decrypted->getLogicalSize(), bufLen);
+		ASSERT_EQ(memcmp(decrypted->begin(), &orgData[0], bufLen), 0);
+
+		TraceEvent("BlobCipherTestDecryptDone").log();
+
+		// induce encryption header corruption - headerVersion corrupted
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.headerVersion += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - encryptionMode corrupted
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		headerCopy.flags.encryptMode += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - cipherText authToken mismatch
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		int hIdx = deterministicRandom()->randomInt(0, AUTH_TOKEN_AES_CMAC_SIZE - 1);
+		headerCopy.multiAuthTokens.cipherTextAuthToken[hIdx] += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		// induce encryption header corruption - header authToken mismatch
+		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
+		       reinterpret_cast<const uint8_t*>(&header),
+		       sizeof(BlobCipherEncryptHeader));
+		hIdx = deterministicRandom()->randomInt(0, AUTH_TOKEN_AES_CMAC_SIZE - 1);
+		headerCopy.multiAuthTokens.headerAuthToken[hIdx] += 1;
+		try {
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
+			ASSERT(false); // error expected
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		try {
+			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
+			uint8_t temp[bufLen];
+			memcpy(encrypted->begin(), &temp[0], bufLen);
+			int tIdx = deterministicRandom()->randomInt(0, bufLen - 1);
+			temp[tIdx] += 1;
+			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv, BlobCipherMetrics::TEST);
+			decrypted = decryptor.decrypt(&temp[0], bufLen, header, arena);
+		} catch (Error& e) {
+			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
+				throw;
+			}
+		}
+
+		TraceEvent("MultiAuthModeAesCmacDone");
+	}
+
+	// Validate dropping encryptDomainId cached keys
+	const EncryptCipherDomainId candidate = deterministicRandom()->randomInt(minDomainId, maxDomainId);
+	cipherKeyCache->resetEncryptDomainId(candidate);
+	std::vector<Reference<BlobCipherKey>> cachedKeys = cipherKeyCache->getAllCiphers(candidate);
+	ASSERT(cachedKeys.empty());
+
+	// Validate dropping all cached cipherKeys
+	cipherKeyCache->cleanup();
+	for (int dId = minDomainId; dId < maxDomainId; dId++) {
+		std::vector<Reference<BlobCipherKey>> cachedKeys = cipherKeyCache->getAllCiphers(dId);
+		ASSERT(cachedKeys.empty());
+	}
+
+	TraceEvent("BlobCipherTestDone");
+	return Void();
+}
diff --git a/fdbclient/BlobGranuleCommon.cpp b/fdbclient/BlobGranuleCommon.cpp
new file mode 100644
index 0000000000..44f32bcb25
--- /dev/null
+++ b/fdbclient/BlobGranuleCommon.cpp
@@ -0,0 +1,45 @@
+/*
+ * BlobGranuleCommon.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/BlobGranuleCommon.h"
+
+BlobGranuleSummaryRef summarizeGranuleChunk(Arena& ar, const BlobGranuleChunkRef& chunk) {
+	BlobGranuleSummaryRef summary;
+	ASSERT(chunk.snapshotFile.present());
+	ASSERT(chunk.snapshotVersion != invalidVersion);
+	ASSERT(chunk.includedVersion >= chunk.snapshotVersion);
+	ASSERT(chunk.newDeltas.empty());
+
+	if (chunk.tenantPrefix.present()) {
+		summary.keyRange = KeyRangeRef(ar, chunk.keyRange.removePrefix(chunk.tenantPrefix.get()));
+	} else {
+		summary.keyRange = KeyRangeRef(ar, chunk.keyRange);
+	}
+
+	summary.snapshotVersion = chunk.snapshotVersion;
+	summary.snapshotSize = chunk.snapshotFile.get().length;
+	summary.deltaVersion = chunk.includedVersion;
+	summary.deltaSize = 0;
+	for (auto& it : chunk.deltaFiles) {
+		summary.deltaSize += it.length;
+	}
+
+	return summary;
+}
\ No newline at end of file
diff --git a/fdbclient/BlobGranuleFiles.cpp b/fdbclient/BlobGranuleFiles.cpp
index 4250ee7815..3747824437 100644
--- a/fdbclient/BlobGranuleFiles.cpp
+++ b/fdbclient/BlobGranuleFiles.cpp
@@ -20,15 +20,19 @@
 
 #include "fdbclient/BlobGranuleFiles.h"
 
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/BlobGranuleCommon.h"
 #include "fdbclient/ClientKnobs.h"
+#include "fdbclient/CommitTransaction.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/SystemData.h" // for allKeys unit test - could remove
 
-#include "flow/BlobCipher.h"
+#include "flow/Arena.h"
 #include "flow/CompressionUtils.h"
 #include "flow/DeterministicRandom.h"
+#include "flow/EncryptUtils.h"
 #include "flow/IRandom.h"
+#include "flow/Knobs.h"
 #include "flow/Trace.h"
 #include "flow/serialize.h"
 #include "flow/UnitTest.h"
@@ -37,11 +41,12 @@
 #include "fmt/format.h"
 
 #include <cstring>
+#include <fstream> // for perf microbenchmark
+#include <limits>
 #include <vector>
 
 #define BG_READ_DEBUG false
-
-// FIXME: implement actual proper file format for this
+#define BG_FILES_TEST_DEBUG false
 
 // Implements granule file parsing and materialization with normal c++ functions (non-actors) so that this can be used
 // outside the FDB network thread.
@@ -57,6 +62,111 @@ uint16_t MIN_SUPPORTED_BG_FORMAT_VERSION = 1;
 const uint8_t SNAPSHOT_FILE_TYPE = 'S';
 const uint8_t DELTA_FILE_TYPE = 'D';
 
+// Deltas in key order
+
+// For key-ordered delta files, the format for both sets and range clears is that you store boundaries ordered by key.
+// Each boundary has a corresponding key, zero or more versioned updates (ValueAndVersionRef), and optionally a clear
+// from keyAfter(key) to the next boundary, at a version.
+// A streaming merge is more efficient than applying deltas one by one to restore to a later version from the snapshot.
+// The concept of this versioned mutation boundaries is repurposed directly from a prior version of redwood, back when
+// it supported versioned data.
+struct ValueAndVersionRef {
+	Version version;
+	MutationRef::Type op; // only set/clear
+	ValueRef value; // only present for set
+
+	ValueAndVersionRef() {}
+	// create clear
+	explicit ValueAndVersionRef(Version version) : version(version), op(MutationRef::Type::ClearRange) {}
+	// create set
+	explicit ValueAndVersionRef(Version version, ValueRef value)
+	  : version(version), op(MutationRef::Type::SetValue), value(value) {}
+	ValueAndVersionRef(Arena& arena, const ValueAndVersionRef& copyFrom)
+	  : version(copyFrom.version), op(copyFrom.op), value(arena, copyFrom.value) {}
+
+	bool isSet() const { return op == MutationRef::SetValue; }
+	bool isClear() const { return op == MutationRef::ClearRange; }
+
+	int totalSize() const { return sizeof(ValueAndVersionRef) + value.size(); }
+	int expectedSize() const { return value.size(); }
+
+	struct OrderByVersion {
+		bool operator()(ValueAndVersionRef const& a, ValueAndVersionRef const& b) const {
+			return a.version < b.version;
+		}
+	};
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version, op, value);
+	}
+};
+
+// Effectively the single DeltaBoundaryRef reduced to one update, but also with the key and clear after information.
+// Sometimes at a given version, the boundary may only be necessary to represent a clear version after this key, or just
+// an update/clear to this key, or both.
+struct ParsedDeltaBoundaryRef {
+	KeyRef key;
+	MutationRef::Type op; // SetValue, ClearRange, or NoOp
+	ValueRef value; // null unless op == SetValue
+	bool clearAfter;
+
+	// op constructor
+	ParsedDeltaBoundaryRef() {}
+	explicit ParsedDeltaBoundaryRef(KeyRef key, bool clearAfter, const ValueAndVersionRef& valueAndVersion)
+	  : key(key), op(valueAndVersion.op), value(valueAndVersion.value), clearAfter(clearAfter) {}
+	// noop constructor
+	explicit ParsedDeltaBoundaryRef(KeyRef key, bool clearAfter)
+	  : key(key), op(MutationRef::Type::NoOp), clearAfter(clearAfter) {}
+	// from snapshot set constructor
+	explicit ParsedDeltaBoundaryRef(const KeyValueRef& kv)
+	  : key(kv.key), op(MutationRef::Type::SetValue), value(kv.value), clearAfter(false) {}
+
+	ParsedDeltaBoundaryRef(Arena& arena, const ParsedDeltaBoundaryRef& copyFrom)
+	  : key(arena, copyFrom.key), op(copyFrom.op), clearAfter(copyFrom.clearAfter) {
+		if (copyFrom.isSet()) {
+			value = StringRef(arena, copyFrom.value);
+		}
+	}
+
+	bool isSet() const { return op == MutationRef::SetValue; }
+	bool isClear() const { return op == MutationRef::ClearRange; }
+	bool isNoOp() const { return op == MutationRef::NoOp; }
+	bool redundant(bool prevClearAfter) const { return op == MutationRef::Type::NoOp && clearAfter == prevClearAfter; }
+};
+
+struct DeltaBoundaryRef {
+	// key
+	KeyRef key;
+	// updates to exactly this key
+	VectorRef<ValueAndVersionRef> values;
+	// clear version from keyAfter(key) up to the next boundary
+	Optional<Version> clearVersion;
+
+	DeltaBoundaryRef() {}
+	DeltaBoundaryRef(Arena& ar, const DeltaBoundaryRef& copyFrom)
+	  : key(ar, copyFrom.key), values(ar, copyFrom.values), clearVersion(copyFrom.clearVersion) {}
+
+	int totalSize() { return sizeof(DeltaBoundaryRef) + key.expectedSize() + values.expectedSize(); }
+	int expectedSize() const { return key.expectedSize() + values.expectedSize(); }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, key, values, clearVersion);
+	}
+};
+
+struct GranuleSortedDeltas {
+	constexpr static FileIdentifier file_identifier = 8183903;
+
+	VectorRef<DeltaBoundaryRef> boundaries;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, boundaries);
+	}
+};
+
 struct ChildBlockPointerRef {
 	StringRef key;
 	uint32_t offset;
@@ -87,16 +197,21 @@ namespace {
 BlobGranuleFileEncryptionKeys getEncryptBlobCipherKey(const BlobGranuleCipherKeysCtx cipherKeysCtx) {
 	BlobGranuleFileEncryptionKeys eKeys;
 
+	// Cipher key reconstructed is 'never' inserted into BlobCipherKey cache, choose 'neverExpire'
 	eKeys.textCipherKey = makeReference<BlobCipherKey>(cipherKeysCtx.textCipherKey.encryptDomainId,
 	                                                   cipherKeysCtx.textCipherKey.baseCipherId,
 	                                                   cipherKeysCtx.textCipherKey.baseCipher.begin(),
 	                                                   cipherKeysCtx.textCipherKey.baseCipher.size(),
-	                                                   cipherKeysCtx.textCipherKey.salt);
+	                                                   cipherKeysCtx.textCipherKey.salt,
+	                                                   std::numeric_limits<int64_t>::max(),
+	                                                   std::numeric_limits<int64_t>::max());
 	eKeys.headerCipherKey = makeReference<BlobCipherKey>(cipherKeysCtx.headerCipherKey.encryptDomainId,
 	                                                     cipherKeysCtx.headerCipherKey.baseCipherId,
 	                                                     cipherKeysCtx.headerCipherKey.baseCipher.begin(),
 	                                                     cipherKeysCtx.headerCipherKey.baseCipher.size(),
-	                                                     cipherKeysCtx.headerCipherKey.salt);
+	                                                     cipherKeysCtx.headerCipherKey.salt,
+	                                                     std::numeric_limits<int64_t>::max(),
+	                                                     std::numeric_limits<int64_t>::max());
 
 	return eKeys;
 }
@@ -172,12 +287,14 @@ struct IndexBlockRef {
 			TraceEvent(SevDebug, "IndexBlockEncrypt_Before").detail("Chksum", chksum);
 		}
 
-		EncryptBlobCipherAes265Ctr encryptor(eKeys.textCipherKey,
-		                                     eKeys.headerCipherKey,
-		                                     cipherKeysCtx.ivRef.begin(),
-		                                     AES_256_IV_LENGTH,
-		                                     ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
-		Value serializedBuff = ObjectWriter::toValue(block, Unversioned());
+		EncryptBlobCipherAes265Ctr encryptor(
+		    eKeys.textCipherKey,
+		    eKeys.headerCipherKey,
+		    cipherKeysCtx.ivRef.begin(),
+		    AES_256_IV_LENGTH,
+		    getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
+		    BlobCipherMetrics::BLOB_GRANULE);
+		Value serializedBuff = ObjectWriter::toValue(block, IncludeVersion(ProtocolVersion::withBlobGranuleFile()));
 		BlobCipherEncryptHeader header;
 		buffer = encryptor.encrypt(serializedBuff.contents().begin(), serializedBuff.contents().size(), &header, arena)
 		             ->toStringRef();
@@ -204,7 +321,8 @@ struct IndexBlockRef {
 
 		validateEncryptionHeaderDetails(eKeys, header, cipherKeysCtx.ivRef);
 
-		DecryptBlobCipherAes256Ctr decryptor(eKeys.textCipherKey, eKeys.headerCipherKey, cipherKeysCtx.ivRef.begin());
+		DecryptBlobCipherAes256Ctr decryptor(
+		    eKeys.textCipherKey, eKeys.headerCipherKey, cipherKeysCtx.ivRef.begin(), BlobCipherMetrics::BLOB_GRANULE);
 		StringRef decrypted =
 		    decryptor.decrypt(idxRef.buffer.begin(), idxRef.buffer.size(), header, arena)->toStringRef();
 
@@ -213,20 +331,22 @@ struct IndexBlockRef {
 			TraceEvent(SevDebug, "IndexBlockEncrypt_After").detail("Chksum", chksum);
 		}
 
-		// TODO: Add version?
-		ObjectReader dataReader(decrypted.begin(), Unversioned());
+		ObjectReader dataReader(decrypted.begin(), IncludeVersion());
 		dataReader.deserialize(FileIdentifierFor<IndexBlock>::value, idxRef.block, arena);
 	}
 
 	void init(Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx, Arena& arena) {
 		if (encryptHeaderRef.present()) {
+			CODE_PROBE(true, "reading encrypted chunked file");
 			ASSERT(cipherKeysCtx.present());
+
 			decrypt(cipherKeysCtx.get(), *this, arena);
 		} else {
-			TraceEvent("IndexBlockSize").detail("Sz", buffer.size());
+			if (BG_ENCRYPT_COMPRESS_DEBUG) {
+				TraceEvent("IndexBlockSize").detail("Sz", buffer.size());
+			}
 
-			// TODO: Add version?
-			ObjectReader dataReader(buffer.begin(), Unversioned());
+			ObjectReader dataReader(buffer.begin(), IncludeVersion());
 			dataReader.deserialize(FileIdentifierFor<IndexBlock>::value, block, arena);
 		}
 	}
@@ -242,10 +362,15 @@ struct IndexBlockRef {
 			encrypt(cipherKeysCtx.get(), arena);
 		} else {
 			encryptHeaderRef.reset();
-			buffer = StringRef(arena, ObjectWriter::toValue(block, Unversioned()).contents());
+			buffer = StringRef(
+			    arena, ObjectWriter::toValue(block, IncludeVersion(ProtocolVersion::withBlobGranuleFile())).contents());
 		}
 
-		TraceEvent(SevDebug, "IndexBlockSize").detail("Sz", buffer.size()).detail("Encrypted", cipherKeysCtx.present());
+		if (BG_ENCRYPT_COMPRESS_DEBUG) {
+			TraceEvent(SevDebug, "IndexBlockSize")
+			    .detail("Sz", buffer.size())
+			    .detail("Encrypted", cipherKeysCtx.present());
+		}
 	}
 
 	template <class Ar>
@@ -259,8 +384,8 @@ struct IndexBlockRef {
 // Encryption: A 'chunk' gets encrypted before getting persisted if enabled. Encryption header is persisted along with
 // the chunk data to assist decryption on reads.
 //
-// Compression: A 'chunk' gets compressed before getting persisted if enabled. Compression filter (algoritm) infomration
-// is persisted as part of 'chunk metadata' to assist decompression on reads.
+// Compression: A 'chunk' gets compressed before getting persisted if enabled. Compression filter (algorithm)
+// information is persisted as part of 'chunk metadata' to assist decompression on reads.
 
 struct IndexBlobGranuleFileChunkRef {
 	constexpr static FileIdentifier file_identifier = 2814019;
@@ -286,11 +411,13 @@ struct IndexBlobGranuleFileChunkRef {
 			TraceEvent(SevDebug, "BlobChunkEncrypt_Before").detail("Chksum", chksum);
 		}
 
-		EncryptBlobCipherAes265Ctr encryptor(eKeys.textCipherKey,
-		                                     eKeys.headerCipherKey,
-		                                     cipherKeysCtx.ivRef.begin(),
-		                                     AES_256_IV_LENGTH,
-		                                     ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
+		EncryptBlobCipherAes265Ctr encryptor(
+		    eKeys.textCipherKey,
+		    eKeys.headerCipherKey,
+		    cipherKeysCtx.ivRef.begin(),
+		    AES_256_IV_LENGTH,
+		    getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
+		    BlobCipherMetrics::BLOB_GRANULE);
 		BlobCipherEncryptHeader header;
 		chunkRef.buffer =
 		    encryptor.encrypt(chunkRef.buffer.begin(), chunkRef.buffer.size(), &header, arena)->toStringRef();
@@ -319,7 +446,8 @@ struct IndexBlobGranuleFileChunkRef {
 
 		validateEncryptionHeaderDetails(eKeys, header, cipherKeysCtx.ivRef);
 
-		DecryptBlobCipherAes256Ctr decryptor(eKeys.textCipherKey, eKeys.headerCipherKey, cipherKeysCtx.ivRef.begin());
+		DecryptBlobCipherAes256Ctr decryptor(
+		    eKeys.textCipherKey, eKeys.headerCipherKey, cipherKeysCtx.ivRef.begin(), BlobCipherMetrics::BLOB_GRANULE);
 		StringRef decrypted =
 		    decryptor.decrypt(chunkRef.buffer.begin(), chunkRef.buffer.size(), header, arena)->toStringRef();
 
@@ -336,7 +464,10 @@ struct IndexBlobGranuleFileChunkRef {
 	                     const CompressionFilter compFilter,
 	                     Arena& arena) {
 		chunkRef.compressionFilter = compFilter;
-		chunkRef.buffer = CompressionUtils::compress(chunkRef.compressionFilter.get(), chunk.contents(), arena);
+		chunkRef.buffer = CompressionUtils::compress(chunkRef.compressionFilter.get(),
+		                                             chunk.contents(),
+		                                             CompressionUtils::getDefaultCompressionLevel(compFilter),
+		                                             arena);
 
 		if (BG_ENCRYPT_COMPRESS_DEBUG) {
 			XXH64_hash_t chunkChksum = XXH3_64bits(chunk.contents().begin(), chunk.contents().size());
@@ -378,19 +509,18 @@ struct IndexBlobGranuleFileChunkRef {
 			                                 : CompressionUtils::toString(CompressionFilter::NONE));
 		}
 
-		// TODO: Add version?
-		return ObjectWriter::toValue(chunkRef, Unversioned());
+		return ObjectWriter::toValue(chunkRef, IncludeVersion(ProtocolVersion::withBlobGranuleFile()));
 	}
 
 	static IndexBlobGranuleFileChunkRef fromBytes(Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx,
 	                                              StringRef buffer,
 	                                              Arena& arena) {
 		IndexBlobGranuleFileChunkRef chunkRef;
-		// TODO: Add version?
-		ObjectReader dataReader(buffer.begin(), Unversioned());
+		ObjectReader dataReader(buffer.begin(), IncludeVersion());
 		dataReader.deserialize(FileIdentifierFor<IndexBlobGranuleFileChunkRef>::value, chunkRef, arena);
 
 		if (chunkRef.encryptHeaderRef.present()) {
+			CODE_PROBE(true, "reading encrypted file chunk");
 			ASSERT(cipherKeysCtx.present());
 			chunkRef.chunkBytes = IndexBlobGranuleFileChunkRef::decrypt(cipherKeysCtx.get(), chunkRef, arena);
 		} else {
@@ -398,6 +528,7 @@ struct IndexBlobGranuleFileChunkRef {
 		}
 
 		if (chunkRef.compressionFilter.present()) {
+			CODE_PROBE(true, "reading compressed file chunk");
 			chunkRef.chunkBytes = IndexBlobGranuleFileChunkRef::decompress(chunkRef, arena);
 		} else if (!chunkRef.chunkBytes.present()) {
 			// 'Encryption' & 'Compression' aren't enabled.
@@ -441,9 +572,9 @@ struct IndexedBlobGranuleFile {
 	// Non-serialized member fields
 	StringRef fileBytes;
 
-	void init(const Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
+	void init(uint8_t fType, const Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
 		formatVersion = LATEST_BG_FORMAT_VERSION;
-		fileType = SNAPSHOT_FILE_TYPE;
+		fileType = fType;
 		chunkStartOffset = -1;
 	}
 
@@ -459,8 +590,7 @@ struct IndexedBlobGranuleFile {
 		// parse index block at head of file
 		Arena arena;
 		IndexedBlobGranuleFile file;
-		// TODO: version?
-		ObjectReader dataReader(fileBytes.begin(), Unversioned());
+		ObjectReader dataReader(fileBytes.begin(), IncludeVersion());
 		dataReader.deserialize(FileIdentifierFor<IndexedBlobGranuleFile>::value, file, arena);
 
 		file.init(fileBytes, arena, cipherKeysCtx);
@@ -521,8 +651,7 @@ struct IndexedBlobGranuleFile {
 		    IndexBlobGranuleFileChunkRef::fromBytes(cipherKeysCtx, childData, childArena);
 
 		ChildType child;
-		// TODO: version?
-		ObjectReader dataReader(chunkRef.chunkBytes.get().begin(), Unversioned());
+		ObjectReader dataReader(chunkRef.chunkBytes.get().begin(), IncludeVersion());
 		dataReader.deserialize(FileIdentifierFor<ChildType>::value, child, childArena);
 
 		// TODO implement some sort of decrypted+decompressed+deserialized cache, if this object gets reused?
@@ -542,29 +671,68 @@ struct IndexedBlobGranuleFile {
 Value serializeIndexBlock(Standalone<IndexedBlobGranuleFile>& file, Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
 	file.indexBlockRef.finalize(cipherKeysCtx, file.arena());
 
-	// TODO: version?
-	Value serialized = ObjectWriter::toValue(file, Unversioned());
+	Value serialized = ObjectWriter::toValue(file, IncludeVersion(ProtocolVersion::withBlobGranuleFile()));
 	file.chunkStartOffset = serialized.contents().size();
 
 	if (BG_ENCRYPT_COMPRESS_DEBUG) {
 		TraceEvent(SevDebug, "SerializeIndexBlock").detail("StartOffset", file.chunkStartOffset);
 	}
 
-	return ObjectWriter::toValue(file, Unversioned());
+	return ObjectWriter::toValue(file, IncludeVersion(ProtocolVersion::withBlobGranuleFile()));
 }
 
-// TODO: this should probably be in actor file with yields?
+Value serializeFileFromChunks(Standalone<IndexedBlobGranuleFile>& file,
+                              Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx,
+                              std::vector<Value>& chunks,
+                              int previousChunkBytes) {
+	Value indexBlockBytes = serializeIndexBlock(file, cipherKeysCtx);
+	int32_t indexSize = indexBlockBytes.size();
+	chunks[0] = indexBlockBytes;
+
+	// TODO: write this directly to stream to avoid extra copy?
+	Arena ret;
+
+	size_t size = indexSize + previousChunkBytes;
+	uint8_t* buffer = new (ret) uint8_t[size];
+	uint8_t* bufferStart = buffer;
+
+	int idx = 0;
+	for (auto& it : chunks) {
+		if (BG_ENCRYPT_COMPRESS_DEBUG) {
+			TraceEvent(SevDebug, "SerializeFile")
+			    .detail("ChunkIdx", idx++)
+			    .detail("Size", it.size())
+			    .detail("Offset", buffer - bufferStart);
+		}
+		buffer = it.copyTo(buffer);
+	}
+	ASSERT(size == buffer - bufferStart);
+
+	return Standalone<StringRef>(StringRef(bufferStart, size), ret);
+}
+
+// TODO: this should probably be in actor file with yields? - move writing logic to separate actor file in server?
 // TODO: optimize memory copying
 // TODO: sanity check no oversized files
-Value serializeChunkedSnapshot(Standalone<GranuleSnapshot> snapshot,
-                               int chunkCount,
+Value serializeChunkedSnapshot(const Standalone<StringRef>& fileNameRef,
+                               const Standalone<GranuleSnapshot>& snapshot,
+                               int targetChunkBytes,
                                Optional<CompressionFilter> compressFilter,
                                Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
+
+	if (BG_ENCRYPT_COMPRESS_DEBUG) {
+		TraceEvent(SevDebug, "SerializeChunkedSnapshot")
+		    .detail("FileName", fileNameRef.toString())
+		    .detail("Encrypted", cipherKeysCtx.present())
+		    .detail("Compressed", compressFilter.present());
+	}
+
+	CODE_PROBE(compressFilter.present(), "serializing compressed snapshot file");
+	CODE_PROBE(cipherKeysCtx.present(), "serializing encrypted snapshot file");
 	Standalone<IndexedBlobGranuleFile> file;
 
-	file.init(cipherKeysCtx);
+	file.init(SNAPSHOT_FILE_TYPE, cipherKeysCtx);
 
-	size_t targetChunkBytes = snapshot.expectedSize() / chunkCount;
 	size_t currentChunkBytesEstimate = 0;
 	size_t previousChunkBytes = 0;
 
@@ -572,7 +740,6 @@ Value serializeChunkedSnapshot(Standalone<GranuleSnapshot> snapshot,
 	chunks.push_back(Value()); // dummy value for index block
 	Standalone<GranuleSnapshot> currentChunk;
 
-	// fmt::print("Chunk index:\n");
 	for (int i = 0; i < snapshot.size(); i++) {
 		// TODO REMOVE sanity check
 		if (i > 0) {
@@ -583,8 +750,8 @@ Value serializeChunkedSnapshot(Standalone<GranuleSnapshot> snapshot,
 		currentChunkBytesEstimate += snapshot[i].expectedSize();
 
 		if (currentChunkBytesEstimate >= targetChunkBytes || i == snapshot.size() - 1) {
-			// TODO: protocol version
-			Value serialized = ObjectWriter::toValue(currentChunk, Unversioned());
+			Value serialized =
+			    ObjectWriter::toValue(currentChunk, IncludeVersion(ProtocolVersion::withBlobGranuleFile()));
 			Value chunkBytes =
 			    IndexBlobGranuleFileChunkRef::toBytes(cipherKeysCtx, compressFilter, serialized, file.arena());
 			chunks.push_back(chunkBytes);
@@ -613,40 +780,24 @@ Value serializeChunkedSnapshot(Standalone<GranuleSnapshot> snapshot,
 		    file.arena(), keyAfter(snapshot.back().key), previousChunkBytes);
 	}
 
-	Value indexBlockBytes = serializeIndexBlock(file, cipherKeysCtx);
-	int32_t indexSize = indexBlockBytes.size();
-	chunks[0] = indexBlockBytes;
-
-	// TODO: write this directly to stream to avoid extra copy?
-	Arena ret;
-
-	size_t size = indexSize + previousChunkBytes;
-	uint8_t* buffer = new (ret) uint8_t[size];
-
-	previousChunkBytes = 0;
-	int idx = 0;
-	for (auto& it : chunks) {
-		if (BG_ENCRYPT_COMPRESS_DEBUG) {
-			TraceEvent(SevDebug, "SerializeSnapshot")
-			    .detail("ChunkIdx", idx++)
-			    .detail("Size", it.size())
-			    .detail("Offset", previousChunkBytes);
-		}
-
-		memcpy(buffer + previousChunkBytes, it.begin(), it.size());
-		previousChunkBytes += it.size();
-	}
-	ASSERT(size == previousChunkBytes);
-
-	return Standalone<StringRef>(StringRef(buffer, size), ret);
+	return serializeFileFromChunks(file, cipherKeysCtx, chunks, previousChunkBytes);
 }
 
 // TODO: use redwood prefix trick to optimize cpu comparison
-static Arena loadSnapshotFile(const StringRef& snapshotData,
-                              KeyRangeRef keyRange,
-                              std::map<KeyRef, ValueRef>& dataMap,
-                              Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
-	Arena rootArena;
+static Standalone<VectorRef<ParsedDeltaBoundaryRef>> loadSnapshotFile(
+    const Standalone<StringRef>& fileName,
+    const StringRef& snapshotData,
+    const KeyRangeRef& keyRange,
+    Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
+	Standalone<VectorRef<ParsedDeltaBoundaryRef>> results;
+
+	if (BG_ENCRYPT_COMPRESS_DEBUG) {
+		TraceEvent(SevDebug, "LoadChunkedSnapshot")
+		    .detail("FileName", fileName.toString())
+		    .detail("RangeBegin", keyRange.begin.printable())
+		    .detail("RangeEnd", keyRange.end.printable())
+		    .detail("Encrypted", cipherKeysCtx.present());
+	}
 
 	Standalone<IndexedBlobGranuleFile> file = IndexedBlobGranuleFile::fromFileBytes(snapshotData, cipherKeysCtx);
 
@@ -655,21 +806,25 @@ static Arena loadSnapshotFile(const StringRef& snapshotData,
 
 	// empty snapshot file
 	if (file.indexBlockRef.block.children.empty()) {
-		return rootArena;
+		return results;
 	}
 
 	ASSERT(file.indexBlockRef.block.children.size() >= 2);
 
-	// TODO: refactor this out of delta tree
-	// int commonPrefixLen = commonPrefixLength(index.dataBlockOffsets.front().first,
-	// index.dataBlockOffsets.back().first);
-
 	// find range of blocks needed to read
 	ChildBlockPointerRef* currentBlock = file.findStartBlock(keyRange.begin);
 
-	// FIXME: optimize cpu comparisons here in first/last partial blocks, doing entire blocks at once based on
-	// comparison, and using shared prefix for key comparison
-	while (currentBlock != (file.indexBlockRef.block.children.end() - 1) && keyRange.end > currentBlock->key) {
+	if (currentBlock == (file.indexBlockRef.block.children.end() - 1) || keyRange.end <= currentBlock->key) {
+		return results;
+	}
+
+	bool lastBlock = false;
+
+	// FIXME: shared prefix for key comparison
+	while (!lastBlock) {
+		auto nextBlock = currentBlock;
+		nextBlock++;
+		lastBlock = (nextBlock == (file.indexBlockRef.block.children.end() - 1)) || (keyRange.end <= nextBlock->key);
 		Standalone<GranuleSnapshot> dataBlock =
 		    file.getChild<GranuleSnapshot>(currentBlock, cipherKeysCtx, file.chunkStartOffset);
 		ASSERT(!dataBlock.empty());
@@ -677,21 +832,332 @@ static Arena loadSnapshotFile(const StringRef& snapshotData,
 
 		bool anyRows = false;
 		for (auto& entry : dataBlock) {
-			if (entry.key >= keyRange.begin && entry.key < keyRange.end) {
-				dataMap.insert({ entry.key, entry.value });
+			if (!results.empty() && !lastBlock) {
+				// no key comparisons needed
+				results.emplace_back(results.arena(), entry);
 				anyRows = true;
+			} else if ((!results.empty() || entry.key >= keyRange.begin) && (!lastBlock || entry.key < keyRange.end)) {
+				results.emplace_back(results.arena(), entry);
+				anyRows = true;
+			} else if (!results.empty() && lastBlock) {
+				break;
 			}
 		}
 		if (anyRows) {
-			rootArena.dependsOn(dataBlock.arena());
+			results.arena().dependsOn(dataBlock.arena());
 		}
 		currentBlock++;
 	}
 
-	return rootArena;
+	return results;
 }
 
-static void applyDelta(KeyRangeRef keyRange, MutationRef m, std::map<KeyRef, ValueRef>& dataMap) {
+typedef std::map<Key, Standalone<DeltaBoundaryRef>> SortedDeltasT;
+
+// FIXME: optimize all of this with common prefix comparison stuff
+SortedDeltasT::iterator insertMutationBoundary(SortedDeltasT& deltasByKey, const KeyRef& boundary) {
+	// Find the first split point in buffer that is >= key
+	auto it = deltasByKey.lower_bound(boundary);
+
+	// Since the map contains fileRange already, we had to have found something
+	ASSERT(it != deltasByKey.end());
+	if (it->first == boundary) {
+		return it;
+	}
+
+	// new boundary, using find as insert hint
+	it = deltasByKey.insert(it, { boundary, Standalone<DeltaBoundaryRef>() });
+
+	// look back at previous entry to see if this boundary is already cleared to at a prior version
+	ASSERT(it != deltasByKey.begin());
+	auto itPrev = it;
+	--itPrev;
+
+	if (itPrev->second.clearVersion.present()) {
+		it->second.clearVersion = itPrev->second.clearVersion;
+		it->second.values.push_back(it->second.arena(), ValueAndVersionRef(it->second.clearVersion.get()));
+	}
+
+	return it;
+}
+
+void updateMutationBoundary(Standalone<DeltaBoundaryRef>& boundary, const ValueAndVersionRef& update) {
+	if (update.isSet()) {
+		if (boundary.values.empty() || boundary.values.back().version < update.version) {
+			// duplicate same set even if it's the same as the last one, so beginVersion reads still get updates
+			boundary.values.push_back(boundary.arena(), update);
+		} else {
+			CODE_PROBE(true, "multiple boundary updates at same version (set)");
+			// preserve inter-mutation order by replacing this one
+			boundary.values.back() = update;
+		}
+	} else {
+		if (boundary.values.empty() ||
+		    (boundary.values.back().isSet() && boundary.values.back().version < update.version)) {
+			// don't duplicate single-key clears in order if previous was also a clear, since it's a no-op when starting
+			// with beginVersion
+			boundary.values.push_back(boundary.arena(), update);
+		} else if (!boundary.values.empty() && boundary.values.back().version == update.version) {
+			CODE_PROBE(true, "multiple boundary updates at same version (clear)");
+			if (boundary.values.back().isSet()) {
+				// if the last 2 updates were clear @ v1 and set @ v2, and we now have a clear at v2, just pop off the
+				// set and leave the previous clear. Otherwise, just set the last set to a clear
+				if (boundary.values.size() >= 2 && boundary.values[boundary.values.size() - 2].isClear()) {
+					CODE_PROBE(true, "clear then set/clear at same version optimization");
+					boundary.values.pop_back();
+				} else {
+					boundary.values.back() = update;
+				}
+			} // else we have 2 consecutive clears at this version, no-op
+		}
+	}
+}
+
+void insertSortedDelta(const MutationRef& m,
+                       const Version version,
+                       const KeyRangeRef& fileRange,
+                       SortedDeltasT& deltasByKey) {
+	// TODO REMOVE validation
+	ASSERT(fileRange.contains(m.param1));
+	if (m.type == MutationRef::ClearRange) {
+		ASSERT(m.param2 <= fileRange.end);
+		// handle single key clear more efficiently
+		if (equalsKeyAfter(m.param1, m.param2)) {
+			SortedDeltasT::iterator key = insertMutationBoundary(deltasByKey, m.param1);
+			updateMutationBoundary(key->second, ValueAndVersionRef(version));
+		} else {
+			// Update each boundary in the cleared range
+			SortedDeltasT::iterator begin = insertMutationBoundary(deltasByKey, m.param1);
+			SortedDeltasT::iterator end = insertMutationBoundary(deltasByKey, m.param2);
+			while (begin != end) {
+				// Set the rangeClearedVersion if not set
+				if (!begin->second.clearVersion.present()) {
+					begin->second.clearVersion = version;
+				}
+
+				// Add a clear to values if it's empty or the last item is not a clear
+				if (begin->second.values.empty() || begin->second.values.back().isSet()) {
+					updateMutationBoundary(begin->second, ValueAndVersionRef(version));
+				}
+				++begin;
+			}
+		}
+	} else {
+		Standalone<DeltaBoundaryRef>& bound = insertMutationBoundary(deltasByKey, m.param1)->second;
+		updateMutationBoundary(bound, ValueAndVersionRef(version, m.param2));
+	}
+}
+
+// TODO: investigate more cpu-efficient sorting methods. Potential options:
+// 1) Replace std::map with ART mutation buffer
+// 2) sort updates and clear endpoints by (key, version), and keep track of active clears.
+void sortDeltasByKey(const Standalone<GranuleDeltas>& deltasByVersion,
+                     const KeyRangeRef& fileRange,
+                     SortedDeltasT& deltasByKey) {
+	if (deltasByVersion.empty()) {
+		return;
+	}
+	if (deltasByKey.empty()) {
+		deltasByKey.insert({ fileRange.begin, Standalone<DeltaBoundaryRef>() });
+		deltasByKey.insert({ fileRange.end, Standalone<DeltaBoundaryRef>() });
+	}
+	for (auto& it : deltasByVersion) {
+		for (auto& m : it.mutations) {
+			insertSortedDelta(m, it.version, fileRange, deltasByKey);
+		}
+	}
+
+	// TODO: could do a scan through map and coalesce clears (if any boundaries with exactly 1 mutation (clear) and same
+	// clearVersion as previous guy)
+}
+
+// FIXME: Could maybe reduce duplicated code between this and chunkedSnapshot for chunking
+Value serializeChunkedDeltaFile(const Standalone<StringRef>& fileNameRef,
+                                const Standalone<GranuleDeltas>& deltas,
+                                const KeyRangeRef& fileRange,
+                                int chunkSize,
+                                Optional<CompressionFilter> compressFilter,
+                                Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
+	if (BG_ENCRYPT_COMPRESS_DEBUG) {
+		TraceEvent(SevDebug, "SerializeChunkedDelta")
+		    .detail("Filename", fileNameRef.toString())
+		    .detail("RangeBegin", fileRange.begin.printable())
+		    .detail("RangeEnd", fileRange.end.printable())
+		    .detail("Encrypted", cipherKeysCtx.present())
+		    .detail("Compressed", compressFilter.present());
+	}
+
+	CODE_PROBE(compressFilter.present(), "serializing compressed delta file");
+	CODE_PROBE(cipherKeysCtx.present(), "serializing encrypted delta file");
+	Standalone<IndexedBlobGranuleFile> file;
+
+	file.init(DELTA_FILE_TYPE, cipherKeysCtx);
+
+	// build in-memory version of boundaries - TODO separate functions
+	SortedDeltasT boundaries;
+	sortDeltasByKey(deltas, fileRange, boundaries);
+
+	std::vector<Value> chunks;
+	chunks.push_back(Value()); // dummy value for index block
+
+	Standalone<GranuleSortedDeltas> currentChunk;
+	size_t currentChunkBytesEstimate = 0;
+	size_t previousChunkBytes = 0;
+
+	// TODO REMOVE - for validation
+	KeyRef lastKey;
+	int i = 0;
+	for (auto& it : boundaries) {
+		// TODO REMOVE sanity check
+		if (i > 0) {
+			ASSERT(lastKey < it.first);
+		}
+		lastKey = it.first;
+		it.second.key = it.first;
+
+		currentChunk.boundaries.push_back_deep(currentChunk.arena(), it.second);
+		currentChunkBytesEstimate += it.second.totalSize();
+
+		if (currentChunkBytesEstimate >= chunkSize || i == boundaries.size() - 1) {
+			Value serialized =
+			    ObjectWriter::toValue(currentChunk, IncludeVersion(ProtocolVersion::withBlobGranuleFile()));
+			Value chunkBytes =
+			    IndexBlobGranuleFileChunkRef::toBytes(cipherKeysCtx, compressFilter, serialized, file.arena());
+			chunks.push_back(chunkBytes);
+
+			// TODO remove validation
+			if (!file.indexBlockRef.block.children.empty()) {
+				ASSERT(file.indexBlockRef.block.children.back().key < currentChunk.boundaries.begin()->key);
+			}
+			file.indexBlockRef.block.children.emplace_back_deep(
+			    file.arena(), currentChunk.boundaries.begin()->key, previousChunkBytes);
+
+			if (BG_ENCRYPT_COMPRESS_DEBUG) {
+				TraceEvent(SevDebug, "ChunkSize")
+				    .detail("ChunkBytes", chunkBytes.size())
+				    .detail("PrvChunkBytes", previousChunkBytes);
+			}
+
+			previousChunkBytes += chunkBytes.size();
+			currentChunkBytesEstimate = 0;
+			currentChunk = Standalone<GranuleSortedDeltas>();
+		}
+		i++;
+	}
+	ASSERT(currentChunk.boundaries.empty());
+	if (!deltas.empty()) {
+		file.indexBlockRef.block.children.emplace_back_deep(file.arena(), fileRange.end, previousChunkBytes);
+	}
+
+	return serializeFileFromChunks(file, cipherKeysCtx, chunks, previousChunkBytes);
+}
+
+ParsedDeltaBoundaryRef deltaAtVersion(const DeltaBoundaryRef& delta, Version beginVersion, Version readVersion) {
+	bool clearAfter = delta.clearVersion.present() && readVersion >= delta.clearVersion.get() &&
+	                  beginVersion <= delta.clearVersion.get();
+	if (delta.values.empty()) {
+		return ParsedDeltaBoundaryRef(delta.key, clearAfter);
+	}
+	auto valueAtVersion = std::lower_bound(delta.values.begin(),
+	                                       delta.values.end(),
+	                                       ValueAndVersionRef(readVersion),
+	                                       ValueAndVersionRef::OrderByVersion());
+	if (valueAtVersion == delta.values.begin() && readVersion < valueAtVersion->version) {
+		// deltas are all higher than read version
+		return ParsedDeltaBoundaryRef(delta.key, clearAfter);
+	}
+	// lower_bound() found version >= readVersion, so if we're at the end or it's not equal, go back one
+	if (valueAtVersion == delta.values.end() || valueAtVersion->version > readVersion) {
+		valueAtVersion--;
+	}
+	ASSERT(readVersion >= valueAtVersion->version);
+	// now, handle beginVersion (if update < beginVersion, it's a noop)
+	if (valueAtVersion->version < beginVersion) {
+		return ParsedDeltaBoundaryRef(delta.key, clearAfter);
+	} else {
+		return ParsedDeltaBoundaryRef(delta.key, clearAfter, *valueAtVersion);
+	}
+}
+
+// The arena owns the BoundaryDeltaRef struct data but the StringRef pointers point to data in deltaData, to avoid extra
+// copying
+Standalone<VectorRef<ParsedDeltaBoundaryRef>> loadChunkedDeltaFile(const Standalone<StringRef>& fileNameRef,
+                                                                   const StringRef& deltaData,
+                                                                   const KeyRangeRef& keyRange,
+                                                                   Version beginVersion,
+                                                                   Version readVersion,
+                                                                   Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx,
+                                                                   bool& startClear) {
+	Standalone<VectorRef<ParsedDeltaBoundaryRef>> deltas;
+	Standalone<IndexedBlobGranuleFile> file = IndexedBlobGranuleFile::fromFileBytes(deltaData, cipherKeysCtx);
+
+	ASSERT(file.fileType == DELTA_FILE_TYPE);
+	ASSERT(file.chunkStartOffset > 0);
+
+	// empty delta file
+	if (file.indexBlockRef.block.children.empty()) {
+		return deltas;
+	}
+
+	ASSERT(file.indexBlockRef.block.children.size() >= 2);
+
+	// find range of blocks needed to read
+	ChildBlockPointerRef* currentBlock = file.findStartBlock(keyRange.begin);
+
+	if (currentBlock == (file.indexBlockRef.block.children.end() - 1) || keyRange.end <= currentBlock->key) {
+		// empty, done
+		return deltas;
+	}
+
+	// FIXME: shared prefix for key comparison
+	// FIXME: could cpu optimize first block a bit more by seeking right to start
+	bool lastBlock = false;
+	bool prevClearAfter = false;
+	while (!lastBlock) {
+		auto nextBlock = currentBlock;
+		nextBlock++;
+		lastBlock = (nextBlock == file.indexBlockRef.block.children.end() - 1) || keyRange.end <= nextBlock->key;
+
+		Standalone<GranuleSortedDeltas> deltaBlock =
+		    file.getChild<GranuleSortedDeltas>(currentBlock, cipherKeysCtx, file.chunkStartOffset);
+		ASSERT(!deltaBlock.boundaries.empty());
+		ASSERT(currentBlock->key == deltaBlock.boundaries.front().key);
+
+		// TODO refactor this into function to share with memory deltas
+		bool blockMemoryUsed = false;
+
+		for (auto& entry : deltaBlock.boundaries) {
+			ParsedDeltaBoundaryRef boundary = deltaAtVersion(entry, beginVersion, readVersion);
+			if (deltas.empty() && entry.key < keyRange.begin) {
+				startClear = boundary.clearAfter;
+				prevClearAfter = boundary.clearAfter;
+			} else if (!lastBlock || entry.key < keyRange.end) {
+				if (!boundary.redundant(prevClearAfter)) {
+					deltas.push_back(deltas.arena(), boundary);
+					blockMemoryUsed = true;
+					prevClearAfter = boundary.clearAfter;
+				}
+			} else {
+				// TODO REMOVE validation
+				ASSERT(lastBlock);
+				break;
+			}
+		}
+		if (blockMemoryUsed) {
+			deltas.arena().dependsOn(deltaBlock.arena());
+		}
+		currentBlock++;
+	}
+
+	// TODO REMOVE eventually? order sanity check for parsed deltas
+	for (int i = 0; i < deltas.size() - 1; i++) {
+		ASSERT(deltas[i].key < deltas[i + 1].key);
+	}
+
+	return deltas;
+}
+
+static void applyDelta(const KeyRangeRef& keyRange, const MutationRef& m, std::map<KeyRef, ValueRef>& dataMap) {
 	if (m.type == MutationRef::ClearRange) {
 		if (m.param2 <= keyRange.begin || m.param1 >= keyRange.end) {
 			return;
@@ -728,12 +1194,12 @@ static void applyDelta(KeyRangeRef keyRange, MutationRef m, std::map<KeyRef, Val
 	}
 }
 
-static void applyDeltas(const GranuleDeltas& deltas,
-                        KeyRangeRef keyRange,
-                        Version beginVersion,
-                        Version readVersion,
-                        Version& lastFileEndVersion,
-                        std::map<KeyRef, ValueRef>& dataMap) {
+static void applyDeltasByVersion(const GranuleDeltas& deltas,
+                                 const KeyRangeRef& keyRange,
+                                 Version beginVersion,
+                                 Version readVersion,
+                                 Version& lastFileEndVersion,
+                                 std::map<KeyRef, ValueRef>& dataMap) {
 	if (deltas.empty()) {
 		return;
 	}
@@ -768,32 +1234,186 @@ static void applyDeltas(const GranuleDeltas& deltas,
 	lastFileEndVersion = deltas.back().version;
 }
 
-static Arena loadDeltaFile(StringRef deltaData,
-                           KeyRangeRef keyRange,
-                           Version beginVersion,
-                           Version readVersion,
-                           Version& lastFileEndVersion,
-                           std::map<KeyRef, ValueRef>& dataMap) {
-	Arena parseArena;
-	GranuleDeltas deltas;
-	ObjectReader reader(deltaData.begin(), Unversioned());
-	reader.deserialize(FileIdentifierFor<GranuleDeltas>::value, deltas, parseArena);
+// TODO: could optimize this slightly to avoid tracking multiple updates for the same key at all since it's always then
+// collapsed to the last one
+Standalone<VectorRef<ParsedDeltaBoundaryRef>> sortMemoryDeltas(const GranuleDeltas& memoryDeltas,
+                                                               const KeyRangeRef& granuleRange,
+                                                               const KeyRangeRef& readRange,
+                                                               Version beginVersion,
+                                                               Version readVersion) {
+	ASSERT(!memoryDeltas.empty());
 
-	if (BG_READ_DEBUG) {
-		fmt::print("Parsed {} deltas from file\n", deltas.size());
+	// filter by request range first
+	SortedDeltasT versionedBoundaries;
+	if (versionedBoundaries.empty()) {
+		versionedBoundaries.insert({ readRange.begin, Standalone<DeltaBoundaryRef>() });
+		versionedBoundaries.insert({ readRange.end, Standalone<DeltaBoundaryRef>() });
 	}
-
-	// TODO REMOVE sanity check
-	for (int i = 0; i < deltas.size() - 1; i++) {
-		if (deltas[i].version > deltas[i + 1].version) {
-			fmt::print(
-			    "BG VERSION ORDER VIOLATION IN DELTA FILE: '{0}', '{1}'\n", deltas[i].version, deltas[i + 1].version);
+	for (auto& it : memoryDeltas) {
+		for (auto& m : it.mutations) {
+			if (m.type == MutationRef::ClearRange) {
+				if (m.param2 > readRange.begin && m.param1 < readRange.end) {
+					KeyRangeRef clearRangeClipped = readRange & KeyRangeRef(m.param1, m.param2);
+					MutationRef clearClipped(
+					    MutationRef::Type::ClearRange, clearRangeClipped.begin, clearRangeClipped.end);
+					insertSortedDelta(clearClipped, it.version, granuleRange, versionedBoundaries);
+				}
+			} else {
+				ASSERT(m.type == MutationRef::SetValue);
+				if (readRange.contains(m.param1)) {
+					insertSortedDelta(m, it.version, granuleRange, versionedBoundaries);
+				}
+			}
 		}
-		ASSERT(deltas[i].version <= deltas[i + 1].version);
 	}
 
-	applyDeltas(deltas, keyRange, beginVersion, readVersion, lastFileEndVersion, dataMap);
-	return parseArena;
+	// parse and collapse based on version
+	bool prevClearAfter = false;
+	Standalone<VectorRef<ParsedDeltaBoundaryRef>> deltas;
+
+	// remove extra ranges inserted from clears that partially overlap read range
+	auto itBegin = versionedBoundaries.begin();
+	while (itBegin->first < readRange.begin) {
+		++itBegin;
+	}
+	auto itEnd = versionedBoundaries.end();
+	itEnd--;
+	while (itEnd->first > readRange.end) {
+		itEnd--;
+	}
+	itEnd++;
+
+	while (itBegin != itEnd) {
+		itBegin->second.key = itBegin->first;
+		ParsedDeltaBoundaryRef boundary = deltaAtVersion(itBegin->second, beginVersion, readVersion);
+		if (!boundary.redundant(prevClearAfter)) {
+			deltas.push_back_deep(deltas.arena(), boundary);
+			prevClearAfter = boundary.clearAfter;
+		}
+		++itBegin;
+	}
+
+	return deltas;
+}
+
+// does a sorted merge of the delta streams.
+// In terms of write precedence, streams[i] < streams[i+1]
+// Handles range clears by tracking the active clears when they start
+struct MergeStreamNext {
+	KeyRef key;
+	int16_t streamIdx;
+	int dataIdx;
+};
+
+// the sort order is logically lower by key, and then higher by streamIdx
+// because a priority queue is backwards, we invert that
+struct OrderForPriorityQueue {
+	int commonPrefixLen;
+	OrderForPriorityQueue(int commonPrefixLen) : commonPrefixLen(commonPrefixLen) {}
+
+	bool operator()(MergeStreamNext const& a, MergeStreamNext const& b) const {
+		int keyCmp = a.key.compareSuffix(b.key, commonPrefixLen);
+		if (keyCmp != 0) {
+			return keyCmp > 0; // reverse
+		}
+		return a.streamIdx < b.streamIdx;
+	}
+};
+
+typedef std::priority_queue<MergeStreamNext, std::vector<MergeStreamNext>, OrderForPriorityQueue> MergePQ;
+
+static RangeResult mergeDeltaStreams(const BlobGranuleChunkRef& chunk,
+                                     const std::vector<Standalone<VectorRef<ParsedDeltaBoundaryRef>>>& streams,
+                                     const std::vector<bool> startClears) {
+	ASSERT(streams.size() < std::numeric_limits<int16_t>::max());
+	ASSERT(startClears.size() == streams.size());
+
+	int prefixLen = commonPrefixLength(chunk.keyRange.begin, chunk.keyRange.end);
+
+	// next element for each stream
+	MergePQ next = MergePQ(OrderForPriorityQueue(prefixLen));
+
+	// efficiently find the highest stream's active clear
+	std::set<int16_t, std::greater<int16_t>> activeClears;
+	int16_t maxActiveClear = -1;
+
+	// check if a given stream is actively clearing
+	bool clearActive[streams.size()];
+	for (int16_t i = 0; i < streams.size(); i++) {
+		clearActive[i] = startClears[i];
+		if (startClears[i]) {
+			activeClears.insert(i);
+			maxActiveClear = i;
+		}
+		if (streams[i].empty()) {
+			// single clear that entirely encases partial read bounds
+			ASSERT(clearActive[i]);
+		} else {
+			MergeStreamNext item;
+			item.key = streams[i][0].key;
+			item.streamIdx = i;
+			item.dataIdx = 0;
+			next.push(item);
+		}
+	}
+
+	RangeResult result;
+	std::vector<MergeStreamNext> cur;
+	cur.reserve(streams.size());
+	while (!next.empty()) {
+		cur.clear();
+		cur.push_back(next.top());
+		next.pop();
+
+		// next.top().key == cur.front().key but with suffix comparison
+		while (!next.empty() && cur.front().key.compareSuffix(next.top().key, prefixLen) == 0) {
+			cur.push_back(next.top());
+			next.pop();
+		}
+
+		// un-set clears and find latest value for key (if present)
+		bool foundValue = false;
+		for (auto& it : cur) {
+			auto& v = streams[it.streamIdx][it.dataIdx];
+			if (clearActive[it.streamIdx]) {
+				clearActive[it.streamIdx] = false;
+				activeClears.erase(it.streamIdx);
+				if (it.streamIdx == maxActiveClear) {
+					// re-get max active clear
+					maxActiveClear = activeClears.empty() ? -1 : *activeClears.begin();
+				}
+			}
+
+			// find value for this key (if any)
+			if (!foundValue && !v.isNoOp()) {
+				foundValue = true;
+				// if it's a clear, or maxActiveClear is higher, no value for this key
+				if (v.isSet() && maxActiveClear < it.streamIdx) {
+					KeyRef finalKey =
+					    chunk.tenantPrefix.present() ? v.key.removePrefix(chunk.tenantPrefix.get()) : v.key;
+					result.push_back_deep(result.arena(), KeyValueRef(finalKey, v.value));
+				}
+			}
+		}
+
+		// advance streams and start clearAfter
+		for (auto& it : cur) {
+			if (streams[it.streamIdx][it.dataIdx].clearAfter) {
+				clearActive[it.streamIdx] = true;
+				activeClears.insert(it.streamIdx);
+				maxActiveClear = std::max(maxActiveClear, it.streamIdx);
+			}
+			// TODO: implement skipping if large clear!!
+			// if (maxClearIdx > it.streamIdx) - skip
+			it.dataIdx++;
+			if (it.dataIdx < streams[it.streamIdx].size()) {
+				it.key = streams[it.streamIdx][it.dataIdx].key;
+				next.push(it);
+			}
+		}
+	}
+
+	return result;
 }
 
 RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
@@ -811,8 +1431,6 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
 	// FIXME: probably some threshold of a small percentage of the data is actually changed, where it makes sense to
 	// just to dependsOn instead of copy, to use a little extra memory footprint to help cpu?
 	Arena arena;
-	std::map<KeyRef, ValueRef> dataMap;
-	Version lastFileEndVersion = invalidVersion;
 	KeyRange requestRange;
 	if (chunk.tenantPrefix.present()) {
 		requestRange = keyRange.withPrefix(chunk.tenantPrefix.get());
@@ -820,41 +1438,79 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
 		requestRange = keyRange;
 	}
 
+	std::vector<Standalone<VectorRef<ParsedDeltaBoundaryRef>>> streams;
+	std::vector<bool> startClears;
+	// +1 for possible snapshot, +1 for possible memory deltas
+	streams.reserve(chunk.deltaFiles.size() + 2);
+
 	if (snapshotData.present()) {
-		Arena snapshotArena = loadSnapshotFile(snapshotData.get(), requestRange, dataMap, chunk.cipherKeysCtx);
-		arena.dependsOn(snapshotArena);
+		ASSERT(chunk.snapshotFile.present());
+		Standalone<VectorRef<ParsedDeltaBoundaryRef>> snapshotRows =
+		    loadSnapshotFile(chunk.snapshotFile.get().filename,
+		                     snapshotData.get(),
+		                     requestRange,
+		                     chunk.snapshotFile.get().cipherKeysCtx);
+		if (!snapshotRows.empty()) {
+			streams.push_back(snapshotRows);
+			startClears.push_back(false);
+			arena.dependsOn(streams.back().arena());
+		}
 	}
 
 	if (BG_READ_DEBUG) {
 		fmt::print("Applying {} delta files\n", chunk.deltaFiles.size());
 	}
 	for (int deltaIdx = 0; deltaIdx < chunk.deltaFiles.size(); deltaIdx++) {
-		Arena deltaArena = loadDeltaFile(
-		    deltaFileData[deltaIdx], requestRange, beginVersion, readVersion, lastFileEndVersion, dataMap);
-		arena.dependsOn(deltaArena);
+		bool startClear = false;
+		auto deltaRows = loadChunkedDeltaFile(chunk.deltaFiles[deltaIdx].filename,
+		                                      deltaFileData[deltaIdx],
+		                                      requestRange,
+		                                      beginVersion,
+		                                      readVersion,
+		                                      chunk.deltaFiles[deltaIdx].cipherKeysCtx,
+		                                      startClear);
+		if (startClear || !deltaRows.empty()) {
+			streams.push_back(deltaRows);
+			startClears.push_back(startClear);
+			arena.dependsOn(streams.back().arena());
+		}
+		arena.dependsOn(deltaRows.arena());
 	}
 	if (BG_READ_DEBUG) {
 		fmt::print("Applying {} memory deltas\n", chunk.newDeltas.size());
 	}
-	applyDeltas(chunk.newDeltas, requestRange, beginVersion, readVersion, lastFileEndVersion, dataMap);
-
-	RangeResult ret;
-	for (auto& it : dataMap) {
-		ret.push_back_deep(
-		    ret.arena(),
-		    KeyValueRef(chunk.tenantPrefix.present() ? it.first.removePrefix(chunk.tenantPrefix.get()) : it.first,
-		                it.second));
+	if (!chunk.newDeltas.empty()) {
+		// TODO REMOVE validation
+		ASSERT(beginVersion <= chunk.newDeltas.front().version);
+		ASSERT(readVersion >= chunk.newDeltas.back().version);
+		auto memoryRows = sortMemoryDeltas(chunk.newDeltas, chunk.keyRange, requestRange, beginVersion, readVersion);
+		if (!memoryRows.empty()) {
+			streams.push_back(memoryRows);
+			startClears.push_back(false);
+			arena.dependsOn(streams.back().arena());
+		}
 	}
 
-	return ret;
+	return mergeDeltaStreams(chunk, streams, startClears);
 }
 
+struct GranuleLoadFreeHandle : NonCopyable, ReferenceCounted<GranuleLoadFreeHandle> {
+	const ReadBlobGranuleContext* granuleContext;
+	int64_t loadId;
+
+	GranuleLoadFreeHandle(const ReadBlobGranuleContext* granuleContext, int64_t loadId)
+	  : granuleContext(granuleContext), loadId(loadId) {}
+
+	~GranuleLoadFreeHandle() { granuleContext->free_load_f(loadId, granuleContext->userContext); }
+};
+
 struct GranuleLoadIds {
 	Optional<int64_t> snapshotId;
 	std::vector<int64_t> deltaIds;
+	std::vector<Reference<GranuleLoadFreeHandle>> freeHandles;
 };
 
-static void startLoad(const ReadBlobGranuleContext granuleContext,
+static void startLoad(const ReadBlobGranuleContext* granuleContext,
                       const BlobGranuleChunkRef& chunk,
                       GranuleLoadIds& loadIds) {
 
@@ -864,12 +1520,13 @@ static void startLoad(const ReadBlobGranuleContext granuleContext,
 		// FIXME: remove when we implement file multiplexing
 		ASSERT(chunk.snapshotFile.get().offset == 0);
 		ASSERT(chunk.snapshotFile.get().length == chunk.snapshotFile.get().fullFileLength);
-		loadIds.snapshotId = granuleContext.start_load_f(snapshotFname.c_str(),
-		                                                 snapshotFname.size(),
-		                                                 chunk.snapshotFile.get().offset,
-		                                                 chunk.snapshotFile.get().length,
-		                                                 chunk.snapshotFile.get().fullFileLength,
-		                                                 granuleContext.userContext);
+		loadIds.snapshotId = granuleContext->start_load_f(snapshotFname.c_str(),
+		                                                  snapshotFname.size(),
+		                                                  chunk.snapshotFile.get().offset,
+		                                                  chunk.snapshotFile.get().length,
+		                                                  chunk.snapshotFile.get().fullFileLength,
+		                                                  granuleContext->userContext);
+		loadIds.freeHandles.push_back(makeReference<GranuleLoadFreeHandle>(granuleContext, loadIds.snapshotId.get()));
 	}
 	loadIds.deltaIds.reserve(chunk.deltaFiles.size());
 	for (int deltaFileIdx = 0; deltaFileIdx < chunk.deltaFiles.size(); deltaFileIdx++) {
@@ -877,13 +1534,14 @@ static void startLoad(const ReadBlobGranuleContext granuleContext,
 		// FIXME: remove when we implement file multiplexing
 		ASSERT(chunk.deltaFiles[deltaFileIdx].offset == 0);
 		ASSERT(chunk.deltaFiles[deltaFileIdx].length == chunk.deltaFiles[deltaFileIdx].fullFileLength);
-		int64_t deltaLoadId = granuleContext.start_load_f(deltaFName.c_str(),
-		                                                  deltaFName.size(),
-		                                                  chunk.deltaFiles[deltaFileIdx].offset,
-		                                                  chunk.deltaFiles[deltaFileIdx].length,
-		                                                  chunk.deltaFiles[deltaFileIdx].fullFileLength,
-		                                                  granuleContext.userContext);
+		int64_t deltaLoadId = granuleContext->start_load_f(deltaFName.c_str(),
+		                                                   deltaFName.size(),
+		                                                   chunk.deltaFiles[deltaFileIdx].offset,
+		                                                   chunk.deltaFiles[deltaFileIdx].length,
+		                                                   chunk.deltaFiles[deltaFileIdx].fullFileLength,
+		                                                   granuleContext->userContext);
 		loadIds.deltaIds.push_back(deltaLoadId);
+		loadIds.freeHandles.push_back(makeReference<GranuleLoadFreeHandle>(granuleContext, deltaLoadId));
 	}
 }
 
@@ -891,7 +1549,8 @@ ErrorOr<RangeResult> loadAndMaterializeBlobGranules(const Standalone<VectorRef<B
                                                     const KeyRangeRef& keyRange,
                                                     Version beginVersion,
                                                     Version readVersion,
-                                                    ReadBlobGranuleContext granuleContext) {
+                                                    ReadBlobGranuleContext granuleContext,
+                                                    GranuleMaterializeStats& stats) {
 	int64_t parallelism = granuleContext.granuleParallelism;
 	if (parallelism < 1) {
 		parallelism = 1;
@@ -901,18 +1560,19 @@ ErrorOr<RangeResult> loadAndMaterializeBlobGranules(const Standalone<VectorRef<B
 	}
 
 	GranuleLoadIds loadIds[files.size()];
-
-	// Kick off first file reads if parallelism > 1
-	for (int i = 0; i < parallelism - 1 && i < files.size(); i++) {
-		startLoad(granuleContext, files[i], loadIds[i]);
-	}
+	int64_t inputBytes = 0;
+	int64_t outputBytes = 0;
 
 	try {
+		// Kick off first file reads if parallelism > 1
+		for (int i = 0; i < parallelism - 1 && i < files.size(); i++) {
+			startLoad(&granuleContext, files[i], loadIds[i]);
+		}
 		RangeResult results;
 		for (int chunkIdx = 0; chunkIdx < files.size(); chunkIdx++) {
 			// Kick off files for this granule if parallelism == 1, or future granule if parallelism > 1
 			if (chunkIdx + parallelism - 1 < files.size()) {
-				startLoad(granuleContext, files[chunkIdx + parallelism - 1], loadIds[chunkIdx + parallelism - 1]);
+				startLoad(&granuleContext, files[chunkIdx + parallelism - 1], loadIds[chunkIdx + parallelism - 1]);
 			}
 
 			RangeResult chunkRows;
@@ -926,9 +1586,11 @@ ErrorOr<RangeResult> loadAndMaterializeBlobGranules(const Standalone<VectorRef<B
 				if (!snapshotData.get().begin()) {
 					return ErrorOr<RangeResult>(blob_granule_file_load_error());
 				}
+				inputBytes += snapshotData.get().size();
 			}
 
-			StringRef deltaData[files[chunkIdx].deltaFiles.size()];
+			// +1 to avoid UBSAN variable length array of size zero
+			StringRef deltaData[files[chunkIdx].deltaFiles.size() + 1];
 			for (int i = 0; i < files[chunkIdx].deltaFiles.size(); i++) {
 				deltaData[i] =
 				    StringRef(granuleContext.get_load_f(loadIds[chunkIdx].deltaIds[i], granuleContext.userContext),
@@ -937,22 +1599,25 @@ ErrorOr<RangeResult> loadAndMaterializeBlobGranules(const Standalone<VectorRef<B
 				if (!deltaData[i].begin()) {
 					return ErrorOr<RangeResult>(blob_granule_file_load_error());
 				}
+				inputBytes += deltaData[i].size();
 			}
 
+			inputBytes += files[chunkIdx].newDeltas.expectedSize();
+
 			// materialize rows from chunk
 			chunkRows =
 			    materializeBlobGranule(files[chunkIdx], keyRange, beginVersion, readVersion, snapshotData, deltaData);
 
+			outputBytes += chunkRows.expectedSize();
+
 			results.arena().dependsOn(chunkRows.arena());
 			results.append(results.arena(), chunkRows.begin(), chunkRows.size());
 
-			if (loadIds[chunkIdx].snapshotId.present()) {
-				granuleContext.free_load_f(loadIds[chunkIdx].snapshotId.get(), granuleContext.userContext);
-			}
-			for (int i = 0; i < loadIds[chunkIdx].deltaIds.size(); i++) {
-				granuleContext.free_load_f(loadIds[chunkIdx].deltaIds[i], granuleContext.userContext);
-			}
+			// free once done by forcing FreeHandles to trigger
+			loadIds[chunkIdx].freeHandles.clear();
 		}
+		stats.inputBytes = inputBytes;
+		stats.outputBytes = outputBytes;
 		return ErrorOr<RangeResult>(results);
 	} catch (Error& e) {
 		return ErrorOr<RangeResult>(e);
@@ -976,7 +1641,7 @@ const EncryptCipherRandomSalt encryptSalt = deterministicRandom()->randomUInt64(
 
 Standalone<StringRef> getBaseCipher() {
 	Standalone<StringRef> baseCipher = makeString(AES_256_KEY_LENGTH);
-	generateRandomData(mutateString(baseCipher), baseCipher.size());
+	deterministicRandom()->randomBytes(mutateString(baseCipher), baseCipher.size());
 	return baseCipher;
 }
 
@@ -996,7 +1661,7 @@ BlobGranuleCipherKeysCtx getCipherKeysCtx(Arena& arena) {
 	cipherKeysCtx.headerCipherKey.baseCipher = StringRef(arena, encryptBaseCipher);
 
 	cipherKeysCtx.ivRef = makeString(AES_256_IV_LENGTH, arena);
-	generateRandomData(mutateString(cipherKeysCtx.ivRef), AES_256_IV_LENGTH);
+	deterministicRandom()->randomBytes(mutateString(cipherKeysCtx.ivRef), AES_256_IV_LENGTH);
 
 	return cipherKeysCtx;
 }
@@ -1007,23 +1672,13 @@ TEST_CASE("/blobgranule/files/applyDelta") {
 	printf("Testing blob granule delta applying\n");
 	Arena a;
 
-	// do this 2 phase arena creation of string refs instead of LiteralStringRef because there is no char* StringRef
-	// constructor, and valgrind might complain if the stringref data isn't in the arena
-	std::string sk_a = "A";
-	std::string sk_ab = "AB";
-	std::string sk_b = "B";
-	std::string sk_c = "C";
-	std::string sk_z = "Z";
-	std::string sval1 = "1";
-	std::string sval2 = "2";
-
-	StringRef k_a = StringRef(a, sk_a);
-	StringRef k_ab = StringRef(a, sk_ab);
-	StringRef k_b = StringRef(a, sk_b);
-	StringRef k_c = StringRef(a, sk_c);
-	StringRef k_z = StringRef(a, sk_z);
-	StringRef val1 = StringRef(a, sval1);
-	StringRef val2 = StringRef(a, sval2);
+	StringRef k_a = StringRef(a, "A"_sr);
+	StringRef k_ab = StringRef(a, "AB"_sr);
+	StringRef k_b = StringRef(a, "B"_sr);
+	StringRef k_c = StringRef(a, "C"_sr);
+	StringRef k_z = StringRef(a, "Z"_sr);
+	StringRef val1 = StringRef(a, "1"_sr);
+	StringRef val2 = StringRef(a, "2"_sr);
 
 	std::map<KeyRef, ValueRef> data;
 	data.insert({ k_a, val1 });
@@ -1150,92 +1805,440 @@ TEST_CASE("/blobgranule/files/applyDelta") {
 	return Void();
 }
 
-// picks a number between 2^minExp and 2^maxExp, but uniformly distributed over exponential buckets 2^n an 2^n+1
-int randomExp(int minExp, int maxExp) {
-	if (minExp == maxExp) { // N=2, case
-		return 1 << minExp;
+void checkDeltaAtVersion(const ParsedDeltaBoundaryRef& expected,
+                         const DeltaBoundaryRef& boundary,
+                         Version beginVersion,
+                         Version readVersion) {
+	ParsedDeltaBoundaryRef actual = deltaAtVersion(boundary, beginVersion, readVersion);
+	ASSERT(expected.clearAfter == actual.clearAfter);
+	ASSERT(expected.op == actual.op);
+	if (expected.isSet()) {
+		ASSERT(expected.value == actual.value);
+	} else {
+		ASSERT(actual.value.empty());
 	}
-	int val = 1 << deterministicRandom()->randomInt(minExp, maxExp);
-	ASSERT(val > 0);
-	return deterministicRandom()->randomInt(val, val * 2);
 }
 
-void checkEmpty(const Value& serialized, Key begin, Key end, Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
-	std::map<KeyRef, ValueRef> result;
-	Arena ar = loadSnapshotFile(serialized, KeyRangeRef(begin, end), result, cipherKeysCtx);
+TEST_CASE("/blobgranule/files/deltaAtVersion") {
+	Arena ar;
+	std::string keyStr = "k";
+	std::string aStr = "a";
+
+	KeyRef key(ar, keyStr);
+	ValueAndVersionRef vv_a_3(3, ValueRef(ar, aStr));
+	ValueAndVersionRef vv_clear_5(5);
+
+	ParsedDeltaBoundaryRef resultEmpty(key, false);
+	ParsedDeltaBoundaryRef resultEmptyWithClear(key, true);
+	ParsedDeltaBoundaryRef resultSetA(key, false, vv_a_3);
+	ParsedDeltaBoundaryRef resultClearA(key, true, vv_clear_5);
+
+	// test empty boundary ref
+	DeltaBoundaryRef boundaryEmpty;
+	boundaryEmpty.key = key;
+	checkDeltaAtVersion(resultEmpty, boundaryEmpty, 0, 2);
+
+	// test empty boundary with clear
+	DeltaBoundaryRef boundaryEmptyWithClear;
+	boundaryEmptyWithClear.key = key;
+	boundaryEmptyWithClear.clearVersion = 5;
+
+	// higher read version includes clear
+	checkDeltaAtVersion(resultEmptyWithClear, boundaryEmptyWithClear, 0, 5);
+	checkDeltaAtVersion(resultEmptyWithClear, boundaryEmptyWithClear, 0, 10);
+	checkDeltaAtVersion(resultEmptyWithClear, boundaryEmptyWithClear, 2, 5);
+	checkDeltaAtVersion(resultEmptyWithClear, boundaryEmptyWithClear, 2, 10);
+	checkDeltaAtVersion(resultEmptyWithClear, boundaryEmptyWithClear, 5, 10);
+	checkDeltaAtVersion(resultEmptyWithClear, boundaryEmptyWithClear, 5, 5);
+
+	// lower read version does not include clear
+	checkDeltaAtVersion(resultEmpty, boundaryEmptyWithClear, 0, 4);
+	checkDeltaAtVersion(resultEmpty, boundaryEmptyWithClear, 3, 4);
+
+	// higher read version but also higher beginVersion does not include clear
+	checkDeltaAtVersion(resultEmpty, boundaryEmptyWithClear, 6, 10);
+
+	// check values
+	DeltaBoundaryRef fullBoundary;
+	fullBoundary.key = key;
+	fullBoundary.values.push_back(ar, vv_a_3);
+	fullBoundary.values.push_back(ar, vv_clear_5);
+	fullBoundary.clearVersion = 5;
+
+	checkDeltaAtVersion(resultEmpty, fullBoundary, 0, 2);
+	checkDeltaAtVersion(resultEmpty, fullBoundary, 6, 10);
+	checkDeltaAtVersion(resultEmpty, fullBoundary, 4, 4);
+
+	checkDeltaAtVersion(resultSetA, fullBoundary, 0, 3);
+	checkDeltaAtVersion(resultSetA, fullBoundary, 3, 4);
+
+	checkDeltaAtVersion(resultClearA, fullBoundary, 0, 5);
+	checkDeltaAtVersion(resultClearA, fullBoundary, 0, 10);
+	checkDeltaAtVersion(resultClearA, fullBoundary, 3, 5);
+	checkDeltaAtVersion(resultClearA, fullBoundary, 4, 5);
+
+	return Void();
+}
+
+void checkSnapshotEmpty(const Value& serialized, Key begin, Key end, Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
+	Standalone<StringRef> fileNameRef = StringRef();
+	Standalone<VectorRef<ParsedDeltaBoundaryRef>> result =
+	    loadSnapshotFile(fileNameRef, serialized, KeyRangeRef(begin, end), cipherKeysCtx);
 	ASSERT(result.empty());
 }
 
 // endIdx is exclusive
-void checkRead(const Standalone<GranuleSnapshot>& snapshot,
-               const Value& serialized,
-               int beginIdx,
-               int endIdx,
-               Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
+void checkSnapshotRead(const Standalone<StringRef>& fileNameRef,
+                       const Standalone<GranuleSnapshot>& snapshot,
+                       const Value& serialized,
+                       int beginIdx,
+                       int endIdx,
+                       Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) {
 	ASSERT(beginIdx < endIdx);
 	ASSERT(endIdx <= snapshot.size());
-	std::map<KeyRef, ValueRef> result;
 	KeyRef beginKey = snapshot[beginIdx].key;
 	Key endKey = endIdx == snapshot.size() ? keyAfter(snapshot.back().key) : snapshot[endIdx].key;
 	KeyRangeRef range(beginKey, endKey);
 
-	Arena ar = loadSnapshotFile(serialized, range, result, cipherKeysCtx);
+	fmt::print("Reading [{0} - {1})\n", beginKey.printable(), endKey.printable());
+
+	Standalone<VectorRef<ParsedDeltaBoundaryRef>> result =
+	    loadSnapshotFile(fileNameRef, serialized, range, cipherKeysCtx);
 
 	if (result.size() != endIdx - beginIdx) {
 		fmt::print("Read {0} rows != {1}\n", result.size(), endIdx - beginIdx);
 	}
+
+	if (BG_FILES_TEST_DEBUG) {
+		fmt::print("Expected Data {0}:\n", result.size());
+		for (auto& it : result) {
+			fmt::print("  {0}=\n", it.key.printable());
+		}
+		fmt::print("Actual Data {0}:\n", endIdx - beginIdx);
+		for (int i = beginIdx; i < endIdx; i++) {
+			fmt::print("  {0}=\n", snapshot[i].key.printable());
+		}
+	}
+
 	ASSERT(result.size() == endIdx - beginIdx);
 	for (auto& it : result) {
-		if (it.first != snapshot[beginIdx].key) {
-			fmt::print("Key {0} != {1}\n", it.first.printable(), snapshot[beginIdx].key.printable());
+		ASSERT(it.isSet());
+		if (it.key != snapshot[beginIdx].key) {
+			fmt::print("Key {0} != {1}\n", it.key.printable(), snapshot[beginIdx].key.printable());
 		}
-		ASSERT(it.first == snapshot[beginIdx].key);
-		if (it.first != snapshot[beginIdx].key) {
+		ASSERT(it.key == snapshot[beginIdx].key);
+		if (it.key != snapshot[beginIdx].key) {
 			fmt::print("Value {0} != {1} for Key {2}\n",
-			           it.second.printable(),
+			           it.value.printable(),
 			           snapshot[beginIdx].value.printable(),
-			           it.first.printable());
+			           it.key.printable());
 		}
-		ASSERT(it.second == snapshot[beginIdx].value);
+		ASSERT(it.value == snapshot[beginIdx].value);
 		beginIdx++;
 	}
 }
 
-TEST_CASE("/blobgranule/files/snapshotFormatUnitTest") {
-	// snapshot files are likely to have a non-trivial shared prefix since they're for a small contiguous key range
-	std::string sharedPrefix = deterministicRandom()->randomUniqueID().toString();
-	int uidSize = sharedPrefix.size();
-	int sharedPrefixLen = deterministicRandom()->randomInt(0, uidSize);
-	int targetKeyLength = deterministicRandom()->randomInt(4, uidSize);
-	sharedPrefix = sharedPrefix.substr(0, sharedPrefixLen) + "_";
+namespace {
 
-	int targetValueLen = randomExp(0, 12);
-	int targetChunks = randomExp(0, 9);
-	int targetDataBytes = randomExp(0, 25);
+size_t uidSize = 32;
 
-	std::unordered_set<std::string> usedKeys;
-	Standalone<GranuleSnapshot> data;
-	int totalDataBytes = 0;
-	while (totalDataBytes < targetDataBytes) {
-		int keySize = deterministicRandom()->randomInt(targetKeyLength / 2, targetKeyLength * 3 / 2);
-		keySize = std::min(keySize, uidSize);
-		std::string key = sharedPrefix + deterministicRandom()->randomUniqueID().toString().substr(0, keySize);
-		if (usedKeys.insert(key).second) {
-			int valueSize = deterministicRandom()->randomInt(targetValueLen / 2, targetValueLen * 3 / 2);
-			std::string value = deterministicRandom()->randomUniqueID().toString();
-			if (value.size() > valueSize) {
-				value = value.substr(0, valueSize);
-			}
-			if (value.size() < valueSize) {
-				value += std::string(valueSize - value.size(), 'x');
-			}
+struct KeyValueGen {
+	Arena ar;
+	std::string sharedPrefix;
+	int targetKeyLength;
+	int targetValueLength;
+	std::set<std::string> usedKeys;
+	std::vector<StringRef> usedKeysList;
+	double clearFrequency;
+	double clearUnsetFrequency;
+	double updateExistingKeyFrequency;
+	int minVersionIncrease;
+	int maxVersionIncrease;
+	int targetMutationsPerDelta;
+	KeyRange allRange;
 
-			data.push_back_deep(data.arena(), KeyValueRef(KeyRef(key), ValueRef(value)));
-			totalDataBytes += key.size() + value.size();
+	Version version = 0;
+
+	// encryption/compression settings
+	// TODO: possibly different cipher keys or meta context per file?
+	Optional<BlobGranuleCipherKeysCtx> cipherKeys;
+	Optional<CompressionFilter> compressFilter;
+
+	KeyValueGen() {
+		sharedPrefix = deterministicRandom()->randomUniqueID().toString();
+		ASSERT(sharedPrefix.size() == uidSize);
+		int sharedPrefixLen = deterministicRandom()->randomInt(0, uidSize);
+		targetKeyLength = deterministicRandom()->randomInt(4, uidSize);
+		sharedPrefix = sharedPrefix.substr(0, sharedPrefixLen) + "_";
+		targetValueLength = deterministicRandom()->randomExp(0, 12);
+		allRange = KeyRangeRef(StringRef(sharedPrefix),
+		                       sharedPrefix.size() == 0 ? "\xff"_sr : strinc(StringRef(sharedPrefix)));
+
+		if (deterministicRandom()->coinflip()) {
+			clearFrequency = 0.0;
+			clearUnsetFrequency = 0.0;
+		} else {
+			clearFrequency = deterministicRandom()->random01() / 2;
+			// clearing an unset value has no effect on the results, we mostly just want to make sure the format doesn't
+			// barf
+			clearUnsetFrequency = deterministicRandom()->random01() / 10;
+		}
+		if (deterministicRandom()->random01() < 0.2) {
+			// no updates, only new writes
+			updateExistingKeyFrequency = 0.0;
+		} else {
+			updateExistingKeyFrequency = deterministicRandom()->random01();
+		}
+		if (deterministicRandom()->coinflip()) {
+			// sequential versions
+			minVersionIncrease = 1;
+			maxVersionIncrease = 2;
+		} else {
+			minVersionIncrease = deterministicRandom()->randomExp(0, 25);
+			maxVersionIncrease = minVersionIncrease + deterministicRandom()->randomExp(0, 25);
+		}
+		if (deterministicRandom()->coinflip()) {
+			targetMutationsPerDelta = 1;
+		} else {
+			targetMutationsPerDelta = deterministicRandom()->randomExp(1, 5);
+		}
+
+		if (deterministicRandom()->coinflip()) {
+			cipherKeys = getCipherKeysCtx(ar);
+		}
+		if (deterministicRandom()->coinflip()) {
+			compressFilter = CompressionUtils::getRandomFilter();
 		}
 	}
 
+	Optional<StringRef> newKey() {
+		for (int nAttempt = 0; nAttempt < 1000; nAttempt++) {
+			size_t keySize = deterministicRandom()->randomInt(targetKeyLength / 2, targetKeyLength * 3 / 2);
+			keySize = std::min(keySize, uidSize);
+			std::string key = sharedPrefix + deterministicRandom()->randomUniqueID().toString().substr(0, keySize);
+			if (usedKeys.insert(key).second) {
+				StringRef k(ar, key);
+				usedKeysList.push_back(k);
+				return k;
+			}
+		}
+		return {};
+	}
+
+	StringRef value() {
+		int valueSize = deterministicRandom()->randomInt(targetValueLength / 2, targetValueLength * 3 / 2);
+		std::string value = deterministicRandom()->randomUniqueID().toString();
+		if (value.size() > valueSize) {
+			value = value.substr(0, valueSize);
+		}
+		if (value.size() < valueSize) {
+			// repeated string so it's compressible
+			value += std::string(valueSize - value.size(), 'x');
+		}
+		return StringRef(ar, value);
+	}
+
+	KeyRef randomUsedKey() const { return usedKeysList[deterministicRandom()->randomInt(0, usedKeysList.size())]; }
+
+	KeyRange randomKeyRange() const {
+		ASSERT(!usedKeysList.empty());
+		Key begin = randomUsedKey();
+		if (deterministicRandom()->coinflip()) {
+			begin = keyAfter(begin);
+		}
+		if (usedKeysList.size() == 1) {
+			return KeyRange(KeyRangeRef(begin, keyAfter(begin)));
+		} else {
+			Key end = begin;
+			while (end == begin) {
+				end = randomUsedKey();
+			}
+			if (deterministicRandom()->coinflip()) {
+				end = keyAfter(end);
+			}
+			if (begin < end) {
+				return KeyRangeRef(begin, end);
+			} else {
+				return KeyRangeRef(end, begin);
+			}
+		}
+	}
+
+	StringRef keyForUpdate(double probUseExisting) {
+		if (!usedKeysList.empty() && deterministicRandom()->random01() < probUseExisting) {
+			return randomUsedKey();
+		} else {
+			auto k = newKey();
+			if (k.present()) {
+				return k.get();
+			} else {
+				// use existing key instead
+				ASSERT(!usedKeysList.empty());
+				return randomUsedKey();
+			}
+		}
+	}
+
+	Version nextVersion() {
+		Version jump = deterministicRandom()->randomInt(minVersionIncrease, maxVersionIncrease);
+		version += jump;
+		return version;
+	}
+
+	MutationRef newMutation() {
+		if (deterministicRandom()->random01() < clearFrequency) {
+			// The algorithm for generating clears of varying sizes is, to generate clear sizes based on an exponential
+			// distribution, such that the expected value of the clear size is 2.
+			int clearWidth = 1;
+			while (clearWidth < usedKeys.size() && deterministicRandom()->coinflip()) {
+				clearWidth *= 2;
+			}
+			bool clearPastEnd = deterministicRandom()->coinflip();
+			if (clearPastEnd) {
+				clearWidth--;
+			}
+			StringRef begin = keyForUpdate(1.0 - clearUnsetFrequency);
+			std::string beginStr = begin.toString();
+			auto it = usedKeys.find(beginStr);
+			ASSERT(it != usedKeys.end());
+			while (it != usedKeys.end() && clearWidth > 0) {
+				it++;
+				clearWidth--;
+			}
+			if (it == usedKeys.end()) {
+				it--;
+				clearPastEnd = true;
+			}
+			std::string endKey = *it;
+			if (clearPastEnd) {
+				Key end = keyAfter(StringRef(ar, endKey));
+				ar.dependsOn(end.arena());
+				return MutationRef(MutationRef::ClearRange, begin, end);
+			} else {
+				// clear up to end
+				return MutationRef(MutationRef::ClearRange, begin, StringRef(ar, endKey));
+			}
+
+		} else {
+			return MutationRef(MutationRef::SetValue, keyForUpdate(updateExistingKeyFrequency), value());
+		}
+	}
+
+	MutationsAndVersionRef newDelta() {
+		Version v = nextVersion();
+		int mutationCount = deterministicRandom()->randomInt(1, targetMutationsPerDelta * 2);
+		MutationsAndVersionRef ret(v, v);
+		for (int i = 0; i < mutationCount; i++) {
+			ret.mutations.push_back(ar, newMutation());
+		}
+		return ret;
+	}
+};
+
+} // namespace
+
+Standalone<GranuleSnapshot> genSnapshot(KeyValueGen& kvGen, int targetDataBytes) {
+	Standalone<GranuleSnapshot> data;
+	int totalDataBytes = 0;
+	while (totalDataBytes < targetDataBytes) {
+		Optional<StringRef> key = kvGen.newKey();
+		if (!key.present()) {
+			break;
+		}
+		StringRef value = kvGen.value();
+
+		data.push_back_deep(data.arena(), KeyValueRef(KeyRef(key.get()), ValueRef(value)));
+		totalDataBytes += key.get().size() + value.size();
+	}
+
 	std::sort(data.begin(), data.end(), KeyValueRef::OrderByKey());
+	return data;
+}
+
+Standalone<GranuleDeltas> genDeltas(KeyValueGen& kvGen, int targetBytes) {
+	Standalone<GranuleDeltas> data;
+	int totalDataBytes = 0;
+	while (totalDataBytes < targetBytes) {
+		data.push_back(data.arena(), kvGen.newDelta());
+		totalDataBytes += data.back().expectedSize();
+	}
+	return data;
+}
+
+TEST_CASE("/blobgranule/files/validateEncryptionCompression") {
+	KeyValueGen kvGen;
+
+	int targetSnapshotChunks = deterministicRandom()->randomExp(0, 9);
+	int targetDeltaChunks = deterministicRandom()->randomExp(0, 8);
+	int targetDataBytes = deterministicRandom()->randomExp(12, 25);
+	int targetSnapshotBytes = (int)(deterministicRandom()->randomInt(0, targetDataBytes));
+	int targetDeltaBytes = targetDataBytes - targetSnapshotBytes;
+
+	int targetSnapshotChunkSize = targetSnapshotBytes / targetSnapshotChunks;
+	int targetDeltaChunkSize = targetDeltaBytes / targetDeltaChunks;
+
+	Standalone<GranuleSnapshot> snapshotData = genSnapshot(kvGen, targetSnapshotBytes);
+	Standalone<GranuleDeltas> deltaData = genDeltas(kvGen, targetDeltaBytes);
+	fmt::print("{0} snapshot rows and {1} deltas\n", snapshotData.size(), deltaData.size());
+
+	Standalone<StringRef> fileNameRef = StringRef();
+
+	Arena ar;
+	BlobGranuleCipherKeysCtx cipherKeys = getCipherKeysCtx(ar);
+	std::vector<bool> encryptionModes = { false, true };
+	std::vector<Optional<CompressionFilter>> compressionModes;
+	compressionModes.insert(
+	    compressionModes.end(), CompressionUtils::supportedFilters.begin(), CompressionUtils::supportedFilters.end());
+
+	std::vector<Value> snapshotValues;
+	for (bool encryptionMode : encryptionModes) {
+		Optional<BlobGranuleCipherKeysCtx> keys = encryptionMode ? cipherKeys : Optional<BlobGranuleCipherKeysCtx>();
+		for (auto& compressionMode : compressionModes) {
+			Value v =
+			    serializeChunkedSnapshot(fileNameRef, snapshotData, targetSnapshotChunkSize, compressionMode, keys);
+			fmt::print("snapshot({0}, {1}): {2}\n",
+			           encryptionMode,
+			           compressionMode.present() ? CompressionUtils::toString(compressionMode.get()) : "",
+			           v.size());
+			for (auto& v2 : snapshotValues) {
+				ASSERT(v != v2);
+			}
+			snapshotValues.push_back(v);
+		}
+	}
+	fmt::print("Validated {0} encryption/compression combos for snapshot\n", snapshotValues.size());
+
+	std::vector<Value> deltaValues;
+	for (bool encryptionMode : encryptionModes) {
+		Optional<BlobGranuleCipherKeysCtx> keys = encryptionMode ? cipherKeys : Optional<BlobGranuleCipherKeysCtx>();
+		for (auto& compressionMode : compressionModes) {
+			Value v = serializeChunkedDeltaFile(
+			    fileNameRef, deltaData, kvGen.allRange, targetDeltaChunkSize, compressionMode, keys);
+			fmt::print("delta({0}, {1}): {2}\n",
+			           encryptionMode,
+			           compressionMode.present() ? CompressionUtils::toString(compressionMode.get()) : "",
+			           v.size());
+			for (auto& v2 : deltaValues) {
+				ASSERT(v != v2);
+			}
+			deltaValues.push_back(v);
+		}
+	}
+	fmt::print("Validated {0} encryption/compression combos for delta\n", deltaValues.size());
+
+	return Void();
+}
+
+TEST_CASE("/blobgranule/files/snapshotFormatUnitTest") {
+	// snapshot files are likely to have a non-trivial shared prefix since they're for a small contiguous key range
+	KeyValueGen kvGen;
+
+	int targetChunks = deterministicRandom()->randomExp(0, 9);
+	int targetDataBytes = deterministicRandom()->randomExp(0, 25);
+	int targetChunkSize = targetDataBytes / targetChunks;
+	Standalone<StringRef> fnameRef = StringRef(std::string("test"));
+
+	Standalone<GranuleSnapshot> data = genSnapshot(kvGen, targetDataBytes);
 
 	int maxExp = 0;
 	while (1 << maxExp < data.size()) {
@@ -1248,24 +2251,10 @@ TEST_CASE("/blobgranule/files/snapshotFormatUnitTest") {
 		ASSERT(data[i].key < data[i + 1].key);
 	}
 
-	fmt::print(
-	    "Constructing snapshot with {0} rows, {1} bytes, and {2} chunks\n", data.size(), totalDataBytes, targetChunks);
+	fmt::print("Constructing snapshot with {0} rows, {1} chunks\n", data.size(), targetChunks);
 
-	Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx = Optional<BlobGranuleCipherKeysCtx>();
-	Arena arena;
-	if (deterministicRandom()->coinflip()) {
-		cipherKeysCtx = getCipherKeysCtx(arena);
-	}
-
-	Optional<CompressionFilter> compressFilter;
-	if (deterministicRandom()->coinflip()) {
-#ifdef ZLIB_LIB_SUPPORTED
-		compressFilter = CompressionFilter::GZIP;
-#else
-		compressFilter = CompressionFilter::NONE;
-#endif
-	}
-	Value serialized = serializeChunkedSnapshot(data, targetChunks, compressFilter, cipherKeysCtx);
+	Value serialized =
+	    serializeChunkedSnapshot(fnameRef, data, targetChunkSize, kvGen.compressFilter, kvGen.cipherKeys);
 
 	fmt::print("Snapshot serialized! {0} bytes\n", serialized.size());
 
@@ -1276,31 +2265,851 @@ TEST_CASE("/blobgranule/files/snapshotFormatUnitTest") {
 
 	fmt::print("Initial read starting\n");
 
-	checkRead(data, serialized, 0, data.size(), cipherKeysCtx);
+	checkSnapshotRead(fnameRef, data, serialized, 0, data.size(), kvGen.cipherKeys);
 
 	fmt::print("Initial read complete\n");
 
 	if (data.size() > 1) {
 		for (int i = 0; i < std::min(100, data.size() * 2); i++) {
-			int width = randomExp(0, maxExp);
+			int width = deterministicRandom()->randomExp(0, maxExp);
 			ASSERT(width <= data.size());
 			int start = deterministicRandom()->randomInt(0, data.size() - width);
-			checkRead(data, serialized, start, start + width, cipherKeysCtx);
+			checkSnapshotRead(fnameRef, data, serialized, start, start + width, kvGen.cipherKeys);
 		}
 
 		fmt::print("Doing empty checks\n");
 		int randomIdx = deterministicRandom()->randomInt(0, data.size() - 1);
-		checkEmpty(serialized, keyAfter(data[randomIdx].key), data[randomIdx + 1].key, cipherKeysCtx);
+		checkSnapshotEmpty(serialized, keyAfter(data[randomIdx].key), data[randomIdx + 1].key, kvGen.cipherKeys);
 	} else {
 		fmt::print("Doing empty checks\n");
 	}
 
-	checkEmpty(serialized, normalKeys.begin, data.front().key, cipherKeysCtx);
-	checkEmpty(serialized, normalKeys.begin, LiteralStringRef("\x00"), cipherKeysCtx);
-	checkEmpty(serialized, keyAfter(data.back().key), normalKeys.end, cipherKeysCtx);
-	checkEmpty(serialized, LiteralStringRef("\xfe"), normalKeys.end, cipherKeysCtx);
+	checkSnapshotEmpty(serialized, normalKeys.begin, data.front().key, kvGen.cipherKeys);
+	checkSnapshotEmpty(serialized, normalKeys.begin, "\x00"_sr, kvGen.cipherKeys);
+	checkSnapshotEmpty(serialized, keyAfter(data.back().key), normalKeys.end, kvGen.cipherKeys);
+	checkSnapshotEmpty(serialized, "\xfe"_sr, normalKeys.end, kvGen.cipherKeys);
 
 	fmt::print("Snapshot format test done!\n");
 
 	return Void();
 }
+
+void checkDeltaRead(const KeyValueGen& kvGen,
+                    const KeyRangeRef& range,
+                    Version beginVersion,
+                    Version readVersion,
+                    const Standalone<GranuleDeltas>& data,
+                    StringRef* serialized) {
+	// expected answer
+	std::map<KeyRef, ValueRef> expectedData;
+	Version lastFileEndVersion = 0;
+
+	fmt::print("Delta Read [{0} - {1}) @ {2} - {3}\n",
+	           range.begin.printable(),
+	           range.end.printable(),
+	           beginVersion,
+	           readVersion);
+
+	applyDeltasByVersion(data, range, beginVersion, readVersion, lastFileEndVersion, expectedData);
+
+	// actual answer
+	std::string filename = randomBGFilename(
+	    deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta");
+	Standalone<BlobGranuleChunkRef> chunk;
+	chunk.deltaFiles.emplace_back_deep(
+	    chunk.arena(), filename, 0, serialized->size(), serialized->size(), kvGen.cipherKeys);
+	chunk.keyRange = kvGen.allRange;
+	chunk.includedVersion = readVersion;
+	chunk.snapshotVersion = invalidVersion;
+
+	RangeResult actualData = materializeBlobGranule(chunk, range, beginVersion, readVersion, {}, serialized);
+
+	if (expectedData.size() != actualData.size()) {
+		fmt::print("Expected Data {0}:\n", expectedData.size());
+		/*for (auto& it : expectedData) {
+		    fmt::print("  {0}=\n", it.first.printable());
+		}*/
+		fmt::print("Actual Data {0}:\n", actualData.size());
+		/*for (auto& it : actualData) {
+		    fmt::print("  {0}=\n", it.key.printable());
+		}*/
+	}
+
+	ASSERT(expectedData.size() == actualData.size());
+	int i = 0;
+	for (auto& it : expectedData) {
+		ASSERT(it.first == actualData[i].key);
+		ASSERT(it.second == actualData[i].value);
+		i++;
+	}
+}
+
+static std::tuple<KeyRange, Version, Version> randomizeKeyAndVersions(const KeyValueGen& kvGen,
+                                                                      const Standalone<GranuleDeltas> data) {
+	// either randomize just keyrange, just version range, or both
+	double rand = deterministicRandom()->randomInt(0, 3);
+	bool randomizeKeyRange = rand == 0 || rand == 2;
+	bool randomizeVersionRange = rand == 1 || rand == 2;
+	KeyRange readRange = kvGen.allRange;
+	Version beginVersion = 0;
+	Version readVersion = data.back().version;
+
+	if (randomizeKeyRange) {
+		readRange = kvGen.randomKeyRange();
+	}
+
+	if (randomizeVersionRange) {
+		if (deterministicRandom()->coinflip()) {
+			beginVersion = 0;
+		} else {
+			beginVersion = data[deterministicRandom()->randomInt(0, data.size())].version;
+			beginVersion += deterministicRandom()->randomInt(0, 3) - 1; // randomize between -1, 0, and +1
+		}
+		readVersion = data[deterministicRandom()->randomInt(0, data.size())].version;
+		readVersion += deterministicRandom()->randomInt(0, 3) - 1; // randomize between -1, 0, and +1
+		if (readVersion < beginVersion) {
+			std::swap(beginVersion, readVersion);
+		}
+	}
+
+	return { readRange, beginVersion, readVersion };
+}
+
+TEST_CASE("/blobgranule/files/deltaFormatUnitTest") {
+	KeyValueGen kvGen;
+	Standalone<StringRef> fileNameRef = StringRef(std::string("test"));
+
+	int targetChunks = deterministicRandom()->randomExp(0, 8);
+	int targetDataBytes = deterministicRandom()->randomExp(0, 21);
+	int targetChunkSize = targetDataBytes / targetChunks;
+
+	Standalone<GranuleDeltas> data = genDeltas(kvGen, targetDataBytes);
+
+	fmt::print("Deltas ({0})\n", data.size());
+	/*for (auto& it : data) {
+	    fmt::print("  {0}) ({1})\n", it.version, it.mutations.size());
+	    for (auto& it2 : it.mutations) {
+	        if (it2.type == MutationRef::Type::SetValue) {
+	            fmt::print("    {0}=\n", it2.param1.printable());
+	        } else {
+	            fmt::print("    {0} - {1}\n", it2.param1.printable(), it2.param2.printable());
+	        }
+	    }
+	}*/
+	Value serialized = serializeChunkedDeltaFile(
+	    fileNameRef, data, kvGen.allRange, targetChunkSize, kvGen.compressFilter, kvGen.cipherKeys);
+
+	// check whole file
+	checkDeltaRead(kvGen, kvGen.allRange, 0, data.back().version, data, &serialized);
+
+	for (int i = 0; i < std::min((size_t)100, kvGen.usedKeysList.size() * data.size()); i++) {
+		auto params = randomizeKeyAndVersions(kvGen, data);
+		checkDeltaRead(kvGen, std::get<0>(params), std::get<1>(params), std::get<2>(params), data, &serialized);
+	}
+
+	return Void();
+}
+
+void checkGranuleRead(const KeyValueGen& kvGen,
+                      const KeyRangeRef& range,
+                      Version beginVersion,
+                      Version readVersion,
+                      const Standalone<GranuleSnapshot>& snapshotData,
+                      const Standalone<GranuleDeltas>& deltaData,
+                      const Value& serializedSnapshot,
+                      const std::vector<std::pair<Version, Value>>& serializedDeltas,
+                      const Standalone<GranuleDeltas>& inMemoryDeltas) {
+	// expected answer
+	std::map<KeyRef, ValueRef> expectedData;
+	if (beginVersion == 0) {
+		for (auto& it : snapshotData) {
+			if (range.contains(it.key)) {
+				expectedData.insert({ it.key, it.value });
+			}
+		}
+	}
+	Version lastFileEndVersion = 0;
+	applyDeltasByVersion(deltaData, range, beginVersion, readVersion, lastFileEndVersion, expectedData);
+
+	// actual answer
+	Standalone<BlobGranuleChunkRef> chunk;
+	if (beginVersion == 0) {
+		std::string snapshotFilename = randomBGFilename(
+		    deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), 0, ".snapshot");
+		chunk.snapshotFile = BlobFilePointerRef(
+		    chunk.arena(), snapshotFilename, 0, serializedSnapshot.size(), serializedSnapshot.size(), kvGen.cipherKeys);
+	}
+	int deltaIdx = 0;
+	while (deltaIdx < serializedDeltas.size() && serializedDeltas[deltaIdx].first < beginVersion) {
+		deltaIdx++;
+	}
+	std::vector<StringRef> deltaPtrsVector;
+	while (deltaIdx < serializedDeltas.size()) {
+		std::string deltaFilename = randomBGFilename(
+		    deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta");
+		size_t fsize = serializedDeltas[deltaIdx].second.size();
+		chunk.deltaFiles.emplace_back_deep(chunk.arena(), deltaFilename, 0, fsize, fsize, kvGen.cipherKeys);
+		deltaPtrsVector.push_back(serializedDeltas[deltaIdx].second);
+
+		if (serializedDeltas[deltaIdx].first >= readVersion) {
+			break;
+		}
+		deltaIdx++;
+	}
+	StringRef deltaPtrs[deltaPtrsVector.size()];
+	for (int i = 0; i < deltaPtrsVector.size(); i++) {
+		deltaPtrs[i] = deltaPtrsVector[i];
+	}
+
+	// add in memory deltas
+	chunk.arena().dependsOn(inMemoryDeltas.arena());
+	for (auto& it : inMemoryDeltas) {
+		if (beginVersion <= it.version && it.version <= readVersion) {
+			chunk.newDeltas.push_back(chunk.arena(), it);
+		}
+	}
+
+	chunk.keyRange = kvGen.allRange;
+	chunk.includedVersion = readVersion;
+	chunk.snapshotVersion = (beginVersion == 0) ? 0 : invalidVersion;
+
+	Optional<StringRef> snapshotPtr;
+	if (beginVersion == 0) {
+		snapshotPtr = serializedSnapshot;
+	}
+	RangeResult actualData = materializeBlobGranule(chunk, range, beginVersion, readVersion, snapshotPtr, deltaPtrs);
+
+	if (expectedData.size() != actualData.size()) {
+		fmt::print("Expected Size {0} != Actual Size {1}\n", expectedData.size(), actualData.size());
+	}
+	if (BG_FILES_TEST_DEBUG) {
+		fmt::print("Expected Data {0}:\n", expectedData.size());
+		for (auto& it : expectedData) {
+			fmt::print("  {0}=\n", it.first.printable());
+		}
+		fmt::print("Actual Data {0}:\n", actualData.size());
+		for (auto& it : actualData) {
+			fmt::print("  {0}=\n", it.key.printable());
+		}
+	}
+
+	ASSERT(expectedData.size() == actualData.size());
+	int i = 0;
+	for (auto& it : expectedData) {
+		if (it.first != actualData[i].key) {
+			fmt::print("expected {0} != actual {1}\n", it.first.printable(), actualData[i].key.printable());
+		}
+		ASSERT(it.first == actualData[i].key);
+		ASSERT(it.second == actualData[i].value);
+		i++;
+	}
+}
+
+TEST_CASE("/blobgranule/files/granuleReadUnitTest") {
+	KeyValueGen kvGen;
+	Standalone<StringRef> fileNameRef = StringRef(std::string("testSnap"));
+
+	int targetSnapshotChunks = deterministicRandom()->randomExp(0, 9);
+	int targetDeltaChunks = deterministicRandom()->randomExp(0, 8);
+	int targetDataBytes = deterministicRandom()->randomExp(12, 25);
+	int targetSnapshotBytes = (int)(deterministicRandom()->randomInt(0, targetDataBytes));
+	int targetDeltaBytes = targetDataBytes - targetSnapshotBytes;
+
+	if (BG_FILES_TEST_DEBUG) {
+		fmt::print("Snapshot Chunks: {0}\nDelta Chunks: {1}\nSnapshot Bytes: {2}\nDelta Bytes: {3}\n",
+		           targetSnapshotChunks,
+		           targetDeltaChunks,
+		           targetSnapshotBytes,
+		           targetDeltaBytes);
+	}
+
+	int targetSnapshotChunkSize = targetSnapshotBytes / targetSnapshotChunks;
+	int targetDeltaChunkSize = targetDeltaBytes / targetDeltaChunks;
+
+	Standalone<GranuleSnapshot> snapshotData = genSnapshot(kvGen, targetSnapshotBytes);
+	if (BG_FILES_TEST_DEBUG) {
+		fmt::print("Snapshot data: {0}\n", snapshotData.size());
+		for (auto& it : snapshotData) {
+			fmt::print("  {0}=\n", it.key.printable());
+		}
+	}
+	Standalone<GranuleDeltas> deltaData = genDeltas(kvGen, targetDeltaBytes);
+	fmt::print("{0} snapshot rows and {1} deltas\n", snapshotData.size(), deltaData.size());
+
+	if (BG_FILES_TEST_DEBUG) {
+		fmt::print("Delta data: {0}\n", deltaData.size());
+		for (auto& it : deltaData) {
+			fmt::print("  {0}) ({1})\n", it.version, it.mutations.size());
+			for (auto& it2 : it.mutations) {
+				if (it2.type == MutationRef::Type::SetValue) {
+					fmt::print("    {0}=\n", it2.param1.printable());
+				} else {
+					fmt::print("    {0} - {1}\n", it2.param1.printable(), it2.param2.printable());
+				}
+			}
+		}
+	}
+
+	Value serializedSnapshot = serializeChunkedSnapshot(
+	    fileNameRef, snapshotData, targetSnapshotChunkSize, kvGen.compressFilter, kvGen.cipherKeys);
+
+	// split deltas up across multiple files
+	int deltaFiles = std::min(deltaData.size(), deterministicRandom()->randomInt(1, 21));
+	int deltasPerFile = deltaData.size() / deltaFiles + 1;
+	std::vector<std::pair<Version, Value>> serializedDeltaFiles;
+	Standalone<GranuleDeltas> inMemoryDeltas;
+	serializedDeltaFiles.reserve(deltaFiles);
+	for (int i = 0; i < deltaFiles; i++) {
+		Standalone<GranuleDeltas> fileData;
+		int j;
+		for (j = i * deltasPerFile; j < (i + 1) * deltasPerFile && j < deltaData.size(); j++) {
+			fileData.push_back_deep(fileData.arena(), deltaData[j]);
+		}
+		if (!fileData.empty()) {
+			if (j == deltaData.size() && deterministicRandom()->coinflip()) {
+				// if it's the last set of deltas, sometimes make them the memory deltas instead
+				fmt::print("Memory Deltas {0} - {1}\n", fileData.front().version, fileData.back().version);
+				inMemoryDeltas = fileData;
+			} else {
+				fmt::print("Delta file {0} - {1}\n", fileData.front().version, fileData.back().version);
+				Standalone<StringRef> fileNameRef = StringRef("delta" + std::to_string(i));
+				Value serializedDelta = serializeChunkedDeltaFile(fileNameRef,
+				                                                  fileData,
+				                                                  kvGen.allRange,
+				                                                  targetDeltaChunkSize,
+				                                                  kvGen.compressFilter,
+				                                                  kvGen.cipherKeys);
+				serializedDeltaFiles.emplace_back(fileData.back().version, serializedDelta);
+			}
+		}
+	}
+
+	fmt::print("Full test\n");
+	checkGranuleRead(kvGen,
+	                 kvGen.allRange,
+	                 0,
+	                 deltaData.back().version,
+	                 snapshotData,
+	                 deltaData,
+	                 serializedSnapshot,
+	                 serializedDeltaFiles,
+	                 inMemoryDeltas);
+
+	// prevent overflow by doing min before multiply
+	int maxRuns = 100;
+	int snapshotAndDeltaSize = 5 + std::min(maxRuns, snapshotData.size()) * std::min(maxRuns, deltaData.size());
+	int lim = std::min(maxRuns, snapshotAndDeltaSize);
+	for (int i = 0; i < lim; i++) {
+		auto params = randomizeKeyAndVersions(kvGen, deltaData);
+		fmt::print("Partial test {0}: [{1} - {2}) @ {3} - {4}\n",
+		           i,
+		           std::get<0>(params).begin.printable(),
+		           std::get<0>(params).end.printable(),
+		           std::get<1>(params),
+		           std::get<2>(params));
+		checkGranuleRead(kvGen,
+		                 std::get<0>(params),
+		                 std::get<1>(params),
+		                 std::get<2>(params),
+		                 snapshotData,
+		                 deltaData,
+		                 serializedSnapshot,
+		                 serializedDeltaFiles,
+		                 inMemoryDeltas);
+	}
+
+	return Void();
+}
+
+// performance micro-benchmarks
+
+struct FileSet {
+	std::tuple<std::string, Version, Value, Standalone<GranuleSnapshot>> snapshotFile;
+	std::vector<std::tuple<std::string, Version, Value, Standalone<GranuleDeltas>>> deltaFiles;
+	Key commonPrefix;
+	KeyRange range;
+};
+
+std::pair<std::string, Version> parseFilename(const std::string& fname) {
+	auto dotPos = fname.find(".");
+	ASSERT(dotPos > 0);
+	std::string type = fname.substr(dotPos + 1);
+	ASSERT(type == "snapshot" || type == "delta");
+	auto lastUnderscorePos = fname.rfind("_");
+	ASSERT('V' == fname[lastUnderscorePos + 1]);
+	std::string versionString = fname.substr(lastUnderscorePos + 2, dotPos);
+	Version version = std::stoll(versionString);
+	return { type, version };
+}
+
+Value loadFileData(std::string filename) {
+	std::ifstream input(filename, std::ios::binary);
+	ASSERT(input.good());
+
+	// copies all data into buffer
+	std::vector<uint8_t> buffer(std::istreambuf_iterator<char>(input), {});
+	Value v(StringRef(&buffer[0], buffer.size()));
+	fmt::print("Loaded {0} file bytes from {1}\n", v.size(), filename);
+
+	input.close();
+	return v;
+}
+
+struct CommonPrefixStats {
+	// for computing common prefix details and stats
+	Key key;
+	int len = -1;
+	int64_t totalKeySize = 0;
+	int totalKeys = 0;
+	int minKeySize = 1000000000;
+	int maxKeySize = 0;
+
+	void addKey(const KeyRef& k) {
+		if (len == -1) {
+			key = k;
+			len = k.size();
+		} else {
+			len = std::min(len, commonPrefixLength(k, key));
+		}
+		totalKeys++;
+		totalKeySize += k.size();
+		minKeySize = std::min(minKeySize, k.size());
+		maxKeySize = std::max(maxKeySize, k.size());
+	}
+
+	Key done() {
+		ASSERT(len >= 0);
+		fmt::print("Common prefix: {0}\nCommon Prefix Length: {1}\nAverage Key Size: {2}\nMin Key Size: {3}, Max Key "
+		           "Size: {4}\n",
+		           key.substr(0, len).printable(),
+		           len,
+		           totalKeySize / totalKeys,
+		           minKeySize,
+		           maxKeySize);
+		return key.substr(0, len);
+	}
+};
+
+FileSet loadFileSet(std::string basePath, const std::vector<std::string>& filenames) {
+	FileSet files;
+	CommonPrefixStats stats;
+	for (int i = 0; i < filenames.size(); i++) {
+		auto parts = parseFilename(filenames[i]);
+		std::string type = parts.first;
+		Version version = parts.second;
+		if (type == "snapshot") {
+			std::string fpath = basePath + filenames[i];
+			Value data = loadFileData(fpath);
+
+			Arena arena;
+			GranuleSnapshot file;
+			ObjectReader dataReader(data.begin(), Unversioned());
+			dataReader.deserialize(FileIdentifierFor<GranuleSnapshot>::value, file, arena);
+			Standalone<GranuleSnapshot> parsed(file, arena);
+
+			fmt::print("Loaded {0} rows from snapshot file\n", parsed.size());
+			files.snapshotFile = { filenames[i], version, data, parsed };
+
+			for (auto& it : parsed) {
+				stats.addKey(it.key);
+			}
+		} else {
+			std::string fpath = basePath + filenames[i];
+			Value data = loadFileData(fpath);
+
+			Arena arena;
+			GranuleDeltas file;
+			ObjectReader dataReader(data.begin(), Unversioned());
+			dataReader.deserialize(FileIdentifierFor<GranuleDeltas>::value, file, arena);
+			Standalone<GranuleDeltas> parsed(file, arena);
+
+			fmt::print("Loaded {0} deltas from delta file\n", parsed.size());
+			files.deltaFiles.push_back({ filenames[i], version, data, parsed });
+
+			for (auto& it : parsed) {
+				for (auto& it2 : it.mutations) {
+					stats.addKey(it2.param1);
+					if (it2.type == MutationRef::Type::ClearRange) {
+						stats.addKey(it2.param2);
+					}
+				}
+			}
+		}
+	}
+
+	files.commonPrefix = stats.done();
+	if (files.commonPrefix.size() == 0) {
+		files.range = normalKeys;
+	} else {
+		files.range = KeyRangeRef(files.commonPrefix, strinc(files.commonPrefix));
+	}
+	fmt::print("Range: [{0} - {1})\n", files.range.begin.printable(), files.range.end.printable());
+
+	return files;
+}
+
+int WRITE_RUNS = 5;
+
+std::pair<int64_t, double> doSnapshotWriteBench(const Standalone<GranuleSnapshot>& data,
+                                                bool chunked,
+                                                Optional<BlobGranuleCipherKeysCtx> cipherKeys,
+                                                Optional<CompressionFilter> compressionFilter) {
+	Standalone<StringRef> fileNameRef = StringRef();
+	int64_t serializedBytes = 0;
+	double elapsed = -timer_monotonic();
+	for (int runI = 0; runI < WRITE_RUNS; runI++) {
+		if (!chunked) {
+			serializedBytes = ObjectWriter::toValue(data, Unversioned()).size();
+		} else {
+			serializedBytes =
+			    serializeChunkedSnapshot(fileNameRef, data, 64 * 1024, compressionFilter, cipherKeys).size();
+		}
+	}
+	elapsed += timer_monotonic();
+	elapsed /= WRITE_RUNS;
+	return { serializedBytes, elapsed };
+}
+
+std::pair<int64_t, double> doDeltaWriteBench(const Standalone<GranuleDeltas>& data,
+                                             const KeyRangeRef& fileRange,
+                                             bool chunked,
+                                             Optional<BlobGranuleCipherKeysCtx> cipherKeys,
+                                             Optional<CompressionFilter> compressionFilter) {
+	Standalone<StringRef> fileNameRef = StringRef();
+	int64_t serializedBytes = 0;
+	double elapsed = -timer_monotonic();
+	for (int runI = 0; runI < WRITE_RUNS; runI++) {
+		if (!chunked) {
+			serializedBytes = ObjectWriter::toValue(data, Unversioned()).size();
+		} else {
+			serializedBytes =
+			    serializeChunkedDeltaFile(fileNameRef, data, fileRange, 32 * 1024, compressionFilter, cipherKeys)
+			        .size();
+		}
+	}
+	elapsed += timer_monotonic();
+	elapsed /= WRITE_RUNS;
+	return { serializedBytes, elapsed };
+}
+
+FileSet rewriteChunkedFileSet(const FileSet& fileSet,
+                              Optional<BlobGranuleCipherKeysCtx> keys,
+                              Optional<CompressionFilter> compressionFilter) {
+	Standalone<StringRef> fileNameRef = StringRef();
+	FileSet newFiles;
+	newFiles.snapshotFile = fileSet.snapshotFile;
+	newFiles.deltaFiles = fileSet.deltaFiles;
+	newFiles.commonPrefix = fileSet.commonPrefix;
+	newFiles.range = fileSet.range;
+
+	std::get<2>(newFiles.snapshotFile) =
+	    serializeChunkedSnapshot(fileNameRef, std::get<3>(newFiles.snapshotFile), 64 * 1024, compressionFilter, keys);
+	for (auto& deltaFile : newFiles.deltaFiles) {
+		std::get<2>(deltaFile) = serializeChunkedDeltaFile(
+		    fileNameRef, std::get<3>(deltaFile), fileSet.range, 32 * 1024, compressionFilter, keys);
+	}
+
+	return newFiles;
+}
+
+int READ_RUNS = 20;
+std::pair<int64_t, double> doReadBench(const FileSet& fileSet,
+                                       bool chunked,
+                                       KeyRange readRange,
+                                       bool clearAllAtEnd,
+                                       Optional<BlobGranuleCipherKeysCtx> keys,
+                                       Optional<CompressionFilter> compressionFilter) {
+	Version readVersion = std::get<1>(fileSet.deltaFiles.back());
+
+	Standalone<BlobGranuleChunkRef> chunk;
+	StringRef deltaPtrs[fileSet.deltaFiles.size()];
+
+	MutationRef clearAllAtEndMutation;
+	if (clearAllAtEnd) {
+		clearAllAtEndMutation = MutationRef(MutationRef::Type::ClearRange, readRange.begin, readRange.end);
+	}
+	if (chunked) {
+		size_t snapshotSize = std::get<3>(fileSet.snapshotFile).size();
+		chunk.snapshotFile =
+		    BlobFilePointerRef(chunk.arena(), std::get<0>(fileSet.snapshotFile), 0, snapshotSize, snapshotSize, keys);
+
+		for (int i = 0; i < fileSet.deltaFiles.size(); i++) {
+			size_t deltaSize = std::get<3>(fileSet.deltaFiles[i]).size();
+			chunk.deltaFiles.emplace_back_deep(
+			    chunk.arena(), std::get<0>(fileSet.deltaFiles[i]), 0, deltaSize, deltaSize, keys);
+			deltaPtrs[i] = std::get<2>(fileSet.deltaFiles[i]);
+		}
+
+		if (clearAllAtEnd) {
+			readVersion++;
+			MutationsAndVersionRef lastDelta;
+			lastDelta.version = readVersion;
+			lastDelta.mutations.push_back(chunk.arena(), clearAllAtEndMutation);
+
+			chunk.newDeltas.push_back_deep(chunk.arena(), lastDelta);
+		}
+
+		chunk.keyRange = fileSet.range;
+		chunk.includedVersion = readVersion;
+		chunk.snapshotVersion = std::get<1>(fileSet.snapshotFile);
+	}
+
+	int64_t serializedBytes = 0;
+	double elapsed = -timer_monotonic();
+	for (int runI = 0; runI < READ_RUNS; runI++) {
+		if (!chunked) {
+			std::map<KeyRef, ValueRef> data;
+			for (auto& it : std::get<3>(fileSet.snapshotFile)) {
+				data.insert({ it.key, it.value });
+			}
+			Version lastFileEndVersion = 0;
+			for (auto& deltaFile : fileSet.deltaFiles) {
+				applyDeltasByVersion(std::get<3>(deltaFile), readRange, 0, readVersion, lastFileEndVersion, data);
+			}
+			if (clearAllAtEnd) {
+				applyDelta(readRange, clearAllAtEndMutation, data);
+			}
+			RangeResult actualData;
+			for (auto& it : data) {
+				actualData.push_back_deep(actualData.arena(), KeyValueRef(it.first, it.second));
+			}
+			serializedBytes += actualData.expectedSize();
+		} else {
+			RangeResult actualData =
+			    materializeBlobGranule(chunk, readRange, 0, readVersion, std::get<2>(fileSet.snapshotFile), deltaPtrs);
+			serializedBytes += actualData.expectedSize();
+		}
+	}
+	elapsed += timer_monotonic();
+	elapsed /= READ_RUNS;
+	serializedBytes /= READ_RUNS;
+	return { serializedBytes, elapsed };
+}
+
+void printMetrics(int64_t diskBytes, double elapsed, int64_t processesBytes, int64_t logicalSize) {
+	double storageAmp = (1.0 * diskBytes) / logicalSize;
+
+	double MBperCPUsec = (elapsed == 0.0) ? 0.0 : (processesBytes / 1024.0 / 1024.0) / elapsed;
+	fmt::print("{}", fmt::format(" {:.6} {:.6}", storageAmp, MBperCPUsec));
+}
+
+TEST_CASE("!/blobgranule/files/benchFromFiles") {
+	std::string basePath = "SET_ME";
+	std::vector<std::vector<std::string>> fileSetNames = { { "SET_ME" } };
+	Arena ar;
+	BlobGranuleCipherKeysCtx cipherKeys = getCipherKeysCtx(ar);
+	std::vector<bool> chunkModes = { false, true };
+	std::vector<bool> encryptionModes = { false, true };
+	std::vector<Optional<CompressionFilter>> compressionModes;
+	compressionModes.push_back({});
+	compressionModes.insert(
+	    compressionModes.end(), CompressionUtils::supportedFilters.begin(), CompressionUtils::supportedFilters.end());
+
+	std::vector<std::string> runNames = { "logical" };
+	std::vector<std::pair<int64_t, double>> snapshotMetrics;
+	std::vector<std::pair<int64_t, double>> deltaMetrics;
+
+	std::vector<FileSet> fileSets;
+	int64_t logicalSnapshotSize = 0;
+	int64_t logicalDeltaSize = 0;
+	for (auto& it : fileSetNames) {
+		FileSet fileSet = loadFileSet(basePath, it);
+		fileSets.push_back(fileSet);
+		logicalSnapshotSize += std::get<3>(fileSet.snapshotFile).expectedSize();
+		for (auto& deltaFile : fileSet.deltaFiles) {
+			logicalDeltaSize += std::get<3>(deltaFile).expectedSize();
+		}
+	}
+	snapshotMetrics.push_back({ logicalSnapshotSize, 0.0 });
+	deltaMetrics.push_back({ logicalDeltaSize, 0.0 });
+
+	for (bool chunk : chunkModes) {
+		for (bool encrypt : encryptionModes) {
+			if (!chunk && encrypt) {
+				continue;
+			}
+			Optional<BlobGranuleCipherKeysCtx> keys = encrypt ? cipherKeys : Optional<BlobGranuleCipherKeysCtx>();
+			for (auto& compressionFilter : compressionModes) {
+				if (!chunk && compressionFilter.present()) {
+					continue;
+				}
+				if (compressionFilter.present() && CompressionFilter::NONE == compressionFilter.get()) {
+					continue;
+				}
+
+				std::string name;
+				if (!chunk) {
+					name = "old";
+				} else {
+					if (encrypt) {
+						name += "ENC";
+					}
+					if (compressionFilter.present()) {
+						name += "CMP";
+					}
+					if (name.empty()) {
+						name = "chunked";
+					}
+				}
+				runNames.push_back(name);
+				int64_t snapshotTotalBytes = 0;
+				double snapshotTotalElapsed = 0.0;
+				for (auto& fileSet : fileSets) {
+					auto res = doSnapshotWriteBench(std::get<3>(fileSet.snapshotFile), chunk, keys, compressionFilter);
+					snapshotTotalBytes += res.first;
+					snapshotTotalElapsed += res.second;
+				}
+				snapshotMetrics.push_back({ snapshotTotalBytes, snapshotTotalElapsed });
+
+				int64_t deltaTotalBytes = 0;
+				double deltaTotalElapsed = 0.0;
+				for (auto& fileSet : fileSets) {
+					for (auto& deltaFile : fileSet.deltaFiles) {
+						auto res =
+						    doDeltaWriteBench(std::get<3>(deltaFile), fileSet.range, chunk, keys, compressionFilter);
+						deltaTotalBytes += res.first;
+						deltaTotalElapsed += res.second;
+					}
+				}
+				deltaMetrics.push_back({ deltaTotalBytes, deltaTotalElapsed });
+			}
+		}
+	}
+
+	fmt::print("\n\n\n\nWrite Results:\n");
+
+	ASSERT(runNames.size() == snapshotMetrics.size());
+	ASSERT(runNames.size() == deltaMetrics.size());
+	for (int i = 0; i < runNames.size(); i++) {
+		fmt::print("{0}", runNames[i]);
+
+		printMetrics(
+		    snapshotMetrics[i].first, snapshotMetrics[i].second, snapshotMetrics[i].first, snapshotMetrics[0].first);
+		printMetrics(deltaMetrics[i].first, deltaMetrics[i].second, deltaMetrics[i].first, deltaMetrics[0].first);
+
+		int64_t logicalTotalBytes = snapshotMetrics[0].first + deltaMetrics[0].first;
+		int64_t totalBytes = deltaMetrics[i].first + snapshotMetrics[i].first;
+		double logicalTotalElapsed = (snapshotMetrics[i].second == 0.0 || deltaMetrics[i].second == 0.0)
+		                                 ? 0.0
+		                                 : snapshotMetrics[i].second + deltaMetrics[i].second;
+		printMetrics(totalBytes, logicalTotalElapsed, deltaMetrics[i].first, logicalTotalBytes);
+
+		fmt::print("\n");
+	}
+
+	std::vector<std::string> readRunNames = {};
+	std::vector<std::pair<int64_t, double>> readMetrics;
+
+	bool doEdgeCaseReadTests = true;
+	std::vector<double> clearAllReadMetrics;
+	std::vector<double> readSingleKeyMetrics;
+
+	for (bool chunk : chunkModes) {
+		for (bool encrypt : encryptionModes) {
+			if (!chunk && encrypt) {
+				continue;
+			}
+
+			Optional<BlobGranuleCipherKeysCtx> keys = encrypt ? cipherKeys : Optional<BlobGranuleCipherKeysCtx>();
+			for (auto& compressionFilter : compressionModes) {
+				if (!chunk && compressionFilter.present()) {
+					continue;
+				}
+				if (compressionFilter.present() && CompressionFilter::NONE == compressionFilter.get()) {
+					continue;
+				}
+				std::string name;
+				if (!chunk) {
+					name = "old";
+				} else {
+					if (encrypt) {
+						name += "ENC";
+					}
+					if (compressionFilter.present()) {
+						name += "CMP";
+					}
+					if (name.empty()) {
+						name = "chunked";
+					}
+				}
+				readRunNames.push_back(name);
+
+				int64_t totalBytesRead = 0;
+				double totalElapsed = 0.0;
+				double totalElapsedClearAll = 0.0;
+				double totalElapsedSingleKey = 0.0;
+				for (auto& fileSet : fileSets) {
+					FileSet newFileSet;
+					if (!chunk) {
+						newFileSet = fileSet;
+					} else {
+						newFileSet = rewriteChunkedFileSet(fileSet, keys, compressionFilter);
+					}
+
+					auto res = doReadBench(newFileSet, chunk, fileSet.range, false, keys, compressionFilter);
+					totalBytesRead += res.first;
+					totalElapsed += res.second;
+
+					if (doEdgeCaseReadTests) {
+						totalElapsedClearAll +=
+						    doReadBench(newFileSet, chunk, fileSet.range, true, keys, compressionFilter).second;
+						Key k = std::get<3>(fileSet.snapshotFile).front().key;
+						KeyRange singleKeyRange(KeyRangeRef(k, keyAfter(k)));
+						totalElapsedSingleKey +=
+						    doReadBench(newFileSet, chunk, singleKeyRange, false, keys, compressionFilter).second;
+					}
+				}
+				readMetrics.push_back({ totalBytesRead, totalElapsed });
+				if (doEdgeCaseReadTests) {
+					clearAllReadMetrics.push_back(totalElapsedClearAll);
+					readSingleKeyMetrics.push_back(totalElapsedSingleKey);
+				}
+			}
+		}
+	}
+
+	fmt::print("\n\nRead Results:\n");
+
+	ASSERT(readRunNames.size() == readMetrics.size());
+	for (int i = 0; i < readRunNames.size(); i++) {
+		fmt::print("{0}", readRunNames[i]);
+
+		double MBperCPUsec = (readMetrics[i].first / 1024.0 / 1024.0) / readMetrics[i].second;
+		fmt::print(" {:.6}", MBperCPUsec);
+
+		fmt::print("\n");
+	}
+
+	if (doEdgeCaseReadTests) {
+		ASSERT(readRunNames.size() == clearAllReadMetrics.size());
+		ASSERT(readRunNames.size() == readSingleKeyMetrics.size());
+		fmt::print("\n\nEdge Case Read Results:\n");
+
+		for (int i = 0; i < readRunNames.size(); i++) {
+			fmt::print("{0}", readRunNames[i]);
+
+			// use MB from full read test but elapsed from these tests so the numbers make sense relatively
+			double MBperCPUsecClearAll = (readMetrics[i].first / 1024.0 / 1024.0) / clearAllReadMetrics[i];
+			double MBperCPUsecSingleKey = (readMetrics[i].first / 1024.0 / 1024.0) / readSingleKeyMetrics[i];
+			fmt::print(" {:.6} {:.6}", MBperCPUsecClearAll, MBperCPUsecSingleKey);
+
+			fmt::print("\n");
+		}
+	}
+
+	fmt::print("\n\nCombined Results:\n");
+	ASSERT(readRunNames.size() == runNames.size() - 1);
+	for (int i = 0; i < readRunNames.size(); i++) {
+		fmt::print("{0}", readRunNames[i]);
+		int64_t logicalBytes = deltaMetrics[i + 1].first;
+		double totalElapsed = snapshotMetrics[i + 1].second + deltaMetrics[i + 1].second + readMetrics[i].second;
+		double MBperCPUsec = (logicalBytes / 1024.0 / 1024.0) / totalElapsed;
+		fmt::print(" {:.6}", MBperCPUsec);
+
+		fmt::print("\n");
+	}
+
+	fmt::print("\n\nBenchmark Complete!\n");
+
+	return Void();
+}
diff --git a/fdbclient/BlobGranuleReader.actor.cpp b/fdbclient/BlobGranuleReader.actor.cpp
index e0f627a9da..583da353f7 100644
--- a/fdbclient/BlobGranuleReader.actor.cpp
+++ b/fdbclient/BlobGranuleReader.actor.cpp
@@ -31,13 +31,6 @@
 #include "fdbclient/FDBTypes.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-// TODO more efficient data structure besides std::map? PTree is unnecessary since this isn't versioned, but some other
-// sorted thing could work. And if it used arenas it'd probably be more efficient with allocations, since everything
-// else is in 1 arena and discarded at the end.
-
-// TODO could refactor the file reading code from here and the delta file function into another actor,
-// then this part would also be testable? but meh
-
 ACTOR Future<Standalone<StringRef>> readFile(Reference<BlobConnectionProvider> bstoreProvider, BlobFilePointerRef f) {
 	try {
 		state Arena arena;
@@ -140,3 +133,66 @@ ACTOR Future<Void> readBlobGranules(BlobGranuleFileRequest request,
 
 	return Void();
 }
+
+// Return true if a given range is fully covered by blob chunks
+bool isRangeFullyCovered(KeyRange range, Standalone<VectorRef<BlobGranuleChunkRef>> blobChunks) {
+	std::vector<KeyRangeRef> blobRanges;
+	for (const BlobGranuleChunkRef& chunk : blobChunks) {
+		blobRanges.push_back(chunk.keyRange);
+	}
+
+	return range.isCovered(blobRanges);
+}
+
+void testAddChunkRange(KeyRef begin, KeyRef end, Standalone<VectorRef<BlobGranuleChunkRef>>& chunks) {
+	BlobGranuleChunkRef chunk;
+	chunk.keyRange = KeyRangeRef(begin, end);
+	chunks.push_back(chunks.arena(), chunk);
+}
+
+TEST_CASE("/fdbserver/blobgranule/isRangeCoveredByBlob") {
+	Standalone<VectorRef<BlobGranuleChunkRef>> chunks;
+	// chunk1 key_a1 - key_a9
+	testAddChunkRange("key_a1"_sr, "key_a9"_sr, chunks);
+	// chunk2 key_b1 - key_b9
+	testAddChunkRange("key_b1"_sr, "key_b9"_sr, chunks);
+
+	// check empty range. not covered
+	{ ASSERT(isRangeFullyCovered(KeyRangeRef(), chunks) == false); }
+
+	// check empty chunks. not covered
+	{
+		Standalone<VectorRef<BlobGranuleChunkRef>> empyChunks;
+		ASSERT(isRangeFullyCovered(KeyRangeRef(), empyChunks) == false);
+	}
+
+	// check '' to \xff
+	{ ASSERT(isRangeFullyCovered(KeyRangeRef(""_sr, "\xff"_sr), chunks) == false); }
+
+	// check {key_a1, key_a9}
+	{ ASSERT(isRangeFullyCovered(KeyRangeRef("key_a1"_sr, "key_a9"_sr), chunks)); }
+
+	// check {key_a1, key_a3}
+	{ ASSERT(isRangeFullyCovered(KeyRangeRef("key_a1"_sr, "key_a3"_sr), chunks)); }
+
+	// check {key_a0, key_a3}
+	{ ASSERT(isRangeFullyCovered(KeyRangeRef("key_a0"_sr, "key_a3"_sr), chunks) == false); }
+
+	// check {key_a5, key_b2}
+	{
+		auto range = KeyRangeRef("key_a5"_sr, "key_b5"_sr);
+		ASSERT(isRangeFullyCovered(range, chunks) == false);
+		ASSERT(range.begin == "key_a5"_sr);
+		ASSERT(range.end == "key_b5"_sr);
+	}
+
+	// check continued chunks
+	{
+		Standalone<VectorRef<BlobGranuleChunkRef>> continuedChunks;
+		testAddChunkRange("key_a1"_sr, "key_a9"_sr, continuedChunks);
+		testAddChunkRange("key_a9"_sr, "key_b1"_sr, continuedChunks);
+		testAddChunkRange("key_b1"_sr, "key_b9"_sr, continuedChunks);
+		ASSERT(isRangeFullyCovered(KeyRangeRef("key_a1"_sr, "key_b9"_sr), continuedChunks) == false);
+	}
+	return Void();
+}
diff --git a/fdbclient/BuildFlags.h.in b/fdbclient/BuildFlags.h.in
index 6f94c540f8..b55c7e55dd 100644
--- a/fdbclient/BuildFlags.h.in
+++ b/fdbclient/BuildFlags.h.in
@@ -33,6 +33,9 @@
 #define C_VERSION_MINOR 0
 #endif
 
+const char* kDate = __DATE__;
+const char* kTime = __TIME__;
+
 // FDB info.
 const std::string kGitHash = "@CURRENT_GIT_VERSION_WNL@";
 const std::string kFdbVersion = "@FDB_VERSION@";
@@ -43,7 +46,7 @@ const std::string kArch = "@CMAKE_SYSTEM@";
 const std::string kCompiler = "@CMAKE_CXX_COMPILER_ID@";
 
 // Library versions.
-const std::string kBoostVersion = "@Boost_LIB_VERSION@";
+const std::string kBoostVersion = BOOST_LIB_VERSION;
 
 // Build info and flags.
 const std::string kCMakeVersion = "@CMAKE_VERSION@";
@@ -61,6 +64,9 @@ std::string jsonBuildInformation() {
 	json_spirit::mValue json;	
 	JSONDoc doc(json);
 
+	doc.create("build_date") = kDate;
+	doc.create("build_time") = kTime;
+
 	doc.create("git_hash") = kGitHash;
 	doc.create("fdb_version") = kFdbVersion;
 
diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt
index ab0e5b3931..2953a360e7 100644
--- a/fdbclient/CMakeLists.txt
+++ b/fdbclient/CMakeLists.txt
@@ -1,8 +1,5 @@
 fdb_find_sources(FDBCLIENT_SRCS)
-list(APPEND FDBCLIENT_SRCS
-  sha1/SHA1.cpp
-  libb64/cdecode.c
-  libb64/cencode.c)
+list(APPEND FDBCLIENT_SRCS sha1/SHA1.cpp)
 
 message(STATUS "FDB version is ${FDB_VERSION}")
 message(STATUS "FDB package name is ${FDB_PACKAGE_NAME}")
@@ -71,13 +68,6 @@ if(WITH_AWS_BACKUP)
   include(awssdk)
 endif()
 
-find_package(ZLIB)
-if(ZLIB_FOUND)
-  add_compile_definitions(ZLIB_LIB_SUPPORTED)
-else()
-  message(STATUS "ZLIB package not found")
-endif()
-
 add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
 target_include_directories(fdbclient PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/versions.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/versions.h)
@@ -100,8 +90,8 @@ add_flow_target(LINK_TEST NAME fdbclientlinktest SRCS LinkTest.cpp)
 target_link_libraries(fdbclientlinktest PRIVATE fdbclient rapidxml) # re-link rapidxml due to private link interface
 
 if(BUILD_AZURE_BACKUP)
-  target_link_libraries(fdbclient PRIVATE curl uuid azure-storage-lite)
-  target_link_libraries(fdbclient_sampling PRIVATE curl uuid azure-storage-lite)
+  target_link_libraries(fdbclient PRIVATE curl azure-storage-lite)
+  target_link_libraries(fdbclient_sampling PRIVATE curl azure-storage-lite)
 endif()
 
 if(BUILD_AWS_BACKUP)
diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 20d48846cc..b15f7c9583 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -22,6 +22,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/Tenant.h"
+#include "flow/IRandom.h"
 #include "flow/UnitTest.h"
 
 #define init(...) KNOB_FN(__VA_ARGS__, INIT_ATOMIC_KNOB, INIT_KNOB)(__VA_ARGS__)
@@ -41,10 +42,6 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( FAILURE_MAX_DELAY,                       5.0 );
 	init( FAILURE_MIN_DELAY,                       4.0 ); if( randomize && BUGGIFY ) FAILURE_MIN_DELAY = 1.0;
-	init( FAILURE_TIMEOUT_DELAY,     FAILURE_MIN_DELAY );
-	init( CLIENT_FAILURE_TIMEOUT_DELAY, FAILURE_MIN_DELAY );
-	init( FAILURE_EMERGENCY_DELAY,                30.0 );
-	init( FAILURE_MAX_GENERATIONS,                  10 );
 	init( RECOVERY_DELAY_START_GENERATION,          70 );
 	init( RECOVERY_DELAY_SECONDS_PER_GENERATION,  60.0 );
 	init( MAX_GENERATIONS,                         100 );
@@ -63,6 +60,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( WRONG_SHARD_SERVER_DELAY,                .01 ); if( randomize && BUGGIFY ) WRONG_SHARD_SERVER_DELAY = deterministicRandom()->random01(); // FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test)
 	init( FUTURE_VERSION_RETRY_DELAY,              .01 ); if( randomize && BUGGIFY ) FUTURE_VERSION_RETRY_DELAY = deterministicRandom()->random01();// FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY;
+	init( GRV_ERROR_RETRY_DELAY,                   5.0 ); if( randomize && BUGGIFY ) GRV_ERROR_RETRY_DELAY = 0.01 + 5 * deterministicRandom()->random01();
 	init( UNKNOWN_TENANT_RETRY_DELAY,              0.0 ); if( randomize && BUGGIFY ) UNKNOWN_TENANT_RETRY_DELAY = deterministicRandom()->random01();
 	init( REPLY_BYTE_LIMIT,                      80000 );
 	init( DEFAULT_BACKOFF,                         .01 ); if( randomize && BUGGIFY ) DEFAULT_BACKOFF = deterministicRandom()->random01();
@@ -81,9 +79,9 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( METADATA_VERSION_CACHE_SIZE,            1000 );
 	init( CHANGE_FEED_LOCATION_LIMIT,            10000 );
 	init( CHANGE_FEED_CACHE_SIZE,               100000 ); if( randomize && BUGGIFY ) CHANGE_FEED_CACHE_SIZE = 1;
-	init( CHANGE_FEED_POP_TIMEOUT,                 5.0 );
+	init( CHANGE_FEED_POP_TIMEOUT,                10.0 );
 	init( CHANGE_FEED_STREAM_MIN_BYTES,            1e4 ); if( randomize && BUGGIFY ) CHANGE_FEED_STREAM_MIN_BYTES = 1;
-	init( TENANT_PREFIX_SIZE_LIMIT,                 28 ); ASSERT(TENANT_PREFIX_SIZE_LIMIT >= TenantMapEntry::ROOT_PREFIX_SIZE); // includes 8-byte ID and optional tenant subspace
+	init( CHANGE_FEED_START_INTERVAL,             10.0 );
 
 	init( MAX_BATCH_SIZE,                         1000 ); if( randomize && BUGGIFY ) MAX_BATCH_SIZE = 1;
 	init( GRV_BATCH_TIMEOUT,                     0.005 ); if( randomize && BUGGIFY ) GRV_BATCH_TIMEOUT = 0.1;
@@ -111,7 +109,6 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( RANGESTREAM_BUFFERED_FRAGMENTS_LIMIT,     20 );
 	init( QUARANTINE_TSS_ON_MISMATCH,             true ); if( randomize && BUGGIFY ) QUARANTINE_TSS_ON_MISMATCH = false; // if true, a tss mismatch will put the offending tss in quarantine. If false, it will just be killed
 	init( CHANGE_FEED_EMPTY_BATCH_TIME,          0.005 );
-	init( SHARD_ENCODE_LOCATION_METADATA,        false ); if( randomize && BUGGIFY )  SHARD_ENCODE_LOCATION_METADATA = true;
 
 	//KeyRangeMap
 	init( KRM_GET_RANGE_LIMIT,                     1e5 ); if( randomize && BUGGIFY ) KRM_GET_RANGE_LIMIT = 10;
@@ -160,8 +157,6 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL, 60);
 	init( BACKUP_AGGREGATE_POLL_RATE,              2.0 ); // polls per second target for all agents on the cluster
 	init( BACKUP_LOG_WRITE_BATCH_MAX_SIZE,         1e6 ); //Must be much smaller than TRANSACTION_SIZE_LIMIT
-	init( BACKUP_LOG_ATOMIC_OPS_SIZE,			  1000 );
-	init( BACKUP_OPERATION_COST_OVERHEAD,		    50 );
 	init( BACKUP_MAX_LOG_RANGES,                    21 ); if( randomize && BUGGIFY ) BACKUP_MAX_LOG_RANGES = 4;
 	init( BACKUP_SIM_COPY_LOG_RANGES,              100 );
 	init( BACKUP_VERSION_DELAY,           5*CORE_VERSIONSPERSECOND );
@@ -204,14 +199,13 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( DEFAULT_COMMIT_GRV_PROXIES_RATIO,          3 );
 	init( DEFAULT_MAX_GRV_PROXIES,                   4 );
 
+	init( GLOBAL_CONFIG_REFRESH_BACKOFF,           0.5 );
+	init( GLOBAL_CONFIG_REFRESH_MAX_BACKOFF,      60.0 );
+	init( GLOBAL_CONFIG_REFRESH_TIMEOUT,          10.0 );
+
 	init( IS_ACCEPTABLE_DELAY,                     1.5 );
 
-	init( HTTP_READ_SIZE,                     128*1024 );
-	init( HTTP_SEND_SIZE,                      32*1024 );
-	init( HTTP_VERBOSE_LEVEL,                        0 );
-	init( HTTP_REQUEST_ID_HEADER,                   "" );
 	init( HTTP_REQUEST_AWS_V4_HEADER,             true );
-	init( HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT, false );
 	init( BLOBSTORE_ENCRYPTION_TYPE,                "" );
 	init( BLOBSTORE_CONNECT_TRIES,                  10 );
 	init( BLOBSTORE_CONNECT_TIMEOUT,                10 );
@@ -270,27 +264,36 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( MAX_TAGS_PER_TRANSACTION,                   5 );
 	init( MAX_TRANSACTION_TAG_LENGTH,                16 );
 	init( COMMIT_SAMPLE_COST,                       100 ); if( randomize && BUGGIFY ) COMMIT_SAMPLE_COST = 10;
-	init( WRITE_COST_BYTE_FACTOR,                 16384 ); if( randomize && BUGGIFY ) WRITE_COST_BYTE_FACTOR = 4096;
 	init( INCOMPLETE_SHARD_PLUS,                   4096 );
 	init( READ_TAG_SAMPLE_RATE,                    0.01 ); if( randomize && BUGGIFY ) READ_TAG_SAMPLE_RATE = 1.0; // Communicated to clients from cluster
 	init( TAG_THROTTLE_SMOOTHING_WINDOW,            2.0 );
 	init( TAG_THROTTLE_RECHECK_INTERVAL,            5.0 ); if( randomize && BUGGIFY ) TAG_THROTTLE_RECHECK_INTERVAL = 0.0;
 	init( TAG_THROTTLE_EXPIRATION_INTERVAL,        60.0 ); if( randomize && BUGGIFY ) TAG_THROTTLE_EXPIRATION_INTERVAL = 1.0;
+	init( WRITE_COST_BYTE_FACTOR,                 16384 ); if( randomize && BUGGIFY ) WRITE_COST_BYTE_FACTOR = 4096;
+	init( READ_COST_BYTE_FACTOR,                  16384 ); if( randomize && BUGGIFY ) READ_COST_BYTE_FACTOR = 4096;
 
 	// busyness reporting
 	init( BUSYNESS_SPIKE_START_THRESHOLD,         0.100 );
 	init( BUSYNESS_SPIKE_SATURATED_THRESHOLD,     0.500 );
 
-	// multi-version client control
-	init( MVC_CLIENTLIB_CHUNK_SIZE,              8*1024 );
-	init( MVC_CLIENTLIB_CHUNKS_PER_TRANSACTION,      32 );
-
 	// Blob granules
 	init( BG_MAX_GRANULE_PARALLELISM,                10 );
+	init( BG_TOO_MANY_GRANULES,                   10000 );
 
 	init( CHANGE_QUORUM_BAD_STATE_RETRY_TIMES,        3 );
 	init( CHANGE_QUORUM_BAD_STATE_RETRY_DELAY,      2.0 );
 
+	// Tenants and Metacluster
+	init( MAX_TENANTS_PER_CLUSTER,                  1e6 );
+	init( TENANT_TOMBSTONE_CLEANUP_INTERVAL,         60 ); if ( randomize && BUGGIFY ) TENANT_TOMBSTONE_CLEANUP_INTERVAL = deterministicRandom()->random01() * 30;
+	init( MAX_DATA_CLUSTERS,                        1e5 );
+	init( REMOVE_CLUSTER_TENANT_BATCH_SIZE,         1e4 ); if ( randomize && BUGGIFY ) REMOVE_CLUSTER_TENANT_BATCH_SIZE = 1;
+	init( METACLUSTER_ASSIGNMENT_CLUSTERS_TO_CHECK,   5 ); if ( randomize && BUGGIFY ) METACLUSTER_ASSIGNMENT_CLUSTERS_TO_CHECK = 1;
+	init( METACLUSTER_ASSIGNMENT_FIRST_CHOICE_DELAY, 1.0 ); if ( randomize && BUGGIFY ) METACLUSTER_ASSIGNMENT_FIRST_CHOICE_DELAY = deterministicRandom()->random01() * 60;
+	init( METACLUSTER_ASSIGNMENT_AVAILABILITY_TIMEOUT, 10.0 ); if ( randomize && BUGGIFY ) METACLUSTER_ASSIGNMENT_AVAILABILITY_TIMEOUT = 1 + deterministicRandom()->random01() * 59;
+	init( TENANT_ENTRY_CACHE_LIST_REFRESH_INTERVAL,  2 ); if( randomize && BUGGIFY ) TENANT_ENTRY_CACHE_LIST_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10);
+
+	init( ENABLE_ENCRYPTION_CPU_TIME_LOGGING,     false );
 	// clang-format on
 }
 
diff --git a/fdbclient/ConfigKnobs.cpp b/fdbclient/ConfigKnobs.cpp
index 03b4b09fbd..283e8208c2 100644
--- a/fdbclient/ConfigKnobs.cpp
+++ b/fdbclient/ConfigKnobs.cpp
@@ -144,10 +144,7 @@ std::string configDBTypeToString(ConfigDBType configDBType) {
 }
 
 TEST_CASE("/fdbclient/ConfigDB/ConfigKey/EncodeDecode") {
-	Tuple tuple;
-	tuple << "class-A"_sr
-	      << "test_long"_sr;
-	auto packed = tuple.pack();
+	auto packed = Tuple::makeTuple("class-A"_sr, "test_long"_sr).pack();
 	auto unpacked = ConfigKeyRef::decodeKey(packed);
 	ASSERT(unpacked.configClass.get() == "class-A"_sr);
 	ASSERT(unpacked.knobName == "test_long"_sr);
@@ -169,18 +166,8 @@ void decodeFailureTest(KeyRef key) {
 } // namespace
 
 TEST_CASE("/fdbclient/ConfigDB/ConfigKey/DecodeFailure") {
-	{
-		Tuple tuple;
-		tuple << "s1"_sr
-		      << "s2"_sr
-		      << "s3"_sr;
-		decodeFailureTest(tuple.pack());
-	}
-	{
-		Tuple tuple;
-		tuple << "s1"_sr << 5;
-		decodeFailureTest(tuple.pack());
-	}
+	decodeFailureTest(Tuple::makeTuple("s1"_sr, "s2"_sr, "s3"_sr).pack());
+	decodeFailureTest(Tuple::makeTuple("s1"_sr, 5).pack());
 	decodeFailureTest("non-tuple-key"_sr);
 	return Void();
 }
diff --git a/fdbclient/DatabaseBackupAgent.actor.cpp b/fdbclient/DatabaseBackupAgent.actor.cpp
index f549205e25..548f03c46c 100644
--- a/fdbclient/DatabaseBackupAgent.actor.cpp
+++ b/fdbclient/DatabaseBackupAgent.actor.cpp
@@ -37,11 +37,11 @@
 
 #include "flow/actorcompiler.h" // has to be last include
 
-const Key DatabaseBackupAgent::keyAddPrefix = LiteralStringRef("add_prefix");
-const Key DatabaseBackupAgent::keyRemovePrefix = LiteralStringRef("remove_prefix");
-const Key DatabaseBackupAgent::keyRangeVersions = LiteralStringRef("range_versions");
-const Key DatabaseBackupAgent::keyCopyStop = LiteralStringRef("copy_stop");
-const Key DatabaseBackupAgent::keyDatabasesInSync = LiteralStringRef("databases_in_sync");
+const Key DatabaseBackupAgent::keyAddPrefix = "add_prefix"_sr;
+const Key DatabaseBackupAgent::keyRemovePrefix = "remove_prefix"_sr;
+const Key DatabaseBackupAgent::keyRangeVersions = "range_versions"_sr;
+const Key DatabaseBackupAgent::keyCopyStop = "copy_stop"_sr;
+const Key DatabaseBackupAgent::keyDatabasesInSync = "databases_in_sync"_sr;
 const int DatabaseBackupAgent::LATEST_DR_VERSION = 1;
 
 DatabaseBackupAgent::DatabaseBackupAgent()
@@ -75,14 +75,13 @@ DatabaseBackupAgent::DatabaseBackupAgent(Database src)
 class DRConfig {
 public:
 	DRConfig(UID uid = UID())
-	  : uid(uid),
-	    configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(databaseBackupPrefixRange.begin), uid)) {}
+	  : uid(uid), configSpace(uidPrefixKey("uid->config/"_sr.withPrefix(databaseBackupPrefixRange.begin), uid)) {}
 	DRConfig(Reference<Task> task)
 	  : DRConfig(BinaryReader::fromStringRef<UID>(task->params[BackupAgentBase::keyConfigLogUid], Unversioned())) {}
 
-	KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(__FUNCTION__sr); }
 
 	void clear(Reference<ReadYourWritesTransaction> tr) { tr->clear(configSpace.range()); }
 
@@ -137,7 +136,7 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 	static constexpr uint32_t version = 1;
 
 	static struct {
-		static TaskParam<int64_t> bytesWritten() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<int64_t> bytesWritten() { return __FUNCTION__sr; }
 	} Params;
 
 	static const Key keyAddBackupRangeTasks;
@@ -203,7 +202,7 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	ACTOR static Future<Void> _execute(Database cx,
@@ -363,7 +362,7 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 					if ((!prevAdjacent || !nextAdjacent) &&
 					    rangeCount > ((prevAdjacent || nextAdjacent) ? CLIENT_KNOBS->BACKUP_MAP_KEY_UPPER_LIMIT
 					                                                 : CLIENT_KNOBS->BACKUP_MAP_KEY_LOWER_LIMIT)) {
-						TEST(true); // range insert delayed because too versionMap is too large
+						CODE_PROBE(true, "range insert delayed because too versionMap is too large");
 
 						if (rangeCount > CLIENT_KNOBS->BACKUP_MAP_KEY_UPPER_LIMIT)
 							TraceEvent(SevWarnAlways, "DBA_KeyRangeMapTooLarge").log();
@@ -405,10 +404,10 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 							break;
 
 						if (backupVersions.get()[versionLoc + 1].key ==
-						    (removePrefix == StringRef() ? normalKeys.end : strinc(removePrefix))) {
+						    (removePrefix == StringRef() ? allKeys.end : strinc(removePrefix))) {
 							tr->clear(KeyRangeRef(
 							    backupVersions.get()[versionLoc].key.removePrefix(removePrefix).withPrefix(addPrefix),
-							    addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix)));
+							    addPrefix == StringRef() ? allKeys.end : strinc(addPrefix)));
 						} else {
 							tr->clear(KeyRangeRef(backupVersions.get()[versionLoc].key,
 							                      backupVersions.get()[versionLoc + 1].key)
@@ -536,9 +535,9 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 		return Void();
 	}
 };
-StringRef BackupRangeTaskFunc::name = LiteralStringRef("dr_backup_range");
-const Key BackupRangeTaskFunc::keyAddBackupRangeTasks = LiteralStringRef("addBackupRangeTasks");
-const Key BackupRangeTaskFunc::keyBackupRangeBeginKey = LiteralStringRef("backupRangeBeginKey");
+StringRef BackupRangeTaskFunc::name = "dr_backup_range"_sr;
+const Key BackupRangeTaskFunc::keyAddBackupRangeTasks = "addBackupRangeTasks"_sr;
+const Key BackupRangeTaskFunc::keyBackupRangeBeginKey = "backupRangeBeginKey"_sr;
 REGISTER_TASKFUNC(BackupRangeTaskFunc);
 
 struct FinishFullBackupTaskFunc : TaskFuncBase {
@@ -588,7 +587,7 @@ struct FinishFullBackupTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -606,7 +605,7 @@ struct FinishFullBackupTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef FinishFullBackupTaskFunc::name = LiteralStringRef("dr_finish_full_backup");
+StringRef FinishFullBackupTaskFunc::name = "dr_finish_full_backup"_sr;
 REGISTER_TASKFUNC(FinishFullBackupTaskFunc);
 
 struct EraseLogRangeTaskFunc : TaskFuncBase {
@@ -683,7 +682,7 @@ struct EraseLogRangeTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@@ -697,7 +696,7 @@ struct EraseLogRangeTaskFunc : TaskFuncBase {
 		return Void();
 	}
 };
-StringRef EraseLogRangeTaskFunc::name = LiteralStringRef("dr_erase_log_range");
+StringRef EraseLogRangeTaskFunc::name = "dr_erase_log_range"_sr;
 REGISTER_TASKFUNC(EraseLogRangeTaskFunc);
 
 struct CopyLogRangeTaskFunc : TaskFuncBase {
@@ -705,7 +704,7 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 	static constexpr uint32_t version = 1;
 
 	static struct {
-		static TaskParam<int64_t> bytesWritten() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<int64_t> bytesWritten() { return __FUNCTION__sr; }
 	} Params;
 
 	static const Key keyNextBeginVersion;
@@ -958,7 +957,7 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@@ -989,8 +988,8 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 		return Void();
 	}
 };
-StringRef CopyLogRangeTaskFunc::name = LiteralStringRef("dr_copy_log_range");
-const Key CopyLogRangeTaskFunc::keyNextBeginVersion = LiteralStringRef("nextBeginVersion");
+StringRef CopyLogRangeTaskFunc::name = "dr_copy_log_range"_sr;
+const Key CopyLogRangeTaskFunc::keyNextBeginVersion = "nextBeginVersion"_sr;
 REGISTER_TASKFUNC(CopyLogRangeTaskFunc);
 
 struct CopyLogsTaskFunc : TaskFuncBase {
@@ -1125,7 +1124,7 @@ struct CopyLogsTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -1143,7 +1142,7 @@ struct CopyLogsTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef CopyLogsTaskFunc::name = LiteralStringRef("dr_copy_logs");
+StringRef CopyLogsTaskFunc::name = "dr_copy_logs"_sr;
 REGISTER_TASKFUNC(CopyLogsTaskFunc);
 
 struct FinishedFullBackupTaskFunc : TaskFuncBase {
@@ -1235,7 +1234,7 @@ struct FinishedFullBackupTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@@ -1283,8 +1282,8 @@ struct FinishedFullBackupTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef FinishedFullBackupTaskFunc::name = LiteralStringRef("dr_finished_full_backup");
-const Key FinishedFullBackupTaskFunc::keyInsertTask = LiteralStringRef("insertTask");
+StringRef FinishedFullBackupTaskFunc::name = "dr_finished_full_backup"_sr;
+const Key FinishedFullBackupTaskFunc::keyInsertTask = "insertTask"_sr;
 REGISTER_TASKFUNC(FinishedFullBackupTaskFunc);
 
 struct CopyDiffLogsTaskFunc : TaskFuncBase {
@@ -1396,7 +1395,7 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -1414,7 +1413,7 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef CopyDiffLogsTaskFunc::name = LiteralStringRef("dr_copy_diff_logs");
+StringRef CopyDiffLogsTaskFunc::name = "dr_copy_diff_logs"_sr;
 REGISTER_TASKFUNC(CopyDiffLogsTaskFunc);
 
 // Skip unneeded EraseLogRangeTaskFunc in 5.1
@@ -1446,7 +1445,7 @@ struct SkipOldEraseLogRangeTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef SkipOldEraseLogRangeTaskFunc::name = LiteralStringRef("dr_skip_legacy_task");
+StringRef SkipOldEraseLogRangeTaskFunc::name = "dr_skip_legacy_task"_sr;
 REGISTER_TASKFUNC(SkipOldEraseLogRangeTaskFunc);
 REGISTER_TASKFUNC_ALIAS(SkipOldEraseLogRangeTaskFunc, db_erase_log_range);
 
@@ -1456,7 +1455,7 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 	static constexpr uint32_t version = 1;
 
 	static struct {
-		static TaskParam<int64_t> bytesWritten() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<int64_t> bytesWritten() { return __FUNCTION__sr; }
 	} Params;
 
 	static const Key keyNextBeginVersion;
@@ -1652,7 +1651,7 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@@ -1683,8 +1682,8 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 		return Void();
 	}
 };
-StringRef OldCopyLogRangeTaskFunc::name = LiteralStringRef("db_copy_log_range");
-const Key OldCopyLogRangeTaskFunc::keyNextBeginVersion = LiteralStringRef("nextBeginVersion");
+StringRef OldCopyLogRangeTaskFunc::name = "db_copy_log_range"_sr;
+const Key OldCopyLogRangeTaskFunc::keyNextBeginVersion = "nextBeginVersion"_sr;
 REGISTER_TASKFUNC(OldCopyLogRangeTaskFunc);
 
 struct AbortOldBackupTaskFunc : TaskFuncBase {
@@ -1753,7 +1752,7 @@ struct AbortOldBackupTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -1771,7 +1770,7 @@ struct AbortOldBackupTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef AbortOldBackupTaskFunc::name = LiteralStringRef("dr_abort_legacy_backup");
+StringRef AbortOldBackupTaskFunc::name = "dr_abort_legacy_backup"_sr;
 REGISTER_TASKFUNC(AbortOldBackupTaskFunc);
 REGISTER_TASKFUNC_ALIAS(AbortOldBackupTaskFunc, db_backup_range);
 REGISTER_TASKFUNC_ALIAS(AbortOldBackupTaskFunc, db_finish_full_backup);
@@ -1834,13 +1833,16 @@ struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase {
 					return Void();
 				}
 
-				if (backupRanges.size() == 1) {
+				if (backupRanges.size() == 1 || isDefaultBackup(backupRanges)) {
 					RangeResult existingDestUidValues = wait(srcTr->getRange(
 					    KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
 					bool found = false;
+					KeyRangeRef targetRange =
+					    (backupRanges.size() == 1) ? backupRanges[0] : getDefaultBackupSharedRange();
 					for (auto it : existingDestUidValues) {
-						if (BinaryReader::fromStringRef<KeyRange>(it.key.removePrefix(destUidLookupPrefix),
-						                                          IncludeVersion()) == backupRanges[0]) {
+						KeyRange uidRange = BinaryReader::fromStringRef<KeyRange>(
+						    it.key.removePrefix(destUidLookupPrefix), IncludeVersion());
+						if (uidRange == targetRange) {
 							if (destUidValue != it.value) {
 								// existing backup/DR is running
 								return Void();
@@ -1856,7 +1858,7 @@ struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase {
 					}
 
 					srcTr->set(
-					    BinaryWriter::toValue(backupRanges[0], IncludeVersion(ProtocolVersion::withSharedMutations()))
+					    BinaryWriter::toValue(targetRange, IncludeVersion(ProtocolVersion::withSharedMutations()))
 					        .withPrefix(destUidLookupPrefix),
 					    destUidValue);
 				}
@@ -1918,7 +1920,7 @@ struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef CopyDiffLogsUpgradeTaskFunc::name = LiteralStringRef("db_copy_diff_logs");
+StringRef CopyDiffLogsUpgradeTaskFunc::name = "db_copy_diff_logs"_sr;
 REGISTER_TASKFUNC(CopyDiffLogsUpgradeTaskFunc);
 
 struct BackupRestorableTaskFunc : TaskFuncBase {
@@ -2031,7 +2033,7 @@ struct BackupRestorableTaskFunc : TaskFuncBase {
 		                           task,
 		                           parentTask->params[Task::reservedTaskParamValidKey],
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -2049,7 +2051,7 @@ struct BackupRestorableTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef BackupRestorableTaskFunc::name = LiteralStringRef("dr_backup_restorable");
+StringRef BackupRestorableTaskFunc::name = "dr_backup_restorable"_sr;
 REGISTER_TASKFUNC(BackupRestorableTaskFunc);
 
 struct StartFullBackupTaskFunc : TaskFuncBase {
@@ -2078,24 +2080,29 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 				srcTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
 				// Initialize destUid
-				if (backupRanges.size() == 1) {
+				if (backupRanges.size() == 1 || isDefaultBackup(backupRanges)) {
 					RangeResult existingDestUidValues = wait(srcTr->getRange(
 					    KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
+					KeyRangeRef targetRange =
+					    (backupRanges.size() == 1) ? backupRanges[0] : getDefaultBackupSharedRange();
 					bool found = false;
 					for (auto it : existingDestUidValues) {
-						if (BinaryReader::fromStringRef<KeyRange>(it.key.removePrefix(destUidLookupPrefix),
-						                                          IncludeVersion()) == backupRanges[0]) {
+						KeyRange uidRange = BinaryReader::fromStringRef<KeyRange>(
+						    it.key.removePrefix(destUidLookupPrefix), IncludeVersion());
+						if (uidRange == targetRange) {
 							destUidValue = it.value;
 							found = true;
+							CODE_PROBE(targetRange == getDefaultBackupSharedRange(),
+							           "DR mutation sharing with default backup");
 							break;
 						}
 					}
 					if (!found) {
 						destUidValue = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
-						srcTr->set(BinaryWriter::toValue(backupRanges[0],
-						                                 IncludeVersion(ProtocolVersion::withSharedMutations()))
-						               .withPrefix(destUidLookupPrefix),
-						           destUidValue);
+						srcTr->set(
+						    BinaryWriter::toValue(targetRange, IncludeVersion(ProtocolVersion::withSharedMutations()))
+						        .withPrefix(destUidLookupPrefix),
+						    destUidValue);
 					}
 				}
 
@@ -2281,7 +2288,7 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 		task->params[BackupAgentBase::keyConfigBackupRanges] = keyConfigBackupRanges;
 		task->params[BackupAgentBase::keyTagName] = tagName;
 		task->params[DatabaseBackupAgent::keyDatabasesInSync] =
-		    backupAction == DatabaseBackupAgent::PreBackupAction::NONE ? LiteralStringRef("t") : LiteralStringRef("f");
+		    backupAction == DatabaseBackupAgent::PreBackupAction::NONE ? "t"_sr : "f"_sr;
 
 		if (!waitFor) {
 			return taskBucket->addTask(tr,
@@ -2301,7 +2308,7 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 		                               .get(logUid)
 		                               .pack(BackupAgentBase::keyFolderId),
 		                           task->params[BackupAgentBase::keyFolderId]));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -2319,7 +2326,7 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef StartFullBackupTaskFunc::name = LiteralStringRef("dr_start_full_backup");
+StringRef StartFullBackupTaskFunc::name = "dr_start_full_backup"_sr;
 REGISTER_TASKFUNC(StartFullBackupTaskFunc);
 } // namespace dbBackup
 
@@ -2625,7 +2632,7 @@ public:
 
 		int64_t startCount = 0;
 		state Key mapPrefix = logUidValue.withPrefix(applyMutationsKeyVersionMapRange.begin);
-		Key mapEnd = normalKeys.end.withPrefix(mapPrefix);
+		Key mapEnd = allKeys.end.withPrefix(mapPrefix);
 		tr->set(logUidValue.withPrefix(applyMutationsAddPrefixRange.begin), addPrefix);
 		tr->set(logUidValue.withPrefix(applyMutationsRemovePrefixRange.begin), removePrefix);
 		tr->set(logUidValue.withPrefix(applyMutationsKeyVersionCountRange.begin), StringRef((uint8_t*)&startCount, 8));
@@ -2780,7 +2787,7 @@ public:
 				Version destVersion = wait(tr3.getReadVersion());
 				TraceEvent("DBA_SwitchoverVersionUpgrade").detail("Src", commitVersion).detail("Dest", destVersion);
 				if (destVersion <= commitVersion) {
-					TEST(true); // Forcing dest backup cluster to higher version
+					CODE_PROBE(true, "Forcing dest backup cluster to higher version");
 					tr3.set(minRequiredCommitVersionKey, BinaryWriter::toValue(commitVersion + 1, Unversioned()));
 					wait(tr3.commit());
 				} else {
@@ -2933,7 +2940,7 @@ public:
 					Version applied = BinaryReader::fromStringRef<Version>(lastApplied.get(), Unversioned());
 					TraceEvent("DBA_AbortVersionUpgrade").detail("Src", applied).detail("Dest", current);
 					if (current <= applied) {
-						TEST(true); // Upgrading version of local database.
+						CODE_PROBE(true, "Upgrading version of local database.");
 						// The +1 is because we want to make sure that a versionstamped operation can't reuse
 						// the same version as an already-applied transaction.
 						tr->set(minRequiredCommitVersionKey, BinaryWriter::toValue(applied + 1, Unversioned()));
@@ -3061,6 +3068,9 @@ public:
 
 		loop {
 			try {
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+
 				wait(success(tr->getReadVersion())); // get the read version before getting a version from the source
 				                                     // database to prevent the time differential from going negative
 
@@ -3072,9 +3082,6 @@ public:
 
 				state UID logUid = wait(backupAgent->getLogUid(tr, tagName));
 
-				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-
 				state Future<Optional<Value>> fPaused = tr->get(backupAgent->taskBucket->getPauseKey());
 				state Future<RangeResult> fErrorValues =
 				    errorLimit > 0
diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index f19782346a..76fded095c 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/DatabaseConfiguration.h"
+#include "fdbclient/FDBTypes.h"
 #include "fdbclient/SystemData.h"
 #include "flow/ITrace.h"
 #include "flow/Trace.h"
@@ -53,6 +54,7 @@ void DatabaseConfiguration::resetInternal() {
 	storageMigrationType = StorageMigrationType::DEFAULT;
 	blobGranulesEnabled = false;
 	tenantMode = TenantMode::DISABLED;
+	encryptionAtRestMode = EncryptionAtRestMode::DISABLED;
 }
 
 int toInt(ValueRef const& v) {
@@ -64,6 +66,16 @@ void parse(int* i, ValueRef const& v) {
 	*i = atoi(v.toString().c_str());
 }
 
+void parse(int64_t* i, ValueRef const& v) {
+	// FIXME: Sanity checking
+	*i = atoll(v.toString().c_str());
+}
+
+void parse(double* i, ValueRef const& v) {
+	// FIXME: Sanity checking
+	*i = atof(v.toString().c_str());
+}
+
 void parseReplicationPolicy(Reference<IReplicationPolicy>* policy, ValueRef const& v) {
 	BinaryReader reader(v, IncludeVersion());
 	serializeReplicationPolicy(reader, *policy);
@@ -213,7 +225,8 @@ bool DatabaseConfiguration::isValid() const {
 	      (perpetualStorageWiggleSpeed == 0 || perpetualStorageWiggleSpeed == 1) &&
 	      isValidPerpetualStorageWiggleLocality(perpetualStorageWiggleLocality) &&
 	      storageMigrationType != StorageMigrationType::UNSET && tenantMode >= TenantMode::DISABLED &&
-	      tenantMode < TenantMode::END)) {
+	      tenantMode < TenantMode::END && encryptionAtRestMode >= EncryptionAtRestMode::DISABLED &&
+	      encryptionAtRestMode < EncryptionAtRestMode::END)) {
 		return false;
 	}
 	std::set<Key> dcIds;
@@ -413,6 +426,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 	result["storage_migration_type"] = storageMigrationType.toString();
 	result["blob_granules_enabled"] = (int32_t)blobGranulesEnabled;
 	result["tenant_mode"] = tenantMode.toString();
+	result["encryption_at_rest_mode"] = encryptionAtRestMode.toString();
 	return result;
 }
 
@@ -546,38 +560,38 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 	KeyRef ck = key.removePrefix(configKeysPrefix);
 	int type;
 
-	if (ck == LiteralStringRef("initialized")) {
+	if (ck == "initialized"_sr) {
 		initialized = true;
-	} else if (ck == LiteralStringRef("commit_proxies")) {
+	} else if (ck == "commit_proxies"_sr) {
 		commitProxyCount = toInt(value);
 		if (commitProxyCount == -1)
 			overwriteProxiesCount();
-	} else if (ck == LiteralStringRef("grv_proxies")) {
+	} else if (ck == "grv_proxies"_sr) {
 		grvProxyCount = toInt(value);
 		if (grvProxyCount == -1)
 			overwriteProxiesCount();
-	} else if (ck == LiteralStringRef("resolvers")) {
+	} else if (ck == "resolvers"_sr) {
 		parse(&resolverCount, value);
-	} else if (ck == LiteralStringRef("logs")) {
+	} else if (ck == "logs"_sr) {
 		parse(&desiredTLogCount, value);
-	} else if (ck == LiteralStringRef("log_replicas")) {
+	} else if (ck == "log_replicas"_sr) {
 		parse(&tLogReplicationFactor, value);
 		tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
-	} else if (ck == LiteralStringRef("log_anti_quorum")) {
+	} else if (ck == "log_anti_quorum"_sr) {
 		parse(&tLogWriteAntiQuorum, value);
 		if (tLogReplicationFactor > 0) {
 			tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
 		}
-	} else if (ck == LiteralStringRef("storage_replicas")) {
+	} else if (ck == "storage_replicas"_sr) {
 		parse(&storageTeamSize, value);
-	} else if (ck == LiteralStringRef("tss_count")) {
+	} else if (ck == "tss_count"_sr) {
 		parse(&desiredTSSCount, value);
-	} else if (ck == LiteralStringRef("log_version")) {
+	} else if (ck == "log_version"_sr) {
 		parse((&type), value);
 		type = std::max((int)TLogVersion::MIN_RECRUITABLE, type);
 		type = std::min((int)TLogVersion::MAX_SUPPORTED, type);
 		tLogVersion = (TLogVersion::Version)type;
-	} else if (ck == LiteralStringRef("log_engine")) {
+	} else if (ck == "log_engine"_sr) {
 		parse((&type), value);
 		tLogDataStoreType = (KeyValueStoreType::StoreType)type;
 		// TODO:  Remove this once Redwood works as a log engine
@@ -588,61 +602,63 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 		if (tLogDataStoreType == KeyValueStoreType::MEMORY_RADIXTREE) {
 			tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2;
 		}
-	} else if (ck == LiteralStringRef("log_spill")) {
+	} else if (ck == "log_spill"_sr) {
 		parse((&type), value);
 		tLogSpillType = (TLogSpillType::SpillType)type;
-	} else if (ck == LiteralStringRef("storage_engine")) {
+	} else if (ck == "storage_engine"_sr) {
 		parse((&type), value);
 		storageServerStoreType = (KeyValueStoreType::StoreType)type;
-	} else if (ck == LiteralStringRef("tss_storage_engine")) {
+	} else if (ck == "tss_storage_engine"_sr) {
 		parse((&type), value);
 		testingStorageServerStoreType = (KeyValueStoreType::StoreType)type;
-	} else if (ck == LiteralStringRef("auto_commit_proxies")) {
+	} else if (ck == "auto_commit_proxies"_sr) {
 		parse(&autoCommitProxyCount, value);
-	} else if (ck == LiteralStringRef("auto_grv_proxies")) {
+	} else if (ck == "auto_grv_proxies"_sr) {
 		parse(&autoGrvProxyCount, value);
-	} else if (ck == LiteralStringRef("auto_resolvers")) {
+	} else if (ck == "auto_resolvers"_sr) {
 		parse(&autoResolverCount, value);
-	} else if (ck == LiteralStringRef("auto_logs")) {
+	} else if (ck == "auto_logs"_sr) {
 		parse(&autoDesiredTLogCount, value);
-	} else if (ck == LiteralStringRef("storage_replication_policy")) {
+	} else if (ck == "storage_replication_policy"_sr) {
 		parseReplicationPolicy(&storagePolicy, value);
-	} else if (ck == LiteralStringRef("log_replication_policy")) {
+	} else if (ck == "log_replication_policy"_sr) {
 		parseReplicationPolicy(&tLogPolicy, value);
-	} else if (ck == LiteralStringRef("log_routers")) {
+	} else if (ck == "log_routers"_sr) {
 		parse(&desiredLogRouterCount, value);
-	} else if (ck == LiteralStringRef("remote_logs")) {
+	} else if (ck == "remote_logs"_sr) {
 		parse(&remoteDesiredTLogCount, value);
-	} else if (ck == LiteralStringRef("remote_log_replicas")) {
+	} else if (ck == "remote_log_replicas"_sr) {
 		parse(&remoteTLogReplicationFactor, value);
-	} else if (ck == LiteralStringRef("remote_log_policy")) {
+	} else if (ck == "remote_log_policy"_sr) {
 		parseReplicationPolicy(&remoteTLogPolicy, value);
-	} else if (ck == LiteralStringRef("backup_worker_enabled")) {
+	} else if (ck == "backup_worker_enabled"_sr) {
 		parse((&type), value);
 		backupWorkerEnabled = (type != 0);
-	} else if (ck == LiteralStringRef("usable_regions")) {
+	} else if (ck == "usable_regions"_sr) {
 		parse(&usableRegions, value);
-	} else if (ck == LiteralStringRef("repopulate_anti_quorum")) {
+	} else if (ck == "repopulate_anti_quorum"_sr) {
 		parse(&repopulateRegionAntiQuorum, value);
-	} else if (ck == LiteralStringRef("regions")) {
+	} else if (ck == "regions"_sr) {
 		parse(&regions, value);
-	} else if (ck == LiteralStringRef("perpetual_storage_wiggle")) {
+	} else if (ck == "perpetual_storage_wiggle"_sr) {
 		parse(&perpetualStorageWiggleSpeed, value);
-	} else if (ck == LiteralStringRef("perpetual_storage_wiggle_locality")) {
+	} else if (ck == "perpetual_storage_wiggle_locality"_sr) {
 		if (!isValidPerpetualStorageWiggleLocality(value.toString())) {
 			return false;
 		}
 		perpetualStorageWiggleLocality = value.toString();
-	} else if (ck == LiteralStringRef("storage_migration_type")) {
+	} else if (ck == "storage_migration_type"_sr) {
 		parse((&type), value);
 		storageMigrationType = (StorageMigrationType::MigrationType)type;
-	} else if (ck == LiteralStringRef("tenant_mode")) {
+	} else if (ck == "tenant_mode"_sr) {
 		tenantMode = TenantMode::fromValue(value);
-	} else if (ck == LiteralStringRef("proxies")) {
+	} else if (ck == "proxies"_sr) {
 		overwriteProxiesCount();
-	} else if (ck == LiteralStringRef("blob_granules_enabled")) {
+	} else if (ck == "blob_granules_enabled"_sr) {
 		parse((&type), value);
 		blobGranulesEnabled = (type != 0);
+	} else if (ck == "encryption_at_rest_mode"_sr) {
+		encryptionAtRestMode = EncryptionAtRestMode::fromValue(value);
 	} else {
 		return false;
 	}
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 333cff9a81..bd34eedb43 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -21,14 +21,27 @@
 #include "fmt/format.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/DatabaseContext.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/JsonBuilder.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/RestoreInterface.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/SystemData.h"
-#include "fdbclient/KeyBackedTypes.h"
-#include "fdbclient/JsonBuilder.h"
+#include "fdbclient/Tenant.h"
+#include "fdbclient/TenantEntryCache.actor.h"
+
+#include "flow/Arena.h"
+#include "flow/CodeProbe.h"
+#include "flow/EncryptUtils.h"
+#include "flow/network.h"
+#include "flow/ObjectSerializer.h"
+#include "flow/ProtocolVersion.h"
+#include "flow/serialize.h"
+#include "flow/Trace.h"
 
 #include <cinttypes>
 #include <ctime>
@@ -36,10 +49,15 @@
 #include "flow/IAsyncFile.h"
 #include "flow/genericactors.actor.h"
 #include "flow/Hash3.h"
+#include "flow/xxhash.h"
+
+#include <memory>
 #include <numeric>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/classification.hpp>
 #include <algorithm>
+#include <unordered_map>
+#include <utility>
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -47,7 +65,7 @@ FDB_DEFINE_BOOLEAN_PARAM(IncrementalBackupOnly);
 FDB_DEFINE_BOOLEAN_PARAM(OnlyApplyMutationLogs);
 
 #define SevFRTestInfo SevVerbose
-//#define SevFRTestInfo SevInfo
+// #define SevFRTestInfo SevInfo
 
 static std::string boolToYesOrNo(bool val) {
 	return val ? std::string("Yes") : std::string("No");
@@ -90,7 +108,7 @@ std::string secondsToTimeFormat(int64_t seconds) {
 		return format("%lld second(s)", seconds);
 }
 
-const Key FileBackupAgent::keyLastRestorable = LiteralStringRef("last_restorable");
+const Key FileBackupAgent::keyLastRestorable = "last_restorable"_sr;
 
 // For convenience
 typedef FileBackupAgent::ERestoreState ERestoreState;
@@ -98,19 +116,19 @@ typedef FileBackupAgent::ERestoreState ERestoreState;
 StringRef FileBackupAgent::restoreStateText(ERestoreState id) {
 	switch (id) {
 	case ERestoreState::UNITIALIZED:
-		return LiteralStringRef("unitialized");
+		return "unitialized"_sr;
 	case ERestoreState::QUEUED:
-		return LiteralStringRef("queued");
+		return "queued"_sr;
 	case ERestoreState::STARTING:
-		return LiteralStringRef("starting");
+		return "starting"_sr;
 	case ERestoreState::RUNNING:
-		return LiteralStringRef("running");
+		return "running"_sr;
 	case ERestoreState::COMPLETED:
-		return LiteralStringRef("completed");
+		return "completed"_sr;
 	case ERestoreState::ABORTED:
-		return LiteralStringRef("aborted");
+		return "aborted"_sr;
 	default:
-		return LiteralStringRef("Unknown");
+		return "Unknown"_sr;
 	}
 }
 
@@ -123,9 +141,9 @@ ACTOR Future<std::vector<KeyBackedTag>> TagUidMap::getAll_impl(TagUidMap* tagsMa
                                                                Reference<ReadYourWritesTransaction> tr,
                                                                Snapshot snapshot) {
 	state Key prefix = tagsMap->prefix; // Copying it here as tagsMap lifetime is not tied to this actor
-	TagMap::PairsType tagPairs = wait(tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot));
+	TagMap::RangeResultType tagPairs = wait(tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot));
 	std::vector<KeyBackedTag> results;
-	for (auto& p : tagPairs)
+	for (auto& p : tagPairs.results)
 		results.push_back(KeyBackedTag(p.first, prefix));
 	return results;
 }
@@ -139,41 +157,37 @@ public:
 	RestoreConfig(UID uid = UID()) : KeyBackedConfig(fileRestorePrefixRange.begin, uid) {}
 	RestoreConfig(Reference<Task> task) : KeyBackedConfig(fileRestorePrefixRange.begin, task) {}
 
-	KeyBackedProperty<ERestoreState> stateEnum() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<ERestoreState> stateEnum() { return configSpace.pack(__FUNCTION__sr); }
 	Future<StringRef> stateText(Reference<ReadYourWritesTransaction> tr) {
 		return map(stateEnum().getD(tr),
 		           [](ERestoreState s) -> StringRef { return FileBackupAgent::restoreStateText(s); });
 	}
-	KeyBackedProperty<Key> addPrefix() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
-	KeyBackedProperty<Key> removePrefix() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
-	KeyBackedProperty<bool> onlyApplyMutationLogs() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
-	KeyBackedProperty<bool> inconsistentSnapshotOnly() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Key> addPrefix() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedProperty<Key> removePrefix() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedProperty<bool> onlyApplyMutationLogs() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedProperty<bool> inconsistentSnapshotOnly() { return configSpace.pack(__FUNCTION__sr); }
 	// XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges
-	KeyBackedProperty<KeyRange> restoreRange() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
-	KeyBackedProperty<std::vector<KeyRange>> restoreRanges() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
-	}
-	KeyBackedProperty<Key> batchFuture() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
-	KeyBackedProperty<Version> beginVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
-	KeyBackedProperty<Version> restoreVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
-	KeyBackedProperty<Version> firstConsistentVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<KeyRange> restoreRange() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedProperty<std::vector<KeyRange>> restoreRanges() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedProperty<Key> batchFuture() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedProperty<Version> beginVersion() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedProperty<Version> restoreVersion() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedProperty<Version> firstConsistentVersion() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedProperty<Reference<IBackupContainer>> sourceContainer() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
-	}
+	KeyBackedProperty<Reference<IBackupContainer>> sourceContainer() { return configSpace.pack(__FUNCTION__sr); }
 	// Get the source container as a bare URL, without creating a container instance
-	KeyBackedProperty<Value> sourceContainerURL() { return configSpace.pack(LiteralStringRef("sourceContainer")); }
+	KeyBackedProperty<Value> sourceContainerURL() { return configSpace.pack("sourceContainer"_sr); }
 
 	// Total bytes written by all log and range restore tasks.
-	KeyBackedBinaryValue<int64_t> bytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> bytesWritten() { return configSpace.pack(__FUNCTION__sr); }
 	// File blocks that have had tasks created for them by the Dispatch task
-	KeyBackedBinaryValue<int64_t> filesBlocksDispatched() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> filesBlocksDispatched() { return configSpace.pack(__FUNCTION__sr); }
 	// File blocks whose tasks have finished
-	KeyBackedBinaryValue<int64_t> fileBlocksFinished() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> fileBlocksFinished() { return configSpace.pack(__FUNCTION__sr); }
 	// Total number of files in the fileMap
-	KeyBackedBinaryValue<int64_t> fileCount() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> fileCount() { return configSpace.pack(__FUNCTION__sr); }
 	// Total number of file blocks in the fileMap
-	KeyBackedBinaryValue<int64_t> fileBlockCount() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> fileBlockCount() { return configSpace.pack(__FUNCTION__sr); }
 
 	Future<std::vector<KeyRange>> getRestoreRangesOrDefault(Reference<ReadYourWritesTransaction> tr) {
 		return getRestoreRangesOrDefault_impl(this, tr);
@@ -200,13 +214,7 @@ public:
 		Version endVersion{ ::invalidVersion }; // not meaningful for range files
 
 		Tuple pack() const {
-			return Tuple()
-			    .append(version)
-			    .append(StringRef(fileName))
-			    .append(isRange)
-			    .append(fileSize)
-			    .append(blockSize)
-			    .append(endVersion);
+			return Tuple::makeTuple(version, fileName, (int)isRange, fileSize, blockSize, endVersion);
 		}
 		static RestoreFile unpack(Tuple const& t) {
 			RestoreFile r;
@@ -222,7 +230,7 @@ public:
 	};
 
 	typedef KeyBackedSet<RestoreFile> FileSetT;
-	FileSetT fileSet() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	FileSetT fileSet() { return configSpace.pack(__FUNCTION__sr); }
 
 	Future<bool> isRunnable(Reference<ReadYourWritesTransaction> tr) {
 		return map(stateEnum().getD(tr), [](ERestoreState s) -> bool {
@@ -464,8 +472,445 @@ Value makePadding(int size) {
 	return pad.substr(0, size);
 }
 
+struct IRangeFileWriter {
+public:
+	virtual Future<Void> padEnd(bool final) = 0;
+
+	virtual Future<Void> writeKV(Key k, Value v) = 0;
+
+	virtual Future<Void> writeKey(Key k) = 0;
+
+	virtual Future<Void> finish() = 0;
+
+	virtual ~IRangeFileWriter() {}
+};
+
+struct SnapshotFileBackupEncryptionKeys {
+	Reference<BlobCipherKey> textCipherKey;
+	EncryptCipherDomainName textDomain;
+	Reference<BlobCipherKey> headerCipherKey;
+	StringRef ivRef;
+};
+
 // File Format handlers.
-// Both Range and Log formats are designed to be readable starting at any 1MB boundary
+// Both Range and Log formats are designed to be readable starting at any BACKUP_RANGEFILE_BLOCK_SIZE boundary
+// so they can be read in parallel.
+//
+// Writer instances must be kept alive while any member actors are in progress.
+//
+// EncryptedRangeFileWriter must be used as follows:
+//   1 - writeKey(key) the queried key range begin
+//   2 - writeKV(k, v) each kv pair to restore
+//   3 - writeKey(key) the queried key range end
+//   4 - finish()
+//
+// EncryptedRangeFileWriter will insert the required padding, header, and extra
+// end/begin keys around the 1MB boundaries as needed.
+//
+// Example:
+//   The range a-z is queries and returns c-j which covers 3 blocks across 2 tenants.
+//   The client code writes keys in this sequence:
+//             t1a t1c t1d t1e t1f t1g t2h t2i t2j t2z
+//
+//   H = header   P = padding   a...z = keys  v = value | = block boundary
+//
+//   Encoded file:  H t1a t1cv t1dv t1ev P | H t1e t1ev t1fv t1gv t2 P | H t2 t2hv t2iv t2jv t2z
+//   Decoded in blocks yields:
+//           Block 1: range [t1a, t1e) with kv pairs t1cv, t1dv
+//           Block 2: range [t1e, t2) with kv pairs t1ev, t1fv, t1gv
+//           Block 3: range [t2, t2z) with kv pairs t2hv, t2iv, t2jv
+//
+//   NOTE: All blocks except for the final block will have one last
+//   value which will not be used.  This isn't actually a waste since
+//   if the next KV pair wouldn't fit within the block after the value
+//   then the space after the final key to the next 1MB boundary would
+//   just be padding anyway.
+//
+//   NOTE: For the EncryptedRangeFileWriter blocks will be split either on the BACKUP_RANGEFILE_BLOCK_SIZE boundary or
+//   when a new tenant id is encountered. If a block is split for crossing tenant boundaries then the last key will be
+//   truncated to just the tenant prefix and the value will be empty (to avoid having sensitive data of one tenant be
+//   encrypted with a key for a different tenant)
+struct EncryptedRangeFileWriter : public IRangeFileWriter {
+	struct Options {
+		constexpr static FileIdentifier file_identifier = 3152016;
+
+		// TODO: Compression is not currently supported so this should always be false
+		bool compressionEnabled = false;
+
+		Options() {}
+
+		template <class Ar>
+		void serialize(Ar& ar) {
+			serializer(ar, compressionEnabled);
+		}
+	};
+
+	EncryptedRangeFileWriter(Database cx,
+	                         Arena* arena,
+	                         Reference<TenantEntryCache<Void>> tenantCache,
+	                         Reference<IBackupFile> file = Reference<IBackupFile>(),
+	                         int blockSize = 0,
+	                         Options options = Options())
+	  : cx(cx), arena(arena), tenantCache(tenantCache), file(file), blockSize(blockSize), blockEnd(0),
+	    fileVersion(BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION), options(options) {
+		buffer = makeString(blockSize);
+		wPtr = mutateString(buffer);
+	}
+
+	static void validateEncryptionHeader(Reference<BlobCipherKey> headerCipherKey,
+	                                     Reference<BlobCipherKey> textCipherKey,
+	                                     BlobCipherEncryptHeader& header) {
+		// Validate encryption header 'cipherHeader' details
+		if (!(header.cipherHeaderDetails.baseCipherId == headerCipherKey->getBaseCipherId() &&
+		      header.cipherHeaderDetails.encryptDomainId == headerCipherKey->getDomainId() &&
+		      header.cipherHeaderDetails.salt == headerCipherKey->getSalt())) {
+			TraceEvent(SevWarn, "EncryptionHeader_CipherHeaderMismatch")
+			    .detail("HeaderDomainId", headerCipherKey->getDomainId())
+			    .detail("ExpectedHeaderDomainId", header.cipherHeaderDetails.encryptDomainId)
+			    .detail("HeaderBaseCipherId", headerCipherKey->getBaseCipherId())
+			    .detail("ExpectedHeaderBaseCipherId", header.cipherHeaderDetails.baseCipherId)
+			    .detail("HeaderSalt", headerCipherKey->getSalt())
+			    .detail("ExpectedHeaderSalt", header.cipherHeaderDetails.salt);
+			throw encrypt_header_metadata_mismatch();
+		}
+
+		// Validate encryption text 'cipherText' details sanity
+		if (!(header.cipherTextDetails.baseCipherId == textCipherKey->getBaseCipherId() &&
+		      header.cipherTextDetails.encryptDomainId == textCipherKey->getDomainId() &&
+		      header.cipherTextDetails.salt == textCipherKey->getSalt())) {
+			TraceEvent(SevWarn, "EncryptionHeader_CipherTextMismatch")
+			    .detail("TextDomainId", textCipherKey->getDomainId())
+			    .detail("ExpectedTextDomainId", header.cipherTextDetails.encryptDomainId)
+			    .detail("TextBaseCipherId", textCipherKey->getBaseCipherId())
+			    .detail("ExpectedTextBaseCipherId", header.cipherTextDetails.baseCipherId)
+			    .detail("TextSalt", textCipherKey->getSalt())
+			    .detail("ExpectedTextSalt", header.cipherTextDetails.salt);
+			throw encrypt_header_metadata_mismatch();
+		}
+	}
+
+	ACTOR static Future<StringRef> decryptImpl(Database cx,
+	                                           StringRef headerS,
+	                                           const uint8_t* dataP,
+	                                           int64_t dataLen,
+	                                           Arena* arena) {
+		Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
+		state BlobCipherEncryptHeader header = BlobCipherEncryptHeader::fromStringRef(headerS);
+		TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::BACKUP));
+		ASSERT(cipherKeys.cipherHeaderKey.isValid() && cipherKeys.cipherTextKey.isValid());
+		validateEncryptionHeader(cipherKeys.cipherHeaderKey, cipherKeys.cipherTextKey, header);
+		DecryptBlobCipherAes256Ctr decryptor(
+		    cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::BACKUP);
+		return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef();
+	}
+
+	static Future<StringRef> decrypt(Database cx,
+	                                 StringRef headerS,
+	                                 const uint8_t* dataP,
+	                                 int64_t dataLen,
+	                                 Arena* arena) {
+		return decryptImpl(cx, headerS, dataP, dataLen, arena);
+	}
+
+	ACTOR static Future<Reference<BlobCipherKey>> refreshKey(EncryptedRangeFileWriter* self,
+	                                                         EncryptCipherDomainId domainId,
+	                                                         EncryptCipherDomainName domainName) {
+		Reference<AsyncVar<ClientDBInfo> const> dbInfo = self->cx->clientInfo;
+		TextAndHeaderCipherKeys cipherKeys =
+		    wait(getLatestEncryptCipherKeysForDomain(dbInfo, domainId, domainName, BlobCipherMetrics::BACKUP));
+		return cipherKeys.cipherTextKey;
+	}
+
+	ACTOR static Future<Void> encrypt(EncryptedRangeFileWriter* self) {
+		ASSERT(self->cipherKeys.headerCipherKey.isValid() && self->cipherKeys.textCipherKey.isValid());
+		// Ensure that the keys we got are still valid before flushing the block
+		if (self->cipherKeys.headerCipherKey->isExpired() || self->cipherKeys.headerCipherKey->needsRefresh()) {
+			Reference<BlobCipherKey> cipherKey =
+			    wait(refreshKey(self, self->cipherKeys.headerCipherKey->getDomainId(), FDB_ENCRYPT_HEADER_DOMAIN_NAME));
+			self->cipherKeys.headerCipherKey = cipherKey;
+		}
+		if (self->cipherKeys.textCipherKey->isExpired() || self->cipherKeys.textCipherKey->needsRefresh()) {
+			Reference<BlobCipherKey> cipherKey =
+			    wait(refreshKey(self, self->cipherKeys.textCipherKey->getDomainId(), self->cipherKeys.textDomain));
+			self->cipherKeys.textCipherKey = cipherKey;
+		}
+		EncryptBlobCipherAes265Ctr encryptor(self->cipherKeys.textCipherKey,
+		                                     self->cipherKeys.headerCipherKey,
+		                                     self->cipherKeys.ivRef.begin(),
+		                                     AES_256_IV_LENGTH,
+		                                     ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE,
+		                                     BlobCipherMetrics::BACKUP);
+		Arena arena;
+		int64_t payloadSize = self->wPtr - self->dataPayloadStart;
+		auto encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, self->encryptHeader, arena);
+
+		// re-write encrypted data to buffer
+		std::memcpy(self->dataPayloadStart, encryptedData->begin(), payloadSize);
+		return Void();
+	}
+
+	ACTOR static Future<Void> updateEncryptionKeysCtx(EncryptedRangeFileWriter* self,
+	                                                  KeyRef key,
+	                                                  Reference<TenantEntryCache<Void>> cache) {
+		state std::pair<int64_t, TenantName> curTenantInfo = wait(getEncryptionDomainDetails(key, cache));
+		state Reference<AsyncVar<ClientDBInfo> const> dbInfo = self->cx->clientInfo;
+
+		// Get text and header cipher key
+		TextAndHeaderCipherKeys textAndHeaderCipherKeys = wait(getLatestEncryptCipherKeysForDomain(
+		    dbInfo, curTenantInfo.first, curTenantInfo.second, BlobCipherMetrics::BACKUP));
+		self->cipherKeys.textCipherKey = textAndHeaderCipherKeys.cipherTextKey;
+		self->cipherKeys.textDomain = curTenantInfo.second;
+		self->cipherKeys.headerCipherKey = textAndHeaderCipherKeys.cipherHeaderKey;
+
+		// Set ivRef
+		self->cipherKeys.ivRef = makeString(AES_256_IV_LENGTH, *self->arena);
+		deterministicRandom()->randomBytes(mutateString(self->cipherKeys.ivRef), AES_256_IV_LENGTH);
+		return Void();
+	}
+
+	// Returns the number of bytes that have been written to the buffer
+	static int64_t currentBufferSize(EncryptedRangeFileWriter* self) { return self->wPtr - self->buffer.begin(); }
+
+	static int64_t expectedFileSize(EncryptedRangeFileWriter* self) {
+		// Return what has already been written to file plus the size of the current buffer
+		// which indicates how many bytes the file will contain once the buffer is written
+		return self->file->size() + currentBufferSize(self);
+	}
+
+	static void copyToBuffer(EncryptedRangeFileWriter* self, const void* src, size_t size) {
+		if (size > 0) {
+			std::memcpy(self->wPtr, src, size);
+			self->wPtr += size;
+			ASSERT(currentBufferSize(self) <= self->blockSize);
+		}
+	}
+
+	static void appendStringRefWithLenToBuffer(EncryptedRangeFileWriter* self, StringRef* s) {
+		// Append the string length followed by the string to the buffer
+		uint32_t lenBuf = bigEndian32((uint32_t)s->size());
+		copyToBuffer(self, &lenBuf, sizeof(lenBuf));
+		copyToBuffer(self, s->begin(), s->size());
+	}
+
+	static bool isSystemKey(KeyRef key) { return key.size() && key[0] == systemKeys.begin[0]; }
+
+	ACTOR static Future<std::pair<int64_t, TenantName>> getEncryptionDomainDetailsImpl(
+	    KeyRef key,
+	    Reference<TenantEntryCache<Void>> tenantCache) {
+		if (isSystemKey(key)) {
+			return std::make_pair(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
+		}
+		if (key.size() < TENANT_PREFIX_SIZE) {
+			return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
+		}
+		KeyRef tenantPrefix = KeyRef(key.begin(), TENANT_PREFIX_SIZE);
+		state int64_t tenantId = TenantMapEntry::prefixToId(tenantPrefix);
+		Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache->getById(tenantId));
+		if (payload.present()) {
+			return std::make_pair(tenantId, payload.get().name);
+		}
+		return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
+	}
+
+	static Future<std::pair<int64_t, TenantName>> getEncryptionDomainDetails(
+	    KeyRef key,
+	    Reference<TenantEntryCache<Void>> tenantCache) {
+		return getEncryptionDomainDetailsImpl(key, tenantCache);
+	}
+
+	// Handles the first block and internal blocks.  Ends current block if needed.
+	// The final flag is used in simulation to pad the file's final block to a whole block size
+	ACTOR static Future<Void> newBlock(EncryptedRangeFileWriter* self,
+	                                   int bytesNeeded,
+	                                   KeyRef lastKey,
+	                                   bool writeValue,
+	                                   bool final = false) {
+		// Write padding to finish current block if needed
+		int bytesLeft = self->blockEnd - expectedFileSize(self);
+		ASSERT(bytesLeft >= 0);
+		if (bytesLeft > 0) {
+			state Value paddingFFs = makePadding(bytesLeft);
+			copyToBuffer(self, paddingFFs.begin(), bytesLeft);
+		}
+
+		if (expectedFileSize(self) > 0) {
+			// write buffer to file since block is finished
+			ASSERT(currentBufferSize(self) == self->blockSize);
+			wait(encrypt(self));
+			wait(self->file->append(self->buffer.begin(), self->blockSize));
+
+			// reset write pointer to beginning of StringRef
+			self->wPtr = mutateString(self->buffer);
+		}
+
+		if (final) {
+			ASSERT(g_network->isSimulated());
+			return Void();
+		}
+
+		// Set new blockEnd
+		self->blockEnd += self->blockSize;
+
+		// write Header
+		copyToBuffer(self, (uint8_t*)&self->fileVersion, sizeof(self->fileVersion));
+
+		// write options struct
+		Value serialized =
+		    ObjectWriter::toValue(self->options, IncludeVersion(ProtocolVersion::withEncryptedSnapshotBackupFile()));
+		appendStringRefWithLenToBuffer(self, &serialized);
+
+		// leave space for encryption header
+		self->encryptHeader = (BlobCipherEncryptHeader*)self->wPtr;
+		self->wPtr += BlobCipherEncryptHeader::headerSize;
+		self->dataPayloadStart = self->wPtr;
+
+		// If this is NOT the first block then write duplicate stuff needed from last block
+		if (self->blockEnd > self->blockSize) {
+			appendStringRefWithLenToBuffer(self, &lastKey);
+			appendStringRefWithLenToBuffer(self, &self->lastKey);
+			if (writeValue) {
+				appendStringRefWithLenToBuffer(self, &self->lastValue);
+			}
+		}
+
+		// There must now be room in the current block for bytesNeeded or the block size is too small
+		if (expectedFileSize(self) + bytesNeeded > self->blockEnd) {
+			throw backup_bad_block_size();
+		}
+
+		return Void();
+	}
+
+	Future<Void> padEnd(bool final) {
+		if (expectedFileSize(this) > 0) {
+			return newBlock(this, 0, StringRef(), true, final);
+		}
+		return Void();
+	}
+
+	// Ends the current block if necessary based on bytesNeeded.
+	ACTOR static Future<Void> newBlockIfNeeded(EncryptedRangeFileWriter* self, int bytesNeeded) {
+		if (expectedFileSize(self) + bytesNeeded > self->blockEnd) {
+			wait(newBlock(self, bytesNeeded, self->lastKey, true));
+		}
+		return Void();
+	}
+
+	ACTOR static Future<Void> handleTenantBondary(EncryptedRangeFileWriter* self,
+	                                              Key k,
+	                                              Value v,
+	                                              bool writeValue,
+	                                              std::pair<int64_t, TenantName> curKeyTenantInfo) {
+		state KeyRef endKey = k;
+		// If we are crossing a boundary with a key that has a tenant prefix then truncate it
+		if (curKeyTenantInfo.first != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID &&
+		    curKeyTenantInfo.first != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
+			endKey = StringRef(k.begin(), TENANT_PREFIX_SIZE);
+		}
+		state ValueRef newValue = StringRef();
+		self->lastKey = k;
+		self->lastValue = v;
+		appendStringRefWithLenToBuffer(self, &endKey);
+		appendStringRefWithLenToBuffer(self, &newValue);
+		wait(newBlock(self, 0, endKey, writeValue));
+		wait(updateEncryptionKeysCtx(self, self->lastKey, self->tenantCache));
+		return Void();
+	}
+
+	ACTOR static Future<bool> finishCurTenantBlockStartNewIfNeeded(EncryptedRangeFileWriter* self,
+	                                                               Key k,
+	                                                               Value v,
+	                                                               bool writeValue) {
+		// Don't want to start a new block if the current key or previous key is empty
+		if (self->lastKey.size() == 0 || k.size() == 0) {
+			return false;
+		}
+		state std::pair<int64_t, TenantName> curKeyTenantInfo = wait(getEncryptionDomainDetails(k, self->tenantCache));
+		state std::pair<int64_t, TenantName> prevKeyTenantInfo =
+		    wait(getEncryptionDomainDetails(self->lastKey, self->tenantCache));
+		// crossing tenant boundaries so finish the current block using only the tenant prefix of the new key
+		if (curKeyTenantInfo.first != prevKeyTenantInfo.first) {
+			CODE_PROBE(true, "crossed tenant boundaries");
+			wait(handleTenantBondary(self, k, v, writeValue, curKeyTenantInfo));
+			return true;
+		}
+		return false;
+	}
+
+	// Start a new block if needed, then write the key and value
+	ACTOR static Future<Void> writeKV_impl(EncryptedRangeFileWriter* self, Key k, Value v) {
+		if (!self->cipherKeys.headerCipherKey.isValid() || !self->cipherKeys.textCipherKey.isValid()) {
+			wait(updateEncryptionKeysCtx(self, k, self->tenantCache));
+		}
+		state int toWrite = sizeof(int32_t) + k.size() + sizeof(int32_t) + v.size();
+		wait(newBlockIfNeeded(self, toWrite));
+		bool createdNewBlock = wait(finishCurTenantBlockStartNewIfNeeded(self, k, v, true));
+		if (createdNewBlock) {
+			return Void();
+		}
+		appendStringRefWithLenToBuffer(self, &k);
+		appendStringRefWithLenToBuffer(self, &v);
+		self->lastKey = k;
+		self->lastValue = v;
+		return Void();
+	}
+
+	Future<Void> writeKV(Key k, Value v) { return writeKV_impl(this, k, v); }
+
+	// Write begin key or end key.
+	ACTOR static Future<Void> writeKey_impl(EncryptedRangeFileWriter* self, Key k) {
+		// TODO (Nim): Is it possible to write empty begin and end keys?
+		if (k.size() > 0 &&
+		    (!self->cipherKeys.headerCipherKey.isValid() || !self->cipherKeys.textCipherKey.isValid())) {
+			wait(updateEncryptionKeysCtx(self, k, self->tenantCache));
+		}
+
+		// Need to account for extra "empty" value being written in the case of crossing tenant boundaries
+		int toWrite = sizeof(uint32_t) + k.size() + sizeof(uint32_t);
+		wait(newBlockIfNeeded(self, toWrite));
+		bool createdNewBlock = wait(finishCurTenantBlockStartNewIfNeeded(self, k, StringRef(), false));
+		if (createdNewBlock) {
+			return Void();
+		}
+		appendStringRefWithLenToBuffer(self, &k);
+		self->lastKey = k;
+		return Void();
+	}
+
+	Future<Void> writeKey(Key k) { return writeKey_impl(this, k); }
+
+	ACTOR static Future<Void> finish_impl(EncryptedRangeFileWriter* self) {
+		// Write any outstanding bytes to the file
+		if (currentBufferSize(self) > 0) {
+			wait(encrypt(self));
+			wait(self->file->append(self->buffer.begin(), currentBufferSize(self)));
+		}
+		return Void();
+	}
+
+	Future<Void> finish() { return finish_impl(this); }
+
+	Database cx;
+	Arena* arena;
+	Reference<TenantEntryCache<Void>> tenantCache;
+	Reference<IBackupFile> file;
+	int blockSize;
+
+private:
+	Standalone<StringRef> buffer;
+	uint8_t* wPtr;
+	BlobCipherEncryptHeader* encryptHeader;
+	uint8_t* dataPayloadStart;
+	int64_t blockEnd;
+	uint32_t fileVersion;
+	Options options;
+	Key lastKey;
+	Key lastValue;
+	SnapshotFileBackupEncryptionKeys cipherKeys;
+};
+
+// File Format handlers.
+// Both Range and Log formats are designed to be readable starting at any BACKUP_RANGEFILE_BLOCK_SIZE boundary
 // so they can be read in parallel.
 //
 // Writer instances must be kept alive while any member actors are in progress.
@@ -474,6 +919,7 @@ Value makePadding(int size) {
 //   1 - writeKey(key) the queried key range begin
 //   2 - writeKV(k, v) each kv pair to restore
 //   3 - writeKey(key) the queried key range end
+//	 4 - finish()
 //
 // RangeFileWriter will insert the required padding, header, and extra
 // end/begin keys around the 1MB boundaries as needed.
@@ -496,7 +942,7 @@ Value makePadding(int size) {
 //   if the next KV pair wouldn't fit within the block after the value
 //   then the space after the final key to the next 1MB boundary would
 //   just be padding anyway.
-struct RangeFileWriter {
+struct RangeFileWriter : public IRangeFileWriter {
 	RangeFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0)
 	  : file(file), blockSize(blockSize), blockEnd(0), fileVersion(BACKUP_AGENT_SNAPSHOT_FILE_VERSION) {}
 
@@ -536,10 +982,10 @@ struct RangeFileWriter {
 	}
 
 	// Used in simulation only to create backup file sizes which are an integer multiple of the block size
-	Future<Void> padEnd() {
+	Future<Void> padEnd(bool final) {
 		ASSERT(g_network->isSimulated());
 		if (file->size() > 0) {
-			return newBlock(this, 0, true);
+			return newBlock(this, 0, final);
 		}
 		return Void();
 	}
@@ -574,6 +1020,8 @@ struct RangeFileWriter {
 
 	Future<Void> writeKey(Key k) { return writeKey_impl(this, k); }
 
+	Future<Void> finish() { return Void(); }
+
 	Reference<IBackupFile> file;
 	int blockSize;
 
@@ -584,9 +1032,49 @@ private:
 	Key lastValue;
 };
 
+ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
+                                        Standalone<VectorRef<KeyValueRef>>* results,
+                                        bool encryptedBlock,
+                                        Optional<Database> cx,
+                                        Reference<TenantEntryCache<Void>> tenantCache) {
+	// Read begin key, if this fails then block was invalid.
+	state uint32_t kLen = reader->consumeNetworkUInt32();
+	state const uint8_t* k = reader->consume(kLen);
+	results->push_back(results->arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
+
+	// Read kv pairs and end key
+	while (1) {
+		// Read a key.
+		kLen = reader->consumeNetworkUInt32();
+		k = reader->consume(kLen);
+
+		// If eof reached or first value len byte is 0xFF then a valid block end was reached.
+		if (reader->eof() || *reader->rptr == 0xFF) {
+			results->push_back(results->arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
+			break;
+		}
+
+		// Read a value, which must exist or the block is invalid
+		uint32_t vLen = reader->consumeNetworkUInt32();
+		const uint8_t* v = reader->consume(vLen);
+		results->push_back(results->arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
+
+		// If eof reached or first byte of next key len is 0xFF then a valid block end was reached.
+		if (reader->eof() || *reader->rptr == 0xFF)
+			break;
+	}
+
+	// Make sure any remaining bytes in the block are 0xFF
+	for (auto b : reader->remainder())
+		if (b != 0xFF)
+			throw restore_corrupted_data_padding();
+	return Void();
+}
+
 ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
                                                                       int64_t offset,
-                                                                      int len) {
+                                                                      int len,
+                                                                      Optional<Database> cx) {
 	state Standalone<StringRef> buf = makeString(len);
 	int rLen = wait(file->read(mutateString(buf), len, offset));
 	if (rLen != len)
@@ -594,48 +1082,44 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
 
 	simulateBlobFailure();
 
-	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
+	state Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
 	state StringRefReader reader(buf, restore_corrupted_data());
+	state Arena arena;
 
 	try {
-		// Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION
-		if (reader.consume<int32_t>() != BACKUP_AGENT_SNAPSHOT_FILE_VERSION)
+		// Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION or
+		// BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION
+		int32_t file_version = reader.consume<int32_t>();
+		if (file_version == BACKUP_AGENT_SNAPSHOT_FILE_VERSION) {
+			wait(decodeKVPairs(&reader, &results, false, cx, Reference<TenantEntryCache<Void>>()));
+		} else if (file_version == BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION) {
+			CODE_PROBE(true, "decoding encrypted block");
+			ASSERT(cx.present());
+			// decode options struct
+			uint32_t optionsLen = reader.consumeNetworkUInt32();
+			const uint8_t* o = reader.consume(optionsLen);
+			StringRef optionsStringRef = StringRef(o, optionsLen);
+			EncryptedRangeFileWriter::Options options =
+			    ObjectReader::fromStringRef<EncryptedRangeFileWriter::Options>(optionsStringRef, IncludeVersion());
+			ASSERT(!options.compressionEnabled);
+
+			// read encryption header
+			const uint8_t* headerStart = reader.consume(BlobCipherEncryptHeader::headerSize);
+			StringRef header = StringRef(headerStart, BlobCipherEncryptHeader::headerSize);
+			const uint8_t* dataPayloadStart = headerStart + BlobCipherEncryptHeader::headerSize;
+			// calculate the total bytes read up to (and including) the header
+			int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + optionsLen + BlobCipherEncryptHeader::headerSize;
+			// get the size of the encrypted payload and decrypt it
+			int64_t dataLen = len - bytesRead;
+			StringRef decryptedData =
+			    wait(EncryptedRangeFileWriter::decrypt(cx.get(), header, dataPayloadStart, dataLen, &results.arena()));
+			reader = StringRefReader(decryptedData, restore_corrupted_data());
+			Reference<TenantEntryCache<Void>> tenantCache = makeReference<TenantEntryCache<Void>>(cx.get());
+			wait(decodeKVPairs(&reader, &results, true, cx, tenantCache));
+		} else {
 			throw restore_unsupported_file_version();
-
-		// Read begin key, if this fails then block was invalid.
-		uint32_t kLen = reader.consumeNetworkUInt32();
-		const uint8_t* k = reader.consume(kLen);
-		results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
-
-		// Read kv pairs and end key
-		while (1) {
-			// Read a key.
-			kLen = reader.consumeNetworkUInt32();
-			k = reader.consume(kLen);
-
-			// If eof reached or first value len byte is 0xFF then a valid block end was reached.
-			if (reader.eof() || *reader.rptr == 0xFF) {
-				results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
-				break;
-			}
-
-			// Read a value, which must exist or the block is invalid
-			uint32_t vLen = reader.consumeNetworkUInt32();
-			const uint8_t* v = reader.consume(vLen);
-			results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
-
-			// If eof reached or first byte of next key len is 0xFF then a valid block end was reached.
-			if (reader.eof() || *reader.rptr == 0xFF)
-				break;
 		}
-
-		// Make sure any remaining bytes in the block are 0xFF
-		for (auto b : reader.remainder())
-			if (b != 0xFF)
-				throw restore_corrupted_data_padding();
-
 		return results;
-
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "FileRestoreDecodeRangeFileBlockFailed")
 		    .error(e)
@@ -781,8 +1265,7 @@ ACTOR static Future<Void> abortFiveZeroBackup(FileBackupAgent* backupAgent,
 
 	state Subspace statusSpace = backupAgent->subspace.get(BackupAgentBase::keyStates).get(uid.toString());
 	state Subspace globalConfig = backupAgent->subspace.get(BackupAgentBase::keyConfig).get(uid.toString());
-	state Subspace newConfigSpace =
-	    uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(fileBackupPrefixRange.begin), uid);
+	state Subspace newConfigSpace = uidPrefixKey("uid->config/"_sr.withPrefix(fileBackupPrefixRange.begin), uid);
 
 	Optional<Value> statusStr = wait(tr->get(statusSpace.pack(FileBackupAgent::keyStateStatus)));
 	state EBackupState status =
@@ -822,8 +1305,6 @@ struct AbortFiveZeroBackupTask : TaskFuncBase {
 		state FileBackupAgent backupAgent;
 		state std::string tagName = task->params[BackupAgentBase::keyConfigBackupTag].toString();
 
-		TEST(true); // Canceling old backup task
-
 		TraceEvent(SevInfo, "FileBackupCancelOldTask")
 		    .detail("Task", task->params[Task::reservedTaskParamKeyType])
 		    .detail("TagName", tagName);
@@ -853,7 +1334,7 @@ struct AbortFiveZeroBackupTask : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef AbortFiveZeroBackupTask::name = LiteralStringRef("abort_legacy_backup");
+StringRef AbortFiveZeroBackupTask::name = "abort_legacy_backup"_sr;
 REGISTER_TASKFUNC(AbortFiveZeroBackupTask);
 REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_backup_diff_logs);
 REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_backup_log_range);
@@ -908,8 +1389,6 @@ struct AbortFiveOneBackupTask : TaskFuncBase {
 		state BackupConfig config(task);
 		state std::string tagName = wait(config.tag().getOrThrow(tr));
 
-		TEST(true); // Canceling 5.1 backup task
-
 		TraceEvent(SevInfo, "FileBackupCancelFiveOneTask")
 		    .detail("Task", task->params[Task::reservedTaskParamKeyType])
 		    .detail("TagName", tagName);
@@ -939,7 +1418,7 @@ struct AbortFiveOneBackupTask : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef AbortFiveOneBackupTask::name = LiteralStringRef("abort_legacy_backup_5.2");
+StringRef AbortFiveOneBackupTask::name = "abort_legacy_backup_5.2"_sr;
 REGISTER_TASKFUNC(AbortFiveOneBackupTask);
 REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_write_range);
 REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_dispatch_ranges);
@@ -978,7 +1457,7 @@ ACTOR static Future<Key> addBackupTask(StringRef name,
 	}
 	wait(waitFor->onSetAddTask(tr, taskBucket, task));
 
-	return LiteralStringRef("OnSetAddTask");
+	return "OnSetAddTask"_sr;
 }
 
 // Clears the backup ID from "backupStartedKey" to pause backup workers.
@@ -1050,9 +1529,9 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 	static constexpr uint32_t version = 1;
 
 	static struct {
-		static TaskParam<Key> beginKey() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<Key> endKey() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<bool> addBackupRangeTasks() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<Key> beginKey() { return __FUNCTION__sr; }
+		static TaskParam<Key> endKey() { return __FUNCTION__sr; }
+		static TaskParam<bool> addBackupRangeTasks() { return __FUNCTION__sr; }
 	} Params;
 
 	std::string toString(Reference<Task> task) const override {
@@ -1077,8 +1556,8 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	// Finish (which flushes/syncs) the file, and then in a single transaction, make some range backup progress durable.
-	// This means:
+	// Finish (which flushes/syncs) the file, and then in a single transaction, make some range backup progress
+	// durable. This means:
 	//  - increment the backup config's range bytes written
 	//  - update the range file map
 	//  - update the task begin key
@@ -1186,8 +1665,8 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		    .detail("EndKey", Params.endKey().get(task).printable())
 		    .detail("TaskKey", task->key.printable());
 
-		// When a key range task saves the last chunk of progress and then the executor dies, when the task continues
-		// its beginKey and endKey will be equal but there is no work to be done.
+		// When a key range task saves the last chunk of progress and then the executor dies, when the task
+		// continues its beginKey and endKey will be equal but there is no work to be done.
 		if (beginKey == endKey)
 			return Void();
 
@@ -1200,8 +1679,8 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		}
 
 		// Read everything from beginKey to endKey, write it to an output file, run the output file processor, and
-		// then set on_done. If we are still writing after X seconds, end the output file and insert a new backup_range
-		// task for the remainder.
+		// then set on_done. If we are still writing after X seconds, end the output file and insert a new
+		// backup_range task for the remainder.
 		state Reference<IBackupFile> outFile;
 		state Version outVersion = invalidVersion;
 		state Key lastKey;
@@ -1216,11 +1695,13 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		                                      Terminator::True,
 		                                      AccessSystemKeys::True,
 		                                      LockAware::True);
-		state RangeFileWriter rangeFile;
+		state std::unique_ptr<IRangeFileWriter> rangeFile;
 		state BackupConfig backup(task);
+		state Arena arena;
+		state Reference<TenantEntryCache<Void>> tenantCache = makeReference<TenantEntryCache<Void>>(cx);
 
-		// Don't need to check keepRunning(task) here because we will do that while finishing each output file, but if
-		// bc is false then clearly the backup is no longer in progress
+		// Don't need to check keepRunning(task) here because we will do that while finishing each output file, but
+		// if bc is false then clearly the backup is no longer in progress
 		state Reference<IBackupContainer> bc = wait(backup.backupContainer().getD(cx.getReference()));
 		if (!bc) {
 			return Void();
@@ -1228,6 +1709,7 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 
 		state bool done = false;
 		state int64_t nrKeys = 0;
+		state bool encryptionEnabled = false;
 
 		loop {
 			state RangeResultWithVersion values;
@@ -1242,17 +1724,20 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 					throw;
 			}
 
-			// If we've seen a new read version OR hit the end of the stream, then if we were writing a file finish it.
+			// If we've seen a new read version OR hit the end of the stream, then if we were writing a file finish
+			// it.
 			if (values.second != outVersion || done) {
 				if (outFile) {
-					TEST(outVersion != invalidVersion); // Backup range task wrote multiple versions
+					CODE_PROBE(outVersion != invalidVersion, "Backup range task wrote multiple versions");
 					state Key nextKey = done ? endKey : keyAfter(lastKey);
-					wait(rangeFile.writeKey(nextKey));
+					wait(rangeFile->writeKey(nextKey));
 
 					if (BUGGIFY) {
-						wait(rangeFile.padEnd());
+						wait(rangeFile->padEnd(true));
 					}
 
+					wait(rangeFile->finish());
+
 					bool usedFile = wait(
 					    finishRangeFile(outFile, cx, task, taskBucket, KeyRangeRef(beginKey, nextKey), outVersion));
 					TraceEvent("FileBackupWroteRangeFile")
@@ -1275,8 +1760,8 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 				// Start writing a new file after verifying this task should keep running as of a new read version
 				// (which must be >= outVersion)
 				outVersion = values.second;
-				// block size must be at least large enough for 3 max size keys and 2 max size values + overhead so 250k
-				// conservatively.
+				// block size must be at least large enough for 3 max size keys and 2 max size values + overhead so
+				// 250k conservatively.
 				state int blockSize =
 				    BUGGIFY ? deterministicRandom()->randomInt(250e3, 4e6) : CLIENT_KNOBS->BACKUP_RANGEFILE_BLOCK_SIZE;
 				state Version snapshotBeginVersion;
@@ -1290,6 +1775,7 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 
 						wait(taskBucket->keepRunning(tr, task) &&
 						     storeOrThrow(snapshotBeginVersion, backup.snapshotBeginVersion().get(tr)) &&
+						     storeOrThrow(encryptionEnabled, backup.enableSnapshotBackupEncryption().get(tr)) &&
 						     store(snapshotRangeFileCount, backup.snapshotRangeFileCount().getD(tr)));
 
 						break;
@@ -1302,16 +1788,22 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 				    wait(bc->writeRangeFile(snapshotBeginVersion, snapshotRangeFileCount, outVersion, blockSize));
 				outFile = f;
 
+				encryptionEnabled = encryptionEnabled && cx->clientInfo->get().isEncryptionEnabled;
 				// Initialize range file writer and write begin key
-				rangeFile = RangeFileWriter(outFile, blockSize);
-				wait(rangeFile.writeKey(beginKey));
+				if (encryptionEnabled) {
+					CODE_PROBE(true, "using encrypted snapshot file writer");
+					rangeFile = std::make_unique<EncryptedRangeFileWriter>(cx, &arena, tenantCache, outFile, blockSize);
+				} else {
+					rangeFile = std::make_unique<RangeFileWriter>(outFile, blockSize);
+				}
+				wait(rangeFile->writeKey(beginKey));
 			}
 
 			// write kvData to file, update lastKey and key count
 			if (values.first.size() != 0) {
 				state size_t i = 0;
 				for (; i < values.first.size(); ++i) {
-					wait(rangeFile.writeKV(values.first[i].key, values.first[i].value));
+					wait(rangeFile->writeKV(values.first[i].key, values.first[i].value));
 				}
 				lastKey = values.first.back().key;
 				nrKeys += values.first.size();
@@ -1376,7 +1868,6 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 	                                  Reference<FutureBucket> futureBucket,
 	                                  Reference<Task> task) {
 		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-
 		if (Params.addBackupRangeTasks().get(task)) {
 			wait(startBackupRangeInternal(tr, taskBucket, futureBucket, task, taskFuture));
 		} else {
@@ -1395,7 +1886,7 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		return Void();
 	}
 };
-StringRef BackupRangeTaskFunc::name = LiteralStringRef("file_backup_write_range_5.2");
+StringRef BackupRangeTaskFunc::name = "file_backup_write_range_5.2"_sr;
 REGISTER_TASKFUNC(BackupRangeTaskFunc);
 
 struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
@@ -1404,11 +1895,11 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 
 	static struct {
 		// Set by Execute, used by Finish
-		static TaskParam<int64_t> shardsBehind() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<int64_t> shardsBehind() { return __FUNCTION__sr; }
 		// Set by Execute, used by Finish
-		static TaskParam<bool> snapshotFinished() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<bool> snapshotFinished() { return __FUNCTION__sr; }
 		// Set by Execute, used by Finish
-		static TaskParam<Version> nextDispatchVersion() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<Version> nextDispatchVersion() { return __FUNCTION__sr; }
 	} Params;
 
 	StringRef getName() const override { return name; };
@@ -1461,12 +1952,12 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		state double startTime = timer();
 		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
-		// The shard map will use 3 values classes.  Exactly SKIP, exactly DONE, then any number >= NOT_DONE_MIN which
-		// will mean not done. This is to enable an efficient coalesce() call to squash adjacent ranges which are not
-		// yet finished to enable efficiently finding random database shards which are not done.
+		// The shard map will use 3 values classes.  Exactly SKIP, exactly DONE, then any number >= NOT_DONE_MIN
+		// which will mean not done. This is to enable an efficient coalesce() call to squash adjacent ranges which
+		// are not yet finished to enable efficiently finding random database shards which are not done.
 		state int notDoneSequence = NOT_DONE_MIN;
-		state KeyRangeMap<int> shardMap(notDoneSequence++, normalKeys.end);
-		state Key beginKey = normalKeys.begin;
+		state KeyRangeMap<int> shardMap(notDoneSequence++);
+		state Key beginKey = allKeys.begin;
 
 		// Read all shard boundaries and add them to the map
 		loop {
@@ -1475,7 +1966,7 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 				state Future<Standalone<VectorRef<KeyRef>>> shardBoundaries =
-				    getBlockOfShards(tr, beginKey, normalKeys.end, CLIENT_KNOBS->TOO_MANY);
+				    getBlockOfShards(tr, beginKey, allKeys.end, CLIENT_KNOBS->TOO_MANY);
 				wait(success(shardBoundaries) && taskBucket->keepRunning(tr, task));
 
 				if (shardBoundaries.get().size() == 0)
@@ -1520,7 +2011,8 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 				     store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
 				     store(recentReadVersion, tr->getReadVersion()) && taskBucket->keepRunning(tr, task));
 
-				// If the snapshot batch future key does not exist, this is the first execution of this dispatch task so
+				// If the snapshot batch future key does not exist, this is the first execution of this dispatch
+				// task so
 				//    - create and set the snapshot batch future key
 				//    - initialize the batch size to 0
 				//    - initialize the target snapshot end version if it is not yet set
@@ -1532,7 +2024,8 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 					config.snapshotBatchSize().set(tr, snapshotBatchSize.get());
 
 					// The dispatch of this batch can take multiple separate executions if the executor fails
-					// so store a completion key for the dispatch finish() to set when dispatching the batch is done.
+					// so store a completion key for the dispatch finish() to set when dispatching the batch is
+					// done.
 					state TaskCompletionKey dispatchCompletionKey = TaskCompletionKey::joinWith(snapshotBatchFuture);
 					// this is a bad hack - but flow doesn't work well with lambda functions and caputring
 					// state variables...
@@ -1558,24 +2051,28 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		// Read all dispatched ranges
 		state std::vector<std::pair<Key, bool>> dispatchBoundaries;
 		tr->reset();
-		beginKey = normalKeys.begin;
+		beginKey = allKeys.begin;
 		loop {
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state Future<std::vector<std::pair<Key, bool>>> bounds = config.snapshotRangeDispatchMap().getRange(
-				    tr, beginKey, keyAfter(normalKeys.end), CLIENT_KNOBS->TOO_MANY);
+				state Future<BackupConfig::RangeDispatchMapT::RangeResultType> bounds =
+				    config.snapshotRangeDispatchMap().getRange(
+				        tr, beginKey, keyAfter(allKeys.end), CLIENT_KNOBS->TOO_MANY);
 				wait(success(bounds) && taskBucket->keepRunning(tr, task) &&
 				     store(recentReadVersion, tr->getReadVersion()));
 
-				if (bounds.get().empty())
+				if (!bounds.get().results.empty()) {
+					dispatchBoundaries.reserve(dispatchBoundaries.size() + bounds.get().results.size());
+					dispatchBoundaries.insert(
+					    dispatchBoundaries.end(), bounds.get().results.begin(), bounds.get().results.end());
+				}
+
+				if (!bounds.get().more)
 					break;
 
-				dispatchBoundaries.reserve(dispatchBoundaries.size() + bounds.get().size());
-				dispatchBoundaries.insert(dispatchBoundaries.end(), bounds.get().begin(), bounds.get().end());
-
-				beginKey = keyAfter(bounds.get().back().first);
+				beginKey = keyAfter(bounds.get().results.back().first);
 				tr->reset();
 			} catch (Error& e) {
 				wait(tr->onError(e));
@@ -1603,8 +2100,8 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 
 				// If this was the end of a dispatched range
 				if (!boundary.second) {
-					// Ensure that the dispatched boundaries exist AND set all shard ranges in the dispatched range to
-					// DONE.
+					// Ensure that the dispatched boundaries exist AND set all shard ranges in the dispatched range
+					// to DONE.
 					RangeMap<Key, int, KeyRangeRef>::Ranges shardRanges =
 					    shardMap.modify(KeyRangeRef(lastKey, boundary.first));
 					iShard = shardRanges.begin();
@@ -1625,7 +2122,7 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		// Set anything outside the backup ranges to SKIP.  We can use insert() here instead of modify()
 		// because it's OK to delete shard boundaries in the skipped ranges.
 		if (backupRanges.size() > 0) {
-			shardMap.insert(KeyRangeRef(normalKeys.begin, backupRanges.front().begin), SKIP);
+			shardMap.insert(KeyRangeRef(allKeys.begin, backupRanges.front().begin), SKIP);
 			wait(yield());
 
 			for (i = 0; i < backupRanges.size() - 1; ++i) {
@@ -1633,7 +2130,7 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 				wait(yield());
 			}
 
-			shardMap.insert(KeyRangeRef(backupRanges.back().end, normalKeys.end), SKIP);
+			shardMap.insert(KeyRangeRef(backupRanges.back().end, allKeys.end), SKIP);
 			wait(yield());
 		}
 
@@ -1654,7 +2151,7 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		}
 
 		// Coalesce the shard map to make random selection below more efficient.
-		shardMap.coalesce(normalKeys);
+		shardMap.coalesce(allKeys);
 		wait(yield());
 
 		// In this context "all" refers to all of the shards relevant for this particular backup
@@ -1686,10 +2183,10 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 			nextDispatchVersion = recentReadVersion + CLIENT_KNOBS->CORE_VERSIONSPERSECOND *
 			                                              CLIENT_KNOBS->BACKUP_SNAPSHOT_DISPATCH_INTERVAL_SEC;
 
-		// If nextDispatchVersion is greater than snapshotTargetEndVersion (which could be in the past) then just use
-		// the greater of recentReadVersion or snapshotTargetEndVersion.  Any range tasks created in this dispatch will
-		// be scheduled at a random time between recentReadVersion and nextDispatchVersion,
-		// so nextDispatchVersion shouldn't be less than recentReadVersion.
+		// If nextDispatchVersion is greater than snapshotTargetEndVersion (which could be in the past) then just
+		// use the greater of recentReadVersion or snapshotTargetEndVersion.  Any range tasks created in this
+		// dispatch will be scheduled at a random time between recentReadVersion and nextDispatchVersion, so
+		// nextDispatchVersion shouldn't be less than recentReadVersion.
 		if (nextDispatchVersion > snapshotTargetEndVersion)
 			nextDispatchVersion = std::max(recentReadVersion, snapshotTargetEndVersion);
 
@@ -1709,12 +2206,12 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		state int countShardsToDispatch = std::max<int>(0, countExpectedShardsDone - countShardsDone);
 
 		// Calculate the number of shards that would have been dispatched by a normal (on-schedule)
-		// BackupSnapshotDispatchTask given the dispatch window and the start and expected-end versions of the current
-		// snapshot.
+		// BackupSnapshotDispatchTask given the dispatch window and the start and expected-end versions of the
+		// current snapshot.
 		int64_t dispatchWindow = nextDispatchVersion - recentReadVersion;
 
-		// If the scheduled snapshot interval is 0 (such as for initial, as-fast-as-possible snapshot) then all shards
-		// are considered late
+		// If the scheduled snapshot interval is 0 (such as for initial, as-fast-as-possible snapshot) then all
+		// shards are considered late
 		int countShardsExpectedPerNormalWindow;
 		if (snapshotScheduledVersionInterval == 0) {
 			countShardsExpectedPerNormalWindow = 0;
@@ -1725,8 +2222,8 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 			    (double(dispatchWindow) / snapshotScheduledVersionInterval) * countAllShards;
 		}
 
-		// The number of shards 'behind' the snapshot is the count of how may additional shards beyond normal are being
-		// dispatched, if any.
+		// The number of shards 'behind' the snapshot is the count of how may additional shards beyond normal are
+		// being dispatched, if any.
 		int countShardsBehind =
 		    std::max<int64_t>(0, countShardsToDispatch + snapshotBatchSize.get() - countShardsExpectedPerNormalWindow);
 		Params.shardsBehind().set(task, countShardsBehind);
@@ -1804,8 +2301,8 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 					wait(store(snapshotBatchSize.get(), config.snapshotBatchSize().getOrThrow(tr)) &&
 					     waitForAll(beginReads) && waitForAll(endReads) && taskBucket->keepRunning(tr, task));
 
-					// Snapshot batch size should be either oldBatchSize or newBatchSize. If new, this transaction is
-					// already done.
+					// Snapshot batch size should be either oldBatchSize or newBatchSize. If new, this transaction
+					// is already done.
 					if (snapshotBatchSize.get() == newBatchSize) {
 						break;
 					} else {
@@ -1843,8 +2340,8 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 							}
 
 							Version scheduledVersion = invalidVersion;
-							// If the next dispatch version is in the future, choose a random version at which to start
-							// the new task.
+							// If the next dispatch version is in the future, choose a random version at which to
+							// start the new task.
 							if (nextDispatchVersion > recentReadVersion)
 								scheduledVersion = recentReadVersion + deterministicRandom()->random01() *
 								                                           (nextDispatchVersion - recentReadVersion);
@@ -1870,8 +2367,8 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 							    .detail("BeginKey", range.begin.printable())
 							    .detail("EndKey", range.end.printable());
 						} else {
-							// This shouldn't happen because if the transaction was already done or if another execution
-							// of this task is making progress it should have been detected above.
+							// This shouldn't happen because if the transaction was already done or if another
+							// execution of this task is making progress it should have been detected above.
 							ASSERT(false);
 						}
 					}
@@ -1903,9 +2400,9 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 	}
 
 	// This function is just a wrapper for BackupSnapshotManifest::addTask() which is defined below.
-	// The BackupSnapshotDispatchTask and BackupSnapshotManifest tasks reference each other so in order to keep their
-	// execute and finish phases defined together inside their class definitions this wrapper is declared here but
-	// defined after BackupSnapshotManifest is defined.
+	// The BackupSnapshotDispatchTask and BackupSnapshotManifest tasks reference each other so in order to keep
+	// their execute and finish phases defined together inside their class definitions this wrapper is declared here
+	// but defined after BackupSnapshotManifest is defined.
 	static Future<Key> addSnapshotManifestTask(Reference<ReadYourWritesTransaction> tr,
 	                                           Reference<TaskBucket> taskBucket,
 	                                           Reference<Task> parentTask,
@@ -1938,9 +2435,9 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 
 		state Reference<TaskFuture> snapshotFinishedFuture = task->getDoneFuture(futureBucket);
 
-		// If the snapshot is finished, the next task is to write a snapshot manifest, otherwise it's another snapshot
-		// dispatch task. In either case, the task should wait for snapshotBatchFuture. The snapshot done key, passed to
-		// the current task, is also passed on.
+		// If the snapshot is finished, the next task is to write a snapshot manifest, otherwise it's another
+		// snapshot dispatch task. In either case, the task should wait for snapshotBatchFuture. The snapshot done
+		// key, passed to the current task, is also passed on.
 		if (Params.snapshotFinished().getOrDefault(task, false)) {
 			wait(success(addSnapshotManifestTask(
 			    tr, taskBucket, task, TaskCompletionKey::signal(snapshotFinishedFuture), snapshotBatchFuture)));
@@ -1962,7 +2459,7 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		return Void();
 	}
 };
-StringRef BackupSnapshotDispatchTask::name = LiteralStringRef("file_backup_dispatch_ranges_5.2");
+StringRef BackupSnapshotDispatchTask::name = "file_backup_dispatch_ranges_5.2"_sr;
 REGISTER_TASKFUNC(BackupSnapshotDispatchTask);
 
 struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
@@ -1970,10 +2467,10 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 	static constexpr uint32_t version = 1;
 
 	static struct {
-		static TaskParam<bool> addBackupLogRangeTasks() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<int64_t> fileSize() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<Version> beginVersion() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<Version> endVersion() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<bool> addBackupLogRangeTasks() { return __FUNCTION__sr; }
+		static TaskParam<int64_t> fileSize() { return __FUNCTION__sr; }
+		static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
+		static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
 	} Params;
 
 	StringRef getName() const override { return name; };
@@ -2034,10 +2531,10 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 		Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
 
 		// Get the set of key ranges that hold mutations for (beginVersion, endVersion).  They will be queried in
-		// parallel below and there is a limit on how many we want to process in a single BackupLogRangeTask so if that
-		// limit is exceeded then set the addBackupLogRangeTasks boolean in Params and stop, signalling the finish()
-		// step to break up the (beginVersion, endVersion) range into smaller intervals which are then processed by
-		// individual BackupLogRangeTasks.
+		// parallel below and there is a limit on how many we want to process in a single BackupLogRangeTask so if
+		// that limit is exceeded then set the addBackupLogRangeTasks boolean in Params and stop, signalling the
+		// finish() step to break up the (beginVersion, endVersion) range into smaller intervals which are then
+		// processed by individual BackupLogRangeTasks.
 		state Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(beginVersion, endVersion, destUidValue);
 		if (ranges.size() > CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES) {
 			Params.addBackupLogRangeTasks().set(task, true);
@@ -2051,9 +2548,9 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 		state Reference<IBackupFile> outFile = wait(bc->writeLogFile(beginVersion, endVersion, blockSize));
 		state LogFileWriter logFile(outFile, blockSize);
 
-		// Query all key ranges covering (beginVersion, endVersion) in parallel, writing their results to the results
-		// promise stream as they are received.  Note that this means the records read from the results stream are not
-		// likely to be in increasing Version order.
+		// Query all key ranges covering (beginVersion, endVersion) in parallel, writing their results to the
+		// results promise stream as they are received.  Note that this means the records read from the results
+		// stream are not likely to be in increasing Version order.
 		state PromiseStream<RangeResultWithVersion> results;
 		state std::vector<Future<Void>> rc;
 
@@ -2062,7 +2559,7 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 			    readCommitted(cx, results, lock, range, Terminator::False, AccessSystemKeys::True, LockAware::True));
 		}
 
-		state Future<Void> sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr<Void> const& result) {
+		state Future<Void> sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr<Void> const& result) mutable {
 			if (result.isError())
 				results.sendError(result.getError());
 			else
@@ -2201,7 +2698,7 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 	}
 };
 
-StringRef BackupLogRangeTaskFunc::name = LiteralStringRef("file_backup_write_logs_5.2");
+StringRef BackupLogRangeTaskFunc::name = "file_backup_write_logs_5.2"_sr;
 REGISTER_TASKFUNC(BackupLogRangeTaskFunc);
 
 // This task stopped being used in 6.2, however the code remains here to handle upgrades.
@@ -2211,9 +2708,9 @@ struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
 	StringRef getName() const override { return name; };
 
 	static struct {
-		static TaskParam<Version> beginVersion() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<Version> endVersion() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<Key> destUidValue() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
+		static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
+		static TaskParam<Key> destUidValue() { return __FUNCTION__sr; }
 	} Params;
 
 	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
@@ -2232,7 +2729,8 @@ struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
 		    BackupConfig(logUid),
 		    waitFor,
 		    [=](Reference<Task> task) {
-			    Params.beginVersion().set(task, 1); // FIXME: remove in 6.X, only needed for 5.2 backward compatibility
+			    Params.beginVersion().set(task,
+			                              1); // FIXME: remove in 6.X, only needed for 5.2 backward compatibility
 			    Params.endVersion().set(task, endVersion);
 			    Params.destUidValue().set(task, destUidValue);
 		    },
@@ -2276,7 +2774,7 @@ struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef EraseLogRangeTaskFunc::name = LiteralStringRef("file_backup_erase_logs_5.2");
+StringRef EraseLogRangeTaskFunc::name = "file_backup_erase_logs_5.2"_sr;
 REGISTER_TASKFUNC(EraseLogRangeTaskFunc);
 
 struct BackupLogsDispatchTask : BackupTaskFuncBase {
@@ -2284,8 +2782,8 @@ struct BackupLogsDispatchTask : BackupTaskFuncBase {
 	static constexpr uint32_t version = 1;
 
 	static struct {
-		static TaskParam<Version> prevBeginVersion() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<Version> beginVersion() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<Version> prevBeginVersion() { return __FUNCTION__sr; }
+		static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
 	} Params;
 
 	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@@ -2354,8 +2852,8 @@ struct BackupLogsDispatchTask : BackupTaskFuncBase {
 		state int priority = latestSnapshotEndVersion.present() ? 1 : 0;
 
 		if (!partitionedLog.present() || !partitionedLog.get()) {
-			// Add the initial log range task to read/copy the mutations and the next logs dispatch task which will run
-			// after this batch is done
+			// Add the initial log range task to read/copy the mutations and the next logs dispatch task which will
+			// run after this batch is done
 			wait(success(BackupLogRangeTaskFunc::addTask(tr,
 			                                             taskBucket,
 			                                             task,
@@ -2446,7 +2944,7 @@ struct BackupLogsDispatchTask : BackupTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef BackupLogsDispatchTask::name = LiteralStringRef("file_backup_dispatch_logs_5.2");
+StringRef BackupLogsDispatchTask::name = "file_backup_dispatch_logs_5.2"_sr;
 REGISTER_TASKFUNC(BackupLogsDispatchTask);
 
 struct FileBackupFinishedTask : BackupTaskFuncBase {
@@ -2506,14 +3004,14 @@ struct FileBackupFinishedTask : BackupTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef FileBackupFinishedTask::name = LiteralStringRef("file_backup_finished_5.2");
+StringRef FileBackupFinishedTask::name = "file_backup_finished_5.2"_sr;
 REGISTER_TASKFUNC(FileBackupFinishedTask);
 
 struct BackupSnapshotManifest : BackupTaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 	static struct {
-		static TaskParam<Version> endVersion() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
 	} Params;
 
 	ACTOR static Future<Void> _execute(Database cx,
@@ -2525,8 +3023,8 @@ struct BackupSnapshotManifest : BackupTaskFuncBase {
 
 		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
-		// Read the entire range file map into memory, then walk it backwards from its last entry to produce a list of
-		// non overlapping key range files
+		// Read the entire range file map into memory, then walk it backwards from its last entry to produce a list
+		// of non overlapping key range files
 		state std::map<Key, BackupConfig::RangeSlice> localmap;
 		state Key startKey;
 		state int batchSize = BUGGIFY ? 1 : 1000000;
@@ -2543,17 +3041,17 @@ struct BackupSnapshotManifest : BackupTaskFuncBase {
 					wait(store(bc, config.backupContainer().getOrThrow(tr)));
 				}
 
-				BackupConfig::RangeFileMapT::PairsType rangeresults =
+				BackupConfig::RangeFileMapT::RangeResultType rangeresults =
 				    wait(config.snapshotRangeFileMap().getRange(tr, startKey, {}, batchSize));
 
-				for (auto& p : rangeresults) {
+				for (auto& p : rangeresults.results) {
 					localmap.insert(p);
 				}
 
-				if (rangeresults.size() < batchSize)
+				if (!rangeresults.more)
 					break;
 
-				startKey = keyAfter(rangeresults.back().first);
+				startKey = keyAfter(rangeresults.results.back().first);
 				tr->reset();
 			} catch (Error& e) {
 				wait(tr->onError(e));
@@ -2590,10 +3088,11 @@ struct BackupSnapshotManifest : BackupTaskFuncBase {
 				totalBytes += r.fileSize;
 
 				// Jump to file that either ends where this file begins or has the greatest end that is less than
-				// the begin of this file.  In other words find the map key that is <= begin of this file.  To do this
-				// find the first end strictly greater than begin and then back up one.
+				// the begin of this file.  In other words find the map key that is <= begin of this file.  To do
+				// this find the first end strictly greater than begin and then back up one.
 				i = localmap.upper_bound(i->second.begin);
-				// If we get begin then we're done, there are no more ranges that end at or before the last file's begin
+				// If we get begin then we're done, there are no more ranges that end at or before the last file's
+				// begin
 				if (i == localmap.begin())
 					break;
 				--i;
@@ -2695,7 +3194,7 @@ struct BackupSnapshotManifest : BackupTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef BackupSnapshotManifest::name = LiteralStringRef("file_backup_write_snapshot_manifest_5.2");
+StringRef BackupSnapshotManifest::name = "file_backup_write_snapshot_manifest_5.2"_sr;
 REGISTER_TASKFUNC(BackupSnapshotManifest);
 
 Future<Key> BackupSnapshotDispatchTask::addSnapshotManifestTask(Reference<ReadYourWritesTransaction> tr,
@@ -2711,7 +3210,7 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase {
 	static constexpr uint32_t version = 1;
 
 	static struct {
-		static TaskParam<Version> beginVersion() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
 	} Params;
 
 	ACTOR static Future<Void> _execute(Database cx,
@@ -2841,8 +3340,8 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase {
 		wait(success(BackupLogsDispatchTask::addTask(
 		    tr, taskBucket, task, 1, 0, beginVersion, TaskCompletionKey::joinWith(backupFinished))));
 
-		// If a clean stop is requested, the log and snapshot tasks will quit after the backup is restorable, then the
-		// following task will clean up and set the completed state.
+		// If a clean stop is requested, the log and snapshot tasks will quit after the backup is restorable, then
+		// the following task will clean up and set the completed state.
 		wait(success(
 		    FileBackupFinishedTask::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished)));
 
@@ -2881,7 +3380,7 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef StartFullBackupTaskFunc::name = LiteralStringRef("file_backup_start_5.2");
+StringRef StartFullBackupTaskFunc::name = "file_backup_start_5.2"_sr;
 REGISTER_TASKFUNC(StartFullBackupTaskFunc);
 
 struct RestoreCompleteTaskFunc : RestoreTaskFuncBase {
@@ -2897,13 +3396,14 @@ struct RestoreCompleteTaskFunc : RestoreTaskFuncBase {
 		// Clear the file map now since it could be huge.
 		restore.fileSet().clear(tr);
 
-		// TODO:  Validate that the range version map has exactly the restored ranges in it.  This means that for any
-		// restore operation the ranges to restore must be within the backed up ranges, otherwise from the restore
-		// perspective it will appear that some key ranges were missing and so the backup set is incomplete and the
-		// restore has failed. This validation cannot be done currently because Restore only supports a single restore
-		// range but backups can have many ranges.
+		// TODO:  Validate that the range version map has exactly the restored ranges in it.  This means that for
+		// any restore operation the ranges to restore must be within the backed up ranges, otherwise from the
+		// restore perspective it will appear that some key ranges were missing and so the backup set is incomplete
+		// and the restore has failed. This validation cannot be done currently because Restore only supports a
+		// single restore range but backups can have many ranges.
 
-		// Clear the applyMutations stuff, including any unapplied mutations from versions beyond the restored version.
+		// Clear the applyMutations stuff, including any unapplied mutations from versions beyond the restored
+		// version.
 		restore.clearApplyMutationsKeys(tr);
 
 		wait(taskBucket->finish(tr, task));
@@ -2928,7 +3428,7 @@ struct RestoreCompleteTaskFunc : RestoreTaskFuncBase {
 		}
 
 		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	static StringRef name;
@@ -2948,14 +3448,14 @@ struct RestoreCompleteTaskFunc : RestoreTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef RestoreCompleteTaskFunc::name = LiteralStringRef("restore_complete");
+StringRef RestoreCompleteTaskFunc::name = "restore_complete"_sr;
 REGISTER_TASKFUNC(RestoreCompleteTaskFunc);
 
 struct RestoreFileTaskFuncBase : RestoreTaskFuncBase {
 	struct InputParams {
-		static TaskParam<RestoreFile> inputFile() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<int64_t> readOffset() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<int64_t> readLen() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<RestoreFile> inputFile() { return __FUNCTION__sr; }
+		static TaskParam<int64_t> readOffset() { return __FUNCTION__sr; }
+		static TaskParam<int64_t> readLen() { return __FUNCTION__sr; }
 	} Params;
 
 	std::string toString(Reference<Task> task) const override {
@@ -2970,8 +3470,8 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 	static struct : InputParams {
 		// The range of data that the (possibly empty) data represented, which is set if it intersects the target
 		// restore range
-		static TaskParam<KeyRange> originalFileRange() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<std::vector<KeyRange>> originalFileRanges() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<KeyRange> originalFileRange() { return __FUNCTION__sr; }
+		static TaskParam<std::vector<KeyRange>> originalFileRanges() { return __FUNCTION__sr; }
 
 		static std::vector<KeyRange> getOriginalFileRanges(Reference<Task> task) {
 			if (originalFileRanges().exists(task)) {
@@ -3040,7 +3540,8 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 		}
 
 		state Reference<IAsyncFile> inFile = wait(bc.get()->readFile(rangeFile.fileName));
-		state Standalone<VectorRef<KeyValueRef>> blockData = wait(decodeRangeFileBlock(inFile, readOffset, readLen));
+		state Standalone<VectorRef<KeyValueRef>> blockData =
+		    wait(decodeRangeFileBlock(inFile, readOffset, readLen, cx));
 
 		// First and last key are the range for this file
 		state KeyRange fileRange = KeyRangeRef(blockData.front().key, blockData.back().key);
@@ -3067,16 +3568,16 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 			state VectorRef<KeyValueRef> data = blockData.slice(rangeStart, rangeEnd);
 
 			// Shrink file range to be entirely within restoreRange and translate it to the new prefix
-			// First, use the untranslated file range to create the shrunk original file range which must be used in the
-			// kv range version map for applying mutations
+			// First, use the untranslated file range to create the shrunk original file range which must be used in
+			// the kv range version map for applying mutations
 			state KeyRange originalFileRange =
 			    KeyRangeRef(std::max(fileRange.begin, restoreRange.begin), std::min(fileRange.end, restoreRange.end));
 			originalFileRanges.push_back(originalFileRange);
 
 			// Now shrink and translate fileRange
 			Key fileEnd = std::min(fileRange.end, restoreRange.end);
-			if (fileEnd == (removePrefix.get() == StringRef() ? normalKeys.end : strinc(removePrefix.get()))) {
-				fileEnd = addPrefix.get() == StringRef() ? normalKeys.end : strinc(addPrefix.get());
+			if (fileEnd == (removePrefix.get() == StringRef() ? allKeys.end : strinc(removePrefix.get()))) {
+				fileEnd = addPrefix.get() == StringRef() ? allKeys.end : strinc(addPrefix.get());
 			} else {
 				fileEnd = fileEnd.removePrefix(removePrefix.get()).withPrefix(addPrefix.get());
 			}
@@ -3114,7 +3615,6 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 					                 : data[start].key.removePrefix(removePrefix.get()).withPrefix(addPrefix.get()),
 					    (iend == end) ? fileRange.end
 					                  : data[iend].key.removePrefix(removePrefix.get()).withPrefix(addPrefix.get()));
-
 					tr->clear(trRange);
 
 					for (; i < iend; ++i) {
@@ -3220,7 +3720,7 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 		}
 
 		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	static StringRef name;
@@ -3240,7 +3740,7 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef RestoreRangeTaskFunc::name = LiteralStringRef("restore_range_data");
+StringRef RestoreRangeTaskFunc::name = "restore_range_data"_sr;
 REGISTER_TASKFUNC(RestoreRangeTaskFunc);
 
 // Decodes a mutation log key, which contains (hash, commitVersion, chunkNumber) and
@@ -3335,6 +3835,14 @@ bool AccumulatedMutations::matchesAnyRange(const std::vector<KeyRange>& ranges)
 	std::vector<MutationRef> mutations = decodeMutationLogValue(serializedMutations);
 	for (auto& m : mutations) {
 		for (auto& r : ranges) {
+			if (m.type == MutationRef::Encrypted) {
+				// TODO:  In order to filter out encrypted mutations that are not relevant to the
+				// target range, they would have to be decrypted here in order to check relevance
+				// below, however the staged mutations would still need to remain encrypted for
+				// staging into the destination database.  Without decrypting, we must assume that
+				// some data could match the range and return true here.
+				return true;
+			}
 			if (m.type == MutationRef::ClearRange) {
 				if (r.intersects(KeyRangeRef(m.param1, m.param2))) {
 					return true;
@@ -3430,8 +3938,8 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 		state Standalone<VectorRef<KeyValueRef>> dataOriginal =
 		    wait(decodeMutationLogFileBlock(inFile, readOffset, readLen));
 
-		// Filter the KV pairs extracted from the log file block to remove any records known to not be needed for this
-		// restore based on the restore range set.
+		// Filter the KV pairs extracted from the log file block to remove any records known to not be needed for
+		// this restore based on the restore range set.
 		state std::vector<KeyValueRef> dataFiltered = filterLogMutationKVPairs(dataOriginal, ranges);
 
 		state int start = 0;
@@ -3505,8 +4013,8 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 
 		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
-		// TODO:  Check to see if there is a leak in the FutureBucket since an invalid task (validation key fails) will
-		// never set its taskFuture.
+		// TODO:  Check to see if there is a leak in the FutureBucket since an invalid task (validation key fails)
+		// will never set its taskFuture.
 		wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
 
 		return Void();
@@ -3534,7 +4042,7 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 		}
 
 		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	Future<Void> execute(Database cx,
@@ -3550,7 +4058,7 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef RestoreLogDataTaskFunc::name = LiteralStringRef("restore_log_data");
+StringRef RestoreLogDataTaskFunc::name = "restore_log_data"_sr;
 REGISTER_TASKFUNC(RestoreLogDataTaskFunc);
 
 struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
@@ -3559,11 +4067,11 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 	StringRef getName() const override { return name; };
 
 	static struct {
-		static TaskParam<Version> beginVersion() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<std::string> beginFile() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<int64_t> beginBlock() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<int64_t> batchSize() { return LiteralStringRef(__FUNCTION__); }
-		static TaskParam<int64_t> remainingInBatch() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
+		static TaskParam<std::string> beginFile() { return __FUNCTION__sr; }
+		static TaskParam<int64_t> beginBlock() { return __FUNCTION__sr; }
+		static TaskParam<int64_t> batchSize() { return __FUNCTION__sr; }
+		static TaskParam<int64_t> remainingInBatch() { return __FUNCTION__sr; }
 	} Params;
 
 	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@@ -3614,10 +4122,10 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		}
 
 		state std::string beginFile = Params.beginFile().getOrDefault(task);
-		// Get a batch of files.  We're targeting batchSize blocks being dispatched so query for batchSize files (each
-		// of which is 0 or more blocks).
+		// Get a batch of files.  We're targeting batchSize blocks being dispatched so query for batchSize files
+		// (each of which is 0 or more blocks).
 		state int taskBatchSize = BUGGIFY ? 1 : CLIENT_KNOBS->RESTORE_DISPATCH_ADDTASK_SIZE;
-		state RestoreConfig::FileSetT::Values files = wait(restore.fileSet().getRange(
+		state RestoreConfig::FileSetT::RangeResultType files = wait(restore.fileSet().getRange(
 		    tr, Optional<RestoreConfig::RestoreFile>({ beginVersion, beginFile }), {}, taskBatchSize));
 
 		// allPartsDone will be set once all block tasks in the current batch are finished.
@@ -3636,9 +4144,9 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		}
 
 		// If there were no files to load then this batch is done and restore is almost done.
-		if (files.size() == 0) {
-			// If adding to existing batch then blocks could be in progress so create a new Dispatch task that waits for
-			// them to finish
+		if (files.results.size() == 0) {
+			// If adding to existing batch then blocks could be in progress so create a new Dispatch task that waits
+			// for them to finish
 			if (addingToExistingBatch) {
 				// Setting next begin to restoreVersion + 1 so that any files in the file map at the restore version
 				// won't be dispatched again.
@@ -3688,8 +4196,8 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 				    .detail("Decision", "restore_complete")
 				    .detail("TaskInstance", THIS_ADDR);
 			} else {
-				// Applying of mutations is not yet finished so wait a small amount of time and then re-add this same
-				// task.
+				// Applying of mutations is not yet finished so wait a small amount of time and then re-add this
+				// same task.
 				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
 				wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, beginVersion, "", 0, batchSize)));
 
@@ -3714,17 +4222,17 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		// blocks per Dispatch task and target batchSize total per batch but a batch must end on a complete version
 		// boundary so exceed the limit if necessary to reach the end of a version of files.
 		state std::vector<Future<Key>> addTaskFutures;
-		state Version endVersion = files[0].version;
+		state Version endVersion = files.results[0].version;
 		state int blocksDispatched = 0;
 		state int64_t beginBlock = Params.beginBlock().getOrDefault(task);
 		state int i = 0;
 
-		for (; i < files.size(); ++i) {
-			RestoreConfig::RestoreFile& f = files[i];
+		for (; i < files.results.size(); ++i) {
+			RestoreConfig::RestoreFile& f = files.results[i];
 
-			// Here we are "between versions" (prior to adding the first block of the first file of a new version) so
-			// this is an opportunity to end the current dispatch batch (which must end on a version boundary) if the
-			// batch size has been reached or exceeded
+			// Here we are "between versions" (prior to adding the first block of the first file of a new version)
+			// so this is an opportunity to end the current dispatch batch (which must end on a version boundary) if
+			// the batch size has been reached or exceeded
 			if (f.version != endVersion && remainingInBatch <= 0) {
 				// Next start will be at the first version after endVersion at the first file first block
 				++endVersion;
@@ -3786,13 +4294,13 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 			    .detail("TaskInstance", THIS_ADDR);
 		}
 
-		// If no blocks were dispatched then the next dispatch task should run now and be joined with the allPartsDone
-		// future
+		// If no blocks were dispatched then the next dispatch task should run now and be joined with the
+		// allPartsDone future
 		if (blocksDispatched == 0) {
 			std::string decision;
 
-			// If no files were dispatched either then the batch size wasn't large enough to catch all of the files at
-			// the next lowest non-dispatched version, so increase the batch size.
+			// If no files were dispatched either then the batch size wasn't large enough to catch all of the files
+			// at the next lowest non-dispatched version, so increase the batch size.
 			if (i == 0) {
 				batchSize *= 2;
 				decision = "increased_batch_size";
@@ -3835,13 +4343,13 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		restore.filesBlocksDispatched().atomicOp(tr, blocksDispatched, MutationRef::Type::AddValue);
 
 		// If beginFile is not empty then we had to stop in the middle of a version (possibly within a file) so we
-		// cannot end the batch here because we do not know if we got all of the files and blocks from the last version
-		// queued, so make sure remainingInBatch is at least 1.
+		// cannot end the batch here because we do not know if we got all of the files and blocks from the last
+		// version queued, so make sure remainingInBatch is at least 1.
 		if (!beginFile.empty())
 			remainingInBatch = std::max<int64_t>(1, remainingInBatch);
 
-		// If more blocks need to be dispatched in this batch then add a follow-on task that is part of the allPartsDone
-		// group which will won't wait to run and will add more block tasks.
+		// If more blocks need to be dispatched in this batch then add a follow-on task that is part of the
+		// allPartsDone group which will won't wait to run and will add more block tasks.
 		if (remainingInBatch > 0)
 			addTaskFutures.push_back(RestoreDispatchTaskFunc::addTask(tr,
 			                                                          taskBucket,
@@ -3918,7 +4426,7 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		}
 
 		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	Future<Void> execute(Database cx,
@@ -3934,7 +4442,7 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef RestoreDispatchTaskFunc::name = LiteralStringRef("restore_dispatch");
+StringRef RestoreDispatchTaskFunc::name = "restore_dispatch"_sr;
 REGISTER_TASKFUNC(RestoreDispatchTaskFunc);
 
 ACTOR Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName) {
@@ -4032,7 +4540,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 	static constexpr uint32_t version = 1;
 
 	static struct {
-		static TaskParam<Version> firstVersion() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<Version> firstVersion() { return __FUNCTION__sr; }
 	} Params;
 
 	// Find all files needed for the restore and save them in the RestoreConfig for the task.
@@ -4098,7 +4606,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 				    .detail("RestoreVersion", restoreVersion)
 				    .detail("Dest", destVersion);
 				if (destVersion <= restoreVersion) {
-					TEST(true); // Forcing restored cluster to higher version
+					CODE_PROBE(true, "Forcing restored cluster to higher version");
 					tr->set(minRequiredCommitVersionKey, BinaryWriter::toValue(restoreVersion + 1, Unversioned()));
 					wait(tr->commit());
 				} else {
@@ -4118,7 +4626,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 			keyRangesFilter.push_back_deep(keyRangesFilter.arena(), KeyRangeRef(r));
 		}
 		state Optional<RestorableFileSet> restorable =
-		    wait(bc->getRestoreSet(restoreVersion, keyRangesFilter, logsOnly, beginVersion));
+		    wait(bc->getRestoreSet(restoreVersion, cx, keyRangesFilter, logsOnly, beginVersion));
 		if (!restorable.present())
 			throw restore_missing_data();
 
@@ -4130,8 +4638,8 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 			if (!inconsistentSnapshotOnly) {
 				for (const RangeFile& f : restorable.get().ranges) {
 					files.push_back({ f.version, f.fileName, true, f.blockSize, f.fileSize });
-					// In a restore with both snapshots and logs, the firstConsistentVersion is the highest version of
-					// any range file.
+					// In a restore with both snapshots and logs, the firstConsistentVersion is the highest version
+					// of any range file.
 					firstConsistentVersion = std::max(firstConsistentVersion, f.version);
 				}
 			} else {
@@ -4250,7 +4758,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 			// If this is an incremental restore, we need to set the applyMutationsMapPrefix
 			// to the earliest log version so no mutations are missed
 			Value versionEncoded = BinaryWriter::toValue(Params.firstVersion().get(task), Unversioned());
-			wait(krmSetRange(tr, restore.applyMutationsMapPrefix(), normalKeys, versionEncoded));
+			wait(krmSetRange(tr, restore.applyMutationsMapPrefix(), allKeys, versionEncoded));
 		}
 		return Void();
 	}
@@ -4276,7 +4784,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 		}
 
 		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return LiteralStringRef("OnSetAddTask");
+		return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -4294,7 +4802,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 };
-StringRef StartFullRestoreTaskFunc::name = LiteralStringRef("restore_start");
+StringRef StartFullRestoreTaskFunc::name = "restore_start"_sr;
 REGISTER_TASKFUNC(StartFullRestoreTaskFunc);
 } // namespace fileBackup
 
@@ -4380,7 +4888,7 @@ public:
 			    .detail("OverrideTargetVersion", targetVersion);
 		}
 
-		Optional<RestorableFileSet> restoreSet = wait(bc->getRestoreSet(targetVersion));
+		Optional<RestorableFileSet> restoreSet = wait(bc->getRestoreSet(targetVersion, cx));
 
 		if (!restoreSet.present()) {
 			TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
@@ -4517,6 +5025,7 @@ public:
 	                                       int snapshotIntervalSeconds,
 	                                       std::string tagName,
 	                                       Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                                       bool encryptionEnabled,
 	                                       StopWhenDone stopWhenDone,
 	                                       UsePartitionedLog partitionedLog,
 	                                       IncrementalBackupOnly incrementalBackupOnly,
@@ -4595,24 +5104,28 @@ public:
 		config.clear(tr);
 
 		state Key destUidValue(BinaryWriter::toValue(uid, Unversioned()));
-		if (normalizedRanges.size() == 1) {
+		if (normalizedRanges.size() == 1 || isDefaultBackup(normalizedRanges)) {
 			RangeResult existingDestUidValues = wait(
 			    tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
 			bool found = false;
+			KeyRangeRef targetRange =
+			    normalizedRanges.size() == 1 ? normalizedRanges[0] : getDefaultBackupSharedRange();
 			for (auto it : existingDestUidValues) {
-				if (BinaryReader::fromStringRef<KeyRange>(it.key.removePrefix(destUidLookupPrefix), IncludeVersion()) ==
-				    normalizedRanges[0]) {
+				KeyRange uidRange =
+				    BinaryReader::fromStringRef<KeyRange>(it.key.removePrefix(destUidLookupPrefix), IncludeVersion());
+				if (uidRange == targetRange) {
 					destUidValue = it.value;
 					found = true;
+					CODE_PROBE(targetRange == getDefaultBackupSharedRange(),
+					           "Backup mutation sharing with default backup");
 					break;
 				}
 			}
 			if (!found) {
 				destUidValue = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
-				tr->set(
-				    BinaryWriter::toValue(normalizedRanges[0], IncludeVersion(ProtocolVersion::withSharedMutations()))
-				        .withPrefix(destUidLookupPrefix),
-				    destUidValue);
+				tr->set(BinaryWriter::toValue(targetRange, IncludeVersion(ProtocolVersion::withSharedMutations()))
+				            .withPrefix(destUidLookupPrefix),
+				        destUidValue);
 			}
 		}
 
@@ -4635,6 +5148,7 @@ public:
 		config.snapshotIntervalSeconds().set(tr, snapshotIntervalSeconds);
 		config.partitionedLogEnabled().set(tr, partitionedLog);
 		config.incrementalBackupOnly().set(tr, incrementalBackupOnly);
+		config.enableSnapshotBackupEncryption().set(tr, encryptionEnabled);
 
 		Key taskKey = wait(fileBackup::StartFullBackupTaskFunc::addTask(
 		    tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal()));
@@ -4887,7 +5401,7 @@ public:
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
 			try {
-				tr->set(backupPausedKey, pause ? LiteralStringRef("1") : LiteralStringRef("0"));
+				tr->set(backupPausedKey, pause ? "1"_sr : "0"_sr);
 				wait(tr->commit());
 				break;
 			} catch (Error& e) {
@@ -5057,11 +5571,11 @@ public:
 						doc.setKey("CurrentSnapshot", snapshot);
 					}
 
-					KeyBackedMap<int64_t, std::pair<std::string, Version>>::PairsType errors =
+					KeyBackedMap<int64_t, std::pair<std::string, Version>>::RangeResultType errors =
 					    wait(config.lastErrorPerType().getRange(
 					        tr, 0, std::numeric_limits<int>::max(), CLIENT_KNOBS->TOO_MANY));
 					JsonBuilderArray errorList;
-					for (auto& e : errors) {
+					for (auto& e : errors.results) {
 						std::string msg = e.second.first;
 						Version ver = e.second.second;
 
@@ -5209,13 +5723,13 @@ public:
 
 					// Append the errors, if requested
 					if (showErrors) {
-						KeyBackedMap<int64_t, std::pair<std::string, Version>>::PairsType errors =
+						KeyBackedMap<int64_t, std::pair<std::string, Version>>::RangeResultType errors =
 						    wait(config.lastErrorPerType().getRange(
 						        tr, 0, std::numeric_limits<int>::max(), CLIENT_KNOBS->TOO_MANY));
 						std::string recentErrors;
 						std::string pastErrors;
 
-						for (auto& e : errors) {
+						for (auto& e : errors.results) {
 							Version v = e.second.second;
 							std::string msg = format(
 							    "%s ago : %s\n",
@@ -5342,7 +5856,7 @@ public:
 		}
 
 		Optional<RestorableFileSet> restoreSet =
-		    wait(bc->getRestoreSet(targetVersion, ranges, onlyApplyMutationLogs, beginVersion));
+		    wait(bc->getRestoreSet(targetVersion, cx, ranges, onlyApplyMutationLogs, beginVersion));
 
 		if (!restoreSet.present()) {
 			TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
@@ -5624,6 +6138,60 @@ Future<Version> FileBackupAgent::restore(Database cx,
 	                                    deterministicRandom()->randomUniqueID());
 }
 
+Future<Version> FileBackupAgent::restore(Database cx,
+                                         Optional<Database> cxOrig,
+                                         Key tagName,
+                                         Key url,
+                                         Optional<std::string> proxy,
+                                         WaitForComplete waitForComplete,
+                                         Version targetVersion,
+                                         Verbose verbose,
+                                         KeyRange range,
+                                         Key addPrefix,
+                                         Key removePrefix,
+                                         LockDB lockDB,
+                                         OnlyApplyMutationLogs onlyApplyMutationLogs,
+                                         InconsistentSnapshotOnly inconsistentSnapshotOnly,
+                                         Version beginVersion,
+                                         Optional<std::string> const& encryptionKeyFileName) {
+	Standalone<VectorRef<KeyRangeRef>> rangeRef;
+	if (range.begin.empty() && range.end.empty()) {
+		addDefaultBackupRanges(rangeRef);
+	} else {
+		rangeRef.push_back_deep(rangeRef.arena(), range);
+	}
+	return restore(cx,
+	               cxOrig,
+	               tagName,
+	               url,
+	               proxy,
+	               rangeRef,
+	               waitForComplete,
+	               targetVersion,
+	               verbose,
+	               addPrefix,
+	               removePrefix,
+	               lockDB,
+	               onlyApplyMutationLogs,
+	               inconsistentSnapshotOnly,
+	               beginVersion,
+	               encryptionKeyFileName);
+}
+
+Future<Version> FileBackupAgent::atomicRestore(Database cx,
+                                               Key tagName,
+                                               KeyRange range,
+                                               Key addPrefix,
+                                               Key removePrefix) {
+	Standalone<VectorRef<KeyRangeRef>> rangeRef;
+	if (range.begin.empty() && range.end.empty()) {
+		addDefaultBackupRanges(rangeRef);
+	} else {
+		rangeRef.push_back_deep(rangeRef.arena(), range);
+	}
+	return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
+}
+
 Future<Version> FileBackupAgent::atomicRestore(Database cx,
                                                Key tagName,
                                                Standalone<VectorRef<KeyRangeRef>> ranges,
@@ -5656,6 +6224,7 @@ Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction>
                                            int snapshotIntervalSeconds,
                                            std::string const& tagName,
                                            Standalone<VectorRef<KeyRangeRef>> backupRanges,
+                                           bool encryptionEnabled,
                                            StopWhenDone stopWhenDone,
                                            UsePartitionedLog partitionedLog,
                                            IncrementalBackupOnly incrementalBackupOnly,
@@ -5668,6 +6237,7 @@ Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction>
 	                                         snapshotIntervalSeconds,
 	                                         tagName,
 	                                         backupRanges,
+	                                         encryptionEnabled,
 	                                         stopWhenDone,
 	                                         partitionedLog,
 	                                         incrementalBackupOnly,
@@ -5768,7 +6338,7 @@ ACTOR static Future<Void> writeKVs(Database cx, Standalone<VectorRef<KeyValueRef
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			KeyRef k1 = kvs[begin].key;
-			KeyRef k2 = end < kvs.size() ? kvs[end].key : normalKeys.end;
+			KeyRef k2 = end < kvs.size() ? kvs[end].key : allKeys.end;
 			TraceEvent(SevFRTestInfo, "TransformDatabaseContentsWriteKVReadBack")
 			    .detail("Range", KeyRangeRef(k1, k2))
 			    .detail("Begin", begin)
diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp
index c345e3669e..45db5aa45b 100644
--- a/fdbclient/GlobalConfig.actor.cpp
+++ b/fdbclient/GlobalConfig.actor.cpp
@@ -28,14 +28,14 @@
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-const KeyRef fdbClientInfoTxnSampleRate = LiteralStringRef("config/fdb_client_info/client_txn_sample_rate");
-const KeyRef fdbClientInfoTxnSizeLimit = LiteralStringRef("config/fdb_client_info/client_txn_size_limit");
+const KeyRef fdbClientInfoTxnSampleRate = "config/fdb_client_info/client_txn_sample_rate"_sr;
+const KeyRef fdbClientInfoTxnSizeLimit = "config/fdb_client_info/client_txn_size_limit"_sr;
 
-const KeyRef transactionTagSampleRate = LiteralStringRef("config/transaction_tag_sample_rate");
-const KeyRef transactionTagSampleCost = LiteralStringRef("config/transaction_tag_sample_cost");
+const KeyRef transactionTagSampleRate = "config/transaction_tag_sample_rate"_sr;
+const KeyRef transactionTagSampleCost = "config/transaction_tag_sample_cost"_sr;
 
-const KeyRef samplingFrequency = LiteralStringRef("visibility/sampling/frequency");
-const KeyRef samplingWindow = LiteralStringRef("visibility/sampling/window");
+const KeyRef samplingFrequency = "visibility/sampling/frequency"_sr;
+const KeyRef samplingWindow = "visibility/sampling/window"_sr;
 
 GlobalConfig::GlobalConfig(DatabaseContext* cx) : cx(cx), lastUpdate(0) {}
 
@@ -62,7 +62,7 @@ void GlobalConfig::applyChanges(Transaction& tr,
 
 	// Write version key to trigger update in cluster controller.
 	tr.atomicOp(globalConfigVersionKey,
-	            LiteralStringRef("0123456789\x00\x00\x00\x00"), // versionstamp
+	            "0123456789\x00\x00\x00\x00"_sr, // versionstamp
 	            MutationRef::SetVersionstampedValue);
 }
 
@@ -101,7 +101,7 @@ void GlobalConfig::trigger(KeyRef key, std::function<void(std::optional<std::any
 }
 
 void GlobalConfig::insert(KeyRef key, ValueRef value) {
-	// TraceEvent(SevInfo, "GlobalConfig_Insert").detail("Key", key).detail("Value", value);
+	// TraceEvent(SevInfo, "GlobalConfigInsert").detail("Key", key).detail("Value", value);
 	data.erase(key);
 
 	Arena arena(key.expectedSize() + value.expectedSize());
@@ -139,7 +139,7 @@ void GlobalConfig::erase(Key key) {
 }
 
 void GlobalConfig::erase(KeyRangeRef range) {
-	// TraceEvent(SevInfo, "GlobalConfig_Erase").detail("Range", range);
+	// TraceEvent(SevInfo, "GlobalConfigErase").detail("Range", range);
 	auto it = data.begin();
 	while (it != data.end()) {
 		if (range.contains(it->first)) {
@@ -153,95 +153,29 @@ void GlobalConfig::erase(KeyRangeRef range) {
 	}
 }
 
-// Older FDB versions used different keys for client profiling data. This
-// function performs a one-time migration of data in these keys to the new
-// global configuration key space.
-ACTOR Future<Void> GlobalConfig::migrate(GlobalConfig* self) {
-	state Key migratedKey("\xff\x02/fdbClientInfo/migrated/"_sr);
-	state Reference<ReadYourWritesTransaction> tr;
-	try {
-		state Backoff backoff;
-		loop {
-			tr = makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(self->cx)));
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-
-			try {
-				state Optional<Value> migrated = wait(tr->get(migratedKey));
-				if (migrated.present()) {
-					// Already performed migration.
-					return Void();
-				}
-
-				state Optional<Value> sampleRate =
-				    wait(tr->get(Key("\xff\x02/fdbClientInfo/client_txn_sample_rate/"_sr)));
-				state Optional<Value> sizeLimit =
-				    wait(tr->get(Key("\xff\x02/fdbClientInfo/client_txn_size_limit/"_sr)));
-
-				tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-				// The value doesn't matter too much, as long as the key is set.
-				tr->set(migratedKey.contents(), "1"_sr);
-				if (sampleRate.present()) {
-					const double sampleRateDbl =
-					    BinaryReader::fromStringRef<double>(sampleRate.get().contents(), Unversioned());
-					Tuple rate = Tuple().appendDouble(sampleRateDbl);
-					tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack());
-				}
-				if (sizeLimit.present()) {
-					const int64_t sizeLimitInt =
-					    BinaryReader::fromStringRef<int64_t>(sizeLimit.get().contents(), Unversioned());
-					Tuple size = Tuple().append(sizeLimitInt);
-					tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack());
-				}
-
-				wait(tr->commit());
-				break;
-			} catch (Error& e) {
-				// If multiple fdbserver processes are started at once, they will all
-				// attempt this migration at the same time, sometimes resulting in
-				// aborts due to conflicts. Purposefully avoid retrying, making this
-				// migration best-effort.
-				TraceEvent(SevInfo, "GlobalConfig_RetryableMigrationError").errorUnsuppressed(e).suppressFor(1.0);
-				wait(tr->onError(e));
-				tr.clear();
-				// tr is cleared, so it won't backoff properly. Use custom backoff logic here.
-				wait(backoff.onError());
-			}
-		}
-	} catch (Error& e) {
-		// Catch non-retryable errors (and do nothing).
-		TraceEvent(SevWarnAlways, "GlobalConfig_MigrationError").error(e);
-	}
-	return Void();
-}
-
 // Updates local copy of global configuration by reading the entire key-range
-// from storage.
-ACTOR Future<Void> GlobalConfig::refresh(GlobalConfig* self) {
-	// TraceEvent trace(SevInfo, "GlobalConfig_Refresh");
+// from storage (proxied through the GrvProxies).
+ACTOR Future<Void> GlobalConfig::refresh(GlobalConfig* self, Version lastKnown) {
+	// TraceEvent trace(SevInfo, "GlobalConfigRefresh");
 	self->erase(KeyRangeRef(""_sr, "\xff"_sr));
 
-	state Backoff backoff;
-
-	state Reference<ReadYourWritesTransaction> tr;
+	state Backoff backoff(CLIENT_KNOBS->GLOBAL_CONFIG_REFRESH_BACKOFF, CLIENT_KNOBS->GLOBAL_CONFIG_REFRESH_MAX_BACKOFF);
 	loop {
 		try {
-			tr = makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(self->cx)));
-			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			RangeResult result = wait(tr->getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY));
-			for (const auto& kv : result) {
+			GlobalConfigRefreshReply reply =
+			    wait(timeoutError(basicLoadBalance(self->cx->getGrvProxies(UseProvisionalProxies::False),
+			                                       &GrvProxyInterface::refreshGlobalConfig,
+			                                       GlobalConfigRefreshRequest{ lastKnown }),
+			                      CLIENT_KNOBS->GLOBAL_CONFIG_REFRESH_TIMEOUT));
+			for (const auto& kv : reply.result) {
 				KeyRef systemKey = kv.key.removePrefix(globalConfigKeysPrefix);
 				self->insert(systemKey, kv.value);
 			}
-			break;
+			return Void();
 		} catch (Error& e) {
-			TraceEvent("GlobalConfigRefreshError").errorUnsuppressed(e).suppressFor(1.0);
-			wait(tr->onError(e));
-			tr.clear();
-			// tr is cleared, so it won't backoff properly. Use custom backoff logic here.
 			wait(backoff.onError());
 		}
 	}
-	return Void();
 }
 
 // Applies updates to the local copy of the global configuration when this
@@ -251,9 +185,8 @@ ACTOR Future<Void> GlobalConfig::updater(GlobalConfig* self, const ClientDBInfo*
 		try {
 			if (self->initialized.canBeSet()) {
 				wait(self->cx->onConnected());
-				wait(self->migrate(self));
 
-				wait(self->refresh(self));
+				wait(self->refresh(self, -1));
 				self->initialized.send(Void());
 			}
 
@@ -270,7 +203,7 @@ ACTOR Future<Void> GlobalConfig::updater(GlobalConfig* self, const ClientDBInfo*
 						// This process missed too many global configuration
 						// history updates or the protocol version changed, so it
 						// must re-read the entire configuration range.
-						wait(self->refresh(self));
+						wait(self->refresh(self, history.back().version));
 						if (dbInfo->history.size() > 0) {
 							self->lastUpdate = dbInfo->history.back().version;
 						}
diff --git a/fdbclient/KeyRangeMap.actor.cpp b/fdbclient/KeyRangeMap.actor.cpp
index c736c714bf..a678c28e4a 100644
--- a/fdbclient/KeyRangeMap.actor.cpp
+++ b/fdbclient/KeyRangeMap.actor.cpp
@@ -23,6 +23,7 @@
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/ReadYourWrites.h"
+#include "flow/UnitTest.h"
 #include "flow/actorcompiler.h" // has to be last include
 
 void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, std::vector<KeyRange>& affectedRanges) {
@@ -35,32 +36,54 @@ void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, std
 		affectedRanges.push_back(KeyRangeRef(keys.end, e.end()));
 }
 
-RangeResult krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, RangeResult kv) {
+RangeResult krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, RangeResult kv, bool align) {
 	ASSERT(!kv.more || kv.size() > 1);
 	KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
 
-	ValueRef beginValue, endValue;
-	if (kv.size() && kv[0].key.startsWith(mapPrefix))
-		beginValue = kv[0].value;
-	if (kv.size() && kv.end()[-1].key.startsWith(mapPrefix))
-		endValue = kv.end()[-1].value;
-
 	RangeResult result;
 	result.arena().dependsOn(kv.arena());
 	result.arena().dependsOn(keys.arena());
 
-	result.push_back(result.arena(), KeyValueRef(keys.begin, beginValue));
+	// Always push a kv pair <= keys.begin.
+	KeyRef beginKey = keys.begin;
+	if (!align && !kv.empty() && kv.front().key.startsWith(mapPrefix) && kv.front().key < withPrefix.begin) {
+		beginKey = kv[0].key.removePrefix(mapPrefix);
+	}
+	ValueRef beginValue;
+	if (!kv.empty() && kv.front().key.startsWith(mapPrefix) && kv.front().key <= withPrefix.begin) {
+		beginValue = kv.front().value;
+	}
+	result.push_back(result.arena(), KeyValueRef(beginKey, beginValue));
+
 	for (int i = 0; i < kv.size(); i++) {
 		if (kv[i].key > withPrefix.begin && kv[i].key < withPrefix.end) {
 			KeyRef k = kv[i].key.removePrefix(mapPrefix);
 			result.push_back(result.arena(), KeyValueRef(k, kv[i].value));
-		} else if (kv[i].key >= withPrefix.end)
+		} else if (kv[i].key >= withPrefix.end) {
 			kv.more = false;
+			// There should be at most 1 value past mapPrefix + keys.end.
+			ASSERT(i == kv.size() - 1);
+			break;
+		}
 	}
 
-	if (!kv.more)
-		result.push_back(result.arena(), KeyValueRef(keys.end, endValue));
+	if (!kv.more) {
+		KeyRef endKey = keys.end;
+		if (!align && !kv.empty() && kv.back().key.startsWith(mapPrefix) && kv.back().key >= withPrefix.end) {
+			endKey = kv.back().key.removePrefix(mapPrefix);
+		}
+		ValueRef endValue;
+		if (!kv.empty()) {
+			// In the aligned case, carry the last value to be the end value.
+			if (align && kv.back().key.startsWith(mapPrefix) && kv.back().key > withPrefix.end) {
+				endValue = result.back().value;
+			} else {
+				endValue = kv.back().value;
+			}
+		}
+		result.push_back(result.arena(), KeyValueRef(endKey, endValue));
+	}
 	result.more = kv.more;
 
 	return result;
@@ -93,6 +116,43 @@ ACTOR Future<RangeResult> krmGetRanges(Reference<ReadYourWritesTransaction> tr,
 	return krmDecodeRanges(mapPrefix, keys, kv);
 }
 
+// Returns keys.begin, all transitional points in keys, and keys.end, and their values
+ACTOR Future<RangeResult> krmGetRangesUnaligned(Transaction* tr,
+                                                Key mapPrefix,
+                                                KeyRange keys,
+                                                int limit,
+                                                int limitBytes) {
+	KeyRange withPrefix =
+	    KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
+
+	state GetRangeLimits limits(limit, limitBytes);
+	limits.minRows = 2;
+	// wait to include the next highest row >= keys.end in the result, so since end is exclusive, we need +2 and
+	// !orEqual
+	RangeResult kv =
+	    wait(tr->getRange(lastLessOrEqual(withPrefix.begin), KeySelectorRef(withPrefix.end, false, +2), limits));
+
+	return krmDecodeRanges(mapPrefix, keys, kv, false);
+}
+
+ACTOR Future<RangeResult> krmGetRangesUnaligned(Reference<ReadYourWritesTransaction> tr,
+                                                Key mapPrefix,
+                                                KeyRange keys,
+                                                int limit,
+                                                int limitBytes) {
+	KeyRange withPrefix =
+	    KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
+
+	state GetRangeLimits limits(limit, limitBytes);
+	limits.minRows = 2;
+	// wait to include the next highest row >= keys.end in the result, so since end is exclusive, we need +2 and
+	// !orEqual
+	RangeResult kv =
+	    wait(tr->getRange(lastLessOrEqual(withPrefix.begin), KeySelectorRef(withPrefix.end, false, +2), limits));
+
+	return krmDecodeRanges(mapPrefix, keys, kv, false);
+}
+
 void krmSetPreviouslyEmptyRange(Transaction* tr,
                                 const KeyRef& mapPrefix,
                                 const KeyRangeRef& keys,
@@ -186,7 +246,7 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
 	// Determine how far to extend this range at the beginning
 	auto beginRange = keys[0].get();
 	bool hasBegin = beginRange.size() > 0 && beginRange[0].key.startsWith(mapPrefix);
-	Value beginValue = hasBegin ? beginRange[0].value : LiteralStringRef("");
+	Value beginValue = hasBegin ? beginRange[0].value : ""_sr;
 
 	state Key beginKey = withPrefix.begin;
 	if (beginValue == value) {
@@ -199,7 +259,7 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
 	bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
 	bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) ||
 	               (endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
-	Value existingValue = hasEnd ? endRange[0].value : LiteralStringRef("");
+	Value existingValue = hasEnd ? endRange[0].value : ""_sr;
 	bool valueMatches = value == existingValue;
 
 	KeyRange conflictRange = KeyRangeRef(hasBegin ? beginRange[0].key : mapPrefix, withPrefix.begin);
@@ -254,3 +314,107 @@ Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& t
                                    Value const& value) {
 	return holdWhile(tr, krmSetRangeCoalescing_(tr.getPtr(), mapPrefix, range, maxRange, value));
 }
+
+TEST_CASE("/keyrangemap/decoderange/aligned") {
+	Arena arena;
+	Key prefix = "/prefix/"_sr;
+	StringRef fullKeyA = StringRef(arena, "/prefix/a"_sr);
+	StringRef fullKeyB = StringRef(arena, "/prefix/b"_sr);
+	StringRef fullKeyC = StringRef(arena, "/prefix/c"_sr);
+	StringRef fullKeyD = StringRef(arena, "/prefix/d"_sr);
+
+	StringRef keyA = StringRef(arena, "a"_sr);
+	StringRef keyB = StringRef(arena, "b"_sr);
+	StringRef keyC = StringRef(arena, "c"_sr);
+	StringRef keyD = StringRef(arena, "d"_sr);
+	StringRef keyE = StringRef(arena, "e"_sr);
+	StringRef keyAB = StringRef(arena, "ab"_sr);
+	StringRef keyAC = StringRef(arena, "ac"_sr);
+	StringRef keyCD = StringRef(arena, "cd"_sr);
+
+	// Fake getRange() call.
+	RangeResult kv;
+	kv.push_back(arena, KeyValueRef(fullKeyA, keyA));
+	kv.push_back(arena, KeyValueRef(fullKeyB, keyB));
+
+	// [A, AB(start), AC(start), B]
+	RangeResult decodedRanges = krmDecodeRanges(prefix, KeyRangeRef(keyAB, keyAC), kv);
+	ASSERT(decodedRanges.size() == 2);
+	ASSERT(decodedRanges.front().key == keyAB);
+	ASSERT(decodedRanges.front().value == keyA);
+	ASSERT(decodedRanges.back().key == keyAC);
+	ASSERT(decodedRanges.back().value == keyA);
+
+	kv.push_back(arena, KeyValueRef(fullKeyC, keyC));
+	kv.push_back(arena, KeyValueRef(fullKeyD, keyD));
+
+	// [A, AB(start), B, C, CD(end), D]
+	decodedRanges = krmDecodeRanges(prefix, KeyRangeRef(keyAB, keyCD), kv);
+	ASSERT(decodedRanges.size() == 4);
+	ASSERT(decodedRanges.front().key == keyAB);
+	ASSERT(decodedRanges.front().value == keyA);
+	ASSERT(decodedRanges.back().key == keyCD);
+	ASSERT(decodedRanges.back().value == keyC);
+
+	// [""(start), A, B, C, D, E(end)]
+	decodedRanges = krmDecodeRanges(prefix, KeyRangeRef(StringRef(), keyE), kv);
+	ASSERT(decodedRanges.size() == 6);
+	ASSERT(decodedRanges.front().key == StringRef());
+	ASSERT(decodedRanges.front().value == StringRef());
+	ASSERT(decodedRanges.back().key == keyE);
+	ASSERT(decodedRanges.back().value == keyD);
+
+	return Void();
+}
+
+TEST_CASE("/keyrangemap/decoderange/unaligned") {
+	Arena arena;
+	Key prefix = "/prefix/"_sr;
+	StringRef fullKeyA = StringRef(arena, "/prefix/a"_sr);
+	StringRef fullKeyB = StringRef(arena, "/prefix/b"_sr);
+	StringRef fullKeyC = StringRef(arena, "/prefix/c"_sr);
+	StringRef fullKeyD = StringRef(arena, "/prefix/d"_sr);
+
+	StringRef keyA = StringRef(arena, "a"_sr);
+	StringRef keyB = StringRef(arena, "b"_sr);
+	StringRef keyC = StringRef(arena, "c"_sr);
+	StringRef keyD = StringRef(arena, "d"_sr);
+	StringRef keyE = StringRef(arena, "e"_sr);
+	StringRef keyAB = StringRef(arena, "ab"_sr);
+	StringRef keyAC = StringRef(arena, "ac"_sr);
+	StringRef keyCD = StringRef(arena, "cd"_sr);
+
+	// Fake getRange() call.
+	RangeResult kv;
+	kv.push_back(arena, KeyValueRef(fullKeyA, keyA));
+	kv.push_back(arena, KeyValueRef(fullKeyB, keyB));
+
+	// [A, AB(start), AC(start), B]
+	RangeResult decodedRanges = krmDecodeRanges(prefix, KeyRangeRef(keyAB, keyAC), kv, false);
+	ASSERT(decodedRanges.size() == 2);
+	ASSERT(decodedRanges.front().key == keyA);
+	ASSERT(decodedRanges.front().value == keyA);
+	ASSERT(decodedRanges.back().key == keyB);
+	ASSERT(decodedRanges.back().value == keyB);
+
+	kv.push_back(arena, KeyValueRef(fullKeyC, keyC));
+	kv.push_back(arena, KeyValueRef(fullKeyD, keyD));
+
+	// [A, AB(start), B, C, CD(end), D]
+	decodedRanges = krmDecodeRanges(prefix, KeyRangeRef(keyAB, keyCD), kv, false);
+	ASSERT(decodedRanges.size() == 4);
+	ASSERT(decodedRanges.front().key == keyA);
+	ASSERT(decodedRanges.front().value == keyA);
+	ASSERT(decodedRanges.back().key == keyD);
+	ASSERT(decodedRanges.back().value == keyD);
+
+	// [""(start), A, B, C, D, E(end)]
+	decodedRanges = krmDecodeRanges(prefix, KeyRangeRef(StringRef(), keyE), kv, false);
+	ASSERT(decodedRanges.size() == 6);
+	ASSERT(decodedRanges.front().key == StringRef());
+	ASSERT(decodedRanges.front().value == StringRef());
+	ASSERT(decodedRanges.back().key == keyE);
+	ASSERT(decodedRanges.back().value == keyD);
+
+	return Void();
+}
\ No newline at end of file
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 0ff62e276f..4119940d44 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <vector>
 
+#include "fdbclient/GenericManagementAPI.actor.h"
 #include "fmt/format.h"
 #include "fdbclient/Knobs.h"
 #include "flow/Arena.h"
@@ -200,6 +201,20 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 			}
 			out[p + key] = format("%d", tenantMode);
 		}
+
+		if (key == "encryption_at_rest_mode") {
+			EncryptionAtRestMode mode;
+			if (value == "disabled") {
+				mode = EncryptionAtRestMode::DISABLED;
+			} else if (value == "aes_256_ctr") {
+				mode = EncryptionAtRestMode::AES_256_CTR;
+			} else {
+				printf("Error: Only disabled|aes_256_ctr are valid for encryption_at_rest_mode.\n");
+				return out;
+			}
+			out[p + key] = format("%d", mode);
+		}
+
 		return out;
 	}
 
@@ -803,6 +818,8 @@ ACTOR Future<Optional<ClusterConnectionString>> getConnectionString(Database cx)
 	}
 }
 
+static std::vector<std::string> connectionStrings;
+
 namespace {
 
 ACTOR Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromStorageServer(Transaction* tr) {
@@ -820,6 +837,19 @@ ACTOR Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromSt
 
 		Version readVersion = wait(tr->getReadVersion());
 		state Optional<Value> currentKey = wait(tr->get(coordinatorsKey));
+		if (g_network->isSimulated() && currentKey.present()) {
+			// If the change coordinators request succeeded, the coordinators
+			// should have changed to the connection string of the most
+			// recently issued request. If instead the connection string is
+			// equal to one of the previously issued requests, there is a bug
+			// and we are breaking the promises we make with
+			// commit_unknown_result (the transaction must no longer be in
+			// progress when receiving commit_unknown_result).
+			int n = connectionStrings.size() > 0 ? connectionStrings.size() - 1 : 0; // avoid underflow
+			for (int i = 0; i < n; ++i) {
+				ASSERT(currentKey.get() != connectionStrings.at(i));
+			}
+		}
 
 		if (!currentKey.present()) {
 			// Someone deleted this key entirely?
@@ -842,12 +872,59 @@ ACTOR Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromSt
 	}
 }
 
+ACTOR Future<Void> verifyConfigurationDatabaseAlive(Database cx) {
+	state Backoff backoff;
+	state Reference<ISingleThreadTransaction> configTr;
+	loop {
+		try {
+			// Attempt to read a random value from the configuration
+			// database to make sure it is online.
+			configTr = ISingleThreadTransaction::create(ISingleThreadTransaction::Type::PAXOS_CONFIG, cx);
+			Tuple tuple;
+			tuple.appendNull(); // config class
+			tuple << "test"_sr;
+			Optional<Value> serializedValue = wait(configTr->get(tuple.pack()));
+			TraceEvent("ChangeQuorumCheckerNewCoordinatorsOnline").log();
+			return Void();
+		} catch (Error& e) {
+			TraceEvent("ChangeQuorumCheckerNewCoordinatorsError").error(e);
+			if (e.code() == error_code_coordinators_changed) {
+				wait(backoff.onError());
+				configTr->reset();
+			} else {
+				wait(configTr->onError(e));
+			}
+		}
+	}
+}
+
+ACTOR Future<Void> resetPreviousCoordinatorsKey(Database cx) {
+	loop {
+		// When the change coordinators transaction succeeds, it uses the
+		// special key space error message to return a message to the client.
+		// This causes the underlying transaction to not be committed. In order
+		// to make sure we clear the previous coordinators key, we have to use
+		// a new transaction here.
+		state Reference<ISingleThreadTransaction> clearTr =
+		    ISingleThreadTransaction::create(ISingleThreadTransaction::Type::RYW, cx);
+		try {
+			clearTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			clearTr->clear(previousCoordinatorsKey);
+			wait(clearTr->commit());
+			return Void();
+		} catch (Error& e2) {
+			wait(clearTr->onError(e2));
+		}
+	}
+}
+
 } // namespace
 
 ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
                                                                ClusterConnectionString* conn,
-                                                               std::string newName) {
-
+                                                               std::string newName,
+                                                               bool disableConfigDB) {
+	TraceEvent("ChangeQuorumCheckerStart").detail("NewConnectionString", conn->toString());
 	state Optional<ClusterConnectionString> clusterConnectionStringOptional =
 	    wait(getClusterConnectionStringFromStorageServer(tr));
 
@@ -862,7 +939,7 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 		conn->hostnames = old.hostnames;
 		conn->coords = old.coords;
 	}
-	std::vector<NetworkAddress> desiredCoordinators = wait(conn->tryResolveHostnames());
+	state std::vector<NetworkAddress> desiredCoordinators = wait(conn->tryResolveHostnames());
 	if (desiredCoordinators.size() != conn->hostnames.size() + conn->coords.size()) {
 		TraceEvent("ChangeQuorumCheckerEarlyTermination")
 		    .detail("Reason", "One or more hostnames are unresolvable")
@@ -878,16 +955,25 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 	std::sort(old.hostnames.begin(), old.hostnames.end());
 	std::sort(old.coords.begin(), old.coords.end());
 	if (conn->hostnames == old.hostnames && conn->coords == old.coords && old.clusterKeyName() == newName) {
+		connectionStrings.clear();
+		if (g_network->isSimulated() && g_simulator->configDBType == ConfigDBType::DISABLED) {
+			disableConfigDB = true;
+		}
+		if (!disableConfigDB) {
+			wait(verifyConfigurationDatabaseAlive(tr->getDatabase()));
+		}
+		wait(resetPreviousCoordinatorsKey(tr->getDatabase()));
 		return CoordinatorsResult::SAME_NETWORK_ADDRESSES;
 	}
 
 	conn->parseKey(newName + ':' + deterministicRandom()->randomAlphaNumeric(32));
+	connectionStrings.push_back(conn->toString());
 
 	if (g_network->isSimulated()) {
 		int i = 0;
 		int protectedCount = 0;
 		while ((protectedCount < ((desiredCoordinators.size() / 2) + 1)) && (i < desiredCoordinators.size())) {
-			auto process = g_simulator.getProcessByAddress(desiredCoordinators[i]);
+			auto process = g_simulator->getProcessByAddress(desiredCoordinators[i]);
 			auto addresses = process->addresses;
 
 			if (!process->isReliable()) {
@@ -895,9 +981,9 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 				continue;
 			}
 
-			g_simulator.protectedAddresses.insert(process->addresses.address);
+			g_simulator->protectedAddresses.insert(process->addresses.address);
 			if (addresses.secondaryAddress.present()) {
-				g_simulator.protectedAddresses.insert(process->addresses.secondaryAddress.get());
+				g_simulator->protectedAddresses.insert(process->addresses.secondaryAddress.get());
 			}
 			TraceEvent("ProtectCoordinator").detail("Address", desiredCoordinators[i]).backtrace();
 			protectedCount++;
@@ -924,8 +1010,13 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 
 	choose {
 		when(wait(waitForAll(leaderServers))) {}
-		when(wait(delay(5.0))) { return CoordinatorsResult::COORDINATOR_UNREACHABLE; }
+		when(wait(delay(5.0))) {
+			return CoordinatorsResult::COORDINATOR_UNREACHABLE;
+		}
 	}
+	TraceEvent("ChangeQuorumCheckerSetCoordinatorsKey")
+	    .detail("CurrentCoordinators", old.toString())
+	    .detail("NewCoordinators", conn->toString());
 	tr->set(coordinatorsKey, conn->toString());
 	return Optional<CoordinatorsResult>();
 }
@@ -988,12 +1079,12 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 
 			if (g_network->isSimulated()) {
 				for (int i = 0; i < (desiredCoordinators.size() / 2) + 1; i++) {
-					auto process = g_simulator.getProcessByAddress(desiredCoordinators[i]);
+					auto process = g_simulator->getProcessByAddress(desiredCoordinators[i]);
 					ASSERT(process->isReliable() || process->rebooting);
 
-					g_simulator.protectedAddresses.insert(process->addresses.address);
+					g_simulator->protectedAddresses.insert(process->addresses.address);
 					if (process->addresses.secondaryAddress.present()) {
-						g_simulator.protectedAddresses.insert(process->addresses.secondaryAddress.get());
+						g_simulator->protectedAddresses.insert(process->addresses.secondaryAddress.get());
 					}
 					TraceEvent("ProtectCoordinator").detail("Address", desiredCoordinators[i]).backtrace();
 				}
@@ -1002,8 +1093,8 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 			TraceEvent("AttemptingQuorumChange")
 			    .detail("FromCS", oldClusterConnectionString.toString())
 			    .detail("ToCS", newClusterConnectionString.toString());
-			TEST(oldClusterKeyName != newClusterKeyName); // Quorum change with new name
-			TEST(oldClusterKeyName == newClusterKeyName); // Quorum change with unchanged name
+			CODE_PROBE(oldClusterKeyName != newClusterKeyName, "Quorum change with new name");
+			CODE_PROBE(oldClusterKeyName == newClusterKeyName, "Quorum change with unchanged name");
 
 			state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 			state ClientCoordinators coord(Reference<ClusterConnectionMemoryRecord>(
@@ -1023,7 +1114,9 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 				                                           TaskPriority::CoordinationReply));
 			choose {
 				when(wait(waitForAll(leaderServers))) {}
-				when(wait(delay(5.0))) { return CoordinatorsResult::COORDINATOR_UNREACHABLE; }
+				when(wait(delay(5.0))) {
+					return CoordinatorsResult::COORDINATOR_UNREACHABLE;
+				}
 			}
 
 			tr.set(coordinatorsKey, newClusterConnectionString.toString());
@@ -1067,10 +1160,8 @@ struct AutoQuorumChange final : IQuorumChange {
 	}
 
 	ACTOR static Future<int> getRedundancy(AutoQuorumChange* self, Transaction* tr) {
-		state Future<Optional<Value>> fStorageReplicas =
-		    tr->get(LiteralStringRef("storage_replicas").withPrefix(configKeysPrefix));
-		state Future<Optional<Value>> fLogReplicas =
-		    tr->get(LiteralStringRef("log_replicas").withPrefix(configKeysPrefix));
+		state Future<Optional<Value>> fStorageReplicas = tr->get("storage_replicas"_sr.withPrefix(configKeysPrefix));
+		state Future<Optional<Value>> fLogReplicas = tr->get("log_replicas"_sr.withPrefix(configKeysPrefix));
 		wait(success(fStorageReplicas) && success(fLogReplicas));
 		int redundancy = std::min(atoi(fStorageReplicas.get().get().toString().c_str()),
 		                          atoi(fLogReplicas.get().get().toString().c_str()));
@@ -1232,10 +1323,7 @@ struct AutoQuorumChange final : IQuorumChange {
 		std::map<StringRef, std::map<StringRef, int>> currentCounts;
 		std::map<StringRef, int> hardLimits;
 
-		std::vector<StringRef> fields({ LiteralStringRef("dcid"),
-		                                LiteralStringRef("data_hall"),
-		                                LiteralStringRef("zoneid"),
-		                                LiteralStringRef("machineid") });
+		std::vector<StringRef> fields({ "dcid"_sr, "data_hall"_sr, "zoneid"_sr, "machineid"_sr });
 
 		for (auto field = fields.begin(); field != fields.end(); field++) {
 			if (field->toString() == "zoneid") {
@@ -1252,7 +1340,7 @@ struct AutoQuorumChange final : IQuorumChange {
 					continue;
 				}
 				// Exclude faulty node due to machine assassination
-				if (g_network->isSimulated() && !g_simulator.getProcessByAddress(worker->address)->isReliable()) {
+				if (g_network->isSimulated() && !g_simulator->getProcessByAddress(worker->address)->isReliable()) {
 					TraceEvent("AutoSelectCoordinators").detail("SkipUnreliableWorker", worker->address.toString());
 					continue;
 				}
@@ -1261,7 +1349,7 @@ struct AutoQuorumChange final : IQuorumChange {
 					if (maxCounts[*field] == 0) {
 						maxCounts[*field] = 1;
 					}
-					auto value = worker->locality.get(*field).orDefault(LiteralStringRef(""));
+					auto value = worker->locality.get(*field).orDefault(""_sr);
 					auto currentCount = currentCounts[*field][value];
 					if (currentCount >= maxCounts[*field]) {
 						valid = false;
@@ -1270,7 +1358,7 @@ struct AutoQuorumChange final : IQuorumChange {
 				}
 				if (valid) {
 					for (auto field = fields.begin(); field != fields.end(); field++) {
-						auto value = worker->locality.get(*field).orDefault(LiteralStringRef(""));
+						auto value = worker->locality.get(*field).orDefault(""_sr);
 						currentCounts[*field][value] += 1;
 					}
 					chosen.push_back(worker->address);
@@ -1323,6 +1411,7 @@ ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> ser
 		state ReadYourWritesTransaction ryw(cx);
 		loop {
 			try {
+				ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
 				ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				ryw.set(
 				    SpecialKeySpace::getManagementApiCommandOptionSpecialKey(failed ? "failed" : "excluded", "force"),
@@ -1385,6 +1474,7 @@ ACTOR Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string
 		state ReadYourWritesTransaction ryw(cx);
 		loop {
 			try {
+				ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
 				ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				ryw.set(SpecialKeySpace::getManagementApiCommandOptionSpecialKey(
 				            failed ? "failed_locality" : "excluded_locality", "force"),
@@ -1427,6 +1517,7 @@ ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> ser
 		state ReadYourWritesTransaction ryw(cx);
 		loop {
 			try {
+				ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
 				ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				for (auto& s : servers) {
 					if (!s.isValid()) {
@@ -1449,8 +1540,7 @@ ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> ser
 						// This is why we now make two clears: first only of the ip
 						// address, the second will delete all ports.
 						if (s.isWholeMachine())
-							ryw.clear(KeyRangeRef(addr.withSuffix(LiteralStringRef(":")),
-							                      addr.withSuffix(LiteralStringRef(";"))));
+							ryw.clear(KeyRangeRef(addr.withSuffix(":"_sr), addr.withSuffix(";"_sr)));
 					}
 				}
 				TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed);
@@ -1530,6 +1620,7 @@ ACTOR Future<Void> includeLocalities(Database cx, std::vector<std::string> local
 		state ReadYourWritesTransaction ryw(cx);
 		loop {
 			try {
+				ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
 				ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				if (includeAll) {
 					if (failed) {
@@ -2029,9 +2120,7 @@ ACTOR Future<Void> lockDatabase(Transaction* tr, UID id) {
 	}
 
 	tr->atomicOp(databaseLockedKey,
-	             BinaryWriter::toValue(id, Unversioned())
-	                 .withPrefix(LiteralStringRef("0123456789"))
-	                 .withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
+	             BinaryWriter::toValue(id, Unversioned()).withPrefix("0123456789"_sr).withSuffix("\x00\x00\x00\x00"_sr),
 	             MutationRef::SetVersionstampedValue);
 	tr->addWriteConflictRange(normalKeys);
 	return Void();
@@ -2052,9 +2141,7 @@ ACTOR Future<Void> lockDatabase(Reference<ReadYourWritesTransaction> tr, UID id)
 	}
 
 	tr->atomicOp(databaseLockedKey,
-	             BinaryWriter::toValue(id, Unversioned())
-	                 .withPrefix(LiteralStringRef("0123456789"))
-	                 .withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
+	             BinaryWriter::toValue(id, Unversioned()).withPrefix("0123456789"_sr).withSuffix("\x00\x00\x00\x00"_sr),
 	             MutationRef::SetVersionstampedValue);
 	tr->addWriteConflictRange(normalKeys);
 	return Void();
@@ -2173,7 +2260,7 @@ ACTOR Future<Void> updateChangeFeed(Transaction* tr, Key rangeID, ChangeFeedStat
 	} else if (status == ChangeFeedStatus::CHANGE_FEED_DESTROY) {
 		if (val.present()) {
 			if (g_network->isSimulated()) {
-				g_simulator.validationData.allDestroyedChangeFeedIDs.insert(rangeID.toString());
+				g_simulator->validationData.allDestroyedChangeFeedIDs.insert(rangeID.toString());
 			}
 			tr->set(rangeIDKey,
 			        changeFeedValue(std::get<0>(decodeChangeFeedValue(val.get())),
@@ -2211,7 +2298,7 @@ ACTOR Future<Void> updateChangeFeed(Reference<ReadYourWritesTransaction> tr,
 	} else if (status == ChangeFeedStatus::CHANGE_FEED_DESTROY) {
 		if (val.present()) {
 			if (g_network->isSimulated()) {
-				g_simulator.validationData.allDestroyedChangeFeedIDs.insert(rangeID.toString());
+				g_simulator->validationData.allDestroyedChangeFeedIDs.insert(rangeID.toString());
 			}
 			tr->set(rangeIDKey,
 			        changeFeedValue(std::get<0>(decodeChangeFeedValue(val.get())),
@@ -2461,6 +2548,21 @@ bool schemaMatch(json_spirit::mValue const& schemaValue,
 	}
 }
 
+void setStorageQuota(Transaction& tr, StringRef tenantName, uint64_t quota) {
+	tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+	auto key = storageQuotaKey(tenantName);
+	tr.set(key, BinaryWriter::toValue<uint64_t>(quota, Unversioned()));
+}
+
+ACTOR Future<Optional<uint64_t>> getStorageQuota(Transaction* tr, StringRef tenantName) {
+	tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+	state Optional<Value> v = wait(tr->get(storageQuotaKey(tenantName)));
+	if (!v.present()) {
+		return Optional<uint64_t>();
+	}
+	return BinaryReader::fromStringRef<uint64_t>(v.get(), Unversioned());
+}
+
 std::string ManagementAPI::generateErrorMessage(const CoordinatorsResult& res) {
 	// Note: the error message here should not be changed if possible
 	// If you do change the message here,
@@ -2509,24 +2611,24 @@ TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") {
 		auto dataHall = dataCenter + std::to_string(i / 2 % 2);
 		auto rack = dataHall + std::to_string(i % 2);
 		auto machineId = rack + std::to_string(i);
-		data.locality.set(LiteralStringRef("dcid"), StringRef(dataCenter));
-		data.locality.set(LiteralStringRef("data_hall"), StringRef(dataHall));
-		data.locality.set(LiteralStringRef("rack"), StringRef(rack));
-		data.locality.set(LiteralStringRef("zoneid"), StringRef(rack));
-		data.locality.set(LiteralStringRef("machineid"), StringRef(machineId));
+		data.locality.set("dcid"_sr, StringRef(dataCenter));
+		data.locality.set("data_hall"_sr, StringRef(dataHall));
+		data.locality.set("rack"_sr, StringRef(rack));
+		data.locality.set("zoneid"_sr, StringRef(rack));
+		data.locality.set("machineid"_sr, StringRef(machineId));
 		data.address.ip = IPAddress(i);
 
 		if (g_network->isSimulated()) {
-			g_simulator.newProcess("TestCoordinator",
-			                       data.address.ip,
-			                       data.address.port,
-			                       false,
-			                       1,
-			                       data.locality,
-			                       ProcessClass(ProcessClass::CoordinatorClass, ProcessClass::CommandLineSource),
-			                       "",
-			                       "",
-			                       currentProtocolVersion);
+			g_simulator->newProcess("TestCoordinator",
+			                        data.address.ip,
+			                        data.address.port,
+			                        false,
+			                        1,
+			                        data.locality,
+			                        ProcessClass(ProcessClass::CoordinatorClass, ProcessClass::CommandLineSource),
+			                        "",
+			                        "",
+			                        currentProtocolVersion());
 		}
 
 		workers.push_back(data);
@@ -2539,10 +2641,7 @@ TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") {
 	std::map<StringRef, std::set<StringRef>> chosenValues;
 
 	ASSERT(chosen.size() == 5);
-	std::vector<StringRef> fields({ LiteralStringRef("dcid"),
-	                                LiteralStringRef("data_hall"),
-	                                LiteralStringRef("zoneid"),
-	                                LiteralStringRef("machineid") });
+	std::vector<StringRef> fields({ "dcid"_sr, "data_hall"_sr, "zoneid"_sr, "machineid"_sr });
 	for (auto worker = chosen.begin(); worker != chosen.end(); worker++) {
 		ASSERT(worker->ip.toV4() < workers.size());
 		LocalityData data = workers[worker->ip.toV4()].locality;
@@ -2551,10 +2650,10 @@ TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") {
 		}
 	}
 
-	ASSERT(chosenValues[LiteralStringRef("dcid")].size() == 2);
-	ASSERT(chosenValues[LiteralStringRef("data_hall")].size() == 4);
-	ASSERT(chosenValues[LiteralStringRef("zoneid")].size() == 5);
-	ASSERT(chosenValues[LiteralStringRef("machineid")].size() == 5);
+	ASSERT(chosenValues["dcid"_sr].size() == 2);
+	ASSERT(chosenValues["data_hall"_sr].size() == 4);
+	ASSERT(chosenValues["zoneid"_sr].size() == 5);
+	ASSERT(chosenValues["machineid"_sr].size() == 5);
 	ASSERT(std::find(chosen.begin(), chosen.end(), workers[noAssignIndex].address) != chosen.end());
 
 	return Void();
diff --git a/fdbclient/Metacluster.cpp b/fdbclient/Metacluster.cpp
new file mode 100644
index 0000000000..6463033db8
--- /dev/null
+++ b/fdbclient/Metacluster.cpp
@@ -0,0 +1,71 @@
+/*
+ * Metacluster.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/Metacluster.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
+
+FDB_DEFINE_BOOLEAN_PARAM(AddNewTenants);
+FDB_DEFINE_BOOLEAN_PARAM(RemoveMissingTenants);
+
+std::string DataClusterEntry::clusterStateToString(DataClusterState clusterState) {
+	switch (clusterState) {
+	case DataClusterState::READY:
+		return "ready";
+	case DataClusterState::REMOVING:
+		return "removing";
+	case DataClusterState::RESTORING:
+		return "restoring";
+	default:
+		UNREACHABLE();
+	}
+}
+
+DataClusterState DataClusterEntry::stringToClusterState(std::string stateStr) {
+	if (stateStr == "ready") {
+		return DataClusterState::READY;
+	} else if (stateStr == "removing") {
+		return DataClusterState::REMOVING;
+	} else if (stateStr == "restoring") {
+		return DataClusterState::RESTORING;
+	}
+
+	UNREACHABLE();
+}
+
+json_spirit::mObject DataClusterEntry::toJson() const {
+	json_spirit::mObject obj;
+	obj["capacity"] = capacity.toJson();
+	obj["allocated"] = allocated.toJson();
+	obj["cluster_state"] = DataClusterEntry::clusterStateToString(clusterState);
+	return obj;
+}
+
+json_spirit::mObject ClusterUsage::toJson() const {
+	json_spirit::mObject obj;
+	obj["num_tenant_groups"] = numTenantGroups;
+	return obj;
+}
+
+KeyBackedObjectProperty<MetaclusterRegistrationEntry, decltype(IncludeVersion())>&
+MetaclusterMetadata::metaclusterRegistration() {
+	static KeyBackedObjectProperty<MetaclusterRegistrationEntry, decltype(IncludeVersion())> instance(
+	    "\xff/metacluster/clusterRegistration"_sr, IncludeVersion());
+	return instance;
+}
\ No newline at end of file
diff --git a/fdbclient/MetaclusterManagement.actor.cpp b/fdbclient/MetaclusterManagement.actor.cpp
new file mode 100644
index 0000000000..33403300bd
--- /dev/null
+++ b/fdbclient/MetaclusterManagement.actor.cpp
@@ -0,0 +1,67 @@
+/*
+ * MetaclusterManagement.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
+#include "fdbclient/ThreadSafeTransaction.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+namespace MetaclusterAPI {
+
+ACTOR Future<Reference<IDatabase>> openDatabase(ClusterConnectionString connectionString) {
+	if (g_network->isSimulated()) {
+		Reference<IClusterConnectionRecord> clusterFile =
+		    makeReference<ClusterConnectionMemoryRecord>(connectionString);
+		Database nativeDb = Database::createDatabase(clusterFile, -1);
+		Reference<IDatabase> threadSafeDb =
+		    wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(nativeDb)));
+		return MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeDb);
+	} else {
+		return MultiVersionApi::api->createDatabaseFromConnectionString(connectionString.toString().c_str());
+	}
+}
+
+KeyBackedObjectMap<ClusterName, DataClusterEntry, decltype(IncludeVersion())>&
+ManagementClusterMetadata::dataClusters() {
+	static KeyBackedObjectMap<ClusterName, DataClusterEntry, decltype(IncludeVersion())> instance(
+	    "metacluster/dataCluster/metadata/"_sr, IncludeVersion());
+	return instance;
+}
+
+KeyBackedMap<ClusterName,
+             ClusterConnectionString,
+             TupleCodec<ClusterName>,
+             ManagementClusterMetadata::ConnectionStringCodec>
+    ManagementClusterMetadata::dataClusterConnectionRecords("metacluster/dataCluster/connectionString/"_sr);
+
+KeyBackedSet<Tuple> ManagementClusterMetadata::clusterCapacityIndex("metacluster/clusterCapacityIndex/"_sr);
+KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>>
+    ManagementClusterMetadata::clusterTenantCount("metacluster/clusterTenantCount/"_sr);
+KeyBackedSet<Tuple> ManagementClusterMetadata::clusterTenantIndex("metacluster/dataCluster/tenantMap/"_sr);
+KeyBackedSet<Tuple> ManagementClusterMetadata::clusterTenantGroupIndex("metacluster/dataCluster/tenantGroupMap/"_sr);
+
+TenantMetadataSpecification& ManagementClusterMetadata::tenantMetadata() {
+	static TenantMetadataSpecification instance(""_sr);
+	return instance;
+}
+
+}; // namespace MetaclusterAPI
\ No newline at end of file
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index f0215ea36d..6c34369e6a 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -248,7 +248,7 @@ TEST_CASE("/fdbclient/MonitorLeader/ConnectionString/hostname") {
 		hostnames.push_back(Hostname::parse(hn1 + ":" + port1));
 		hostnames.push_back(Hostname::parse(hn2 + ":" + port2));
 
-		ClusterConnectionString cs(hostnames, LiteralStringRef("TestCluster:0"));
+		ClusterConnectionString cs(hostnames, "TestCluster:0"_sr);
 		ASSERT(cs.hostnames.size() == 2);
 		ASSERT(cs.coords.size() == 0);
 		ASSERT(cs.toString() == connectionString);
@@ -259,7 +259,7 @@ TEST_CASE("/fdbclient/MonitorLeader/ConnectionString/hostname") {
 		hostnames.push_back(Hostname::parse(hn1 + ":" + port1));
 		hostnames.push_back(Hostname::parse(hn1 + ":" + port1));
 		try {
-			ClusterConnectionString cs(hostnames, LiteralStringRef("TestCluster:0"));
+			ClusterConnectionString cs(hostnames, "TestCluster:0"_sr);
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_connection_string_invalid);
 		}
@@ -367,7 +367,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
 		auto c = connectionString.begin();
 		while (c != connectionString.end()) {
 			if (deterministicRandom()->random01() < 0.1) // Add whitespace character
-				output += deterministicRandom()->randomChoice(LiteralStringRef(" \t\n\r"));
+				output += deterministicRandom()->randomChoice(" \t\n\r"_sr);
 			if (deterministicRandom()->random01() < 0.5) { // Add one of the input characters
 				output += *c;
 				++c;
@@ -376,9 +376,9 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
 				output += "#";
 				int charCount = deterministicRandom()->randomInt(0, 20);
 				for (int i = 0; i < charCount; i++) {
-					output += deterministicRandom()->randomChoice(LiteralStringRef("asdfzxcv123345:!@#$#$&()<\"\' \t"));
+					output += deterministicRandom()->randomChoice("asdfzxcv123345:!@#$#$&()<\"\' \t"_sr);
 				}
-				output += deterministicRandom()->randomChoice(LiteralStringRef("\n\r"));
+				output += deterministicRandom()->randomChoice("\n\r"_sr);
 			}
 		}
 
@@ -501,6 +501,7 @@ ACTOR Future<Void> monitorNominee(Key key,
                                   Optional<LeaderInfo>* info) {
 	loop {
 		state Optional<LeaderInfo> li;
+		wait(Future<Void>(Void())); // Make sure we weren't cancelled
 		if (coord.hostname.present()) {
 			wait(store(li,
 			           retryGetReplyFromHostname(GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
@@ -663,69 +664,43 @@ ACTOR Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> s
 	}
 }
 
-struct ClientStatusStats {
-	int count;
-	std::vector<std::pair<NetworkAddress, Key>> examples;
+namespace {
 
-	ClientStatusStats() : count(0) { examples.reserve(CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT); }
-};
+void tryInsertIntoSamples(OpenDatabaseRequest::Samples& samples,
+                          const NetworkAddress& networkAddress,
+                          const Key& traceLogGroup) {
+	++samples.count;
+	if (samples.samples.size() < static_cast<size_t>(CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT)) {
+		samples.samples.insert({ networkAddress, traceLogGroup });
+	}
+}
+
+} // namespace
 
 OpenDatabaseRequest ClientData::getRequest() {
 	OpenDatabaseRequest req;
 
-	std::map<StringRef, ClientStatusStats> issueMap;
-	std::map<ClientVersionRef, ClientStatusStats> versionMap;
-	std::map<StringRef, ClientStatusStats> maxProtocolMap;
-	int clientCount = 0;
-
-	// SOMEDAY: add a yield in this loop
 	for (auto& ci : clientStatusInfoMap) {
-		for (auto& it : ci.second.issues) {
-			auto& entry = issueMap[it];
-			entry.count++;
-			if (entry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
-				entry.examples.emplace_back(ci.first, ci.second.traceLogGroup);
-			}
-		}
-		if (ci.second.versions.size()) {
-			clientCount++;
-			StringRef maxProtocol;
-			for (auto& it : ci.second.versions) {
-				maxProtocol = std::max(maxProtocol, it.protocolVersion);
-				auto& entry = versionMap[it];
-				entry.count++;
-				if (entry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
-					entry.examples.emplace_back(ci.first, ci.second.traceLogGroup);
-				}
-			}
-			auto& maxEntry = maxProtocolMap[maxProtocol];
-			maxEntry.count++;
-			if (maxEntry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
-				maxEntry.examples.emplace_back(ci.first, ci.second.traceLogGroup);
-			}
-		} else {
-			auto& entry = versionMap[ClientVersionRef()];
-			entry.count++;
-			if (entry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
-				entry.examples.emplace_back(ci.first, ci.second.traceLogGroup);
-			}
-		}
-	}
+		const auto& networkAddress = ci.first;
+		const auto& traceLogGroup = ci.second.traceLogGroup;
 
-	req.issues.reserve(issueMap.size());
-	for (auto& it : issueMap) {
-		req.issues.push_back(ItemWithExamples<Key>(it.first, it.second.count, it.second.examples));
+		for (auto& issue : ci.second.issues) {
+			tryInsertIntoSamples(req.issues[issue], networkAddress, traceLogGroup);
+		}
+
+		if (!ci.second.versions.size()) {
+			tryInsertIntoSamples(req.supportedVersions[ClientVersionRef()], networkAddress, traceLogGroup);
+			continue;
+		}
+
+		++req.clientCount;
+		StringRef maxProtocol;
+		for (auto& it : ci.second.versions) {
+			maxProtocol = std::max(maxProtocol, it.protocolVersion);
+			tryInsertIntoSamples(req.supportedVersions[it], networkAddress, traceLogGroup);
+		}
+		tryInsertIntoSamples(req.maxProtocolSupported[maxProtocol], networkAddress, traceLogGroup);
 	}
-	req.supportedVersions.reserve(versionMap.size());
-	for (auto& it : versionMap) {
-		req.supportedVersions.push_back(
-		    ItemWithExamples<Standalone<ClientVersionRef>>(it.first, it.second.count, it.second.examples));
-	}
-	req.maxProtocolSupported.reserve(maxProtocolMap.size());
-	for (auto& it : maxProtocolMap) {
-		req.maxProtocolSupported.push_back(ItemWithExamples<Key>(it.first, it.second.count, it.second.examples));
-	}
-	req.clientCount = clientCount;
 
 	return req;
 }
@@ -887,6 +862,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 	for (const auto& c : cs.coords) {
 		clientLeaderServers.push_back(ClientLeaderRegInterface(c));
 	}
+	ASSERT(clientLeaderServers.size() > 0);
 
 	deterministicRandom()->randomShuffle(clientLeaderServers);
 
@@ -906,7 +882,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			bool upToDate = wait(connRecord->upToDate(storedConnectionString));
 			if (upToDate) {
 				incorrectTime = Optional<double>();
-			} else if (allConnectionsFailed) {
+			} else if (allConnectionsFailed && storedConnectionString.getNumberOfCoordinators() > 0) {
 				// Failed to connect to all coordinators from the current connection string,
 				// so it is not possible to get any new updates from the cluster. It can be that
 				// all the coordinators have changed, but the client missed that, because it had
@@ -920,7 +896,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 				info.intermediateConnRecord = connRecord;
 				return info;
 			} else {
-				req.issues.push_back_deep(req.issues.arena(), LiteralStringRef("incorrect_cluster_file_contents"));
+				req.issues.push_back_deep(req.issues.arena(), "incorrect_cluster_file_contents"_sr);
 				std::string connectionString = connRecord->getConnectionString().toString();
 				if (!incorrectTime.present()) {
 					incorrectTime = now();
@@ -964,6 +940,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 				    .detail("OldConnStr", info.intermediateConnRecord->getConnectionString().toString());
 				info.intermediateConnRecord = connRecord->makeIntermediateRecord(
 				    ClusterConnectionString(rep.get().read().forward.get().toString()));
+				ASSERT(info.intermediateConnRecord->getConnectionString().getNumberOfCoordinators() > 0);
 				return info;
 			}
 			if (connRecord != info.intermediateConnRecord) {
@@ -987,8 +964,8 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			successIndex = index;
 			allConnectionsFailed = false;
 		} else {
-			TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cant talk to cluster controller
-			TEST(rep.getError().code() == error_code_lookup_failed); // Coordinator hostname resolving failure
+			CODE_PROBE(rep.getError().code() == error_code_failed_to_progress,
+			           "Coordinator cant talk to cluster controller");
 			TraceEvent("MonitorProxiesConnectFailed")
 			    .detail("Error", rep.getError().name())
 			    .detail("Coordinator", clientLeaderServer.getAddressString());
@@ -1009,6 +986,7 @@ ACTOR Future<Void> monitorProxies(
     Key traceLogGroup) {
 	state MonitorLeaderInfo info(connRecord->get());
 	loop {
+		ASSERT(connRecord->get().isValid());
 		choose {
 			when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
 			         connRecord->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup))) {
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index b08457a3ac..775cfa54d0 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -257,13 +257,14 @@ ThreadFuture<Standalone<VectorRef<KeyRef>>> DLTransaction::getRangeSplitPoints(c
 	});
 }
 
-ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> DLTransaction::getBlobGranuleRanges(const KeyRangeRef& keyRange) {
+ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> DLTransaction::getBlobGranuleRanges(const KeyRangeRef& keyRange,
+                                                                                     int rangeLimit) {
 	if (!api->transactionGetBlobGranuleRanges) {
 		return unsupported_operation();
 	}
 
 	FdbCApi::FDBFuture* f = api->transactionGetBlobGranuleRanges(
-	    tr, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size());
+	    tr, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size(), rangeLimit);
 	return toThreadFuture<Standalone<VectorRef<KeyRangeRef>>>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
 		const FdbCApi::FDBKeyRange* keyRanges;
 		int keyRangesLength;
@@ -279,10 +280,46 @@ ThreadResult<RangeResult> DLTransaction::readBlobGranules(const KeyRangeRef& key
                                                           Version beginVersion,
                                                           Optional<Version> readVersion,
                                                           ReadBlobGranuleContext granuleContext) {
-	if (!api->transactionReadBlobGranules) {
+	return unsupported_operation();
+}
+
+ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> DLTransaction::readBlobGranulesStart(
+    const KeyRangeRef& keyRange,
+    Version beginVersion,
+    Optional<Version> readVersion,
+    Version* readVersionOut) {
+	if (!api->transactionReadBlobGranulesStart) {
 		return unsupported_operation();
 	}
 
+	int64_t rv = readVersion.present() ? readVersion.get() : latestVersion;
+
+	FdbCApi::FDBFuture* f = api->transactionReadBlobGranulesStart(tr,
+	                                                              keyRange.begin.begin(),
+	                                                              keyRange.begin.size(),
+	                                                              keyRange.end.begin(),
+	                                                              keyRange.end.size(),
+	                                                              beginVersion,
+	                                                              rv,
+	                                                              readVersionOut);
+
+	return ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>>(
+	    (ThreadSingleAssignmentVar<Standalone<VectorRef<BlobGranuleChunkRef>>>*)(f));
+};
+
+ThreadResult<RangeResult> DLTransaction::readBlobGranulesFinish(
+    ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> startFuture,
+    const KeyRangeRef& keyRange,
+    Version beginVersion,
+    Version readVersion,
+    ReadBlobGranuleContext granuleContext) {
+	if (!api->transactionReadBlobGranulesFinish) {
+		return unsupported_operation();
+	}
+
+	// convert back to fdb future for API
+	FdbCApi::FDBFuture* f = (FdbCApi::FDBFuture*)(startFuture.extractPtr());
+
 	// FIXME: better way to convert here?
 	FdbCApi::FDBReadBlobGranuleContext context;
 	context.userContext = granuleContext.userContext;
@@ -292,17 +329,40 @@ ThreadResult<RangeResult> DLTransaction::readBlobGranules(const KeyRangeRef& key
 	context.debugNoMaterialize = granuleContext.debugNoMaterialize;
 	context.granuleParallelism = granuleContext.granuleParallelism;
 
-	int64_t rv = readVersion.present() ? readVersion.get() : latestVersion;
+	FdbCApi::FDBResult* r = api->transactionReadBlobGranulesFinish(tr,
+	                                                               f,
+	                                                               keyRange.begin.begin(),
+	                                                               keyRange.begin.size(),
+	                                                               keyRange.end.begin(),
+	                                                               keyRange.end.size(),
+	                                                               beginVersion,
+	                                                               readVersion,
+	                                                               &context);
 
-	FdbCApi::FDBResult* r = api->transactionReadBlobGranules(tr,
-	                                                         keyRange.begin.begin(),
-	                                                         keyRange.begin.size(),
-	                                                         keyRange.end.begin(),
-	                                                         keyRange.end.size(),
-	                                                         beginVersion,
-	                                                         rv,
-	                                                         context);
 	return ThreadResult<RangeResult>((ThreadSingleAssignmentVar<RangeResult>*)(r));
+};
+
+ThreadFuture<Standalone<VectorRef<BlobGranuleSummaryRef>>>
+DLTransaction::summarizeBlobGranules(const KeyRangeRef& keyRange, Optional<Version> summaryVersion, int rangeLimit) {
+	if (!api->transactionSummarizeBlobGranules) {
+		return unsupported_operation();
+	}
+
+	int64_t sv = summaryVersion.present() ? summaryVersion.get() : latestVersion;
+
+	FdbCApi::FDBFuture* f = api->transactionSummarizeBlobGranules(
+	    tr, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size(), sv, rangeLimit);
+
+	return toThreadFuture<Standalone<VectorRef<BlobGranuleSummaryRef>>>(
+	    api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
+		    const FdbCApi::FDBGranuleSummary* summaries;
+		    int summariesLength;
+		    FdbCApi::fdb_error_t error = api->futureGetGranuleSummaryArray(f, &summaries, &summariesLength);
+		    ASSERT(!error);
+		    // The memory for this is stored in the FDBFuture and is released when the future gets destroyed
+		    return Standalone<VectorRef<BlobGranuleSummaryRef>>(
+		        VectorRef<BlobGranuleSummaryRef>((BlobGranuleSummaryRef*)summaries, summariesLength), Arena());
+	    });
 }
 
 void DLTransaction::addReadConflictRange(const KeyRangeRef& keys) {
@@ -583,6 +643,73 @@ ThreadFuture<Void> DLDatabase::waitPurgeGranulesComplete(const KeyRef& purgeKey)
 	return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { return Void(); });
 }
 
+ThreadFuture<bool> DLDatabase::blobbifyRange(const KeyRangeRef& keyRange) {
+	if (!api->databaseBlobbifyRange) {
+		return unsupported_operation();
+	}
+
+	FdbCApi::FDBFuture* f = api->databaseBlobbifyRange(
+	    db, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size());
+
+	return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
+		FdbCApi::fdb_bool_t ret = false;
+		ASSERT(!api->futureGetBool(f, &ret));
+		return ret;
+	});
+}
+
+ThreadFuture<bool> DLDatabase::unblobbifyRange(const KeyRangeRef& keyRange) {
+	if (!api->databaseUnblobbifyRange) {
+		return unsupported_operation();
+	}
+
+	FdbCApi::FDBFuture* f = api->databaseUnblobbifyRange(
+	    db, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size());
+
+	return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
+		FdbCApi::fdb_bool_t ret = false;
+		ASSERT(!api->futureGetBool(f, &ret));
+		return ret;
+	});
+}
+
+ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> DLDatabase::listBlobbifiedRanges(const KeyRangeRef& keyRange,
+                                                                                  int rangeLimit) {
+	if (!api->databaseListBlobbifiedRanges) {
+		return unsupported_operation();
+	}
+
+	FdbCApi::FDBFuture* f = api->databaseListBlobbifiedRanges(
+	    db, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size(), rangeLimit);
+
+	return toThreadFuture<Standalone<VectorRef<KeyRangeRef>>>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
+		const FdbCApi::FDBKeyRange* keyRanges;
+		int keyRangesLength;
+		FdbCApi::fdb_error_t error = api->futureGetKeyRangeArray(f, &keyRanges, &keyRangesLength);
+		ASSERT(!error);
+		// The memory for this is stored in the FDBFuture and is released when the future gets destroyed.
+		return Standalone<VectorRef<KeyRangeRef>>(VectorRef<KeyRangeRef>((KeyRangeRef*)keyRanges, keyRangesLength),
+		                                          Arena());
+	});
+}
+
+ThreadFuture<Version> DLDatabase::verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) {
+	if (!api->databaseVerifyBlobRange) {
+		return unsupported_operation();
+	}
+
+	Version readVersion = version.present() ? version.get() : latestVersion;
+
+	FdbCApi::FDBFuture* f = api->databaseVerifyBlobRange(
+	    db, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size(), readVersion);
+
+	return toThreadFuture<Version>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
+		Version version = invalidVersion;
+		ASSERT(!api->futureGetInt64(f, &version));
+		return version;
+	});
+}
+
 // DLApi
 
 // Loads the specified function from a dynamic library
@@ -626,6 +753,11 @@ void DLApi::init() {
 
 	loadClientFunction(&api->selectApiVersion, lib, fdbCPath, "fdb_select_api_version_impl", headerVersion >= 0);
 	loadClientFunction(&api->getClientVersion, lib, fdbCPath, "fdb_get_client_version", headerVersion >= 410);
+	loadClientFunction(&api->useFutureProtocolVersion,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_use_future_protocol_version",
+	                   headerVersion >= ApiVersion::withFutureProtocolVersionApi().version());
 	loadClientFunction(&api->setNetworkOption, lib, fdbCPath, "fdb_network_set_option", headerVersion >= 0);
 	loadClientFunction(&api->setupNetwork, lib, fdbCPath, "fdb_setup_network", headerVersion >= 0);
 	loadClientFunction(&api->runNetwork, lib, fdbCPath, "fdb_run_network", headerVersion >= 0);
@@ -635,7 +767,7 @@ void DLApi::init() {
 	                   lib,
 	                   fdbCPath,
 	                   "fdb_create_database_from_connection_string",
-	                   headerVersion >= 720);
+	                   headerVersion >= ApiVersion::withCreateDBFromConnString().version());
 
 	loadClientFunction(&api->databaseOpenTenant, lib, fdbCPath, "fdb_database_open_tenant", headerVersion >= 710);
 	loadClientFunction(
@@ -668,16 +800,39 @@ void DLApi::init() {
 	                   fdbCPath,
 	                   "fdb_database_wait_purge_granules_complete",
 	                   headerVersion >= 710);
+	loadClientFunction(&api->databaseBlobbifyRange,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_database_blobbify_range",
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
+	loadClientFunction(&api->databaseUnblobbifyRange,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_database_unblobbify_range",
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
+	loadClientFunction(&api->databaseListBlobbifiedRanges,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_database_list_blobbified_ranges",
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
+	loadClientFunction(&api->databaseVerifyBlobRange,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_database_verify_blob_range",
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
 
 	loadClientFunction(
 	    &api->tenantCreateTransaction, lib, fdbCPath, "fdb_tenant_create_transaction", headerVersion >= 710);
-	loadClientFunction(
-	    &api->tenantPurgeBlobGranules, lib, fdbCPath, "fdb_tenant_purge_blob_granules", headerVersion >= 720);
+	loadClientFunction(&api->tenantPurgeBlobGranules,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_tenant_purge_blob_granules",
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
 	loadClientFunction(&api->tenantWaitPurgeGranulesComplete,
 	                   lib,
 	                   fdbCPath,
 	                   "fdb_tenant_wait_purge_granules_complete",
-	                   headerVersion >= 720);
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
 	loadClientFunction(&api->tenantDestroy, lib, fdbCPath, "fdb_tenant_destroy", headerVersion >= 710);
 
 	loadClientFunction(&api->transactionSetOption, lib, fdbCPath, "fdb_transaction_set_option", headerVersion >= 0);
@@ -737,11 +892,31 @@ void DLApi::init() {
 	                   headerVersion >= 710);
 	loadClientFunction(
 	    &api->transactionReadBlobGranules, lib, fdbCPath, "fdb_transaction_read_blob_granules", headerVersion >= 710);
+	loadClientFunction(&api->transactionReadBlobGranulesStart,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_transaction_read_blob_granules_start",
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
+	loadClientFunction(&api->transactionReadBlobGranulesFinish,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_transaction_read_blob_granules_finish",
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
+	loadClientFunction(&api->transactionSummarizeBlobGranules,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_transaction_summarize_blob_granules",
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
 	loadClientFunction(&api->futureGetInt64,
 	                   lib,
 	                   fdbCPath,
 	                   headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version",
 	                   headerVersion >= 0);
+	loadClientFunction(&api->futureGetBool,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_future_get_bool",
+	                   headerVersion >= ApiVersion::withFutureGetBool().version());
 	loadClientFunction(&api->futureGetUInt64, lib, fdbCPath, "fdb_future_get_uint64", headerVersion >= 700);
 	loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error", headerVersion >= 0);
 	loadClientFunction(&api->futureGetKey, lib, fdbCPath, "fdb_future_get_key", headerVersion >= 0);
@@ -754,6 +929,11 @@ void DLApi::init() {
 	    &api->futureGetKeyValueArray, lib, fdbCPath, "fdb_future_get_keyvalue_array", headerVersion >= 0);
 	loadClientFunction(
 	    &api->futureGetMappedKeyValueArray, lib, fdbCPath, "fdb_future_get_mappedkeyvalue_array", headerVersion >= 710);
+	loadClientFunction(&api->futureGetGranuleSummaryArray,
+	                   lib,
+	                   fdbCPath,
+	                   "fdb_future_get_granule_summary_array",
+	                   headerVersion >= ApiVersion::withBlobRangeApi().version());
 	loadClientFunction(&api->futureGetSharedState, lib, fdbCPath, "fdb_future_get_shared_state", headerVersion >= 710);
 	loadClientFunction(&api->futureSetCallback, lib, fdbCPath, "fdb_future_set_callback", headerVersion >= 0);
 	loadClientFunction(&api->futureCancel, lib, fdbCPath, "fdb_future_cancel", headerVersion >= 0);
@@ -788,6 +968,14 @@ const char* DLApi::getClientVersion() {
 	return api->getClientVersion();
 }
 
+void DLApi::useFutureProtocolVersion() {
+	if (!api->useFutureProtocolVersion) {
+		return;
+	}
+
+	api->useFutureProtocolVersion();
+}
+
 void DLApi::setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value) {
 	throwIfError(api->setNetworkOption(static_cast<FDBNetworkOption>(option),
 	                                   value.present() ? value.get().begin() : nullptr,
@@ -1069,9 +1257,10 @@ ThreadFuture<Standalone<VectorRef<KeyRef>>> MultiVersionTransaction::getRangeSpl
 }
 
 ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> MultiVersionTransaction::getBlobGranuleRanges(
-    const KeyRangeRef& keyRange) {
+    const KeyRangeRef& keyRange,
+    int rangeLimit) {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getBlobGranuleRanges(keyRange)
+	auto f = tr.transaction ? tr.transaction->getBlobGranuleRanges(keyRange, rangeLimit)
 	                        : makeTimeout<Standalone<VectorRef<KeyRangeRef>>>();
 	return abortableFuture(f, tr.onChange);
 }
@@ -1080,14 +1269,55 @@ ThreadResult<RangeResult> MultiVersionTransaction::readBlobGranules(const KeyRan
                                                                     Version beginVersion,
                                                                     Optional<Version> readVersion,
                                                                     ReadBlobGranuleContext granuleContext) {
+	// FIXME: prevent from calling this from another main thread?
 	auto tr = getTransaction();
 	if (tr.transaction) {
-		return tr.transaction->readBlobGranules(keyRange, beginVersion, readVersion, granuleContext);
+		Version readVersionOut;
+		auto f = tr.transaction->readBlobGranulesStart(keyRange, beginVersion, readVersion, &readVersionOut);
+		auto abortableF = abortableFuture(f, tr.onChange);
+		abortableF.blockUntilReadyCheckOnMainThread();
+		if (abortableF.isError()) {
+			return ThreadResult<RangeResult>(abortableF.getError());
+		}
+		if (granuleContext.debugNoMaterialize) {
+			return ThreadResult<RangeResult>(blob_granule_not_materialized());
+		}
+		return tr.transaction->readBlobGranulesFinish(
+		    abortableF, keyRange, beginVersion, readVersionOut, granuleContext);
 	} else {
 		return abortableTimeoutResult<RangeResult>(tr.onChange);
 	}
 }
 
+ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> MultiVersionTransaction::readBlobGranulesStart(
+    const KeyRangeRef& keyRange,
+    Version beginVersion,
+    Optional<Version> readVersion,
+    Version* readVersionOut) {
+	// can't call this directly
+	return ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>>(unsupported_operation());
+}
+
+ThreadResult<RangeResult> MultiVersionTransaction::readBlobGranulesFinish(
+    ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> startFuture,
+    const KeyRangeRef& keyRange,
+    Version beginVersion,
+    Version readVersion,
+    ReadBlobGranuleContext granuleContext) {
+	// can't call this directly
+	return ThreadResult<RangeResult>(unsupported_operation());
+}
+
+ThreadFuture<Standalone<VectorRef<BlobGranuleSummaryRef>>> MultiVersionTransaction::summarizeBlobGranules(
+    const KeyRangeRef& keyRange,
+    Optional<Version> summaryVersion,
+    int rangeLimit) {
+	auto tr = getTransaction();
+	auto f = tr.transaction ? tr.transaction->summarizeBlobGranules(keyRange, summaryVersion, rangeLimit)
+	                        : makeTimeout<Standalone<VectorRef<BlobGranuleSummaryRef>>>();
+	return abortableFuture(f, tr.onChange);
+}
+
 void MultiVersionTransaction::atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) {
 	auto tr = getTransaction();
 	if (tr.transaction) {
@@ -1182,7 +1412,7 @@ void MultiVersionTransaction::setOption(FDBTransactionOptions::Option option, Op
 		throw invalid_option();
 	}
 
-	if (MultiVersionApi::apiVersionAtLeast(610) && itr->second.persistent) {
+	if (MultiVersionApi::api->getApiVersion().hasPersistentOptions() && itr->second.persistent) {
 		persistentOptions.emplace_back(option, value.castTo<Standalone<StringRef>>());
 	}
 
@@ -1579,6 +1809,32 @@ ThreadFuture<Void> MultiVersionDatabase::waitPurgeGranulesComplete(const KeyRef&
 	return abortableFuture(f, dbState->dbVar->get().onChange);
 }
 
+ThreadFuture<bool> MultiVersionDatabase::blobbifyRange(const KeyRangeRef& keyRange) {
+	auto dbVar = dbState->dbVar->get();
+	auto f = dbVar.value ? dbVar.value->blobbifyRange(keyRange) : ThreadFuture<bool>(Never());
+	return abortableFuture(f, dbVar.onChange);
+}
+
+ThreadFuture<bool> MultiVersionDatabase::unblobbifyRange(const KeyRangeRef& keyRange) {
+	auto dbVar = dbState->dbVar->get();
+	auto f = dbVar.value ? dbVar.value->unblobbifyRange(keyRange) : ThreadFuture<bool>(Never());
+	return abortableFuture(f, dbVar.onChange);
+}
+
+ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> MultiVersionDatabase::listBlobbifiedRanges(const KeyRangeRef& keyRange,
+                                                                                            int rangeLimit) {
+	auto dbVar = dbState->dbVar->get();
+	auto f = dbVar.value ? dbVar.value->listBlobbifiedRanges(keyRange, rangeLimit)
+	                     : ThreadFuture<Standalone<VectorRef<KeyRangeRef>>>(Never());
+	return abortableFuture(f, dbVar.onChange);
+}
+
+ThreadFuture<Version> MultiVersionDatabase::verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) {
+	auto dbVar = dbState->dbVar->get();
+	auto f = dbVar.value ? dbVar.value->verifyBlobRange(keyRange, version) : ThreadFuture<Version>(Never());
+	return abortableFuture(f, dbVar.onChange);
+}
+
 // Returns the protocol version reported by the coordinator this client is connected to
 // If an expected version is given, the future won't return until the protocol version is different than expected
 // Note: this will never return if the server is running a protocol from FDB 5.0 or older
@@ -1644,7 +1900,7 @@ ThreadFuture<Void> MultiVersionDatabase::DatabaseState::monitorProtocolVersion()
 		}
 
 		ProtocolVersion clusterVersion =
-		    !cv.isError() ? cv.get() : self->dbProtocolVersion.orDefault(currentProtocolVersion);
+		    !cv.isError() ? cv.get() : self->dbProtocolVersion.orDefault(currentProtocolVersion());
 		onMainThreadVoid([self, clusterVersion]() { self->protocolVersionChanged(clusterVersion); });
 		return ErrorOr<Void>(Void());
 	});
@@ -1674,7 +1930,7 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion
 		    .detail("OldProtocolVersion", dbProtocolVersion);
 		// When the protocol version changes, clear the corresponding entry in the shared state map
 		// so it can be re-initialized. Only do so if there was a valid previous protocol version.
-		if (dbProtocolVersion.present() && MultiVersionApi::apiVersionAtLeast(710)) {
+		if (dbProtocolVersion.present() && MultiVersionApi::api->getApiVersion().hasClusterSharedStateMap()) {
 			MultiVersionApi::api->clearClusterSharedStateMapEntry(clusterId, dbProtocolVersion.get());
 		}
 
@@ -1703,7 +1959,7 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion
 				return;
 			}
 
-			if (client->external && !MultiVersionApi::apiVersionAtLeast(610)) {
+			if (client->external && !MultiVersionApi::api->getApiVersion().hasInlineUpdateDatabase()) {
 				// Old API versions return a future when creating the database, so we need to wait for it
 				Reference<DatabaseState> self = Reference<DatabaseState>::addRef(this);
 				dbReady = mapThreadFuture<Void, Void>(
@@ -1787,7 +2043,8 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
 			    .detail("ConnectionRecord", connectionRecord);
 		}
 	}
-	if (db.isValid() && dbProtocolVersion.present() && MultiVersionApi::apiVersionAtLeast(710)) {
+	if (db.isValid() && dbProtocolVersion.present() &&
+	    MultiVersionApi::api->getApiVersion().hasClusterSharedStateMap()) {
 		Future<std::string> updateResult =
 		    MultiVersionApi::api->updateClusterSharedStateMap(connectionRecord, dbProtocolVersion.get(), db);
 		sharedStateUpdater = map(errorOr(updateResult), [this](ErrorOr<std::string> result) {
@@ -1907,11 +2164,6 @@ void MultiVersionDatabase::LegacyVersionMonitor::close() {
 }
 
 // MultiVersionApi
-bool MultiVersionApi::apiVersionAtLeast(int minVersion) {
-	ASSERT_NE(MultiVersionApi::api->apiVersion, 0);
-	return MultiVersionApi::api->apiVersion >= minVersion || MultiVersionApi::api->apiVersion < 0;
-}
-
 void MultiVersionApi::runOnExternalClientsAllThreads(std::function<void(Reference<ClientInfo>)> func,
                                                      bool runOnFailedClients) {
 	for (int i = 0; i < threadCount; i++) {
@@ -1957,23 +2209,28 @@ Reference<ClientInfo> MultiVersionApi::getLocalClient() {
 }
 
 void MultiVersionApi::selectApiVersion(int apiVersion) {
+	ApiVersion newApiVersion(apiVersion);
 	if (!localClient) {
 		localClient = makeReference<ClientInfo>(getLocalClientAPI());
 		ASSERT(localClient);
 	}
 
-	if (this->apiVersion != 0 && this->apiVersion != apiVersion) {
+	if (this->apiVersion.isValid() && this->apiVersion != newApiVersion) {
 		throw api_version_already_set();
 	}
 
 	localClient->api->selectApiVersion(apiVersion);
-	this->apiVersion = apiVersion;
+	this->apiVersion = newApiVersion;
 }
 
 const char* MultiVersionApi::getClientVersion() {
 	return localClient->api->getClientVersion();
 }
 
+void MultiVersionApi::useFutureProtocolVersion() {
+	localClient->api->useFutureProtocolVersion();
+}
+
 namespace {
 
 void validateOption(Optional<StringRef> value, bool canBePresent, bool canBeAbsent, bool canBeEmpty = true) {
@@ -1991,7 +2248,7 @@ void validateOption(Optional<StringRef> value, bool canBePresent, bool canBeAbse
 
 void MultiVersionApi::disableMultiVersionClientApi() {
 	MutexHolder holder(lock);
-	if (networkStartSetup || localClientDisabled) {
+	if (networkStartSetup || localClientDisabled || disableBypass) {
 		throw invalid_option();
 	}
 
@@ -2006,7 +2263,7 @@ void MultiVersionApi::setCallbacksOnExternalThreads() {
 
 	callbackOnMainThread = false;
 }
-void MultiVersionApi::addExternalLibrary(std::string path) {
+void MultiVersionApi::addExternalLibrary(std::string path, bool useFutureVersion) {
 	std::string filename = basename(path);
 
 	if (filename.empty() || !fileExists(path)) {
@@ -2023,8 +2280,8 @@ void MultiVersionApi::addExternalLibrary(std::string path) {
 	threadCount = std::max(threadCount, 1);
 
 	if (externalClientDescriptions.count(filename) == 0) {
-		TraceEvent("AddingExternalClient").detail("LibraryPath", filename);
-		externalClientDescriptions.emplace(std::make_pair(filename, ClientDesc(path, true)));
+		TraceEvent("AddingExternalClient").detail("LibraryPath", filename).detail("UseFutureVersion", useFutureVersion);
+		externalClientDescriptions.emplace(std::make_pair(filename, ClientDesc(path, true, useFutureVersion)));
 	}
 }
 
@@ -2044,7 +2301,7 @@ void MultiVersionApi::addExternalLibraryDirectory(std::string path) {
 		std::string lib = abspath(joinPath(path, filename));
 		if (externalClientDescriptions.count(filename) == 0) {
 			TraceEvent("AddingExternalClient").detail("LibraryPath", filename);
-			externalClientDescriptions.emplace(std::make_pair(filename, ClientDesc(lib, true)));
+			externalClientDescriptions.emplace(std::make_pair(filename, ClientDesc(lib, true, false)));
 		}
 	}
 }
@@ -2182,7 +2439,7 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 		setCallbacksOnExternalThreads();
 	} else if (option == FDBNetworkOptions::EXTERNAL_CLIENT_LIBRARY) {
 		validateOption(value, true, false, false);
-		addExternalLibrary(abspath(value.get().toString()));
+		addExternalLibrary(abspath(value.get().toString()), false);
 	} else if (option == FDBNetworkOptions::EXTERNAL_CLIENT_DIRECTORY) {
 		validateOption(value, true, false, false);
 		addExternalLibraryDirectory(value.get().toString());
@@ -2198,6 +2455,13 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 		externalClient = true;
 		bypassMultiClientApi = true;
 		forwardOption = true;
+	} else if (option == FDBNetworkOptions::DISABLE_CLIENT_BYPASS) {
+		MutexHolder holder(lock);
+		ASSERT(!networkStartSetup);
+		if (bypassMultiClientApi) {
+			throw invalid_option();
+		}
+		disableBypass = true;
 	} else if (option == FDBNetworkOptions::CLIENT_THREADS_PER_VERSION) {
 		MutexHolder holder(lock);
 		validateOption(value, true, false, false);
@@ -2213,6 +2477,21 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 	} else if (option == FDBNetworkOptions::CLIENT_TMP_DIR) {
 		validateOption(value, true, false, false);
 		tmpDir = abspath(value.get().toString());
+	} else if (option == FDBNetworkOptions::FUTURE_VERSION_CLIENT_LIBRARY) {
+		validateOption(value, true, false, false);
+		addExternalLibrary(abspath(value.get().toString()), true);
+	} else if (option == FDBNetworkOptions::TRACE_FILE_IDENTIFIER) {
+		validateOption(value, true, false, true);
+		traceFileIdentifier = value.get().toString();
+		{
+			MutexHolder holder(lock);
+			// Forward the option unmodified only to the the local client and let it validate it.
+			// While for external clients the trace file identifiers are determined in setupNetwork
+			localClient->api->setNetworkOption(option, value);
+		}
+	} else if (option == FDBNetworkOptions::TRACE_SHARE_AMONG_CLIENT_THREADS) {
+		validateOption(value, false, true);
+		traceShareBaseNameAmongThreads = true;
 	} else {
 		forwardOption = true;
 	}
@@ -2251,13 +2530,18 @@ void MultiVersionApi::setupNetwork() {
 		for (auto i : externalClientDescriptions) {
 			std::string path = i.second.libPath;
 			std::string filename = basename(path);
+			bool useFutureVersion = i.second.useFutureVersion;
 
 			// Copy external lib for each thread
 			if (externalClients.count(filename) == 0) {
 				externalClients[filename] = {};
-				for (const auto& tmp : copyExternalLibraryPerThread(path)) {
+				auto libCopies = copyExternalLibraryPerThread(path);
+				for (int idx = 0; idx < libCopies.size(); ++idx) {
 					externalClients[filename].push_back(Reference<ClientInfo>(
-					    new ClientInfo(new DLApi(tmp.first, tmp.second /*unlink on load*/), path)));
+					    new ClientInfo(new DLApi(libCopies[idx].first, libCopies[idx].second /*unlink on load*/),
+					                   path,
+					                   useFutureVersion,
+					                   idx)));
 				}
 			}
 		}
@@ -2276,7 +2560,7 @@ void MultiVersionApi::setupNetwork() {
 
 		networkStartSetup = true;
 
-		if (externalClients.empty()) {
+		if (externalClients.empty() && !disableBypass) {
 			bypassMultiClientApi = true; // SOMEDAY: we won't be able to set this option once it becomes possible to add
 			                             // clients after setupNetwork is called
 		}
@@ -2296,17 +2580,30 @@ void MultiVersionApi::setupNetwork() {
 	if (!bypassMultiClientApi) {
 		runOnExternalClientsAllThreads([this](Reference<ClientInfo> client) {
 			TraceEvent("InitializingExternalClient").detail("LibraryPath", client->libPath);
-			client->api->selectApiVersion(apiVersion);
+			client->api->selectApiVersion(apiVersion.version());
+			if (client->useFutureVersion) {
+				client->api->useFutureProtocolVersion();
+			}
 			client->loadVersion();
 		});
 
+		std::string baseTraceFileId;
+		if (apiVersion.hasTraceFileIdentifier()) {
+			// TRACE_FILE_IDENTIFIER option is supported since 6.3
+			baseTraceFileId = traceFileIdentifier.empty() ? format("%d", getpid()) : traceFileIdentifier;
+		}
+
 		MutexHolder holder(lock);
-		runOnExternalClientsAllThreads([this, transportId](Reference<ClientInfo> client) {
+		runOnExternalClientsAllThreads([this, transportId, baseTraceFileId](Reference<ClientInfo> client) {
 			for (auto option : options) {
 				client->api->setNetworkOption(option.first, option.second.castTo<StringRef>());
 			}
 			client->api->setNetworkOption(FDBNetworkOptions::EXTERNAL_CLIENT_TRANSPORT_ID, std::to_string(transportId));
-
+			if (!baseTraceFileId.empty()) {
+				client->api->setNetworkOption(
+				    FDBNetworkOptions::TRACE_FILE_IDENTIFIER,
+				    traceShareBaseNameAmongThreads ? baseTraceFileId : client->getTraceFileIdentifier(baseTraceFileId));
+			}
 			client->api->setupNetwork();
 		});
 
@@ -2345,21 +2642,17 @@ void MultiVersionApi::runNetwork() {
 
 	std::vector<THREAD_HANDLE> handles;
 	if (!bypassMultiClientApi) {
-		for (int threadNum = 0; threadNum < threadCount; threadNum++) {
-			runOnExternalClients(threadNum, [&handles, threadNum](Reference<ClientInfo> client) {
-				if (client->external) {
-					std::string threadName = format("fdb-%s-%d", client->releaseVersion.c_str(), threadNum);
-					if (threadName.size() > 15) {
-						threadName = format("fdb-%s", client->releaseVersion.c_str());
-						if (threadName.size() > 15) {
-							threadName = "fdb-external";
-						}
-					}
-					handles.push_back(
-					    g_network->startThread(&runNetworkThread, client.getPtr(), 0, threadName.c_str()));
+		runOnExternalClientsAllThreads([&handles](Reference<ClientInfo> client) {
+			ASSERT(client->external);
+			std::string threadName = format("fdb-%s-%d", client->releaseVersion.c_str(), client->threadIndex);
+			if (threadName.size() > 15) {
+				threadName = format("fdb-%s", client->releaseVersion.c_str());
+				if (threadName.size() > 15) {
+					threadName = "fdb-external";
 				}
-			});
-		}
+			}
+			handles.push_back(g_network->startThread(&runNetworkThread, client.getPtr(), 0, threadName.c_str()));
+		});
 	}
 
 	localClient->api->runNetwork();
@@ -2472,9 +2765,9 @@ ACTOR Future<std::string> updateClusterSharedStateMapImpl(MultiVersionApi* self,
                                                           ProtocolVersion dbProtocolVersion,
                                                           Reference<IDatabase> db) {
 	// The cluster ID will be the connection record string (either a filename or the connection string itself)
-	// in API versions before we could read the cluster ID.
+	// in versions before we could read the cluster ID.
 	state std::string clusterId = connectionRecord.toString();
-	if (MultiVersionApi::apiVersionAtLeast(720)) {
+	if (dbProtocolVersion.hasClusterIdSpecialKey()) {
 		state Reference<ITransaction> tr = db->createTransaction();
 		loop {
 			try {
@@ -2648,8 +2941,8 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() {
 
 MultiVersionApi::MultiVersionApi()
   : callbackOnMainThread(true), localClientDisabled(false), networkStartSetup(false), networkSetup(false),
-    bypassMultiClientApi(false), externalClient(false), apiVersion(0), threadCount(0), tmpDir("/tmp"),
-    envOptionsLoaded(false) {}
+    disableBypass(false), bypassMultiClientApi(false), externalClient(false), apiVersion(0), threadCount(0),
+    tmpDir("/tmp"), traceShareBaseNameAmongThreads(false), envOptionsLoaded(false) {}
 
 MultiVersionApi* MultiVersionApi::api = new MultiVersionApi();
 
@@ -2686,6 +2979,12 @@ bool ClientInfo::canReplace(Reference<ClientInfo> other) const {
 	return !protocolVersion.isCompatible(other->protocolVersion);
 }
 
+std::string ClientInfo::getTraceFileIdentifier(const std::string& baseIdentifier) {
+	std::string versionStr = releaseVersion;
+	std::replace(versionStr.begin(), versionStr.end(), '.', '_');
+	return format("%s_v%st%d", baseIdentifier.c_str(), versionStr.c_str(), threadIndex);
+}
+
 // UNIT TESTS
 TEST_CASE("/fdbclient/multiversionclient/EnvironmentVariableParsing") {
 	auto vals = parseOptionValues("a");
@@ -2971,7 +3270,7 @@ struct AbortableTest {
 	}
 };
 
-TEST_CASE("/fdbclient/multiversionclient/AbortableSingleAssignmentVar") {
+TEST_CASE("fdbclient/multiversionclient/AbortableSingleAssignmentVar") {
 	state volatile bool done = false;
 	state THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<AbortableTest>, (void*)&done);
 
@@ -3048,7 +3347,7 @@ struct DLTest {
 	}
 };
 
-TEST_CASE("/fdbclient/multiversionclient/DLSingleAssignmentVar") {
+TEST_CASE("fdbclient/multiversionclient/DLSingleAssignmentVar") {
 	state volatile bool done = false;
 
 	MultiVersionApi::api->callbackOnMainThread = true;
@@ -3092,7 +3391,7 @@ struct MapTest {
 	}
 };
 
-TEST_CASE("/fdbclient/multiversionclient/MapSingleAssignmentVar") {
+TEST_CASE("fdbclient/multiversionclient/MapSingleAssignmentVar") {
 	state volatile bool done = false;
 	state THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<MapTest>, (void*)&done);
 
@@ -3131,7 +3430,7 @@ struct FlatMapTest {
 	}
 };
 
-TEST_CASE("/fdbclient/multiversionclient/FlatMapSingleAssignmentVar") {
+TEST_CASE("fdbclient/multiversionclient/FlatMapSingleAssignmentVar") {
 	state volatile bool done = false;
 	state THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<FlatMapTest>, (void*)&done);
 
diff --git a/fdbclient/MutationLogReader.actor.cpp b/fdbclient/MutationLogReader.actor.cpp
index 5919fdc66b..d6e3adc8dd 100644
--- a/fdbclient/MutationLogReader.actor.cpp
+++ b/fdbclient/MutationLogReader.actor.cpp
@@ -67,7 +67,7 @@ ACTOR Future<Void> PipelinedReader::getNext_impl(PipelinedReader* self, Database
 	state Transaction tr(cx);
 
 	state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
-	                            (g_network->isSimulated() && !g_simulator.speedUpSimulation)
+	                            (g_network->isSimulated() && !g_simulator->speedUpSimulation)
 	                                ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
 	                                : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
 
@@ -179,7 +179,7 @@ ACTOR Future<Standalone<RangeResultRef>> MutationLogReader::getNext_impl(Mutatio
 namespace {
 // UNIT TESTS
 TEST_CASE("/fdbclient/mutationlogreader/VersionKeyRefConversion") {
-	Key prefix = LiteralStringRef("foos");
+	Key prefix = "foos"_sr;
 
 	ASSERT(keyRefToVersion(versionToKey(0, prefix), prefix.size()) == 0);
 	ASSERT(keyRefToVersion(versionToKey(1, prefix), prefix.size()) == 1);
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 9c2e3cb153..c390020b64 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -23,6 +23,7 @@
 #include <algorithm>
 #include <cstdio>
 #include <iterator>
+#include <limits>
 #include <memory>
 #include <regex>
 #include <unordered_set>
@@ -44,6 +45,7 @@
 #include "fdbclient/BlobGranuleCommon.h"
 #include "fdbclient/ClusterInterface.h"
 #include "fdbclient/ClusterConnectionFile.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/GlobalConfig.actor.h"
@@ -102,6 +104,8 @@
 #endif
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+FDB_DEFINE_BOOLEAN_PARAM(CacheResult);
+
 extern const char* getSourceVersion();
 
 namespace {
@@ -152,8 +156,8 @@ NetworkOptions::NetworkOptions()
     supportedVersions(new ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>()), runLoopProfilingEnabled(false),
     primaryClient(true) {}
 
-static const Key CLIENT_LATENCY_INFO_PREFIX = LiteralStringRef("client_latency/");
-static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = LiteralStringRef("client_latency_counter/");
+static const Key CLIENT_LATENCY_INFO_PREFIX = "client_latency/"_sr;
+static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = "client_latency_counter/"_sr;
 
 void DatabaseContext::addTssMapping(StorageServerInterface const& ssi, StorageServerInterface const& tssi) {
 	auto result = tssMapping.find(ssi.id());
@@ -167,14 +171,8 @@ void DatabaseContext::addTssMapping(StorageServerInterface const& ssi, StorageSe
 			tssMetrics[tssi.id()] = metrics;
 			tssMapping[ssi.id()] = tssi;
 		} else {
-			if (result->second.id() == tssi.id()) {
-				metrics = tssMetrics[tssi.id()];
-			} else {
-				TEST(true); // SS now maps to new TSS! This will probably never happen in practice
-				tssMetrics.erase(result->second.id());
-				metrics = makeReference<TSSMetrics>();
-				tssMetrics[tssi.id()] = metrics;
-			}
+			ASSERT(result->second.id() == tssi.id());
+			metrics = tssMetrics[tssi.id()];
 			result->second = tssi;
 		}
 
@@ -230,8 +228,9 @@ void DatabaseContext::getLatestCommitVersions(const Reference<LocationInfo>& loc
                                               VersionVector& latestCommitVersions) {
 	latestCommitVersions.clear();
 
-	if (info->debugID.present()) {
-		g_traceBatch.addEvent("TransactionDebug", info->debugID.get().first(), "NativeAPI.getLatestCommitVersions");
+	if (info->readOptions.present() && info->readOptions.get().debugID.present()) {
+		g_traceBatch.addEvent(
+		    "TransactionDebug", info->readOptions.get().debugID.get().first(), "NativeAPI.getLatestCommitVersions");
 	}
 
 	if (!info->readVersionObtainedFromGrvProxy) {
@@ -242,7 +241,7 @@ void DatabaseContext::getLatestCommitVersions(const Reference<LocationInfo>& loc
 		return;
 	}
 
-	if (ssVersionVectorCache.getMaxVersion() != invalidVersion && readVersion > ssVersionVectorCache.getMaxVersion()) {
+	if (readVersion > ssVersionVectorCache.getMaxVersion()) {
 		if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !info->options.skipGrvCache && info->options.useGrvCache) {
 			return;
 		} else {
@@ -255,16 +254,32 @@ void DatabaseContext::getLatestCommitVersions(const Reference<LocationInfo>& loc
 
 	std::map<Version, std::set<Tag>> versionMap; // order the versions to be returned
 	for (int i = 0; i < locationInfo->locations()->size(); i++) {
-		UID uid = locationInfo->locations()->getId(i);
-		if (ssidTagMapping.find(uid) != ssidTagMapping.end()) {
-			Tag tag = ssidTagMapping[uid];
+		bool updatedVersionMap = false;
+		Version commitVersion = invalidVersion;
+		Tag tag = invalidTag;
+		auto iter = ssidTagMapping.find(locationInfo->locations()->getId(i));
+		if (iter != ssidTagMapping.end()) {
+			tag = iter->second;
 			if (ssVersionVectorCache.hasVersion(tag)) {
-				Version commitVersion = ssVersionVectorCache.getVersion(tag); // latest commit version
+				commitVersion = ssVersionVectorCache.getVersion(tag); // latest commit version
 				if (commitVersion < readVersion) {
+					updatedVersionMap = true;
 					versionMap[commitVersion].insert(tag);
 				}
 			}
 		}
+		// Do not log if commitVersion >= readVersion.
+		if (!updatedVersionMap && commitVersion == invalidVersion) {
+			TraceEvent(SevDebug, "CommitVersionNotFoundForSS")
+			    .detail("InSSIDMap", iter != ssidTagMapping.end() ? 1 : 0)
+			    .detail("Tag", tag)
+			    .detail("CommitVersion", commitVersion)
+			    .detail("ReadVersion", readVersion)
+			    .detail("VersionVector", ssVersionVectorCache.toString())
+			    .setMaxEventLength(11000)
+			    .setMaxFieldLength(10000);
+			++transactionCommitVersionNotFoundForSS;
+		}
 	}
 
 	// insert the commit versions in the version vector.
@@ -444,7 +459,7 @@ void DatabaseContext::validateVersion(Version version) const {
 		throw client_invalid_operation();
 	}
 	if (switchable && version < minAcceptableReadVersion) {
-		TEST(true); // Attempted to read a version lower than any this client has seen from the current cluster
+		CODE_PROBE(true, "Attempted to read a version lower than any this client has seen from the current cluster");
 		throw transaction_too_old();
 	}
 
@@ -584,7 +599,8 @@ ACTOR Future<Void> databaseLogger(DatabaseContext* cx) {
 	loop {
 		wait(delay(CLIENT_KNOBS->SYSTEM_MONITOR_INTERVAL, TaskPriority::FlushTrace));
 
-		if (!g_network->isSimulated()) {
+		bool logTraces = !g_network->isSimulated() || BUGGIFY_WITH_PROB(0.01);
+		if (logTraces) {
 			TraceEvent ev("TransactionMetrics", cx->dbId);
 
 			ev.detail("Elapsed", (lastLogged == 0) ? 0 : now() - lastLogged)
@@ -637,6 +653,19 @@ ACTOR Future<Void> databaseLogger(DatabaseContext* cx) {
 		cx->bgLatencies.clear();
 		cx->bgGranulesPerRequest.clear();
 
+		if (cx->usedAnyChangeFeeds && logTraces) {
+			TraceEvent feedEv("ChangeFeedClientMetrics", cx->dbId);
+
+			feedEv.detail("Elapsed", (lastLogged == 0) ? 0 : now() - lastLogged)
+			    .detail("Cluster",
+			            cx->getConnectionRecord()
+			                ? cx->getConnectionRecord()->getConnectionString().clusterKeyName().toString()
+			                : "")
+			    .detail("Internal", cx->internal);
+
+			cx->ccFeed.logToTraceEvent(feedEv);
+		}
+
 		lastLogged = now();
 	}
 }
@@ -710,6 +739,7 @@ ACTOR static Future<Void> delExcessClntTxnEntriesActor(Transaction* tr, int64_t
 				tr->clear(KeyRangeRef(txEntries[0].key, strinc(endKey)));
 				TraceEvent(SevInfo, "DeletingExcessCntTxnEntries").detail("BytesToBeDeleted", numBytesToDel);
 				int64_t bytesDel = -numBytesToDel;
+
 				tr->atomicOp(clientLatencyAtomicCtr, StringRef((uint8_t*)&bytesDel, 8), MutationRef::AddValue);
 				wait(tr->commit());
 			}
@@ -1114,8 +1144,8 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 			state bool quarantine = CLIENT_KNOBS->QUARANTINE_TSS_ON_MISMATCH;
 			TraceEvent(SevWarnAlways, quarantine ? "TSS_QuarantineMismatch" : "TSS_KillMismatch")
 			    .detail("TSSID", data.first.toString());
-			TEST(quarantine); // Quarantining TSS because it got mismatch
-			TEST(!quarantine); // Killing TSS because it got mismatch
+			CODE_PROBE(quarantine, "Quarantining TSS because it got mismatch");
+			CODE_PROBE(!quarantine, "Killing TSS because it got mismatch");
 
 			tr = makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(cx)));
 			state int tries = 0;
@@ -1124,7 +1154,7 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 					tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					if (quarantine) {
-						tr->set(tssQuarantineKeyFor(data.first), LiteralStringRef(""));
+						tr->set(tssQuarantineKeyFor(data.first), ""_sr);
 					} else {
 						tr->clear(serverTagKeyFor(data.first));
 					}
@@ -1132,10 +1162,9 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 
 					for (const DetailedTSSMismatch& d : data.second) {
 						// <tssid, time, mismatchid> -> mismatch data
-						tssMismatchDB.set(
-						    tr,
-						    Tuple().append(data.first.toString()).append(d.timestamp).append(d.mismatchId.toString()),
-						    d.traceString);
+						tssMismatchDB.set(tr,
+						                  Tuple::makeTuple(data.first.toString(), d.timestamp, d.mismatchId.toString()),
+						                  d.traceString);
 					}
 
 					wait(tr->commit());
@@ -1154,7 +1183,7 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 			// clear out txn so that the extra DatabaseContext ref gets decref'd and we can free cx
 			tr = makeReference<ReadYourWritesTransaction>();
 		} else {
-			TEST(true); // Not handling TSS with mismatch because it's already gone
+			CODE_PROBE(true, "Not handling TSS with mismatch because it's already gone");
 		}
 	}
 }
@@ -1253,7 +1282,7 @@ void DatabaseContext::registerSpecialKeysImpl(SpecialKeySpace::MODULE module,
                                               std::unique_ptr<SpecialKeyRangeReadImpl>&& impl,
                                               int deprecatedVersion) {
 	// if deprecated, add the implementation when the api version is less than the deprecated version
-	if (deprecatedVersion == -1 || apiVersion < deprecatedVersion) {
+	if (deprecatedVersion == -1 || apiVersion.version() < deprecatedVersion) {
 		specialKeySpace->registerKeyRange(module, type, impl->getKeyRange(), impl.get());
 		specialKeySpaceModules.push_back(std::move(impl));
 	}
@@ -1262,32 +1291,6 @@ void DatabaseContext::registerSpecialKeysImpl(SpecialKeySpace::MODULE module,
 ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> clusterRecord);
 ACTOR Future<Optional<Value>> getJSON(Database db);
 
-struct WorkerInterfacesSpecialKeyImpl : SpecialKeyRangeReadImpl {
-	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
-	                             KeyRangeRef kr,
-	                             GetRangeLimits limitsHint) const override {
-		if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
-			Key prefix = Key(getKeyRange().begin);
-			return map(getWorkerInterfaces(ryw->getDatabase()->getConnectionRecord()),
-			           [prefix = prefix, kr = KeyRange(kr)](const RangeResult& in) {
-				           RangeResult result;
-				           for (const auto& [k_, v] : in) {
-					           auto k = k_.withPrefix(prefix);
-					           if (kr.contains(k))
-						           result.push_back_deep(result.arena(), KeyValueRef(k, v));
-				           }
-
-				           std::sort(result.begin(), result.end(), KeyValueRef::OrderByKey{});
-				           return result;
-			           });
-		} else {
-			return RangeResult();
-		}
-	}
-
-	explicit WorkerInterfacesSpecialKeyImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
-};
-
 struct SingleSpecialKeyImpl : SpecialKeyRangeReadImpl {
 	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
 	                             KeyRangeRef kr,
@@ -1302,12 +1305,17 @@ struct SingleSpecialKeyImpl : SpecialKeyRangeReadImpl {
 		});
 	}
 
-	SingleSpecialKeyImpl(KeyRef k, const std::function<Future<Optional<Value>>(ReadYourWritesTransaction*)>& f)
-	  : SpecialKeyRangeReadImpl(singleKeyRange(k)), k(k), f(f) {}
+	SingleSpecialKeyImpl(KeyRef k,
+	                     const std::function<Future<Optional<Value>>(ReadYourWritesTransaction*)>& f,
+	                     bool supportsTenants = false)
+	  : SpecialKeyRangeReadImpl(singleKeyRange(k)), k(k), f(f), tenantSupport(supportsTenants) {}
+
+	bool supportsTenants() const override { return tenantSupport; };
 
 private:
 	Key k;
 	std::function<Future<Optional<Value>>(ReadYourWritesTransaction*)> f;
+	bool tenantSupport;
 };
 
 class HealthMetricsRangeImpl : public SpecialKeyRangeAsyncImpl {
@@ -1322,7 +1330,7 @@ static RangeResult healthMetricsToKVPairs(const HealthMetrics& metrics, KeyRange
 	RangeResult result;
 	if (CLIENT_BUGGIFY)
 		return result;
-	if (kr.contains(LiteralStringRef("\xff\xff/metrics/health/aggregate")) && metrics.worstStorageDurabilityLag != 0) {
+	if (kr.contains("\xff\xff/metrics/health/aggregate"_sr) && metrics.worstStorageDurabilityLag != 0) {
 		json_spirit::mObject statsObj;
 		statsObj["batch_limited"] = metrics.batchLimited;
 		statsObj["tps_limit"] = metrics.tpsLimit;
@@ -1334,15 +1342,13 @@ static RangeResult healthMetricsToKVPairs(const HealthMetrics& metrics, KeyRange
 		std::string statsString =
 		    json_spirit::write_string(json_spirit::mValue(statsObj), json_spirit::Output_options::raw_utf8);
 		ValueRef bytes(result.arena(), statsString);
-		result.push_back(result.arena(), KeyValueRef(LiteralStringRef("\xff\xff/metrics/health/aggregate"), bytes));
+		result.push_back(result.arena(), KeyValueRef("\xff\xff/metrics/health/aggregate"_sr, bytes));
 	}
 	// tlog stats
 	{
 		int phase = 0; // Avoid comparing twice per loop iteration
 		for (const auto& [uid, logStats] : metrics.tLogQueue) {
-			StringRef k{
-				StringRef(uid.toString()).withPrefix(LiteralStringRef("\xff\xff/metrics/health/log/"), result.arena())
-			};
+			StringRef k{ StringRef(uid.toString()).withPrefix("\xff\xff/metrics/health/log/"_sr, result.arena()) };
 			if (phase == 0 && k >= kr.begin) {
 				phase = 1;
 			}
@@ -1364,8 +1370,7 @@ static RangeResult healthMetricsToKVPairs(const HealthMetrics& metrics, KeyRange
 	{
 		int phase = 0; // Avoid comparing twice per loop iteration
 		for (const auto& [uid, storageStats] : metrics.storageStats) {
-			StringRef k{ StringRef(uid.toString())
-				             .withPrefix(LiteralStringRef("\xff\xff/metrics/health/storage/"), result.arena()) };
+			StringRef k{ StringRef(uid.toString()).withPrefix("\xff\xff/metrics/health/storage/"_sr, result.arena()) };
 			if (phase == 0 && k >= kr.begin) {
 				phase = 1;
 			}
@@ -1391,10 +1396,9 @@ static RangeResult healthMetricsToKVPairs(const HealthMetrics& metrics, KeyRange
 
 ACTOR static Future<RangeResult> healthMetricsGetRangeActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
 	HealthMetrics metrics = wait(ryw->getDatabase()->getHealthMetrics(
-	    /*detailed ("per process")*/ kr.intersects(KeyRangeRef(LiteralStringRef("\xff\xff/metrics/health/storage/"),
-	                                                           LiteralStringRef("\xff\xff/metrics/health/storage0"))) ||
-	    kr.intersects(KeyRangeRef(LiteralStringRef("\xff\xff/metrics/health/log/"),
-	                              LiteralStringRef("\xff\xff/metrics/health/log0")))));
+	    /*detailed ("per process")*/ kr.intersects(
+	        KeyRangeRef("\xff\xff/metrics/health/storage/"_sr, "\xff\xff/metrics/health/storage0"_sr)) ||
+	    kr.intersects(KeyRangeRef("\xff\xff/metrics/health/log/"_sr, "\xff\xff/metrics/health/log0"_sr))));
 	return healthMetricsToKVPairs(metrics, kr);
 }
 
@@ -1432,7 +1436,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
                                  EnableLocalityLoadBalance enableLocalityLoadBalance,
                                  LockAware lockAware,
                                  IsInternal internal,
-                                 int apiVersion,
+                                 int _apiVersion,
                                  IsSwitchable switchable,
                                  Optional<TenantName> defaultTenant)
   : lockAware(lockAware), switchable(switchable), connectionRecord(connectionRecord), proxyProvisional(false),
@@ -1467,16 +1471,24 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
     transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
     transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
-    latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
-    bytesPerCommit(1000), bgLatencies(1000), bgGranulesPerRequest(1000), outstandingWatches(0), sharedStatePtr(nullptr),
-    lastGrvTime(0.0), cachedReadVersion(0), lastRkBatchThrottleTime(0.0), lastRkDefaultThrottleTime(0.0),
-    lastProxyRequestTime(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
-    clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
-    healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
-    smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
+    transactionCommitVersionNotFoundForSS("CommitVersionNotFoundForSS", cc), bgReadInputBytes("BGReadInputBytes", cc),
+    bgReadOutputBytes("BGReadOutputBytes", cc), usedAnyChangeFeeds(false), ccFeed("ChangeFeedClientMetrics"),
+    feedStreamStarts("FeedStreamStarts", ccFeed), feedMergeStreamStarts("FeedMergeStreamStarts", ccFeed),
+    feedErrors("FeedErrors", ccFeed), feedNonRetriableErrors("FeedNonRetriableErrors", ccFeed),
+    feedPops("FeedPops", ccFeed), feedPopsFallback("FeedPopsFallback", ccFeed), latencies(1000), readLatencies(1000),
+    commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), bgLatencies(1000),
+    bgGranulesPerRequest(1000), outstandingWatches(0), sharedStatePtr(nullptr), lastGrvTime(0.0), cachedReadVersion(0),
+    lastRkBatchThrottleTime(0.0), lastRkDefaultThrottleTime(0.0), lastProxyRequestTime(0.0),
+    transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor),
+    coordinator(coordinator), apiVersion(_apiVersion), mvCacheInsertLocation(0), healthMetricsLastUpdated(0),
+    detailedHealthMetricsLastUpdated(0), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)),
     connectToDatabaseEventCacheHolder(format("ConnectToDatabase/%s", dbId.toString().c_str())) {
+
 	dbId = deterministicRandom()->randomUniqueID();
+
+	TraceEvent("DatabaseContextCreated", dbId).backtrace();
+
 	connected = (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size())
 	                ? Void()
 	                : clientInfo->onChange();
@@ -1484,7 +1496,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 	metadataVersionCache.resize(CLIENT_KNOBS->METADATA_VERSION_CACHE_SIZE);
 	maxOutstandingWatches = CLIENT_KNOBS->DEFAULT_MAX_OUTSTANDING_WATCHES;
 
-	snapshotRywEnabled = apiVersionAtLeast(300) ? 1 : 0;
+	snapshotRywEnabled = apiVersion.hasSnapshotRYW() ? 1 : 0;
 
 	logger = databaseLogger(this) && tssLogger(this);
 	locationCacheSize = g_network->isSimulated() ? CLIENT_KNOBS->LOCATION_CACHE_EVICTION_SIZE_SIM
@@ -1492,8 +1504,8 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 	tenantCacheSize = g_network->isSimulated() ? CLIENT_KNOBS->TENANT_CACHE_EVICTION_SIZE_SIM
 	                                           : CLIENT_KNOBS->TENANT_CACHE_EVICTION_SIZE;
 
-	getValueSubmitted.init(LiteralStringRef("NativeAPI.GetValueSubmitted"));
-	getValueCompleted.init(LiteralStringRef("NativeAPI.GetValueCompleted"));
+	getValueSubmitted.init("NativeAPI.GetValueSubmitted"_sr);
+	getValueCompleted.init("NativeAPI.GetValueCompleted"_sr);
 
 	clientDBInfoMonitor = monitorClientDBInfoChange(this, clientInfo, &proxiesChangeTrigger);
 	tssMismatchHandler = handleTssMismatches(this);
@@ -1503,34 +1515,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 	smoothMidShardSize.reset(CLIENT_KNOBS->INIT_MID_SHARD_BYTES);
 	globalConfig = std::make_unique<GlobalConfig>(this);
 
-	if (apiVersionAtLeast(720)) {
-		registerSpecialKeysImpl(
-		    SpecialKeySpace::MODULE::CLUSTERID,
-		    SpecialKeySpace::IMPLTYPE::READONLY,
-		    std::make_unique<SingleSpecialKeyImpl>(
-		        LiteralStringRef("\xff\xff/cluster_id"), [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
-			        try {
-				        if (ryw->getDatabase().getPtr()) {
-					        return map(getClusterId(ryw->getDatabase()),
-					                   [](UID id) { return Optional<Value>(StringRef(id.toString())); });
-				        }
-			        } catch (Error& e) {
-				        return e;
-			        }
-			        return Optional<Value>();
-		        }));
-		registerSpecialKeysImpl(
-		    SpecialKeySpace::MODULE::MANAGEMENT,
-		    SpecialKeySpace::IMPLTYPE::READWRITE,
-		    std::make_unique<TenantRangeImpl<true>>(SpecialKeySpace::getManagementApiCommandRange("tenant")));
-	}
-	if (apiVersionAtLeast(710) && !apiVersionAtLeast(720)) {
-		registerSpecialKeysImpl(
-		    SpecialKeySpace::MODULE::MANAGEMENT,
-		    SpecialKeySpace::IMPLTYPE::READWRITE,
-		    std::make_unique<TenantRangeImpl<false>>(SpecialKeySpace::getManagementApiCommandRange("tenantmap")));
-	}
-	if (apiVersionAtLeast(700)) {
+	if (apiVersion.version() >= 700) {
 		registerSpecialKeysImpl(SpecialKeySpace::MODULE::ERRORMSG,
 		                        SpecialKeySpace::IMPLTYPE::READONLY,
 		                        std::make_unique<SingleSpecialKeyImpl>(
@@ -1540,12 +1525,13 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 				                            return Optional<Value>(ryw->getSpecialKeySpaceErrorMsg().get());
 			                            else
 				                            return Optional<Value>();
-		                            }));
+		                            },
+		                            true));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<ManagementCommandsOptionsImpl>(
-		        KeyRangeRef(LiteralStringRef("options/"), LiteralStringRef("options0"))
+		        KeyRangeRef("options/"_sr, "options0"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
@@ -1567,31 +1553,31 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<ExclusionInProgressRangeImpl>(
-		        KeyRangeRef(LiteralStringRef("in_progress_exclusion/"), LiteralStringRef("in_progress_exclusion0"))
+		        KeyRangeRef("in_progress_exclusion/"_sr, "in_progress_exclusion0"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::CONFIGURATION,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<ProcessClassRangeImpl>(
-		        KeyRangeRef(LiteralStringRef("process/class_type/"), LiteralStringRef("process/class_type0"))
+		        KeyRangeRef("process/class_type/"_sr, "process/class_type0"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::CONFIGURATION,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<ProcessClassSourceRangeImpl>(
-		        KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0"))
+		        KeyRangeRef("process/class_source/"_sr, "process/class_source0"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<LockDatabaseImpl>(
-		        singleKeyRange(LiteralStringRef("db_locked"))
+		        singleKeyRange("db_locked"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<ConsistencyCheckImpl>(
-		        singleKeyRange(LiteralStringRef("consistency_check_suspended"))
+		        singleKeyRange("consistency_check_suspended"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::GLOBALCONFIG,
@@ -1605,44 +1591,44 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 		    SpecialKeySpace::MODULE::CONFIGURATION,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<CoordinatorsImpl>(
-		        KeyRangeRef(LiteralStringRef("coordinators/"), LiteralStringRef("coordinators0"))
+		        KeyRangeRef("coordinators/"_sr, "coordinators0"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<CoordinatorsAutoImpl>(
-		        singleKeyRange(LiteralStringRef("auto_coordinators"))
+		        singleKeyRange("auto_coordinators"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<AdvanceVersionImpl>(
-		        singleKeyRange(LiteralStringRef("min_required_commit_version"))
+		        singleKeyRange("min_required_commit_version"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<VersionEpochImpl>(
-		        singleKeyRange(LiteralStringRef("version_epoch"))
+		        singleKeyRange("version_epoch"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<ClientProfilingImpl>(
-		        KeyRangeRef(LiteralStringRef("profiling/"), LiteralStringRef("profiling0"))
+		        KeyRangeRef("profiling/"_sr, "profiling0"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)),
 		    /* deprecated */ 720);
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<MaintenanceImpl>(
-		        KeyRangeRef(LiteralStringRef("maintenance/"), LiteralStringRef("maintenance0"))
+		        KeyRangeRef("maintenance/"_sr, "maintenance0"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::MANAGEMENT,
 		    SpecialKeySpace::IMPLTYPE::READWRITE,
 		    std::make_unique<DataDistributionImpl>(
-		        KeyRangeRef(LiteralStringRef("data_distribution/"), LiteralStringRef("data_distribution0"))
+		        KeyRangeRef("data_distribution/"_sr, "data_distribution0"_sr)
 		            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::ACTORLINEAGE,
@@ -1653,7 +1639,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 		                        std::make_unique<ActorProfilerConf>(
 		                            SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ACTOR_PROFILER_CONF)));
 	}
-	if (apiVersionAtLeast(630)) {
+	if (apiVersion.version() >= 630) {
 		registerSpecialKeysImpl(SpecialKeySpace::MODULE::TRANSACTION,
 		                        SpecialKeySpace::IMPLTYPE::READONLY,
 		                        std::make_unique<ConflictingKeysImpl>(conflictingKeysRange));
@@ -1666,20 +1652,18 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 		registerSpecialKeysImpl(SpecialKeySpace::MODULE::METRICS,
 		                        SpecialKeySpace::IMPLTYPE::READONLY,
 		                        std::make_unique<DDStatsRangeImpl>(ddStatsRange));
-		registerSpecialKeysImpl(
-		    SpecialKeySpace::MODULE::METRICS,
-		    SpecialKeySpace::IMPLTYPE::READONLY,
-		    std::make_unique<HealthMetricsRangeImpl>(KeyRangeRef(LiteralStringRef("\xff\xff/metrics/health/"),
-		                                                         LiteralStringRef("\xff\xff/metrics/health0"))));
-		registerSpecialKeysImpl(
-		    SpecialKeySpace::MODULE::WORKERINTERFACE,
-		    SpecialKeySpace::IMPLTYPE::READONLY,
-		    std::make_unique<WorkerInterfacesSpecialKeyImpl>(KeyRangeRef(
-		        LiteralStringRef("\xff\xff/worker_interfaces/"), LiteralStringRef("\xff\xff/worker_interfaces0"))));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::METRICS,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<HealthMetricsRangeImpl>(
+		                            KeyRangeRef("\xff\xff/metrics/health/"_sr, "\xff\xff/metrics/health0"_sr)));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::WORKERINTERFACE,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<WorkerInterfacesSpecialKeyImpl>(
+		                            KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr)));
 		registerSpecialKeysImpl(SpecialKeySpace::MODULE::STATUSJSON,
 		                        SpecialKeySpace::IMPLTYPE::READONLY,
 		                        std::make_unique<SingleSpecialKeyImpl>(
-		                            LiteralStringRef("\xff\xff/status/json"),
+		                            "\xff\xff/status/json"_sr,
 		                            [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
 			                            if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
 				                            ++ryw->getDatabase()->transactionStatusRequests;
@@ -1687,11 +1671,12 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 			                            } else {
 				                            return Optional<Value>();
 			                            }
-		                            }));
+		                            },
+		                            true));
 		registerSpecialKeysImpl(SpecialKeySpace::MODULE::CLUSTERFILEPATH,
 		                        SpecialKeySpace::IMPLTYPE::READONLY,
 		                        std::make_unique<SingleSpecialKeyImpl>(
-		                            LiteralStringRef("\xff\xff/cluster_file_path"),
+		                            "\xff\xff/cluster_file_path"_sr,
 		                            [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
 			                            try {
 				                            if (ryw->getDatabase().getPtr() &&
@@ -1704,13 +1689,14 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 				                            return e;
 			                            }
 			                            return Optional<Value>();
-		                            }));
+		                            },
+		                            true));
 
 		registerSpecialKeysImpl(
 		    SpecialKeySpace::MODULE::CONNECTIONSTRING,
 		    SpecialKeySpace::IMPLTYPE::READONLY,
 		    std::make_unique<SingleSpecialKeyImpl>(
-		        LiteralStringRef("\xff\xff/connection_string"),
+		        "\xff\xff/connection_string"_sr,
 		        [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
 			        try {
 				        if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
@@ -1722,7 +1708,30 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 				        return e;
 			        }
 			        return Optional<Value>();
-		        }));
+		        },
+		        true));
+		registerSpecialKeysImpl(SpecialKeySpace::MODULE::CLUSTERID,
+		                        SpecialKeySpace::IMPLTYPE::READONLY,
+		                        std::make_unique<SingleSpecialKeyImpl>(
+		                            "\xff\xff/cluster_id"_sr,
+		                            [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
+			                            try {
+				                            if (ryw->getDatabase().getPtr()) {
+					                            return map(getClusterId(ryw->getDatabase()), [](UID id) {
+						                            return Optional<Value>(StringRef(id.toString()));
+					                            });
+				                            }
+			                            } catch (Error& e) {
+				                            return e;
+			                            }
+			                            return Optional<Value>();
+		                            },
+		                            true));
+
+		registerSpecialKeysImpl(
+		    SpecialKeySpace::MODULE::MANAGEMENT,
+		    SpecialKeySpace::IMPLTYPE::READWRITE,
+		    std::make_unique<TenantRangeImpl>(SpecialKeySpace::getManagementApiCommandRange("tenant")));
 	}
 	throttleExpirer = recurring([this]() { expireThrottles(); }, CLIENT_KNOBS->TAG_THROTTLE_EXPIRATION_INTERVAL);
 
@@ -1762,8 +1771,13 @@ DatabaseContext::DatabaseContext(const Error& err)
     transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
     transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
     transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
-    latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
-    bytesPerCommit(1000), bgLatencies(1000), bgGranulesPerRequest(1000), transactionTracingSample(false),
+    transactionCommitVersionNotFoundForSS("CommitVersionNotFoundForSS", cc), bgReadInputBytes("BGReadInputBytes", cc),
+    bgReadOutputBytes("BGReadOutputBytes", cc), usedAnyChangeFeeds(false), ccFeed("ChangeFeedClientMetrics"),
+    feedStreamStarts("FeedStreamStarts", ccFeed), feedMergeStreamStarts("FeedMergeStreamStarts", ccFeed),
+    feedErrors("FeedErrors", ccFeed), feedNonRetriableErrors("FeedNonRetriableErrors", ccFeed),
+    feedPops("FeedPops", ccFeed), feedPopsFallback("FeedPopsFallback", ccFeed), latencies(1000), readLatencies(1000),
+    commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), bgLatencies(1000),
+    bgGranulesPerRequest(1000), sharedStatePtr(nullptr), transactionTracingSample(false),
     smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     connectToDatabaseEventCacheHolder(format("ConnectToDatabase/%s", dbId.toString().c_str())) {}
 
@@ -1805,9 +1819,17 @@ DatabaseContext::~DatabaseContext() {
 		it->second->notifyContextDestroyed();
 	ASSERT_ABORT(server_interf.empty());
 	locationCache.insert(allKeys, Reference<LocationInfo>());
+	for (auto& it : notAtLatestChangeFeeds) {
+		it.second->context = nullptr;
+	}
+	for (auto& it : changeFeedUpdaters) {
+		it.second->context = nullptr;
+	}
+
+	TraceEvent("DatabaseContextDestructed", dbId).backtrace();
 }
 
-Optional<KeyRangeLocationInfo> DatabaseContext::getCachedLocation(const Optional<TenantName>& tenantName,
+Optional<KeyRangeLocationInfo> DatabaseContext::getCachedLocation(const Optional<TenantNameRef>& tenantName,
                                                                   const KeyRef& key,
                                                                   Reverse isBackward) {
 	TenantMapEntry tenantEntry;
@@ -1833,7 +1855,7 @@ Optional<KeyRangeLocationInfo> DatabaseContext::getCachedLocation(const Optional
 	return Optional<KeyRangeLocationInfo>();
 }
 
-bool DatabaseContext::getCachedLocations(const Optional<TenantName>& tenantName,
+bool DatabaseContext::getCachedLocations(const Optional<TenantNameRef>& tenantName,
                                          const KeyRangeRef& range,
                                          std::vector<KeyRangeLocationInfo>& result,
                                          int limit,
@@ -1860,7 +1882,7 @@ bool DatabaseContext::getCachedLocations(const Optional<TenantName>& tenantName,
 	loop {
 		auto r = reverse ? end : begin;
 		if (!r->value()) {
-			TEST(result.size()); // had some but not all cached locations
+			CODE_PROBE(result.size(), "had some but not all cached locations");
 			result.clear();
 			return false;
 		}
@@ -1890,7 +1912,7 @@ void DatabaseContext::cacheTenant(const TenantName& tenant, const TenantMapEntry
 	}
 }
 
-Reference<LocationInfo> DatabaseContext::setCachedLocation(const Optional<TenantName>& tenant,
+Reference<LocationInfo> DatabaseContext::setCachedLocation(const Optional<TenantNameRef>& tenant,
                                                            const TenantMapEntry& tenantEntry,
                                                            const KeyRangeRef& absoluteKeys,
                                                            const std::vector<StorageServerInterface>& servers) {
@@ -1907,7 +1929,7 @@ Reference<LocationInfo> DatabaseContext::setCachedLocation(const Optional<Tenant
 	int maxEvictionAttempts = 100, attempts = 0;
 	auto loc = makeReference<LocationInfo>(serverRefs);
 	while (locationCache.size() > locationCacheSize && attempts < maxEvictionAttempts) {
-		TEST(true); // NativeAPI storage server locationCache entry evicted
+		CODE_PROBE(true, "NativeAPI storage server locationCache entry evicted");
 		attempts++;
 		auto r = locationCache.randomRange();
 		Key begin = r.begin(), end = r.end(); // insert invalidates r, so can't be passed a mere reference into it
@@ -2091,7 +2113,7 @@ Future<Void> DatabaseContext::onConnected() {
 
 ACTOR static Future<Void> switchConnectionRecordImpl(Reference<IClusterConnectionRecord> connRecord,
                                                      DatabaseContext* self) {
-	TEST(true); // Switch connection file
+	CODE_PROBE(true, "Switch connection file");
 	TraceEvent("SwitchConnectionRecord")
 	    .detail("ClusterFile", connRecord->toString())
 	    .detail("ConnectionString", connRecord->getConnectionString().toString());
@@ -2152,7 +2174,7 @@ void DatabaseContext::expireThrottles() {
 	for (auto& priorityItr : throttledTags) {
 		for (auto tagItr = priorityItr.second.begin(); tagItr != priorityItr.second.end();) {
 			if (tagItr->second.expired()) {
-				TEST(true); // Expiring client throttle
+				CODE_PROBE(true, "Expiring client throttle");
 				tagItr = priorityItr.second.erase(tagItr);
 			} else {
 				++tagItr;
@@ -2285,6 +2307,13 @@ Database Database::createDatabase(std::string connFileName,
 	return Database::createDatabase(rccr, apiVersion, internal, clientLocality);
 }
 
+Database Database::createSimulatedExtraDatabase(std::string connectionString, Optional<TenantName> defaultTenant) {
+	auto extraFile = makeReference<ClusterConnectionMemoryRecord>(ClusterConnectionString(connectionString));
+	Database db = Database::createDatabase(extraFile, ApiVersion::LATEST_VERSION);
+	db->defaultTenant = defaultTenant;
+	return db;
+}
+
 Reference<WatchMetadata> DatabaseContext::getWatchMetadata(int64_t tenantId, KeyRef key) const {
 	const auto it = watchMap.find(std::make_pair(tenantId, key));
 	if (it == watchMap.end())
@@ -2458,7 +2487,7 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
 		ASSERT(value.present());
 
 		Standalone<VectorRef<ClientVersionRef>> supportedVersions;
-		std::vector<StringRef> supportedVersionsStrings = value.get().splitAny(LiteralStringRef(";"));
+		std::vector<StringRef> supportedVersionsStrings = value.get().splitAny(";"_sr);
 		for (StringRef versionString : supportedVersionsStrings) {
 #ifdef ADDRESS_SANITIZER
 			__lsan_disable();
@@ -2638,7 +2667,7 @@ bool DatabaseContext::isCurrentGrvProxy(UID proxyId) const {
 		if (proxy.id() == proxyId)
 			return true;
 	}
-	TEST(true); // stale GRV proxy detected
+	CODE_PROBE(true, "stale GRV proxy detected");
 	return false;
 }
 
@@ -2831,7 +2860,7 @@ void updateTagMappings(Database cx, const GetKeyServerLocationsReply& reply) {
 // If isBackward == true, returns the shard containing the key before 'key' (an infinitely long, inexpressible key).
 // Otherwise returns the shard containing key
 ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
-                                                           Optional<TenantName> tenant,
+                                                           TenantInfo tenant,
                                                            Key key,
                                                            SpanContext spanContext,
                                                            Optional<UID> debugID,
@@ -2854,27 +2883,22 @@ ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
 			++cx->transactionKeyServerLocationRequests;
 			choose {
 				when(wait(cx->onProxiesChanged())) {}
-				when(GetKeyServerLocationsReply rep =
-				         wait(basicLoadBalance(cx->getCommitProxies(useProvisionalProxies),
-				                               &CommitProxyInterface::getKeyServersLocations,
-				                               GetKeyServerLocationsRequest(span.context,
-				                                                            tenant.castTo<TenantNameRef>(),
-				                                                            key,
-				                                                            Optional<KeyRef>(),
-				                                                            100,
-				                                                            isBackward,
-				                                                            version,
-				                                                            key.arena()),
-				                               TaskPriority::DefaultPromiseEndpoint))) {
+				when(GetKeyServerLocationsReply rep = wait(basicLoadBalance(
+				         cx->getCommitProxies(useProvisionalProxies),
+				         &CommitProxyInterface::getKeyServersLocations,
+				         GetKeyServerLocationsRequest(
+				             span.context, tenant, key, Optional<KeyRef>(), 100, isBackward, version, key.arena()),
+				         TaskPriority::DefaultPromiseEndpoint))) {
 					++cx->transactionKeyServerLocationRequestsCompleted;
 					if (debugID.present())
 						g_traceBatch.addEvent(
 						    "TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocation.After");
 					ASSERT(rep.results.size() == 1);
 
-					auto locationInfo =
-					    cx->setCachedLocation(tenant, rep.tenantEntry, rep.results[0].first, rep.results[0].second);
+					auto locationInfo = cx->setCachedLocation(
+					    tenant.name, rep.tenantEntry, rep.results[0].first, rep.results[0].second);
 					updateTssMappings(cx, rep);
+					updateTagMappings(cx, rep);
 
 					return KeyRangeLocationInfo(
 					    rep.tenantEntry,
@@ -2885,8 +2909,8 @@ ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_tenant_not_found) {
-			ASSERT(tenant.present());
-			cx->invalidateCachedTenant(tenant.get());
+			ASSERT(tenant.name.present());
+			cx->invalidateCachedTenant(tenant.name.get());
 		}
 
 		throw;
@@ -2924,7 +2948,7 @@ bool checkOnlyEndpointFailed(const Database& cx, const Endpoint& endpoint) {
 
 template <class F>
 Future<KeyRangeLocationInfo> getKeyLocation(Database const& cx,
-                                            Optional<TenantName> const& tenant,
+                                            TenantInfo const& tenant,
                                             Key const& key,
                                             F StorageServerInterface::*member,
                                             SpanContext spanContext,
@@ -2933,7 +2957,7 @@ Future<KeyRangeLocationInfo> getKeyLocation(Database const& cx,
                                             Reverse isBackward,
                                             Version version) {
 	// we first check whether this range is cached
-	Optional<KeyRangeLocationInfo> locationInfo = cx->getCachedLocation(tenant, key, isBackward);
+	Optional<KeyRangeLocationInfo> locationInfo = cx->getCachedLocation(tenant.name, key, isBackward);
 	if (!locationInfo.present()) {
 		return getKeyLocation_internal(
 		    cx, tenant, key, spanContext, debugID, useProvisionalProxies, isBackward, version);
@@ -2965,18 +2989,18 @@ Future<KeyRangeLocationInfo> getKeyLocation(Reference<TransactionState> trState,
                                             UseTenant useTenant,
                                             Version version) {
 	auto f = getKeyLocation(trState->cx,
-	                        useTenant ? trState->tenant() : Optional<TenantName>(),
+	                        useTenant ? trState->getTenantInfo(AllowInvalidTenantID::True) : TenantInfo(),
 	                        key,
 	                        member,
 	                        trState->spanContext,
-	                        trState->debugID,
+	                        trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
 	                        trState->useProvisionalProxies,
 	                        isBackward,
 	                        version);
 
-	if (trState->tenant().present() && useTenant) {
+	if (trState->tenant().present() && useTenant && trState->tenantId() == TenantInfo::INVALID_TENANT) {
 		return map(f, [trState](const KeyRangeLocationInfo& locationInfo) {
-			trState->tenantId = locationInfo.tenantEntry.id;
+			trState->trySetTenantId(locationInfo.tenantEntry.id);
 			return locationInfo;
 		});
 	} else {
@@ -2986,7 +3010,7 @@ Future<KeyRangeLocationInfo> getKeyLocation(Reference<TransactionState> trState,
 
 ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
     Database cx,
-    Optional<TenantName> tenant,
+    TenantInfo tenant,
     KeyRange keys,
     int limit,
     Reverse reverse,
@@ -3003,18 +3027,12 @@ ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
 			++cx->transactionKeyServerLocationRequests;
 			choose {
 				when(wait(cx->onProxiesChanged())) {}
-				when(GetKeyServerLocationsReply _rep =
-				         wait(basicLoadBalance(cx->getCommitProxies(useProvisionalProxies),
-				                               &CommitProxyInterface::getKeyServersLocations,
-				                               GetKeyServerLocationsRequest(span.context,
-				                                                            tenant.castTo<TenantNameRef>(),
-				                                                            keys.begin,
-				                                                            keys.end,
-				                                                            limit,
-				                                                            reverse,
-				                                                            version,
-				                                                            keys.arena()),
-				                               TaskPriority::DefaultPromiseEndpoint))) {
+				when(GetKeyServerLocationsReply _rep = wait(basicLoadBalance(
+				         cx->getCommitProxies(useProvisionalProxies),
+				         &CommitProxyInterface::getKeyServersLocations,
+				         GetKeyServerLocationsRequest(
+				             span.context, tenant, keys.begin, keys.end, limit, reverse, version, keys.arena()),
+				         TaskPriority::DefaultPromiseEndpoint))) {
 					++cx->transactionKeyServerLocationRequestsCompleted;
 					state GetKeyServerLocationsReply rep = _rep;
 					if (debugID.present())
@@ -3031,7 +3049,7 @@ ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
 						    rep.tenantEntry,
 						    (toRelativeRange(rep.results[shard].first, rep.tenantEntry.prefix) & keys),
 						    cx->setCachedLocation(
-						        tenant, rep.tenantEntry, rep.results[shard].first, rep.results[shard].second));
+						        tenant.name, rep.tenantEntry, rep.results[shard].first, rep.results[shard].second));
 						wait(yield());
 					}
 					updateTssMappings(cx, rep);
@@ -3043,8 +3061,8 @@ ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_tenant_not_found) {
-			ASSERT(tenant.present());
-			cx->invalidateCachedTenant(tenant.get());
+			ASSERT(tenant.name.present());
+			cx->invalidateCachedTenant(tenant.name.get());
 		}
 
 		throw;
@@ -3059,7 +3077,7 @@ ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
 // [([a, b1), locationInfo), ([b1, c), locationInfo), ([c, d1), locationInfo)].
 template <class F>
 Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Database const& cx,
-                                                               Optional<TenantName> tenant,
+                                                               TenantInfo const& tenant,
                                                                KeyRange const& keys,
                                                                int limit,
                                                                Reverse reverse,
@@ -3072,7 +3090,7 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Database const& c
 	ASSERT(!keys.empty());
 
 	std::vector<KeyRangeLocationInfo> locations;
-	if (!cx->getCachedLocations(tenant, keys, locations, limit, reverse)) {
+	if (!cx->getCachedLocations(tenant.name, keys, locations, limit, reverse)) {
 		return getKeyRangeLocations_internal(
 		    cx, tenant, keys, limit, reverse, spanContext, debugID, useProvisionalProxies, version);
 	}
@@ -3110,20 +3128,20 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Reference<Transac
                                                                UseTenant useTenant,
                                                                Version version) {
 	auto f = getKeyRangeLocations(trState->cx,
-	                              useTenant ? trState->tenant() : Optional<TenantName>(),
+	                              useTenant ? trState->getTenantInfo(AllowInvalidTenantID::True) : TenantInfo(),
 	                              keys,
 	                              limit,
 	                              reverse,
 	                              member,
 	                              trState->spanContext,
-	                              trState->debugID,
+	                              trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
 	                              trState->useProvisionalProxies,
 	                              version);
 
-	if (trState->tenant().present() && useTenant) {
+	if (trState->tenant().present() && useTenant && trState->tenantId() == TenantInfo::INVALID_TENANT) {
 		return map(f, [trState](const std::vector<KeyRangeLocationInfo>& locationInfo) {
 			ASSERT(!locationInfo.empty());
-			trState->tenantId = locationInfo[0].tenantEntry.id;
+			trState->trySetTenantId(locationInfo[0].tenantEntry.id);
 			return locationInfo;
 		});
 	} else {
@@ -3138,16 +3156,16 @@ ACTOR Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange
 	state Version version = wait(fVersion);
 
 	loop {
-		std::vector<KeyRangeLocationInfo> locations =
-		    wait(getKeyRangeLocations_internal(trState->cx,
-		                                       trState->tenant(),
-		                                       keys,
-		                                       CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT,
-		                                       Reverse::False,
-		                                       trState->spanContext,
-		                                       trState->debugID,
-		                                       trState->useProvisionalProxies,
-		                                       version));
+		std::vector<KeyRangeLocationInfo> locations = wait(getKeyRangeLocations_internal(
+		    trState->cx,
+		    trState->getTenantInfo(),
+		    keys,
+		    CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT,
+		    Reverse::False,
+		    trState->spanContext,
+		    trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
+		    trState->useProvisionalProxies,
+		    version));
 		totalRanges += CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT;
 		totalRequests++;
 		if (locations.size() == 0 || totalRanges >= trState->cx->locationCacheSize ||
@@ -3190,6 +3208,8 @@ SpanContext generateSpanID(bool transactionTracingSample, SpanContext parentCont
 	    deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUInt64(), TraceFlags::unsampled);
 }
 
+FDB_DEFINE_BOOLEAN_PARAM(AllowInvalidTenantID);
+
 TransactionState::TransactionState(Database cx,
                                    Optional<TenantName> tenant,
                                    TaskPriority taskID,
@@ -3213,28 +3233,44 @@ Reference<TransactionState> TransactionState::cloneAndReset(Reference<Transactio
 	newState->startTime = startTime;
 	newState->committedVersion = committedVersion;
 	newState->conflictingKeys = conflictingKeys;
+	newState->authToken = authToken;
 	newState->tenantSet = tenantSet;
 
 	return newState;
 }
 
-TenantInfo TransactionState::getTenantInfo() {
+TenantInfo TransactionState::getTenantInfo(AllowInvalidTenantID allowInvalidId /* = false */) {
 	Optional<TenantName> const& t = tenant();
 
 	if (options.rawAccess) {
 		return TenantInfo();
+	} else if (!cx->internal && cx->clientInfo->get().clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+		throw management_cluster_invalid_access();
 	} else if (!cx->internal && cx->clientInfo->get().tenantMode == TenantMode::REQUIRED && !t.present()) {
 		throw tenant_name_required();
 	} else if (!t.present()) {
 		return TenantInfo();
 	} else if (cx->clientInfo->get().tenantMode == TenantMode::DISABLED && t.present()) {
-		throw tenants_disabled();
+		// If we are running provisional proxies, we allow a tenant request to go through since we don't know the tenant
+		// mode. Such a transaction would not be allowed to commit without enabling provisional commits because either
+		// the commit proxies will be provisional or the read version will be too old.
+		if (!cx->clientInfo->get().grvProxies.empty() && !cx->clientInfo->get().grvProxies[0].provisional) {
+			throw tenants_disabled();
+		} else {
+			ASSERT(!useProvisionalProxies);
+		}
 	}
 
-	ASSERT(tenantId != TenantInfo::INVALID_TENANT);
-	return TenantInfo(t.get(), tenantId);
+	ASSERT(allowInvalidId || tenantId_ != TenantInfo::INVALID_TENANT);
+	return TenantInfo(t, authToken, tenantId_);
 }
 
+// Returns the tenant used in this transaction. If the tenant is unset and raw access isn't specified, then the default
+// tenant from DatabaseContext is applied to this transaction (note: the default tenant is typically unset, but in
+// simulation could be something different).
+//
+// This function should not be called in the transaction constructor or in the setOption function to allow a user the
+// opportunity to set raw access.
 Optional<TenantName> const& TransactionState::tenant() {
 	if (tenantSet) {
 		return tenant_;
@@ -3247,10 +3283,20 @@ Optional<TenantName> const& TransactionState::tenant() {
 	}
 }
 
+// Returns true if the tenant has been set, but does not cause default tenant resolution. This is useful in setOption
+// (where we do not want to call tenant()) if we want to enforce that an option not be set on a Tenant transaction (e.g.
+// for raw access).
 bool TransactionState::hasTenant() const {
 	return tenantSet && tenant_.present();
 }
 
+Future<Void> TransactionState::handleUnknownTenant() {
+	tenantId_ = TenantInfo::INVALID_TENANT;
+	ASSERT(tenant().present());
+	cx->invalidateCachedTenant(tenant().get());
+	return delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, taskID);
+}
+
 Future<Void> Transaction::warmRange(KeyRange keys) {
 	return warmRange_impl(trState, keys, getReadVersion());
 }
@@ -3277,12 +3323,16 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
 		state uint64_t startTime;
 		state double startTimeD;
 		state VersionVector ssLatestCommitVersions;
+		state Optional<ReadOptions> readOptions = trState->readOptions;
+
 		trState->cx->getLatestCommitVersions(locationInfo.locations, ver, trState, ssLatestCommitVersions);
 		try {
-			if (trState->debugID.present()) {
+			if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
 				getValueID = nondeterministicRandom()->randomUniqueID();
+				readOptions.get().debugID = getValueID;
 
-				g_traceBatch.addAttach("GetValueAttachID", trState->debugID.get().first(), getValueID.get().first());
+				g_traceBatch.addAttach(
+				    "GetValueAttachID", trState->readOptions.get().debugID.get().first(), getValueID.get().first());
 				g_traceBatch.addEvent("GetValueDebug",
 				                      getValueID.get().first(),
 				                      "NativeAPI.getValue.Before"); //.detail("TaskID", g_network->getCurrentTask());
@@ -3315,7 +3365,7 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
 					                         ver,
 					                         trState->cx->sampleReadTags() ? trState->options.readTags
 					                                                       : Optional<TagSet>(),
-					                         getValueID,
+					                         readOptions,
 					                         ssLatestCommitVersions),
 					         TaskPriority::DefaultPromiseEndpoint,
 					         AtMostOnce::False,
@@ -3369,9 +3419,8 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
 				trState->cx->invalidateCache(locationInfo.tenantEntry.prefix, key);
 				wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
 			} else if (e.code() == error_code_unknown_tenant) {
-				ASSERT(useTenant && trState->tenant().present());
-				trState->cx->invalidateCachedTenant(trState->tenant().get());
-				wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
+				ASSERT(useTenant);
+				wait(trState->handleUnknownTenant());
 			} else {
 				if (trState->trLogInfo && recordLogInfo)
 					trState->trLogInfo->addLog(FdbClientLogEvents::EventGetError(startTimeD,
@@ -3391,12 +3440,16 @@ ACTOR Future<Key> getKey(Reference<TransactionState> trState,
                          UseTenant useTenant = UseTenant::True) {
 	wait(success(version));
 
-	state Optional<UID> getKeyID = Optional<UID>();
-	state Span span("NAPI:getKey"_loc, trState->spanContext);
-	if (trState->debugID.present()) {
-		getKeyID = nondeterministicRandom()->randomUniqueID();
+	state Optional<UID> getKeyID;
+	state Optional<ReadOptions> readOptions = trState->readOptions;
 
-		g_traceBatch.addAttach("GetKeyAttachID", trState->debugID.get().first(), getKeyID.get().first());
+	state Span span("NAPI:getKey"_loc, trState->spanContext);
+	if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
+		getKeyID = nondeterministicRandom()->randomUniqueID();
+		readOptions.get().debugID = getKeyID;
+
+		g_traceBatch.addAttach(
+		    "GetKeyAttachID", trState->readOptions.get().debugID.get().first(), getKeyID.get().first());
 		g_traceBatch.addEvent(
 		    "GetKeyDebug",
 		    getKeyID.get().first(),
@@ -3439,7 +3492,7 @@ ACTOR Future<Key> getKey(Reference<TransactionState> trState,
 			                  k,
 			                  version.get(),
 			                  trState->cx->sampleReadTags() ? trState->options.readTags : Optional<TagSet>(),
-			                  getKeyID,
+			                  readOptions,
 			                  ssLatestCommitVersions);
 			req.arena.dependsOn(k.arena());
 
@@ -3480,9 +3533,8 @@ ACTOR Future<Key> getKey(Reference<TransactionState> trState,
 
 				wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
 			} else if (e.code() == error_code_unknown_tenant) {
-				ASSERT(useTenant && trState->tenant().present());
-				trState->cx->invalidateCachedTenant(trState->tenant().get());
-				wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
+				ASSERT(useTenant);
+				wait(trState->handleUnknownTenant());
 			} else {
 				TraceEvent(SevInfo, "GetKeyError").error(e).detail("AtKey", k.getKey()).detail("Offset", k.offset);
 				throw e;
@@ -3493,8 +3545,8 @@ ACTOR Future<Key> getKey(Reference<TransactionState> trState,
 
 ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version, SpanContext spanContext) {
 	state Span span("NAPI:waitForCommittedVersion"_loc, spanContext);
-	try {
-		loop {
+	loop {
+		try {
 			choose {
 				when(wait(cx->onProxiesChanged())) {}
 				when(GetReadVersionReply v = wait(basicLoadBalance(
@@ -3520,10 +3572,16 @@ ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version, Span
 					wait(delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, cx->taskID));
 				}
 			}
+		} catch (Error& e) {
+			if (e.code() == error_code_batch_transaction_throttled ||
+			    e.code() == error_code_grv_proxy_memory_limit_exceeded) {
+				// GRV Proxy returns an error
+				wait(delayJittered(CLIENT_KNOBS->GRV_ERROR_RETRY_DELAY));
+			} else {
+				TraceEvent(SevError, "WaitForCommittedVersionError").error(e);
+				throw;
+			}
 		}
-	} catch (Error& e) {
-		TraceEvent(SevError, "WaitForCommittedVersionError").error(e);
-		throw;
 	}
 }
 
@@ -3566,7 +3624,7 @@ ACTOR Future<Version> watchValue(Database cx, Reference<const WatchParameters> p
 
 	loop {
 		state KeyRangeLocationInfo locationInfo = wait(getKeyLocation(cx,
-		                                                              parameters->tenant.name,
+		                                                              parameters->tenant,
 		                                                              parameters->key,
 		                                                              &StorageServerInterface::watchValue,
 		                                                              parameters->spanContext,
@@ -3629,13 +3687,13 @@ ACTOR Future<Version> watchValue(Database cx, Reference<const WatchParameters> p
 				wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, parameters->taskID));
 			} else if (e.code() == error_code_watch_cancelled || e.code() == error_code_process_behind) {
 				// clang-format off
-				TEST(e.code() == error_code_watch_cancelled); // Too many watches on the storage server, poll for changes instead
-				TEST(e.code() == error_code_process_behind); // The storage servers are all behind
+				CODE_PROBE(e.code() == error_code_watch_cancelled, "Too many watches on the storage server, poll for changes instead");
+				CODE_PROBE(e.code() == error_code_process_behind, "The storage servers are all behind");
 				// clang-format on
 				wait(delay(CLIENT_KNOBS->WATCH_POLLING_TIME, parameters->taskID));
 			} else if (e.code() == error_code_timed_out) { // The storage server occasionally times out watches in case
 				                                           // it was cancelled
-				TEST(true); // A watch timed out
+				CODE_PROBE(true, "A watch timed out");
 				wait(delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, parameters->taskID));
 			} else {
 				state Error err = e;
@@ -3667,7 +3725,8 @@ ACTOR Future<Void> watchStorageServerResp(int64_t tenantId, Key key, Database cx
 			}
 			// ABA happens
 			else {
-				TEST(true); // ABA issue where the version returned from the server is less than the version in the map
+				CODE_PROBE(true,
+				           "ABA issue where the version returned from the server is less than the version in the map");
 
 				// case 2: version_1 < version_2 and future_count == 1
 				if (metadata->watchPromise.getFutureReferenceCount() == 1) {
@@ -3696,7 +3755,7 @@ ACTOR Future<Void> watchStorageServerResp(int64_t tenantId, Key key, Database cx
 }
 
 ACTOR Future<Void> sameVersionDiffValue(Database cx, Reference<WatchParameters> parameters) {
-	state ReadYourWritesTransaction tr(cx, parameters->tenant.name);
+	state ReadYourWritesTransaction tr(cx, parameters->tenant.name.castTo<TenantName>());
 	loop {
 		try {
 			if (!parameters->tenant.name.present()) {
@@ -3715,7 +3774,7 @@ ACTOR Future<Void> sameVersionDiffValue(Database cx, Reference<WatchParameters>
 			}
 
 			// val_3 == val_2 (storage server value matches value passed into the function -> new watch)
-			if (valSS == parameters->value && tr.getTransactionState()->tenantId == parameters->tenant.tenantId) {
+			if (valSS == parameters->value && tr.getTransactionState()->tenantId() == parameters->tenant.tenantId) {
 				metadata = makeReference<WatchMetadata>(parameters);
 				cx->setWatchMetadata(metadata);
 
@@ -3758,7 +3817,8 @@ Future<Void> getWatchFuture(Database cx, Reference<WatchParameters> parameters)
 	// case 3: val_1 != val_2 && version_2 > version_1 (received watch with different value and a higher version so
 	// recreate in SS)
 	else if (parameters->version > metadata->parameters->version) {
-		TEST(true); // Setting a watch that has a different value than the one in the map but a higher version (newer)
+		CODE_PROBE(true,
+		           "Setting a watch that has a different value than the one in the map but a higher version (newer)");
 		cx->deleteWatchMetadata(parameters->tenant.tenantId, parameters->key);
 
 		metadata->watchPromise.send(parameters->version);
@@ -3773,10 +3833,10 @@ Future<Void> getWatchFuture(Database cx, Reference<WatchParameters> parameters)
 	}
 	// case 5: val_1 != val_2 && version_1 == version_2 (received watch with different value but same version)
 	else if (metadata->parameters->version == parameters->version) {
-		TEST(true); // Setting a watch which has a different value than the one in the map but the same version
+		CODE_PROBE(true, "Setting a watch which has a different value than the one in the map but the same version");
 		return sameVersionDiffValue(cx, parameters);
 	}
-	TEST(true); // Setting a watch which has a different value than the one in the map but a lower version (older)
+	CODE_PROBE(true, "Setting a watch which has a different value than the one in the map but a lower version (older)");
 
 	// case 4: val_1 != val_2 && version_2 < version_1
 	return Void();
@@ -3897,13 +3957,15 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
 
 			// FIXME: buggify byte limits on internal functions that use them, instead of globally
 			req.tags = trState->cx->sampleReadTags() ? trState->options.readTags : Optional<TagSet>();
-			req.debugID = trState->debugID;
+
+			req.options = trState->readOptions;
 
 			try {
-				if (trState->debugID.present()) {
-					g_traceBatch.addEvent(
-					    "TransactionDebug", trState->debugID.get().first(), "NativeAPI.getExactRange.Before");
-					/*TraceEvent("TransactionDebugGetExactRangeInfo", trState->debugID.get())
+				if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
+					g_traceBatch.addEvent("TransactionDebug",
+					                      trState->readOptions.get().debugID.get().first(),
+					                      "NativeAPI.getExactRange.Before");
+					/*TraceEvent("TransactionDebugGetExactRangeInfo", trState->readOptions.debugID.get())
 					    .detail("ReqBeginKey", req.begin.getKey())
 					    .detail("ReqEndKey", req.end.getKey())
 					    .detail("ReqLimit", req.limit)
@@ -3933,9 +3995,10 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
 					++trState->cx->transactionPhysicalReadsCompleted;
 					throw;
 				}
-				if (trState->debugID.present())
-					g_traceBatch.addEvent(
-					    "TransactionDebug", trState->debugID.get().first(), "NativeAPI.getExactRange.After");
+				if (trState->readOptions.present() && trState->readOptions.get().debugID.present())
+					g_traceBatch.addEvent("TransactionDebug",
+					                      trState->readOptions.get().debugID.get().first(),
+					                      "NativeAPI.getExactRange.After");
 				output.arena().dependsOn(rep.arena);
 				output.append(output.arena(), rep.data.begin(), rep.data.size());
 
@@ -3970,7 +4033,7 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
 						    .detail("BlockBytes", rep.data.expectedSize());
 						ASSERT(false);
 					}
-					TEST(true); // GetKeyValuesFamilyReply.more in getExactRange
+					CODE_PROBE(true, "GetKeyValuesFamilyReply.more in getExactRange");
 					// Make next request to the same shard with a beginning key just after the last key returned
 					if (reverse)
 						locations[shard].range =
@@ -3981,7 +4044,7 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
 				}
 
 				if (!more || locations[shard].range.empty()) {
-					TEST(true); // getExactrange (!more || locations[shard].first.empty())
+					CODE_PROBE(true, "getExactrange (!more || locations[shard].first.empty())");
 					if (shard == locations.size() - 1) {
 						const KeyRangeRef& range = locations[shard].range;
 						KeyRef begin = reverse ? keys.begin : range.end;
@@ -3991,7 +4054,7 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
 							output.more = false;
 							return output;
 						}
-						TEST(true); // Multiple requests of key locations
+						CODE_PROBE(true, "Multiple requests of key locations");
 
 						keys = KeyRangeRef(begin, end);
 						break;
@@ -4022,9 +4085,8 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
 					wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
 					break;
 				} else if (e.code() == error_code_unknown_tenant) {
-					ASSERT(useTenant && trState->tenant().present());
-					trState->cx->invalidateCachedTenant(trState->tenant().get());
-					wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
+					ASSERT(useTenant);
+					wait(trState->handleUnknownTenant());
 					break;
 				} else {
 					TraceEvent(SevInfo, "GetExactRangeError")
@@ -4264,7 +4326,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 			req.arena.dependsOn(mapper.arena());
 			setMatchIndex<GetKeyValuesFamilyRequest>(req, matchIndex);
 			req.tenantInfo = useTenant ? trState->getTenantInfo() : TenantInfo();
-			req.isFetchKeys = (trState->taskID == TaskPriority::FetchKeys);
+			req.options = trState->readOptions;
 			req.version = readVersion;
 
 			trState->cx->getLatestCommitVersions(
@@ -4302,13 +4364,13 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 			ASSERT(req.limitBytes > 0 && req.limit != 0 && req.limit < 0 == reverse);
 
 			req.tags = trState->cx->sampleReadTags() ? trState->options.readTags : Optional<TagSet>();
-			req.debugID = trState->debugID;
 			req.spanContext = span.context;
 			try {
-				if (trState->debugID.present()) {
-					g_traceBatch.addEvent(
-					    "TransactionDebug", trState->debugID.get().first(), "NativeAPI.getRange.Before");
-					/*TraceEvent("TransactionDebugGetRangeInfo", trState->debugID.get())
+				if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
+					g_traceBatch.addEvent("TransactionDebug",
+					                      trState->readOptions.get().debugID.get().first(),
+					                      "NativeAPI.getRange.Before");
+					/*TraceEvent("TransactionDebugGetRangeInfo", trState->readOptions.debugID.get())
 					    .detail("ReqBeginKey", req.begin.getKey())
 					    .detail("ReqEndKey", req.end.getKey())
 					    .detail("OriginalBegin", originalBegin.toString())
@@ -4347,11 +4409,11 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 					throw;
 				}
 
-				if (trState->debugID.present()) {
+				if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
 					g_traceBatch.addEvent("TransactionDebug",
-					                      trState->debugID.get().first(),
+					                      trState->readOptions.get().debugID.get().first(),
 					                      "NativeAPI.getRange.After"); //.detail("SizeOf", rep.data.size());
-					/*TraceEvent("TransactionDebugGetRangeDone", trState->debugID.get())
+					/*TraceEvent("TransactionDebugGetRangeDone", trState->readOptions.debugID.get())
 					    .detail("ReqBeginKey", req.begin.getKey())
 					    .detail("ReqEndKey", req.end.getKey())
 					    .detail("RepIsMore", rep.more)
@@ -4431,7 +4493,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 
 				if (!rep.more) {
 					ASSERT(modifiedSelectors);
-					TEST(true); // !GetKeyValuesFamilyReply.more and modifiedSelectors in getRange
+					CODE_PROBE(true, "!GetKeyValuesFamilyReply.more and modifiedSelectors in getRange");
 
 					if (!rep.data.size()) {
 						RangeResultFamily result = wait(
@@ -4455,7 +4517,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 					else
 						begin = firstGreaterOrEqual(shard.end);
 				} else {
-					TEST(true); // GetKeyValuesFamilyReply.more in getRange
+					CODE_PROBE(true, "GetKeyValuesFamilyReply.more in getRange");
 					if (reverse)
 						end = firstGreaterOrEqual(output[output.size() - 1].key);
 					else
@@ -4463,10 +4525,11 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 				}
 
 			} catch (Error& e) {
-				if (trState->debugID.present()) {
-					g_traceBatch.addEvent(
-					    "TransactionDebug", trState->debugID.get().first(), "NativeAPI.getRange.Error");
-					TraceEvent("TransactionDebugError", trState->debugID.get()).error(e);
+				if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
+					g_traceBatch.addEvent("TransactionDebug",
+					                      trState->readOptions.get().debugID.get().first(),
+					                      "NativeAPI.getRange.Error");
+					TraceEvent("TransactionDebugError", trState->readOptions.get().debugID.get()).error(e);
 				}
 				if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
 				    (e.code() == error_code_transaction_too_old && readVersion == latestVersion)) {
@@ -4493,9 +4556,8 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 
 					wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
 				} else if (e.code() == error_code_unknown_tenant) {
-					ASSERT(useTenant && trState->tenant().present());
-					trState->cx->invalidateCachedTenant(trState->tenant().get());
-					wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
+					ASSERT(useTenant);
+					wait(trState->handleUnknownTenant());
 				} else {
 					if (trState->trLogInfo)
 						trState->trLogInfo->addLog(
@@ -4574,7 +4636,7 @@ static Future<Void> tssStreamComparison(Request request,
 			} else {
 				tssData.metrics->ssError(e.code());
 			}
-			TEST(e.code() != error_code_end_of_stream); // SS got error in TSS stream comparison
+			CODE_PROBE(e.code() != error_code_end_of_stream, "SS got error in TSS stream comparison");
 		}
 
 		state double sleepTime = std::max(startTime + FLOW_KNOBS->LOAD_BALANCE_TSS_TIMEOUT - now(), 0.0);
@@ -4586,7 +4648,7 @@ static Future<Void> tssStreamComparison(Request request,
 				}
 				when(wait(delay(sleepTime))) {
 					++tssData.metrics->tssTimeouts;
-					TEST(true); // Got TSS timeout in stream comparison
+					CODE_PROBE(true, "Got TSS timeout in stream comparison");
 				}
 			}
 		} catch (Error& e) {
@@ -4601,7 +4663,7 @@ static Future<Void> tssStreamComparison(Request request,
 			} else {
 				tssData.metrics->tssError(e.code());
 			}
-			TEST(e.code() != error_code_end_of_stream); // TSS got error in TSS stream comparison
+			CODE_PROBE(e.code() != error_code_end_of_stream, "TSS got error in TSS stream comparison");
 		}
 
 		if (!ssEndOfStream || !tssEndOfStream) {
@@ -4614,13 +4676,12 @@ static Future<Void> tssStreamComparison(Request request,
 			// FIXME: this code is pretty much identical to LoadBalance.h
 			// TODO could add team check logic in if we added synchronous way to turn this into a fixed getRange request
 			// and send it to the whole team and compare? I think it's fine to skip that for streaming though
-			TEST(ssEndOfStream != tssEndOfStream); // SS or TSS stream finished early!
 
 			// skip tss comparison if both are end of stream
 			if ((!ssEndOfStream || !tssEndOfStream) && !TSS_doCompare(ssReply.get(), tssReply.get())) {
-				TEST(true); // TSS mismatch in stream comparison
+				CODE_PROBE(true, "TSS mismatch in stream comparison");
 				TraceEvent mismatchEvent(
-				    (g_network->isSimulated() && g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations)
+				    (g_network->isSimulated() && g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations)
 				        ? SevWarnAlways
 				        : SevError,
 				    TSS_mismatchTraceName(request));
@@ -4630,10 +4691,10 @@ static Future<Void> tssStreamComparison(Request request,
 				if (tssData.metrics->shouldRecordDetailedMismatch()) {
 					TSS_traceMismatch(mismatchEvent, request, ssReply.get(), tssReply.get());
 
-					TEST(FLOW_KNOBS
-					         ->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Full TSS Mismatch in stream comparison
-					TEST(!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Partial TSS Mismatch in stream
-					                                                         // comparison and storing the rest in FDB
+					CODE_PROBE(FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL,
+					           "Tracing Full TSS Mismatch in stream comparison");
+					CODE_PROBE(!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL,
+					           "Tracing Partial TSS Mismatch in stream comparison and storing the rest in FDB");
 
 					if (!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL) {
 						mismatchEvent.disable();
@@ -4642,7 +4703,7 @@ static Future<Void> tssStreamComparison(Request request,
 
 						// record a summarized trace event instead
 						TraceEvent summaryEvent((g_network->isSimulated() &&
-						                         g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations)
+						                         g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations)
 						                            ? SevWarnAlways
 						                            : SevError,
 						                        TSS_mismatchTraceName(request));
@@ -4673,7 +4734,7 @@ maybeDuplicateTSSStreamFragment(Request& req, QueueModel* model, RequestStream<R
 		Optional<TSSEndpointData> tssData = model->getTssData(ssStream->getEndpoint().token.first());
 
 		if (tssData.present()) {
-			TEST(true); // duplicating stream to TSS
+			CODE_PROBE(true, "duplicating stream to TSS");
 			resetReply(req);
 			// FIXME: optimize to avoid creating new netNotifiedQueueWithAcknowledgements for each stream duplication
 			RequestStream<Request> tssRequestStream(tssData.get().endpoint);
@@ -4719,9 +4780,8 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 			req.spanContext = spanContext;
 			req.limit = reverse ? -CLIENT_KNOBS->REPLY_BYTE_LIMIT : CLIENT_KNOBS->REPLY_BYTE_LIMIT;
 			req.limitBytes = std::numeric_limits<int>::max();
-			// leaving the flag off for now to prevent data fetches stall under heavy load
-			// it is used to inform the storage that the rangeRead is for Fetch
-			// req.isFetchKeys = (trState->taskID == TaskPriority::FetchKeys);
+			req.options = trState->readOptions;
+
 			trState->cx->getLatestCommitVersions(
 			    locations[shard].locations, req.version, trState, req.ssLatestCommitVersions);
 
@@ -4732,12 +4792,12 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 
 			// FIXME: buggify byte limits on internal functions that use them, instead of globally
 			req.tags = trState->cx->sampleReadTags() ? trState->options.readTags : Optional<TagSet>();
-			req.debugID = trState->debugID;
 
 			try {
-				if (trState->debugID.present()) {
-					g_traceBatch.addEvent(
-					    "TransactionDebug", trState->debugID.get().first(), "NativeAPI.RangeStream.Before");
+				if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
+					g_traceBatch.addEvent("TransactionDebug",
+					                      trState->readOptions.get().debugID.get().first(),
+					                      "NativeAPI.RangeStream.Before");
 				}
 				++trState->cx->transactionPhysicalReads;
 				state GetKeyValuesStreamReply rep;
@@ -4831,9 +4891,10 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 						}
 						rep = GetKeyValuesStreamReply();
 					}
-					if (trState->debugID.present())
-						g_traceBatch.addEvent(
-						    "TransactionDebug", trState->debugID.get().first(), "NativeAPI.getExactRange.After");
+					if (trState->readOptions.present() && trState->readOptions.get().debugID.present())
+						g_traceBatch.addEvent("TransactionDebug",
+						                      trState->readOptions.get().debugID.get().first(),
+						                      "NativeAPI.getExactRange.After");
 					RangeResult output(RangeResultRef(rep.data, rep.more), rep.arena);
 
 					if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) {
@@ -4873,7 +4934,7 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 							    .detail("BlockBytes", rep.data.expectedSize());
 							ASSERT(false);
 						}
-						TEST(true); // GetKeyValuesStreamReply.more in getRangeStream
+						CODE_PROBE(true, "GetKeyValuesStreamReply.more in getRangeStream");
 						// Make next request to the same shard with a beginning key just after the last key returned
 						if (reverse)
 							locations[shard].range =
@@ -4941,7 +5002,7 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 					throw;
 				}
 				if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
-				    e.code() == error_code_connection_failed) {
+				    e.code() == error_code_connection_failed || e.code() == error_code_request_maybe_delivered) {
 					const KeyRangeRef& range = locations[shard].range;
 
 					if (reverse)
@@ -4954,9 +5015,7 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 					wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
 					break;
 				} else if (e.code() == error_code_unknown_tenant) {
-					ASSERT(trState->tenant().present());
-					trState->cx->invalidateCachedTenant(trState->tenant().get());
-					wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
+					wait(trState->handleUnknownTenant());
 					break;
 				} else {
 					results->sendError(e);
@@ -5239,7 +5298,7 @@ ACTOR Future<TenantInfo> getTenantMetadata(Reference<TransactionState> trState,
 Future<TenantInfo> populateAndGetTenant(Reference<TransactionState> trState, Key const& key, Version version) {
 	if (!trState->tenant().present() || key == metadataVersionKey) {
 		return TenantInfo();
-	} else if (trState->tenantId != TenantInfo::INVALID_TENANT) {
+	} else if (trState->tenantId() != TenantInfo::INVALID_TENANT) {
 		return trState->getTenantInfo();
 	} else {
 		return getTenantMetadata(trState, key, version);
@@ -5271,7 +5330,7 @@ ACTOR Future<Void> watch(Reference<Watch> watch,
 						when(wait(watch->watchFuture)) { break; }
 
 						when(wait(cx->connectionFileChanged())) {
-							TEST(true); // Recreated a watch after switch
+							CODE_PROBE(true, "Recreated a watch after switch");
 							cx->clearWatchMetadata();
 							watch->watchFuture = watchValueMap(cx->minAcceptableReadVersion,
 							                                   tenantInfo,
@@ -5314,7 +5373,7 @@ Future<Void> Transaction::watch(Reference<Watch> watch) {
 	    trState->options.readTags,
 	    trState->spanContext,
 	    trState->taskID,
-	    trState->debugID,
+	    trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
 	    trState->useProvisionalProxies);
 }
 
@@ -5444,18 +5503,18 @@ Future<RangeResultFamily> Transaction::getRangeInternal(const KeySelector& begin
 
 	KeySelector b = begin;
 	if (b.orEqual) {
-		TEST(true); // Native begin orEqual==true
+		CODE_PROBE(true, "Native begin orEqual==true");
 		b.removeOrEqual(b.arena());
 	}
 
 	KeySelector e = end;
 	if (e.orEqual) {
-		TEST(true); // Native end orEqual==true
+		CODE_PROBE(true, "Native end orEqual==true");
 		e.removeOrEqual(e.arena());
 	}
 
 	if (b.offset >= e.offset && b.getKey() >= e.getKey()) {
-		TEST(true); // Native range inverted
+		CODE_PROBE(true, "Native range inverted");
 		return RangeResultFamily();
 	}
 
@@ -5504,7 +5563,7 @@ Future<RangeResult> Transaction::getRange(const KeySelector& begin,
 
 // A method for streaming data from the storage server that is more efficient than getRange when reading large amounts
 // of data
-Future<Void> Transaction::getRangeStream(const PromiseStream<RangeResult>& results,
+Future<Void> Transaction::getRangeStream(PromiseStream<RangeResult>& results,
                                          const KeySelector& begin,
                                          const KeySelector& end,
                                          GetRangeLimits limits,
@@ -5518,18 +5577,18 @@ Future<Void> Transaction::getRangeStream(const PromiseStream<RangeResult>& resul
 
 	KeySelector b = begin;
 	if (b.orEqual) {
-		TEST(true); // Native stream begin orEqual==true
+		CODE_PROBE(true, "Native stream begin orEqual==true");
 		b.removeOrEqual(b.arena());
 	}
 
 	KeySelector e = end;
 	if (e.orEqual) {
-		TEST(true); // Native stream end orEqual==true
+		CODE_PROBE(true, "Native stream end orEqual==true");
 		e.removeOrEqual(e.arena());
 	}
 
 	if (b.offset >= e.offset && b.getKey() >= e.getKey()) {
-		TEST(true); // Native stream range inverted
+		CODE_PROBE(true, "Native stream range inverted");
 		results.sendError(end_of_stream());
 		return Void();
 	}
@@ -5543,7 +5602,7 @@ Future<Void> Transaction::getRangeStream(const PromiseStream<RangeResult>& resul
 	    ::getRangeStream(trState, results, getReadVersion(), b, e, limits, conflictRange, snapshot, reverse), results);
 }
 
-Future<Void> Transaction::getRangeStream(const PromiseStream<RangeResult>& results,
+Future<Void> Transaction::getRangeStream(PromiseStream<RangeResult>& results,
                                          const KeySelector& begin,
                                          const KeySelector& end,
                                          int limit,
@@ -5580,7 +5639,7 @@ void Transaction::addReadConflictRange(KeyRangeRef const& keys) {
 
 void Transaction::makeSelfConflicting() {
 	BinaryWriter wr(Unversioned());
-	wr.serializeBytes(LiteralStringRef("\xFF/SC/"));
+	wr.serializeBytes("\xFF/SC/"_sr);
 	wr << deterministicRandom()->randomUniqueID();
 	auto r = singleKeyRange(wr.toValue(), tr.arena);
 	tr.transaction.read_conflict_ranges.push_back(tr.arena, r);
@@ -5632,7 +5691,7 @@ void Transaction::atomicOp(const KeyRef& key,
 	if (addConflictRange && operationType != MutationRef::SetVersionstampedKey)
 		t.write_conflict_ranges.push_back(req.arena, r);
 
-	TEST(true); // NativeAPI atomic operation
+	CODE_PROBE(true, "NativeAPI atomic operation");
 }
 
 void Transaction::clear(const KeyRangeRef& range, AddConflictRange addConflictRange) {
@@ -5718,7 +5777,7 @@ double Transaction::getBackoff(int errCode) {
 			if (priorityItr != trState->cx->throttledTags.end()) {
 				auto tagItr = priorityItr->second.find(tag);
 				if (tagItr != priorityItr->second.end()) {
-					TEST(true); // Returning throttle backoff
+					CODE_PROBE(true, "Returning throttle backoff");
 					returnedBackoff = std::max(
 					    returnedBackoff,
 					    std::min(CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL, tagItr->second.throttleDuration()));
@@ -5733,7 +5792,9 @@ double Transaction::getBackoff(int errCode) {
 	returnedBackoff *= deterministicRandom()->random01();
 
 	// Set backoff for next time
-	if (errCode == error_code_proxy_memory_limit_exceeded) {
+	if (errCode == error_code_commit_proxy_memory_limit_exceeded ||
+	    errCode == error_code_grv_proxy_memory_limit_exceeded) {
+
 		backoff = std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, CLIENT_KNOBS->RESOURCE_CONSTRAINED_MAX_BACKOFF);
 	} else {
 		backoff = std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, trState->options.maxBackoff);
@@ -5924,8 +5985,12 @@ ACTOR void checkWrites(Reference<TransactionState> trState,
 	}
 }
 
-ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trState, KeyRange range) {
-	state Transaction tr(trState->cx);
+FDB_BOOLEAN_PARAM(TenantPrefixPrepended);
+
+ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trState,
+                                                 KeyRange range,
+                                                 TenantPrefixPrepended tenantPrefixPrepended) {
+	state Transaction tr(trState->cx, trState->tenant());
 	state int retries = 0;
 	state Span span("NAPI:dummyTransaction"_loc, trState->spanContext);
 	tr.span.setParent(span.context);
@@ -5934,7 +5999,14 @@ ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trS
 			TraceEvent("CommitDummyTransaction").detail("Key", range.begin).detail("Retries", retries);
 			tr.trState->options = trState->options;
 			tr.trState->taskID = trState->taskID;
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.trState->authToken = trState->authToken;
+			tr.trState->trySetTenantId(trState->tenantId());
+			if (!trState->hasTenant()) {
+				tr.setOption(FDBTransactionOptions::RAW_ACCESS);
+			} else {
+				tr.trState->skipApplyTenantPrefix = tenantPrefixPrepended;
+				CODE_PROBE(true, "Commit of a dummy transaction in tenant keyspace");
+			}
 			tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.addReadConflictRange(range);
@@ -5942,6 +6014,10 @@ ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trS
 			wait(tr.commit());
 			return Void();
 		} catch (Error& e) {
+			// If the tenant is gone, then our original transaction won't be able to commit
+			if (e.code() == error_code_unknown_tenant) {
+				return Void();
+			}
 			TraceEvent("CommitDummyTransactionError")
 			    .errorUnsuppressed(e)
 			    .detail("Key", range.begin)
@@ -5965,16 +6041,17 @@ void Transaction::setupWatches() {
 		Future<Version> watchVersion = getCommittedVersion() > 0 ? getCommittedVersion() : getReadVersion();
 
 		for (int i = 0; i < watches.size(); ++i)
-			watches[i]->setWatch(watchValueMap(watchVersion,
-			                                   trState->getTenantInfo(),
-			                                   watches[i]->key,
-			                                   watches[i]->value,
-			                                   trState->cx,
-			                                   trState->options.readTags,
-			                                   trState->spanContext,
-			                                   trState->taskID,
-			                                   trState->debugID,
-			                                   trState->useProvisionalProxies));
+			watches[i]->setWatch(
+			    watchValueMap(watchVersion,
+			                  trState->getTenantInfo(),
+			                  watches[i]->key,
+			                  watches[i]->value,
+			                  trState->cx,
+			                  trState->options.readTags,
+			                  trState->spanContext,
+			                  trState->taskID,
+			                  trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
+			                  trState->useProvisionalProxies));
 
 		watches.clear();
 	} catch (Error&) {
@@ -6063,30 +6140,46 @@ ACTOR Future<Optional<ClientTrCommitCostEstimation>> estimateCommitCosts(Referen
 // TODO: send the prefix as part of the commit request and ship it all the way
 // through to the storage servers
 void applyTenantPrefix(CommitTransactionRequest& req, Key tenantPrefix) {
+	VectorRef<MutationRef> updatedMutations;
+	updatedMutations.reserve(req.arena, req.transaction.mutations.size());
 	for (auto& m : req.transaction.mutations) {
+		StringRef param1 = m.param1;
+		StringRef param2 = m.param2;
 		if (m.param1 != metadataVersionKey) {
-			m.param1 = m.param1.withPrefix(tenantPrefix, req.arena);
+			param1 = m.param1.withPrefix(tenantPrefix, req.arena);
 			if (m.type == MutationRef::ClearRange) {
-				m.param2 = m.param2.withPrefix(tenantPrefix, req.arena);
+				param2 = m.param2.withPrefix(tenantPrefix, req.arena);
 			} else if (m.type == MutationRef::SetVersionstampedKey) {
-				uint8_t* key = mutateString(m.param1);
-				int* offset = reinterpret_cast<int*>(&key[m.param1.size() - 4]);
+				uint8_t* key = mutateString(param1);
+				int* offset = reinterpret_cast<int*>(&key[param1.size() - 4]);
 				*offset += tenantPrefix.size();
 			}
 		}
+		updatedMutations.push_back(req.arena, MutationRef(MutationRef::Type(m.type), param1, param2));
 	}
+	req.transaction.mutations = updatedMutations;
 
-	for (auto& rc : req.transaction.read_conflict_ranges) {
+	VectorRef<KeyRangeRef> updatedReadConflictRanges;
+	updatedReadConflictRanges.reserve(req.arena, req.transaction.read_conflict_ranges.size());
+	for (auto const& rc : req.transaction.read_conflict_ranges) {
 		if (rc.begin != metadataVersionKey) {
-			rc = rc.withPrefix(tenantPrefix, req.arena);
+			updatedReadConflictRanges.push_back(req.arena, rc.withPrefix(tenantPrefix, req.arena));
+		} else {
+			updatedReadConflictRanges.push_back(req.arena, rc);
 		}
 	}
+	req.transaction.read_conflict_ranges = updatedReadConflictRanges;
 
+	VectorRef<KeyRangeRef> updatedWriteConflictRanges;
+	updatedWriteConflictRanges.reserve(req.arena, req.transaction.write_conflict_ranges.size());
 	for (auto& wc : req.transaction.write_conflict_ranges) {
 		if (wc.begin != metadataVersionKey) {
-			wc = wc.withPrefix(tenantPrefix, req.arena);
+			updatedWriteConflictRanges.push_back(req.arena, wc.withPrefix(tenantPrefix, req.arena));
+		} else {
+			updatedWriteConflictRanges.push_back(req.arena, wc);
 		}
 	}
+	req.transaction.write_conflict_ranges = updatedWriteConflictRanges;
 }
 
 ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
@@ -6095,14 +6188,18 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
 	state TraceInterval interval("TransactionCommit");
 	state double startTime = now();
 	state Span span("NAPI:tryCommit"_loc, trState->spanContext);
-	state Optional<UID> debugID = trState->debugID;
+	state Optional<UID> debugID = trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>();
+	state TenantPrefixPrepended tenantPrefixPrepended = TenantPrefixPrepended::False;
 	if (debugID.present()) {
 		TraceEvent(interval.begin()).detail("Parent", debugID.get());
 	}
 	try {
 		if (CLIENT_BUGGIFY) {
-			throw deterministicRandom()->randomChoice(std::vector<Error>{
-			    not_committed(), transaction_too_old(), proxy_memory_limit_exceeded(), commit_unknown_result() });
+			throw deterministicRandom()->randomChoice(std::vector<Error>{ not_committed(),
+			                                                              transaction_too_old(),
+			                                                              commit_proxy_memory_limit_exceeded(),
+			                                                              grv_proxy_memory_limit_exceeded(),
+			                                                              commit_unknown_result() });
 		}
 
 		if (req.tagSet.present() && trState->options.priority < TransactionPriority::IMMEDIATE) {
@@ -6113,7 +6210,9 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
 		}
 
 		state Key tenantPrefix;
-		if (trState->tenant().present()) {
+		// skipApplyTenantPrefix is set only in the context of a commitDummyTransaction()
+		// (see member declaration)
+		if (trState->tenant().present() && !trState->skipApplyTenantPrefix) {
 			KeyRangeLocationInfo locationInfo = wait(getKeyLocation(trState,
 			                                                        ""_sr,
 			                                                        &StorageServerInterface::getValue,
@@ -6121,11 +6220,11 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
 			                                                        UseTenant::True,
 			                                                        req.transaction.read_snapshot));
 			applyTenantPrefix(req, locationInfo.tenantEntry.prefix);
+			tenantPrefixPrepended = TenantPrefixPrepended::True;
 			tenantPrefix = locationInfo.tenantEntry.prefix;
 		}
-
+		CODE_PROBE(trState->skipApplyTenantPrefix, "Tenant prefix prepend skipped for dummy transaction");
 		req.tenantInfo = trState->getTenantInfo();
-
 		startTime = now();
 		state Optional<UID> commitID = Optional<UID>();
 
@@ -6249,21 +6348,25 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
 				KeyRangeRef selfConflictingRange =
 				    intersects(req.transaction.write_conflict_ranges, req.transaction.read_conflict_ranges).get();
 
-				TEST(true); // Waiting for dummy transaction to report commit_unknown_result
+				CODE_PROBE(true, "Waiting for dummy transaction to report commit_unknown_result");
 
-				wait(commitDummyTransaction(trState, singleKeyRange(selfConflictingRange.begin)));
+				wait(
+				    commitDummyTransaction(trState, singleKeyRange(selfConflictingRange.begin), tenantPrefixPrepended));
 			}
 
 			// The user needs to be informed that we aren't sure whether the commit happened.  Standard retry loops
 			// retry it anyway (relying on transaction idempotence) but a client might do something else.
 			throw commit_unknown_result();
 		} else if (e.code() == error_code_unknown_tenant) {
+			// Rather than reset the tenant and retry just the commit, we need to throw this error to the user and let
+			// them retry the whole transaction
 			ASSERT(trState->tenant().present());
 			trState->cx->invalidateCachedTenant(trState->tenant().get());
 			throw;
 		} else {
 			if (e.code() != error_code_transaction_too_old && e.code() != error_code_not_committed &&
-			    e.code() != error_code_database_locked && e.code() != error_code_proxy_memory_limit_exceeded &&
+			    e.code() != error_code_database_locked && e.code() != error_code_commit_proxy_memory_limit_exceeded &&
+			    e.code() != error_code_grv_proxy_memory_limit_exceeded &&
 			    e.code() != error_code_batch_transaction_throttled && e.code() != error_code_tag_throttled &&
 			    e.code() != error_code_process_behind && e.code() != error_code_future_version &&
 			    e.code() != error_code_tenant_not_found) {
@@ -6489,10 +6592,10 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 			    makeReference<TransactionLogInfo>(value.get().printable(), TransactionLogInfo::DONT_LOG);
 			trState->trLogInfo->maxFieldLength = trState->options.maxTransactionLoggingFieldLength;
 		}
-		if (trState->debugID.present()) {
+		if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
 			TraceEvent(SevInfo, "TransactionBeingTraced")
 			    .detail("DebugTransactionID", trState->trLogInfo->identifier)
-			    .detail("ServerTraceID", trState->debugID.get());
+			    .detail("ServerTraceID", trState->readOptions.get().debugID.get());
 		}
 		break;
 
@@ -6524,10 +6627,11 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 	case FDBTransactionOptions::SERVER_REQUEST_TRACING:
 		validateOptionValueNotPresent(value);
 		debugTransaction(deterministicRandom()->randomUniqueID());
-		if (trState->trLogInfo && !trState->trLogInfo->identifier.empty()) {
+		if (trState->trLogInfo && !trState->trLogInfo->identifier.empty() && trState->readOptions.present() &&
+		    trState->readOptions.get().debugID.present()) {
 			TraceEvent(SevInfo, "TransactionBeingTraced")
 			    .detail("DebugTransactionID", trState->trLogInfo->identifier)
-			    .detail("ServerTraceID", trState->debugID.get());
+			    .detail("ServerTraceID", trState->readOptions.get().debugID.get());
 		}
 		break;
 
@@ -6562,6 +6666,11 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 
 	case FDBTransactionOptions::USE_PROVISIONAL_PROXIES:
 		validateOptionValueNotPresent(value);
+		if (trState->hasTenant()) {
+			Error e = invalid_option();
+			TraceEvent(SevWarn, "TenantTransactionUseProvisionalProxies").error(e).detail("Tenant", trState->tenant());
+			throw e;
+		}
 		trState->options.getReadVersionFlags |= GetReadVersionRequest::FLAG_USE_PROVISIONAL_PROXIES;
 		trState->useProvisionalProxies = UseProvisionalProxies::True;
 		break;
@@ -6587,7 +6696,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 		if (value.get().size() != 33) {
 			throw invalid_option_value();
 		}
-		TEST(true); // Adding link in FDBTransactionOptions::SPAN_PARENT
+		CODE_PROBE(true, "Adding link in FDBTransactionOptions::SPAN_PARENT");
 		span.setParent(BinaryReader::fromStringRef<SpanContext>(value.get(), IncludeVersion()));
 		break;
 
@@ -6603,6 +6712,9 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 
 	case FDBTransactionOptions::USE_GRV_CACHE:
 		validateOptionValueNotPresent(value);
+		if (apiVersionAtLeast(720) && !trState->cx->sharedStatePtr) {
+			throw invalid_option();
+		}
 		if (trState->numErrors == 0) {
 			trState->options.useGrvCache = true;
 		}
@@ -6626,6 +6738,13 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 		trState->options.rawAccess = true;
 		break;
 
+	case FDBTransactionOptions::AUTHORIZATION_TOKEN:
+		if (value.present())
+			trState->authToken = Standalone<StringRef>(value.get());
+		else
+			trState->authToken.reset();
+		break;
+
 	default:
 		break;
 	}
@@ -6667,10 +6786,10 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanContext parentSpa
 						for (auto& tag : tags) {
 							auto itr = v.tagThrottleInfo.find(tag.first);
 							if (itr == v.tagThrottleInfo.end()) {
-								TEST(true); // Removing client throttle
+								CODE_PROBE(true, "Removing client throttle");
 								priorityThrottledTags.erase(tag.first);
 							} else {
-								TEST(true); // Setting client throttle
+								CODE_PROBE(true, "Setting client throttle");
 								auto result = priorityThrottledTags.try_emplace(tag.first, itr->second);
 								if (!result.second) {
 									result.first->second.update(itr->second);
@@ -6695,10 +6814,15 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanContext parentSpa
 				}
 			}
 		} catch (Error& e) {
-			if (e.code() != error_code_broken_promise && e.code() != error_code_batch_transaction_throttled)
+			if (e.code() != error_code_broken_promise && e.code() != error_code_batch_transaction_throttled &&
+			    e.code() != error_code_grv_proxy_memory_limit_exceeded)
 				TraceEvent(SevError, "GetConsistentReadVersionError").error(e);
 			if (e.code() == error_code_batch_transaction_throttled && !cx->apiVersionAtLeast(630)) {
 				wait(delayJittered(5.0));
+			} else if (e.code() == error_code_grv_proxy_memory_limit_exceeded) {
+				// FIXME(xwang): the better way is to let this error broadcast to transaction.onError(e), otherwise the
+				// txn->cx counter doesn't make sense
+				wait(delayJittered(CLIENT_KNOBS->GRV_ERROR_RETRY_DELAY));
 			} else {
 				throw;
 			}
@@ -6716,26 +6840,22 @@ ACTOR Future<Void> readVersionBatcher(DatabaseContext* cx,
 	state Future<Void> timeout;
 	state Optional<UID> debugID;
 	state bool send_batch;
-	state Reference<Histogram> batchSizeDist = Histogram::getHistogram(LiteralStringRef("GrvBatcher"),
-	                                                                   LiteralStringRef("ClientGrvBatchSize"),
-	                                                                   Histogram::Unit::countLinear,
-	                                                                   0,
-	                                                                   CLIENT_KNOBS->MAX_BATCH_SIZE * 2);
+	state Reference<Histogram> batchSizeDist = Histogram::getHistogram(
+	    "GrvBatcher"_sr, "ClientGrvBatchSize"_sr, Histogram::Unit::countLinear, 0, CLIENT_KNOBS->MAX_BATCH_SIZE * 2);
 	state Reference<Histogram> batchIntervalDist =
-	    Histogram::getHistogram(LiteralStringRef("GrvBatcher"),
-	                            LiteralStringRef("ClientGrvBatchInterval"),
+	    Histogram::getHistogram("GrvBatcher"_sr,
+	                            "ClientGrvBatchInterval"_sr,
 	                            Histogram::Unit::microseconds,
 	                            0,
 	                            CLIENT_KNOBS->GRV_BATCH_TIMEOUT * 1000000 * 2);
-	state Reference<Histogram> grvReplyLatencyDist = Histogram::getHistogram(
-	    LiteralStringRef("GrvBatcher"), LiteralStringRef("ClientGrvReplyLatency"), Histogram::Unit::microseconds);
+	state Reference<Histogram> grvReplyLatencyDist =
+	    Histogram::getHistogram("GrvBatcher"_sr, "ClientGrvReplyLatency"_sr, Histogram::Unit::microseconds);
 	state double lastRequestTime = now();
 
 	state TransactionTagMap<uint32_t> tags;
 
 	// dynamic batching
 	state PromiseStream<double> replyTimes;
-	state PromiseStream<Error> _errorStream;
 	state double batchTime = 0;
 	state Span span("NAPI:readVersionBatcher"_loc);
 	loop {
@@ -6853,7 +6973,7 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
 				if (itr->second.expired()) {
 					priorityThrottledTags.erase(itr);
 				} else if (itr->second.throttleDuration() > 0) {
-					TEST(true); // throttling transaction after getting read version
+					CODE_PROBE(true, "throttling transaction after getting read version");
 					++trState->cx->transactionReadVersionsThrottled;
 					throw tag_throttled();
 				}
@@ -6959,12 +7079,12 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 			}
 
 			if (maxThrottleDelay > 0.0 && !canRecheck) { // TODO: allow delaying?
-				TEST(true); // Throttling tag before GRV request
+				CODE_PROBE(true, "Throttling tag before GRV request");
 				++trState->cx->transactionReadVersionsThrottled;
 				readVersion = tag_throttled();
 				return readVersion;
 			} else {
-				TEST(maxThrottleDelay > 0.0); // Rechecking throttle
+				CODE_PROBE(maxThrottleDelay > 0.0, "Rechecking throttle");
 			}
 
 			for (auto& tag : trState->options.tags) {
@@ -6983,7 +7103,9 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 
 		Location location = "NAPI:getReadVersion"_loc;
 		SpanContext spanContext = generateSpanID(trState->cx->transactionTracingSample, trState->spanContext);
-		auto const req = DatabaseContext::VersionRequest(spanContext, trState->options.tags, trState->debugID);
+		Optional<UID> versionDebugID =
+		    trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>();
+		auto const req = DatabaseContext::VersionRequest(spanContext, trState->options.tags, versionDebugID);
 		batcher.stream.send(req);
 		trState->startTime = now();
 		readVersion = extractReadVersion(trState, location, spanContext, req.reply.getFuture(), metadataVersion);
@@ -7141,14 +7263,16 @@ Future<Void> Transaction::onError(Error const& e) {
 		return client_invalid_operation();
 	}
 	if (e.code() == error_code_not_committed || e.code() == error_code_commit_unknown_result ||
-	    e.code() == error_code_database_locked || e.code() == error_code_proxy_memory_limit_exceeded ||
-	    e.code() == error_code_process_behind || e.code() == error_code_batch_transaction_throttled ||
-	    e.code() == error_code_tag_throttled) {
+	    e.code() == error_code_database_locked || e.code() == error_code_commit_proxy_memory_limit_exceeded ||
+	    e.code() == error_code_grv_proxy_memory_limit_exceeded || e.code() == error_code_process_behind ||
+	    e.code() == error_code_batch_transaction_throttled || e.code() == error_code_tag_throttled ||
+	    e.code() == error_code_blob_granule_request_failed) {
 		if (e.code() == error_code_not_committed)
 			++trState->cx->transactionsNotCommitted;
 		else if (e.code() == error_code_commit_unknown_result)
 			++trState->cx->transactionsMaybeCommitted;
-		else if (e.code() == error_code_proxy_memory_limit_exceeded)
+		else if (e.code() == error_code_commit_proxy_memory_limit_exceeded ||
+		         e.code() == error_code_grv_proxy_memory_limit_exceeded)
 			++trState->cx->transactionsResourceConstrained;
 		else if (e.code() == error_code_process_behind)
 			++trState->cx->transactionsProcessBehind;
@@ -7205,7 +7329,7 @@ ACTOR Future<StorageMetrics> doGetStorageMetrics(Database cx, KeyRange keys, Ref
 ACTOR Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx, KeyRange keys) {
 	state Span span("NAPI:GetStorageMetricsLargeKeyRange"_loc);
 	std::vector<KeyRangeLocationInfo> locations = wait(getKeyRangeLocations(cx,
-	                                                                        Optional<TenantName>(),
+	                                                                        TenantInfo(),
 	                                                                        keys,
 	                                                                        std::numeric_limits<int>::max(),
 	                                                                        Reverse::False,
@@ -7307,7 +7431,7 @@ ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Da
 		                          // to find the read-hot sub ranges within a read-hot shard.
 		std::vector<KeyRangeLocationInfo> locations =
 		    wait(getKeyRangeLocations(cx,
-		                              Optional<TenantName>(),
+		                              TenantInfo(),
 		                              keys,
 		                              shardLimit,
 		                              Reverse::False,
@@ -7343,10 +7467,10 @@ ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Da
 			wait(waitForAll(fReplies));
 
 			if (nLocs == 1) {
-				TEST(true); // Single-shard read hot range request
+				CODE_PROBE(true, "Single-shard read hot range request");
 				return fReplies[0].get().readHotRanges;
 			} else {
-				TEST(true); // Multi-shard read hot range request
+				CODE_PROBE(true, "Multi-shard read hot range request");
 				Standalone<VectorRef<ReadHotRangeWithMetrics>> results;
 				for (int i = 0; i < nLocs; i++) {
 					results.append(results.arena(),
@@ -7378,7 +7502,7 @@ ACTOR Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(Databa
 	state Span span("NAPI:WaitStorageMetrics"_loc, generateSpanID(cx->transactionTracingSample));
 	loop {
 		std::vector<KeyRangeLocationInfo> locations = wait(getKeyRangeLocations(cx,
-		                                                                        Optional<TenantName>(),
+		                                                                        TenantInfo(),
 		                                                                        keys,
 		                                                                        shardLimit,
 		                                                                        Reverse::False,
@@ -7536,9 +7660,7 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(Reference<Transa
 				trState->cx->invalidateCache(locations[0].tenantEntry.prefix, keys);
 				wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
 			} else if (e.code() == error_code_unknown_tenant) {
-				ASSERT(trState->tenant().present());
-				trState->cx->invalidateCachedTenant(trState->tenant().get());
-				wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
+				wait(trState->handleUnknownTenant());
 			} else {
 				TraceEvent(SevError, "GetRangeSplitPoints").error(e);
 				throw;
@@ -7552,18 +7674,19 @@ ACTOR Future<TenantMapEntry> blobGranuleGetTenantEntry(Transaction* self, Key ra
 	Optional<KeyRangeLocationInfo> cachedLocationInfo =
 	    self->trState->cx->getCachedLocation(self->getTenant().get(), rangeStartKey, Reverse::False);
 	if (!cachedLocationInfo.present()) {
-		KeyRangeLocationInfo l = wait(getKeyLocation_internal(self->trState->cx,
-		                                                      self->getTenant().get(),
-		                                                      rangeStartKey,
-		                                                      self->trState->spanContext,
-		                                                      self->trState->debugID,
-		                                                      self->trState->useProvisionalProxies,
-		                                                      Reverse::False,
-		                                                      latestVersion));
-		self->trState->tenantId = l.tenantEntry.id;
+		KeyRangeLocationInfo l = wait(getKeyLocation_internal(
+		    self->trState->cx,
+		    self->trState->getTenantInfo(AllowInvalidTenantID::True),
+		    rangeStartKey,
+		    self->trState->spanContext,
+		    self->trState->readOptions.present() ? self->trState->readOptions.get().debugID : Optional<UID>(),
+		    self->trState->useProvisionalProxies,
+		    Reverse::False,
+		    latestVersion));
+		self->trState->trySetTenantId(l.tenantEntry.id);
 		return l.tenantEntry;
 	} else {
-		self->trState->tenantId = cachedLocationInfo.get().tenantEntry.id;
+		self->trState->trySetTenantId(cachedLocationInfo.get().tenantEntry.id);
 		return cachedLocationInfo.get().tenantEntry;
 	}
 }
@@ -7577,7 +7700,9 @@ Future<Standalone<VectorRef<KeyRef>>> Transaction::getRangeSplitPoints(KeyRange
 
 // the blob granule requests are a bit funky because they piggyback off the existing transaction to read from the system
 // keyspace
-ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRangesActor(Transaction* self, KeyRange keyRange) {
+ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRangesActor(Transaction* self,
+                                                                           KeyRange keyRange,
+                                                                           int rangeLimit) {
 	// FIXME: use streaming range read
 	state KeyRange currentRange = keyRange;
 	state Standalone<VectorRef<KeyRangeRef>> results;
@@ -7598,23 +7723,24 @@ ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRangesActor(Trans
 		if (tenantPrefix.present()) {
 			state Standalone<StringRef> mappingPrefix = tenantPrefix.get().withPrefix(blobGranuleMappingKeys.begin);
 
-			// basically krmGetRange, but enable it to not use tenant without RAW_ACCESS by doing manual getRange with
-			// UseTenant::False
-			GetRangeLimits limits(1000);
+			// basically krmGetRangeUnaligned, but enable it to not use tenant without RAW_ACCESS by doing manual
+			// getRange with UseTenant::False
+			GetRangeLimits limits(2 * rangeLimit + 2);
 			limits.minRows = 2;
+
 			RangeResult rawMapping = wait(getRange(self->trState,
 			                                       self->getReadVersion(),
 			                                       lastLessOrEqual(keyRange.begin.withPrefix(mappingPrefix)),
-			                                       firstGreaterThan(keyRange.end.withPrefix(mappingPrefix)),
+			                                       KeySelectorRef(keyRange.end.withPrefix(mappingPrefix), false, +2),
 			                                       limits,
 			                                       Reverse::False,
 			                                       UseTenant::False));
 			// strip off mapping prefix
-			blobGranuleMapping = krmDecodeRanges(mappingPrefix, currentRange, rawMapping);
+			blobGranuleMapping = krmDecodeRanges(mappingPrefix, currentRange, rawMapping, false);
 		} else {
 			wait(store(
 			    blobGranuleMapping,
-			    krmGetRanges(
+			    krmGetRangesUnaligned(
 			        self, blobGranuleMappingKeys.begin, currentRange, 1000, GetRangeLimits::BYTE_LIMIT_UNLIMITED)));
 		}
 
@@ -7622,6 +7748,9 @@ ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRangesActor(Trans
 			if (blobGranuleMapping[i].value.size()) {
 				results.push_back(results.arena(),
 				                  KeyRangeRef(blobGranuleMapping[i].key, blobGranuleMapping[i + 1].key));
+				if (results.size() == rangeLimit) {
+					return results;
+				}
 			}
 		}
 		results.arena().dependsOn(blobGranuleMapping.arena());
@@ -7633,8 +7762,8 @@ ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRangesActor(Trans
 	}
 }
 
-Future<Standalone<VectorRef<KeyRangeRef>>> Transaction::getBlobGranuleRanges(const KeyRange& range) {
-	return ::getBlobGranuleRangesActor(this, range);
+Future<Standalone<VectorRef<KeyRangeRef>>> Transaction::getBlobGranuleRanges(const KeyRange& range, int rangeLimit) {
+	return ::getBlobGranuleRangesActor(this, range, rangeLimit);
 }
 
 // hack (for now) to get blob worker interface into load balance
@@ -7648,7 +7777,11 @@ ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesActor(
     KeyRange range,
     Version begin,
     Optional<Version> read,
-    Version* readVersionOut) { // read not present is "use transaction version"
+    Version* readVersionOut,
+    int chunkLimit,
+    bool summarize) { // read not present is "use transaction version"
+
+	ASSERT(chunkLimit > 0);
 
 	state RangeResult blobGranuleMapping;
 	state Key granuleStartKey;
@@ -7689,7 +7822,7 @@ ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesActor(
 
 		// basically krmGetRange, but enable it to not use tenant without RAW_ACCESS by doing manual getRange with
 		// UseTenant::False
-		GetRangeLimits limits(1000);
+		GetRangeLimits limits(CLIENT_KNOBS->BG_TOO_MANY_GRANULES);
 		limits.minRows = 2;
 		RangeResult rawMapping = wait(getRange(self->trState,
 		                                       self->getReadVersion(),
@@ -7704,19 +7837,24 @@ ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesActor(
 		blobGranuleMapping = krmDecodeRanges(prefix, range, rawMapping);
 	} else {
 		self->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		wait(store(
-		    blobGranuleMapping,
-		    krmGetRanges(self, blobGranuleMappingKeys.begin, keyRange, 1000, GetRangeLimits::BYTE_LIMIT_UNLIMITED)));
+		wait(store(blobGranuleMapping,
+		           krmGetRanges(self,
+		                        blobGranuleMappingKeys.begin,
+		                        keyRange,
+		                        CLIENT_KNOBS->BG_TOO_MANY_GRANULES,
+		                        GetRangeLimits::BYTE_LIMIT_UNLIMITED)));
 	}
 	if (blobGranuleMapping.more) {
 		if (BG_REQUEST_DEBUG) {
 			fmt::print(
 			    "BG Mapping for [{0} - %{1}) too large!\n", keyRange.begin.printable(), keyRange.end.printable());
 		}
-		TraceEvent(SevWarn, "BGMappingTooLarge").detail("Range", range).detail("Max", 1000);
+		TraceEvent(SevWarn, "BGMappingTooLarge")
+		    .detail("Range", range)
+		    .detail("Max", CLIENT_KNOBS->BG_TOO_MANY_GRANULES);
 		throw unsupported_operation();
 	}
-	ASSERT(!blobGranuleMapping.more && blobGranuleMapping.size() < CLIENT_KNOBS->TOO_MANY);
+	ASSERT(!blobGranuleMapping.more && blobGranuleMapping.size() <= CLIENT_KNOBS->BG_TOO_MANY_GRANULES);
 
 	if (blobGranuleMapping.size() < 2) {
 		throw blob_granule_transaction_too_old();
@@ -7735,7 +7873,6 @@ ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesActor(
 				fmt::print("Key range [{0} - {1}) missing worker assignment!\n",
 				           granuleStartKey.printable(),
 				           granuleEndKey.printable());
-				// TODO probably new exception type instead
 			}
 			throw blob_granule_transaction_too_old();
 		}
@@ -7762,11 +7899,9 @@ ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesActor(
 			    getValue(self->trState, blobWorkerListKeyFor(workerId), self->getReadVersion(), UseTenant::False)));
 			// from the time the mapping was read from the db, the associated blob worker
 			// could have died and so its interface wouldn't be present as part of the blobWorkerList
-			// we persist in the db. So throw wrong_shard_server to get the new mapping
+			// we persist in the db. So throw blob_granule_request_failed to get the new mapping
 			if (!workerInterface.present()) {
-				// need to re-read mapping, throw transaction_too_old so client retries. TODO better error?
-				// throw wrong_shard_server();
-				throw transaction_too_old();
+				throw blob_granule_request_failed();
 			}
 			// FIXME: maybe just want to insert here if there are racing queries for the same worker or something?
 			self->trState->cx->blobWorker_interf[workerId] = decodeBlobWorkerListValue(workerInterface.get());
@@ -7799,6 +7934,7 @@ ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesActor(
 		req.readVersion = rv;
 		req.tenantInfo = self->getTenant().present() ? self->trState->getTenantInfo() : TenantInfo();
 		req.canCollapseBegin = true; // TODO make this a parameter once we support it
+		req.summarize = summarize;
 
 		std::vector<Reference<ReferencedInterface<BlobWorkerInterface>>> v;
 		v.push_back(
@@ -7855,7 +7991,7 @@ ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesActor(
 						if (!results.empty() && results.back().keyRange.end != chunk.keyRange.begin) {
 							ASSERT(results.back().keyRange.end > chunk.keyRange.begin);
 							ASSERT(results.back().keyRange.end <= chunk.keyRange.end);
-							TEST(true); // Merge while reading granule range
+							CODE_PROBE(true, "Merge while reading granule range");
 							while (!results.empty() && results.back().keyRange.begin >= chunk.keyRange.begin) {
 								// TODO: we can't easily un-depend the arenas for these guys, but that's ok as this
 								// should be rare
@@ -7869,6 +8005,12 @@ ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesActor(
 							chunkEndKey = chunkEndKey.removePrefix(tenantPrefix.get());
 						}
 						keyRange = KeyRangeRef(std::min(chunkEndKey, keyRange.end), keyRange.end);
+						if (summarize && results.size() == chunkLimit) {
+							break;
+						}
+					}
+					if (summarize && results.size() == chunkLimit) {
+						break;
 					}
 				}
 				// if we detect that this blob worker fails, cancel the request, as otherwise load balance will
@@ -7894,10 +8036,8 @@ ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesActor(
 				           e.name());
 			}
 			// worker is up but didn't actually have granule, or connection failed
-			if (e.code() == error_code_wrong_shard_server || e.code() == error_code_connection_failed ||
-			    e.code() == error_code_unknown_tenant) {
-				// need to re-read mapping, throw transaction_too_old so client retries. TODO better error?
-				throw transaction_too_old();
+			if (e.code() == error_code_wrong_shard_server || e.code() == error_code_connection_failed) {
+				throw blob_granule_request_failed();
 			}
 			throw e;
 		}
@@ -7917,7 +8057,36 @@ Future<Standalone<VectorRef<BlobGranuleChunkRef>>> Transaction::readBlobGranules
                                                                                  Version begin,
                                                                                  Optional<Version> readVersion,
                                                                                  Version* readVersionOut) {
-	return readBlobGranulesActor(this, range, begin, readVersion, readVersionOut);
+	return readBlobGranulesActor(
+	    this, range, begin, readVersion, readVersionOut, std::numeric_limits<int>::max(), false);
+}
+
+ACTOR Future<Standalone<VectorRef<BlobGranuleSummaryRef>>> summarizeBlobGranulesActor(Transaction* self,
+                                                                                      KeyRange range,
+                                                                                      Optional<Version> summaryVersion,
+                                                                                      int rangeLimit) {
+	state Version readVersionOut;
+	Standalone<VectorRef<BlobGranuleChunkRef>> chunks =
+	    wait(readBlobGranulesActor(self, range, 0, summaryVersion, &readVersionOut, rangeLimit, true));
+	ASSERT(chunks.size() <= rangeLimit);
+	ASSERT(!summaryVersion.present() || readVersionOut == summaryVersion.get());
+	Standalone<VectorRef<BlobGranuleSummaryRef>> summaries;
+	summaries.reserve(summaries.arena(), chunks.size());
+	for (auto& it : chunks) {
+		summaries.push_back(summaries.arena(), summarizeGranuleChunk(summaries.arena(), it));
+	}
+
+	return summaries;
+}
+
+Future<Standalone<VectorRef<BlobGranuleSummaryRef>>>
+Transaction::summarizeBlobGranules(const KeyRange& range, Optional<Version> summaryVersion, int rangeLimit) {
+	return summarizeBlobGranulesActor(this, range, summaryVersion, rangeLimit);
+}
+
+void Transaction::addGranuleMaterializeStats(const GranuleMaterializeStats& stats) {
+	trState->cx->bgReadInputBytes += stats.inputBytes;
+	trState->cx->bgReadOutputBytes += stats.outputBytes;
 }
 
 ACTOR Future<Version> setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware) {
@@ -7941,6 +8110,112 @@ ACTOR Future<Version> setPerpetualStorageWiggle(Database cx, bool enable, LockAw
 	return version;
 }
 
+ACTOR Future<Version> checkBlobSubrange(Database db, KeyRange keyRange, Optional<Version> version) {
+	state Transaction tr(db);
+	loop {
+		try {
+			state Version summaryVersion;
+			if (version.present()) {
+				summaryVersion = version.get();
+			} else {
+				wait(store(summaryVersion, tr.getReadVersion()));
+			}
+			// same properties as a read for validating granule is readable, just much less memory and network bandwidth
+			// used
+			wait(success(tr.summarizeBlobGranules(keyRange, summaryVersion, std::numeric_limits<int>::max())));
+			return summaryVersion;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+ACTOR Future<Version> verifyBlobRangeActor(Reference<DatabaseContext> cx, KeyRange range, Optional<Version> version) {
+	state Database db(cx);
+	state Transaction tr(db);
+	state Standalone<VectorRef<KeyRangeRef>> allRanges;
+	state KeyRange curRegion = KeyRangeRef(range.begin, range.begin);
+	state Version readVersionOut = invalidVersion;
+	state int batchSize = BUGGIFY ? deterministicRandom()->randomInt(2, 10) : CLIENT_KNOBS->BG_TOO_MANY_GRANULES / 2;
+	state int loadSize = (BUGGIFY ? deterministicRandom()->randomInt(1, 20) : 20) * batchSize;
+
+	if (version.present()) {
+		if (version.get() == latestVersion) {
+			loop {
+				try {
+					Version _version = wait(tr.getReadVersion());
+					version = _version;
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+		}
+		if (version.get() <= 0) {
+			TraceEvent("VerifyBlobInvalidVersion").detail("Range", range).detail("Version", version);
+			throw unsupported_operation();
+		}
+	}
+
+	loop {
+		if (curRegion.begin >= range.end) {
+			return readVersionOut;
+		}
+		loop {
+			try {
+				wait(store(allRanges, tr.getBlobGranuleRanges(KeyRangeRef(curRegion.begin, range.end), loadSize)));
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		if (allRanges.empty()) {
+			if (curRegion.begin < range.end) {
+				return invalidVersion;
+			}
+			return readVersionOut;
+		}
+
+		state std::vector<Future<Version>> checkParts;
+		// Chunk up to smaller ranges than this limit. Must be smaller than BG_TOO_MANY_GRANULES to not hit the limit
+		int batchCount = 0;
+		for (auto& it : allRanges) {
+			if (it.begin > curRegion.end) {
+				return invalidVersion;
+			}
+
+			curRegion = KeyRangeRef(curRegion.begin, it.end);
+			batchCount++;
+
+			if (batchCount == batchSize) {
+				checkParts.push_back(checkBlobSubrange(db, curRegion, version));
+				batchCount = 0;
+				curRegion = KeyRangeRef(curRegion.end, curRegion.end);
+			}
+		}
+		if (!curRegion.empty()) {
+			checkParts.push_back(checkBlobSubrange(db, curRegion, version));
+		}
+
+		try {
+			wait(waitForAll(checkParts));
+		} catch (Error& e) {
+			if (e.code() == error_code_blob_granule_transaction_too_old) {
+				return invalidVersion;
+			}
+			throw e;
+		}
+		ASSERT(!checkParts.empty());
+		readVersionOut = checkParts.back().get();
+		curRegion = KeyRangeRef(curRegion.end, curRegion.end);
+	}
+}
+
+Future<Version> DatabaseContext::verifyBlobRange(const KeyRange& range, Optional<Version> version) {
+	return verifyBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, version);
+}
+
 ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
                                                                                       bool primary,
                                                                                       bool use_system_priority) {
@@ -7948,7 +8223,8 @@ ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleV
 	state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap(readKey,
 	                                                                                          IncludeVersion());
 	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	state std::vector<std::pair<UID, StorageWiggleValue>> res;
+	state KeyBackedRangeResult<std::pair<UID, StorageWiggleValue>> res;
+
 	// read the wiggling pairs
 	loop {
 		try {
@@ -7964,7 +8240,7 @@ ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleV
 			wait(tr->onError(e));
 		}
 	}
-	return res;
+	return res.results;
 }
 
 ACTOR Future<Void> splitStorageMetricsStream(PromiseStream<Key> resultStream,
@@ -7982,7 +8258,7 @@ ACTOR Future<Void> splitStorageMetricsStream(PromiseStream<Key> resultStream,
 	loop {
 		state std::vector<KeyRangeLocationInfo> locations =
 		    wait(getKeyRangeLocations(cx,
-		                              Optional<TenantName>(),
+		                              TenantInfo(),
 		                              KeyRangeRef(beginKey, keys.end),
 		                              CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
 		                              Reverse::False,
@@ -8082,7 +8358,7 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(Database cx,
 	loop {
 		state std::vector<KeyRangeLocationInfo> locations =
 		    wait(getKeyRangeLocations(cx,
-		                              Optional<TenantName>(),
+		                              TenantInfo(),
 		                              keys,
 		                              CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
 		                              Reverse::False,
@@ -8167,7 +8443,7 @@ Reference<TransactionLogInfo> Transaction::createTrLogInfoProbabilistically(cons
 		    cx->globalConfig->get<double>(fdbClientInfoTxnSampleRate, CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY);
 		if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) &&
 		    deterministicRandom()->random01() < clientSamplingProbability &&
-		    (!g_network->isSimulated() || !g_simulator.speedUpSimulation)) {
+		    (!g_network->isSimulated() || !g_simulator->speedUpSimulation)) {
 			return makeReference<TransactionLogInfo>(TransactionLogInfo::DATABASE);
 		}
 	}
@@ -8327,7 +8603,7 @@ ACTOR Future<std::vector<CheckpointMetaData>> getCheckpointMetaData(Database cx,
 		try {
 			state std::vector<KeyRangeLocationInfo> locations =
 			    wait(getKeyRangeLocations(cx,
-			                              Optional<TenantName>(),
+			                              TenantInfo(),
 			                              keys,
 			                              CLIENT_KNOBS->TOO_MANY,
 			                              Reverse::False,
@@ -8488,16 +8764,13 @@ ACTOR static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef add
 	for (const auto& it : kvs) {
 		ClientWorkerInterface workerInterf =
 		    BinaryReader::fromStringRef<ClientWorkerInterface>(it.value, IncludeVersion());
-		Key primaryAddress =
-		    it.key.endsWith(LiteralStringRef(":tls")) ? it.key.removeSuffix(LiteralStringRef(":tls")) : it.key;
+		Key primaryAddress = it.key.endsWith(":tls"_sr) ? it.key.removeSuffix(":tls"_sr) : it.key;
 		workerInterfaces[primaryAddress] = workerInterf;
 		// Also add mapping from a worker's second address(if present) to its interface
 		if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
 			Key secondAddress =
 			    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
-			secondAddress = secondAddress.endsWith(LiteralStringRef(":tls"))
-			                    ? secondAddress.removeSuffix(LiteralStringRef(":tls"))
-			                    : secondAddress;
+			secondAddress = secondAddress.endsWith(":tls"_sr) ? secondAddress.removeSuffix(":tls"_sr) : secondAddress;
 			workerInterfaces[secondAddress] = workerInterf;
 		}
 	}
@@ -8569,38 +8842,28 @@ Future<DatabaseSharedState*> DatabaseContext::initSharedState() {
 }
 
 void DatabaseContext::setSharedState(DatabaseSharedState* p) {
-	ASSERT(p->protocolVersion == currentProtocolVersion);
+	ASSERT(p->protocolVersion == currentProtocolVersion());
 	sharedStatePtr = p;
 	sharedStatePtr->refCount++;
 }
 
 ACTOR Future<Void> storageFeedVersionUpdater(StorageServerInterface interf, ChangeFeedStorageData* self) {
-	state Promise<Void> destroyed = self->destroyed;
 	loop {
-		if (destroyed.isSet()) {
-			return Void();
-		}
 		if (self->version.get() < self->desired.get()) {
 			wait(delay(CLIENT_KNOBS->CHANGE_FEED_EMPTY_BATCH_TIME) || self->version.whenAtLeast(self->desired.get()));
-			if (destroyed.isSet()) {
-				return Void();
-			}
 			if (self->version.get() < self->desired.get()) {
 				try {
 					ChangeFeedVersionUpdateReply rep = wait(brokenPromiseToNever(
 					    interf.changeFeedVersionUpdate.getReply(ChangeFeedVersionUpdateRequest(self->desired.get()))));
-
 					if (rep.version > self->version.get()) {
 						self->version.set(rep.version);
 					}
 				} catch (Error& e) {
-					if (e.code() == error_code_server_overloaded) {
-						if (FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY > CLIENT_KNOBS->CHANGE_FEED_EMPTY_BATCH_TIME) {
-							wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY -
-							           CLIENT_KNOBS->CHANGE_FEED_EMPTY_BATCH_TIME));
-						}
-					} else {
-						throw e;
+					if (e.code() != error_code_server_overloaded) {
+						throw;
+					}
+					if (FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY > CLIENT_KNOBS->CHANGE_FEED_EMPTY_BATCH_TIME) {
+						wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY - CLIENT_KNOBS->CHANGE_FEED_EMPTY_BATCH_TIME));
 					}
 				}
 			}
@@ -8619,10 +8882,53 @@ Reference<ChangeFeedStorageData> DatabaseContext::getStorageData(StorageServerIn
 		newStorageUpdater->id = interf.id();
 		newStorageUpdater->interfToken = token;
 		newStorageUpdater->updater = storageFeedVersionUpdater(interf, newStorageUpdater.getPtr());
-		changeFeedUpdaters[token] = newStorageUpdater;
+		newStorageUpdater->context = this;
+		newStorageUpdater->created = now();
+		changeFeedUpdaters[token] = newStorageUpdater.getPtr();
 		return newStorageUpdater;
 	}
-	return it->second;
+	return Reference<ChangeFeedStorageData>::addRef(it->second);
+}
+
+Version DatabaseContext::getMinimumChangeFeedVersion() {
+	Version minVersion = std::numeric_limits<Version>::max();
+	for (auto& it : changeFeedUpdaters) {
+		if (now() - it.second->created > CLIENT_KNOBS->CHANGE_FEED_START_INTERVAL) {
+			minVersion = std::min(minVersion, it.second->version.get());
+		}
+	}
+	for (auto& it : notAtLatestChangeFeeds) {
+		if (now() - it.second->created > CLIENT_KNOBS->CHANGE_FEED_START_INTERVAL) {
+			minVersion = std::min(minVersion, it.second->getVersion());
+		}
+	}
+	return minVersion;
+}
+
+void DatabaseContext::setDesiredChangeFeedVersion(Version v) {
+	for (auto& it : changeFeedUpdaters) {
+		if (it.second->version.get() < v && it.second->desired.get() < v) {
+			it.second->desired.set(v);
+		}
+	}
+}
+
+ChangeFeedStorageData::~ChangeFeedStorageData() {
+	if (context) {
+		context->changeFeedUpdaters.erase(interfToken);
+	}
+}
+
+ChangeFeedData::ChangeFeedData(DatabaseContext* context)
+  : dbgid(deterministicRandom()->randomUniqueID()), context(context), notAtLatest(1), created(now()) {
+	if (context) {
+		context->notAtLatestChangeFeeds[dbgid] = this;
+	}
+}
+ChangeFeedData::~ChangeFeedData() {
+	if (context) {
+		context->notAtLatestChangeFeeds.erase(dbgid);
+	}
 }
 
 Version ChangeFeedData::getVersion() {
@@ -8816,6 +9122,9 @@ ACTOR Future<Void> partialChangeFeedStream(StorageServerInterface interf,
 					if (refresh.canBeSet() && !atLatestVersion && rep.atLatestVersion) {
 						atLatestVersion = true;
 						feedData->notAtLatest.set(feedData->notAtLatest.get() - 1);
+						if (feedData->notAtLatest.get() == 0 && feedData->context) {
+							feedData->context->notAtLatestChangeFeeds.erase(feedData->dbgid);
+						}
 					}
 					if (refresh.canBeSet() && rep.minStreamVersion > storageData->version.get()) {
 						storageData->version.set(rep.minStreamVersion);
@@ -8980,8 +9289,8 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 	state std::vector<Future<Void>> onErrors(interfs.size());
 	state std::vector<MutationAndVersionStream> streams(interfs.size());
 
-	TEST(interfs.size() > 10); // Large change feed merge cursor
-	TEST(interfs.size() > 100); // Very large change feed merge cursor
+	CODE_PROBE(interfs.size() > 10, "Large change feed merge cursor");
+	CODE_PROBE(interfs.size() > 100, "Very large change feed merge cursor");
 
 	state UID mergeCursorUID = UID();
 	state std::vector<UID> debugUIDs;
@@ -9006,11 +9315,6 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 		results->streams.push_back(it.first.changeFeedStream.getReplyStream(req));
 	}
 
-	for (auto& it : results->storageData) {
-		if (it->debugGetReferenceCount() == 2) {
-			db->changeFeedUpdaters.erase(it->interfToken);
-		}
-	}
 	results->maxSeenVersion = invalidVersion;
 	results->storageData.clear();
 	Promise<Void> refresh = results->refresh;
@@ -9019,6 +9323,10 @@ ACTOR Future<Void> mergeChangeFeedStream(Reference<DatabaseContext> db,
 		results->storageData.push_back(db->getStorageData(interfs[i].first));
 	}
 	results->notAtLatest.set(interfs.size());
+	if (results->context) {
+		results->context->notAtLatestChangeFeeds[results->dbgid] = results.getPtr();
+		results->created = now();
+	}
 	refresh.send(Void());
 
 	for (int i = 0; i < interfs.size(); i++) {
@@ -9061,6 +9369,8 @@ ACTOR Future<KeyRange> getChangeFeedRange(Reference<DatabaseContext> db, Databas
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			Version readVer = wait(tr.getReadVersion());
 			if (readVer < begin) {
 				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
@@ -9107,10 +9417,21 @@ ACTOR Future<Void> singleChangeFeedStreamInternal(KeyRange range,
 	// update lastReturned once the previous mutation has been consumed
 	if (*begin - 1 > results->lastReturnedVersion.get()) {
 		results->lastReturnedVersion.set(*begin - 1);
+		if (!refresh.canBeSet()) {
+			try {
+				// refresh is set if and only if this actor is cancelled
+				wait(Future<Void>(Void()));
+				// Catch any unexpected behavior if the above contract is broken
+				ASSERT(false);
+			} catch (Error& e) {
+				ASSERT(e.code() == error_code_actor_cancelled);
+				throw;
+			}
+		}
 	}
 
 	loop {
-
+		ASSERT(refresh.canBeSet());
 		state ChangeFeedStreamReply feedReply = waitNext(results->streams[0].getFuture());
 		*begin = feedReply.mutations.back().version + 1;
 
@@ -9160,6 +9481,9 @@ ACTOR Future<Void> singleChangeFeedStreamInternal(KeyRange range,
 		if (!atLatest && feedReply.atLatestVersion) {
 			atLatest = true;
 			results->notAtLatest.set(0);
+			if (results->context) {
+				results->context->notAtLatestChangeFeeds.erase(results->dbgid);
+			}
 		}
 
 		if (feedReply.minStreamVersion > results->storageData[0]->version.get()) {
@@ -9198,11 +9522,6 @@ ACTOR Future<Void> singleChangeFeedStream(Reference<DatabaseContext> db,
 
 	results->streams.clear();
 
-	for (auto& it : results->storageData) {
-		if (it->debugGetReferenceCount() == 2) {
-			db->changeFeedUpdaters.erase(it->interfToken);
-		}
-	}
 	results->streams.push_back(interf.changeFeedStream.getReplyStream(req));
 
 	results->maxSeenVersion = invalidVersion;
@@ -9211,6 +9530,10 @@ ACTOR Future<Void> singleChangeFeedStream(Reference<DatabaseContext> db,
 	Promise<Void> refresh = results->refresh;
 	results->refresh = Promise<Void>();
 	results->notAtLatest.set(1);
+	if (results->context) {
+		results->context->notAtLatestChangeFeeds[results->dbgid] = results.getPtr();
+		results->created = now();
+	}
 	refresh.send(Void());
 
 	wait(results->streams[0].onError() || singleChangeFeedStreamInternal(range, results, rangeID, begin, end));
@@ -9228,6 +9551,7 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
                                             bool canReadPopped) {
 	state Database cx(db);
 	state Span span("NAPI:GetChangeFeedStream"_loc);
+	db->usedAnyChangeFeeds = true;
 
 	results->endVersion = end;
 
@@ -9242,7 +9566,7 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 			keys = fullRange & range;
 			state std::vector<KeyRangeLocationInfo> locations =
 			    wait(getKeyRangeLocations(cx,
-			                              Optional<TenantName>(),
+			                              TenantInfo(),
 			                              keys,
 			                              CLIENT_KNOBS->CHANGE_FEED_LOCATION_LIMIT,
 			                              Reverse::False,
@@ -9279,6 +9603,10 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 				if (useIdx >= 0) {
 					chosenLocations[loc] = useIdx;
 					loc++;
+					if (g_network->isSimulated() && !g_simulator->speedUpSimulation && BUGGIFY_WITH_PROB(0.01)) {
+						// simulate as if we had to wait for all alternatives delayed, before the next one
+						wait(delay(deterministicRandom()->random01()));
+					}
 					continue;
 				}
 
@@ -9299,19 +9627,22 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 				loc = 0;
 			}
 
+			++db->feedStreamStarts;
+
 			if (locations.size() > 1) {
+				++db->feedMergeStreamStarts;
 				std::vector<std::pair<StorageServerInterface, KeyRange>> interfs;
 				for (int i = 0; i < locations.size(); i++) {
 					interfs.emplace_back(locations[i].locations->getInterface(chosenLocations[i]),
 					                     locations[i].range & range);
 				}
-				TEST(true); // Change feed merge cursor
+				CODE_PROBE(true, "Change feed merge cursor");
 				// TODO (jslocum): validate connectionFileChanged behavior
 				wait(
 				    mergeChangeFeedStream(db, interfs, results, rangeID, &begin, end, replyBufferSize, canReadPopped) ||
 				    cx->connectionFileChanged());
 			} else {
-				TEST(true); // Change feed single cursor
+				CODE_PROBE(true, "Change feed single cursor");
 				StorageServerInterface interf = locations[0].locations->getInterface(chosenLocations[0]);
 				wait(singleChangeFeedStream(
 				         db, interf, range, results, rangeID, &begin, end, replyBufferSize, canReadPopped) ||
@@ -9319,15 +9650,11 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 			}
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled || e.code() == error_code_change_feed_popped) {
-				for (auto& it : results->storageData) {
-					if (it->debugGetReferenceCount() == 2) {
-						db->changeFeedUpdaters.erase(it->interfToken);
-					}
-				}
 				results->streams.clear();
 				results->storageData.clear();
 				if (e.code() == error_code_change_feed_popped) {
-					TEST(true); // getChangeFeedStreamActor got popped
+					++db->feedNonRetriableErrors;
+					CODE_PROBE(true, "getChangeFeedStreamActor got popped");
 					results->mutations.sendError(e);
 					results->refresh.sendError(e);
 				} else {
@@ -9337,29 +9664,40 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 			}
 			if (results->notAtLatest.get() == 0) {
 				results->notAtLatest.set(1);
+				if (results->context) {
+					results->context->notAtLatestChangeFeeds[results->dbgid] = results.getPtr();
+					results->created = now();
+				}
 			}
 
 			if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
 			    e.code() == error_code_connection_failed || e.code() == error_code_unknown_change_feed ||
-			    e.code() == error_code_broken_promise) {
+			    e.code() == error_code_broken_promise || e.code() == error_code_future_version ||
+			    e.code() == error_code_request_maybe_delivered ||
+			    e.code() == error_code_storage_too_many_feed_streams) {
+				++db->feedErrors;
 				db->changeFeedCache.erase(rangeID);
 				cx->invalidateCache(Key(), keys);
-				if (begin == lastBeginVersion) {
+				if (begin == lastBeginVersion || e.code() == error_code_storage_too_many_feed_streams) {
 					// We didn't read anything since the last failure before failing again.
-					// Do exponential backoff, up to 1 second
-					sleepWithBackoff = std::min(1.0, sleepWithBackoff * 1.5);
+					// Back off quickly and exponentially, up to 1 second
+					sleepWithBackoff = std::min(2.0, sleepWithBackoff * 5);
+					sleepWithBackoff = std::max(0.1, sleepWithBackoff);
 				} else {
 					sleepWithBackoff = CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY;
 				}
+				TraceEvent("ChangeFeedClientError")
+				    .errorUnsuppressed(e)
+				    .suppressFor(30.0)
+				    .detail("AnyProgress", begin != lastBeginVersion);
 				wait(delay(sleepWithBackoff));
 			} else {
+				if (e.code() != error_code_end_of_stream) {
+					++db->feedNonRetriableErrors;
+					TraceEvent("ChangeFeedClientErrorNonRetryable").errorUnsuppressed(e).suppressFor(5.0);
+				}
 				results->mutations.sendError(e);
 				results->refresh.sendError(change_feed_cancelled());
-				for (auto& it : results->storageData) {
-					if (it->debugGetReferenceCount() == 2) {
-						db->changeFeedUpdaters.erase(it->interfToken);
-					}
-				}
 				results->streams.clear();
 				results->storageData.clear();
 				return Void();
@@ -9379,11 +9717,20 @@ Future<Void> DatabaseContext::getChangeFeedStream(Reference<ChangeFeedData> resu
 	    Reference<DatabaseContext>::addRef(this), results, rangeID, begin, end, range, replyBufferSize, canReadPopped);
 }
 
-ACTOR Future<std::vector<OverlappingChangeFeedEntry>> singleLocationOverlappingChangeFeeds(
-    Database cx,
-    Reference<LocationInfo> location,
-    KeyRangeRef range,
-    Version minVersion) {
+Version OverlappingChangeFeedsInfo::getFeedMetadataVersion(const KeyRangeRef& range) const {
+	Version v = invalidVersion;
+	for (auto& it : feedMetadataVersions) {
+		if (it.second > v && it.first.intersects(range)) {
+			v = it.second;
+		}
+	}
+	return v;
+}
+
+ACTOR Future<OverlappingChangeFeedsReply> singleLocationOverlappingChangeFeeds(Database cx,
+                                                                               Reference<LocationInfo> location,
+                                                                               KeyRangeRef range,
+                                                                               Version minVersion) {
 	state OverlappingChangeFeedsRequest req;
 	req.range = range;
 	req.minVersion = minVersion;
@@ -9395,16 +9742,16 @@ ACTOR Future<std::vector<OverlappingChangeFeedEntry>> singleLocationOverlappingC
 	                                                   TaskPriority::DefaultPromiseEndpoint,
 	                                                   AtMostOnce::False,
 	                                                   cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr));
-	return rep.rangeIds;
+	return rep;
 }
 
 bool compareChangeFeedResult(const OverlappingChangeFeedEntry& i, const OverlappingChangeFeedEntry& j) {
-	return i.rangeId < j.rangeId;
+	return i.feedId < j.feedId;
 }
 
-ACTOR Future<std::vector<OverlappingChangeFeedEntry>> getOverlappingChangeFeedsActor(Reference<DatabaseContext> db,
-                                                                                     KeyRangeRef range,
-                                                                                     Version minVersion) {
+ACTOR Future<OverlappingChangeFeedsInfo> getOverlappingChangeFeedsActor(Reference<DatabaseContext> db,
+                                                                        KeyRangeRef range,
+                                                                        Version minVersion) {
 	state Database cx(db);
 	state Span span("NAPI:GetOverlappingChangeFeeds"_loc);
 
@@ -9412,7 +9759,7 @@ ACTOR Future<std::vector<OverlappingChangeFeedEntry>> getOverlappingChangeFeedsA
 		try {
 			state std::vector<KeyRangeLocationInfo> locations =
 			    wait(getKeyRangeLocations(cx,
-			                              Optional<TenantName>(),
+			                              TenantInfo(),
 			                              range,
 			                              CLIENT_KNOBS->CHANGE_FEED_LOCATION_LIMIT,
 			                              Reverse::False,
@@ -9430,22 +9777,37 @@ ACTOR Future<std::vector<OverlappingChangeFeedEntry>> getOverlappingChangeFeedsA
 				throw all_alternatives_failed();
 			}
 
-			state std::vector<Future<std::vector<OverlappingChangeFeedEntry>>> allOverlappingRequests;
+			state std::vector<Future<OverlappingChangeFeedsReply>> allOverlappingRequests;
 			for (auto& it : locations) {
 				allOverlappingRequests.push_back(
 				    singleLocationOverlappingChangeFeeds(cx, it.locations, it.range & range, minVersion));
 			}
 			wait(waitForAll(allOverlappingRequests));
 
-			std::vector<OverlappingChangeFeedEntry> result;
-			for (auto& it : allOverlappingRequests) {
-				result.insert(result.end(), it.get().begin(), it.get().end());
+			OverlappingChangeFeedsInfo result;
+			std::unordered_map<KeyRef, OverlappingChangeFeedEntry> latestFeedMetadata;
+			for (int i = 0; i < locations.size(); i++) {
+				result.arena.dependsOn(allOverlappingRequests[i].get().arena);
+				result.arena.dependsOn(locations[i].range.arena());
+				result.feedMetadataVersions.push_back(
+				    { locations[i].range, allOverlappingRequests[i].get().feedMetadataVersion });
+				for (auto& it : allOverlappingRequests[i].get().feeds) {
+					auto res = latestFeedMetadata.insert({ it.feedId, it });
+					if (!res.second) {
+						CODE_PROBE(true, "deduping fetched overlapping feed by higher metadata version");
+						if (res.first->second.feedMetadataVersion < it.feedMetadataVersion) {
+							res.first->second = it;
+						}
+					}
+				}
+			}
+			for (auto& it : latestFeedMetadata) {
+				result.feeds.push_back(result.arena, it.second);
 			}
-			std::sort(result.begin(), result.end(), compareChangeFeedResult);
-			result.resize(std::unique(result.begin(), result.end()) - result.begin());
 			return result;
 		} catch (Error& e) {
-			if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
+			if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
+			    e.code() == error_code_future_version) {
 				cx->invalidateCache(Key(), range);
 				wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY));
 			} else {
@@ -9455,16 +9817,18 @@ ACTOR Future<std::vector<OverlappingChangeFeedEntry>> getOverlappingChangeFeedsA
 	}
 }
 
-Future<std::vector<OverlappingChangeFeedEntry>> DatabaseContext::getOverlappingChangeFeeds(KeyRangeRef range,
-                                                                                           Version minVersion) {
+Future<OverlappingChangeFeedsInfo> DatabaseContext::getOverlappingChangeFeeds(KeyRangeRef range, Version minVersion) {
 	return getOverlappingChangeFeedsActor(Reference<DatabaseContext>::addRef(this), range, minVersion);
 }
 
 ACTOR static Future<Void> popChangeFeedBackup(Database cx, Key rangeID, Version version) {
+	++cx->feedPopsFallback;
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			state Key rangeIDKey = rangeID.withPrefix(changeFeedPrefix);
 			Optional<Value> val = wait(tr.get(rangeIDKey));
 			if (val.present()) {
@@ -9496,12 +9860,14 @@ ACTOR Future<Void> popChangeFeedMutationsActor(Reference<DatabaseContext> db, Ke
 	state Database cx(db);
 	state Key rangeIDKey = rangeID.withPrefix(changeFeedPrefix);
 	state Span span("NAPI:PopChangeFeedMutations"_loc);
+	db->usedAnyChangeFeeds = true;
+	++db->feedPops;
 
 	state KeyRange keys = wait(getChangeFeedRange(db, cx, rangeID));
 
 	state std::vector<KeyRangeLocationInfo> locations =
 	    wait(getKeyRangeLocations(cx,
-	                              Optional<TenantName>(),
+	                              TenantInfo(),
 	                              keys,
 	                              3,
 	                              Reverse::False,
@@ -9568,6 +9934,7 @@ Reference<DatabaseContext::TransactionT> DatabaseContext::createTransaction() {
 	return makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(this)));
 }
 
+// BlobGranule API.
 ACTOR Future<Key> purgeBlobGranulesActor(Reference<DatabaseContext> db,
                                          KeyRange range,
                                          Version purgeVersion,
@@ -9579,8 +9946,21 @@ ACTOR Future<Key> purgeBlobGranulesActor(Reference<DatabaseContext> db,
 	state KeyRange purgeRange = range;
 	state bool loadedTenantPrefix = false;
 
-	// FIXME: implement force
-	if (!force) {
+	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+	if (purgeVersion == latestVersion) {
+		loop {
+			try {
+				Version _purgeVersion = wait(tr.getReadVersion());
+				purgeVersion = _purgeVersion;
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+		tr.reset();
+	}
+	if (purgeVersion <= 0) {
+		TraceEvent("PurgeInvalidVersion").detail("Range", range).detail("Version", purgeVersion).detail("Force", force);
 		throw unsupported_operation();
 	}
 
@@ -9588,6 +9968,7 @@ ACTOR Future<Key> purgeBlobGranulesActor(Reference<DatabaseContext> db,
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			if (tenant.present() && !loadedTenantPrefix) {
 				TenantMapEntry tenantEntry = wait(blobGranuleGetTenantEntry(&tr, range.begin));
@@ -9595,6 +9976,18 @@ ACTOR Future<Key> purgeBlobGranulesActor(Reference<DatabaseContext> db,
 				purgeRange = purgeRange.withPrefix(tenantEntry.prefix);
 			}
 
+			// must be aligned to blob range(s)
+			state Future<Optional<Value>> beginPresent = tr.get(purgeRange.begin.withPrefix(blobRangeKeys.begin));
+			state Future<Optional<Value>> endPresent = tr.get(purgeRange.end.withPrefix(blobRangeKeys.begin));
+			wait(success(beginPresent) && success(endPresent));
+			if (!beginPresent.get().present() || !endPresent.get().present()) {
+				TraceEvent("UnalignedPurge")
+				    .detail("Range", range)
+				    .detail("Version", purgeVersion)
+				    .detail("Force", force);
+				throw unsupported_operation();
+			}
+
 			Value purgeValue = blobGranulePurgeValueFor(purgeVersion, range, force);
 			tr.atomicOp(
 			    addVersionStampAtEnd(blobGranulePurgeKeys.begin), purgeValue, MutationRef::SetVersionstampedKey);
@@ -9664,6 +10057,108 @@ Future<Void> DatabaseContext::waitPurgeGranulesComplete(Key purgeKey) {
 	return waitPurgeGranulesCompleteActor(Reference<DatabaseContext>::addRef(this), purgeKey);
 }
 
+ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getBlobRanges(Reference<ReadYourWritesTransaction> tr,
+                                                               KeyRange range,
+                                                               int batchLimit) {
+	state Standalone<VectorRef<KeyRangeRef>> blobRanges;
+	state Key beginKey = range.begin;
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+
+			state RangeResult results = wait(
+			    krmGetRangesUnaligned(tr, blobRangeKeys.begin, KeyRangeRef(beginKey, range.end), 2 * batchLimit + 2));
+
+			blobRanges.arena().dependsOn(results.arena());
+			for (int i = 0; i < results.size() - 1; i++) {
+				if (results[i].value == blobRangeActive) {
+					blobRanges.push_back(blobRanges.arena(), KeyRangeRef(results[i].key, results[i + 1].key));
+				}
+				if (blobRanges.size() == batchLimit) {
+					return blobRanges;
+				}
+			}
+
+			if (!results.more) {
+				return blobRanges;
+			}
+			beginKey = results.back().key;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+}
+
+ACTOR Future<bool> setBlobRangeActor(Reference<DatabaseContext> cx, KeyRange range, bool active) {
+	state Database db(cx);
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
+
+	state Value value = active ? blobRangeActive : blobRangeInactive;
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+
+			Standalone<VectorRef<KeyRangeRef>> startBlobRanges = wait(getBlobRanges(tr, range, 1));
+
+			if (active) {
+				// Idempotent request.
+				if (!startBlobRanges.empty()) {
+					return startBlobRanges.front().begin == range.begin && startBlobRanges.front().end == range.end;
+				}
+			} else {
+				// An unblobbify request must be aligned to boundaries.
+				// It is okay to unblobbify multiple regions all at once.
+				if (startBlobRanges.empty()) {
+					// already unblobbified
+					return true;
+				} else if (startBlobRanges.front().begin != range.begin) {
+					// If there is a blob at the beginning of the range and it isn't aligned
+					return false;
+				}
+				// if blob range does start at the specified, key, we need to make sure the end of also a boundary of a
+				// blob range
+				Optional<Value> endPresent = wait(tr->get(range.end.withPrefix(blobRangeKeys.begin)));
+				if (!endPresent.present()) {
+					return false;
+				}
+			}
+
+			tr->set(blobRangeChangeKey, deterministicRandom()->randomUniqueID().toString());
+			// This is not coalescing because we want to keep each range logically separate.
+			wait(krmSetRange(tr, blobRangeKeys.begin, range, value));
+			wait(tr->commit());
+			return true;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+}
+
+Future<bool> DatabaseContext::blobbifyRange(KeyRange range) {
+	return setBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, true);
+}
+
+Future<bool> DatabaseContext::unblobbifyRange(KeyRange range) {
+	return setBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, false);
+}
+
+ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRangesActor(Reference<DatabaseContext> cx,
+                                                                           KeyRange range,
+                                                                           int rangeLimit) {
+	state Database db(cx);
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
+
+	state Standalone<VectorRef<KeyRangeRef>> blobRanges = wait(getBlobRanges(tr, range, rangeLimit));
+
+	return blobRanges;
+}
+
+Future<Standalone<VectorRef<KeyRangeRef>>> DatabaseContext::listBlobbifiedRanges(KeyRange range, int rowLimit) {
+	return listBlobbifiedRangesActor(Reference<DatabaseContext>::addRef(this), range, rowLimit);
+}
+
 int64_t getMaxKeySize(KeyRef const& key) {
 	return getMaxWriteKeySize(key, true);
 }
@@ -9673,7 +10168,7 @@ int64_t getMaxReadKeySize(KeyRef const& key) {
 }
 
 int64_t getMaxWriteKeySize(KeyRef const& key, bool hasRawAccess) {
-	int64_t tenantSize = hasRawAccess ? CLIENT_KNOBS->TENANT_PREFIX_SIZE_LIMIT : 0;
+	int64_t tenantSize = hasRawAccess ? TenantMapEntry::PREFIX_SIZE : 0;
 	return key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
 	                                        : CLIENT_KNOBS->KEY_SIZE_LIMIT + tenantSize;
 }
diff --git a/fdbclient/PaxosConfigTransaction.actor.cpp b/fdbclient/PaxosConfigTransaction.actor.cpp
index 4b7c19c05a..b940aced7f 100644
--- a/fdbclient/PaxosConfigTransaction.actor.cpp
+++ b/fdbclient/PaxosConfigTransaction.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/DatabaseContext.h"
+#include "fdbclient/MonitorLeader.h"
 #include "fdbclient/PaxosConfigTransaction.h"
 #include "flow/actorcompiler.h" // must be last include
 
@@ -34,8 +35,9 @@ class CommitQuorum {
 	Standalone<VectorRef<ConfigMutationRef>> mutations;
 	ConfigCommitAnnotation annotation;
 
-	ConfigTransactionCommitRequest getCommitRequest(ConfigGeneration generation) const {
-		return ConfigTransactionCommitRequest(generation, mutations, annotation);
+	ConfigTransactionCommitRequest getCommitRequest(ConfigGeneration generation,
+	                                                CoordinatorsHash coordinatorsHash) const {
+		return ConfigTransactionCommitRequest(coordinatorsHash, generation, mutations, annotation);
 	}
 
 	void updateResult() {
@@ -62,14 +64,16 @@ class CommitQuorum {
 
 	ACTOR static Future<Void> addRequestActor(CommitQuorum* self,
 	                                          ConfigGeneration generation,
+	                                          CoordinatorsHash coordinatorsHash,
 	                                          ConfigTransactionInterface cti) {
 		try {
 			if (cti.hostname.present()) {
-				wait(timeoutError(retryGetReplyFromHostname(
-				                      self->getCommitRequest(generation), cti.hostname.get(), WLTOKEN_CONFIGTXN_COMMIT),
+				wait(timeoutError(retryGetReplyFromHostname(self->getCommitRequest(generation, coordinatorsHash),
+				                                            cti.hostname.get(),
+				                                            WLTOKEN_CONFIGTXN_COMMIT),
 				                  CLIENT_KNOBS->COMMIT_QUORUM_TIMEOUT));
 			} else {
-				wait(timeoutError(cti.commit.getReply(self->getCommitRequest(generation)),
+				wait(timeoutError(cti.commit.getReply(self->getCommitRequest(generation, coordinatorsHash)),
 				                  CLIENT_KNOBS->COMMIT_QUORUM_TIMEOUT));
 			}
 			++self->successful;
@@ -109,11 +113,11 @@ public:
 	}
 	void setTimestamp() { annotation.timestamp = now(); }
 	size_t expectedSize() const { return annotation.expectedSize() + mutations.expectedSize(); }
-	Future<Void> commit(ConfigGeneration generation) {
+	Future<Void> commit(ConfigGeneration generation, CoordinatorsHash coordinatorsHash) {
 		// Send commit message to all replicas, even those that did not return the used replica.
 		// This way, slow replicas are kept up date.
 		for (const auto& cti : ctis) {
-			actors.add(addRequestActor(this, generation, cti));
+			actors.add(addRequestActor(this, generation, coordinatorsHash, cti));
 		}
 		return result.getFuture();
 	}
@@ -122,11 +126,13 @@ public:
 
 class GetGenerationQuorum {
 	ActorCollection actors{ false };
+	CoordinatorsHash coordinatorsHash{ 0 };
 	std::vector<ConfigTransactionInterface> ctis;
 	std::map<ConfigGeneration, std::vector<ConfigTransactionInterface>> seenGenerations;
 	Promise<ConfigGeneration> result;
 	size_t totalRepliesReceived{ 0 };
 	size_t maxAgreement{ 0 };
+	Future<Void> coordinatorsChangedFuture;
 	Optional<Version> lastSeenLiveVersion;
 	Future<ConfigGeneration> getGenerationFuture;
 
@@ -137,14 +143,15 @@ class GetGenerationQuorum {
 				if (cti.hostname.present()) {
 					wait(timeoutError(store(reply,
 					                        retryGetReplyFromHostname(
-					                            ConfigTransactionGetGenerationRequest{ self->lastSeenLiveVersion },
+					                            ConfigTransactionGetGenerationRequest{ self->coordinatorsHash,
+					                                                                   self->lastSeenLiveVersion },
 					                            cti.hostname.get(),
 					                            WLTOKEN_CONFIGTXN_GETGENERATION)),
 					                  CLIENT_KNOBS->GET_GENERATION_QUORUM_TIMEOUT));
 				} else {
 					wait(timeoutError(store(reply,
-					                        cti.getGeneration.getReply(
-					                            ConfigTransactionGetGenerationRequest{ self->lastSeenLiveVersion })),
+					                        cti.getGeneration.getReply(ConfigTransactionGetGenerationRequest{
+					                            self->coordinatorsHash, self->lastSeenLiveVersion })),
 					                  CLIENT_KNOBS->GET_GENERATION_QUORUM_TIMEOUT));
 				}
 
@@ -155,6 +162,14 @@ class GetGenerationQuorum {
 				auto& replicas = self->seenGenerations[gen];
 				replicas.push_back(cti);
 				self->maxAgreement = std::max(replicas.size(), self->maxAgreement);
+				// TraceEvent("ConfigTransactionGotGenerationReply")
+				// 		.detail("From", cti.getGeneration.getEndpoint().getPrimaryAddress())
+				// 		.detail("TotalRepliesReceived", self->totalRepliesReceived)
+				// 		.detail("ReplyGeneration", gen.toString())
+				// 		.detail("Replicas", replicas.size())
+				// 		.detail("Coordinators", self->ctis.size())
+				// 		.detail("MaxAgreement", self->maxAgreement)
+				// 		.detail("LastSeenLiveVersion", self->lastSeenLiveVersion);
 				if (replicas.size() >= self->ctis.size() / 2 + 1 && !self->result.isSet()) {
 					self->result.send(gen);
 				} else if (self->maxAgreement + (self->ctis.size() - self->totalRepliesReceived) <
@@ -199,9 +214,19 @@ class GetGenerationQuorum {
 				}
 			} catch (Error& e) {
 				if (e.code() == error_code_failed_to_reach_quorum) {
-					TEST(true); // Failed to reach quorum getting generation
-					wait(delayJittered(
-					    std::clamp(0.005 * (1 << retries), 0.0, CLIENT_KNOBS->TIMEOUT_RETRY_UPPER_BOUND)));
+					CODE_PROBE(true, "Failed to reach quorum getting generation");
+					if (self->coordinatorsChangedFuture.isReady()) {
+						throw coordinators_changed();
+					}
+					wait(delayJittered(std::clamp(
+					    0.005 * (1 << std::min(retries, 30)), 0.0, CLIENT_KNOBS->TIMEOUT_RETRY_UPPER_BOUND)));
+					if (deterministicRandom()->random01() < 0.05) {
+						// Randomly inject a delay of at least the generation
+						// reply timeout, to try to prevent contention between
+						// clients.
+						wait(delay(CLIENT_KNOBS->GET_GENERATION_QUORUM_TIMEOUT *
+						           (deterministicRandom()->random01() + 1.0)));
+					}
 					++retries;
 					self->actors.clear(false);
 					self->seenGenerations.clear();
@@ -217,9 +242,12 @@ class GetGenerationQuorum {
 
 public:
 	GetGenerationQuorum() = default;
-	explicit GetGenerationQuorum(std::vector<ConfigTransactionInterface> const& ctis,
+	explicit GetGenerationQuorum(CoordinatorsHash coordinatorsHash,
+	                             std::vector<ConfigTransactionInterface> const& ctis,
+	                             Future<Void> coordinatorsChangedFuture,
 	                             Optional<Version> const& lastSeenLiveVersion = {})
-	  : ctis(ctis), lastSeenLiveVersion(lastSeenLiveVersion) {}
+	  : coordinatorsHash(coordinatorsHash), ctis(ctis), coordinatorsChangedFuture(coordinatorsChangedFuture),
+	    lastSeenLiveVersion(lastSeenLiveVersion) {}
 	Future<ConfigGeneration> getGeneration() {
 		if (!getGenerationFuture.isValid()) {
 			getGenerationFuture = getGenerationActor(this);
@@ -240,12 +268,14 @@ public:
 };
 
 class PaxosConfigTransactionImpl {
+	CoordinatorsHash coordinatorsHash{ 0 };
 	std::vector<ConfigTransactionInterface> ctis;
 	GetGenerationQuorum getGenerationQuorum;
 	CommitQuorum commitQuorum;
 	int numRetries{ 0 };
 	Optional<UID> dID;
 	Database cx;
+	Future<Void> watchClusterFileFuture;
 
 	ACTOR static Future<Optional<Value>> get(PaxosConfigTransactionImpl* self, Key key) {
 		state ConfigKey configKey = ConfigKey::decodeKey(key);
@@ -263,18 +293,19 @@ class PaxosConfigTransactionImpl {
 				}
 				wait(waitForAll(fs));
 				state Reference<ConfigTransactionInfo> configNodes(new ConfigTransactionInfo(readReplicas));
-				ConfigTransactionGetReply reply =
-				    wait(timeoutError(basicLoadBalance(configNodes,
-				                                       &ConfigTransactionInterface::get,
-				                                       ConfigTransactionGetRequest{ generation, configKey }),
-				                      CLIENT_KNOBS->GET_KNOB_TIMEOUT));
+				ConfigTransactionGetReply reply = wait(timeoutError(
+				    basicLoadBalance(configNodes,
+				                     &ConfigTransactionInterface::get,
+				                     ConfigTransactionGetRequest{ self->coordinatorsHash, generation, configKey }),
+				    CLIENT_KNOBS->GET_KNOB_TIMEOUT));
 				if (reply.value.present()) {
 					return reply.value.get().toValue();
 				} else {
 					return Optional<Value>{};
 				}
 			} catch (Error& e) {
-				if (e.code() != error_code_timed_out && e.code() != error_code_broken_promise) {
+				if (e.code() != error_code_timed_out && e.code() != error_code_broken_promise &&
+				    e.code() != error_code_coordinators_changed) {
 					throw;
 				}
 				self->reset();
@@ -283,58 +314,87 @@ class PaxosConfigTransactionImpl {
 	}
 
 	ACTOR static Future<RangeResult> getConfigClasses(PaxosConfigTransactionImpl* self) {
-		state ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
-		state std::vector<ConfigTransactionInterface> readReplicas = self->getGenerationQuorum.getReadReplicas();
-		std::vector<Future<Void>> fs;
-		for (ConfigTransactionInterface& readReplica : readReplicas) {
-			if (readReplica.hostname.present()) {
-				fs.push_back(tryInitializeRequestStream(
-				    &readReplica.getClasses, readReplica.hostname.get(), WLTOKEN_CONFIGTXN_GETCLASSES));
+		loop {
+			try {
+				state ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
+				state std::vector<ConfigTransactionInterface> readReplicas =
+				    self->getGenerationQuorum.getReadReplicas();
+				std::vector<Future<Void>> fs;
+				for (ConfigTransactionInterface& readReplica : readReplicas) {
+					if (readReplica.hostname.present()) {
+						fs.push_back(tryInitializeRequestStream(
+						    &readReplica.getClasses, readReplica.hostname.get(), WLTOKEN_CONFIGTXN_GETCLASSES));
+					}
+				}
+				wait(waitForAll(fs));
+				state Reference<ConfigTransactionInfo> configNodes(new ConfigTransactionInfo(readReplicas));
+				ConfigTransactionGetConfigClassesReply reply = wait(
+				    basicLoadBalance(configNodes,
+				                     &ConfigTransactionInterface::getClasses,
+				                     ConfigTransactionGetConfigClassesRequest{ self->coordinatorsHash, generation }));
+				RangeResult result;
+				result.reserve(result.arena(), reply.configClasses.size());
+				for (const auto& configClass : reply.configClasses) {
+					result.push_back_deep(result.arena(), KeyValueRef(configClass, ""_sr));
+				}
+				return result;
+			} catch (Error& e) {
+				if (e.code() != error_code_coordinators_changed) {
+					throw;
+				}
+				self->reset();
 			}
 		}
-		wait(waitForAll(fs));
-		state Reference<ConfigTransactionInfo> configNodes(new ConfigTransactionInfo(readReplicas));
-		ConfigTransactionGetConfigClassesReply reply =
-		    wait(basicLoadBalance(configNodes,
-		                          &ConfigTransactionInterface::getClasses,
-		                          ConfigTransactionGetConfigClassesRequest{ generation }));
-		RangeResult result;
-		result.reserve(result.arena(), reply.configClasses.size());
-		for (const auto& configClass : reply.configClasses) {
-			result.push_back_deep(result.arena(), KeyValueRef(configClass, ""_sr));
-		}
-		return result;
 	}
 
 	ACTOR static Future<RangeResult> getKnobs(PaxosConfigTransactionImpl* self, Optional<Key> configClass) {
-		state ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
-		state std::vector<ConfigTransactionInterface> readReplicas = self->getGenerationQuorum.getReadReplicas();
-		std::vector<Future<Void>> fs;
-		for (ConfigTransactionInterface& readReplica : readReplicas) {
-			if (readReplica.hostname.present()) {
-				fs.push_back(tryInitializeRequestStream(
-				    &readReplica.getKnobs, readReplica.hostname.get(), WLTOKEN_CONFIGTXN_GETKNOBS));
+		loop {
+			try {
+				state ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
+				state std::vector<ConfigTransactionInterface> readReplicas =
+				    self->getGenerationQuorum.getReadReplicas();
+				std::vector<Future<Void>> fs;
+				for (ConfigTransactionInterface& readReplica : readReplicas) {
+					if (readReplica.hostname.present()) {
+						fs.push_back(tryInitializeRequestStream(
+						    &readReplica.getKnobs, readReplica.hostname.get(), WLTOKEN_CONFIGTXN_GETKNOBS));
+					}
+				}
+				wait(waitForAll(fs));
+				state Reference<ConfigTransactionInfo> configNodes(new ConfigTransactionInfo(readReplicas));
+				ConfigTransactionGetKnobsReply reply = wait(basicLoadBalance(
+				    configNodes,
+				    &ConfigTransactionInterface::getKnobs,
+				    ConfigTransactionGetKnobsRequest{ self->coordinatorsHash, generation, configClass }));
+				RangeResult result;
+				result.reserve(result.arena(), reply.knobNames.size());
+				for (const auto& knobName : reply.knobNames) {
+					result.push_back_deep(result.arena(), KeyValueRef(knobName, ""_sr));
+				}
+				return result;
+			} catch (Error& e) {
+				if (e.code() != error_code_coordinators_changed) {
+					throw;
+				}
+				self->reset();
 			}
 		}
-		wait(waitForAll(fs));
-		state Reference<ConfigTransactionInfo> configNodes(new ConfigTransactionInfo(readReplicas));
-		ConfigTransactionGetKnobsReply reply =
-		    wait(basicLoadBalance(configNodes,
-		                          &ConfigTransactionInterface::getKnobs,
-		                          ConfigTransactionGetKnobsRequest{ generation, configClass }));
-		RangeResult result;
-		result.reserve(result.arena(), reply.knobNames.size());
-		for (const auto& knobName : reply.knobNames) {
-			result.push_back_deep(result.arena(), KeyValueRef(knobName, ""_sr));
-		}
-		return result;
 	}
 
 	ACTOR static Future<Void> commit(PaxosConfigTransactionImpl* self) {
-		ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
-		self->commitQuorum.setTimestamp();
-		wait(self->commitQuorum.commit(generation));
-		return Void();
+		loop {
+			try {
+				ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
+				self->commitQuorum.setTimestamp();
+				wait(self->commitQuorum.commit(generation, self->coordinatorsHash));
+				return Void();
+			} catch (Error& e) {
+				if (e.code() != error_code_coordinators_changed) {
+					throw;
+				}
+				self->reset();
+			}
+		}
 	}
 
 	ACTOR static Future<Void> onError(PaxosConfigTransactionImpl* self, Error e) {
@@ -350,6 +410,20 @@ class PaxosConfigTransactionImpl {
 		throw e;
 	}
 
+	// Returns when the cluster interface updates with a new connection string.
+	ACTOR static Future<Void> watchClusterFile(Database cx) {
+		state Future<Void> leaderMonitor =
+		    monitorLeader<ClusterInterface>(cx->getConnectionRecord(), cx->statusClusterInterface);
+		state std::string connectionString = cx->getConnectionRecord()->getConnectionString().toString();
+
+		loop {
+			wait(cx->statusClusterInterface->onChange());
+			if (cx->getConnectionRecord()->getConnectionString().toString() != connectionString) {
+				return Void();
+			}
+		}
+	}
+
 public:
 	Future<Version> getReadVersion() {
 		return map(getGenerationQuorum.getGeneration(), [](auto const& gen) { return gen.committedVersion; });
@@ -395,7 +469,25 @@ public:
 	void debugTransaction(UID dID) { this->dID = dID; }
 
 	void reset() {
-		getGenerationQuorum = GetGenerationQuorum{ ctis };
+		ctis.clear();
+		// Re-read connection string. If the cluster file changed, this will
+		// return the updated value.
+		const ClusterConnectionString& cs = cx->getConnectionRecord()->getConnectionString();
+		ctis.reserve(cs.hostnames.size() + cs.coords.size());
+		for (const auto& h : cs.hostnames) {
+			ctis.emplace_back(h);
+		}
+		for (const auto& c : cs.coords) {
+			ctis.emplace_back(c);
+		}
+		coordinatorsHash = std::hash<std::string>()(cx->getConnectionRecord()->getConnectionString().toString());
+		if (!cx->statusLeaderMon.isValid() || cx->statusLeaderMon.isReady()) {
+			cx->statusClusterInterface = makeReference<AsyncVar<Optional<ClusterInterface>>>();
+			cx->statusLeaderMon = watchClusterFile(cx);
+		}
+		getGenerationQuorum = GetGenerationQuorum{
+			coordinatorsHash, ctis, cx->statusLeaderMon, getGenerationQuorum.getLastSeenLiveVersion()
+		};
 		commitQuorum = CommitQuorum{ ctis };
 	}
 
@@ -416,21 +508,10 @@ public:
 
 	Future<Void> commit() { return commit(this); }
 
-	PaxosConfigTransactionImpl(Database const& cx) : cx(cx) {
-		const ClusterConnectionString& cs = cx->getConnectionRecord()->getConnectionString();
-		ctis.reserve(cs.hostnames.size() + cs.coords.size());
-		for (const auto& h : cs.hostnames) {
-			ctis.emplace_back(h);
-		}
-		for (const auto& c : cs.coords) {
-			ctis.emplace_back(c);
-		}
-		getGenerationQuorum = GetGenerationQuorum{ ctis };
-		commitQuorum = CommitQuorum{ ctis };
-	}
+	PaxosConfigTransactionImpl(Database const& cx) : cx(cx) { reset(); }
 
 	PaxosConfigTransactionImpl(std::vector<ConfigTransactionInterface> const& ctis)
-	  : ctis(ctis), getGenerationQuorum(ctis), commitQuorum(ctis) {}
+	  : ctis(ctis), getGenerationQuorum(0, ctis, Future<Void>()), commitQuorum(ctis) {}
 };
 
 Future<Version> PaxosConfigTransaction::getReadVersion() {
diff --git a/fdbclient/RESTClient.actor.cpp b/fdbclient/RESTClient.actor.cpp
index 015ed1a8b0..e0cb416dec 100644
--- a/fdbclient/RESTClient.actor.cpp
+++ b/fdbclient/RESTClient.actor.cpp
@@ -20,7 +20,7 @@
 
 #include "fdbclient/RESTClient.h"
 
-#include "fdbclient/HTTP.h"
+#include "fdbrpc/HTTP.h"
 #include "flow/IRateControl.h"
 #include "fdbclient/RESTUtils.h"
 #include "flow/Arena.h"
diff --git a/fdbclient/RYWIterator.cpp b/fdbclient/RYWIterator.cpp
index 949f164485..3966df3748 100644
--- a/fdbclient/RYWIterator.cpp
+++ b/fdbclient/RYWIterator.cpp
@@ -231,28 +231,28 @@ void testSnapshotCache() {
 	WriteMap writes(&arena);
 
 	Standalone<VectorRef<KeyValueRef>> keys;
-	keys.push_back_deep(keys.arena(), KeyValueRef(LiteralStringRef("d"), LiteralStringRef("doo")));
-	keys.push_back_deep(keys.arena(), KeyValueRef(LiteralStringRef("e"), LiteralStringRef("eoo")));
-	keys.push_back_deep(keys.arena(), KeyValueRef(LiteralStringRef("e\x00"), LiteralStringRef("zoo")));
-	keys.push_back_deep(keys.arena(), KeyValueRef(LiteralStringRef("f"), LiteralStringRef("foo")));
-	cache.insert(KeyRangeRef(LiteralStringRef("d"), LiteralStringRef("f\x00")), keys);
+	keys.push_back_deep(keys.arena(), KeyValueRef("d"_sr, "doo"_sr));
+	keys.push_back_deep(keys.arena(), KeyValueRef("e"_sr, "eoo"_sr));
+	keys.push_back_deep(keys.arena(), KeyValueRef("e\x00"_sr, "zoo"_sr));
+	keys.push_back_deep(keys.arena(), KeyValueRef("f"_sr, "foo"_sr));
+	cache.insert(KeyRangeRef("d"_sr, "f\x00"_sr), keys);
 
-	cache.insert(KeyRangeRef(LiteralStringRef("g"), LiteralStringRef("h")), Standalone<VectorRef<KeyValueRef>>());
+	cache.insert(KeyRangeRef("g"_sr, "h"_sr), Standalone<VectorRef<KeyValueRef>>());
 
 	Standalone<VectorRef<KeyValueRef>> keys2;
-	keys2.push_back_deep(keys2.arena(), KeyValueRef(LiteralStringRef("k"), LiteralStringRef("koo")));
-	keys2.push_back_deep(keys2.arena(), KeyValueRef(LiteralStringRef("l"), LiteralStringRef("loo")));
-	cache.insert(KeyRangeRef(LiteralStringRef("j"), LiteralStringRef("m")), keys2);
+	keys2.push_back_deep(keys2.arena(), KeyValueRef("k"_sr, "koo"_sr));
+	keys2.push_back_deep(keys2.arena(), KeyValueRef("l"_sr, "loo"_sr));
+	cache.insert(KeyRangeRef("j"_sr, "m"_sr), keys2);
 
-	writes.mutate(LiteralStringRef("c"), MutationRef::SetValue, LiteralStringRef("c--"), true);
-	writes.clear(KeyRangeRef(LiteralStringRef("c\x00"), LiteralStringRef("e")), true);
-	writes.mutate(LiteralStringRef("c\x00"), MutationRef::SetValue, LiteralStringRef("c00--"), true);
+	writes.mutate("c"_sr, MutationRef::SetValue, "c--"_sr, true);
+	writes.clear(KeyRangeRef("c\x00"_sr, "e"_sr), true);
+	writes.mutate("c\x00"_sr, MutationRef::SetValue, "c00--"_sr, true);
 	WriteMap::iterator it3(&writes);
-	writes.mutate(LiteralStringRef("d"), MutationRef::SetValue, LiteralStringRef("d--"), true);
-	writes.mutate(LiteralStringRef("e"), MutationRef::SetValue, LiteralStringRef("e++"), true);
-	writes.mutate(LiteralStringRef("i"), MutationRef::SetValue, LiteralStringRef("i--"), true);
+	writes.mutate("d"_sr, MutationRef::SetValue, "d--"_sr, true);
+	writes.mutate("e"_sr, MutationRef::SetValue, "e++"_sr, true);
+	writes.mutate("i"_sr, MutationRef::SetValue, "i--"_sr, true);
 
-	KeyRange searchKeys = KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z"));
+	KeyRange searchKeys = KeyRangeRef("a"_sr, "z"_sr);
 
 	RYWIterator it(&cache, &writes);
 	it.skip(searchKeys.begin);
@@ -425,7 +425,7 @@ TEST_CASE("/fdbclient/WriteMap/emptiness") {
 	Arena arena = Arena();
 	WriteMap writes = WriteMap(&arena);
 	ASSERT(writes.empty());
-	writes.mutate(LiteralStringRef("apple"), MutationRef::SetValue, LiteralStringRef("red"), true);
+	writes.mutate("apple"_sr, MutationRef::SetValue, "red"_sr, true);
 	ASSERT(!writes.empty());
 	return Void();
 }
@@ -457,11 +457,11 @@ TEST_CASE("/fdbclient/WriteMap/clear") {
 	ASSERT(writes.empty());
 	ASSERT(getWriteMapCount(&writes) == 1);
 
-	writes.mutate(LiteralStringRef("apple"), MutationRef::SetValue, LiteralStringRef("red"), true);
+	writes.mutate("apple"_sr, MutationRef::SetValue, "red"_sr, true);
 	ASSERT(!writes.empty());
 	ASSERT(getWriteMapCount(&writes) == 3);
 
-	KeyRangeRef range = KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("j"));
+	KeyRangeRef range = KeyRangeRef("a"_sr, "j"_sr);
 	writes.clear(range, true);
 	ASSERT(getWriteMapCount(&writes) == 3);
 
@@ -474,22 +474,19 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedKey") {
 	ASSERT(writes.empty());
 	ASSERT(getWriteMapCount(&writes) == 1);
 
-	writes.mutate(LiteralStringRef("stamp:XXXXXXXX\x06\x00\x00\x00"),
-	              MutationRef::SetVersionstampedKey,
-	              LiteralStringRef("1"),
-	              true);
+	writes.mutate("stamp:XXXXXXXX\x06\x00\x00\x00"_sr, MutationRef::SetVersionstampedKey, "1"_sr, true);
 	ASSERT(!writes.empty());
 	ASSERT(getWriteMapCount(&writes) == 3);
 
-	writes.mutate(LiteralStringRef("stamp:ZZZZZZZZZZ"), MutationRef::AddValue, LiteralStringRef("2"), true);
+	writes.mutate("stamp:ZZZZZZZZZZ"_sr, MutationRef::AddValue, "2"_sr, true);
 	ASSERT(getWriteMapCount(&writes) == 5);
 
 	WriteMap::iterator it(&writes);
 	it.skip(allKeys.begin);
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("stamp:XXXXXXXX\x06\x00\x00\x00")) == 0);
+	ASSERT(it.beginKey().compare(""_sr) == 0);
+	ASSERT(it.endKey().compare("stamp:XXXXXXXX\x06\x00\x00\x00"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(!it.is_conflict_range());
 	ASSERT(!it.is_operation());
@@ -498,8 +495,8 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedKey") {
 	++it;
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("stamp:XXXXXXXX\x06\x00\x00\x00")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("stamp:XXXXXXXX\x06\x00\x00\x00\x00")) == 0);
+	ASSERT(it.beginKey().compare("stamp:XXXXXXXX\x06\x00\x00\x00"_sr) == 0);
+	ASSERT(it.endKey().compare("stamp:XXXXXXXX\x06\x00\x00\x00\x00"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(it.is_conflict_range());
 	ASSERT(it.is_operation());
@@ -509,8 +506,8 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedKey") {
 	++it;
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("stamp:XXXXXXXX\x06\x00\x00\x00\x00")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("stamp:ZZZZZZZZZZ")) == 0);
+	ASSERT(it.beginKey().compare("stamp:XXXXXXXX\x06\x00\x00\x00\x00"_sr) == 0);
+	ASSERT(it.endKey().compare("stamp:ZZZZZZZZZZ"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(!it.is_conflict_range());
 	ASSERT(!it.is_operation());
@@ -519,8 +516,8 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedKey") {
 	++it;
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("stamp:ZZZZZZZZZZ")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("stamp:ZZZZZZZZZZ\x00")) == 0);
+	ASSERT(it.beginKey().compare("stamp:ZZZZZZZZZZ"_sr) == 0);
+	ASSERT(it.endKey().compare("stamp:ZZZZZZZZZZ\x00"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(it.is_conflict_range());
 	ASSERT(it.is_operation());
@@ -530,8 +527,8 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedKey") {
 	++it;
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("stamp:ZZZZZZZZZZ\x00")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("\xff\xff")) == 0);
+	ASSERT(it.beginKey().compare("stamp:ZZZZZZZZZZ\x00"_sr) == 0);
+	ASSERT(it.endKey().compare("\xff\xff"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(!it.is_conflict_range());
 	ASSERT(!it.is_operation());
@@ -550,22 +547,19 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedValue") {
 	ASSERT(writes.empty());
 	ASSERT(getWriteMapCount(&writes) == 1);
 
-	writes.mutate(LiteralStringRef("stamp"),
-	              MutationRef::SetVersionstampedValue,
-	              LiteralStringRef("XXXXXXXX\x00\x00\x00\x00\x00\x00"),
-	              true);
+	writes.mutate("stamp"_sr, MutationRef::SetVersionstampedValue, "XXXXXXXX\x00\x00\x00\x00\x00\x00"_sr, true);
 	ASSERT(!writes.empty());
 	ASSERT(getWriteMapCount(&writes) == 3);
 
-	writes.mutate(LiteralStringRef("stamp123"), MutationRef::AddValue, LiteralStringRef("1"), true);
+	writes.mutate("stamp123"_sr, MutationRef::AddValue, "1"_sr, true);
 	ASSERT(getWriteMapCount(&writes) == 5);
 
 	WriteMap::iterator it(&writes);
 	it.skip(allKeys.begin);
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("stamp")) == 0);
+	ASSERT(it.beginKey().compare(""_sr) == 0);
+	ASSERT(it.endKey().compare("stamp"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(!it.is_conflict_range());
 	ASSERT(!it.is_operation());
@@ -574,8 +568,8 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedValue") {
 	++it;
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("stamp")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("stamp\x00")) == 0);
+	ASSERT(it.beginKey().compare("stamp"_sr) == 0);
+	ASSERT(it.endKey().compare("stamp\x00"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(it.is_conflict_range());
 	ASSERT(it.is_operation());
@@ -585,8 +579,8 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedValue") {
 	++it;
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("stamp\x00")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("stamp123")) == 0);
+	ASSERT(it.beginKey().compare("stamp\x00"_sr) == 0);
+	ASSERT(it.endKey().compare("stamp123"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(!it.is_conflict_range());
 	ASSERT(!it.is_operation());
@@ -595,8 +589,8 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedValue") {
 	++it;
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("stamp123")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("stamp123\x00")) == 0);
+	ASSERT(it.beginKey().compare("stamp123"_sr) == 0);
+	ASSERT(it.endKey().compare("stamp123\x00"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(it.is_conflict_range());
 	ASSERT(it.is_operation());
@@ -606,8 +600,8 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedValue") {
 	++it;
 
 	ASSERT(it.beginKey() < allKeys.end);
-	ASSERT(it.beginKey().compare(LiteralStringRef("stamp123\x00")) == 0);
-	ASSERT(it.endKey().compare(LiteralStringRef("\xff\xff")) == 0);
+	ASSERT(it.beginKey().compare("stamp123\x00"_sr) == 0);
+	ASSERT(it.endKey().compare("\xff\xff"_sr) == 0);
 	ASSERT(!it.is_cleared_range());
 	ASSERT(!it.is_conflict_range());
 	ASSERT(!it.is_operation());
@@ -626,10 +620,10 @@ TEST_CASE("/fdbclient/WriteMap/addValue") {
 	ASSERT(writes.empty());
 	ASSERT(getWriteMapCount(&writes) == 1);
 
-	writes.mutate(LiteralStringRef("apple123"), MutationRef::SetValue, LiteralStringRef("17"), true);
+	writes.mutate("apple123"_sr, MutationRef::SetValue, "17"_sr, true);
 	ASSERT(getWriteMapCount(&writes) == 3);
 
-	writes.mutate(LiteralStringRef("apple123"), MutationRef::AddValue, LiteralStringRef("1"), true);
+	writes.mutate("apple123"_sr, MutationRef::AddValue, "1"_sr, true);
 	ASSERT(getWriteMapCount(&writes) == 3);
 
 	return Void();
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index a64a65e58a..c3016c87e3 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -459,7 +459,7 @@ public:
 
 		if (!it.is_unreadable() && !it.is_unknown_range() && key.offset > 1) {
 			*readThroughEnd = true;
-			key.setKey(maxKey); // maxKey is a KeyRef, but points to a LiteralStringRef. TODO: how can we ASSERT this?
+			key.setKey(maxKey); // maxKey is a KeyRef, but points to a literal. TODO: how can we ASSERT this?
 			key.offset = 1;
 			return;
 		}
@@ -681,7 +681,8 @@ public:
 				break;
 
 			if (it.is_unknown_range()) {
-				if (limits.hasByteLimit() && result.size() && itemsPastEnd >= 1 - end.offset) {
+				if (limits.hasByteLimit() && limits.hasSatisfiedMinRows() && result.size() &&
+				    itemsPastEnd >= 1 - end.offset) {
 					result.more = true;
 					break;
 				}
@@ -1213,7 +1214,7 @@ public:
 		// isolation support. But it is not default and is rarely used. So we disallow it until we have thorough test
 		// coverage for it.)
 		if (snapshot) {
-			TEST(true); // getMappedRange not supported for snapshot.
+			CODE_PROBE(true, "getMappedRange not supported for snapshot.");
 			throw unsupported_operation();
 		}
 		// For now, getMappedRange requires read-your-writes being NOT disabled. But the support of RYW is limited
@@ -1222,7 +1223,7 @@ public:
 		// which returns the written value transparently. In another word, it makes sure not break RYW semantics without
 		// actually implementing reading from the writes.
 		if (ryw->options.readYourWritesDisabled) {
-			TEST(true); // getMappedRange not supported for read-your-writes disabled.
+			CODE_PROBE(true, "getMappedRange not supported for read-your-writes disabled.");
 			throw unsupported_operation();
 		}
 
@@ -1242,7 +1243,7 @@ public:
 			++it;
 
 			ASSERT(itCopy->value.size());
-			TEST(itCopy->value.size() > 1); // Multiple watches on the same key triggered by RYOW
+			CODE_PROBE(itCopy->value.size() > 1, "Multiple watches on the same key triggered by RYOW");
 
 			for (int i = 0; i < itCopy->value.size(); i++) {
 				if (itCopy->value[i]->onChangeTrigger.isSet()) {
@@ -1535,15 +1536,15 @@ ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord
 }
 
 Future<Optional<Value>> ReadYourWritesTransaction::get(const Key& key, Snapshot snapshot) {
-	TEST(true); // ReadYourWritesTransaction::get
+	CODE_PROBE(true, "ReadYourWritesTransaction::get");
 
 	if (getDatabase()->apiVersionAtLeast(630)) {
 		if (specialKeys.contains(key)) {
-			TEST(true); // Special keys get
+			CODE_PROBE(true, "Special keys get");
 			return getDatabase()->specialKeySpace->get(this, key);
 		}
 	} else {
-		if (key == LiteralStringRef("\xff\xff/status/json")) {
+		if (key == "\xff\xff/status/json"_sr) {
 			if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionRecord()) {
 				++tr.getDatabase()->transactionStatusRequests;
 				return getJSON(tr.getDatabase());
@@ -1552,7 +1553,7 @@ Future<Optional<Value>> ReadYourWritesTransaction::get(const Key& key, Snapshot
 			}
 		}
 
-		if (key == LiteralStringRef("\xff\xff/cluster_file_path")) {
+		if (key == "\xff\xff/cluster_file_path"_sr) {
 			try {
 				if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionRecord()) {
 					Optional<Value> output = StringRef(tr.getDatabase()->getConnectionRecord()->getLocation());
@@ -1564,7 +1565,7 @@ Future<Optional<Value>> ReadYourWritesTransaction::get(const Key& key, Snapshot
 			return Optional<Value>();
 		}
 
-		if (key == LiteralStringRef("\xff\xff/connection_string")) {
+		if (key == "\xff\xff/connection_string"_sr) {
 			try {
 				if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionRecord()) {
 					Reference<IClusterConnectionRecord> f = tr.getDatabase()->getConnectionRecord();
@@ -1622,11 +1623,11 @@ Future<RangeResult> ReadYourWritesTransaction::getRange(KeySelector begin,
 	if (getDatabase()->apiVersionAtLeast(630)) {
 		if (specialKeys.contains(begin.getKey()) && specialKeys.begin <= end.getKey() &&
 		    end.getKey() <= specialKeys.end) {
-			TEST(true); // Special key space get range
+			CODE_PROBE(true, "Special key space get range");
 			return getDatabase()->specialKeySpace->getRange(this, begin, end, limits, reverse);
 		}
 	} else {
-		if (begin.getKey() == LiteralStringRef("\xff\xff/worker_interfaces")) {
+		if (begin.getKey() == "\xff\xff/worker_interfaces"_sr) {
 			if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionRecord()) {
 				return getWorkerInterfaces(tr.getDatabase()->getConnectionRecord());
 			} else {
@@ -1648,7 +1649,7 @@ Future<RangeResult> ReadYourWritesTransaction::getRange(KeySelector begin,
 
 	// This optimization prevents nullptr operations from being added to the conflict range
 	if (limits.isReached()) {
-		TEST(true); // RYW range read limit 0
+		CODE_PROBE(true, "RYW range read limit 0");
 		return RangeResult();
 	}
 
@@ -1662,7 +1663,7 @@ Future<RangeResult> ReadYourWritesTransaction::getRange(KeySelector begin,
 		end.removeOrEqual(end.arena());
 
 	if (begin.offset >= end.offset && begin.getKey() >= end.getKey()) {
-		TEST(true); // RYW range inverted
+		CODE_PROBE(true, "RYW range inverted");
 		return RangeResult();
 	}
 
@@ -1692,11 +1693,11 @@ Future<MappedRangeResult> ReadYourWritesTransaction::getMappedRange(KeySelector
 	if (getDatabase()->apiVersionAtLeast(630)) {
 		if (specialKeys.contains(begin.getKey()) && specialKeys.begin <= end.getKey() &&
 		    end.getKey() <= specialKeys.end) {
-			TEST(true); // Special key space get range (getMappedRange)
+			CODE_PROBE(true, "Special key space get range (getMappedRange)");
 			throw client_invalid_operation(); // Not support special keys.
 		}
 	} else {
-		if (begin.getKey() == LiteralStringRef("\xff\xff/worker_interfaces")) {
+		if (begin.getKey() == "\xff\xff/worker_interfaces"_sr) {
 			throw client_invalid_operation(); // Not support special keys.
 		}
 	}
@@ -1714,7 +1715,7 @@ Future<MappedRangeResult> ReadYourWritesTransaction::getMappedRange(KeySelector
 
 	// This optimization prevents nullptr operations from being added to the conflict range
 	if (limits.isReached()) {
-		TEST(true); // RYW range read limit 0 (getMappedRange)
+		CODE_PROBE(true, "RYW range read limit 0 (getMappedRange)");
 		return MappedRangeResult();
 	}
 
@@ -1728,7 +1729,7 @@ Future<MappedRangeResult> ReadYourWritesTransaction::getMappedRange(KeySelector
 		end.removeOrEqual(end.arena());
 
 	if (begin.offset >= end.offset && begin.getKey() >= end.getKey()) {
-		TEST(true); // RYW range inverted (getMappedRange)
+		CODE_PROBE(true, "RYW range inverted (getMappedRange)");
 		return MappedRangeResult();
 	}
 
@@ -1783,7 +1784,8 @@ Future<Standalone<VectorRef<KeyRef>>> ReadYourWritesTransaction::getRangeSplitPo
 	return waitOrError(tr.getRangeSplitPoints(range, chunkSize), resetPromise.getFuture());
 }
 
-Future<Standalone<VectorRef<KeyRangeRef>>> ReadYourWritesTransaction::getBlobGranuleRanges(const KeyRange& range) {
+Future<Standalone<VectorRef<KeyRangeRef>>> ReadYourWritesTransaction::getBlobGranuleRanges(const KeyRange& range,
+                                                                                           int rangeLimit) {
 	if (checkUsedDuringCommit()) {
 		return used_during_commit();
 	}
@@ -1794,7 +1796,7 @@ Future<Standalone<VectorRef<KeyRangeRef>>> ReadYourWritesTransaction::getBlobGra
 	if (range.begin > maxKey || range.end > maxKey)
 		return key_outside_legal_range();
 
-	return waitOrError(tr.getBlobGranuleRanges(range), resetPromise.getFuture());
+	return waitOrError(tr.getBlobGranuleRanges(range, rangeLimit), resetPromise.getFuture());
 }
 
 Future<Standalone<VectorRef<BlobGranuleChunkRef>>> ReadYourWritesTransaction::readBlobGranules(
@@ -1821,6 +1823,32 @@ Future<Standalone<VectorRef<BlobGranuleChunkRef>>> ReadYourWritesTransaction::re
 	return waitOrError(tr.readBlobGranules(range, begin, readVersion, readVersionOut), resetPromise.getFuture());
 }
 
+Future<Standalone<VectorRef<BlobGranuleSummaryRef>>> ReadYourWritesTransaction::summarizeBlobGranules(
+    const KeyRange& range,
+    Optional<Version> summaryVersion,
+    int rangeLimit) {
+
+	if (checkUsedDuringCommit()) {
+		return used_during_commit();
+	}
+
+	if (resetPromise.isSet())
+		return resetPromise.getFuture().getError();
+
+	KeyRef maxKey = getMaxReadKey();
+	if (range.begin > maxKey || range.end > maxKey)
+		return key_outside_legal_range();
+
+	return waitOrError(tr.summarizeBlobGranules(range, summaryVersion, rangeLimit), resetPromise.getFuture());
+}
+
+void ReadYourWritesTransaction::addGranuleMaterializeStats(const GranuleMaterializeStats& stats) {
+	if (checkUsedDuringCommit()) {
+		throw used_during_commit();
+	}
+	tr.addGranuleMaterializeStats(stats);
+}
+
 void ReadYourWritesTransaction::addReadConflictRange(KeyRangeRef const& keys) {
 	if (checkUsedDuringCommit()) {
 		throw used_during_commit();
@@ -1998,7 +2026,7 @@ void ReadYourWritesTransaction::setToken(uint64_t token) {
 }
 
 RangeResult ReadYourWritesTransaction::getReadConflictRangeIntersecting(KeyRangeRef kr) {
-	TEST(true); // Special keys read conflict range
+	CODE_PROBE(true, "Special keys read conflict range");
 	ASSERT(readConflictRangeKeysRange.contains(kr));
 	ASSERT(!tr.trState->options.checkWritesEnabled);
 	RangeResult result;
@@ -2012,22 +2040,20 @@ RangeResult ReadYourWritesTransaction::getReadConflictRangeIntersecting(KeyRange
 			if (kr.begin <= iter->begin() && iter->begin() < kr.end) {
 				result.push_back(result.arena(),
 				                 KeyValueRef(iter->begin().withPrefix(readConflictRangeKeysRange.begin, result.arena()),
-				                             iter->value() ? LiteralStringRef("1") : LiteralStringRef("0")));
+				                             iter->value() ? "1"_sr : "0"_sr));
 			}
 		}
 	} else {
-		CoalescedKeyRefRangeMap<ValueRef> readConflicts{ LiteralStringRef("0"), specialKeys.end };
+		CoalescedKeyRefRangeMap<ValueRef> readConflicts{ "0"_sr, specialKeys.end };
 		for (const auto& range : tr.readConflictRanges())
-			readConflicts.insert(range.withPrefix(readConflictRangeKeysRange.begin, result.arena()),
-			                     LiteralStringRef("1"));
+			readConflicts.insert(range.withPrefix(readConflictRangeKeysRange.begin, result.arena()), "1"_sr);
 		for (const auto& range : nativeReadRanges)
-			readConflicts.insert(range.withPrefix(readConflictRangeKeysRange.begin, result.arena()),
-			                     LiteralStringRef("1"));
+			readConflicts.insert(range.withPrefix(readConflictRangeKeysRange.begin, result.arena()), "1"_sr);
 		for (const auto& f : tr.getExtraReadConflictRanges()) {
 			if (f.isReady() && f.get().first < f.get().second)
 				readConflicts.insert(KeyRangeRef(f.get().first, f.get().second)
 				                         .withPrefix(readConflictRangeKeysRange.begin, result.arena()),
-				                     LiteralStringRef("1"));
+				                     "1"_sr);
 		}
 		auto beginIter = readConflicts.rangeContaining(kr.begin);
 		if (beginIter->begin() != kr.begin)
@@ -2040,12 +2066,12 @@ RangeResult ReadYourWritesTransaction::getReadConflictRangeIntersecting(KeyRange
 }
 
 RangeResult ReadYourWritesTransaction::getWriteConflictRangeIntersecting(KeyRangeRef kr) {
-	TEST(true); // Special keys write conflict range
+	CODE_PROBE(true, "Special keys write conflict range");
 	ASSERT(writeConflictRangeKeysRange.contains(kr));
 	RangeResult result;
 
 	// Memory owned by result
-	CoalescedKeyRefRangeMap<ValueRef> writeConflicts{ LiteralStringRef("0"), specialKeys.end };
+	CoalescedKeyRefRangeMap<ValueRef> writeConflicts{ "0"_sr, specialKeys.end };
 
 	if (!options.readYourWritesDisabled) {
 		KeyRangeRef strippedWriteRangePrefix = kr.removePrefix(writeConflictRangeKeysRange.begin);
@@ -2058,15 +2084,13 @@ RangeResult ReadYourWritesTransaction::getWriteConflictRangeIntersecting(KeyRang
 				writeConflicts.insert(
 				    KeyRangeRef(it.beginKey().toArena(result.arena()), it.endKey().toArena(result.arena()))
 				        .withPrefix(writeConflictRangeKeysRange.begin, result.arena()),
-				    LiteralStringRef("1"));
+				    "1"_sr);
 		}
 	} else {
 		for (const auto& range : tr.writeConflictRanges())
-			writeConflicts.insert(range.withPrefix(writeConflictRangeKeysRange.begin, result.arena()),
-			                      LiteralStringRef("1"));
+			writeConflicts.insert(range.withPrefix(writeConflictRangeKeysRange.begin, result.arena()), "1"_sr);
 		for (const auto& range : nativeWriteRanges)
-			writeConflicts.insert(range.withPrefix(writeConflictRangeKeysRange.begin, result.arena()),
-			                      LiteralStringRef("1"));
+			writeConflicts.insert(range.withPrefix(writeConflictRangeKeysRange.begin, result.arena()), "1"_sr);
 	}
 
 	for (const auto& k : versionStampKeys) {
@@ -2086,8 +2110,7 @@ RangeResult ReadYourWritesTransaction::getWriteConflictRangeIntersecting(KeyRang
 		} else {
 			range = getVersionstampKeyRange(result.arena(), k, tr.getCachedReadVersion().orDefault(0), getMaxReadKey());
 		}
-		writeConflicts.insert(range.withPrefix(writeConflictRangeKeysRange.begin, result.arena()),
-		                      LiteralStringRef("1"));
+		writeConflicts.insert(range.withPrefix(writeConflictRangeKeysRange.begin, result.arena()), "1"_sr);
 	}
 
 	auto beginIter = writeConflicts.rangeContaining(kr.begin);
@@ -2133,19 +2156,19 @@ void ReadYourWritesTransaction::atomicOp(const KeyRef& key, const ValueRef& oper
 
 	KeyRef k;
 	if (!tr.apiVersionAtLeast(520) && operationType == MutationRef::SetVersionstampedKey) {
-		k = key.withSuffix(LiteralStringRef("\x00\x00"), arena);
+		k = key.withSuffix("\x00\x00"_sr, arena);
 	} else {
 		k = KeyRef(arena, key);
 	}
 	ValueRef v;
 	if (!tr.apiVersionAtLeast(520) && operationType == MutationRef::SetVersionstampedValue) {
-		v = operand.withSuffix(LiteralStringRef("\x00\x00\x00\x00"), arena);
+		v = operand.withSuffix("\x00\x00\x00\x00"_sr, arena);
 	} else {
 		v = ValueRef(arena, operand);
 	}
 
 	if (operationType == MutationRef::SetVersionstampedKey) {
-		TEST(options.readYourWritesDisabled); // SetVersionstampedKey without ryw enabled
+		CODE_PROBE(options.readYourWritesDisabled, "SetVersionstampedKey without ryw enabled");
 		// this does validation of the key and needs to be performed before the readYourWritesDisabled path
 		KeyRangeRef range = getVersionstampKeyRange(arena, k, tr.getCachedReadVersion().orDefault(0), getMaxReadKey());
 		versionStampKeys.push_back(arena, k);
@@ -2191,17 +2214,17 @@ void ReadYourWritesTransaction::set(const KeyRef& key, const ValueRef& value) {
 		} else {
 			// These three special keys are deprecated in 7.0 and an alternative C API is added
 			// TODO : Rewrite related code using C api
-			if (key == LiteralStringRef("\xff\xff/reboot_worker")) {
+			if (key == "\xff\xff/reboot_worker"_sr) {
 				BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion())
 				    .reboot.send(RebootRequest());
 				return;
 			}
-			if (key == LiteralStringRef("\xff\xff/suspend_worker")) {
+			if (key == "\xff\xff/suspend_worker"_sr) {
 				BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion())
 				    .reboot.send(RebootRequest(false, false, options.timeoutInSeconds));
 				return;
 			}
-			if (key == LiteralStringRef("\xff\xff/reboot_and_check_worker")) {
+			if (key == "\xff\xff/reboot_and_check_worker"_sr) {
 				BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion())
 				    .reboot.send(RebootRequest(false, true));
 				return;
diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index fc45e0a0fd..b32e349631 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -20,8 +20,8 @@
 
 #include "fdbclient/S3BlobStore.h"
 
-#include "fdbclient/md5/md5.h"
-#include "fdbclient/libb64/encode.h"
+#include "md5/md5.h"
+#include "libb64/encode.h"
 #include "fdbclient/sha1/SHA1.h"
 #include <time.h>
 #include <iomanip>
@@ -98,7 +98,7 @@ S3BlobStoreEndpoint::BlobKnobs::BlobKnobs() {
 
 bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
 #define TRY_PARAM(n, sn)                                                                                               \
-	if (name == LiteralStringRef(#n) || name == LiteralStringRef(#sn)) {                                               \
+	if (name == #n || name == #sn) {                                                                                   \
 		n = value;                                                                                                     \
 		return true;                                                                                                   \
 	}
@@ -109,7 +109,7 @@ bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
 	TRY_PARAM(request_tries, rt);
 	TRY_PARAM(request_timeout_min, rtom);
 	// TODO: For backward compatibility because request_timeout was renamed to request_timeout_min
-	if (name == LiteralStringRef("request_timeout") || name == LiteralStringRef("rto")) {
+	if (name == "request_timeout"_sr || name == "rto"_sr) {
 		request_timeout_min = value;
 		return true;
 	}
@@ -187,7 +187,7 @@ std::string guessRegionFromDomain(std::string domain) {
 
 		StringRef h(domain.c_str() + p);
 
-		if (!h.startsWith(LiteralStringRef("oss-"))) {
+		if (!h.startsWith("oss-"_sr)) {
 			h.eat(service); // ignore s3 service
 		}
 
@@ -208,7 +208,7 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 	try {
 		StringRef t(url);
 		StringRef prefix = t.eat("://");
-		if (prefix != LiteralStringRef("blobstore"))
+		if (prefix != "blobstore"_sr)
 			throw format("Invalid blobstore URL prefix '%s'", prefix.toString().c_str());
 
 		Optional<std::string> proxyHost, proxyPort;
@@ -261,7 +261,7 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 			StringRef value = t.eat("&");
 
 			// Special case for header
-			if (name == LiteralStringRef("header")) {
+			if (name == "header"_sr) {
 				StringRef originalValue = value;
 				StringRef headerFieldName = value.eat(":");
 				StringRef headerFieldValue = value;
@@ -282,7 +282,7 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 			}
 
 			// overwrite s3 region from parameter
-			if (name == LiteralStringRef("region")) {
+			if (name == "region"_sr) {
 				region = value.toString();
 				continue;
 			}
@@ -476,7 +476,7 @@ ACTOR Future<Void> deleteRecursively_impl(Reference<S3BlobStoreEndpoint> b,
 	state Future<Void> done = b->listObjectsStream(bucket, resultStream, prefix, '/', std::numeric_limits<int>::max());
 	// Wrap done in an actor which will send end_of_stream since listObjectsStream() does not (so that many calls can
 	// write to the same stream)
-	done = map(done, [=](Void) {
+	done = map(done, [=](Void) mutable {
 		resultStream.sendError(end_of_stream());
 		return Void();
 	});
@@ -737,16 +737,21 @@ ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3B
 		service = b->knobs.secure_connection ? "https" : "http";
 	}
 	bool isTLS = b->knobs.secure_connection == 1;
+	state Reference<IConnection> conn;
 	if (b->useProxy) {
-		// TODO(renxuan): Support http proxy + TLS
-		if (isTLS || b->service == "443") {
-			fprintf(stderr, "ERROR: TLS is not supported yet when using HTTP proxy.\n");
-			throw connection_failed();
+		if (isTLS) {
+			Reference<IConnection> _conn =
+			    wait(HTTP::proxyConnect(host, service, b->proxyHost.get(), b->proxyPort.get()));
+			conn = _conn;
+		} else {
+			host = b->proxyHost.get();
+			service = b->proxyPort.get();
+			Reference<IConnection> _conn = wait(INetworkConnections::net()->connect(host, service, false));
+			conn = _conn;
 		}
-		host = b->proxyHost.get();
-		service = b->proxyPort.get();
+	} else {
+		wait(store(conn, INetworkConnections::net()->connect(host, service, isTLS)));
 	}
-	state Reference<IConnection> conn = wait(INetworkConnections::net()->connect(host, service, isTLS));
 	wait(conn->connectHandshake());
 
 	TraceEvent("S3BlobStoreEndpointNewConnection")
@@ -894,7 +899,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 				canonicalURI += boost::algorithm::join(queryParameters, "&");
 			}
 
-			if (bstore->useProxy) {
+			if (bstore->useProxy && bstore->knobs.secure_connection == 0) {
 				// Has to be in absolute-form.
 				canonicalURI = "http://" + bstore->host + ":" + bstore->service + canonicalURI;
 			}
@@ -1190,7 +1195,7 @@ ACTOR Future<S3BlobStoreEndpoint::ListResult> listObjects_impl(Reference<S3BlobS
 	    bstore->listObjectsStream(bucket, resultStream, prefix, delimiter, maxDepth, recurseFilter);
 	// Wrap done in an actor which sends end_of_stream because list does not so that many lists can write to the same
 	// stream
-	done = map(done, [=](Void) {
+	done = map(done, [=](Void) mutable {
 		resultStream.sendError(end_of_stream());
 		return Void();
 	});
@@ -1427,7 +1432,7 @@ void S3BlobStoreEndpoint::setV4AuthHeaders(std::string const& verb,
 	if (headers.find("Content-MD5") != headers.end())
 		headersList.push_back({ "content-md5", trim_copy(headers["Content-MD5"]) + "\n" });
 	for (auto h : headers) {
-		if (StringRef(h.first).startsWith(LiteralStringRef("x-amz")))
+		if (StringRef(h.first).startsWith("x-amz"_sr))
 			headersList.push_back({ to_lower_copy(h.first), trim_copy(h.second) + "\n" });
 	}
 	std::sort(headersList.begin(), headersList.end());
@@ -1488,7 +1493,7 @@ void S3BlobStoreEndpoint::setAuthHeaders(std::string const& verb, std::string co
 	msg.append("\n");
 	for (auto h : headers) {
 		StringRef name = h.first;
-		if (name.startsWith(LiteralStringRef("x-amz")) || name.startsWith(LiteralStringRef("x-icloud"))) {
+		if (name.startsWith("x-amz"_sr) || name.startsWith("x-icloud"_sr)) {
 			msg.append(h.first);
 			msg.append(":");
 			msg.append(h.second);
diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index 7cc2079e27..cea30ca663 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -21,7 +21,7 @@
 #include "fdbclient/Schemas.h"
 
 // NOTE: also change mr-status-json-schemas.rst.inc
-const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
+const KeyRef JSONSchemas::statusSchema = R"statusSchema(
 {
    "cluster":{
       "storage_wiggler": {
@@ -137,6 +137,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
                         "blob_manager",
                         "blob_worker",
                         "encrypt_key_proxy",
+                        "consistency_scan",
                         "storage_cache",
                         "router",
                         "coordinator"
@@ -427,7 +428,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
                   "log_server_min_free_space",
                   "log_server_min_free_space_ratio",
                   "storage_server_durability_lag",
-                  "storage_server_list_fetch_failed"
+                  "storage_server_list_fetch_failed",
+                  "blob_worker_lag",
+                  "blob_worker_missing"
                ]
             },
             "description":"The database is not being saturated by the workload."
@@ -448,7 +451,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
                   "log_server_min_free_space",
                   "log_server_min_free_space_ratio",
                   "storage_server_durability_lag",
-                  "storage_server_list_fetch_failed"
+                  "storage_server_list_fetch_failed",
+                  "blob_worker_lag",
+                  "blob_worker_missing"
                ]
             },
             "description":"The database is not being saturated by the workload."
@@ -557,6 +562,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
                   "unreachable_ratekeeper_worker",
                   "unreachable_blobManager_worker",
                   "unreachable_encryptKeyProxy_worker",
+                  "unreachable_consistencyScan_worker",
                   "unreadable_configuration",
                   "full_replication_timeout",
                   "client_issues",
@@ -596,7 +602,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
          }
       ],
 )statusSchema"
-                                                          R"statusSchema(
+                                         R"statusSchema(
       "recovery_state":{
          "seconds_since_last_recovered":1,
          "required_resolvers":1,
@@ -844,8 +850,26 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
              "disabled",
              "optional_experimental",
              "required_experimental"
+         ]},
+         "encryption_at_rest_mode": {
+             "$enum":[
+             "disabled",
+             "aes_256_ctr"
          ]}
       },
+      "consistency_scan_info":{
+        "consistency_scan_enabled":false,
+        "restart":false,
+        "max_rate":0,
+        "target_interval":0,
+        "bytes_read_prev_round":0,
+        "last_round_start_datetime":"2022-04-20 00:05:05.123 +0000",
+        "last_round_finish_datetime":"1970-01-01 00:00:00.000 +0000",
+        "last_round_start_timestamp":1648857905.123,
+        "last_round_finish_timestamp":0,
+        "smoothed_round_seconds":1,
+        "finished_rounds":1
+      },
       "data":{
          "least_operating_space_bytes_log_server":0,
          "average_partition_size_bytes":0,
@@ -981,9 +1005,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
          "up_to_date":true
       }
    }
-})statusSchema");
+})statusSchema"_sr;
 
-const KeyRef JSONSchemas::clusterConfigurationSchema = LiteralStringRef(R"configSchema(
+const KeyRef JSONSchemas::clusterConfigurationSchema = R"configSchema(
 {
     "create":{
     "$enum":[
@@ -1053,9 +1077,9 @@ const KeyRef JSONSchemas::clusterConfigurationSchema = LiteralStringRef(R"config
     "auto_logs":3,
     "commit_proxies":5,
     "grv_proxies":1
-})configSchema");
+})configSchema"_sr;
 
-const KeyRef JSONSchemas::latencyBandConfigurationSchema = LiteralStringRef(R"configSchema(
+const KeyRef JSONSchemas::latencyBandConfigurationSchema = R"configSchema(
 {
     "get_read_version":{
         "bands":[
@@ -1075,30 +1099,30 @@ const KeyRef JSONSchemas::latencyBandConfigurationSchema = LiteralStringRef(R"co
         ],
         "max_commit_bytes":0
     }
-})configSchema");
+})configSchema"_sr;
 
-const KeyRef JSONSchemas::dataDistributionStatsSchema = LiteralStringRef(R"""(
+const KeyRef JSONSchemas::dataDistributionStatsSchema = R"""(
 {
   "shard_bytes": 1947000
 }
-)""");
+)"""_sr;
 
-const KeyRef JSONSchemas::logHealthSchema = LiteralStringRef(R"""(
+const KeyRef JSONSchemas::logHealthSchema = R"""(
 {
   "log_queue": 156
 }
-)""");
+)"""_sr;
 
-const KeyRef JSONSchemas::storageHealthSchema = LiteralStringRef(R"""(
+const KeyRef JSONSchemas::storageHealthSchema = R"""(
 {
   "cpu_usage": 3.28629447047675,
   "disk_usage": 0.19997897369207954,
   "storage_durability_lag": 5050809,
   "storage_queue": 2030
 }
-)""");
+)"""_sr;
 
-const KeyRef JSONSchemas::aggregateHealthSchema = LiteralStringRef(R"""(
+const KeyRef JSONSchemas::aggregateHealthSchema = R"""(
 {
   "batch_limited": false,
   "limiting_storage_durability_lag": 5050809,
@@ -1108,12 +1132,12 @@ const KeyRef JSONSchemas::aggregateHealthSchema = LiteralStringRef(R"""(
   "worst_storage_queue": 2030,
   "worst_log_queue": 156
 }
-)""");
+)"""_sr;
 
-const KeyRef JSONSchemas::managementApiErrorSchema = LiteralStringRef(R"""(
+const KeyRef JSONSchemas::managementApiErrorSchema = R"""(
 {
    "retriable": false,
    "command": "exclude",
    "message": "The reason of the error"
 }
-)""");
+)"""_sr;
diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 9f00bdf594..6304db86ce 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/ServerKnobs.h"
+#include "flow/CompressionUtils.h"
 #include "flow/IRandom.h"
 #include "flow/flow.h"
 
@@ -50,7 +51,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// TLogs
 	init( TLOG_TIMEOUT,                                          0.4 ); //cannot buggify because of availability
 	init( TLOG_SLOW_REJOIN_WARN_TIMEOUT_SECS,                     60 ); if( randomize && BUGGIFY ) TLOG_SLOW_REJOIN_WARN_TIMEOUT_SECS = deterministicRandom()->randomInt(5,10);
-	init( RECOVERY_TLOG_SMART_QUORUM_DELAY,                     0.25 ); if( randomize && BUGGIFY ) RECOVERY_TLOG_SMART_QUORUM_DELAY = 0.0; // smaller might be better for bug amplification
 	init( TLOG_STORAGE_MIN_UPDATE_INTERVAL,                      0.5 );
 	init( BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL,               30 );
 	init( DESIRED_TOTAL_BYTES,                                150000 ); if( randomize && BUGGIFY ) DESIRED_TOTAL_BYTES = 10000;
@@ -58,10 +58,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( UPDATE_DELAY,                                        0.001 );
 	init( MAXIMUM_PEEK_BYTES,                                   10e6 );
 	init( APPLY_MUTATION_BYTES,                                  1e6 );
-	init( RECOVERY_DATA_BYTE_LIMIT,                           100000 );
-	init( BUGGIFY_RECOVERY_DATA_LIMIT,                          1000 );
-	init( LONG_TLOG_COMMIT_TIME,                                0.25 ); //cannot buggify because of recovery time
-	init( LARGE_TLOG_COMMIT_BYTES,                             4<<20 );
 	init( BUGGIFY_RECOVER_MEMORY_LIMIT,                          1e6 );
 	init( BUGGIFY_WORKER_REMOVED_MAX_LAG,                         30 );
 	init( UPDATE_STORAGE_BYTE_LIMIT,                             1e6 );
@@ -94,7 +90,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( MAX_CACHE_VERSIONS,                                   10e6 );
 	init( TLOG_IGNORE_POP_AUTO_ENABLE_DELAY,                   300.0 );
 	init( TXS_POPPED_MAX_DELAY,                                  1.0 ); if ( randomize && BUGGIFY ) TXS_POPPED_MAX_DELAY = deterministicRandom()->random01();
-	init( TLOG_MAX_CREATE_DURATION,                             10.0 );
+	// In some rare simulation tests, particularly with log_spill:=1 configured, the 10 second limit is exceeded, causing SevError trace events
+	// and simulation test failure. Increasing the knob value to 15.0 in simulation is a workaround to avoid these failures.
+	init( TLOG_MAX_CREATE_DURATION,                             10.0 ); if (isSimulated) TLOG_MAX_CREATE_DURATION = 15.0;
 	init( PEEK_LOGGING_AMOUNT,                                     5 );
 	init( PEEK_LOGGING_DELAY,                                    5.0 );
 	init( PEEK_RESET_INTERVAL,                                 300.0 ); if ( randomize && BUGGIFY ) PEEK_RESET_INTERVAL = 20.0;
@@ -133,16 +131,15 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( BG_REBALANCE_POLLING_INTERVAL,                        10.0 );
 	init( BG_REBALANCE_SWITCH_CHECK_INTERVAL,                    5.0 ); if (randomize && BUGGIFY) BG_REBALANCE_SWITCH_CHECK_INTERVAL = 1.0;
 	init( DD_QUEUE_LOGGING_INTERVAL,                             5.0 );
+	init( DD_QUEUE_COUNTER_REFRESH_INTERVAL,                    60.0 );
+	// 100 / 60 < 2 trace/sec ~ 2 * 200 = 400b/sec
+	init( DD_QUEUE_COUNTER_MAX_LOG,                              100 ); if( randomize && BUGGIFY ) DD_QUEUE_COUNTER_MAX_LOG = 1;
+	init( DD_QUEUE_COUNTER_SUMMARIZE,                           true );
 	init( RELOCATION_PARALLELISM_PER_SOURCE_SERVER,                2 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_SOURCE_SERVER = 1;
 	init( RELOCATION_PARALLELISM_PER_DEST_SERVER,                 10 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_DEST_SERVER = 1; // Note: if this is smaller than FETCH_KEYS_PARALLELISM, this will artificially reduce performance. The current default of 10 is probably too high but is set conservatively for now.
 	init( DD_QUEUE_MAX_KEY_SERVERS,                              100 ); if( randomize && BUGGIFY ) DD_QUEUE_MAX_KEY_SERVERS = 1;
 	init( DD_REBALANCE_PARALLELISM,                               50 );
 	init( DD_REBALANCE_RESET_AMOUNT,                              30 );
-	init( BG_DD_MAX_WAIT,                                      120.0 );
-	init( BG_DD_MIN_WAIT,                                        0.1 );
-	init( BG_DD_INCREASE_RATE,                                  1.10 );
-	init( BG_DD_DECREASE_RATE,                                  1.02 );
-	init( BG_DD_SATURATION_DELAY,                                1.0 );
 	init( INFLIGHT_PENALTY_HEALTHY,                              1.0 );
 	init( INFLIGHT_PENALTY_UNHEALTHY,                          500.0 );
 	init( INFLIGHT_PENALTY_ONE_LEFT,                          1000.0 );
@@ -165,8 +162,14 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PRIORITY_TEAM_FAILED,                                  805 );
 	init( PRIORITY_TEAM_0_LEFT,                                  809 );
 	init( PRIORITY_SPLIT_SHARD,                                  950 ); if( randomize && BUGGIFY ) PRIORITY_SPLIT_SHARD = 350;
+	init( PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD,           960 ); if( randomize && BUGGIFY ) PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD = 360; // Set as the lowest priority
 
 	// Data distribution
+	init( SHARD_ENCODE_LOCATION_METADATA,                      false ); if( randomize && BUGGIFY )  SHARD_ENCODE_LOCATION_METADATA = true;
+	init( ENABLE_DD_PHYSICAL_SHARD,                            false ); // EXPERIMENTAL; If true, SHARD_ENCODE_LOCATION_METADATA must be true; When true, optimization of data move between DCs is disabled
+	init( MAX_PHYSICAL_SHARD_BYTES,                        500000000 ); // 500 MB; for ENABLE_DD_PHYSICAL_SHARD; smaller leads to larger number of physicalShard per storage server
+ 	init( PHYSICAL_SHARD_METRICS_DELAY,                        300.0 ); // 300 seconds; for ENABLE_DD_PHYSICAL_SHARD
+	init( ANONYMOUS_PHYSICAL_SHARD_TRANSITION_TIME,            600.0 ); if( randomize && BUGGIFY )  ANONYMOUS_PHYSICAL_SHARD_TRANSITION_TIME = 0.0; // 600 seconds; for ENABLE_DD_PHYSICAL_SHARD
 	init( READ_REBALANCE_CPU_THRESHOLD,                         15.0 );
 	init( READ_REBALANCE_SRC_PARALLELISM,                         20 );
 	init( READ_REBALANCE_SHARD_TOPK,  READ_REBALANCE_SRC_PARALLELISM * 2 );
@@ -249,7 +252,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( SERVER_LIST_DELAY,                                     1.0 );
 	init( RECRUITMENT_IDLE_DELAY,                                1.0 );
 	init( STORAGE_RECRUITMENT_DELAY,                            10.0 );
-	init( BLOB_WORKER_RECRUITMENT_DELAY,                        10.0 );
 	init( TSS_HACK_IDENTITY_MAPPING,                           false ); // THIS SHOULD NEVER BE SET IN PROD. Only for performance testing
 	init( TSS_RECRUITMENT_TIMEOUT,       3*STORAGE_RECRUITMENT_DELAY ); if (randomize && BUGGIFY ) TSS_RECRUITMENT_TIMEOUT = 1.0; // Super low timeout should cause tss recruitments to fail
 	init( TSS_DD_CHECK_INTERVAL,                                60.0 ); if (randomize && BUGGIFY ) TSS_DD_CHECK_INTERVAL = 1.0;    // May kill all TSS quickly
@@ -275,7 +277,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_FAILURE_TIME,                                       1.0 ); if( randomize && BUGGIFY ) DD_FAILURE_TIME = 10.0;
 	init( DD_ZERO_HEALTHY_TEAM_DELAY,                            1.0 );
 	init( REMOTE_KV_STORE,                                     false );
-	init( REMOTE_KV_STORE_INIT_DELAY,                            0.1 );
+	init( REBOOT_KV_STORE_DELAY,                                 0.1 );
 	init( REMOTE_KV_STORE_MAX_INIT_DURATION,                    10.0 );
 	init( REBALANCE_MAX_RETRIES,                                 100 );
 	init( DD_OVERLAP_PENALTY,                                  10000 );
@@ -291,8 +293,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY,                    120 ); if( randomize && BUGGIFY ) DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY = 5;
 	init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD,                      10 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000;
 	init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD,                      20 );
+	init( DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC,   isSimulated ? 2 : 21 * 60 * 60 * 24 ); if(randomize && BUGGIFY) DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC = isSimulated ? 0: 120;
 	init( DD_TENANT_AWARENESS_ENABLED,                         false );
-	init( TENANT_CACHE_LIST_REFRESH_INTERVAL,                    2.0 );
+	init( TENANT_CACHE_LIST_REFRESH_INTERVAL,                      2 ); if( randomize && BUGGIFY ) TENANT_CACHE_LIST_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10);
+
 
 	// TeamRemover
 	init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER,                false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true
@@ -370,19 +374,26 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( REPLACE_CONTENTS_BYTES,                                1e5 );
 
 	// KeyValueStoreRocksDB
+	init( ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES,         true ); if( randomize && BUGGIFY )  ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES = false;
+	init( ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE,                  true ); if( randomize && BUGGIFY )  ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE = false;
+	init( ROCKSDB_READ_RANGE_ROW_LIMIT,                        65535 ); if( randomize && BUGGIFY )  ROCKSDB_READ_RANGE_ROW_LIMIT = deterministicRandom()->randomInt(2, 10);
+
 	init( ROCKSDB_BACKGROUND_PARALLELISM,                          4 );
 	init( ROCKSDB_READ_PARALLELISM,                                4 );
 	// Use a smaller memtable in simulation to avoid OOMs.
 	int64_t memtableBytes = isSimulated ? 32 * 1024 : 512 * 1024 * 1024;
 	init( ROCKSDB_MEMTABLE_BYTES,                      memtableBytes );
+	init( ROCKSDB_LEVEL_STYLE_COMPACTION,                       true );
 	init( ROCKSDB_UNSAFE_AUTO_FSYNC,                           false );
 	init( ROCKSDB_PERIODIC_COMPACTION_SECONDS,                     0 );
 	init( ROCKSDB_PREFIX_LEN,                                      0 );
-	init( ROCKSDB_BLOCK_CACHE_SIZE,                                0 );
+	// If rocksdb block cache size is 0, the default 8MB is used.
+	int64_t blockCacheSize = isSimulated ? 0 : 1024 * 1024 * 1024 /* 1GB */;
+	init( ROCKSDB_BLOCK_CACHE_SIZE,                   blockCacheSize );
 	init( ROCKSDB_METRICS_DELAY,                                60.0 );
-	init( ROCKSDB_READ_VALUE_TIMEOUT,                            5.0 );
-	init( ROCKSDB_READ_VALUE_PREFIX_TIMEOUT,                     5.0 );
-	init( ROCKSDB_READ_RANGE_TIMEOUT,                            5.0 );
+	init( ROCKSDB_READ_VALUE_TIMEOUT,      isSimulated ? 5.0 : 200.0 );
+	init( ROCKSDB_READ_VALUE_PREFIX_TIMEOUT, isSimulated ? 5.0 : 200.0 );
+	init( ROCKSDB_READ_RANGE_TIMEOUT,       isSimulated ? 5.0 : 200.0 );
 	init( ROCKSDB_READ_QUEUE_WAIT,                               1.0 );
 	init( ROCKSDB_READ_QUEUE_HARD_MAX,                          1000 );
 	init( ROCKSDB_READ_QUEUE_SOFT_MAX,                           500 );
@@ -396,9 +407,11 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// If true, enables dynamic adjustment of ROCKSDB_WRITE_RATE_LIMITER_BYTES according to the recent demand of background IO.
 	init( ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE,                 true );
 	init( DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY,                    "fdb");
+	init( ROCKSDB_DISABLE_AUTO_COMPACTIONS,                    false ); // RocksDB default
 
 	init( ROCKSDB_PERFCONTEXT_ENABLE,                          false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip() ? false : true;
 	init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE,                    0.0001 );
+	init( ROCKSDB_METRICS_SAMPLE_INTERVAL,						  0.0);
 	init( ROCKSDB_MAX_SUBCOMPACTIONS,                              2 );
 	init( ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT,      64000000000 ); // 64GB, Rocksdb option, Writes will slow down.
 	init( ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT,     100000000000 ); // 100GB, Rocksdb option, Writes will stall.
@@ -411,6 +424,12 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_COMPACTION_READAHEAD_SIZE,                   32768 ); // 32 KB, performs bigger reads when doing compaction.
 	init( ROCKSDB_BLOCK_SIZE,                                  32768 ); // 32 KB, size of the block in rocksdb cache.
  	init( ENABLE_SHARDED_ROCKSDB,                              false );
+	init( ROCKSDB_WRITE_BUFFER_SIZE,                         1 << 30 ); // 1G
+	init( ROCKSDB_CF_WRITE_BUFFER_SIZE,                     64 << 20 ); // 64M, RocksDB default.
+	init( ROCKSDB_MAX_TOTAL_WAL_SIZE,                              0 ); // RocksDB default.
+	init( ROCKSDB_MAX_BACKGROUND_JOBS,                             2 ); // RocksDB default.
+	init( ROCKSDB_DELETE_OBSOLETE_FILE_PERIOD,                 21600 ); // 6h, RocksDB default.
+	init( ROCKSDB_PHYSICAL_SHARD_CLEAN_UP_DELAY, isSimulated ? 10.0 : 300.0 ); // Delays shard clean up, must be larger than ROCKSDB_READ_VALUE_TIMEOUT to prevent reading deleted shard.
 
 	// Leader election
 	bool longLeaderElection = randomize && BUGGIFY;
@@ -474,7 +493,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( REPORT_TRANSACTION_COST_ESTIMATION_DELAY,               0.1 );
 	init( PROXY_REJECT_BATCH_QUEUED_TOO_LONG,                    true );
 
-        bool buggfyUseResolverPrivateMutations = randomize && BUGGIFY && !ENABLE_VERSION_VECTOR_TLOG_UNICAST;
+	bool buggfyUseResolverPrivateMutations = randomize && BUGGIFY && !ENABLE_VERSION_VECTOR_TLOG_UNICAST;
 	init( PROXY_USE_RESOLVER_PRIVATE_MUTATIONS,                 false ); if( buggfyUseResolverPrivateMutations ) PROXY_USE_RESOLVER_PRIVATE_MUTATIONS = deterministicRandom()->coinflip();
 
 	init( RESET_MASTER_BATCHES,                                   200 );
@@ -482,6 +501,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( RESET_MASTER_DELAY,                                   300.0 );
 	init( RESET_RESOLVER_DELAY,                                 300.0 );
 
+	init( GLOBAL_CONFIG_MIGRATE_TIMEOUT,                          5.0 );
+	init( GLOBAL_CONFIG_REFRESH_INTERVAL,                         1.0 ); if ( randomize && BUGGIFY ) GLOBAL_CONFIG_REFRESH_INTERVAL = 0.1;
+	init( GLOBAL_CONFIG_REFRESH_TIMEOUT,                         10.0 ); if ( randomize && BUGGIFY ) GLOBAL_CONFIG_REFRESH_TIMEOUT = 1.0;
+
 	// Master Server
 	// masterCommitter() in the master server will allow lower priority tasks (e.g. DataDistibution)
 	//  by delay()ing for this amount of time between accepted batches of TransactionRequests.
@@ -530,6 +553,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ATTEMPT_RECRUITMENT_DELAY,                           0.035 );
 	init( WAIT_FOR_DISTRIBUTOR_JOIN_DELAY,                       1.0 );
 	init( WAIT_FOR_RATEKEEPER_JOIN_DELAY,                        1.0 );
+	init( WAIT_FOR_CONSISTENCYSCAN_JOIN_DELAY,                   1.0 );
 	init( WAIT_FOR_BLOB_MANAGER_JOIN_DELAY,                      1.0 );
 	init( WAIT_FOR_ENCRYPT_KEY_PROXY_JOIN_DELAY,                 1.0 );
 	init( WORKER_FAILURE_TIME,                                   1.0 ); if( randomize && BUGGIFY ) WORKER_FAILURE_TIME = 10.0;
@@ -540,6 +564,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( CHECK_REMOTE_HEALTH_INTERVAL,                           60 );
 	init( FORCE_RECOVERY_CHECK_DELAY,                            5.0 );
 	init( RATEKEEPER_FAILURE_TIME,                               1.0 );
+	init( CONSISTENCYSCAN_FAILURE_TIME,                          1.0 );
 	init( BLOB_MANAGER_FAILURE_TIME,                             1.0 );
 	init( REPLACE_INTERFACE_DELAY,                              60.0 );
 	init( REPLACE_INTERFACE_CHECK_DELAY,                         5.0 );
@@ -605,9 +630,13 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( SLOW_SMOOTHING_AMOUNT,                                10.0 ); if( slowRatekeeper ) SLOW_SMOOTHING_AMOUNT = 50.0;
 	init( METRIC_UPDATE_RATE,                                     .1 ); if( slowRatekeeper ) METRIC_UPDATE_RATE = 0.5;
 	init( DETAILED_METRIC_UPDATE_RATE,                           5.0 );
-	init (RATEKEEPER_DEFAULT_LIMIT,                              1e6 ); if( randomize && BUGGIFY ) RATEKEEPER_DEFAULT_LIMIT = 0;
+	init( RATEKEEPER_DEFAULT_LIMIT,                              1e6 ); if( randomize && BUGGIFY ) RATEKEEPER_DEFAULT_LIMIT = 0;
 	init( RATEKEEPER_LIMIT_REASON_SAMPLE_RATE,                   0.1 );
 	init( RATEKEEPER_PRINT_LIMIT_REASON,                       false ); if( randomize && BUGGIFY ) RATEKEEPER_PRINT_LIMIT_REASON = true;
+	init( RATEKEEPER_MIN_RATE,                                   0.0 );
+	init( RATEKEEPER_MAX_RATE,                                   1e9 );
+	init( RATEKEEPER_BATCH_MIN_RATE,                             0.0 );
+	init( RATEKEEPER_BATCH_MAX_RATE,                             1e9 );
 
 	bool smallStorageTarget = randomize && BUGGIFY;
 	init( TARGET_BYTES_PER_STORAGE_SERVER,                    1000e6 ); if( smallStorageTarget ) TARGET_BYTES_PER_STORAGE_SERVER = 3000e3;
@@ -617,9 +646,12 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( SPRING_BYTES_STORAGE_SERVER_BATCH,                   100e6 ); if( smallStorageTarget ) SPRING_BYTES_STORAGE_SERVER_BATCH = 150e3;
 	init( STORAGE_HARD_LIMIT_BYTES,                           1500e6 ); if( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES = 4500e3;
 	init( STORAGE_HARD_LIMIT_BYTES_OVERAGE,                   5000e3 ); if( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES_OVERAGE = 100e3; // byte+version overage ensures storage server makes enough progress on freeing up storage queue memory at hard limit by ensuring it advances desiredOldestVersion enough per commit cycle.
+	init( STORAGE_HARD_LIMIT_BYTES_SPEED_UP_SIM, STORAGE_HARD_LIMIT_BYTES ); if( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES_SPEED_UP_SIM *= 10;
+	init( STORAGE_HARD_LIMIT_BYTES_OVERAGE_SPEED_UP_SIM, STORAGE_HARD_LIMIT_BYTES_OVERAGE ); if( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES_OVERAGE_SPEED_UP_SIM *= 10;
 	init( STORAGE_HARD_LIMIT_VERSION_OVERAGE, VERSIONS_PER_SECOND / 4.0 );
 	init( STORAGE_DURABILITY_LAG_HARD_MAX,                    2000e6 ); if( smallStorageTarget ) STORAGE_DURABILITY_LAG_HARD_MAX = 100e6;
 	init( STORAGE_DURABILITY_LAG_SOFT_MAX,                     250e6 ); if( smallStorageTarget ) STORAGE_DURABILITY_LAG_SOFT_MAX = 10e6;
+	init( STORAGE_INCLUDE_FEED_STORAGE_QUEUE,                   true ); if ( randomize && BUGGIFY ) STORAGE_INCLUDE_FEED_STORAGE_QUEUE = false;
 
 	//FIXME: Low priority reads are disabled by assigning very high knob values, reduce knobs for 7.0
 	init( LOW_PRIORITY_STORAGE_QUEUE_BYTES,                    775e8 ); if( smallStorageTarget ) LOW_PRIORITY_STORAGE_QUEUE_BYTES = 1750e3;
@@ -657,6 +689,20 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DURABILITY_LAG_REDUCTION_RATE,                      0.9999 );
 	init( DURABILITY_LAG_INCREASE_RATE,                        1.001 );
 	init( STORAGE_SERVER_LIST_FETCH_TIMEOUT,                    20.0 );
+	init( BW_THROTTLING_ENABLED,                                true );
+
+	bool buggifySmallBWLag = randomize && BUGGIFY;
+	init( TARGET_BW_LAG,                                       240.0 ); if(buggifySmallBWLag) TARGET_BW_LAG = 10.0;
+	init( TARGET_BW_LAG_BATCH,                                 200.0 ); if(buggifySmallBWLag) TARGET_BW_LAG_BATCH = 4.0;
+	init( TARGET_BW_LAG_UPDATE,                                  9.0 ); if(buggifySmallBWLag) TARGET_BW_LAG_UPDATE = 1.0;
+	init( MIN_BW_HISTORY,                                         10 );
+	init( BW_ESTIMATION_INTERVAL,                               10.0 ); if(buggifySmallBWLag) BW_ESTIMATION_INTERVAL = 2.0;
+	init( BW_LAG_INCREASE_AMOUNT,                                1.1 );
+	init( BW_LAG_DECREASE_AMOUNT,                                0.9 );
+	init( BW_FETCH_WORKERS_INTERVAL,                             5.0 );
+	init( BW_RW_LOGGING_INTERVAL,                                5.0 );
+	init( BW_MAX_BLOCKED_INTERVAL,                              10.0 ); if(buggifySmallBWLag) BW_MAX_BLOCKED_INTERVAL = 2.0;
+	init( BW_RK_SIM_QUIESCE_DELAY,                             150.0 );
 
 	init( MAX_AUTO_THROTTLED_TRANSACTION_TAGS,                     5 ); if(randomize && BUGGIFY) MAX_AUTO_THROTTLED_TRANSACTION_TAGS = 1;
 	init( MAX_MANUAL_THROTTLED_TRANSACTION_TAGS,                  40 ); if(randomize && BUGGIFY) MAX_MANUAL_THROTTLED_TRANSACTION_TAGS = 1;
@@ -671,9 +717,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( AUTO_TAG_THROTTLING_ENABLED,                          true ); if(randomize && BUGGIFY) AUTO_TAG_THROTTLING_ENABLED = false;
 	init( SS_THROTTLE_TAGS_TRACKED,                                1 ); if(randomize && BUGGIFY) SS_THROTTLE_TAGS_TRACKED = deterministicRandom()->randomInt(1, 10);
 	init( GLOBAL_TAG_THROTTLING,                               false );
+	init( ENFORCE_TAG_THROTTLING_ON_PROXIES,                   false );
 	init( GLOBAL_TAG_THROTTLING_MIN_RATE,                        1.0 );
 	init( GLOBAL_TAG_THROTTLING_FOLDING_TIME,                   10.0 );
-	init( GLOBAL_TAG_THROTTLING_TRACE_INTERVAL,                  5.0 );
+	init( GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO,            5.0 );
 
 	//Storage Metrics
 	init( STORAGE_METRICS_AVERAGE_INTERVAL,                    120.0 );
@@ -694,12 +741,14 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( STORAGE_LIMIT_BYTES,                                500000 );
 	init( BUGGIFY_LIMIT_BYTES,                                  1000 );
 	init( FETCH_USING_STREAMING,                               false ); if( randomize && isSimulated && BUGGIFY ) FETCH_USING_STREAMING = true; //Determines if fetch keys uses streaming reads
+	init( FETCH_USING_BLOB,                                    false );
 	init( FETCH_BLOCK_BYTES,                                     2e6 );
 	init( FETCH_KEYS_PARALLELISM_BYTES,                          4e6 ); if( randomize && BUGGIFY ) FETCH_KEYS_PARALLELISM_BYTES = 3e6;
 	init( FETCH_KEYS_PARALLELISM,                                  2 );
+	init( FETCH_KEYS_PARALLELISM_FULL,                             6 );
 	init( FETCH_KEYS_LOWER_PRIORITY,                               0 );
-	init( FETCH_CHANGEFEED_PARALLELISM,                            2 );
 	init( SERVE_FETCH_CHECKPOINT_PARALLELISM,                      4 );
+	init( CHANGE_FEED_DISK_READS_PARALLELISM,                   1000 ); if( randomize && BUGGIFY ) CHANGE_FEED_DISK_READS_PARALLELISM = 20;
 	init( BUGGIFY_BLOCK_BYTES,                                 10000 );
 	init( STORAGE_RECOVERY_VERSION_LAG_LIMIT,				2 * MAX_READ_TRANSACTION_LIFE_VERSIONS );
 	init( STORAGE_COMMIT_BYTES,                             10000000 ); if( randomize && BUGGIFY ) STORAGE_COMMIT_BYTES = 2000000;
@@ -707,7 +756,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( STORAGE_DURABILITY_LAG_REJECT_THRESHOLD,              0.25 );
 	init( STORAGE_DURABILITY_LAG_MIN_RATE,                       0.1 );
 	init( STORAGE_COMMIT_INTERVAL,                               0.5 ); if( randomize && BUGGIFY ) STORAGE_COMMIT_INTERVAL = 2.0;
-	init( UPDATE_SHARD_VERSION_INTERVAL,                        0.25 ); if( randomize && BUGGIFY ) UPDATE_SHARD_VERSION_INTERVAL = 1.0;
 	init( BYTE_SAMPLING_FACTOR,                                  250 ); //cannot buggify because of differences in restarting tests
 	init( BYTE_SAMPLING_OVERHEAD,                                100 );
 	init( MAX_STORAGE_SERVER_WATCH_BYTES,                      100e6 ); if( randomize && BUGGIFY ) MAX_STORAGE_SERVER_WATCH_BYTES = 10e3;
@@ -716,7 +764,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( BYTE_SAMPLE_LOAD_PARALLELISM,                            8 ); if( randomize && BUGGIFY ) BYTE_SAMPLE_LOAD_PARALLELISM = 1;
 	init( BYTE_SAMPLE_LOAD_DELAY,                                0.0 ); if( randomize && BUGGIFY ) BYTE_SAMPLE_LOAD_DELAY = 0.1;
 	init( BYTE_SAMPLE_START_DELAY,                               1.0 ); if( randomize && BUGGIFY ) BYTE_SAMPLE_START_DELAY = 0.0;
-	init( UPDATE_STORAGE_PROCESS_STATS_INTERVAL,                 5.0 );
 	init( BEHIND_CHECK_DELAY,                                    2.0 );
 	init( BEHIND_CHECK_COUNT,                                      2 );
 	init( BEHIND_CHECK_VERSIONS,             5 * VERSIONS_PER_SECOND );
@@ -724,7 +771,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( MIN_TAG_READ_PAGES_RATE,                             1.0e4 ); if( randomize && BUGGIFY ) MIN_TAG_READ_PAGES_RATE = 0;
 	init( MIN_TAG_WRITE_PAGES_RATE,                             3200 ); if( randomize && BUGGIFY ) MIN_TAG_WRITE_PAGES_RATE = 0;
 	init( TAG_MEASUREMENT_INTERVAL,                        30.0 ); if( randomize && BUGGIFY ) TAG_MEASUREMENT_INTERVAL = 1.0;
-	init( READ_COST_BYTE_FACTOR,                          16384 ); if( randomize && BUGGIFY ) READ_COST_BYTE_FACTOR = 4096;
 	init( PREFIX_COMPRESS_KVS_MEM_SNAPSHOTS,                    true ); if( randomize && BUGGIFY ) PREFIX_COMPRESS_KVS_MEM_SNAPSHOTS = false;
 	init( REPORT_DD_METRICS,                                    true );
 	init( DD_METRICS_REPORT_INTERVAL,                           30.0 );
@@ -740,6 +786,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( MAX_PARALLEL_QUICK_GET_VALUE,                           50 ); if ( randomize && BUGGIFY ) MAX_PARALLEL_QUICK_GET_VALUE = deterministicRandom()->randomInt(1, 100);
 	init( QUICK_GET_KEY_VALUES_LIMIT,                           2000 );
 	init( QUICK_GET_KEY_VALUES_LIMIT_BYTES,                      1e7 );
+	init( STORAGE_FEED_QUERY_HARD_LIMIT,                      100000 );
 
 	//Wait Failure
 	init( MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS,                 250 ); if( randomize && BUGGIFY ) MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS = 2;
@@ -760,13 +807,14 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ENABLE_WORKER_HEALTH_MONITOR,                        false );
 	init( WORKER_HEALTH_MONITOR_INTERVAL,                       60.0 );
 	init( PEER_LATENCY_CHECK_MIN_POPULATION,                      30 );
-	init( PEER_LATENCY_DEGRADATION_PERCENTILE,                  0.90 );
+	init( PEER_LATENCY_DEGRADATION_PERCENTILE,                  0.50 );
 	init( PEER_LATENCY_DEGRADATION_THRESHOLD,                   0.05 );
-	init( PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE,        0.90 );
+	init( PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE,        0.50 );
 	init( PEER_LATENCY_DEGRADATION_THRESHOLD_SATELLITE,          0.1 );
 	init( PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD,         0.1 );
-	init( PEER_DEGRADATION_CONNECTION_FAILURE_COUNT,               1 );
+	init( PEER_DEGRADATION_CONNECTION_FAILURE_COUNT,               5 );
 	init( WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER,           true );
+	init( STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT,                 false ); if ( randomize && BUGGIFY ) STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT = true;
 
 	// Test harness
 	init( WORKER_POLL_DELAY,                                     1.0 );
@@ -779,7 +827,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	// Dynamic Knobs (implementation)
 	init( COMPACTION_INTERVAL,             isSimulated ? 5.0 : 300.0 );
-	init( UPDATE_NODE_TIMEOUT,                                   3.0 );
+	init( BROADCASTER_SELF_UPDATE_DELAY,                         1.0 );
 	init( GET_COMMITTED_VERSION_TIMEOUT,                         3.0 );
 	init( GET_SNAPSHOT_AND_CHANGES_TIMEOUT,                      3.0 );
 	init( FETCH_CHANGES_TIMEOUT,                                 3.0 );
@@ -795,14 +843,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DISABLE_DUPLICATE_LOG_WARNING,                       false );
 	init( HISTOGRAM_REPORT_INTERVAL,                           300.0 );
 
-	// IPager
-	init( PAGER_RESERVED_PAGES,                                    1 );
-
-	// IndirectShadowPager
-	init( FREE_PAGE_VACUUM_THRESHOLD,                              1 );
-	init( VACUUM_QUEUE_SIZE,                                  100000 );
-	init( VACUUM_BYTES_PER_SECOND,                               1e6 );
-
 	// Timekeeper
 	init( TIME_KEEPER_DELAY,                                      10 );
 	init( TIME_KEEPER_MAX_ENTRIES,                3600 * 24 * 30 * 6 ); if( randomize && BUGGIFY ) { TIME_KEEPER_MAX_ENTRIES = 2; }
@@ -821,11 +861,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( FASTRESTORE_ROLE_LOGGING_DELAY,                          5 ); if( randomize && BUGGIFY ) { FASTRESTORE_ROLE_LOGGING_DELAY = deterministicRandom()->random01() * 60 + 1; }
 	init( FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL,               5 ); if( randomize && BUGGIFY ) { FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL = deterministicRandom()->random01() * 60 + 1; }
 	init( FASTRESTORE_ATOMICOP_WEIGHT,                             1 ); if( randomize && BUGGIFY ) { FASTRESTORE_ATOMICOP_WEIGHT = deterministicRandom()->random01() * 200 + 1; }
-	init( FASTRESTORE_APPLYING_PARALLELISM,               	   10000 ); if( randomize && BUGGIFY ) { FASTRESTORE_APPLYING_PARALLELISM = deterministicRandom()->random01() * 10 + 1; }
 	init( FASTRESTORE_MONITOR_LEADER_DELAY,                        5 ); if( randomize && BUGGIFY ) { FASTRESTORE_MONITOR_LEADER_DELAY = deterministicRandom()->random01() * 100; }
 	init( FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS,                60 ); if( randomize && BUGGIFY ) { FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS = deterministicRandom()->random01() * 240 + 10; }
 	init( FASTRESTORE_TRACK_REQUEST_LATENCY,              	   false ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_REQUEST_LATENCY = false; }
-	init( FASTRESTORE_TRACK_LOADER_SEND_REQUESTS,              false ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_LOADER_SEND_REQUESTS = true; }
 	init( FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT,                 6144 ); if( randomize && BUGGIFY ) { FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT = 1; }
 	init( FASTRESTORE_WAIT_FOR_MEMORY_LATENCY,                    10 ); if( randomize && BUGGIFY ) { FASTRESTORE_WAIT_FOR_MEMORY_LATENCY = 60; }
 	init( FASTRESTORE_HEARTBEAT_DELAY,                            10 ); if( randomize && BUGGIFY ) { FASTRESTORE_HEARTBEAT_DELAY = deterministicRandom()->random01() * 120 + 2; }
@@ -882,52 +920,60 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init ( CLUSTER_RECOVERY_EVENT_NAME_PREFIX,              "Master" );
 
 	// Encryption
-	init( ENABLE_ENCRYPTION,                                   false ); if ( randomize && BUGGIFY ) { ENABLE_ENCRYPTION = deterministicRandom()->coinflip(); }
+	init( ENABLE_ENCRYPTION,                                   false ); if ( randomize && BUGGIFY ) ENABLE_ENCRYPTION = !ENABLE_ENCRYPTION;
 	init( ENCRYPTION_MODE,                             "AES-256-CTR" );
 	init( SIM_KMS_MAX_KEYS,                                     4096 );
 	init( ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH,                 100000 );
-	init( ENABLE_TLOG_ENCRYPTION,                  ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY ) { ENABLE_TLOG_ENCRYPTION = (ENABLE_ENCRYPTION && !PROXY_USE_RESOLVER_PRIVATE_MUTATIONS && deterministicRandom()->coinflip()); }
-	init( ENABLE_BLOB_GRANULE_ENCRYPTION,          ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY ) { ENABLE_BLOB_GRANULE_ENCRYPTION = (ENABLE_ENCRYPTION && deterministicRandom()->coinflip()); }
+	init( ENABLE_TLOG_ENCRYPTION,                  ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION && !PROXY_USE_RESOLVER_PRIVATE_MUTATIONS ) ENABLE_TLOG_ENCRYPTION = true;
+	init( ENABLE_STORAGE_SERVER_ENCRYPTION,        ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY) ENABLE_STORAGE_SERVER_ENCRYPTION = !ENABLE_STORAGE_SERVER_ENCRYPTION;
+	init( ENABLE_BLOB_GRANULE_ENCRYPTION,          ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY) ENABLE_BLOB_GRANULE_ENCRYPTION = !ENABLE_BLOB_GRANULE_ENCRYPTION;
 
 	// encrypt key proxy
 	init( ENABLE_BLOB_GRANULE_COMPRESSION,                     false ); if ( randomize && BUGGIFY ) { ENABLE_BLOB_GRANULE_COMPRESSION = deterministicRandom()->coinflip(); }
-	init( BLOB_GRANULE_COMPRESSION_FILTER,                    "GZIP" ); if ( randomize && BUGGIFY ) { BLOB_GRANULE_COMPRESSION_FILTER = "NONE"; }
+	init( BLOB_GRANULE_COMPRESSION_FILTER,                    "NONE" ); if ( randomize && BUGGIFY ) { BLOB_GRANULE_COMPRESSION_FILTER = CompressionUtils::toString(CompressionUtils::getRandomFilter()); }
 
-
-    // KMS connector type
+	// KMS connector type
 	init( KMS_CONNECTOR_TYPE,                     "RESTKmsConnector" );
 
 	// Blob granlues
-	init( BG_URL,               isSimulated ? "file://fdbblob/" : "" ); // TODO: store in system key space or something, eventually
-	// BlobGranuleVerify* simulation tests use "blobRangeKeys", BlobGranuleCorrectness* use "tenant", default in real clusters is "tenant"
-	init( BG_RANGE_SOURCE,                                  "tenant" );
+	init( BG_URL,               isSimulated ? "file://fdbblob/" : "" ); // TODO: store in system key space or something, eventually	
+	bool buggifyMediumGranules = simulationMediumShards || (randomize && BUGGIFY);
 	// BlobGranuleVerify* simulation tests use "knobs", BlobGranuleCorrectness* use "tenant", default in real clusters is "knobs"
 	init( BG_METADATA_SOURCE,                                "knobs" );
-	init( BG_SNAPSHOT_FILE_TARGET_BYTES,                    10000000 ); if( buggifySmallShards ) BG_SNAPSHOT_FILE_TARGET_BYTES = 100000; else if (simulationMediumShards || (randomize && BUGGIFY) ) BG_SNAPSHOT_FILE_TARGET_BYTES = 1000000; 
-	init( BG_SNAPSHOT_FILE_TARGET_CHUNKS,                        100 ); if ( randomize && BUGGIFY ) BG_SNAPSHOT_FILE_TARGET_CHUNKS = 1 << deterministicRandom()->randomInt(0, 8);
+	init( BG_SNAPSHOT_FILE_TARGET_BYTES,                    10000000 ); if( buggifySmallShards ) BG_SNAPSHOT_FILE_TARGET_BYTES = 100000; else if (buggifyMediumGranules) BG_SNAPSHOT_FILE_TARGET_BYTES = 1000000;
+	init( BG_SNAPSHOT_FILE_TARGET_CHUNK_BYTES,               64*1024 ); if ( randomize && BUGGIFY ) BG_SNAPSHOT_FILE_TARGET_CHUNK_BYTES = BG_SNAPSHOT_FILE_TARGET_BYTES / (1 << deterministicRandom()->randomInt(0, 8));
 	init( BG_DELTA_BYTES_BEFORE_COMPACT, BG_SNAPSHOT_FILE_TARGET_BYTES/2 );
 	init( BG_DELTA_FILE_TARGET_BYTES,   BG_DELTA_BYTES_BEFORE_COMPACT/10 );
+	init( BG_DELTA_FILE_TARGET_CHUNK_BYTES,                  32*1024 ); if ( randomize && BUGGIFY ) BG_DELTA_FILE_TARGET_CHUNK_BYTES = BG_DELTA_FILE_TARGET_BYTES / (1 << deterministicRandom()->randomInt(0, 7));
 	init( BG_MAX_SPLIT_FANOUT,                                    10 ); if( randomize && BUGGIFY ) BG_MAX_SPLIT_FANOUT = deterministicRandom()->randomInt(5, 15);
 	init( BG_MAX_MERGE_FANIN,                                     10 ); if( randomize && BUGGIFY ) BG_MAX_MERGE_FANIN = deterministicRandom()->randomInt(2, 15);
 	init( BG_HOT_SNAPSHOT_VERSIONS,                          5000000 );
 
 	init( BG_CONSISTENCY_CHECK_ENABLED,                         true ); if (randomize && BUGGIFY) BG_CONSISTENCY_CHECK_ENABLED = false;
 	init( BG_CONSISTENCY_CHECK_TARGET_SPEED_KB,                 1000 ); if (randomize && BUGGIFY) BG_CONSISTENCY_CHECK_TARGET_SPEED_KB *= (deterministicRandom()->randomInt(2, 50) / 10);
+	init( BG_KEY_TUPLE_TRUNCATE_OFFSET,                            0 );
 
 	init( BG_ENABLE_MERGING,                                    true ); if (randomize && BUGGIFY) BG_ENABLE_MERGING = false;
 	init( BG_MERGE_CANDIDATE_THRESHOLD_SECONDS, isSimulated ? 20.0 : 30 * 60 ); if (randomize && BUGGIFY) BG_MERGE_CANDIDATE_THRESHOLD_SECONDS = 5.0;
-	
+	init( BG_MERGE_CANDIDATE_DELAY_SECONDS, BG_MERGE_CANDIDATE_THRESHOLD_SECONDS / 10.0 );
 
 	init( BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM,                8 ); if( randomize && BUGGIFY ) BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM = 1;
+	init( BLOB_WORKER_RESNAPSHOT_PARALLELISM,                     40 ); if( randomize && BUGGIFY ) BLOB_WORKER_RESNAPSHOT_PARALLELISM = deterministicRandom()->randomInt(1, 10);
+	init( BLOB_WORKER_DELTA_FILE_WRITE_PARALLELISM,             2000 ); if( randomize && BUGGIFY ) BLOB_WORKER_DELTA_FILE_WRITE_PARALLELISM = deterministicRandom()->randomInt(10, 100);
 	init( BLOB_WORKER_TIMEOUT,                                  10.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_TIMEOUT = 1.0;
 	init( BLOB_WORKER_REQUEST_TIMEOUT,                           5.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_REQUEST_TIMEOUT = 1.0;
 	init( BLOB_WORKERLIST_FETCH_INTERVAL,                        1.0 );
 	init( BLOB_WORKER_BATCH_GRV_INTERVAL,                        0.1 );
-	
+	init( BLOB_WORKER_DO_REJECT_WHEN_FULL,                      true ); if ( randomize && BUGGIFY ) BLOB_WORKER_DO_REJECT_WHEN_FULL = false;
+	init( BLOB_WORKER_REJECT_WHEN_FULL_THRESHOLD,                0.9 );
+	init( BLOB_WORKER_FORCE_FLUSH_CLEANUP_DELAY,                30.0 ); if ( randomize && BUGGIFY ) BLOB_WORKER_FORCE_FLUSH_CLEANUP_DELAY = deterministicRandom()->randomInt(0, 10) - 1;
 
 	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN,                   0.1 );
 	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX,                   5.0 );
 	init( BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT,              1.5 );
+	init( BLOB_MANAGER_CONCURRENT_MERGE_CHECKS,                   64 ); if( randomize && BUGGIFY ) BLOB_MANAGER_CONCURRENT_MERGE_CHECKS = 1 << deterministicRandom()->randomInt(0, 7);
+	init( BLOB_MANIFEST_BACKUP,                                false );
+	init( BLOB_FULL_RESTORE_MODE,                              false );
 
 	init( BGCC_TIMEOUT,                   isSimulated ? 10.0 : 120.0 );
 	init( BGCC_MIN_INTERVAL,                isSimulated ? 1.0 : 10.0 );
diff --git a/fdbclient/SimpleConfigTransaction.actor.cpp b/fdbclient/SimpleConfigTransaction.actor.cpp
index cea49019bc..dba5d327b7 100644
--- a/fdbclient/SimpleConfigTransaction.actor.cpp
+++ b/fdbclient/SimpleConfigTransaction.actor.cpp
@@ -43,11 +43,13 @@ class SimpleConfigTransactionImpl {
 		state ConfigTransactionGetGenerationReply reply;
 		if (self->cti.hostname.present()) {
 			wait(store(reply,
-			           retryGetReplyFromHostname(ConfigTransactionGetGenerationRequest{},
+			           retryGetReplyFromHostname(ConfigTransactionGetGenerationRequest{ 0, Optional<Version>() },
 			                                     self->cti.hostname.get(),
 			                                     WLTOKEN_CONFIGTXN_GETGENERATION)));
 		} else {
-			wait(store(reply, retryBrokenPromise(self->cti.getGeneration, ConfigTransactionGetGenerationRequest{})));
+			wait(store(reply,
+			           retryBrokenPromise(self->cti.getGeneration,
+			                              ConfigTransactionGetGenerationRequest{ 0, Optional<Version>() })));
 		}
 		if (self->dID.present()) {
 			TraceEvent("SimpleConfigTransactionGotReadVersion", self->dID.get())
@@ -70,11 +72,12 @@ class SimpleConfigTransactionImpl {
 		state ConfigTransactionGetReply reply;
 		if (self->cti.hostname.present()) {
 			wait(store(reply,
-			           retryGetReplyFromHostname(ConfigTransactionGetRequest{ generation, configKey },
+			           retryGetReplyFromHostname(ConfigTransactionGetRequest{ 0, generation, configKey },
 			                                     self->cti.hostname.get(),
 			                                     WLTOKEN_CONFIGTXN_GET)));
 		} else {
-			wait(store(reply, retryBrokenPromise(self->cti.get, ConfigTransactionGetRequest{ generation, configKey })));
+			wait(store(reply,
+			           retryBrokenPromise(self->cti.get, ConfigTransactionGetRequest{ 0, generation, configKey })));
 		}
 		if (self->dID.present()) {
 			TraceEvent("SimpleConfigTransactionGotValue", self->dID.get())
@@ -95,13 +98,13 @@ class SimpleConfigTransactionImpl {
 		state ConfigTransactionGetConfigClassesReply reply;
 		if (self->cti.hostname.present()) {
 			wait(store(reply,
-			           retryGetReplyFromHostname(ConfigTransactionGetConfigClassesRequest{ generation },
+			           retryGetReplyFromHostname(ConfigTransactionGetConfigClassesRequest{ 0, generation },
 			                                     self->cti.hostname.get(),
 			                                     WLTOKEN_CONFIGTXN_GETCLASSES)));
 		} else {
 			wait(store(
 			    reply,
-			    retryBrokenPromise(self->cti.getClasses, ConfigTransactionGetConfigClassesRequest{ generation })));
+			    retryBrokenPromise(self->cti.getClasses, ConfigTransactionGetConfigClassesRequest{ 0, generation })));
 		}
 		RangeResult result;
 		for (const auto& configClass : reply.configClasses) {
@@ -118,13 +121,13 @@ class SimpleConfigTransactionImpl {
 		state ConfigTransactionGetKnobsReply reply;
 		if (self->cti.hostname.present()) {
 			wait(store(reply,
-			           retryGetReplyFromHostname(ConfigTransactionGetKnobsRequest{ generation, configClass },
+			           retryGetReplyFromHostname(ConfigTransactionGetKnobsRequest{ 0, generation, configClass },
 			                                     self->cti.hostname.get(),
 			                                     WLTOKEN_CONFIGTXN_GETKNOBS)));
 		} else {
-			wait(store(
-			    reply,
-			    retryBrokenPromise(self->cti.getKnobs, ConfigTransactionGetKnobsRequest{ generation, configClass })));
+			wait(store(reply,
+			           retryBrokenPromise(self->cti.getKnobs,
+			                              ConfigTransactionGetKnobsRequest{ 0, generation, configClass })));
 		}
 		RangeResult result;
 		for (const auto& knobName : reply.knobNames) {
@@ -137,6 +140,7 @@ class SimpleConfigTransactionImpl {
 		if (!self->getGenerationFuture.isValid()) {
 			self->getGenerationFuture = getGeneration(self);
 		}
+		self->toCommit.coordinatorsHash = 0;
 		wait(store(self->toCommit.generation, self->getGenerationFuture));
 		self->toCommit.annotation.timestamp = now();
 		if (self->cti.hostname.present()) {
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index 85f237cb6d..698d15b38e 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -21,10 +21,12 @@
 #include "boost/lexical_cast.hpp"
 #include "boost/algorithm/string.hpp"
 
+#include <string>
 #include <time.h>
 #include <msgpack.hpp>
 
 #include <exception>
+#include <unordered_set>
 
 #include "fdbclient/ActorLineageProfiler.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
@@ -56,65 +58,46 @@ static bool isAlphaNumeric(const std::string& key) {
 } // namespace
 
 std::unordered_map<SpecialKeySpace::MODULE, KeyRange> SpecialKeySpace::moduleToBoundary = {
-	{ SpecialKeySpace::MODULE::TRANSACTION,
-	  KeyRangeRef(LiteralStringRef("\xff\xff/transaction/"), LiteralStringRef("\xff\xff/transaction0")) },
+	{ SpecialKeySpace::MODULE::TRANSACTION, KeyRangeRef("\xff\xff/transaction/"_sr, "\xff\xff/transaction0"_sr) },
 	{ SpecialKeySpace::MODULE::WORKERINTERFACE,
-	  KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"), LiteralStringRef("\xff\xff/worker_interfaces0")) },
-	{ SpecialKeySpace::MODULE::STATUSJSON, singleKeyRange(LiteralStringRef("\xff\xff/status/json")) },
-	{ SpecialKeySpace::MODULE::CONNECTIONSTRING, singleKeyRange(LiteralStringRef("\xff\xff/connection_string")) },
-	{ SpecialKeySpace::MODULE::CLUSTERFILEPATH, singleKeyRange(LiteralStringRef("\xff\xff/cluster_file_path")) },
-	{ SpecialKeySpace::MODULE::METRICS,
-	  KeyRangeRef(LiteralStringRef("\xff\xff/metrics/"), LiteralStringRef("\xff\xff/metrics0")) },
-	{ SpecialKeySpace::MODULE::MANAGEMENT,
-	  KeyRangeRef(LiteralStringRef("\xff\xff/management/"), LiteralStringRef("\xff\xff/management0")) },
-	{ SpecialKeySpace::MODULE::ERRORMSG, singleKeyRange(LiteralStringRef("\xff\xff/error_message")) },
-	{ SpecialKeySpace::MODULE::CONFIGURATION,
-	  KeyRangeRef(LiteralStringRef("\xff\xff/configuration/"), LiteralStringRef("\xff\xff/configuration0")) },
-	{ SpecialKeySpace::MODULE::GLOBALCONFIG,
-	  KeyRangeRef(LiteralStringRef("\xff\xff/global_config/"), LiteralStringRef("\xff\xff/global_config0")) },
-	{ SpecialKeySpace::MODULE::TRACING,
-	  KeyRangeRef(LiteralStringRef("\xff\xff/tracing/"), LiteralStringRef("\xff\xff/tracing0")) },
-	{ SpecialKeySpace::MODULE::ACTORLINEAGE,
-	  KeyRangeRef(LiteralStringRef("\xff\xff/actor_lineage/"), LiteralStringRef("\xff\xff/actor_lineage0")) },
+	  KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr) },
+	{ SpecialKeySpace::MODULE::STATUSJSON, singleKeyRange("\xff\xff/status/json"_sr) },
+	{ SpecialKeySpace::MODULE::CONNECTIONSTRING, singleKeyRange("\xff\xff/connection_string"_sr) },
+	{ SpecialKeySpace::MODULE::CLUSTERFILEPATH, singleKeyRange("\xff\xff/cluster_file_path"_sr) },
+	{ SpecialKeySpace::MODULE::METRICS, KeyRangeRef("\xff\xff/metrics/"_sr, "\xff\xff/metrics0"_sr) },
+	{ SpecialKeySpace::MODULE::MANAGEMENT, KeyRangeRef("\xff\xff/management/"_sr, "\xff\xff/management0"_sr) },
+	{ SpecialKeySpace::MODULE::ERRORMSG, singleKeyRange("\xff\xff/error_message"_sr) },
+	{ SpecialKeySpace::MODULE::CONFIGURATION, KeyRangeRef("\xff\xff/configuration/"_sr, "\xff\xff/configuration0"_sr) },
+	{ SpecialKeySpace::MODULE::GLOBALCONFIG, KeyRangeRef("\xff\xff/global_config/"_sr, "\xff\xff/global_config0"_sr) },
+	{ SpecialKeySpace::MODULE::TRACING, KeyRangeRef("\xff\xff/tracing/"_sr, "\xff\xff/tracing0"_sr) },
+	{ SpecialKeySpace::MODULE::ACTORLINEAGE, KeyRangeRef("\xff\xff/actor_lineage/"_sr, "\xff\xff/actor_lineage0"_sr) },
 	{ SpecialKeySpace::MODULE::ACTOR_PROFILER_CONF,
-	  KeyRangeRef(LiteralStringRef("\xff\xff/actor_profiler_conf/"),
-	              LiteralStringRef("\xff\xff/actor_profiler_conf0")) },
-	{ SpecialKeySpace::MODULE::CLUSTERID, singleKeyRange(LiteralStringRef("\xff\xff/cluster_id")) },
+	  KeyRangeRef("\xff\xff/actor_profiler_conf/"_sr, "\xff\xff/actor_profiler_conf0"_sr) },
+	{ SpecialKeySpace::MODULE::CLUSTERID, singleKeyRange("\xff\xff/cluster_id"_sr) },
 };
 
 std::unordered_map<std::string, KeyRange> SpecialKeySpace::managementApiCommandToRange = {
-	{ "exclude",
-	  KeyRangeRef(LiteralStringRef("excluded/"), LiteralStringRef("excluded0"))
-	      .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
-	{ "failed",
-	  KeyRangeRef(LiteralStringRef("failed/"), LiteralStringRef("failed0"))
-	      .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
+	{ "exclude", KeyRangeRef("excluded/"_sr, "excluded0"_sr).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
+	{ "failed", KeyRangeRef("failed/"_sr, "failed0"_sr).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
 	{ "excludedlocality",
-	  KeyRangeRef(LiteralStringRef("excluded_locality/"), LiteralStringRef("excluded_locality0"))
+	  KeyRangeRef("excluded_locality/"_sr, "excluded_locality0"_sr)
 	      .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
 	{ "failedlocality",
-	  KeyRangeRef(LiteralStringRef("failed_locality/"), LiteralStringRef("failed_locality0"))
+	  KeyRangeRef("failed_locality/"_sr, "failed_locality0"_sr)
 	      .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
-	{ "lock", singleKeyRange(LiteralStringRef("db_locked")).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
+	{ "lock", singleKeyRange("db_locked"_sr).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
 	{ "consistencycheck",
-	  singleKeyRange(LiteralStringRef("consistency_check_suspended"))
-	      .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
+	  singleKeyRange("consistency_check_suspended"_sr).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
 	{ "coordinators",
-	  KeyRangeRef(LiteralStringRef("coordinators/"), LiteralStringRef("coordinators0"))
-	      .withPrefix(moduleToBoundary[MODULE::CONFIGURATION].begin) },
+	  KeyRangeRef("coordinators/"_sr, "coordinators0"_sr).withPrefix(moduleToBoundary[MODULE::CONFIGURATION].begin) },
 	{ "advanceversion",
-	  singleKeyRange(LiteralStringRef("min_required_commit_version"))
-	      .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
-	{ "versionepoch",
-	  singleKeyRange(LiteralStringRef("version_epoch")).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
-	{ "profile",
-	  KeyRangeRef(LiteralStringRef("profiling/"), LiteralStringRef("profiling0"))
-	      .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
+	  singleKeyRange("min_required_commit_version"_sr).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
+	{ "versionepoch", singleKeyRange("version_epoch"_sr).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
+	{ "profile", KeyRangeRef("profiling/"_sr, "profiling0"_sr).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
 	{ "maintenance",
-	  KeyRangeRef(LiteralStringRef("maintenance/"), LiteralStringRef("maintenance0"))
-	      .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
+	  KeyRangeRef("maintenance/"_sr, "maintenance0"_sr).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
 	{ "datadistribution",
-	  KeyRangeRef(LiteralStringRef("data_distribution/"), LiteralStringRef("data_distribution0"))
+	  KeyRangeRef("data_distribution/"_sr, "data_distribution0"_sr)
 	      .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
 	{ "tenant", KeyRangeRef("tenant/"_sr, "tenant0"_sr).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
 	{ "tenantmap",
@@ -122,18 +105,15 @@ std::unordered_map<std::string, KeyRange> SpecialKeySpace::managementApiCommandT
 };
 
 std::unordered_map<std::string, KeyRange> SpecialKeySpace::actorLineageApiCommandToRange = {
-	{ "state",
-	  KeyRangeRef(LiteralStringRef("state/"), LiteralStringRef("state0"))
-	      .withPrefix(moduleToBoundary[MODULE::ACTORLINEAGE].begin) },
-	{ "time",
-	  KeyRangeRef(LiteralStringRef("time/"), LiteralStringRef("time0"))
-	      .withPrefix(moduleToBoundary[MODULE::ACTORLINEAGE].begin) }
+	{ "state", KeyRangeRef("state/"_sr, "state0"_sr).withPrefix(moduleToBoundary[MODULE::ACTORLINEAGE].begin) },
+	{ "time", KeyRangeRef("time/"_sr, "time0"_sr).withPrefix(moduleToBoundary[MODULE::ACTORLINEAGE].begin) }
 };
 
 std::set<std::string> SpecialKeySpace::options = { "excluded/force",
 	                                               "failed/force",
 	                                               "excluded_locality/force",
-	                                               "failed_locality/force" };
+	                                               "failed_locality/force",
+	                                               "worker_interfaces/verify" };
 
 std::set<std::string> SpecialKeySpace::tracingOptions = { kTracingTransactionIdKey, kTracingTokenKey };
 
@@ -148,13 +128,18 @@ RangeResult rywGetRange(ReadYourWritesTransaction* ryw, const KeyRangeRef& kr, c
 ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl* skrImpl,
                                                  ReadYourWritesTransaction* ryw,
                                                  KeySelector* ks,
-                                                 Optional<RangeResult>* cache) {
+                                                 KeyRangeMap<Optional<RangeResult>>* cache) {
 	// should be removed before calling
 	ASSERT(!ks->orEqual);
 
 	// never being called if KeySelector is already normalized
 	ASSERT(ks->offset != 1);
 
+	// Throw error if module doesn't support tenants and we have a tenant
+	if (ryw->getTenant().present() && !skrImpl->supportsTenants()) {
+		throw illegal_tenant_access();
+	}
+
 	state Key startKey(skrImpl->getKeyRange().begin);
 	state Key endKey(skrImpl->getKeyRange().end);
 	state RangeResult result;
@@ -234,7 +219,7 @@ ACTOR Future<Void> normalizeKeySelectorActor(SpecialKeySpace* sks,
                                              KeyRangeRef boundary,
                                              int* actualOffset,
                                              RangeResult* result,
-                                             Optional<RangeResult>* cache) {
+                                             KeyRangeMap<Optional<RangeResult>>* cache) {
 	// If offset < 1, where we need to move left, iter points to the range containing at least one smaller key
 	// (It's a wasting of time to walk through the range whose begin key is same as ks->key)
 	// (rangeContainingKeyBefore itself handles the case where ks->key == Key())
@@ -319,7 +304,9 @@ ACTOR Future<RangeResult> SpecialKeySpace::checkRYWValid(SpecialKeySpace* sks,
 		         wait(SpecialKeySpace::getRangeAggregationActor(sks, ryw, begin, end, limits, reverse))) {
 			return result;
 		}
-		when(wait(ryw->resetFuture())) { throw internal_error(); }
+		when(wait(ryw->resetFuture())) {
+			throw internal_error();
+		}
 	}
 }
 
@@ -337,8 +324,9 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 	state int actualBeginOffset;
 	state int actualEndOffset;
 	state KeyRangeRef moduleBoundary;
-	// used to cache result from potential first read
-	state Optional<RangeResult> cache;
+	// used to cache results from potential first async read
+	// the current implementation will read the whole range result to save in the cache
+	state KeyRangeMap<Optional<RangeResult>> cache(Optional<RangeResult>(), specialKeys.end);
 
 	if (ryw->specialKeySpaceRelaxed()) {
 		moduleBoundary = sks->range;
@@ -364,16 +352,31 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 	// Handle all corner cases like what RYW does
 	// return if range inverted
 	if (actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) {
-		TEST(true); // inverted range
+		CODE_PROBE(true, "inverted range");
 		return RangeResultRef(false, false);
 	}
 	// If touches begin or end, return with readToBegin and readThroughEnd flags
 	if (begin.getKey() == moduleBoundary.end || end.getKey() == moduleBoundary.begin) {
-		TEST(true); // query touches begin or end
+		CODE_PROBE(true, "query touches begin or end");
 		return result;
 	}
 	state RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::Ranges ranges =
 	    sks->getReadImpls().intersectingRanges(KeyRangeRef(begin.getKey(), end.getKey()));
+
+	// Check tenant legality separately from below iterations
+	// because it may be partially completed and returned
+	// before illegal range is checked due to the limits handler
+	if (ryw->getTenant().present()) {
+		for (auto iter : ranges) {
+			if (iter->value() == nullptr) {
+				continue;
+			}
+			if (!iter->value()->supportsTenants()) {
+				throw illegal_tenant_access();
+			}
+		}
+	}
+
 	// TODO : workaround to write this two together to make the code compact
 	// The issue here is boost::iterator_range<> doest not provide rbegin(), rend()
 	iter = reverse ? ranges.end() : ranges.begin();
@@ -385,7 +388,7 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 			KeyRangeRef kr = iter->range();
 			KeyRef keyStart = kr.contains(begin.getKey()) ? begin.getKey() : kr.begin;
 			KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end;
-			if (iter->value()->isAsync() && cache.present()) {
+			if (iter->value()->isAsync() && cache.rangeContaining(keyStart).value().present()) {
 				const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(iter->value());
 				RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache));
 				pairs = pairs_;
@@ -416,7 +419,7 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 			KeyRangeRef kr = iter->range();
 			KeyRef keyStart = kr.contains(begin.getKey()) ? begin.getKey() : kr.begin;
 			KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end;
-			if (iter->value()->isAsync() && cache.present()) {
+			if (iter->value()->isAsync() && cache.rangeContaining(keyStart).value().present()) {
 				const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(iter->value());
 				RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache));
 				pairs = pairs_;
@@ -453,7 +456,7 @@ Future<RangeResult> SpecialKeySpace::getRange(ReadYourWritesTransaction* ryw,
 	if (!limits.isValid())
 		return range_limits_invalid();
 	if (limits.isReached()) {
-		TEST(true); // read limit 0
+		CODE_PROBE(true, "Special Key Space range read limit 0");
 		return RangeResult();
 	}
 	// make sure orEqual == false
@@ -461,7 +464,7 @@ Future<RangeResult> SpecialKeySpace::getRange(ReadYourWritesTransaction* ryw,
 	end.removeOrEqual(end.arena());
 
 	if (begin.offset >= end.offset && begin.getKey() >= end.getKey()) {
-		TEST(true); // range inverted
+		CODE_PROBE(true, "range inverted");
 		return RangeResult();
 	}
 
@@ -499,6 +502,9 @@ void SpecialKeySpace::set(ReadYourWritesTransaction* ryw, const KeyRef& key, con
 		    .detail("Value", value.toString());
 		throw special_keys_no_write_module_found();
 	}
+	if (!impl->supportsTenants() && ryw->getTenant().present()) {
+		throw illegal_tenant_access();
+	}
 	return impl->set(ryw, key, value);
 }
 
@@ -516,6 +522,9 @@ void SpecialKeySpace::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& r
 		TraceEvent(SevDebug, "SpecialKeySpaceNoWriteModuleFound").detail("Range", range);
 		throw special_keys_no_write_module_found();
 	}
+	if (!begin->supportsTenants() && ryw->getTenant().present()) {
+		throw illegal_tenant_access();
+	}
 	return begin->clear(ryw, range);
 }
 
@@ -525,6 +534,9 @@ void SpecialKeySpace::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) {
 	auto impl = writeImpls[key];
 	if (impl == nullptr)
 		throw special_keys_no_write_module_found();
+	if (!impl->supportsTenants() && ryw->getTenant().present()) {
+		throw illegal_tenant_access();
+	}
 	return impl->clear(ryw, key);
 }
 
@@ -536,8 +548,8 @@ bool validateSnakeCaseNaming(const KeyRef& k) {
 	// Suffix can be \xff\xff or \x00 in single key range
 	if (key.endsWith(specialKeys.begin))
 		key = key.removeSuffix(specialKeys.end);
-	else if (key.endsWith(LiteralStringRef("\x00")))
-		key = key.removeSuffix(LiteralStringRef("\x00"));
+	else if (key.endsWith("\x00"_sr))
+		key = key.removeSuffix("\x00"_sr);
 	for (const char& c : key.toString()) {
 		// only small letters, numbers, '/', '_' is allowed
 		ASSERT((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '/' || c == '_');
@@ -612,6 +624,16 @@ ACTOR Future<Void> commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction*
 		++iter;
 	}
 	state std::vector<SpecialKeyRangeRWImpl*>::const_iterator it;
+	// Check validity of tenant support before iterating through
+	// module ptrs and potentially getting partial commits
+	if (ryw->getTenant().present()) {
+		for (it = writeModulePtrs.begin(); it != writeModulePtrs.end(); ++it) {
+			if (!(*it)->supportsTenants()) {
+				throw illegal_tenant_access();
+			}
+		}
+	}
+
 	for (it = writeModulePtrs.begin(); it != writeModulePtrs.end(); ++it) {
 		Optional<std::string> msg = wait((*it)->commit(ryw));
 		if (msg.present()) {
@@ -629,12 +651,8 @@ Future<Void> SpecialKeySpace::commit(ReadYourWritesTransaction* ryw) {
 	return commitActor(this, ryw);
 }
 
-SKSCTestImpl::SKSCTestImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
-
-Future<RangeResult> SKSCTestImpl::getRange(ReadYourWritesTransaction* ryw,
-                                           KeyRangeRef kr,
-                                           GetRangeLimits limitsHint) const {
-	ASSERT(range.contains(kr));
+// For SKSCTestRWImpl and SKSCTestAsyncReadImpl
+Future<RangeResult> SKSCTestGetRangeBase(ReadYourWritesTransaction* ryw, KeyRangeRef kr, GetRangeLimits limitsHint) {
 	auto resultFuture = ryw->getRange(kr, CLIENT_KNOBS->TOO_MANY);
 	// all keys are written to RYW, since GRV is set, the read should happen locally
 	ASSERT(resultFuture.isReady());
@@ -644,11 +662,29 @@ Future<RangeResult> SKSCTestImpl::getRange(ReadYourWritesTransaction* ryw,
 	return rywGetRange(ryw, kr, kvs);
 }
 
-Future<Optional<std::string>> SKSCTestImpl::commit(ReadYourWritesTransaction* ryw) {
+SKSCTestRWImpl::SKSCTestRWImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
+
+Future<RangeResult> SKSCTestRWImpl::getRange(ReadYourWritesTransaction* ryw,
+                                             KeyRangeRef kr,
+                                             GetRangeLimits limitsHint) const {
+	ASSERT(range.contains(kr));
+	return SKSCTestGetRangeBase(ryw, kr, limitsHint);
+}
+
+Future<Optional<std::string>> SKSCTestRWImpl::commit(ReadYourWritesTransaction* ryw) {
 	ASSERT(false);
 	return Optional<std::string>();
 }
 
+SKSCTestAsyncReadImpl::SKSCTestAsyncReadImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {}
+
+Future<RangeResult> SKSCTestAsyncReadImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                    KeyRangeRef kr,
+                                                    GetRangeLimits limitsHint) const {
+	ASSERT(range.contains(kr));
+	return SKSCTestGetRangeBase(ryw, kr, limitsHint);
+}
+
 ReadConflictRangeImpl::ReadConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
 
 ACTOR static Future<RangeResult> getReadConflictRangeImpl(ReadYourWritesTransaction* ryw, KeyRange kr) {
@@ -679,13 +715,14 @@ Future<RangeResult> ConflictingKeysImpl::getRange(ReadYourWritesTransaction* ryw
 	if (ryw->getTransactionState()->conflictingKeys) {
 		auto krMapPtr = ryw->getTransactionState()->conflictingKeys.get();
 		auto beginIter = krMapPtr->rangeContaining(kr.begin);
-		if (beginIter->begin() != kr.begin)
-			++beginIter;
 		auto endIter = krMapPtr->rangeContaining(kr.end);
+
+		if (!kr.contains(beginIter->begin()) && beginIter != endIter)
+			++beginIter;
 		for (auto it = beginIter; it != endIter; ++it) {
 			result.push_back_deep(result.arena(), KeyValueRef(it->begin(), it->value()));
 		}
-		if (endIter->begin() != kr.end)
+		if (kr.contains(endIter->begin()))
 			result.push_back_deep(result.arena(), KeyValueRef(endIter->begin(), endIter->value()));
 	}
 	return result;
@@ -695,8 +732,8 @@ ACTOR Future<RangeResult> ddMetricsGetRangeActor(ReadYourWritesTransaction* ryw,
 	loop {
 		try {
 			auto keys = kr.removePrefix(ddStatsRange.begin);
-			Standalone<VectorRef<DDMetricsRef>> resultWithoutPrefix = wait(
-			    waitDataDistributionMetricsList(ryw->getDatabase(), keys, CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT));
+			Standalone<VectorRef<DDMetricsRef>> resultWithoutPrefix =
+			    wait(waitDataDistributionMetricsList(ryw->getDatabase(), keys, CLIENT_KNOBS->TOO_MANY));
 			RangeResult result;
 			for (const auto& ddMetricsRef : resultWithoutPrefix) {
 				// each begin key is the previous end key, thus we only encode the begin key in the result
@@ -732,7 +769,7 @@ Future<RangeResult> DDStatsRangeImpl::getRange(ReadYourWritesTransaction* ryw,
 }
 
 Key SpecialKeySpace::getManagementApiCommandOptionSpecialKey(const std::string& command, const std::string& option) {
-	Key prefix = LiteralStringRef("options/").withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin);
+	Key prefix = "options/"_sr.withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin);
 	auto pair = command + "/" + option;
 	ASSERT(options.find(pair) != options.end());
 	return prefix.withSuffix(pair);
@@ -863,11 +900,11 @@ void ExcludeServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef&
 
 Key ExcludeServersRangeImpl::decode(const KeyRef& key) const {
 	return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
-	    .withPrefix(LiteralStringRef("\xff/conf/"));
+	    .withPrefix("\xff/conf/"_sr);
 }
 
 Key ExcludeServersRangeImpl::encode(const KeyRef& key) const {
-	return key.removePrefix(LiteralStringRef("\xff/conf/"))
+	return key.removePrefix("\xff/conf/"_sr)
 	    .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
 }
 
@@ -957,21 +994,13 @@ ACTOR Future<bool> checkExclusion(Database db,
 
 	state int ssTotalCount = 0;
 	state int ssExcludedCount = 0;
-	state double worstFreeSpaceRatio = 1.0;
+
+	state std::unordered_set<std::string> diskLocalities;
+	state int64_t totalKvStoreFreeBytes = 0;
+	state int64_t totalKvStoreUsedBytes = 0;
+	state int64_t totalKvStoreUsedBytesNonExcluded = 0;
 	try {
 		for (auto proc : processesMap.obj()) {
-			bool storageServer = false;
-			StatusArray rolesArray = proc.second.get_obj()["roles"].get_array();
-			for (StatusObjectReader role : rolesArray) {
-				if (role["role"].get_str() == "storage") {
-					storageServer = true;
-					break;
-				}
-			}
-			// Skip non-storage servers in free space calculation
-			if (!storageServer)
-				continue;
-
 			StatusObjectReader process(proc.second);
 			std::string addrStr;
 			if (!process.get("address", addrStr)) {
@@ -981,33 +1010,49 @@ ACTOR Future<bool> checkExclusion(Database db,
 			NetworkAddress addr = NetworkAddress::parse(addrStr);
 			bool excluded =
 			    (process.has("excluded") && process.last().get_bool()) || addressExcluded(*exclusions, addr);
-			ssTotalCount++;
-			if (excluded)
-				ssExcludedCount++;
 
-			if (!excluded) {
-				StatusObjectReader disk;
-				if (!process.get("disk", disk)) {
-					*msg =
-					    ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
-					return false;
+			StatusObjectReader localityObj;
+			std::string disk_id;
+			if (process.get("locality", localityObj)) {
+				process.get("disk_id", disk_id); // its ok if we don't have this field
+			}
+
+			StatusArray rolesArray = proc.second.get_obj()["roles"].get_array();
+			for (StatusObjectReader role : rolesArray) {
+				if (role["role"].get_str() == "storage") {
+					ssTotalCount++;
+
+					int64_t used_bytes;
+					if (!role.get("kvstore_used_bytes", used_bytes)) {
+						*msg = ManagementAPIError::toJsonString(
+						    false, markFailed ? "exclude failed" : "exclude", errorString);
+						return false;
+					}
+
+					int64_t free_bytes;
+					if (!role.get("kvstore_free_bytes", free_bytes)) {
+						*msg = ManagementAPIError::toJsonString(
+						    false, markFailed ? "exclude failed" : "exclude", errorString);
+						return false;
+					}
+
+					totalKvStoreUsedBytes += used_bytes;
+
+					if (!excluded) {
+						totalKvStoreUsedBytesNonExcluded += used_bytes;
+
+						if (disk_id.empty() || diskLocalities.find(disk_id) == diskLocalities.end()) {
+							totalKvStoreFreeBytes += free_bytes;
+							if (!disk_id.empty()) {
+								diskLocalities.insert(disk_id);
+							}
+						}
+					}
 				}
 
-				int64_t total_bytes;
-				if (!disk.get("total_bytes", total_bytes)) {
-					*msg =
-					    ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
-					return false;
+				if (excluded) {
+					ssExcludedCount++;
 				}
-
-				int64_t free_bytes;
-				if (!disk.get("free_bytes", free_bytes)) {
-					*msg =
-					    ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
-					return false;
-				}
-
-				worstFreeSpaceRatio = std::min(worstFreeSpaceRatio, double(free_bytes) / total_bytes);
 			}
 		}
 	} catch (...) // std::exception
@@ -1016,14 +1061,15 @@ ACTOR Future<bool> checkExclusion(Database db,
 		return false;
 	}
 
-	if (ssExcludedCount == ssTotalCount ||
-	    (1 - worstFreeSpaceRatio) * ssTotalCount / (ssTotalCount - ssExcludedCount) > 0.9) {
+	double finalFreeRatio = 1 - (totalKvStoreUsedBytes / (totalKvStoreUsedBytesNonExcluded + totalKvStoreFreeBytes));
+	if (ssExcludedCount == ssTotalCount || finalFreeRatio <= 0.1) {
 		std::string temp = "ERROR: This exclude may cause the total free space in the cluster to drop below 10%.\n"
 		                   "Call set(\"0xff0xff/management/options/exclude/force\", ...) first to exclude without "
 		                   "checking free space.\n";
 		*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp);
 		return false;
 	}
+
 	return true;
 }
 
@@ -1106,11 +1152,11 @@ void FailedServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& k
 
 Key FailedServersRangeImpl::decode(const KeyRef& key) const {
 	return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
-	    .withPrefix(LiteralStringRef("\xff/conf/"));
+	    .withPrefix("\xff/conf/"_sr);
 }
 
 Key FailedServersRangeImpl::encode(const KeyRef& key) const {
-	return key.removePrefix(LiteralStringRef("\xff/conf/"))
+	return key.removePrefix("\xff/conf/"_sr)
 	    .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
 }
 
@@ -1272,8 +1318,7 @@ Future<Optional<std::string>> ProcessClassRangeImpl::commit(ReadYourWritesTransa
 			// validate class type
 			ValueRef processClassType = entry.second.get();
 			ProcessClass processClass(processClassType.toString(), ProcessClass::DBSource);
-			if (processClass.classType() == ProcessClass::InvalidClass &&
-			    processClassType != LiteralStringRef("default")) {
+			if (processClass.classType() == ProcessClass::InvalidClass && processClassType != "default"_sr) {
 				std::string error = "ERROR: \'" + processClassType.toString() + "\' is not a valid process class\n";
 				errorMsg = ManagementAPIError::toJsonString(false, "setclass", error);
 				return errorMsg;
@@ -1373,11 +1418,10 @@ ACTOR Future<Optional<std::string>> lockDatabaseCommitActor(ReadYourWritesTransa
 		throw database_locked();
 	} else if (!val.present()) {
 		// lock database
-		ryw->getTransaction().atomicOp(databaseLockedKey,
-		                               BinaryWriter::toValue(uid, Unversioned())
-		                                   .withPrefix(LiteralStringRef("0123456789"))
-		                                   .withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
-		                               MutationRef::SetVersionstampedValue);
+		ryw->getTransaction().atomicOp(
+		    databaseLockedKey,
+		    BinaryWriter::toValue(uid, Unversioned()).withPrefix("0123456789"_sr).withSuffix("\x00\x00\x00\x00"_sr),
+		    MutationRef::SetVersionstampedValue);
 		ryw->getTransaction().addWriteConflictRange(normalKeys);
 	}
 
@@ -1587,7 +1631,8 @@ Future<RangeResult> TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw,
 
 void TracingOptionsImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) {
 	if (ryw->getApproximateSize() > 0) {
-		ryw->setSpecialKeySpaceErrorMsg("tracing options must be set first");
+		ryw->setSpecialKeySpaceErrorMsg(
+		    ManagementAPIError::toJsonString(false, "configure trace", "tracing options must be set first"));
 		ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>()));
 		return;
 	}
@@ -1600,7 +1645,8 @@ void TracingOptionsImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key,
 		} else if (value.toString() == "false") {
 			ryw->setToken(0);
 		} else {
-			ryw->setSpecialKeySpaceErrorMsg("token must be set to true/false");
+			ryw->setSpecialKeySpaceErrorMsg(
+			    ManagementAPIError::toJsonString(false, "configure trace token", "token must be set to true/false"));
 			throw special_keys_api_failure();
 		}
 	}
@@ -1614,12 +1660,12 @@ Future<Optional<std::string>> TracingOptionsImpl::commit(ReadYourWritesTransacti
 }
 
 void TracingOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) {
-	ryw->setSpecialKeySpaceErrorMsg("clear range disabled");
+	ryw->setSpecialKeySpaceErrorMsg(ManagementAPIError::toJsonString(false, "clear trace", "clear range disabled"));
 	throw special_keys_api_failure();
 }
 
 void TracingOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) {
-	ryw->setSpecialKeySpaceErrorMsg("clear disabled");
+	ryw->setSpecialKeySpaceErrorMsg(ManagementAPIError::toJsonString(false, "clear trace", "clear disabled"));
 	throw special_keys_api_failure();
 }
 
@@ -1629,7 +1675,7 @@ ACTOR Future<RangeResult> coordinatorsGetRangeActor(ReadYourWritesTransaction* r
 	state ClusterConnectionString cs = ryw->getDatabase()->getConnectionRecord()->getConnectionString();
 	state std::vector<NetworkAddress> coordinator_processes = wait(cs.tryResolveHostnames());
 	RangeResult result;
-	Key cluster_decription_key = prefix.withSuffix(LiteralStringRef("cluster_description"));
+	Key cluster_decription_key = prefix.withSuffix("cluster_description"_sr);
 	if (kr.contains(cluster_decription_key)) {
 		result.push_back_deep(result.arena(), KeyValueRef(cluster_decription_key, cs.clusterKeyName()));
 	}
@@ -1644,7 +1690,7 @@ ACTOR Future<RangeResult> coordinatorsGetRangeActor(ReadYourWritesTransaction* r
 			processes_str += ",";
 		processes_str += w.toString();
 	}
-	Key processes_key = prefix.withSuffix(LiteralStringRef("processes"));
+	Key processes_key = prefix.withSuffix("processes"_sr);
 	if (kr.contains(processes_key)) {
 		result.push_back_deep(result.arena(), KeyValueRef(processes_key, Value(processes_str)));
 	}
@@ -1666,7 +1712,7 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 	state bool parse_error = false;
 
 	// check update for coordinators
-	Key processes_key = LiteralStringRef("processes").withPrefix(kr.begin);
+	Key processes_key = "processes"_sr.withPrefix(kr.begin);
 	auto processes_entry = ryw->getSpecialKeySpaceWriteMap()[processes_key];
 	if (processes_entry.first) {
 		ASSERT(processes_entry.second.present()); // no clear should be seen here
@@ -1706,7 +1752,7 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 
 	std::string newName;
 	// check update for cluster_description
-	Key cluster_decription_key = LiteralStringRef("cluster_description").withPrefix(kr.begin);
+	Key cluster_decription_key = "cluster_description"_sr.withPrefix(kr.begin);
 	auto entry = ryw->getSpecialKeySpaceWriteMap()[cluster_decription_key];
 	if (entry.first) {
 		// check valid description [a-zA-Z0-9_]+
@@ -1720,11 +1766,15 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 		}
 	}
 
+	auto configDBEntry = ryw->getSpecialKeySpaceWriteMap()["config_db"_sr.withPrefix(kr.begin)];
+
 	TraceEvent(SevDebug, "SKSChangeCoordinatorsStart")
 	    .detail("NewConnectionString", conn.toString())
-	    .detail("Description", entry.first ? entry.second.get().toString() : "");
+	    .detail("Description", entry.first ? entry.second.get().toString() : "")
+	    .detail("ConfigDBDisabled", configDBEntry.first);
 
-	Optional<CoordinatorsResult> r = wait(changeQuorumChecker(&ryw->getTransaction(), &conn, newName));
+	Optional<CoordinatorsResult> r =
+	    wait(changeQuorumChecker(&ryw->getTransaction(), &conn, newName, configDBEntry.first));
 
 	TraceEvent(SevDebug, "SKSChangeCoordinatorsFinish")
 	    .detail("Result", r.present() ? static_cast<int>(r.get()) : -1); // -1 means success
@@ -1947,7 +1997,7 @@ Future<RangeResult> ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw
 	KeyRef prefix = getKeyRange().begin;
 	RangeResult result = RangeResult();
 	// client_txn_sample_rate
-	Key sampleRateKey = LiteralStringRef("client_txn_sample_rate").withPrefix(prefix);
+	Key sampleRateKey = "client_txn_sample_rate"_sr.withPrefix(prefix);
 
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
 
@@ -1968,7 +2018,7 @@ Future<RangeResult> ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw
 		}
 	}
 	// client_txn_size_limit
-	Key txnSizeLimitKey = LiteralStringRef("client_txn_size_limit").withPrefix(prefix);
+	Key txnSizeLimitKey = "client_txn_size_limit"_sr.withPrefix(prefix);
 	if (kr.contains(txnSizeLimitKey)) {
 		auto entry = ryw->getSpecialKeySpaceWriteMap()[txnSizeLimitKey];
 		if (!ryw->readYourWritesDisabled() && entry.first) {
@@ -1994,7 +2044,7 @@ Future<Optional<std::string>> ClientProfilingImpl::commit(ReadYourWritesTransact
 	Standalone<VectorRef<KeyRangeRef>> clears;
 
 	// client_txn_sample_rate
-	Key sampleRateKey = LiteralStringRef("client_txn_sample_rate").withPrefix(getKeyRange().begin);
+	Key sampleRateKey = "client_txn_sample_rate"_sr.withPrefix(getKeyRange().begin);
 	auto rateEntry = ryw->getSpecialKeySpaceWriteMap()[sampleRateKey];
 
 	if (rateEntry.first && rateEntry.second.present()) {
@@ -2005,7 +2055,7 @@ Future<Optional<std::string>> ClientProfilingImpl::commit(ReadYourWritesTransact
 		} else {
 			try {
 				double sampleRate = boost::lexical_cast<double>(sampleRateStr);
-				Tuple rate = Tuple().appendDouble(sampleRate);
+				Tuple rate = Tuple::makeTuple(sampleRate);
 				insertions.push_back_deep(insertions.arena(), KeyValueRef(fdbClientInfoTxnSampleRate, rate.pack()));
 			} catch (boost::bad_lexical_cast& e) {
 				return Optional<std::string>(ManagementAPIError::toJsonString(
@@ -2014,7 +2064,7 @@ Future<Optional<std::string>> ClientProfilingImpl::commit(ReadYourWritesTransact
 		}
 	}
 	// client_txn_size_limit
-	Key txnSizeLimitKey = LiteralStringRef("client_txn_size_limit").withPrefix(getKeyRange().begin);
+	Key txnSizeLimitKey = "client_txn_size_limit"_sr.withPrefix(getKeyRange().begin);
 	auto sizeLimitEntry = ryw->getSpecialKeySpaceWriteMap()[txnSizeLimitKey];
 	if (sizeLimitEntry.first && sizeLimitEntry.second.present()) {
 		std::string sizeLimitStr = sizeLimitEntry.second.get().toString();
@@ -2024,7 +2074,7 @@ Future<Optional<std::string>> ClientProfilingImpl::commit(ReadYourWritesTransact
 		} else {
 			try {
 				int64_t sizeLimit = boost::lexical_cast<int64_t>(sizeLimitStr);
-				Tuple size = Tuple().append(sizeLimit);
+				Tuple size = Tuple::makeTuple(sizeLimit);
 				insertions.push_back_deep(insertions.arena(), KeyValueRef(fdbClientInfoTxnSizeLimit, size.pack()));
 			} catch (boost::bad_lexical_cast& e) {
 				return Optional<std::string>(ManagementAPIError::toJsonString(
@@ -2059,11 +2109,11 @@ void parse(StringRef& val, double& d) {
 }
 
 void parse(StringRef& val, WaitState& w) {
-	if (val == LiteralStringRef("disk") || val == LiteralStringRef("Disk")) {
+	if (val == "disk"_sr || val == "Disk"_sr) {
 		w = WaitState::Disk;
-	} else if (val == LiteralStringRef("network") || val == LiteralStringRef("Network")) {
+	} else if (val == "network"_sr || val == "Network"_sr) {
 		w = WaitState::Network;
-	} else if (val == LiteralStringRef("running") || val == LiteralStringRef("Running")) {
+	} else if (val == "running"_sr || val == "Running"_sr) {
 		w = WaitState::Running;
 	} else {
 		throw std::range_error("failed to parse run state");
@@ -2164,7 +2214,8 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 	state std::vector<StringRef> endValues = kr.end.removePrefix(prefix).splitAny("/"_sr);
 	// Require index (either "state" or "time") and address:port.
 	if (beginValues.size() < 2 || endValues.size() < 2) {
-		ryw->setSpecialKeySpaceErrorMsg("missing required parameters (index, host)");
+		ryw->setSpecialKeySpaceErrorMsg(
+		    ManagementAPIError::toJsonString(false, "read actor_lineage", "missing required parameters (index, host)"));
 		throw special_keys_api_failure();
 	}
 
@@ -2183,12 +2234,14 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 				parse(endValues.begin() + 1, endValues.end(), endRangeHost, timeEnd, waitStateEnd, seqEnd);
 			}
 		} else {
-			ryw->setSpecialKeySpaceErrorMsg("invalid index in actor_lineage");
+			ryw->setSpecialKeySpaceErrorMsg(
+			    ManagementAPIError::toJsonString(false, "read actor_lineage", "invalid index in actor_lineage"));
 			throw special_keys_api_failure();
 		}
 	} catch (Error& e) {
 		if (e.code() != special_keys_api_failure().code()) {
-			ryw->setSpecialKeySpaceErrorMsg("failed to parse key");
+			ryw->setSpecialKeySpaceErrorMsg(
+			    ManagementAPIError::toJsonString(false, "read actor_lineage", "failed to parse key"));
 			throw special_keys_api_failure();
 		} else {
 			throw e;
@@ -2198,7 +2251,8 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 	if (kr.begin != kr.end && host != endRangeHost) {
 		// The client doesn't know about all the hosts, so a get range covering
 		// multiple hosts has no way of knowing which IP:port combos to use.
-		ryw->setSpecialKeySpaceErrorMsg("the host must remain the same on both ends of the range");
+		ryw->setSpecialKeySpaceErrorMsg(ManagementAPIError::toJsonString(
+		    false, "read actor_lineage", "the host must remain the same on both ends of the range"));
 		throw special_keys_api_failure();
 	}
 
@@ -2459,7 +2513,7 @@ ACTOR static Future<RangeResult> DataDistributionGetRangeActor(ReadYourWritesTra
                                                                KeyRangeRef kr) {
 	state RangeResult result;
 	// dataDistributionModeKey
-	state Key modeKey = LiteralStringRef("mode").withPrefix(prefix);
+	state Key modeKey = "mode"_sr.withPrefix(prefix);
 
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
 
@@ -2475,7 +2529,7 @@ ACTOR static Future<RangeResult> DataDistributionGetRangeActor(ReadYourWritesTra
 		}
 	}
 	// rebalanceDDIgnoreKey
-	state Key rebalanceIgnoredKey = LiteralStringRef("rebalance_ignored").withPrefix(prefix);
+	state Key rebalanceIgnoredKey = "rebalance_ignored"_sr.withPrefix(prefix);
 	if (kr.contains(rebalanceIgnoredKey)) {
 		auto entry = ryw->getSpecialKeySpaceWriteMap()[rebalanceIgnoredKey];
 		if (ryw->readYourWritesDisabled() || !entry.first) {
@@ -2502,8 +2556,8 @@ Future<Optional<std::string>> DataDistributionImpl::commit(ReadYourWritesTransac
 
 	Optional<std::string> msg;
 	KeyRangeRef kr = getKeyRange();
-	Key modeKey = LiteralStringRef("mode").withPrefix(kr.begin);
-	Key rebalanceIgnoredKey = LiteralStringRef("rebalance_ignored").withPrefix(kr.begin);
+	Key modeKey = "mode"_sr.withPrefix(kr.begin);
+	Key rebalanceIgnoredKey = "rebalance_ignored"_sr.withPrefix(kr.begin);
 	auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(kr);
 	for (auto iter = ranges.begin(); iter != ranges.end(); ++iter) {
 		if (!iter->value().first)
@@ -2690,11 +2744,11 @@ void ExcludedLocalitiesRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyR
 
 Key ExcludedLocalitiesRangeImpl::decode(const KeyRef& key) const {
 	return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
-	    .withPrefix(LiteralStringRef("\xff/conf/"));
+	    .withPrefix("\xff/conf/"_sr);
 }
 
 Key ExcludedLocalitiesRangeImpl::encode(const KeyRef& key) const {
-	return key.removePrefix(LiteralStringRef("\xff/conf/"))
+	return key.removePrefix("\xff/conf/"_sr)
 	    .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
 }
 
@@ -2719,11 +2773,11 @@ void FailedLocalitiesRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef
 
 Key FailedLocalitiesRangeImpl::decode(const KeyRef& key) const {
 	return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
-	    .withPrefix(LiteralStringRef("\xff/conf/"));
+	    .withPrefix("\xff/conf/"_sr);
 }
 
 Key FailedLocalitiesRangeImpl::encode(const KeyRef& key) const {
-	return key.removePrefix(LiteralStringRef("\xff/conf/"))
+	return key.removePrefix("\xff/conf/"_sr)
 	    .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
 }
 
@@ -2731,3 +2785,143 @@ Future<Optional<std::string>> FailedLocalitiesRangeImpl::commit(ReadYourWritesTr
 	// exclude locality with failed option as true.
 	return excludeLocalityCommitActor(ryw, true);
 }
+
+// Defined in ReadYourWrites.actor.cpp
+ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> clusterRecord);
+// Defined in NativeAPI.actor.cpp
+ACTOR Future<bool> verifyInterfaceActor(Reference<FlowLock> connectLock, ClientWorkerInterface workerInterf);
+
+ACTOR static Future<RangeResult> workerInterfacesImplGetRangeActor(ReadYourWritesTransaction* ryw,
+                                                                   KeyRef prefix,
+                                                                   KeyRangeRef kr) {
+	if (!ryw->getDatabase().getPtr() || !ryw->getDatabase()->getConnectionRecord())
+		return RangeResult();
+
+	state RangeResult interfs = wait(getWorkerInterfaces(ryw->getDatabase()->getConnectionRecord()));
+	// for options' special keys, the boolean flag indicates if it's a SET operation
+	auto [verify, _] = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandOptionSpecialKey(
+	    "worker_interfaces", "verify")];
+	state RangeResult result;
+	if (verify) {
+		// if verify option is set, we try to talk to every worker and only returns those we can talk to
+		Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
+		state std::vector<Future<bool>> verifyInterfs;
+		for (const auto& [k_, value] : interfs) {
+			auto k = k_.withPrefix(prefix);
+			if (kr.contains(k)) {
+				ClientWorkerInterface workerInterf =
+				    BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion());
+				verifyInterfs.push_back(verifyInterfaceActor(connectLock, workerInterf));
+			} else {
+				verifyInterfs.push_back(false);
+			}
+		}
+		wait(waitForAll(verifyInterfs));
+		// state int index;
+		for (int index = 0; index < interfs.size(); index++) {
+			if (verifyInterfs[index].get()) {
+				// if we can establish a connection, add the kv pair into the result
+				result.push_back_deep(result.arena(),
+				                      KeyValueRef(interfs[index].key.withPrefix(prefix), interfs[index].value));
+			}
+		}
+	} else {
+		for (const auto& [k_, v] : interfs) {
+			auto k = k_.withPrefix(prefix);
+			if (kr.contains(k))
+				result.push_back_deep(result.arena(), KeyValueRef(k, v));
+		}
+	}
+	std::sort(result.begin(), result.end(), KeyValueRef::OrderByKey{});
+	return result;
+}
+
+WorkerInterfacesSpecialKeyImpl::WorkerInterfacesSpecialKeyImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
+
+Future<RangeResult> WorkerInterfacesSpecialKeyImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                             KeyRangeRef kr,
+                                                             GetRangeLimits limitsHint) const {
+	return workerInterfacesImplGetRangeActor(ryw, getKeyRange().begin, kr);
+}
+
+ACTOR Future<Void> validateSpecialSubrangeRead(ReadYourWritesTransaction* ryw,
+                                               KeySelector begin,
+                                               KeySelector end,
+                                               GetRangeLimits limits,
+                                               Reverse reverse,
+                                               RangeResult result) {
+	if (!result.size()) {
+		RangeResult testResult = wait(ryw->getRange(begin, end, limits, Snapshot::True, reverse));
+		ASSERT(testResult == result);
+		return Void();
+	}
+
+	if (reverse) {
+		ASSERT(std::is_sorted(result.begin(), result.end(), KeyValueRef::OrderByKeyBack{}));
+	} else {
+		ASSERT(std::is_sorted(result.begin(), result.end(), KeyValueRef::OrderByKey{}));
+	}
+
+	// Generate a keyrange where we can determine the expected result based solely on the previous readrange, and whose
+	// boundaries may or may not be keys in result.
+	std::vector<Key> candidateKeys;
+	if (reverse) {
+		for (int i = result.size() - 1; i >= 0; --i) {
+			candidateKeys.emplace_back(result[i].key);
+			if (i - 1 >= 0) {
+				candidateKeys.emplace_back(keyBetween(KeyRangeRef(result[i].key, result[i - 1].key)));
+			}
+		}
+	} else {
+		for (int i = 0; i < result.size(); ++i) {
+			candidateKeys.emplace_back(result[i].key);
+			if (i + 1 < result.size()) {
+				candidateKeys.emplace_back(keyBetween(KeyRangeRef(result[i].key, result[i + 1].key)));
+			}
+		}
+	}
+	std::sort(candidateKeys.begin(), candidateKeys.end());
+	int originalSize = candidateKeys.size();
+	// Add more candidate keys so that we might read a range between two adjacent result keys.
+	for (int i = 0; i < originalSize - 1; ++i) {
+		candidateKeys.emplace_back(keyBetween(KeyRangeRef(candidateKeys[i], candidateKeys[i + 1])));
+	}
+	std::vector<Key> keys;
+	keys = { deterministicRandom()->randomChoice(candidateKeys), deterministicRandom()->randomChoice(candidateKeys) };
+	std::sort(keys.begin(), keys.end());
+	state KeySelector testBegin = firstGreaterOrEqual(keys[0]);
+	state KeySelector testEnd = firstGreaterOrEqual(keys[1]);
+
+	// Generate expected result. Linear time is ok here since we're in simulation, and there's a benefit to keeping this
+	// simple (as we're using it as an test oracle)
+	state RangeResult expectedResult;
+	// The reverse parameter should be the same as for the original read, so if
+	// reverse is true then the results are _already_ in reverse order.
+	for (const auto& kr : result) {
+		if (kr.key >= keys[0] && kr.key < keys[1]) {
+			expectedResult.push_back(expectedResult.arena(), kr);
+		}
+	}
+
+	// Test
+	RangeResult testResult = wait(ryw->getRange(testBegin, testEnd, limits, Snapshot::True, reverse));
+	if (testResult != expectedResult) {
+		fmt::print("Reverse: {}\n", reverse);
+		fmt::print("Original range: [{}, {})\n", begin.toString(), end.toString());
+		fmt::print("Original result:\n");
+		for (const auto& kr : result) {
+			fmt::print("	{} -> {}\n", kr.key.printable(), kr.value.printable());
+		}
+		fmt::print("Test range: [{}, {})\n", testBegin.getKey().printable(), testEnd.getKey().printable());
+		fmt::print("Expected:\n");
+		for (const auto& kr : expectedResult) {
+			fmt::print("	{} -> {}\n", kr.key.printable(), kr.value.printable());
+		}
+		fmt::print("Got:\n");
+		for (const auto& kr : testResult) {
+			fmt::print("	{} -> {}\n", kr.key.printable(), kr.value.printable());
+		}
+		ASSERT(testResult == expectedResult);
+	}
+	return Void();
+}
diff --git a/fdbclient/StorageServerInterface.cpp b/fdbclient/StorageServerInterface.cpp
index 8a9b6183a4..535ef8f98f 100644
--- a/fdbclient/StorageServerInterface.cpp
+++ b/fdbclient/StorageServerInterface.cpp
@@ -129,7 +129,7 @@ const char* TSS_mismatchTraceName(const GetKeyValuesRequest& req) {
 static void traceKeyValuesSummary(TraceEvent& event,
                                   const KeySelectorRef& begin,
                                   const KeySelectorRef& end,
-                                  Optional<TenantName> tenant,
+                                  Optional<TenantNameRef> tenant,
                                   Version version,
                                   int limit,
                                   int limitBytes,
@@ -152,7 +152,7 @@ static void traceKeyValuesSummary(TraceEvent& event,
 static void traceKeyValuesDiff(TraceEvent& event,
                                const KeySelectorRef& begin,
                                const KeySelectorRef& end,
-                               Optional<TenantName> tenant,
+                               Optional<TenantNameRef> tenant,
                                Version version,
                                int limit,
                                int limitBytes,
diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp
index c140104335..a720547b56 100644
--- a/fdbclient/SystemData.cpp
+++ b/fdbclient/SystemData.cpp
@@ -31,20 +31,20 @@
 FDB_DEFINE_BOOLEAN_PARAM(AssignEmptyRange);
 FDB_DEFINE_BOOLEAN_PARAM(UnassignShard);
 
-const KeyRef systemKeysPrefix = LiteralStringRef("\xff");
+const KeyRef systemKeysPrefix = "\xff"_sr;
 const KeyRangeRef normalKeys(KeyRef(), systemKeysPrefix);
-const KeyRangeRef systemKeys(systemKeysPrefix, LiteralStringRef("\xff\xff"));
-const KeyRangeRef nonMetadataSystemKeys(LiteralStringRef("\xff\x02"), LiteralStringRef("\xff\x03"));
+const KeyRangeRef systemKeys(systemKeysPrefix, "\xff\xff"_sr);
+const KeyRangeRef nonMetadataSystemKeys("\xff\x02"_sr, "\xff\x03"_sr);
 const KeyRangeRef allKeys = KeyRangeRef(normalKeys.begin, systemKeys.end);
-const KeyRef afterAllKeys = LiteralStringRef("\xff\xff\x00");
-const KeyRangeRef specialKeys = KeyRangeRef(LiteralStringRef("\xff\xff"), LiteralStringRef("\xff\xff\xff"));
+const KeyRef afterAllKeys = "\xff\xff\x00"_sr;
+const KeyRangeRef specialKeys = KeyRangeRef("\xff\xff"_sr, "\xff\xff\xff"_sr);
 
 // keyServersKeys.contains(k) iff k.startsWith(keyServersPrefix)
-const KeyRangeRef keyServersKeys(LiteralStringRef("\xff/keyServers/"), LiteralStringRef("\xff/keyServers0"));
+const KeyRangeRef keyServersKeys("\xff/keyServers/"_sr, "\xff/keyServers0"_sr);
 const KeyRef keyServersPrefix = keyServersKeys.begin;
 const KeyRef keyServersEnd = keyServersKeys.end;
-const KeyRangeRef keyServersKeyServersKeys(LiteralStringRef("\xff/keyServers/\xff/keyServers/"),
-                                           LiteralStringRef("\xff/keyServers/\xff/keyServers0"));
+const KeyRangeRef keyServersKeyServersKeys("\xff/keyServers/\xff/keyServers/"_sr,
+                                           "\xff/keyServers/\xff/keyServers0"_sr);
 const KeyRef keyServersKeyServersKey = keyServersKeyServersKeys.begin;
 
 // These constants are selected to be easily recognized during debugging.
@@ -274,20 +274,17 @@ void decodeKeyServersValue(std::map<Tag, UID> const& tag_uid,
 }
 
 const KeyRangeRef conflictingKeysRange =
-    KeyRangeRef(LiteralStringRef("\xff\xff/transaction/conflicting_keys/"),
-                LiteralStringRef("\xff\xff/transaction/conflicting_keys/\xff\xff"));
-const ValueRef conflictingKeysTrue = LiteralStringRef("1");
-const ValueRef conflictingKeysFalse = LiteralStringRef("0");
+    KeyRangeRef("\xff\xff/transaction/conflicting_keys/"_sr, "\xff\xff/transaction/conflicting_keys/\xff\xff"_sr);
+const ValueRef conflictingKeysTrue = "1"_sr;
+const ValueRef conflictingKeysFalse = "0"_sr;
 
 const KeyRangeRef readConflictRangeKeysRange =
-    KeyRangeRef(LiteralStringRef("\xff\xff/transaction/read_conflict_range/"),
-                LiteralStringRef("\xff\xff/transaction/read_conflict_range/\xff\xff"));
+    KeyRangeRef("\xff\xff/transaction/read_conflict_range/"_sr, "\xff\xff/transaction/read_conflict_range/\xff\xff"_sr);
 
-const KeyRangeRef writeConflictRangeKeysRange =
-    KeyRangeRef(LiteralStringRef("\xff\xff/transaction/write_conflict_range/"),
-                LiteralStringRef("\xff\xff/transaction/write_conflict_range/\xff\xff"));
+const KeyRangeRef writeConflictRangeKeysRange = KeyRangeRef("\xff\xff/transaction/write_conflict_range/"_sr,
+                                                            "\xff\xff/transaction/write_conflict_range/\xff\xff"_sr);
 
-const KeyRef clusterIdKey = LiteralStringRef("\xff/clusterId");
+const KeyRef clusterIdKey = "\xff/clusterId"_sr;
 
 const KeyRef checkpointPrefix = "\xff/checkpoint/"_sr;
 
@@ -344,7 +341,7 @@ DataMoveMetaData decodeDataMoveValue(const ValueRef& value) {
 }
 
 // "\xff/cacheServer/[[UID]] := StorageServerInterface"
-const KeyRangeRef storageCacheServerKeys(LiteralStringRef("\xff/cacheServer/"), LiteralStringRef("\xff/cacheServer0"));
+const KeyRangeRef storageCacheServerKeys("\xff/cacheServer/"_sr, "\xff/cacheServer0"_sr);
 const KeyRef storageCacheServersPrefix = storageCacheServerKeys.begin;
 const KeyRef storageCacheServersEnd = storageCacheServerKeys.end;
 
@@ -356,16 +353,16 @@ const Key storageCacheServerKey(UID id) {
 }
 
 const Value storageCacheServerValue(const StorageServerInterface& ssi) {
-	auto protocolVersion = currentProtocolVersion;
+	auto protocolVersion = currentProtocolVersion();
 	protocolVersion.addObjectSerializerFlag();
 	return ObjectWriter::toValue(ssi, IncludeVersion(protocolVersion));
 }
 
-const KeyRangeRef ddStatsRange = KeyRangeRef(LiteralStringRef("\xff\xff/metrics/data_distribution_stats/"),
-                                             LiteralStringRef("\xff\xff/metrics/data_distribution_stats/\xff\xff"));
+const KeyRangeRef ddStatsRange =
+    KeyRangeRef("\xff\xff/metrics/data_distribution_stats/"_sr, "\xff\xff/metrics/data_distribution_stats/\xff\xff"_sr);
 
 //    "\xff/storageCache/[[begin]]" := "[[vector<uint16_t>]]"
-const KeyRangeRef storageCacheKeys(LiteralStringRef("\xff/storageCache/"), LiteralStringRef("\xff/storageCache0"));
+const KeyRangeRef storageCacheKeys("\xff/storageCache/"_sr, "\xff/storageCache0"_sr);
 const KeyRef storageCachePrefix = storageCacheKeys.begin;
 
 const Key storageCacheKey(const KeyRef& k) {
@@ -427,7 +424,7 @@ const Key serverKeysKey(UID serverID, const KeyRef& key) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverKeysPrefix);
 	wr << serverID;
-	wr.serializeBytes(LiteralStringRef("/"));
+	wr.serializeBytes("/"_sr);
 	wr.serializeBytes(key);
 	return wr.toValue();
 }
@@ -435,7 +432,7 @@ const Key serverKeysPrefixFor(UID serverID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverKeysPrefix);
 	wr << serverID;
-	wr.serializeBytes(LiteralStringRef("/"));
+	wr.serializeBytes("/"_sr);
 	return wr.toValue();
 }
 UID serverKeysDecodeServer(const KeyRef& key) {
@@ -475,18 +472,21 @@ const Value serverKeysValue(const UID& id) {
 
 void decodeServerKeysValue(const ValueRef& value, bool& assigned, bool& emptyRange, UID& id) {
 	if (value.size() == 0) {
-		id = UID();
 		assigned = false;
 		emptyRange = false;
+		id = UID();
 	} else if (value == serverKeysTrue) {
 		assigned = true;
 		emptyRange = false;
+		id = anonymousShardId;
 	} else if (value == serverKeysTrueEmptyRange) {
 		assigned = true;
 		emptyRange = true;
+		id = anonymousShardId;
 	} else if (value == serverKeysFalse) {
 		assigned = false;
 		emptyRange = false;
+		id = UID();
 	} else {
 		BinaryReader rd(value, IncludeVersion());
 		ASSERT(rd.protocolVersion().hasShardEncodeLocationMetaData());
@@ -496,13 +496,13 @@ void decodeServerKeysValue(const ValueRef& value, bool& assigned, bool& emptyRan
 	}
 }
 
-const KeyRef cacheKeysPrefix = LiteralStringRef("\xff\x02/cacheKeys/");
+const KeyRef cacheKeysPrefix = "\xff\x02/cacheKeys/"_sr;
 
 const Key cacheKeysKey(uint16_t idx, const KeyRef& key) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(cacheKeysPrefix);
 	wr << idx;
-	wr.serializeBytes(LiteralStringRef("/"));
+	wr.serializeBytes("/"_sr);
 	wr.serializeBytes(key);
 	return wr.toValue();
 }
@@ -510,7 +510,7 @@ const Key cacheKeysPrefixFor(uint16_t idx) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(cacheKeysPrefix);
 	wr << idx;
-	wr.serializeBytes(LiteralStringRef("/"));
+	wr.serializeBytes("/"_sr);
 	return wr.toValue();
 }
 uint16_t cacheKeysDecodeIndex(const KeyRef& key) {
@@ -523,9 +523,8 @@ KeyRef cacheKeysDecodeKey(const KeyRef& key) {
 	return key.substr(cacheKeysPrefix.size() + sizeof(uint16_t) + 1);
 }
 
-const KeyRef cacheChangeKey = LiteralStringRef("\xff\x02/cacheChangeKey");
-const KeyRangeRef cacheChangeKeys(LiteralStringRef("\xff\x02/cacheChangeKeys/"),
-                                  LiteralStringRef("\xff\x02/cacheChangeKeys0"));
+const KeyRef cacheChangeKey = "\xff\x02/cacheChangeKey"_sr;
+const KeyRangeRef cacheChangeKeys("\xff\x02/cacheChangeKeys/"_sr, "\xff\x02/cacheChangeKeys0"_sr);
 const KeyRef cacheChangePrefix = cacheChangeKeys.begin;
 const Key cacheChangeKeyFor(uint16_t idx) {
 	BinaryWriter wr(Unversioned());
@@ -540,9 +539,9 @@ uint16_t cacheChangeKeyDecodeIndex(const KeyRef& key) {
 	return idx;
 }
 
-const KeyRangeRef tssMappingKeys(LiteralStringRef("\xff/tss/"), LiteralStringRef("\xff/tss0"));
+const KeyRangeRef tssMappingKeys("\xff/tss/"_sr, "\xff/tss0"_sr);
 
-const KeyRangeRef tssQuarantineKeys(LiteralStringRef("\xff/tssQ/"), LiteralStringRef("\xff/tssQ0"));
+const KeyRangeRef tssQuarantineKeys("\xff/tssQ/"_sr, "\xff/tssQ0"_sr);
 
 const Key tssQuarantineKeyFor(UID serverID) {
 	BinaryWriter wr(Unversioned());
@@ -558,22 +557,19 @@ UID decodeTssQuarantineKey(KeyRef const& key) {
 	return serverID;
 }
 
-const KeyRangeRef tssMismatchKeys(LiteralStringRef("\xff/tssMismatch/"), LiteralStringRef("\xff/tssMismatch0"));
+const KeyRangeRef tssMismatchKeys("\xff/tssMismatch/"_sr, "\xff/tssMismatch0"_sr);
 
-const KeyRangeRef serverMetadataKeys(LiteralStringRef("\xff/serverMetadata/"),
-                                     LiteralStringRef("\xff/serverMetadata0"));
+const KeyRangeRef serverMetadataKeys("\xff/serverMetadata/"_sr, "\xff/serverMetadata0"_sr);
 
-const KeyRangeRef serverTagKeys(LiteralStringRef("\xff/serverTag/"), LiteralStringRef("\xff/serverTag0"));
+const KeyRangeRef serverTagKeys("\xff/serverTag/"_sr, "\xff/serverTag0"_sr);
 
 const KeyRef serverTagPrefix = serverTagKeys.begin;
-const KeyRangeRef serverTagConflictKeys(LiteralStringRef("\xff/serverTagConflict/"),
-                                        LiteralStringRef("\xff/serverTagConflict0"));
+const KeyRangeRef serverTagConflictKeys("\xff/serverTagConflict/"_sr, "\xff/serverTagConflict0"_sr);
 const KeyRef serverTagConflictPrefix = serverTagConflictKeys.begin;
 // serverTagHistoryKeys is the old tag a storage server uses before it is migrated to a different location.
 // For example, we can copy a SS file to a remote DC and start the SS there;
 //   The new SS will need to consume the last bits of data from the old tag it is responsible for.
-const KeyRangeRef serverTagHistoryKeys(LiteralStringRef("\xff/serverTagHistory/"),
-                                       LiteralStringRef("\xff/serverTagHistory0"));
+const KeyRangeRef serverTagHistoryKeys("\xff/serverTagHistory/"_sr, "\xff/serverTagHistory0"_sr);
 const KeyRef serverTagHistoryPrefix = serverTagHistoryKeys.begin;
 
 const Key serverTagKeyFor(UID serverID) {
@@ -658,12 +654,11 @@ const Key serverTagConflictKeyFor(Tag tag) {
 	return wr.toValue();
 }
 
-const KeyRangeRef tagLocalityListKeys(LiteralStringRef("\xff/tagLocalityList/"),
-                                      LiteralStringRef("\xff/tagLocalityList0"));
+const KeyRangeRef tagLocalityListKeys("\xff/tagLocalityList/"_sr, "\xff/tagLocalityList0"_sr);
 const KeyRef tagLocalityListPrefix = tagLocalityListKeys.begin;
 
 const Key tagLocalityListKeyFor(Optional<Value> dcID) {
-	BinaryWriter wr(AssumeVersion(currentProtocolVersion));
+	BinaryWriter wr(AssumeVersion(currentProtocolVersion()));
 	wr.serializeBytes(tagLocalityListKeys.begin);
 	wr << dcID;
 	return wr.toValue();
@@ -676,7 +671,7 @@ const Value tagLocalityListValue(int8_t const& tagLocality) {
 }
 Optional<Value> decodeTagLocalityListKey(KeyRef const& key) {
 	Optional<Value> dcID;
-	BinaryReader rd(key.removePrefix(tagLocalityListKeys.begin), AssumeVersion(currentProtocolVersion));
+	BinaryReader rd(key.removePrefix(tagLocalityListKeys.begin), AssumeVersion(currentProtocolVersion()));
 	rd >> dcID;
 	return dcID;
 }
@@ -687,12 +682,11 @@ int8_t decodeTagLocalityListValue(ValueRef const& value) {
 	return s;
 }
 
-const KeyRangeRef datacenterReplicasKeys(LiteralStringRef("\xff\x02/datacenterReplicas/"),
-                                         LiteralStringRef("\xff\x02/datacenterReplicas0"));
+const KeyRangeRef datacenterReplicasKeys("\xff\x02/datacenterReplicas/"_sr, "\xff\x02/datacenterReplicas0"_sr);
 const KeyRef datacenterReplicasPrefix = datacenterReplicasKeys.begin;
 
 const Key datacenterReplicasKeyFor(Optional<Value> dcID) {
-	BinaryWriter wr(AssumeVersion(currentProtocolVersion));
+	BinaryWriter wr(AssumeVersion(currentProtocolVersion()));
 	wr.serializeBytes(datacenterReplicasKeys.begin);
 	wr << dcID;
 	return wr.toValue();
@@ -705,7 +699,7 @@ const Value datacenterReplicasValue(int const& replicas) {
 }
 Optional<Value> decodeDatacenterReplicasKey(KeyRef const& key) {
 	Optional<Value> dcID;
-	BinaryReader rd(key.removePrefix(datacenterReplicasKeys.begin), AssumeVersion(currentProtocolVersion));
+	BinaryReader rd(key.removePrefix(datacenterReplicasKeys.begin), AssumeVersion(currentProtocolVersion()));
 	rd >> dcID;
 	return dcID;
 }
@@ -721,27 +715,26 @@ extern const KeyRangeRef tLogDatacentersKeys;
 extern const KeyRef tLogDatacentersPrefix;
 const Key tLogDatacentersKeyFor(Optional<Value> dcID);
 
-const KeyRangeRef tLogDatacentersKeys(LiteralStringRef("\xff\x02/tLogDatacenters/"),
-                                      LiteralStringRef("\xff\x02/tLogDatacenters0"));
+const KeyRangeRef tLogDatacentersKeys("\xff\x02/tLogDatacenters/"_sr, "\xff\x02/tLogDatacenters0"_sr);
 const KeyRef tLogDatacentersPrefix = tLogDatacentersKeys.begin;
 
 const Key tLogDatacentersKeyFor(Optional<Value> dcID) {
-	BinaryWriter wr(AssumeVersion(currentProtocolVersion));
+	BinaryWriter wr(AssumeVersion(currentProtocolVersion()));
 	wr.serializeBytes(tLogDatacentersKeys.begin);
 	wr << dcID;
 	return wr.toValue();
 }
 Optional<Value> decodeTLogDatacentersKey(KeyRef const& key) {
 	Optional<Value> dcID;
-	BinaryReader rd(key.removePrefix(tLogDatacentersKeys.begin), AssumeVersion(currentProtocolVersion));
+	BinaryReader rd(key.removePrefix(tLogDatacentersKeys.begin), AssumeVersion(currentProtocolVersion()));
 	rd >> dcID;
 	return dcID;
 }
 
-const KeyRef primaryDatacenterKey = LiteralStringRef("\xff/primaryDatacenter");
+const KeyRef primaryDatacenterKey = "\xff/primaryDatacenter"_sr;
 
 // serverListKeys.contains(k) iff k.startsWith( serverListKeys.begin ) because '/'+1 == '0'
-const KeyRangeRef serverListKeys(LiteralStringRef("\xff/serverList/"), LiteralStringRef("\xff/serverList0"));
+const KeyRangeRef serverListKeys("\xff/serverList/"_sr, "\xff/serverList0"_sr);
 const KeyRef serverListPrefix = serverListKeys.begin;
 
 const Key serverListKeyFor(UID serverID) {
@@ -752,7 +745,7 @@ const Key serverListKeyFor(UID serverID) {
 }
 
 const Value serverListValue(StorageServerInterface const& server) {
-	auto protocolVersion = currentProtocolVersion;
+	auto protocolVersion = currentProtocolVersion();
 	protocolVersion.addObjectSerializerFlag();
 	return ObjectWriter::toValue(server, IncludeVersion(protocolVersion));
 }
@@ -784,7 +777,7 @@ StorageServerInterface decodeServerListValue(ValueRef const& value) {
 }
 
 Value swVersionValue(SWVersion const& swversion) {
-	auto protocolVersion = currentProtocolVersion;
+	auto protocolVersion = currentProtocolVersion();
 	protocolVersion.addObjectSerializerFlag();
 	return ObjectWriter::toValue(swversion, IncludeVersion(protocolVersion));
 }
@@ -797,11 +790,11 @@ SWVersion decodeSWVersionValue(ValueRef const& value) {
 }
 
 // processClassKeys.contains(k) iff k.startsWith( processClassKeys.begin ) because '/'+1 == '0'
-const KeyRangeRef processClassKeys(LiteralStringRef("\xff/processClass/"), LiteralStringRef("\xff/processClass0"));
+const KeyRangeRef processClassKeys("\xff/processClass/"_sr, "\xff/processClass0"_sr);
 const KeyRef processClassPrefix = processClassKeys.begin;
-const KeyRef processClassChangeKey = LiteralStringRef("\xff/processClassChanges");
-const KeyRef processClassVersionKey = LiteralStringRef("\xff/processClassChangesVersion");
-const ValueRef processClassVersionValue = LiteralStringRef("1");
+const KeyRef processClassChangeKey = "\xff/processClassChanges"_sr;
+const KeyRef processClassVersionKey = "\xff/processClassChangesVersion"_sr;
+const ValueRef processClassVersionValue = "1"_sr;
 
 const Key processClassKeyFor(StringRef processID) {
 	BinaryWriter wr(Unversioned());
@@ -837,21 +830,23 @@ ProcessClass decodeProcessClassValue(ValueRef const& value) {
 	return s;
 }
 
-const KeyRangeRef configKeys(LiteralStringRef("\xff/conf/"), LiteralStringRef("\xff/conf0"));
+const KeyRangeRef configKeys("\xff/conf/"_sr, "\xff/conf0"_sr);
 const KeyRef configKeysPrefix = configKeys.begin;
 
-const KeyRef perpetualStorageWiggleKey(LiteralStringRef("\xff/conf/perpetual_storage_wiggle"));
-const KeyRef perpetualStorageWiggleLocalityKey(LiteralStringRef("\xff/conf/perpetual_storage_wiggle_locality"));
-const KeyRef perpetualStorageWiggleIDPrefix(
-    LiteralStringRef("\xff/storageWiggleID/")); // withSuffix /primary or /remote
-const KeyRef perpetualStorageWiggleStatsPrefix(
-    LiteralStringRef("\xff/storageWiggleStats/")); // withSuffix /primary or /remote
+const KeyRef perpetualStorageWiggleKey("\xff/conf/perpetual_storage_wiggle"_sr);
+const KeyRef perpetualStorageWiggleLocalityKey("\xff/conf/perpetual_storage_wiggle_locality"_sr);
+const KeyRef perpetualStorageWiggleIDPrefix("\xff/storageWiggleID/"_sr); // withSuffix /primary or /remote
+const KeyRef perpetualStorageWiggleStatsPrefix("\xff/storageWiggleStats/"_sr); // withSuffix /primary or /remote
 
-const KeyRef triggerDDTeamInfoPrintKey(LiteralStringRef("\xff/triggerDDTeamInfoPrint"));
+const KeyRef triggerDDTeamInfoPrintKey("\xff/triggerDDTeamInfoPrint"_sr);
 
-const KeyRangeRef excludedServersKeys(LiteralStringRef("\xff/conf/excluded/"), LiteralStringRef("\xff/conf/excluded0"));
+const KeyRef consistencyScanInfoKey = "\xff/consistencyScanInfo"_sr;
+
+const KeyRef encryptionAtRestModeConfKey("\xff/conf/encryption_at_rest_mode"_sr);
+
+const KeyRangeRef excludedServersKeys("\xff/conf/excluded/"_sr, "\xff/conf/excluded0"_sr);
 const KeyRef excludedServersPrefix = excludedServersKeys.begin;
-const KeyRef excludedServersVersionKey = LiteralStringRef("\xff/conf/excluded");
+const KeyRef excludedServersVersionKey = "\xff/conf/excluded"_sr;
 AddressExclusion decodeExcludedServersKey(KeyRef const& key) {
 	ASSERT(key.startsWith(excludedServersPrefix));
 	// Returns an invalid NetworkAddress if given an invalid key (within the prefix)
@@ -866,10 +861,9 @@ std::string encodeExcludedServersKey(AddressExclusion const& addr) {
 	return excludedServersPrefix.toString() + addr.toString();
 }
 
-const KeyRangeRef excludedLocalityKeys(LiteralStringRef("\xff/conf/excluded_locality/"),
-                                       LiteralStringRef("\xff/conf/excluded_locality0"));
+const KeyRangeRef excludedLocalityKeys("\xff/conf/excluded_locality/"_sr, "\xff/conf/excluded_locality0"_sr);
 const KeyRef excludedLocalityPrefix = excludedLocalityKeys.begin;
-const KeyRef excludedLocalityVersionKey = LiteralStringRef("\xff/conf/excluded_locality");
+const KeyRef excludedLocalityVersionKey = "\xff/conf/excluded_locality"_sr;
 std::string decodeExcludedLocalityKey(KeyRef const& key) {
 	ASSERT(key.startsWith(excludedLocalityPrefix));
 	return key.removePrefix(excludedLocalityPrefix).toString();
@@ -878,9 +872,9 @@ std::string encodeExcludedLocalityKey(std::string const& locality) {
 	return excludedLocalityPrefix.toString() + locality;
 }
 
-const KeyRangeRef failedServersKeys(LiteralStringRef("\xff/conf/failed/"), LiteralStringRef("\xff/conf/failed0"));
+const KeyRangeRef failedServersKeys("\xff/conf/failed/"_sr, "\xff/conf/failed0"_sr);
 const KeyRef failedServersPrefix = failedServersKeys.begin;
-const KeyRef failedServersVersionKey = LiteralStringRef("\xff/conf/failed");
+const KeyRef failedServersVersionKey = "\xff/conf/failed"_sr;
 AddressExclusion decodeFailedServersKey(KeyRef const& key) {
 	ASSERT(key.startsWith(failedServersPrefix));
 	// Returns an invalid NetworkAddress if given an invalid key (within the prefix)
@@ -895,10 +889,9 @@ std::string encodeFailedServersKey(AddressExclusion const& addr) {
 	return failedServersPrefix.toString() + addr.toString();
 }
 
-const KeyRangeRef failedLocalityKeys(LiteralStringRef("\xff/conf/failed_locality/"),
-                                     LiteralStringRef("\xff/conf/failed_locality0"));
+const KeyRangeRef failedLocalityKeys("\xff/conf/failed_locality/"_sr, "\xff/conf/failed_locality0"_sr);
 const KeyRef failedLocalityPrefix = failedLocalityKeys.begin;
-const KeyRef failedLocalityVersionKey = LiteralStringRef("\xff/conf/failed_locality");
+const KeyRef failedLocalityVersionKey = "\xff/conf/failed_locality"_sr;
 std::string decodeFailedLocalityKey(KeyRef const& key) {
 	ASSERT(key.startsWith(failedLocalityPrefix));
 	return key.removePrefix(failedLocalityPrefix).toString();
@@ -907,20 +900,18 @@ std::string encodeFailedLocalityKey(std::string const& locality) {
 	return failedLocalityPrefix.toString() + locality;
 }
 
-// const KeyRangeRef globalConfigKeys( LiteralStringRef("\xff/globalConfig/"), LiteralStringRef("\xff/globalConfig0") );
+// const KeyRangeRef globalConfigKeys( "\xff/globalConfig/"_sr, "\xff/globalConfig0"_sr );
 // const KeyRef globalConfigPrefix = globalConfigKeys.begin;
 
-const KeyRangeRef globalConfigDataKeys(LiteralStringRef("\xff/globalConfig/k/"),
-                                       LiteralStringRef("\xff/globalConfig/k0"));
+const KeyRangeRef globalConfigDataKeys("\xff/globalConfig/k/"_sr, "\xff/globalConfig/k0"_sr);
 const KeyRef globalConfigKeysPrefix = globalConfigDataKeys.begin;
 
-const KeyRangeRef globalConfigHistoryKeys(LiteralStringRef("\xff/globalConfig/h/"),
-                                          LiteralStringRef("\xff/globalConfig/h0"));
+const KeyRangeRef globalConfigHistoryKeys("\xff/globalConfig/h/"_sr, "\xff/globalConfig/h0"_sr);
 const KeyRef globalConfigHistoryPrefix = globalConfigHistoryKeys.begin;
 
-const KeyRef globalConfigVersionKey = LiteralStringRef("\xff/globalConfig/v");
+const KeyRef globalConfigVersionKey = "\xff/globalConfig/v"_sr;
 
-const KeyRangeRef workerListKeys(LiteralStringRef("\xff/worker/"), LiteralStringRef("\xff/worker0"));
+const KeyRangeRef workerListKeys("\xff/worker/"_sr, "\xff/worker0"_sr);
 const KeyRef workerListPrefix = workerListKeys.begin;
 
 const Key workerListKeyFor(StringRef processID) {
@@ -950,11 +941,10 @@ ProcessData decodeWorkerListValue(ValueRef const& value) {
 	return s;
 }
 
-const KeyRangeRef backupProgressKeys(LiteralStringRef("\xff\x02/backupProgress/"),
-                                     LiteralStringRef("\xff\x02/backupProgress0"));
+const KeyRangeRef backupProgressKeys("\xff\x02/backupProgress/"_sr, "\xff\x02/backupProgress0"_sr);
 const KeyRef backupProgressPrefix = backupProgressKeys.begin;
-const KeyRef backupStartedKey = LiteralStringRef("\xff\x02/backupStarted");
-extern const KeyRef backupPausedKey = LiteralStringRef("\xff\x02/backupPaused");
+const KeyRef backupStartedKey = "\xff\x02/backupStarted"_sr;
+extern const KeyRef backupPausedKey = "\xff\x02/backupPaused"_sr;
 
 const Key backupProgressKeyFor(UID workerID) {
 	BinaryWriter wr(Unversioned());
@@ -997,98 +987,91 @@ std::vector<std::pair<UID, Version>> decodeBackupStartedValue(const ValueRef& va
 	return ids;
 }
 
-const KeyRef coordinatorsKey = LiteralStringRef("\xff/coordinators");
-const KeyRef logsKey = LiteralStringRef("\xff/logs");
-const KeyRef minRequiredCommitVersionKey = LiteralStringRef("\xff/minRequiredCommitVersion");
-const KeyRef versionEpochKey = LiteralStringRef("\xff/versionEpoch");
+const KeyRef previousCoordinatorsKey = "\xff/previousCoordinators"_sr;
+const KeyRef coordinatorsKey = "\xff/coordinators"_sr;
+const KeyRef logsKey = "\xff/logs"_sr;
+const KeyRef minRequiredCommitVersionKey = "\xff/minRequiredCommitVersion"_sr;
+const KeyRef versionEpochKey = "\xff/versionEpoch"_sr;
 
-const KeyRef globalKeysPrefix = LiteralStringRef("\xff/globals");
-const KeyRef lastEpochEndKey = LiteralStringRef("\xff/globals/lastEpochEnd");
-const KeyRef lastEpochEndPrivateKey = LiteralStringRef("\xff\xff/globals/lastEpochEnd");
-const KeyRef killStorageKey = LiteralStringRef("\xff/globals/killStorage");
-const KeyRef killStoragePrivateKey = LiteralStringRef("\xff\xff/globals/killStorage");
-const KeyRef rebootWhenDurableKey = LiteralStringRef("\xff/globals/rebootWhenDurable");
-const KeyRef rebootWhenDurablePrivateKey = LiteralStringRef("\xff\xff/globals/rebootWhenDurable");
-const KeyRef primaryLocalityKey = LiteralStringRef("\xff/globals/primaryLocality");
-const KeyRef primaryLocalityPrivateKey = LiteralStringRef("\xff\xff/globals/primaryLocality");
-const KeyRef fastLoggingEnabled = LiteralStringRef("\xff/globals/fastLoggingEnabled");
-const KeyRef fastLoggingEnabledPrivateKey = LiteralStringRef("\xff\xff/globals/fastLoggingEnabled");
+const KeyRef globalKeysPrefix = "\xff/globals"_sr;
+const KeyRef lastEpochEndKey = "\xff/globals/lastEpochEnd"_sr;
+const KeyRef lastEpochEndPrivateKey = "\xff\xff/globals/lastEpochEnd"_sr;
+const KeyRef killStorageKey = "\xff/globals/killStorage"_sr;
+const KeyRef killStoragePrivateKey = "\xff\xff/globals/killStorage"_sr;
+const KeyRef rebootWhenDurableKey = "\xff/globals/rebootWhenDurable"_sr;
+const KeyRef rebootWhenDurablePrivateKey = "\xff\xff/globals/rebootWhenDurable"_sr;
+const KeyRef primaryLocalityKey = "\xff/globals/primaryLocality"_sr;
+const KeyRef primaryLocalityPrivateKey = "\xff\xff/globals/primaryLocality"_sr;
+const KeyRef fastLoggingEnabled = "\xff/globals/fastLoggingEnabled"_sr;
+const KeyRef fastLoggingEnabledPrivateKey = "\xff\xff/globals/fastLoggingEnabled"_sr;
 
 // Whenever configuration changes or DD related system keyspace is changed(e.g.., serverList),
 // actor must grab the moveKeysLockOwnerKey and update moveKeysLockWriteKey.
 // This prevents concurrent write to the same system keyspace.
 // When the owner of the DD related system keyspace changes, DD will reboot
-const KeyRef moveKeysLockOwnerKey = LiteralStringRef("\xff/moveKeysLock/Owner");
-const KeyRef moveKeysLockWriteKey = LiteralStringRef("\xff/moveKeysLock/Write");
+const KeyRef moveKeysLockOwnerKey = "\xff/moveKeysLock/Owner"_sr;
+const KeyRef moveKeysLockWriteKey = "\xff/moveKeysLock/Write"_sr;
 
-const KeyRef dataDistributionModeKey = LiteralStringRef("\xff/dataDistributionMode");
+const KeyRef dataDistributionModeKey = "\xff/dataDistributionMode"_sr;
 const UID dataDistributionModeLock = UID(6345, 3425);
 
 // Keys to view and control tag throttling
-const KeyRangeRef tagThrottleKeys =
-    KeyRangeRef(LiteralStringRef("\xff\x02/throttledTags/tag/"), LiteralStringRef("\xff\x02/throttledTags/tag0"));
+const KeyRangeRef tagThrottleKeys = KeyRangeRef("\xff\x02/throttledTags/tag/"_sr, "\xff\x02/throttledTags/tag0"_sr);
 const KeyRef tagThrottleKeysPrefix = tagThrottleKeys.begin;
-const KeyRef tagThrottleAutoKeysPrefix = LiteralStringRef("\xff\x02/throttledTags/tag/\x01");
-const KeyRef tagThrottleSignalKey = LiteralStringRef("\xff\x02/throttledTags/signal");
-const KeyRef tagThrottleAutoEnabledKey = LiteralStringRef("\xff\x02/throttledTags/autoThrottlingEnabled");
-const KeyRef tagThrottleLimitKey = LiteralStringRef("\xff\x02/throttledTags/manualThrottleLimit");
-const KeyRef tagThrottleCountKey = LiteralStringRef("\xff\x02/throttledTags/manualThrottleCount");
+const KeyRef tagThrottleAutoKeysPrefix = "\xff\x02/throttledTags/tag/\x01"_sr;
+const KeyRef tagThrottleSignalKey = "\xff\x02/throttledTags/signal"_sr;
+const KeyRef tagThrottleAutoEnabledKey = "\xff\x02/throttledTags/autoThrottlingEnabled"_sr;
+const KeyRef tagThrottleLimitKey = "\xff\x02/throttledTags/manualThrottleLimit"_sr;
+const KeyRef tagThrottleCountKey = "\xff\x02/throttledTags/manualThrottleCount"_sr;
 
 // Client status info prefix
-const KeyRangeRef fdbClientInfoPrefixRange(LiteralStringRef("\xff\x02/fdbClientInfo/"),
-                                           LiteralStringRef("\xff\x02/fdbClientInfo0"));
+const KeyRangeRef fdbClientInfoPrefixRange("\xff\x02/fdbClientInfo/"_sr, "\xff\x02/fdbClientInfo0"_sr);
 // See remaining fields in GlobalConfig.actor.h
 
 // ConsistencyCheck settings
-const KeyRef fdbShouldConsistencyCheckBeSuspended = LiteralStringRef("\xff\x02/ConsistencyCheck/Suspend");
+const KeyRef fdbShouldConsistencyCheckBeSuspended = "\xff\x02/ConsistencyCheck/Suspend"_sr;
 
 // Request latency measurement key
-const KeyRef latencyBandConfigKey = LiteralStringRef("\xff\x02/latencyBandConfig");
+const KeyRef latencyBandConfigKey = "\xff\x02/latencyBandConfig"_sr;
 
 // Keyspace to maintain wall clock to version map
-const KeyRangeRef timeKeeperPrefixRange(LiteralStringRef("\xff\x02/timeKeeper/map/"),
-                                        LiteralStringRef("\xff\x02/timeKeeper/map0"));
-const KeyRef timeKeeperVersionKey = LiteralStringRef("\xff\x02/timeKeeper/version");
-const KeyRef timeKeeperDisableKey = LiteralStringRef("\xff\x02/timeKeeper/disable");
+const KeyRangeRef timeKeeperPrefixRange("\xff\x02/timeKeeper/map/"_sr, "\xff\x02/timeKeeper/map0"_sr);
+const KeyRef timeKeeperVersionKey = "\xff\x02/timeKeeper/version"_sr;
+const KeyRef timeKeeperDisableKey = "\xff\x02/timeKeeper/disable"_sr;
 
 // Backup Log Mutation constant variables
-const KeyRef backupEnabledKey = LiteralStringRef("\xff/backupEnabled");
-const KeyRangeRef backupLogKeys(LiteralStringRef("\xff\x02/blog/"), LiteralStringRef("\xff\x02/blog0"));
-const KeyRangeRef applyLogKeys(LiteralStringRef("\xff\x02/alog/"), LiteralStringRef("\xff\x02/alog0"));
+const KeyRef backupEnabledKey = "\xff/backupEnabled"_sr;
+const KeyRangeRef backupLogKeys("\xff\x02/blog/"_sr, "\xff\x02/blog0"_sr);
+const KeyRangeRef applyLogKeys("\xff\x02/alog/"_sr, "\xff\x02/alog0"_sr);
 // static_assert( backupLogKeys.begin.size() == backupLogPrefixBytes, "backupLogPrefixBytes incorrect" );
-const KeyRef backupVersionKey = LiteralStringRef("\xff/backupDataFormat");
-const ValueRef backupVersionValue = LiteralStringRef("4");
+const KeyRef backupVersionKey = "\xff/backupDataFormat"_sr;
+const ValueRef backupVersionValue = "4"_sr;
 const int backupVersion = 4;
 
 // Log Range constant variables
 // \xff/logRanges/[16-byte UID][begin key] := serialize( make_pair([end key], [destination key prefix]),
 // IncludeVersion() )
-const KeyRangeRef logRangesRange(LiteralStringRef("\xff/logRanges/"), LiteralStringRef("\xff/logRanges0"));
+const KeyRangeRef logRangesRange("\xff/logRanges/"_sr, "\xff/logRanges0"_sr);
 
 // Layer status metadata prefix
-const KeyRangeRef layerStatusMetaPrefixRange(LiteralStringRef("\xff\x02/status/"),
-                                             LiteralStringRef("\xff\x02/status0"));
+const KeyRangeRef layerStatusMetaPrefixRange("\xff\x02/status/"_sr, "\xff\x02/status0"_sr);
 
 // Backup agent status root
-const KeyRangeRef backupStatusPrefixRange(LiteralStringRef("\xff\x02/backupstatus/"),
-                                          LiteralStringRef("\xff\x02/backupstatus0"));
+const KeyRangeRef backupStatusPrefixRange("\xff\x02/backupstatus/"_sr, "\xff\x02/backupstatus0"_sr);
 
 // Restore configuration constant variables
-const KeyRangeRef fileRestorePrefixRange(LiteralStringRef("\xff\x02/restore-agent/"),
-                                         LiteralStringRef("\xff\x02/restore-agent0"));
+const KeyRangeRef fileRestorePrefixRange("\xff\x02/restore-agent/"_sr, "\xff\x02/restore-agent0"_sr);
 
 // Backup Agent configuration constant variables
-const KeyRangeRef fileBackupPrefixRange(LiteralStringRef("\xff\x02/backup-agent/"),
-                                        LiteralStringRef("\xff\x02/backup-agent0"));
+const KeyRangeRef fileBackupPrefixRange("\xff\x02/backup-agent/"_sr, "\xff\x02/backup-agent0"_sr);
 
 // DR Agent configuration constant variables
-const KeyRangeRef databaseBackupPrefixRange(LiteralStringRef("\xff\x02/db-backup-agent/"),
-                                            LiteralStringRef("\xff\x02/db-backup-agent0"));
+const KeyRangeRef databaseBackupPrefixRange("\xff\x02/db-backup-agent/"_sr, "\xff\x02/db-backup-agent0"_sr);
 
 // \xff\x02/sharedLogRangesConfig/destUidLookup/[keyRange]
-const KeyRef destUidLookupPrefix = LiteralStringRef("\xff\x02/sharedLogRangesConfig/destUidLookup/");
+const KeyRef destUidLookupPrefix = "\xff\x02/sharedLogRangesConfig/destUidLookup/"_sr;
 // \xff\x02/sharedLogRangesConfig/backuplatestVersions/[destUid]/[logUid]
-const KeyRef backupLatestVersionsPrefix = LiteralStringRef("\xff\x02/sharedLogRangesConfig/backupLatestVersions/");
+const KeyRef backupLatestVersionsPrefix = "\xff\x02/sharedLogRangesConfig/backupLatestVersions/"_sr;
 
 // Returns the encoded key comprised of begin key and log uid
 Key logRangesEncodeKey(KeyRef keyBegin, UID logUid) {
@@ -1143,31 +1126,27 @@ Key uidPrefixKey(KeyRef keyPrefix, UID logUid) {
 // Apply mutations constant variables
 // \xff/applyMutationsEnd/[16-byte UID] := serialize( endVersion, Unversioned() )
 // This indicates what is the highest version the mutation log can be applied
-const KeyRangeRef applyMutationsEndRange(LiteralStringRef("\xff/applyMutationsEnd/"),
-                                         LiteralStringRef("\xff/applyMutationsEnd0"));
+const KeyRangeRef applyMutationsEndRange("\xff/applyMutationsEnd/"_sr, "\xff/applyMutationsEnd0"_sr);
 
 // \xff/applyMutationsBegin/[16-byte UID] := serialize( beginVersion, Unversioned() )
-const KeyRangeRef applyMutationsBeginRange(LiteralStringRef("\xff/applyMutationsBegin/"),
-                                           LiteralStringRef("\xff/applyMutationsBegin0"));
+const KeyRangeRef applyMutationsBeginRange("\xff/applyMutationsBegin/"_sr, "\xff/applyMutationsBegin0"_sr);
 
 // \xff/applyMutationsAddPrefix/[16-byte UID] := addPrefix
-const KeyRangeRef applyMutationsAddPrefixRange(LiteralStringRef("\xff/applyMutationsAddPrefix/"),
-                                               LiteralStringRef("\xff/applyMutationsAddPrefix0"));
+const KeyRangeRef applyMutationsAddPrefixRange("\xff/applyMutationsAddPrefix/"_sr, "\xff/applyMutationsAddPrefix0"_sr);
 
 // \xff/applyMutationsRemovePrefix/[16-byte UID] := removePrefix
-const KeyRangeRef applyMutationsRemovePrefixRange(LiteralStringRef("\xff/applyMutationsRemovePrefix/"),
-                                                  LiteralStringRef("\xff/applyMutationsRemovePrefix0"));
+const KeyRangeRef applyMutationsRemovePrefixRange("\xff/applyMutationsRemovePrefix/"_sr,
+                                                  "\xff/applyMutationsRemovePrefix0"_sr);
 
-const KeyRangeRef applyMutationsKeyVersionMapRange(LiteralStringRef("\xff/applyMutationsKeyVersionMap/"),
-                                                   LiteralStringRef("\xff/applyMutationsKeyVersionMap0"));
-const KeyRangeRef applyMutationsKeyVersionCountRange(LiteralStringRef("\xff\x02/applyMutationsKeyVersionCount/"),
-                                                     LiteralStringRef("\xff\x02/applyMutationsKeyVersionCount0"));
+const KeyRangeRef applyMutationsKeyVersionMapRange("\xff/applyMutationsKeyVersionMap/"_sr,
+                                                   "\xff/applyMutationsKeyVersionMap0"_sr);
+const KeyRangeRef applyMutationsKeyVersionCountRange("\xff\x02/applyMutationsKeyVersionCount/"_sr,
+                                                     "\xff\x02/applyMutationsKeyVersionCount0"_sr);
 
-const KeyRef systemTuplesPrefix = LiteralStringRef("\xff/a/");
-const KeyRef metricConfChangeKey = LiteralStringRef("\x01TDMetricConfChanges\x00");
+const KeyRef systemTuplesPrefix = "\xff/a/"_sr;
+const KeyRef metricConfChangeKey = "\x01TDMetricConfChanges\x00"_sr;
 
-const KeyRangeRef metricConfKeys(LiteralStringRef("\x01TDMetricConf\x00\x01"),
-                                 LiteralStringRef("\x01TDMetricConf\x00\x02"));
+const KeyRangeRef metricConfKeys("\x01TDMetricConf\x00\x01"_sr, "\x01TDMetricConf\x00\x02"_sr);
 const KeyRef metricConfPrefix = metricConfKeys.begin;
 
 /*
@@ -1176,15 +1155,15 @@ const Key metricConfKey( KeyRef const& prefix, MetricNameRef const& name, KeyRef
     wr.serializeBytes( prefix );
     wr.serializeBytes( metricConfPrefix );
     wr.serializeBytes( name.type );
-    wr.serializeBytes( LiteralStringRef("\x00\x01") );
+    wr.serializeBytes( "\x00\x01"_sr );
     wr.serializeBytes( name.name );
-    wr.serializeBytes( LiteralStringRef("\x00\x01") );
+    wr.serializeBytes( "\x00\x01"_sr );
     wr.serializeBytes( name.address );
-    wr.serializeBytes( LiteralStringRef("\x00\x01") );
+    wr.serializeBytes( "\x00\x01"_sr );
     wr.serializeBytes( name.id );
-    wr.serializeBytes( LiteralStringRef("\x00\x01") );
+    wr.serializeBytes( "\x00\x01"_sr );
     wr.serializeBytes( key );
-    wr.serializeBytes( LiteralStringRef("\x00") );
+    wr.serializeBytes( "\x00"_sr );
     return wr.toValue();
 }
 
@@ -1207,23 +1186,22 @@ std::pair<MetricNameRef, KeyRef> decodeMetricConfKey( KeyRef const& prefix, KeyR
 }
 */
 
-const KeyRef maxUIDKey = LiteralStringRef("\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff");
+const KeyRef maxUIDKey = "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"_sr;
 
-const KeyRef databaseLockedKey = LiteralStringRef("\xff/dbLocked");
-const KeyRef databaseLockedKeyEnd = LiteralStringRef("\xff/dbLocked\x00");
-const KeyRef metadataVersionKey = LiteralStringRef("\xff/metadataVersion");
-const KeyRef metadataVersionKeyEnd = LiteralStringRef("\xff/metadataVersion\x00");
-const KeyRef metadataVersionRequiredValue =
-    LiteralStringRef("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00");
-const KeyRef mustContainSystemMutationsKey = LiteralStringRef("\xff/mustContainSystemMutations");
+const KeyRef databaseLockedKey = "\xff/dbLocked"_sr;
+const KeyRef databaseLockedKeyEnd = "\xff/dbLocked\x00"_sr;
+const KeyRef metadataVersionKey = "\xff/metadataVersion"_sr;
+const KeyRef metadataVersionKeyEnd = "\xff/metadataVersion\x00"_sr;
+const KeyRef metadataVersionRequiredValue = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"_sr;
+const KeyRef mustContainSystemMutationsKey = "\xff/mustContainSystemMutations"_sr;
 
-const KeyRangeRef monitorConfKeys(LiteralStringRef("\xff\x02/monitorConf/"), LiteralStringRef("\xff\x02/monitorConf0"));
+const KeyRangeRef monitorConfKeys("\xff\x02/monitorConf/"_sr, "\xff\x02/monitorConf0"_sr);
 
-const KeyRef restoreRequestDoneKey = LiteralStringRef("\xff\x02/restoreRequestDone");
+const KeyRef restoreRequestDoneKey = "\xff\x02/restoreRequestDone"_sr;
 
-const KeyRef healthyZoneKey = LiteralStringRef("\xff\x02/healthyZone");
-const StringRef ignoreSSFailuresZoneString = LiteralStringRef("IgnoreSSFailures");
-const KeyRef rebalanceDDIgnoreKey = LiteralStringRef("\xff\x02/rebalanceDDIgnored");
+const KeyRef healthyZoneKey = "\xff\x02/healthyZone"_sr;
+const StringRef ignoreSSFailuresZoneString = "IgnoreSSFailures"_sr;
+const KeyRef rebalanceDDIgnoreKey = "\xff\x02/rebalanceDDIgnored"_sr;
 
 const Value healthyZoneValue(StringRef const& zoneId, Version version) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withHealthyZoneValue()));
@@ -1240,16 +1218,15 @@ std::pair<Key, Version> decodeHealthyZoneValue(ValueRef const& value) {
 	return std::make_pair(zoneId, version);
 }
 
-const KeyRangeRef testOnlyTxnStateStorePrefixRange(LiteralStringRef("\xff/TESTONLYtxnStateStore/"),
-                                                   LiteralStringRef("\xff/TESTONLYtxnStateStore0"));
+const KeyRangeRef testOnlyTxnStateStorePrefixRange("\xff/TESTONLYtxnStateStore/"_sr, "\xff/TESTONLYtxnStateStore0"_sr);
 
-const KeyRef writeRecoveryKey = LiteralStringRef("\xff/writeRecovery");
-const ValueRef writeRecoveryKeyTrue = LiteralStringRef("1");
-const KeyRef snapshotEndVersionKey = LiteralStringRef("\xff/snapshotEndVersion");
+const KeyRef writeRecoveryKey = "\xff/writeRecovery"_sr;
+const ValueRef writeRecoveryKeyTrue = "1"_sr;
+const KeyRef snapshotEndVersionKey = "\xff/snapshotEndVersion"_sr;
 
-const KeyRangeRef changeFeedKeys(LiteralStringRef("\xff\x02/feed/"), LiteralStringRef("\xff\x02/feed0"));
+const KeyRangeRef changeFeedKeys("\xff\x02/feed/"_sr, "\xff\x02/feed0"_sr);
 const KeyRef changeFeedPrefix = changeFeedKeys.begin;
-const KeyRef changeFeedPrivatePrefix = LiteralStringRef("\xff\xff\x02/feed/");
+const KeyRef changeFeedPrivatePrefix = "\xff\xff\x02/feed/"_sr;
 
 const Value changeFeedValue(KeyRangeRef const& range, Version popVersion, ChangeFeedStatus status) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withChangeFeed()));
@@ -1270,7 +1247,7 @@ std::tuple<KeyRange, Version, ChangeFeedStatus> decodeChangeFeedValue(ValueRef c
 	return std::make_tuple(range, version, status);
 }
 
-const KeyRangeRef changeFeedDurableKeys(LiteralStringRef("\xff\xff/cf/"), LiteralStringRef("\xff\xff/cf0"));
+const KeyRangeRef changeFeedDurableKeys("\xff\xff/cf/"_sr, "\xff\xff/cf0"_sr);
 const KeyRef changeFeedDurablePrefix = changeFeedDurableKeys.begin;
 
 const Value changeFeedDurableKey(Key const& feed, Version version) {
@@ -1310,9 +1287,9 @@ const KeyRangeRef configClassKeys("\xff\xff/configClasses/"_sr, "\xff\xff/config
 
 // key to watch for changes in active blob ranges + KeyRangeMap of active blob ranges
 // Blob Manager + Worker stuff is all \xff\x02 to avoid Transaction State Store
-const KeyRef blobRangeChangeKey = LiteralStringRef("\xff\x02/blobRangeChange");
-const KeyRangeRef blobRangeKeys(LiteralStringRef("\xff\x02/blobRange/"), LiteralStringRef("\xff\x02/blobRange0"));
-const KeyRef blobManagerEpochKey = LiteralStringRef("\xff\x02/blobManagerEpoch");
+const KeyRef blobRangeChangeKey = "\xff\x02/blobRangeChange"_sr;
+const KeyRangeRef blobRangeKeys("\xff\x02/blobRange/"_sr, "\xff\x02/blobRange0"_sr);
+const KeyRef blobManagerEpochKey = "\xff\x02/blobManagerEpoch"_sr;
 
 const Value blobManagerEpochValueFor(int64_t epoch) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withBlobGranule()));
@@ -1328,15 +1305,19 @@ int64_t decodeBlobManagerEpochValue(ValueRef const& value) {
 }
 
 // blob granule data
-const KeyRangeRef blobGranuleFileKeys(LiteralStringRef("\xff\x02/bgf/"), LiteralStringRef("\xff\x02/bgf0"));
-const KeyRangeRef blobGranuleMappingKeys(LiteralStringRef("\xff\x02/bgm/"), LiteralStringRef("\xff\x02/bgm0"));
-const KeyRangeRef blobGranuleLockKeys(LiteralStringRef("\xff\x02/bgl/"), LiteralStringRef("\xff\x02/bgl0"));
-const KeyRangeRef blobGranuleSplitKeys(LiteralStringRef("\xff\x02/bgs/"), LiteralStringRef("\xff\x02/bgs0"));
-const KeyRangeRef blobGranuleMergeKeys(LiteralStringRef("\xff\x02/bgmerge/"), LiteralStringRef("\xff\x02/bgmerge0"));
-const KeyRangeRef blobGranuleHistoryKeys(LiteralStringRef("\xff\x02/bgh/"), LiteralStringRef("\xff\x02/bgh0"));
-const KeyRangeRef blobGranulePurgeKeys(LiteralStringRef("\xff\x02/bgp/"), LiteralStringRef("\xff\x02/bgp0"));
-const KeyRangeRef blobGranuleVersionKeys(LiteralStringRef("\xff\x02/bgv/"), LiteralStringRef("\xff\x02/bgv0"));
-const KeyRef blobGranulePurgeChangeKey = LiteralStringRef("\xff\x02/bgpChange");
+const KeyRef blobRangeActive = "1"_sr;
+const KeyRef blobRangeInactive = StringRef();
+
+const KeyRangeRef blobGranuleFileKeys("\xff\x02/bgf/"_sr, "\xff\x02/bgf0"_sr);
+const KeyRangeRef blobGranuleMappingKeys("\xff\x02/bgm/"_sr, "\xff\x02/bgm0"_sr);
+const KeyRangeRef blobGranuleLockKeys("\xff\x02/bgl/"_sr, "\xff\x02/bgl0"_sr);
+const KeyRangeRef blobGranuleSplitKeys("\xff\x02/bgs/"_sr, "\xff\x02/bgs0"_sr);
+const KeyRangeRef blobGranuleMergeKeys("\xff\x02/bgmerge/"_sr, "\xff\x02/bgmerge0"_sr);
+const KeyRangeRef blobGranuleMergeBoundaryKeys("\xff\x02/bgmergebounds/"_sr, "\xff\x02/bgmergebounds0"_sr);
+const KeyRangeRef blobGranuleHistoryKeys("\xff\x02/bgh/"_sr, "\xff\x02/bgh0"_sr);
+const KeyRangeRef blobGranulePurgeKeys("\xff\x02/bgp/"_sr, "\xff\x02/bgp0"_sr);
+const KeyRangeRef blobGranuleForcePurgedKeys("\xff\x02/bgpforce/"_sr, "\xff\x02/bgpforce0"_sr);
+const KeyRef blobGranulePurgeChangeKey = "\xff\x02/bgpChange"_sr;
 
 const uint8_t BG_FILE_TYPE_DELTA = 'D';
 const uint8_t BG_FILE_TYPE_SNAPSHOT = 'S';
@@ -1524,9 +1505,9 @@ std::pair<BlobGranuleSplitState, Version> decodeBlobGranuleSplitValue(const Valu
 
 const Value blobGranuleMergeValueFor(KeyRange mergeKeyRange,
                                      std::vector<UID> parentGranuleIDs,
-                                     std::vector<KeyRange> parentGranuleRanges,
+                                     std::vector<Key> parentGranuleRanges,
                                      std::vector<Version> parentGranuleStartVersions) {
-	ASSERT(parentGranuleIDs.size() == parentGranuleRanges.size());
+	ASSERT(parentGranuleIDs.size() == parentGranuleRanges.size() - 1);
 	ASSERT(parentGranuleIDs.size() == parentGranuleStartVersions.size());
 
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withBlobGranule()));
@@ -1536,12 +1517,12 @@ const Value blobGranuleMergeValueFor(KeyRange mergeKeyRange,
 	wr << parentGranuleStartVersions;
 	return addVersionStampAtEnd(wr.toValue());
 }
-std::tuple<KeyRange, Version, std::vector<UID>, std::vector<KeyRange>, std::vector<Version>>
-decodeBlobGranuleMergeValue(ValueRef const& value) {
+std::tuple<KeyRange, Version, std::vector<UID>, std::vector<Key>, std::vector<Version>> decodeBlobGranuleMergeValue(
+    ValueRef const& value) {
 	KeyRange range;
 	Version v;
 	std::vector<UID> parentGranuleIDs;
-	std::vector<KeyRange> parentGranuleRanges;
+	std::vector<Key> parentGranuleRanges;
 	std::vector<Version> parentGranuleStartVersions;
 
 	BinaryReader reader(value, IncludeVersion());
@@ -1551,13 +1532,30 @@ decodeBlobGranuleMergeValue(ValueRef const& value) {
 	reader >> parentGranuleStartVersions;
 	reader >> v;
 
-	ASSERT(parentGranuleIDs.size() == parentGranuleRanges.size());
+	ASSERT(parentGranuleIDs.size() == parentGranuleRanges.size() - 1);
 	ASSERT(parentGranuleIDs.size() == parentGranuleStartVersions.size());
 	ASSERT(bigEndian64(v) >= 0);
 
 	return std::tuple(range, bigEndian64(v), parentGranuleIDs, parentGranuleRanges, parentGranuleStartVersions);
 }
 
+const Key blobGranuleMergeBoundaryKeyFor(const KeyRef& key) {
+	return key.withPrefix(blobGranuleMergeBoundaryKeys.begin);
+}
+
+const Value blobGranuleMergeBoundaryValueFor(BlobGranuleMergeBoundary const& boundary) {
+	BinaryWriter wr(IncludeVersion(ProtocolVersion::withBlobGranule()));
+	wr << boundary;
+	return wr.toValue();
+}
+
+Standalone<BlobGranuleMergeBoundary> decodeBlobGranuleMergeBoundaryValue(const ValueRef& value) {
+	Standalone<BlobGranuleMergeBoundary> boundaryValue;
+	BinaryReader reader(value, IncludeVersion());
+	reader >> boundaryValue;
+	return boundaryValue;
+}
+
 const Key blobGranuleHistoryKeyFor(KeyRangeRef const& range, Version version) {
 	BinaryWriter wr(AssumeVersion(ProtocolVersion::withBlobGranule()));
 	wr.serializeBytes(blobGranuleHistoryKeys.begin);
@@ -1581,6 +1579,8 @@ const KeyRange blobGranuleHistoryKeyRangeFor(KeyRangeRef const& range) {
 }
 
 const Value blobGranuleHistoryValueFor(Standalone<BlobGranuleHistoryValue> const& historyValue) {
+	ASSERT(historyValue.parentVersions.empty() ||
+	       historyValue.parentBoundaries.size() - 1 == historyValue.parentVersions.size());
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withBlobGranule()));
 	wr << historyValue;
 	return wr.toValue();
@@ -1590,10 +1590,12 @@ Standalone<BlobGranuleHistoryValue> decodeBlobGranuleHistoryValue(const ValueRef
 	Standalone<BlobGranuleHistoryValue> historyValue;
 	BinaryReader reader(value, IncludeVersion());
 	reader >> historyValue;
+	ASSERT(historyValue.parentVersions.empty() ||
+	       historyValue.parentBoundaries.size() - 1 == historyValue.parentVersions.size());
 	return historyValue;
 }
 
-const KeyRangeRef blobWorkerListKeys(LiteralStringRef("\xff\x02/bwList/"), LiteralStringRef("\xff\x02/bwList0"));
+const KeyRangeRef blobWorkerListKeys("\xff\x02/bwList/"_sr, "\xff\x02/bwList0"_sr);
 
 const Key blobWorkerListKeyFor(UID workerID) {
 	BinaryWriter wr(AssumeVersion(ProtocolVersion::withBlobGranule()));
@@ -1620,11 +1622,12 @@ BlobWorkerInterface decodeBlobWorkerListValue(ValueRef const& value) {
 	return interf;
 }
 
-const KeyRangeRef tenantMapKeys("\xff/tenantMap/"_sr, "\xff/tenantMap0"_sr);
-const KeyRef tenantMapPrefix = tenantMapKeys.begin;
-const KeyRef tenantMapPrivatePrefix = "\xff\xff/tenantMap/"_sr;
-const KeyRef tenantLastIdKey = "\xff/tenantLastId/"_sr;
-const KeyRef tenantDataPrefixKey = "\xff/tenantDataPrefix"_sr;
+const KeyRangeRef storageQuotaKeys("\xff/storageQuota/"_sr, "\xff/storageQuota0"_sr);
+const KeyRef storageQuotaPrefix = storageQuotaKeys.begin;
+
+Key storageQuotaKey(StringRef tenantName) {
+	return tenantName.withPrefix(storageQuotaPrefix);
+}
 
 // for tests
 void testSSISerdes(StorageServerInterface const& ssi) {
diff --git a/fdbclient/TagThrottle.actor.cpp b/fdbclient/TagThrottle.actor.cpp
index b5205fd153..7a1712c4df 100644
--- a/fdbclient/TagThrottle.actor.cpp
+++ b/fdbclient/TagThrottle.actor.cpp
@@ -136,17 +136,11 @@ Key ThrottleApi::getTagQuotaKey(TransactionTagRef tag) {
 }
 
 bool ThrottleApi::TagQuotaValue::isValid() const {
-	return reservedReadQuota <= totalReadQuota && reservedWriteQuota <= totalWriteQuota && reservedReadQuota >= 0 &&
-	       reservedWriteQuota >= 0;
+	return reservedQuota <= totalQuota && reservedQuota >= 0;
 }
 
 Value ThrottleApi::TagQuotaValue::toValue() const {
-	Tuple tuple;
-	tuple.appendDouble(reservedReadQuota);
-	tuple.appendDouble(totalReadQuota);
-	tuple.appendDouble(reservedWriteQuota);
-	tuple.appendDouble(totalWriteQuota);
-	return tuple.pack();
+	return Tuple::makeTuple(reservedQuota, totalQuota).pack();
 }
 
 ThrottleApi::TagQuotaValue ThrottleApi::TagQuotaValue::fromValue(ValueRef value) {
@@ -156,20 +150,16 @@ ThrottleApi::TagQuotaValue ThrottleApi::TagQuotaValue::fromValue(ValueRef value)
 	}
 	TagQuotaValue result;
 	try {
-		result.reservedReadQuota = tuple.getDouble(0);
-		result.totalReadQuota = tuple.getDouble(1);
-		result.reservedWriteQuota = tuple.getDouble(2);
-		result.totalWriteQuota = tuple.getDouble(3);
+		result.reservedQuota = tuple.getDouble(0);
+		result.totalQuota = tuple.getDouble(1);
 	} catch (Error& e) {
 		TraceEvent(SevWarnAlways, "TagQuotaValueFailedToDeserialize").error(e);
 		throw invalid_throttle_quota_value();
 	}
 	if (!result.isValid()) {
 		TraceEvent(SevWarnAlways, "TagQuotaValueInvalidQuotas")
-		    .detail("ReservedReadQuota", result.reservedReadQuota)
-		    .detail("TotalReadQuota", result.totalReadQuota)
-		    .detail("ReservedWriteQuota", result.reservedWriteQuota)
-		    .detail("TotalWriteQuota", result.totalWriteQuota);
+		    .detail("ReservedQuota", result.reservedQuota)
+		    .detail("TotalQuota", result.totalQuota);
 		throw invalid_throttle_quota_value();
 	}
 	return result;
diff --git a/fdbclient/TaskBucket.actor.cpp b/fdbclient/TaskBucket.actor.cpp
index aa9a110086..2e72b301c0 100644
--- a/fdbclient/TaskBucket.actor.cpp
+++ b/fdbclient/TaskBucket.actor.cpp
@@ -66,7 +66,7 @@ struct UnblockFutureTaskFunc : TaskFuncBase {
 		return Void();
 	}
 };
-StringRef UnblockFutureTaskFunc::name = LiteralStringRef("UnblockFuture");
+StringRef UnblockFutureTaskFunc::name = "UnblockFuture"_sr;
 REGISTER_TASKFUNC(UnblockFutureTaskFunc);
 
 struct AddTaskFunc : TaskFuncBase {
@@ -88,7 +88,7 @@ struct AddTaskFunc : TaskFuncBase {
 		return Void();
 	};
 };
-StringRef AddTaskFunc::name = LiteralStringRef("AddTask");
+StringRef AddTaskFunc::name = "AddTask"_sr;
 REGISTER_TASKFUNC(AddTaskFunc);
 
 struct IdleTaskFunc : TaskFuncBase {
@@ -109,18 +109,18 @@ struct IdleTaskFunc : TaskFuncBase {
 		return tb->finish(tr, task);
 	};
 };
-StringRef IdleTaskFunc::name = LiteralStringRef("idle");
+StringRef IdleTaskFunc::name = "idle"_sr;
 REGISTER_TASKFUNC(IdleTaskFunc);
 
-Key Task::reservedTaskParamKeyType = LiteralStringRef("type");
-Key Task::reservedTaskParamKeyAddTask = LiteralStringRef("_add_task");
-Key Task::reservedTaskParamKeyDone = LiteralStringRef("done");
-Key Task::reservedTaskParamKeyPriority = LiteralStringRef("priority");
-Key Task::reservedTaskParamKeyFuture = LiteralStringRef("future");
-Key Task::reservedTaskParamKeyBlockID = LiteralStringRef("blockid");
-Key Task::reservedTaskParamKeyVersion = LiteralStringRef("version");
-Key Task::reservedTaskParamValidKey = LiteralStringRef("_validkey");
-Key Task::reservedTaskParamValidValue = LiteralStringRef("_validvalue");
+Key Task::reservedTaskParamKeyType = "type"_sr;
+Key Task::reservedTaskParamKeyAddTask = "_add_task"_sr;
+Key Task::reservedTaskParamKeyDone = "done"_sr;
+Key Task::reservedTaskParamKeyPriority = "priority"_sr;
+Key Task::reservedTaskParamKeyFuture = "future"_sr;
+Key Task::reservedTaskParamKeyBlockID = "blockid"_sr;
+Key Task::reservedTaskParamKeyVersion = "version"_sr;
+Key Task::reservedTaskParamValidKey = "_validkey"_sr;
+Key Task::reservedTaskParamValidValue = "_validvalue"_sr;
 
 // IMPORTANT:  Task() must result in an EMPTY parameter set, so params should only
 // be set for non-default constructor arguments.  To change this behavior look at all
@@ -199,7 +199,7 @@ public:
 		// many other new tasks get added so that the timed out tasks never get chances to re-run
 		if (deterministicRandom()->random01() < CLIENT_KNOBS->TASKBUCKET_CHECK_TIMEOUT_CHANCE) {
 			bool anyTimeouts = wait(requeueTimedOutTasks(tr, taskBucket));
-			TEST(anyTimeouts); // Found a task that timed out
+			CODE_PROBE(anyTimeouts, "Found a task that timed out");
 		}
 
 		state std::vector<Future<Optional<Key>>> taskKeyFutures(CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY + 1);
@@ -233,7 +233,7 @@ public:
 			bool anyTimeouts = wait(requeueTimedOutTasks(tr, taskBucket));
 			// If there were timeouts, try to get a task since there should now be one in one of the available spaces.
 			if (anyTimeouts) {
-				TEST(true); // Try to get one task from timeouts subspace
+				CODE_PROBE(true, "Try to get one task from timeouts subspace");
 				Reference<Task> task = wait(getOne(tr, taskBucket));
 				return task;
 			}
@@ -651,10 +651,7 @@ public:
 	                                     Reference<Task> task) {
 		taskBucket->setOptions(tr);
 
-		Tuple t;
-		t.append(task->timeoutVersion);
-		t.append(task->key);
-
+		Tuple t = Tuple::makeTuple(task->timeoutVersion, task->key);
 		RangeResult values = wait(tr->getRange(taskBucket->timeouts.range(t), 1));
 		if (values.size() > 0)
 			return false;
@@ -707,7 +704,7 @@ public:
 					wait(delay(CLIENT_KNOBS->TASKBUCKET_CHECK_ACTIVE_DELAY));
 					bool isActiveKey = wait(getActiveKey(tr, taskBucket, startingValue));
 					if (isActiveKey) {
-						TEST(true); // checkActive return true
+						CODE_PROBE(true, "checkActive return true");
 						return true;
 					}
 					break;
@@ -717,7 +714,7 @@ public:
 			}
 		}
 
-		TEST(true); // checkActive return false
+		CODE_PROBE(true, "checkActive return false");
 		return false;
 	}
 
@@ -725,7 +722,7 @@ public:
 	                                          Reference<TaskBucket> taskBucket) {
 		taskBucket->setOptions(tr);
 
-		Optional<Value> val = wait(tr->get(taskBucket->prefix.pack(LiteralStringRef("task_count"))));
+		Optional<Value> val = wait(tr->get(taskBucket->prefix.pack("task_count"_sr)));
 
 		if (!val.present())
 			return 0;
@@ -742,7 +739,7 @@ public:
 	// Returns True if any tasks were affected.
 	ACTOR static Future<bool> requeueTimedOutTasks(Reference<ReadYourWritesTransaction> tr,
 	                                               Reference<TaskBucket> taskBucket) {
-		TEST(true); // Looks for tasks that have timed out and returns them to be available tasks.
+		CODE_PROBE(true, "Looks for tasks that have timed out and returns them to be available tasks.");
 		Version end = wait(tr->getReadVersion());
 		state KeyRange range(
 		    KeyRangeRef(taskBucket->timeouts.get(0).range().begin, taskBucket->timeouts.get(end).range().end));
@@ -849,12 +846,12 @@ public:
 
 		// If we're updating the task params the clear the old space and write params to the new space
 		if (updateParams) {
-			TEST(true); // Extended a task while updating parameters
+			CODE_PROBE(true, "Extended a task while updating parameters");
 			for (auto& p : task->params) {
 				tr->set(newTimeoutSpace.pack(p.key), p.value);
 			}
 		} else {
-			TEST(true); // Extended a task without updating parameters
+			CODE_PROBE(true, "Extended a task without updating parameters");
 			// Otherwise, read and transplant the params from the old to new timeout spaces
 			RangeResult params = wait(tr->getRange(oldTimeoutSpace.range(), CLIENT_KNOBS->TOO_MANY));
 			for (auto& kv : params) {
@@ -876,10 +873,10 @@ TaskBucket::TaskBucket(const Subspace& subspace,
   : cc("TaskBucket"), dispatchSlotChecksStarted("DispatchSlotChecksStarted", cc), dispatchErrors("DispatchErrors", cc),
     dispatchDoTasks("DispatchDoTasks", cc), dispatchEmptyTasks("DispatchEmptyTasks", cc),
     dispatchSlotChecksComplete("DispatchSlotChecksComplete", cc), dbgid(deterministicRandom()->randomUniqueID()),
-    prefix(subspace), active(prefix.get(LiteralStringRef("ac"))), pauseKey(prefix.pack(LiteralStringRef("pause"))),
-    available(prefix.get(LiteralStringRef("av"))), available_prioritized(prefix.get(LiteralStringRef("avp"))),
-    timeouts(prefix.get(LiteralStringRef("to"))), timeout(CLIENT_KNOBS->TASKBUCKET_TIMEOUT_VERSIONS),
-    system_access(sysAccess), priority_batch(priorityBatch), lockAware(lockAware) {}
+    prefix(subspace), active(prefix.get("ac"_sr)), pauseKey(prefix.pack("pause"_sr)), available(prefix.get("av"_sr)),
+    available_prioritized(prefix.get("avp"_sr)), timeouts(prefix.get("to"_sr)),
+    timeout(CLIENT_KNOBS->TASKBUCKET_TIMEOUT_VERSIONS), system_access(sysAccess), priority_batch(priorityBatch),
+    lockAware(lockAware) {}
 
 TaskBucket::~TaskBucket() {}
 
@@ -922,9 +919,7 @@ Key TaskBucket::addTask(Reference<ReadYourWritesTransaction> tr, Reference<Task>
 	for (auto& param : task->params)
 		tr->set(taskSpace.pack(param.key), param.value);
 
-	tr->atomicOp(prefix.pack(LiteralStringRef("task_count")),
-	             LiteralStringRef("\x01\x00\x00\x00\x00\x00\x00\x00"),
-	             MutationRef::AddValue);
+	tr->atomicOp(prefix.pack("task_count"_sr), "\x01\x00\x00\x00\x00\x00\x00\x00"_sr, MutationRef::AddValue);
 
 	return key;
 }
@@ -996,13 +991,9 @@ Future<bool> TaskBucket::isEmpty(Reference<ReadYourWritesTransaction> tr) {
 Future<Void> TaskBucket::finish(Reference<ReadYourWritesTransaction> tr, Reference<Task> task) {
 	setOptions(tr);
 
-	Tuple t;
-	t.append(task->timeoutVersion);
-	t.append(task->key);
+	Tuple t = Tuple::makeTuple(task->timeoutVersion, task->key);
 
-	tr->atomicOp(prefix.pack(LiteralStringRef("task_count")),
-	             LiteralStringRef("\xff\xff\xff\xff\xff\xff\xff\xff"),
-	             MutationRef::AddValue);
+	tr->atomicOp(prefix.pack("task_count"_sr), "\xff\xff\xff\xff\xff\xff\xff\xff"_sr, MutationRef::AddValue);
 	tr->clear(timeouts.range(t));
 
 	return Void();
@@ -1033,7 +1024,7 @@ Future<int64_t> TaskBucket::getTaskCount(Reference<ReadYourWritesTransaction> tr
 }
 
 Future<Void> TaskBucket::watchTaskCount(Reference<ReadYourWritesTransaction> tr) {
-	return tr->watch(prefix.pack(LiteralStringRef("task_count")));
+	return tr->watch(prefix.pack("task_count"_sr));
 }
 
 Future<Void> TaskBucket::debugPrintRange(Reference<ReadYourWritesTransaction> tr, Subspace subspace, Key msg) {
@@ -1108,7 +1099,7 @@ public:
 			Key key = StringRef(deterministicRandom()->randomUniqueID().toString());
 			taskFuture->addBlock(tr, key);
 			auto task = makeReference<Task>();
-			task->params[Task::reservedTaskParamKeyType] = LiteralStringRef("UnblockFuture");
+			task->params[Task::reservedTaskParamKeyType] = "UnblockFuture"_sr;
 			task->params[Task::reservedTaskParamKeyFuture] = taskFuture->key;
 			task->params[Task::reservedTaskParamKeyBlockID] = key;
 			onSetFutures.push_back(vectorFuture[i]->onSet(tr, taskBucket, task));
@@ -1138,10 +1129,10 @@ public:
 		bool is_set = wait(isSet(tr, taskFuture));
 
 		if (is_set) {
-			TEST(true); // is_set == true
+			CODE_PROBE(true, "is_set == true");
 			wait(performAction(tr, taskBucket, taskFuture, task));
 		} else {
-			TEST(true); // is_set == false
+			CODE_PROBE(true, "is_set == false");
 			Subspace callbackSpace =
 			    taskFuture->callbacks.get(StringRef(deterministicRandom()->randomUniqueID().toString()));
 			for (auto& v : task->params) {
@@ -1222,7 +1213,7 @@ public:
 		taskFuture->futureBucket->setOptions(tr);
 
 		task->params[Task::reservedTaskParamKeyAddTask] = task->params[Task::reservedTaskParamKeyType];
-		task->params[Task::reservedTaskParamKeyType] = LiteralStringRef("AddTask");
+		task->params[Task::reservedTaskParamKeyType] = "AddTask"_sr;
 		wait(onSet(tr, taskBucket, taskFuture, task));
 
 		return Void();
@@ -1287,14 +1278,14 @@ TaskFuture::TaskFuture(const Reference<FutureBucket> bucket, Key k) : futureBuck
 	}
 
 	prefix = futureBucket->prefix.get(key);
-	blocks = prefix.get(LiteralStringRef("bl"));
-	callbacks = prefix.get(LiteralStringRef("cb"));
+	blocks = prefix.get("bl"_sr);
+	callbacks = prefix.get("cb"_sr);
 }
 
 TaskFuture::~TaskFuture() {}
 
 void TaskFuture::addBlock(Reference<ReadYourWritesTransaction> tr, StringRef block_id) {
-	tr->set(blocks.pack(block_id), LiteralStringRef(""));
+	tr->set(blocks.pack(block_id), ""_sr);
 }
 
 Future<Void> TaskFuture::set(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket) {
diff --git a/fdbclient/Tenant.cpp b/fdbclient/Tenant.cpp
index 1071059114..2ad1989fd0 100644
--- a/fdbclient/Tenant.cpp
+++ b/fdbclient/Tenant.cpp
@@ -18,47 +18,213 @@
  * limitations under the License.
  */
 
+#include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/Tenant.h"
+#include "fdbrpc/TenantInfo.h"
+#include "flow/BooleanParam.h"
+#include "flow/IRandom.h"
+#include "libb64/decode.h"
+#include "libb64/encode.h"
+#include "flow/ApiVersion.h"
 #include "flow/UnitTest.h"
 
+FDB_DEFINE_BOOLEAN_PARAM(EnforceValidTenantId);
+
 Key TenantMapEntry::idToPrefix(int64_t id) {
 	int64_t swapped = bigEndian64(id);
-	return StringRef(reinterpret_cast<const uint8_t*>(&swapped), 8);
+	return StringRef(reinterpret_cast<const uint8_t*>(&swapped), TENANT_PREFIX_SIZE);
 }
 
-int64_t TenantMapEntry::prefixToId(KeyRef prefix) {
-	ASSERT(prefix.size() == 8);
+int64_t TenantMapEntry::prefixToId(KeyRef prefix, EnforceValidTenantId enforceValidTenantId) {
+	ASSERT(prefix.size() == TENANT_PREFIX_SIZE);
 	int64_t id = *reinterpret_cast<const int64_t*>(prefix.begin());
 	id = bigEndian64(id);
-	ASSERT(id >= 0);
+	if (enforceValidTenantId) {
+		ASSERT(id >= 0);
+	} else if (id < 0) {
+		return TenantInfo::INVALID_TENANT;
+	}
 	return id;
 }
 
-void TenantMapEntry::initPrefix(KeyRef subspace) {
-	ASSERT(id >= 0);
-	prefix = makeString(8 + subspace.size());
-	uint8_t* data = mutateString(prefix);
-	if (subspace.size() > 0) {
-		memcpy(data, subspace.begin(), subspace.size());
+std::string TenantMapEntry::tenantStateToString(TenantState tenantState) {
+	switch (tenantState) {
+	case TenantState::REGISTERING:
+		return "registering";
+	case TenantState::READY:
+		return "ready";
+	case TenantState::REMOVING:
+		return "removing";
+	case TenantState::UPDATING_CONFIGURATION:
+		return "updating configuration";
+	case TenantState::RENAMING_FROM:
+		return "renaming from";
+	case TenantState::RENAMING_TO:
+		return "renaming to";
+	case TenantState::ERROR:
+		return "error";
+	default:
+		UNREACHABLE();
 	}
-	int64_t swapped = bigEndian64(id);
-	memcpy(data + subspace.size(), &swapped, 8);
 }
 
-TenantMapEntry::TenantMapEntry() : id(-1) {}
-TenantMapEntry::TenantMapEntry(int64_t id, KeyRef subspace) : id(id) {
-	initPrefix(subspace);
+TenantState TenantMapEntry::stringToTenantState(std::string stateStr) {
+	if (stateStr == "registering") {
+		return TenantState::REGISTERING;
+	} else if (stateStr == "ready") {
+		return TenantState::READY;
+	} else if (stateStr == "removing") {
+		return TenantState::REMOVING;
+	} else if (stateStr == "updating configuration") {
+		return TenantState::UPDATING_CONFIGURATION;
+	} else if (stateStr == "renaming from") {
+		return TenantState::RENAMING_FROM;
+	} else if (stateStr == "renaming to") {
+		return TenantState::RENAMING_TO;
+	} else if (stateStr == "error") {
+		return TenantState::ERROR;
+	}
+
+	UNREACHABLE();
+}
+
+std::string TenantMapEntry::tenantLockStateToString(TenantLockState tenantState) {
+	switch (tenantState) {
+	case TenantLockState::UNLOCKED:
+		return "unlocked";
+	case TenantLockState::READ_ONLY:
+		return "read only";
+	case TenantLockState::LOCKED:
+		return "locked";
+	default:
+		UNREACHABLE();
+	}
+}
+
+TenantLockState TenantMapEntry::stringToTenantLockState(std::string stateStr) {
+	if (stateStr == "unlocked") {
+		return TenantLockState::UNLOCKED;
+	} else if (stateStr == "read only") {
+		return TenantLockState::READ_ONLY;
+	} else if (stateStr == "locked") {
+		return TenantLockState::LOCKED;
+	}
+
+	UNREACHABLE();
+}
+
+TenantMapEntry::TenantMapEntry() {}
+TenantMapEntry::TenantMapEntry(int64_t id, TenantState tenantState, bool encrypted)
+  : tenantState(tenantState), encrypted(encrypted) {
+	setId(id);
+}
+TenantMapEntry::TenantMapEntry(int64_t id,
+                               TenantState tenantState,
+                               Optional<TenantGroupName> tenantGroup,
+                               bool encrypted)
+  : tenantState(tenantState), tenantGroup(tenantGroup), encrypted(encrypted) {
+	setId(id);
+}
+
+void TenantMapEntry::setId(int64_t id) {
+	ASSERT(id >= 0);
+	this->id = id;
+	prefix = idToPrefix(id);
+}
+
+std::string TenantMapEntry::toJson() const {
+	json_spirit::mObject tenantEntry;
+	tenantEntry["id"] = id;
+	tenantEntry["encrypted"] = encrypted;
+
+	json_spirit::mObject prefixObject;
+	std::string encodedPrefix = base64::encoder::from_string(prefix.toString());
+	// Remove trailing newline
+	encodedPrefix.resize(encodedPrefix.size() - 1);
+
+	prefixObject["base64"] = encodedPrefix;
+	prefixObject["printable"] = printable(prefix);
+	tenantEntry["prefix"] = prefixObject;
+
+	tenantEntry["tenant_state"] = TenantMapEntry::tenantStateToString(tenantState);
+	if (assignedCluster.present()) {
+		tenantEntry["assigned_cluster"] = assignedCluster.get().toString();
+	}
+	if (tenantGroup.present()) {
+		json_spirit::mObject tenantGroupObject;
+		std::string encodedTenantGroup = base64::encoder::from_string(tenantGroup.get().toString());
+		// Remove trailing newline
+		encodedTenantGroup.resize(encodedTenantGroup.size() - 1);
+
+		tenantGroupObject["base64"] = encodedTenantGroup;
+		tenantGroupObject["printable"] = printable(tenantGroup.get());
+		tenantEntry["tenant_group"] = tenantGroupObject;
+	}
+
+	return json_spirit::write_string(json_spirit::mValue(tenantEntry));
+}
+
+bool TenantMapEntry::matchesConfiguration(TenantMapEntry const& other) const {
+	return tenantGroup == other.tenantGroup && encrypted == other.encrypted;
+}
+
+void TenantMapEntry::configure(Standalone<StringRef> parameter, Optional<Value> value) {
+	if (parameter == "tenant_group"_sr) {
+		tenantGroup = value;
+	} else if (parameter == "assigned_cluster"_sr) {
+		assignedCluster = value;
+	} else {
+		TraceEvent(SevWarnAlways, "UnknownTenantConfigurationParameter").detail("Parameter", parameter);
+		throw invalid_tenant_configuration();
+	}
+}
+
+json_spirit::mObject TenantGroupEntry::toJson() const {
+	json_spirit::mObject tenantGroupEntry;
+	if (assignedCluster.present()) {
+		tenantGroupEntry["assigned_cluster"] = assignedCluster.get().toString();
+	}
+
+	return tenantGroupEntry;
+}
+
+TenantMetadataSpecification& TenantMetadata::instance() {
+	static TenantMetadataSpecification _instance = TenantMetadataSpecification("\xff/"_sr);
+	return _instance;
+}
+
+Key TenantMetadata::tenantMapPrivatePrefix() {
+	static Key _prefix = "\xff"_sr.withSuffix(tenantMap().subspace.begin);
+	return _prefix;
+}
+
+TEST_CASE("/fdbclient/libb64/base64decoder") {
+	Standalone<StringRef> buf = makeString(100);
+	for (int i = 0; i < 1000; ++i) {
+		int length = deterministicRandom()->randomInt(0, 100);
+		deterministicRandom()->randomBytes(mutateString(buf), length);
+
+		StringRef str = buf.substr(0, length);
+		std::string encodedStr = base64::encoder::from_string(str.toString());
+		// Remove trailing newline
+		encodedStr.resize(encodedStr.size() - 1);
+
+		std::string decodedStr = base64::decoder::from_string(encodedStr);
+		ASSERT(decodedStr == str.toString());
+	}
+
+	return Void();
 }
 
 TEST_CASE("/fdbclient/TenantMapEntry/Serialization") {
-	TenantMapEntry entry1(1, ""_sr);
+	TenantMapEntry entry1(1, TenantState::READY, false);
 	ASSERT(entry1.prefix == "\x00\x00\x00\x00\x00\x00\x00\x01"_sr);
 	TenantMapEntry entry2 = TenantMapEntry::decode(entry1.encode());
 	ASSERT(entry1.id == entry2.id && entry1.prefix == entry2.prefix);
 
-	TenantMapEntry entry3(std::numeric_limits<int64_t>::max(), "foo"_sr);
-	ASSERT(entry3.prefix == "foo\x7f\xff\xff\xff\xff\xff\xff\xff"_sr);
+	TenantMapEntry entry3(std::numeric_limits<int64_t>::max(), TenantState::READY, false);
+	ASSERT(entry3.prefix == "\x7f\xff\xff\xff\xff\xff\xff\xff"_sr);
 	TenantMapEntry entry4 = TenantMapEntry::decode(entry3.encode());
 	ASSERT(entry3.id == entry4.id && entry3.prefix == entry4.prefix);
 
@@ -68,15 +234,9 @@ TEST_CASE("/fdbclient/TenantMapEntry/Serialization") {
 		int64_t maxPlusOne = std::min<uint64_t>(UINT64_C(1) << bits, std::numeric_limits<int64_t>::max());
 		int64_t id = deterministicRandom()->randomInt64(min, maxPlusOne);
 
-		int subspaceLength = deterministicRandom()->randomInt(0, 20);
-		Standalone<StringRef> subspace = makeString(subspaceLength);
-		generateRandomData(mutateString(subspace), subspaceLength);
-
-		TenantMapEntry entry(id, subspace);
+		TenantMapEntry entry(id, TenantState::READY, false);
 		int64_t bigEndianId = bigEndian64(id);
-		ASSERT(entry.id == id && entry.prefix.startsWith(subspace) &&
-		       entry.prefix.endsWith(StringRef(reinterpret_cast<uint8_t*>(&bigEndianId), 8)) &&
-		       entry.prefix.size() == subspaceLength + 8);
+		ASSERT(entry.id == id && entry.prefix == StringRef(reinterpret_cast<uint8_t*>(&bigEndianId), 8));
 
 		TenantMapEntry decodedEntry = TenantMapEntry::decode(entry.encode());
 		ASSERT(decodedEntry.id == entry.id && decodedEntry.prefix == entry.prefix);
diff --git a/fdbclient/TenantManagement.actor.cpp b/fdbclient/TenantManagement.actor.cpp
new file mode 100644
index 0000000000..608da5c690
--- /dev/null
+++ b/fdbclient/TenantManagement.actor.cpp
@@ -0,0 +1,40 @@
+/*
+ * TenantManagement.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+#include <map>
+#include "fdbclient/SystemData.h"
+#include "fdbclient/TenantManagement.actor.h"
+#include "fdbclient/Tuple.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+namespace TenantAPI {
+
+TenantMode tenantModeForClusterType(ClusterType clusterType, TenantMode tenantMode) {
+	if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+		return TenantMode::DISABLED;
+	} else if (clusterType == ClusterType::METACLUSTER_DATA) {
+		return TenantMode::REQUIRED;
+	} else {
+		return tenantMode;
+	}
+}
+
+} // namespace TenantAPI
diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp
index 1dc5357572..eec04e46e3 100644
--- a/fdbclient/ThreadSafeTransaction.cpp
+++ b/fdbclient/ThreadSafeTransaction.cpp
@@ -21,6 +21,7 @@
 #include "fdbclient/BlobGranuleFiles.h"
 #include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
+#include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/ThreadSafeTransaction.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/versions.h"
@@ -54,7 +55,7 @@ Reference<ITenant> ThreadSafeDatabase::openTenant(TenantNameRef tenantName) {
 }
 
 Reference<ITransaction> ThreadSafeDatabase::createTransaction() {
-	auto type = isConfigDB ? ISingleThreadTransaction::Type::SIMPLE_CONFIG : ISingleThreadTransaction::Type::RYW;
+	auto type = isConfigDB ? ISingleThreadTransaction::Type::PAXOS_CONFIG : ISingleThreadTransaction::Type::RYW;
 	return Reference<ITransaction>(new ThreadSafeTransaction(db, type, Optional<TenantName>()));
 }
 
@@ -87,6 +88,7 @@ ThreadFuture<int64_t> ThreadSafeDatabase::rebootWorker(const StringRef& address,
 	DatabaseContext* db = this->db;
 	Key addressKey = address;
 	return onMainThread([db, addressKey, check, duration]() -> Future<int64_t> {
+		db->checkDeferredError();
 		return db->rebootWorker(addressKey, check, duration);
 	});
 }
@@ -94,14 +96,20 @@ ThreadFuture<int64_t> ThreadSafeDatabase::rebootWorker(const StringRef& address,
 ThreadFuture<Void> ThreadSafeDatabase::forceRecoveryWithDataLoss(const StringRef& dcid) {
 	DatabaseContext* db = this->db;
 	Key dcidKey = dcid;
-	return onMainThread([db, dcidKey]() -> Future<Void> { return db->forceRecoveryWithDataLoss(dcidKey); });
+	return onMainThread([db, dcidKey]() -> Future<Void> {
+		db->checkDeferredError();
+		return db->forceRecoveryWithDataLoss(dcidKey);
+	});
 }
 
 ThreadFuture<Void> ThreadSafeDatabase::createSnapshot(const StringRef& uid, const StringRef& snapshot_command) {
 	DatabaseContext* db = this->db;
 	Key snapUID = uid;
 	Key cmd = snapshot_command;
-	return onMainThread([db, snapUID, cmd]() -> Future<Void> { return db->createSnapshot(snapUID, cmd); });
+	return onMainThread([db, snapUID, cmd]() -> Future<Void> {
+		db->checkDeferredError();
+		return db->createSnapshot(snapUID, cmd);
+	});
 }
 
 ThreadFuture<DatabaseSharedState*> ThreadSafeDatabase::createSharedState() {
@@ -125,14 +133,17 @@ double ThreadSafeDatabase::getMainThreadBusyness() {
 // Note: this will never return if the server is running a protocol from FDB 5.0 or older
 ThreadFuture<ProtocolVersion> ThreadSafeDatabase::getServerProtocol(Optional<ProtocolVersion> expectedVersion) {
 	DatabaseContext* db = this->db;
-	return onMainThread(
-	    [db, expectedVersion]() -> Future<ProtocolVersion> { return db->getClusterProtocol(expectedVersion); });
+	return onMainThread([db, expectedVersion]() -> Future<ProtocolVersion> {
+		db->checkDeferredError();
+		return db->getClusterProtocol(expectedVersion);
+	});
 }
 
 ThreadFuture<Key> ThreadSafeDatabase::purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) {
 	DatabaseContext* db = this->db;
 	KeyRange range = keyRange;
 	return onMainThread([db, range, purgeVersion, force]() -> Future<Key> {
+		db->checkDeferredError();
 		return db->purgeBlobGranules(range, purgeVersion, {}, force);
 	});
 }
@@ -140,16 +151,64 @@ ThreadFuture<Key> ThreadSafeDatabase::purgeBlobGranules(const KeyRangeRef& keyRa
 ThreadFuture<Void> ThreadSafeDatabase::waitPurgeGranulesComplete(const KeyRef& purgeKey) {
 	DatabaseContext* db = this->db;
 	Key key = purgeKey;
-	return onMainThread([db, key]() -> Future<Void> { return db->waitPurgeGranulesComplete(key); });
+	return onMainThread([db, key]() -> Future<Void> {
+		db->checkDeferredError();
+		return db->waitPurgeGranulesComplete(key);
+	});
 }
 
-ThreadSafeDatabase::ThreadSafeDatabase(Reference<IClusterConnectionRecord> connectionRecord, int apiVersion) {
+ThreadFuture<bool> ThreadSafeDatabase::blobbifyRange(const KeyRangeRef& keyRange) {
+	DatabaseContext* db = this->db;
+	KeyRange range = keyRange;
+	return onMainThread([=]() -> Future<bool> {
+		db->checkDeferredError();
+		return db->blobbifyRange(range);
+	});
+}
+
+ThreadFuture<bool> ThreadSafeDatabase::unblobbifyRange(const KeyRangeRef& keyRange) {
+	DatabaseContext* db = this->db;
+	KeyRange range = keyRange;
+	return onMainThread([=]() -> Future<bool> {
+		db->checkDeferredError();
+		return db->unblobbifyRange(range);
+	});
+}
+
+ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> ThreadSafeDatabase::listBlobbifiedRanges(const KeyRangeRef& keyRange,
+                                                                                          int rangeLimit) {
+	DatabaseContext* db = this->db;
+	KeyRange range = keyRange;
+	return onMainThread([=]() -> Future<Standalone<VectorRef<KeyRangeRef>>> {
+		db->checkDeferredError();
+		return db->listBlobbifiedRanges(range, rangeLimit);
+	});
+}
+
+ThreadFuture<Version> ThreadSafeDatabase::verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) {
+	DatabaseContext* db = this->db;
+	KeyRange range = keyRange;
+	return onMainThread([=]() -> Future<Version> {
+		db->checkDeferredError();
+		return db->verifyBlobRange(range, version);
+	});
+}
+
+ThreadSafeDatabase::ThreadSafeDatabase(ConnectionRecordType connectionRecordType,
+                                       std::string connectionRecordString,
+                                       int apiVersion) {
 	// Allocate memory for the Database from this thread (so the pointer is known for subsequent method calls)
 	// but run its constructor on the main thread
 	DatabaseContext* db = this->db = DatabaseContext::allocateOnForeignThread();
 
-	onMainThreadVoid([db, connectionRecord, apiVersion]() {
+	onMainThreadVoid([db, connectionRecordType, connectionRecordString, apiVersion]() {
 		try {
+			Reference<IClusterConnectionRecord> connectionRecord =
+			    connectionRecordType == ConnectionRecordType::FILE
+			        ? Reference<IClusterConnectionRecord>(ClusterConnectionFile::openOrDefault(connectionRecordString))
+			        : Reference<IClusterConnectionRecord>(
+			              new ClusterConnectionMemoryRecord(ClusterConnectionString(connectionRecordString)));
+
 			Database::createDatabase(connectionRecord, apiVersion, IsInternal::False, LocalityData(), db).extractPtr();
 		} catch (Error& e) {
 			new (db) DatabaseContext(e);
@@ -165,7 +224,7 @@ ThreadSafeDatabase::~ThreadSafeDatabase() {
 }
 
 Reference<ITransaction> ThreadSafeTenant::createTransaction() {
-	auto type = db->isConfigDB ? ISingleThreadTransaction::Type::SIMPLE_CONFIG : ISingleThreadTransaction::Type::RYW;
+	auto type = db->isConfigDB ? ISingleThreadTransaction::Type::PAXOS_CONFIG : ISingleThreadTransaction::Type::RYW;
 	return Reference<ITransaction>(new ThreadSafeTransaction(db->db, type, name));
 }
 
@@ -181,7 +240,10 @@ ThreadFuture<Key> ThreadSafeTenant::purgeBlobGranules(const KeyRangeRef& keyRang
 ThreadFuture<Void> ThreadSafeTenant::waitPurgeGranulesComplete(const KeyRef& purgeKey) {
 	DatabaseContext* db = this->db->db;
 	Key key = purgeKey;
-	return onMainThread([db, key]() -> Future<Void> { return db->waitPurgeGranulesComplete(key); });
+	return onMainThread([db, key]() -> Future<Void> {
+		db->checkDeferredError();
+		return db->waitPurgeGranulesComplete(key);
+	});
 }
 
 ThreadSafeTenant::~ThreadSafeTenant() {}
@@ -350,13 +412,14 @@ ThreadFuture<Standalone<VectorRef<const char*>>> ThreadSafeTransaction::getAddre
 }
 
 ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> ThreadSafeTransaction::getBlobGranuleRanges(
-    const KeyRangeRef& keyRange) {
+    const KeyRangeRef& keyRange,
+    int rangeLimit) {
 	ISingleThreadTransaction* tr = this->tr;
 	KeyRange r = keyRange;
 
-	return onMainThread([tr, r]() -> Future<Standalone<VectorRef<KeyRangeRef>>> {
+	return onMainThread([=]() -> Future<Standalone<VectorRef<KeyRangeRef>>> {
 		tr->checkDeferredError();
-		return tr->getBlobGranuleRanges(r);
+		return tr->getBlobGranuleRanges(r, rangeLimit);
 	});
 }
 
@@ -364,34 +427,53 @@ ThreadResult<RangeResult> ThreadSafeTransaction::readBlobGranules(const KeyRange
                                                                   Version beginVersion,
                                                                   Optional<Version> readVersion,
                                                                   ReadBlobGranuleContext granule_context) {
-	// FIXME: prevent from calling this from another main thread!
+	// This should not be called directly, bypassMultiversionApi should not be set
+	return ThreadResult<RangeResult>(unsupported_operation());
+}
 
+ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> ThreadSafeTransaction::readBlobGranulesStart(
+    const KeyRangeRef& keyRange,
+    Version beginVersion,
+    Optional<Version> readVersion,
+    Version* readVersionOut) {
 	ISingleThreadTransaction* tr = this->tr;
 	KeyRange r = keyRange;
 
-	int64_t readVersionOut;
-	ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> getFilesFuture = onMainThread(
-	    [tr, r, beginVersion, readVersion, &readVersionOut]() -> Future<Standalone<VectorRef<BlobGranuleChunkRef>>> {
+	return onMainThread(
+	    [tr, r, beginVersion, readVersion, readVersionOut]() -> Future<Standalone<VectorRef<BlobGranuleChunkRef>>> {
 		    tr->checkDeferredError();
-		    return tr->readBlobGranules(r, beginVersion, readVersion, &readVersionOut);
+		    return tr->readBlobGranules(r, beginVersion, readVersion, readVersionOut);
 	    });
+}
 
-	// FIXME: can this safely avoid another main thread jump?
-	getFilesFuture.blockUntilReadyCheckOnMainThread();
-
-	// propagate error to client
-	if (getFilesFuture.isError()) {
-		return ThreadResult<RangeResult>(getFilesFuture.getError());
-	}
-
-	Standalone<VectorRef<BlobGranuleChunkRef>> files = getFilesFuture.get();
-
+ThreadResult<RangeResult> ThreadSafeTransaction::readBlobGranulesFinish(
+    ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> startFuture,
+    const KeyRangeRef& keyRange,
+    Version beginVersion,
+    Version readVersion,
+    ReadBlobGranuleContext granuleContext) {
 	// do this work off of fdb network threads for performance!
-	if (granule_context.debugNoMaterialize) {
-		return ThreadResult<RangeResult>(blob_granule_not_materialized());
-	} else {
-		return loadAndMaterializeBlobGranules(files, keyRange, beginVersion, readVersionOut, granule_context);
+	Standalone<VectorRef<BlobGranuleChunkRef>> files = startFuture.get();
+	GranuleMaterializeStats stats;
+	auto ret = loadAndMaterializeBlobGranules(files, keyRange, beginVersion, readVersion, granuleContext, stats);
+	if (!ret.isError()) {
+		ISingleThreadTransaction* tr = this->tr;
+		onMainThreadVoid([tr, stats]() { tr->addGranuleMaterializeStats(stats); });
 	}
+	return ret;
+}
+
+ThreadFuture<Standalone<VectorRef<BlobGranuleSummaryRef>>> ThreadSafeTransaction::summarizeBlobGranules(
+    const KeyRangeRef& keyRange,
+    Optional<Version> summaryVersion,
+    int rangeLimit) {
+	ISingleThreadTransaction* tr = this->tr;
+	KeyRange r = keyRange;
+
+	return onMainThread([=]() -> Future<Standalone<VectorRef<BlobGranuleSummaryRef>>> {
+		tr->checkDeferredError();
+		return tr->summarizeBlobGranules(r, summaryVersion, rangeLimit);
+	});
 }
 
 void ThreadSafeTransaction::addReadConflictRange(const KeyRangeRef& keys) {
@@ -489,22 +571,34 @@ Version ThreadSafeTransaction::getCommittedVersion() {
 
 ThreadFuture<VersionVector> ThreadSafeTransaction::getVersionVector() {
 	ISingleThreadTransaction* tr = this->tr;
-	return onMainThread([tr]() -> Future<VersionVector> { return tr->getVersionVector(); });
+	return onMainThread([tr]() -> Future<VersionVector> {
+		tr->checkDeferredError();
+		return tr->getVersionVector();
+	});
 }
 
 ThreadFuture<SpanContext> ThreadSafeTransaction::getSpanContext() {
 	ISingleThreadTransaction* tr = this->tr;
-	return onMainThread([tr]() -> Future<SpanContext> { return tr->getSpanContext(); });
+	return onMainThread([tr]() -> Future<SpanContext> {
+		tr->checkDeferredError();
+		return tr->getSpanContext();
+	});
 }
 
 ThreadFuture<int64_t> ThreadSafeTransaction::getApproximateSize() {
 	ISingleThreadTransaction* tr = this->tr;
-	return onMainThread([tr]() -> Future<int64_t> { return tr->getApproximateSize(); });
+	return onMainThread([tr]() -> Future<int64_t> {
+		tr->checkDeferredError();
+		return tr->getApproximateSize();
+	});
 }
 
 ThreadFuture<Standalone<StringRef>> ThreadSafeTransaction::getVersionstamp() {
 	ISingleThreadTransaction* tr = this->tr;
-	return onMainThread([tr]() -> Future<Standalone<StringRef>> { return tr->getVersionstamp(); });
+	return onMainThread([tr]() -> Future<Standalone<StringRef>> {
+		tr->checkDeferredError();
+		return tr->getVersionstamp();
+	});
 }
 
 void ThreadSafeTransaction::setOption(FDBTransactionOptions::Option option, Optional<StringRef> value) {
@@ -563,19 +657,25 @@ void ThreadSafeTransaction::reset() {
 
 extern const char* getSourceVersion();
 
-ThreadSafeApi::ThreadSafeApi()
-  : apiVersion(-1), clientVersion(format("%s,%s,%llx", FDB_VT_VERSION, getSourceVersion(), currentProtocolVersion)),
-    transportId(0) {}
+ThreadSafeApi::ThreadSafeApi() : apiVersion(-1), transportId(0) {}
 
 void ThreadSafeApi::selectApiVersion(int apiVersion) {
-	this->apiVersion = apiVersion;
+	this->apiVersion = ApiVersion(apiVersion);
 }
 
 const char* ThreadSafeApi::getClientVersion() {
-	// There is only one copy of the ThreadSafeAPI, and it never gets deleted. Also, clientVersion is never modified.
+	// There is only one copy of the ThreadSafeAPI, and it never gets deleted.
+	// Also, clientVersion is initialized on demand and never modified afterwards.
+	if (clientVersion.empty()) {
+		clientVersion = format("%s,%s,%llx", FDB_VT_VERSION, getSourceVersion(), currentProtocolVersion());
+	}
 	return clientVersion.c_str();
 }
 
+void ThreadSafeApi::useFutureProtocolVersion() {
+	::useFutureProtocolVersion();
+}
+
 void ThreadSafeApi::setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value) {
 	if (option == FDBNetworkOptions::EXTERNAL_CLIENT_TRANSPORT_ID) {
 		if (value.present()) {
@@ -632,12 +732,12 @@ void ThreadSafeApi::stopNetwork() {
 
 Reference<IDatabase> ThreadSafeApi::createDatabase(const char* clusterFilePath) {
 	return Reference<IDatabase>(
-	    new ThreadSafeDatabase(ClusterConnectionFile::openOrDefault(clusterFilePath), apiVersion));
+	    new ThreadSafeDatabase(ThreadSafeDatabase::ConnectionRecordType::FILE, clusterFilePath, apiVersion.version()));
 }
 
 Reference<IDatabase> ThreadSafeApi::createDatabaseFromConnectionString(const char* connectionString) {
 	return Reference<IDatabase>(new ThreadSafeDatabase(
-	    makeReference<ClusterConnectionMemoryRecord>(ClusterConnectionString(connectionString)), apiVersion));
+	    ThreadSafeDatabase::ConnectionRecordType::CONNECTION_STRING, connectionString, apiVersion.version()));
 }
 
 void ThreadSafeApi::addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) {
diff --git a/fdbclient/Tracing.actor.cpp b/fdbclient/Tracing.actor.cpp
index 815e568acb..3100b208fc 100644
--- a/fdbclient/Tracing.actor.cpp
+++ b/fdbclient/Tracing.actor.cpp
@@ -497,7 +497,7 @@ TEST_CASE("/flow/Tracing/AddEvents") {
 	auto arena = span1.arena;
 	SmallVectorRef<KeyValueRef> attrs;
 	attrs.push_back(arena, KeyValueRef("foo"_sr, "bar"_sr));
-	span1.addEvent(LiteralStringRef("read_version"), 1.0, attrs);
+	span1.addEvent("read_version"_sr, 1.0, attrs);
 	ASSERT(span1.events[0].name.toString() == "read_version");
 	ASSERT(span1.events[0].time == 1.0);
 	ASSERT(span1.events[0].attributes.begin()->key.toString() == "foo");
@@ -505,7 +505,7 @@ TEST_CASE("/flow/Tracing/AddEvents") {
 
 	// Use helper method to add an OTELEventRef with no attributes to an OTELSpan
 	Span span2("span_with_event"_loc);
-	span2.addEvent(StringRef(span2.arena, LiteralStringRef("commit_succeed")), 1234567.100);
+	span2.addEvent(StringRef(span2.arena, "commit_succeed"_sr), 1234567.100);
 	ASSERT(span2.events[0].name.toString() == "commit_succeed");
 	ASSERT(span2.events[0].time == 1234567.100);
 	ASSERT(span2.events[0].attributes.size() == 0);
@@ -537,8 +537,8 @@ TEST_CASE("/flow/Tracing/AddAttributes") {
 	IKnobCollection::getMutableGlobalKnobCollection().setKnob("tracing_span_attributes_enabled",
 	                                                          KnobValueRef::create(bool{ true }));
 	auto arena = span1.arena;
-	span1.addAttribute(StringRef(arena, LiteralStringRef("foo")), StringRef(arena, LiteralStringRef("bar")));
-	span1.addAttribute(StringRef(arena, LiteralStringRef("operation")), StringRef(arena, LiteralStringRef("grv")));
+	span1.addAttribute(StringRef(arena, "foo"_sr), StringRef(arena, "bar"_sr));
+	span1.addAttribute(StringRef(arena, "operation"_sr), StringRef(arena, "grv"_sr));
 	ASSERT_EQ(span1.attributes.size(), 3); // Includes default attribute of "address"
 	ASSERT(span1.attributes[1] == KeyValueRef("foo"_sr, "bar"_sr));
 	ASSERT(span1.attributes[2] == KeyValueRef("operation"_sr, "grv"_sr));
@@ -548,9 +548,9 @@ TEST_CASE("/flow/Tracing/AddAttributes") {
 	                       deterministicRandom()->randomUInt64(),
 	                       TraceFlags::sampled));
 	auto s2Arena = span2.arena;
-	span2.addAttribute(StringRef(s2Arena, LiteralStringRef("a")), StringRef(s2Arena, LiteralStringRef("1")))
-	    .addAttribute(StringRef(s2Arena, LiteralStringRef("b")), LiteralStringRef("2"))
-	    .addAttribute(StringRef(s2Arena, LiteralStringRef("c")), LiteralStringRef("3"));
+	span2.addAttribute(StringRef(s2Arena, "a"_sr), StringRef(s2Arena, "1"_sr))
+	    .addAttribute(StringRef(s2Arena, "b"_sr), "2"_sr)
+	    .addAttribute(StringRef(s2Arena, "c"_sr), "3"_sr);
 
 	ASSERT_EQ(span2.attributes.size(), 4); // Includes default attribute of "address"
 	ASSERT(span2.attributes[1] == KeyValueRef("a"_sr, "1"_sr));
@@ -718,7 +718,7 @@ TEST_CASE("/flow/Tracing/FastUDPMessagePackEncoding") {
 	attrs.push_back(s3Arena, KeyValueRef("foo"_sr, "bar"_sr));
 	span3.addAttribute("operation"_sr, "grv"_sr)
 	    .addLink(SpanContext(UID(300, 301), 400, TraceFlags::sampled))
-	    .addEvent(StringRef(s3Arena, LiteralStringRef("event1")), 100.101, attrs);
+	    .addEvent(StringRef(s3Arena, "event1"_sr), 100.101, attrs);
 	tracer.serialize_span(span3, request);
 	data = request.buffer.get();
 	ASSERT(data[0] == 0b10011100); // 12 element array.
diff --git a/fdbclient/Tuple.cpp b/fdbclient/Tuple.cpp
index 729c910252..212223abd2 100644
--- a/fdbclient/Tuple.cpp
+++ b/fdbclient/Tuple.cpp
@@ -19,8 +19,11 @@
  */
 
 #include "fdbclient/Tuple.h"
+#include "flow/UnitTest.h"
 
 const uint8_t VERSIONSTAMP_96_CODE = 0x33;
+const uint8_t USER_TYPE_START = 0x40;
+const uint8_t USER_TYPE_END = 0x4f;
 
 // TODO: Many functions copied from bindings/flow/Tuple.cpp. Merge at some point.
 static float bigEndianFloat(float orig) {
@@ -58,7 +61,7 @@ static void adjustFloatingPoint(uint8_t* bytes, size_t size, bool encode) {
 	}
 }
 
-Tuple::Tuple(StringRef const& str, bool exclude_incomplete) {
+Tuple::Tuple(StringRef const& str, bool exclude_incomplete, bool include_user_type) {
 	data.append(data.arena(), str.begin(), str.size());
 
 	size_t i = 0;
@@ -79,6 +82,9 @@ Tuple::Tuple(StringRef const& str, bool exclude_incomplete) {
 			i += 1;
 		} else if (data[i] == VERSIONSTAMP_96_CODE) {
 			i += VERSIONSTAMP_TUPLE_SIZE + 1;
+		} else if (include_user_type && isUserType(data[i])) {
+			// User defined codes must come at the end of a Tuple and are not delimited.
+			i = data.size();
 		} else {
 			throw invalid_tuple_data_type();
 		}
@@ -93,6 +99,14 @@ Tuple Tuple::unpack(StringRef const& str, bool exclude_incomplete) {
 	return Tuple(str, exclude_incomplete);
 }
 
+Tuple Tuple::unpackUserType(StringRef const& str, bool exclude_incomplete) {
+	return Tuple(str, exclude_incomplete, true);
+}
+
+bool Tuple::isUserType(uint8_t code) const {
+	return code >= USER_TYPE_START && code <= USER_TYPE_END;
+}
+
 Tuple& Tuple::append(Tuple const& tuple) {
 	for (size_t offset : tuple.offsets) {
 		offsets.push_back(offset + data.size());
@@ -103,7 +117,7 @@ Tuple& Tuple::append(Tuple const& tuple) {
 	return *this;
 }
 
-Tuple& Tuple::appendVersionstamp(Versionstamp const& vs) {
+Tuple& Tuple::append(Versionstamp const& vs) {
 	offsets.push_back(data.size());
 
 	data.push_back(data.arena(), VERSIONSTAMP_96_CODE);
@@ -134,6 +148,10 @@ Tuple& Tuple::append(StringRef const& str, bool utf8) {
 	return *this;
 }
 
+Tuple& Tuple::append(UnicodeStr const& str) {
+	return append(str.str, true);
+}
+
 Tuple& Tuple::appendRaw(StringRef const& str) {
 	offsets.push_back(data.size());
 
@@ -166,7 +184,11 @@ Tuple& Tuple::append(int64_t value) {
 	return *this;
 }
 
-Tuple& Tuple::appendBool(bool value) {
+Tuple& Tuple::append(int32_t value) {
+	return append((int64_t)value);
+}
+
+Tuple& Tuple::append(bool value) {
 	offsets.push_back(data.size());
 	if (value) {
 		data.push_back(data.arena(), 0x27);
@@ -176,7 +198,7 @@ Tuple& Tuple::appendBool(bool value) {
 	return *this;
 }
 
-Tuple& Tuple::appendFloat(float value) {
+Tuple& Tuple::append(float value) {
 	offsets.push_back(data.size());
 	float swap = bigEndianFloat(value);
 	uint8_t* bytes = (uint8_t*)&swap;
@@ -187,7 +209,7 @@ Tuple& Tuple::appendFloat(float value) {
 	return *this;
 }
 
-Tuple& Tuple::appendDouble(double value) {
+Tuple& Tuple::append(double value) {
 	offsets.push_back(data.size());
 	double swap = value;
 	swap = bigEndianDouble(swap);
@@ -199,12 +221,25 @@ Tuple& Tuple::appendDouble(double value) {
 	return *this;
 }
 
-Tuple& Tuple::appendNull() {
+Tuple& Tuple::append(std::nullptr_t) {
 	offsets.push_back(data.size());
 	data.push_back(data.arena(), (uint8_t)'\x00');
 	return *this;
 }
 
+Tuple& Tuple::appendNull() {
+	return append(nullptr);
+}
+
+Tuple& Tuple::append(Tuple::UserTypeStr const& udt) {
+	offsets.push_back(data.size());
+	ASSERT(isUserType(udt.code));
+	data.push_back(data.arena(), udt.code);
+	data.append(data.arena(), udt.str.begin(), udt.str.size());
+
+	return *this;
+}
+
 Tuple::ElementType Tuple::getType(size_t index) const {
 	if (index >= offsets.size()) {
 		throw invalid_tuple_index();
@@ -228,6 +263,8 @@ Tuple::ElementType Tuple::getType(size_t index) const {
 		return ElementType::BOOL;
 	} else if (code == VERSIONSTAMP_96_CODE) {
 		return ElementType::VERSIONSTAMP;
+	} else if (isUserType(code)) {
+		return ElementType::USER_TYPE;
 	} else {
 		throw invalid_tuple_data_type();
 	}
@@ -388,6 +425,29 @@ Versionstamp Tuple::getVersionstamp(size_t index) const {
 	return Versionstamp(StringRef(data.begin() + offsets[index] + 1, VERSIONSTAMP_TUPLE_SIZE));
 }
 
+Tuple::UserTypeStr Tuple::getUserType(size_t index) const {
+	// Valid index.
+	if (index >= offsets.size()) {
+		throw invalid_tuple_index();
+	}
+
+	// Valid user type code.
+	ASSERT_LT(offsets[index], data.size());
+	uint8_t code = data[offsets[index]];
+	if (!isUserType(code)) {
+		throw invalid_tuple_data_type();
+	}
+
+	size_t start = offsets[index] + 1;
+
+	Standalone<StringRef> str;
+	VectorRef<uint8_t> staging;
+	staging.append(str.arena(), data.begin() + start, data.size() - start);
+	str.StringRef::operator=(StringRef(staging.begin(), staging.size()));
+
+	return Tuple::UserTypeStr(code, str);
+}
+
 KeyRange Tuple::range(Tuple const& tuple) const {
 	VectorRef<uint8_t> begin;
 	VectorRef<uint8_t> end;
@@ -426,3 +486,76 @@ StringRef Tuple::subTupleRawString(size_t index) const {
 	size_t endPos = end < offsets.size() ? offsets[end] : data.size();
 	return StringRef(data.begin() + offsets[index], endPos - offsets[index]);
 }
+
+TEST_CASE("/fdbclient/Tuple/makeTuple") {
+	Tuple t1 = Tuple::makeTuple(1,
+	                            1.0f,
+	                            1.0,
+	                            false,
+	                            "byteStr"_sr,
+	                            Tuple::UnicodeStr("str"_sr),
+	                            nullptr,
+	                            Versionstamp("000000000000"_sr),
+	                            Tuple::UserTypeStr(0x41, "12345678"_sr));
+	Tuple t2 = Tuple()
+	               .append(1)
+	               .append(1.0f)
+	               .append(1.0)
+	               .append(false)
+	               .append("byteStr"_sr)
+	               .append(Tuple::UnicodeStr("str"_sr))
+	               .append(nullptr)
+	               .append(Versionstamp("000000000000"_sr))
+	               .append(Tuple::UserTypeStr(0x41, "12345678"_sr));
+
+	ASSERT(t1.pack() == t2.pack());
+	ASSERT(t1.getType(0) == Tuple::INT);
+	ASSERT(t1.getType(1) == Tuple::FLOAT);
+	ASSERT(t1.getType(2) == Tuple::DOUBLE);
+	ASSERT(t1.getType(3) == Tuple::BOOL);
+	ASSERT(t1.getType(4) == Tuple::BYTES);
+	ASSERT(t1.getType(5) == Tuple::UTF8);
+	ASSERT(t1.getType(6) == Tuple::NULL_TYPE);
+	ASSERT(t1.getType(7) == Tuple::VERSIONSTAMP);
+	ASSERT(t1.getType(8) == Tuple::USER_TYPE);
+	ASSERT(t1.size() == 9);
+
+	return Void();
+}
+
+TEST_CASE("/fdbclient/Tuple/unpack") {
+	Tuple t1 = Tuple::makeTuple(1,
+	                            1.0f,
+	                            1.0,
+	                            false,
+	                            "byteStr"_sr,
+	                            Tuple::UnicodeStr("str"_sr),
+	                            nullptr,
+	                            Versionstamp("000000000000"_sr),
+	                            Tuple::UserTypeStr(0x41, "12345678"_sr));
+
+	Standalone<StringRef> packed = t1.pack();
+	Tuple t2 = Tuple::unpackUserType(packed);
+	ASSERT(t2.pack() == t1.pack());
+	ASSERT(t2.getInt(0) == t1.getInt(0));
+	ASSERT(t2.getFloat(1) == t1.getFloat(1));
+	ASSERT(t2.getDouble(2) == t1.getDouble(2));
+	ASSERT(t2.getBool(3) == t1.getBool(3));
+	ASSERT(t2.getString(4) == t1.getString(4));
+	ASSERT(t2.getString(5) == t1.getString(5));
+	ASSERT(t2.getType(6) == Tuple::NULL_TYPE);
+	ASSERT(t2.getVersionstamp(7) == t1.getVersionstamp(7));
+	ASSERT(t2.getUserType(8) == t1.getUserType(8));
+	ASSERT(t2.size() == 9);
+
+	try {
+		Tuple t3 = Tuple::unpack(packed);
+		ASSERT(false);
+	} catch (Error& e) {
+		if (e.code() != error_code_invalid_tuple_data_type) {
+			throw e;
+		}
+	}
+
+	return Void();
+}
diff --git a/fdbclient/WriteMap.cpp b/fdbclient/WriteMap.cpp
index dcb6ee9581..56aedd8e8c 100644
--- a/fdbclient/WriteMap.cpp
+++ b/fdbclient/WriteMap.cpp
@@ -567,7 +567,7 @@ void WriteMap::clearNoConflict(KeyRangeRef keys) {
 	bool end_conflict = it.is_conflict_range();
 	bool end_unreadable = it.is_unreadable();
 
-	TEST(it.is_conflict_range() != lastConflicted); // not last conflicted
+	CODE_PROBE(it.is_conflict_range() != lastConflicted, "not last conflicted");
 
 	it.tree.clear();
 
diff --git a/fdbclient/azure_backup/BackupContainerAzureBlobStore.actor.cpp b/fdbclient/azure_backup/BackupContainerAzureBlobStore.actor.cpp
index 0c3ac8ba02..8bd8f94872 100644
--- a/fdbclient/azure_backup/BackupContainerAzureBlobStore.actor.cpp
+++ b/fdbclient/azure_backup/BackupContainerAzureBlobStore.actor.cpp
@@ -29,7 +29,7 @@ namespace {
 std::string const notFoundErrorCode = "404";
 
 void printAzureError(std::string const& operationName, azure::storage_lite::storage_error const& err) {
-	printf("(%s) : Error from Azure SDK : %s (%s) : %s",
+	printf("(%s) : Error from Azure SDK : %s (%s) : %s\n",
 	       operationName.c_str(),
 	       err.code_name.c_str(),
 	       err.code.c_str(),
@@ -109,9 +109,9 @@ public:
 
 	class WriteFile final : public IAsyncFile, ReferenceCounted<WriteFile> {
 		AsyncTaskThread* asyncTaskThread;
-		std::shared_ptr<AzureClient> client;
 		std::string containerName;
 		std::string blobName;
+		std::shared_ptr<AzureClient> client;
 		int64_t m_cursor{ 0 };
 		// Ideally this buffer should not be a string, but
 		// the Azure SDK only supports/tests uploading to append
@@ -318,7 +318,7 @@ BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const std::string&
 	std::string accountKey = _accountKey;
 	auto credential = std::make_shared<azure::storage_lite::shared_key_credential>(accountName, accountKey);
 	auto storageAccount = std::make_shared<azure::storage_lite::storage_account>(
-	    accountName, credential, true, format("https://%s", endpoint.c_str()));
+	    accountName, credential, true, fmt::format("https://{}", endpoint));
 	client = std::make_unique<AzureClient>(storageAccount, 1);
 }
 
@@ -333,12 +333,16 @@ Future<Void> BackupContainerAzureBlobStore::create() {
 	TraceEvent(SevDebug, "BCAzureBlobStoreCreateContainer").detail("ContainerName", containerName);
 	Future<Void> createContainerFuture =
 	    asyncTaskThread.execAsync([containerName = this->containerName, client = this->client] {
-		    waitAzureFuture(client->create_container(containerName), "create_container");
+		    auto outcome = client->get_container_properties(containerName).get();
+		    if (!outcome.success()) {
+			    waitAzureFuture(client->create_container(containerName), "create_container");
+		    }
 		    return Void();
 	    });
 	Future<Void> encryptionSetupFuture = usesEncryption() ? encryptionSetupComplete() : Void();
 	return createContainerFuture && encryptionSetupFuture;
 }
+
 Future<bool> BackupContainerAzureBlobStore::exists() {
 	TraceEvent(SevDebug, "BCAzureBlobStoreCheckContainerExists").detail("ContainerName", containerName);
 	return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client] {
diff --git a/fdbclient/azure_backup/README.md b/fdbclient/azure_backup/README.md
new file mode 100644
index 0000000000..4a34683674
--- /dev/null
+++ b/fdbclient/azure_backup/README.md
@@ -0,0 +1,33 @@
+# Set up the Azure Backup Testing Environment
+
+Make sure we built FDB with `-DBUILD_AZURE_BACKUP=ON`
+
+# Test
+
+If you run _BackupToBlob_ and _RestoreFromBlob_ workloads with the paramter _backupURL_ starts with `azure://`,
+the workload will backup to and restore from the azure blob storage.
+For example, _BackupAzureBlobCorrectness.toml_
+
+## Url format
+
+The code now supports the following style urls:
+
+- `azure://<account_name>.blob.core.windows.net/<container_name>` (The formal url format for the blob service provided by the azure storage account)
+- `azure://<ip|hostname>:<port>/<account_name>/<container_name>` (Directly providing the endpoint address for the blob service, usually for local testing)
+
+## Local test environment 
+
+We need to use the _Azurite_ to simulate an Azure blob service locally.
+Please follow the [turtorial](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=docker-hub) to start your service locally.
+
+For example,
+```
+docker run -p 10000:10000 -v `pwd`:<path> -w <path> mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --oauth basic --cert ./<...>.pem --key ./<...>.key.pem --debug ./<log_file_path>
+```
+
+### Notice
+
+- To use uses _https_, we need to provide the certificates via `--cert` and `--key`
+    The detailed [turtorial](https://github.com/Azure/Azurite/blob/main/README.md#https-setup) to setup HTTPS.  (We tested with the `mkcert` method)
+- To use Azure SDKs, we need to pass `--oauth basic` option
+- Please take a look at the [difference](https://github.com/Azure/Azurite/blob/main/README.md#differences-between-azurite-and-azure-storage) between Azurite and Azure Storage
diff --git a/fdbclient/azurestorage.cmake b/fdbclient/azurestorage.cmake
index 36f8e24f6e..b967824948 100644
--- a/fdbclient/azurestorage.cmake
+++ b/fdbclient/azurestorage.cmake
@@ -1,3 +1,5 @@
+cmake_minimum_required(VERSION 3.13)
+
 project(azurestorage-download)
 
 include(ExternalProject)
diff --git a/fdbclient/include/fdbclient/AsyncFileS3BlobStore.actor.h b/fdbclient/include/fdbclient/AsyncFileS3BlobStore.actor.h
index 4244992666..3a17528358 100644
--- a/fdbclient/include/fdbclient/AsyncFileS3BlobStore.actor.h
+++ b/fdbclient/include/fdbclient/AsyncFileS3BlobStore.actor.h
@@ -36,8 +36,8 @@
 #include "flow/Net2Packet.h"
 #include "flow/IRateControl.h"
 #include "fdbclient/S3BlobStore.h"
-#include "fdbclient/md5/md5.h"
-#include "fdbclient/libb64/encode.h"
+#include "md5/md5.h"
+#include "libb64/encode.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 ACTOR template <typename T>
diff --git a/fdbclient/include/fdbclient/Atomic.h b/fdbclient/include/fdbclient/Atomic.h
index 6643bcdafc..61f948e38f 100644
--- a/fdbclient/include/fdbclient/Atomic.h
+++ b/fdbclient/include/fdbclient/Atomic.h
@@ -120,7 +120,7 @@ inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
 	if (!otherOperand.size())
 		return existingValue;
 	if (existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
-		TEST(true) // AppendIfFIts resulted in truncation
+		CODE_PROBE(true, "AppendIfFits resulted in truncation");
 		return existingValue;
 	}
 
diff --git a/fdbclient/include/fdbclient/BackupAgent.actor.h b/fdbclient/include/fdbclient/BackupAgent.actor.h
index 573c9c2c19..314f151fd0 100644
--- a/fdbclient/include/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/include/fdbclient/BackupAgent.actor.h
@@ -143,7 +143,7 @@ public:
 		futureBucket = std::move(r.futureBucket);
 	}
 
-	KeyBackedProperty<Key> lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Key> lastBackupTimestamp() { return config.pack(__FUNCTION__sr); }
 
 	Future<Void> run(Database cx, double pollDelay, int maxConcurrentTasks) {
 		return taskBucket->run(cx, futureBucket, std::make_shared<double const>(pollDelay), maxConcurrentTasks);
@@ -208,33 +208,14 @@ public:
 	                        WaitForComplete waitForComplete = WaitForComplete::True,
 	                        Version targetVersion = ::invalidVersion,
 	                        Verbose verbose = Verbose::True,
-	                        KeyRange range = normalKeys,
+	                        KeyRange range = KeyRange(),
 	                        Key addPrefix = Key(),
 	                        Key removePrefix = Key(),
 	                        LockDB lockDB = LockDB::True,
 	                        OnlyApplyMutationLogs onlyApplyMutationLogs = OnlyApplyMutationLogs::False,
 	                        InconsistentSnapshotOnly inconsistentSnapshotOnly = InconsistentSnapshotOnly::False,
 	                        Version beginVersion = ::invalidVersion,
-	                        Optional<std::string> const& encryptionKeyFileName = {}) {
-		Standalone<VectorRef<KeyRangeRef>> rangeRef;
-		rangeRef.push_back_deep(rangeRef.arena(), range);
-		return restore(cx,
-		               cxOrig,
-		               tagName,
-		               url,
-		               proxy,
-		               rangeRef,
-		               waitForComplete,
-		               targetVersion,
-		               verbose,
-		               addPrefix,
-		               removePrefix,
-		               lockDB,
-		               onlyApplyMutationLogs,
-		               inconsistentSnapshotOnly,
-		               beginVersion,
-		               encryptionKeyFileName);
-	}
+	                        Optional<std::string> const& encryptionKeyFileName = {});
 	Future<Version> atomicRestore(Database cx,
 	                              Key tagName,
 	                              Standalone<VectorRef<KeyRangeRef>> ranges,
@@ -242,13 +223,10 @@ public:
 	                              Key removePrefix = Key());
 	Future<Version> atomicRestore(Database cx,
 	                              Key tagName,
-	                              KeyRange range = normalKeys,
+	                              KeyRange range = KeyRange(),
 	                              Key addPrefix = Key(),
-	                              Key removePrefix = Key()) {
-		Standalone<VectorRef<KeyRangeRef>> rangeRef;
-		rangeRef.push_back_deep(rangeRef.arena(), range);
-		return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
-	}
+	                              Key removePrefix = Key());
+
 	// Tries to abort the restore for a tag.  Returns the final (stable) state of the tag.
 	Future<ERestoreState> abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName);
 	Future<ERestoreState> abortRestore(Database cx, Key tagName);
@@ -272,6 +250,7 @@ public:
 	                          int snapshotIntervalSeconds,
 	                          std::string const& tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                          bool encryptionEnabled,
 	                          StopWhenDone = StopWhenDone::True,
 	                          UsePartitionedLog = UsePartitionedLog::False,
 	                          IncrementalBackupOnly = IncrementalBackupOnly::False,
@@ -283,6 +262,7 @@ public:
 	                          int snapshotIntervalSeconds,
 	                          std::string const& tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                          bool encryptionEnabled,
 	                          StopWhenDone stopWhenDone = StopWhenDone::True,
 	                          UsePartitionedLog partitionedLog = UsePartitionedLog::False,
 	                          IncrementalBackupOnly incrementalBackupOnly = IncrementalBackupOnly::False,
@@ -295,6 +275,7 @@ public:
 			                    snapshotIntervalSeconds,
 			                    tagName,
 			                    backupRanges,
+			                    encryptionEnabled,
 			                    stopWhenDone,
 			                    partitionedLog,
 			                    incrementalBackupOnly,
@@ -361,7 +342,7 @@ public:
 template <>
 inline Standalone<StringRef> TupleCodec<FileBackupAgent::ERestoreState>::pack(
     FileBackupAgent::ERestoreState const& val) {
-	return Tuple().append(val).pack();
+	return Tuple::makeTuple(static_cast<int>(val)).pack();
 }
 template <>
 inline FileBackupAgent::ERestoreState TupleCodec<FileBackupAgent::ERestoreState>::unpack(
@@ -578,7 +559,7 @@ ACTOR Future<Void> cleanupBackup(Database cx, DeleteData deleteData);
 using EBackupState = BackupAgentBase::EnumState;
 template <>
 inline Standalone<StringRef> TupleCodec<EBackupState>::pack(EBackupState const& val) {
-	return Tuple().append(static_cast<int>(val)).pack();
+	return Tuple::makeTuple(static_cast<int>(val)).pack();
 }
 template <>
 inline EBackupState TupleCodec<EBackupState>::unpack(Standalone<StringRef> const& val) {
@@ -621,7 +602,7 @@ class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
 	                                                           Snapshot snapshot);
 
 public:
-	TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
+	TagUidMap(const StringRef& prefix) : TagMap("tag->uid/"_sr.withPrefix(prefix)), prefix(prefix) {}
 
 	Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr,
 	                                         Snapshot snapshot = Snapshot::False) {
@@ -652,11 +633,11 @@ static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadY
 class KeyBackedConfig {
 public:
 	static struct {
-		static TaskParam<UID> uid() { return LiteralStringRef(__FUNCTION__); }
+		static TaskParam<UID> uid() { return __FUNCTION__sr; }
 	} TaskParams;
 
 	KeyBackedConfig(StringRef prefix, UID uid = UID())
-	  : uid(uid), prefix(prefix), configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
+	  : uid(uid), prefix(prefix), configSpace(uidPrefixKey("uid->config/"_sr.withPrefix(prefix), uid)) {}
 
 	KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {}
 
@@ -685,7 +666,7 @@ public:
 		});
 	}
 
-	KeyBackedProperty<std::string> tag() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<std::string> tag() { return configSpace.pack(__FUNCTION__sr); }
 
 	UID getUid() { return uid; }
 
@@ -694,12 +675,10 @@ public:
 	void clear(Reference<ReadYourWritesTransaction> tr) { tr->clear(configSpace.range()); }
 
 	// lastError is a pair of error message and timestamp expressed as an int64_t
-	KeyBackedProperty<std::pair<std::string, Version>> lastError() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
-	}
+	KeyBackedProperty<std::pair<std::string, Version>> lastError() { return configSpace.pack(__FUNCTION__sr); }
 
 	KeyBackedMap<int64_t, std::pair<std::string, Version>> lastErrorPerType() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
+		return configSpace.pack(__FUNCTION__sr);
 	}
 
 	// Updates the error per type map and the last error property
@@ -727,8 +706,7 @@ protected:
 
 template <>
 inline Standalone<StringRef> TupleCodec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const& bc) {
-	Tuple tuple;
-	tuple.append(StringRef(bc->getURL()));
+	Tuple tuple = Tuple::makeTuple(bc->getURL());
 
 	if (bc->getEncryptionKeyFileName().present()) {
 		tuple.append(bc->getEncryptionKeyFileName().get());
@@ -775,9 +753,7 @@ public:
 		Version version;
 		std::string fileName;
 		int64_t fileSize;
-		Tuple pack() const {
-			return Tuple().append(begin).append(version).append(StringRef(fileName)).append(fileSize);
-		}
+		Tuple pack() const { return Tuple::makeTuple(begin, version, fileName, fileSize); }
 		static RangeSlice unpack(Tuple const& t) {
 			RangeSlice r;
 			int i = 0;
@@ -791,47 +767,41 @@ public:
 
 	// Map of range end boundaries to info about the backup file written for that range.
 	typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT;
-	RangeFileMapT snapshotRangeFileMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	RangeFileMapT snapshotRangeFileMap() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Number of kv range files that were both committed to persistent storage AND inserted into
 	// the snapshotRangeFileMap.  Note that since insertions could replace 1 or more existing
 	// map entries this is not necessarily the number of entries currently in the map.
 	// This value exists to help with sizing of kv range folders for BackupContainers that
 	// require it.
-	KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Coalesced set of ranges already dispatched for writing.
 	typedef KeyBackedMap<Key, bool> RangeDispatchMapT;
-	RangeDispatchMapT snapshotRangeDispatchMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	RangeDispatchMapT snapshotRangeDispatchMap() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Interval to use for the first (initial) snapshot.
-	KeyBackedProperty<int64_t> initialSnapshotIntervalSeconds() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
-	}
+	KeyBackedProperty<int64_t> initialSnapshotIntervalSeconds() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Interval to use for determining the target end version for new snapshots
-	KeyBackedProperty<int64_t> snapshotIntervalSeconds() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<int64_t> snapshotIntervalSeconds() { return configSpace.pack(__FUNCTION__sr); }
 
 	// When the current snapshot began
-	KeyBackedProperty<Version> snapshotBeginVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Version> snapshotBeginVersion() { return configSpace.pack(__FUNCTION__sr); }
 
 	// When the current snapshot is desired to end.
 	// This can be changed at runtime to speed up or slow down a snapshot
-	KeyBackedProperty<Version> snapshotTargetEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Version> snapshotTargetEndVersion() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedProperty<int64_t> snapshotBatchSize() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<int64_t> snapshotBatchSize() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedProperty<Key> snapshotBatchFuture() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Key> snapshotBatchFuture() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
-	}
+	KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedProperty<Version> snapshotDispatchLastVersion() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
-	}
+	KeyBackedProperty<Version> snapshotDispatchLastVersion() { return configSpace.pack(__FUNCTION__sr); }
 
 	Future<Void> initNewSnapshot(Reference<ReadYourWritesTransaction> tr, int64_t intervalSeconds = -1) {
 		BackupConfig& copy = *this; // Capture this by value instead of this ptr
@@ -865,51 +835,50 @@ public:
 		});
 	}
 
-	KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedProperty<EBackupState> stateEnum() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<EBackupState> stateEnum() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedProperty<Reference<IBackupContainer>> backupContainer() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
-	}
+	KeyBackedProperty<Reference<IBackupContainer>> backupContainer() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Set to true when all backup workers for saving mutation logs have been started.
-	KeyBackedProperty<bool> allWorkerStarted() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<bool> allWorkerStarted() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Each backup worker adds its (epoch, tag.id) to this property.
 	KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
+		return configSpace.pack(__FUNCTION__sr);
 	}
 
 	// Set to true if backup worker is enabled.
-	KeyBackedProperty<bool> backupWorkerEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<bool> backupWorkerEnabled() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
-	KeyBackedProperty<bool> partitionedLogEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<bool> partitionedLogEnabled() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Set to true if only requesting incremental backup without base snapshot.
-	KeyBackedProperty<bool> incrementalBackupOnly() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<bool> incrementalBackupOnly() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Latest version for which all prior versions have saved by backup workers.
-	KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
-		return configSpace.pack(LiteralStringRef(__FUNCTION__));
-	}
+	KeyBackedProperty<Version> latestBackupWorkerSavedVersion() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Stop differntial logging if already started or don't start after completing KV ranges
-	KeyBackedProperty<bool> stopWhenDone() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<bool> stopWhenDone() { return configSpace.pack(__FUNCTION__sr); }
+
+	// Enable snapshot backup file encryption
+	KeyBackedProperty<bool> enableSnapshotBackupEncryption() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Latest version for which all prior versions have had their log copy tasks completed
-	KeyBackedProperty<Version> latestLogEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Version> latestLogEndVersion() { return configSpace.pack(__FUNCTION__sr); }
 
 	// The end version of the last complete snapshot
-	KeyBackedProperty<Version> latestSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Version> latestSnapshotEndVersion() { return configSpace.pack(__FUNCTION__sr); }
 
 	// The end version of the first complete snapshot
-	KeyBackedProperty<Version> firstSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Version> firstSnapshotEndVersion() { return configSpace.pack(__FUNCTION__sr); }
 
-	KeyBackedProperty<Key> destUidValue() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<Key> destUidValue() { return configSpace.pack(__FUNCTION__sr); }
 
 	Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) {
 		tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@@ -940,7 +909,7 @@ public:
 		           });
 	}
 
-	KeyBackedProperty<std::vector<KeyRange>> backupRanges() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<std::vector<KeyRange>> backupRanges() { return configSpace.pack(__FUNCTION__sr); }
 
 	void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) {
 		Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin);
@@ -1008,7 +977,8 @@ struct StringRefReader {
 namespace fileBackup {
 ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
                                                                       int64_t offset,
-                                                                      int len);
+                                                                      int len,
+                                                                      Optional<Database> cx);
 
 // Reads a mutation log block from file and parses into batch mutation blocks for further parsing.
 ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<IAsyncFile> file,
@@ -1030,5 +1000,41 @@ ACTOR Future<Void> transformRestoredDatabase(Database cx,
 
 void simulateBlobFailure();
 
+// Add the set of ranges that are backed up in a default backup to the given vector. This consists of all normal keys
+// and the system backup ranges.
+void addDefaultBackupRanges(Standalone<VectorRef<KeyRangeRef>>& backupKeys);
+
+// Return a vector containing the key ranges in system key-space that should be backed up in a default backup.
+VectorRef<KeyRangeRef> const& getSystemBackupRanges();
+
+// Return a key-range map that can be used to check whether a system key is a candidate backup key (i.e. whether it is
+// part of any system backup ranges).
+KeyRangeMap<bool> const& systemBackupMutationMask();
+
+// Returns true if the given set of ranges exactly matches the set of ranges included in a default backup.
+template <class Container>
+bool isDefaultBackup(Container ranges) {
+	std::unordered_set<KeyRangeRef> uniqueRanges(ranges.begin(), ranges.end());
+	auto& systemBackupRanges = getSystemBackupRanges();
+
+	if (uniqueRanges.size() != systemBackupRanges.size() + 1) {
+		return false;
+	}
+
+	if (!uniqueRanges.count(normalKeys)) {
+		return false;
+	}
+	for (auto range : getSystemBackupRanges()) {
+		if (!uniqueRanges.count(range)) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+// Returns a key-range used to denote that a shared mutation stream belongs to the default backup set.
+KeyRangeRef const& getDefaultBackupSharedRange();
+
 #include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbclient/include/fdbclient/BackupContainer.h b/fdbclient/include/fdbclient/BackupContainer.h
index 11d5c2ba27..95a1072a05 100644
--- a/fdbclient/include/fdbclient/BackupContainer.h
+++ b/fdbclient/include/fdbclient/BackupContainer.h
@@ -67,6 +67,9 @@ static const uint32_t PARTITIONED_MLOG_VERSION = 4110;
 // Snapshot file version written by FileBackupAgent
 static const uint32_t BACKUP_AGENT_SNAPSHOT_FILE_VERSION = 1001;
 
+// Encrypted Snapshot file version written by FileBackupAgent
+static const uint32_t BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION = 1002;
+
 struct LogFile {
 	Version beginVersion;
 	Version endVersion;
@@ -250,7 +253,7 @@ public:
 
 	// Returns the key ranges in the snapshot file. This is an expensive function
 	// and should only be used in simulation for sanity check.
-	virtual Future<KeyRange> getSnapshotFileKeyRange(const RangeFile& file) = 0;
+	virtual Future<KeyRange> getSnapshotFileKeyRange(const RangeFile& file, Optional<Database> cx) = 0;
 
 	struct ExpireProgress {
 		std::string step;
@@ -289,6 +292,7 @@ public:
 	// If logsOnly is set, only use log files in [beginVersion, targetVervions) in restore set.
 	// Returns non-present if restoring to the given version is not possible.
 	virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
+	                                                          Optional<Database> cx,
 	                                                          VectorRef<KeyRangeRef> keyRangesFilter = {},
 	                                                          bool logsOnly = false,
 	                                                          Version beginVersion = -1) = 0;
diff --git a/fdbclient/include/fdbclient/BackupContainerAzureBlobStore.h b/fdbclient/include/fdbclient/BackupContainerAzureBlobStore.h
index 77285ced16..ed79a56078 100644
--- a/fdbclient/include/fdbclient/BackupContainerAzureBlobStore.h
+++ b/fdbclient/include/fdbclient/BackupContainerAzureBlobStore.h
@@ -25,8 +25,6 @@
 #include "fdbclient/AsyncTaskThread.h"
 #include "fdbclient/BackupContainerFileSystem.h"
 
-#include "storage_credential.h"
-#include "storage_account.h"
 #include "blob/blob_client.h"
 
 class BackupContainerAzureBlobStore final : public BackupContainerFileSystem,
diff --git a/fdbclient/include/fdbclient/BackupContainerFileSystem.h b/fdbclient/include/fdbclient/BackupContainerFileSystem.h
index 17245c9c39..c6819a864e 100644
--- a/fdbclient/include/fdbclient/BackupContainerFileSystem.h
+++ b/fdbclient/include/fdbclient/BackupContainerFileSystem.h
@@ -152,9 +152,10 @@ public:
 	                        ExpireProgress* progress,
 	                        Version restorableBeginVersion) final;
 
-	Future<KeyRange> getSnapshotFileKeyRange(const RangeFile& file) final;
+	Future<KeyRange> getSnapshotFileKeyRange(const RangeFile& file, Optional<Database> cx) final;
 
 	Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
+	                                                  Optional<Database> cx,
 	                                                  VectorRef<KeyRangeRef> keyRangesFilter,
 	                                                  bool logsOnly,
 	                                                  Version beginVersion) final;
diff --git a/flow/include/flow/BlobCipher.h b/fdbclient/include/fdbclient/BlobCipher.h
similarity index 77%
rename from flow/include/flow/BlobCipher.h
rename to fdbclient/include/fdbclient/BlobCipher.h
index 466ebba1b0..be2eae72b8 100644
--- a/flow/include/flow/BlobCipher.h
+++ b/fdbclient/include/fdbclient/BlobCipher.h
@@ -17,23 +17,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#ifndef FLOW_BLOB_CIPHER_H
-#define FLOW_BLOB_CIPHER_H
-#include "flow/ProtocolVersion.h"
-#include "flow/serialize.h"
+#ifndef FDBCLIENT_BLOB_CIPHER_H
+#define FDBCLIENT_BLOB_CIPHER_H
 #pragma once
 
+#include "fdbrpc/Stats.h"
 #include "flow/Arena.h"
 #include "flow/EncryptUtils.h"
 #include "flow/FastRef.h"
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
+#include "flow/Knobs.h"
 #include "flow/network.h"
+#include "flow/Platform.h"
+#include "flow/ProtocolVersion.h"
+#include "flow/serialize.h"
 
 #include <boost/functional/hash.hpp>
 #include <cinttypes>
+#include <limits>
 #include <memory>
 #include <openssl/aes.h>
+#include <openssl/cmac.h>
 #include <openssl/engine.h>
 #include <openssl/evp.h>
 #include <openssl/hmac.h>
@@ -48,6 +53,59 @@
 #define AES_256_KEY_LENGTH 32
 #define AES_256_IV_LENGTH 16
 
+class BlobCipherMetrics : public NonCopyable {
+public:
+	static BlobCipherMetrics* getInstance() {
+		static BlobCipherMetrics* instance = nullptr;
+		if (instance == nullptr) {
+			instance = new BlobCipherMetrics;
+		}
+		return instance;
+	}
+
+	// Order of this enum has to match initializer of counterSets.
+	enum UsageType : int {
+		TLOG = 0,
+		KV_MEMORY,
+		KV_REDWOOD,
+		BLOB_GRANULE,
+		BACKUP,
+		TEST,
+		MAX,
+	};
+
+	struct CounterSet {
+		Counter encryptCPUTimeNS;
+		Counter decryptCPUTimeNS;
+		LatencySample getCipherKeysLatency;
+		LatencySample getLatestCipherKeysLatency;
+
+		CounterSet(CounterCollection& cc, std::string name);
+	};
+
+	static CounterSet& counters(UsageType t) {
+		ASSERT(t < UsageType::MAX);
+		return getInstance()->counterSets[int(t)];
+	}
+
+private:
+	BlobCipherMetrics();
+
+	CounterCollection cc;
+	Future<Void> traceFuture;
+
+public:
+	Counter cipherKeyCacheHit;
+	Counter cipherKeyCacheMiss;
+	Counter cipherKeyCacheExpired;
+	Counter latestCipherKeyCacheHit;
+	Counter latestCipherKeyCacheMiss;
+	Counter latestCipherKeyCacheNeedsRefresh;
+	LatencySample getCipherKeysLatency;
+	LatencySample getLatestCipherKeysLatency;
+	std::array<CounterSet, int(UsageType::MAX)> counterSets;
+};
+
 // Encryption operations buffer management
 // Approach limits number of copies needed during encryption or decryption operations.
 // For encryption EncryptBuf is allocated using client supplied Arena and provided to AES library to capture
@@ -59,7 +117,7 @@ class EncryptBuf : public ReferenceCounted<EncryptBuf>, NonCopyable {
 public:
 	EncryptBuf(int size, Arena& arena) : allocSize(size), logicalSize(size) {
 		if (size > 0) {
-			buffer = new (arena) uint8_t[size];
+			buffer = new (arena) uint8_t[size]();
 		} else {
 			buffer = nullptr;
 		}
@@ -83,9 +141,9 @@ private:
 #pragma pack(push, 1) // exact fit - no padding
 struct BlobCipherDetails {
 	// Encryption domain boundary identifier.
-	EncryptCipherDomainId encryptDomainId = ENCRYPT_INVALID_DOMAIN_ID;
+	EncryptCipherDomainId encryptDomainId = INVALID_ENCRYPT_DOMAIN_ID;
 	// BaseCipher encryption key identifier
-	EncryptCipherBaseKeyId baseCipherId = ENCRYPT_INVALID_CIPHER_KEY_ID;
+	EncryptCipherBaseKeyId baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID;
 	// Random salt
 	EncryptCipherRandomSalt salt{};
 
@@ -135,7 +193,8 @@ typedef struct BlobCipherEncryptHeader {
 			uint8_t headerVersion{};
 			uint8_t encryptMode{};
 			uint8_t authTokenMode{};
-			uint8_t _reserved[4]{};
+			uint8_t authTokenAlgo{};
+			uint8_t _reserved[3]{};
 		} flags;
 		uint64_t _padding{};
 	};
@@ -166,12 +225,12 @@ typedef struct BlobCipherEncryptHeader {
 
 		struct {
 			// Cipher text authentication token
-			uint8_t cipherTextAuthToken[AUTH_TOKEN_SIZE]{};
-			uint8_t headerAuthToken[AUTH_TOKEN_SIZE]{};
+			uint8_t cipherTextAuthToken[AUTH_TOKEN_MAX_SIZE]{};
+			uint8_t headerAuthToken[AUTH_TOKEN_MAX_SIZE]{};
 		} multiAuthTokens;
 		struct {
-			uint8_t authToken[AUTH_TOKEN_SIZE]{};
-			uint8_t _reserved[AUTH_TOKEN_SIZE]{};
+			uint8_t authToken[AUTH_TOKEN_MAX_SIZE]{};
+			uint8_t _reserved[AUTH_TOKEN_MAX_SIZE]{};
 		} singleAuthToken;
 	};
 
@@ -216,15 +275,20 @@ public:
 	BlobCipherKey(const EncryptCipherDomainId& domainId,
 	              const EncryptCipherBaseKeyId& baseCiphId,
 	              const uint8_t* baseCiph,
-	              int baseCiphLen);
+	              int baseCiphLen,
+	              const int64_t refreshAt,
+	              int64_t expireAt);
 	BlobCipherKey(const EncryptCipherDomainId& domainId,
 	              const EncryptCipherBaseKeyId& baseCiphId,
 	              const uint8_t* baseCiph,
 	              int baseCiphLen,
-	              const EncryptCipherRandomSalt& salt);
+	              const EncryptCipherRandomSalt& salt,
+	              const int64_t refreshAt,
+	              const int64_t expireAt);
 
 	uint8_t* data() const { return cipher.get(); }
-	uint64_t getCreationTime() const { return creationTime; }
+	uint64_t getRefreshAtTS() const { return refreshAtTS; }
+	uint64_t getExpireAtTS() const { return expireAtTS; }
 	EncryptCipherDomainId getDomainId() const { return encryptDomainId; }
 	EncryptCipherRandomSalt getSalt() const { return randomSalt; }
 	EncryptCipherBaseKeyId getBaseCipherId() const { return baseCipherId; }
@@ -243,6 +307,20 @@ public:
 		       randomSalt == details.salt;
 	}
 
+	inline bool needsRefresh() {
+		if (refreshAtTS == std::numeric_limits<int64_t>::max()) {
+			return false;
+		}
+		return now() >= refreshAtTS ? true : false;
+	}
+
+	inline bool isExpired() {
+		if (expireAtTS == std::numeric_limits<int64_t>::max()) {
+			return false;
+		}
+		return now() >= expireAtTS ? true : false;
+	}
+
 	void reset();
 
 private:
@@ -254,16 +332,20 @@ private:
 	EncryptCipherBaseKeyId baseCipherId;
 	// Random salt used for encryption cipher key derivation
 	EncryptCipherRandomSalt randomSalt;
-	// Creation timestamp for the derived encryption cipher key
-	uint64_t creationTime;
 	// Derived encryption cipher key
 	std::unique_ptr<uint8_t[]> cipher;
+	// CipherKey needs refreshAt
+	int64_t refreshAtTS;
+	// CipherKey is valid until
+	int64_t expireAtTS;
 
 	void initKey(const EncryptCipherDomainId& domainId,
 	             const uint8_t* baseCiph,
 	             int baseCiphLen,
 	             const EncryptCipherBaseKeyId& baseCiphId,
-	             const EncryptCipherRandomSalt& salt);
+	             const EncryptCipherRandomSalt& salt,
+	             const int64_t refreshAt,
+	             const int64_t expireAt);
 	void applyHmacSha256Derivation();
 };
 
@@ -299,8 +381,7 @@ using BlobCipherKeyIdCacheMapCItr =
 
 struct BlobCipherKeyIdCache : ReferenceCounted<BlobCipherKeyIdCache> {
 public:
-	BlobCipherKeyIdCache();
-	explicit BlobCipherKeyIdCache(EncryptCipherDomainId dId);
+	explicit BlobCipherKeyIdCache(EncryptCipherDomainId dId, size_t* sizeStat);
 
 	BlobCipherKeyIdCacheKey getCacheKey(const EncryptCipherBaseKeyId& baseCipherId,
 	                                    const EncryptCipherRandomSalt& salt);
@@ -326,7 +407,9 @@ public:
 
 	Reference<BlobCipherKey> insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId,
 	                                             const uint8_t* baseCipher,
-	                                             int baseCipherLen);
+	                                             int baseCipherLen,
+	                                             const int64_t refreshAt,
+	                                             const int64_t expireAt);
 
 	// API enables inserting base encryption cipher details to the BlobCipherKeyIdCache
 	// Given cipherKeys are immutable, attempting to re-insert same 'identical' cipherKey
@@ -341,7 +424,9 @@ public:
 	Reference<BlobCipherKey> insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId,
 	                                             const uint8_t* baseCipher,
 	                                             int baseCipherLen,
-	                                             const EncryptCipherRandomSalt& salt);
+	                                             const EncryptCipherRandomSalt& salt,
+	                                             const int64_t refreshAt,
+	                                             const int64_t expireAt);
 
 	// API cleanup the cache by dropping all cached cipherKeys
 	void cleanup();
@@ -349,11 +434,15 @@ public:
 	// API returns list of all 'cached' cipherKeys
 	std::vector<Reference<BlobCipherKey>> getAllCipherKeys();
 
+	// Return number of cipher keys in the cahce.
+	size_t getSize() const { return keyIdCache.size(); }
+
 private:
 	EncryptCipherDomainId domainId;
 	BlobCipherKeyIdCacheMap keyIdCache;
 	Optional<EncryptCipherBaseKeyId> latestBaseCipherKeyId;
 	Optional<EncryptCipherRandomSalt> latestRandomSalt;
+	size_t* sizeStat; // pointer to the outer BlobCipherKeyCache size count.
 };
 
 using BlobCipherDomainCacheMap = std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKeyIdCache>>;
@@ -377,7 +466,9 @@ public:
 	Reference<BlobCipherKey> insertCipherKey(const EncryptCipherDomainId& domainId,
 	                                         const EncryptCipherBaseKeyId& baseCipherId,
 	                                         const uint8_t* baseCipher,
-	                                         int baseCipherLen);
+	                                         int baseCipherLen,
+	                                         const int64_t refreshAt,
+	                                         const int64_t expireAt);
 
 	// Enable clients to insert base encryption cipher details to the BlobCipherKeyCache.
 	// The cipherKeys are indexed using 'baseCipherId', given cipherKeys are immutable,
@@ -394,7 +485,9 @@ public:
 	                                         const EncryptCipherBaseKeyId& baseCipherId,
 	                                         const uint8_t* baseCipher,
 	                                         int baseCipherLen,
-	                                         const EncryptCipherRandomSalt& salt);
+	                                         const EncryptCipherRandomSalt& salt,
+	                                         const int64_t refreshAt,
+	                                         const int64_t expireAt);
 
 	// API returns the last insert cipherKey for a given encryption domain Id.
 	// If domain Id is invalid, it would throw 'encrypt_invalid_id' exception,
@@ -414,10 +507,19 @@ public:
 
 	// API enables dropping all 'cached' cipherKeys for a given encryption domain Id.
 	// Useful to cleanup cache if an encryption domain gets removed/destroyed etc.
-
 	void resetEncryptDomainId(const EncryptCipherDomainId domainId);
 
+	// Total number of cipher keys in the cache.
+	size_t getSize() const { return size; }
+
 	static Reference<BlobCipherKeyCache> getInstance() {
+		static bool cleanupRegistered = false;
+		if (!cleanupRegistered) {
+			// We try to avoid cipher keys appear in core dumps, so we clean them up before crash.
+			// TODO(yiwu): use of MADV_DONTDUMP instead of the crash handler.
+			registerCrashHandlerCallback(BlobCipherKeyCache::cleanup);
+			cleanupRegistered = true;
+		}
 		if (g_network->isSimulated()) {
 			return FlowSingleton<BlobCipherKeyCache>::getInstance(
 			    []() { return makeReference<BlobCipherKeyCache>(g_network->isSimulated()); });
@@ -433,6 +535,7 @@ public:
 
 private:
 	BlobCipherDomainCacheMap domainCacheMap;
+	size_t size = 0;
 
 	BlobCipherKeyCache() {}
 };
@@ -450,17 +553,30 @@ public:
 	                           Reference<BlobCipherKey> hCipherKey,
 	                           const uint8_t* iv,
 	                           const int ivLen,
-	                           const EncryptAuthTokenMode mode);
+	                           const EncryptAuthTokenMode mode,
+	                           BlobCipherMetrics::UsageType usageType);
 	EncryptBlobCipherAes265Ctr(Reference<BlobCipherKey> tCipherKey,
 	                           Reference<BlobCipherKey> hCipherKey,
-	                           const EncryptAuthTokenMode mode);
+	                           const uint8_t* iv,
+	                           const int ivLen,
+	                           const EncryptAuthTokenMode mode,
+	                           const EncryptAuthTokenAlgo algo,
+	                           BlobCipherMetrics::UsageType usageType);
+	EncryptBlobCipherAes265Ctr(Reference<BlobCipherKey> tCipherKey,
+	                           Reference<BlobCipherKey> hCipherKey,
+	                           const EncryptAuthTokenMode mode,
+	                           BlobCipherMetrics::UsageType usageType);
+	EncryptBlobCipherAes265Ctr(Reference<BlobCipherKey> tCipherKey,
+	                           Reference<BlobCipherKey> hCipherKey,
+	                           const EncryptAuthTokenMode mode,
+	                           const EncryptAuthTokenAlgo algo,
+	                           BlobCipherMetrics::UsageType usageType);
 	~EncryptBlobCipherAes265Ctr();
 
 	Reference<EncryptBuf> encrypt(const uint8_t* plaintext,
 	                              const int plaintextLen,
 	                              BlobCipherEncryptHeader* header,
 	                              Arena&);
-	Standalone<StringRef> encryptBlobGranuleChunk(const uint8_t* plaintext, const int plaintextLen);
 
 private:
 	EVP_CIPHER_CTX* ctx;
@@ -468,6 +584,8 @@ private:
 	Reference<BlobCipherKey> headerCipherKey;
 	EncryptAuthTokenMode authTokenMode;
 	uint8_t iv[AES_256_IV_LENGTH];
+	BlobCipherMetrics::UsageType usageType;
+	EncryptAuthTokenAlgo authTokenAlgo;
 
 	void init();
 };
@@ -479,7 +597,8 @@ class DecryptBlobCipherAes256Ctr final : NonCopyable, public ReferenceCounted<De
 public:
 	DecryptBlobCipherAes256Ctr(Reference<BlobCipherKey> tCipherKey,
 	                           Reference<BlobCipherKey> hCipherKey,
-	                           const uint8_t* iv);
+	                           const uint8_t* iv,
+	                           BlobCipherMetrics::UsageType usageType);
 	~DecryptBlobCipherAes256Ctr();
 
 	Reference<EncryptBuf> decrypt(const uint8_t* ciphertext,
@@ -498,22 +617,20 @@ private:
 	Reference<BlobCipherKey> headerCipherKey;
 	bool headerAuthTokenValidationDone;
 	bool authTokensValidationDone;
+	BlobCipherMetrics::UsageType usageType;
 
 	void verifyEncryptHeaderMetadata(const BlobCipherEncryptHeader& header);
 	void verifyAuthTokens(const uint8_t* ciphertext,
 	                      const int ciphertextLen,
 	                      const BlobCipherEncryptHeader& header,
-	                      uint8_t* buff,
 	                      Arena& arena);
 	void verifyHeaderSingleAuthToken(const uint8_t* ciphertext,
 	                                 const int ciphertextLen,
 	                                 const BlobCipherEncryptHeader& header,
-	                                 uint8_t* buff,
 	                                 Arena& arena);
 	void verifyHeaderMultiAuthToken(const uint8_t* ciphertext,
 	                                const int ciphertextLen,
 	                                const BlobCipherEncryptHeader& header,
-	                                uint8_t* buff,
 	                                Arena& arena);
 };
 
@@ -522,16 +639,32 @@ public:
 	HmacSha256DigestGen(const unsigned char* key, size_t len);
 	~HmacSha256DigestGen();
 	HMAC_CTX* getCtx() const { return ctx; }
-	StringRef digest(unsigned char const* data, size_t len, Arena&);
+	unsigned int digest(const std::vector<std::pair<const uint8_t*, size_t>>& payload,
+	                    unsigned char* buf,
+	                    unsigned int bufLen);
 
 private:
 	HMAC_CTX* ctx;
 };
 
-StringRef computeAuthToken(const uint8_t* payload,
-                           const int payloadLen,
-                           const uint8_t* key,
-                           const int keyLen,
-                           Arena& arena);
+class Aes256CmacDigestGen final : NonCopyable {
+public:
+	Aes256CmacDigestGen(const unsigned char* key, size_t len);
+	~Aes256CmacDigestGen();
+	CMAC_CTX* getCtx() const { return ctx; }
+	size_t digest(const std::vector<std::pair<const uint8_t*, size_t>>& payload, uint8_t* digest, int digestlen);
 
-#endif // FLOW_BLOB_CIPHER_H
\ No newline at end of file
+private:
+	CMAC_CTX* ctx;
+};
+
+void computeAuthToken(const std::vector<std::pair<const uint8_t*, size_t>>& payload,
+                      const uint8_t* key,
+                      const int keyLen,
+                      unsigned char* digestBuf,
+                      const EncryptAuthTokenAlgo algo,
+                      unsigned int digestMaxBufSz);
+
+EncryptAuthTokenMode getEncryptAuthTokenMode(const EncryptAuthTokenMode mode);
+
+#endif // FDBCLIENT_BLOB_CIPHER_H
\ No newline at end of file
diff --git a/fdbclient/include/fdbclient/BlobGranuleCommon.h b/fdbclient/include/fdbclient/BlobGranuleCommon.h
index 4561779794..6f530f020d 100644
--- a/fdbclient/include/fdbclient/BlobGranuleCommon.h
+++ b/fdbclient/include/fdbclient/BlobGranuleCommon.h
@@ -22,10 +22,10 @@
 #define FDBCLIENT_BLOBGRANULECOMMON_H
 #pragma once
 
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
 
-#include "flow/BlobCipher.h"
 #include "flow/EncryptUtils.h"
 #include "flow/IRandom.h"
 #include "flow/serialize.h"
@@ -35,7 +35,6 @@
 #define BG_ENCRYPT_COMPRESS_DEBUG false
 
 // file format of actual blob files
-// FIXME: use VecSerStrategy::String serialization for this
 struct GranuleSnapshot : VectorRef<KeyValueRef> {
 
 	constexpr static FileIdentifier file_identifier = 1300395;
@@ -46,6 +45,7 @@ struct GranuleSnapshot : VectorRef<KeyValueRef> {
 	}
 };
 
+// Deltas in version order
 struct GranuleDeltas : VectorRef<MutationsAndVersionRef> {
 	constexpr static FileIdentifier file_identifier = 8563013;
 
@@ -55,6 +55,13 @@ struct GranuleDeltas : VectorRef<MutationsAndVersionRef> {
 	}
 };
 
+struct GranuleMaterializeStats {
+	int64_t inputBytes;
+	int64_t outputBytes;
+
+	GranuleMaterializeStats() : inputBytes(0), outputBytes(0) {}
+};
+
 struct BlobGranuleCipherKeysMeta {
 	EncryptCipherDomainId textDomainId;
 	EncryptCipherBaseKeyId textBaseCipherId;
@@ -142,16 +149,11 @@ struct BlobGranuleCipherKeysMetaRef {
 	StringRef ivRef;
 
 	BlobGranuleCipherKeysMetaRef() {}
-	BlobGranuleCipherKeysMetaRef(Arena& to,
-	                             const EncryptCipherDomainId tDomainId,
-	                             const EncryptCipherBaseKeyId tBaseCipherId,
-	                             const EncryptCipherRandomSalt tSalt,
-	                             const EncryptCipherDomainId hDomainId,
-	                             const EncryptCipherBaseKeyId hBaseCipherId,
-	                             const EncryptCipherRandomSalt hSalt,
-	                             const std::string& ivStr)
-	  : textDomainId(tDomainId), textBaseCipherId(tBaseCipherId), textSalt(tSalt), headerDomainId(hDomainId),
-	    headerBaseCipherId(hBaseCipherId), headerSalt(hSalt), ivRef(StringRef(to, ivStr)) {}
+	BlobGranuleCipherKeysMetaRef(Arena& to, BlobGranuleCipherKeysMeta cipherKeysMeta)
+	  : textDomainId(cipherKeysMeta.textDomainId), textBaseCipherId(cipherKeysMeta.textBaseCipherId),
+	    textSalt(cipherKeysMeta.textSalt), headerDomainId(cipherKeysMeta.headerDomainId),
+	    headerBaseCipherId(cipherKeysMeta.headerBaseCipherId), headerSalt(cipherKeysMeta.headerSalt),
+	    ivRef(StringRef(to, cipherKeysMeta.ivStr)) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -161,16 +163,31 @@ struct BlobGranuleCipherKeysMetaRef {
 
 struct BlobFilePointerRef {
 	constexpr static FileIdentifier file_identifier = 5253554;
+	// Serializable fields
 	StringRef filename;
 	int64_t offset;
 	int64_t length;
 	int64_t fullFileLength;
-	Optional<BlobGranuleCipherKeysMetaRef> cipherKeysMetaRef;
+	Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx;
+
+	// Non-serializable fields
+	Optional<BlobGranuleCipherKeysMetaRef>
+	    cipherKeysMetaRef; // Placeholder to cache information sufficient to lookup encryption ciphers
 
 	BlobFilePointerRef() {}
+
 	BlobFilePointerRef(Arena& to, const std::string& filename, int64_t offset, int64_t length, int64_t fullFileLength)
 	  : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength) {}
 
+	BlobFilePointerRef(Arena& to,
+	                   const std::string& filename,
+	                   int64_t offset,
+	                   int64_t length,
+	                   int64_t fullFileLength,
+	                   Optional<BlobGranuleCipherKeysCtx> ciphKeysCtx)
+	  : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength),
+	    cipherKeysCtx(ciphKeysCtx) {}
+
 	BlobFilePointerRef(Arena& to,
 	                   const std::string& filename,
 	                   int64_t offset,
@@ -179,30 +196,23 @@ struct BlobFilePointerRef {
 	                   Optional<BlobGranuleCipherKeysMeta> ciphKeysMeta)
 	  : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength) {
 		if (ciphKeysMeta.present()) {
-			cipherKeysMetaRef = BlobGranuleCipherKeysMetaRef(to,
-			                                                 ciphKeysMeta.get().textDomainId,
-			                                                 ciphKeysMeta.get().textBaseCipherId,
-			                                                 ciphKeysMeta.get().textSalt,
-			                                                 ciphKeysMeta.get().headerDomainId,
-			                                                 ciphKeysMeta.get().headerBaseCipherId,
-			                                                 ciphKeysMeta.get().headerSalt,
-			                                                 ciphKeysMeta.get().ivStr);
+			cipherKeysMetaRef = BlobGranuleCipherKeysMetaRef(to, ciphKeysMeta.get());
 		}
 	}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, filename, offset, length, fullFileLength, cipherKeysMetaRef);
+		serializer(ar, filename, offset, length, fullFileLength, cipherKeysCtx);
 	}
 
 	std::string toString() const {
 		std::stringstream ss;
 		ss << filename.toString() << ":" << offset << ":" << length << ":" << fullFileLength;
-		if (cipherKeysMetaRef.present()) {
-			ss << ":CipherKeysMeta:TextCipher:" << cipherKeysMetaRef.get().textDomainId << ":"
-			   << cipherKeysMetaRef.get().textBaseCipherId << ":" << cipherKeysMetaRef.get().textSalt
-			   << ":HeaderCipher:" << cipherKeysMetaRef.get().headerDomainId << ":"
-			   << cipherKeysMetaRef.get().headerBaseCipherId << ":" << cipherKeysMetaRef.get().headerSalt;
+		if (cipherKeysCtx.present()) {
+			ss << ":CipherKeysCtx:TextCipher:" << cipherKeysCtx.get().textCipherKey.encryptDomainId << ":"
+			   << cipherKeysCtx.get().textCipherKey.baseCipherId << ":" << cipherKeysCtx.get().textCipherKey.salt
+			   << ":HeaderCipher:" << cipherKeysCtx.get().headerCipherKey.encryptDomainId << ":"
+			   << cipherKeysCtx.get().headerCipherKey.baseCipherId << ":" << cipherKeysCtx.get().headerCipherKey.salt;
 		}
 		return std::move(ss).str();
 	}
@@ -223,32 +233,75 @@ struct BlobGranuleChunkRef {
 	VectorRef<BlobFilePointerRef> deltaFiles;
 	GranuleDeltas newDeltas;
 	Optional<KeyRef> tenantPrefix;
-	Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar,
-		           keyRange,
-		           includedVersion,
-		           snapshotVersion,
-		           snapshotFile,
-		           deltaFiles,
-		           newDeltas,
-		           tenantPrefix,
-		           cipherKeysCtx);
+		serializer(ar, keyRange, includedVersion, snapshotVersion, snapshotFile, deltaFiles, newDeltas, tenantPrefix);
 	}
 };
 
+struct BlobGranuleSummaryRef {
+	constexpr static FileIdentifier file_identifier = 9774587;
+	KeyRangeRef keyRange;
+	Version snapshotVersion;
+	int64_t snapshotSize;
+	Version deltaVersion;
+	int64_t deltaSize;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, keyRange, snapshotVersion, snapshotSize, deltaVersion, deltaSize);
+	}
+};
+
+BlobGranuleSummaryRef summarizeGranuleChunk(Arena& ar, const BlobGranuleChunkRef& chunk);
+
 enum BlobGranuleSplitState { Unknown = 0, Initialized = 1, Assigned = 2, Done = 3 };
 
+// Boundary metadata for each range indexed by the beginning of the range.
+struct BlobGranuleMergeBoundary {
+	constexpr static FileIdentifier file_identifier = 557861;
+
+	// Hard boundaries represent backing regions we want to keep separate.
+	bool buddy;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, buddy);
+	}
+};
+
 struct BlobGranuleHistoryValue {
 	constexpr static FileIdentifier file_identifier = 991434;
 	UID granuleID;
-	VectorRef<std::pair<KeyRangeRef, Version>> parentGranules;
+	VectorRef<KeyRef> parentBoundaries;
+	VectorRef<Version> parentVersions;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, granuleID, parentGranules);
+		serializer(ar, granuleID, parentBoundaries, parentVersions);
+	}
+};
+
+struct GranuleHistory {
+	KeyRange range;
+	Version version;
+	Standalone<BlobGranuleHistoryValue> value;
+
+	GranuleHistory() {}
+
+	GranuleHistory(KeyRange range, Version version, Standalone<BlobGranuleHistoryValue> value)
+	  : range(range), version(version), value(value) {}
+};
+
+// A manifest to assist full fdb restore from blob granule files
+struct BlobManifest {
+	constexpr static FileIdentifier file_identifier = 298872;
+	VectorRef<KeyValueRef> rows;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, rows);
 	}
 };
 
diff --git a/fdbclient/include/fdbclient/BlobGranuleFiles.h b/fdbclient/include/fdbclient/BlobGranuleFiles.h
index 7ee7a62bd4..23faff3d03 100644
--- a/fdbclient/include/fdbclient/BlobGranuleFiles.h
+++ b/fdbclient/include/fdbclient/BlobGranuleFiles.h
@@ -26,18 +26,25 @@
 #include "fdbclient/BlobGranuleCommon.h"
 #include "flow/CompressionUtils.h"
 
-Value serializeChunkedSnapshot(Standalone<GranuleSnapshot> snapshot,
-                               int chunks,
+Value serializeChunkedSnapshot(const Standalone<StringRef>& fileNameRef,
+                               const Standalone<GranuleSnapshot>& snapshot,
+                               int chunkSize,
                                Optional<CompressionFilter> compressFilter,
-                               Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx = Optional<BlobGranuleCipherKeysCtx>());
+                               Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx = {});
 
-// FIXME: support sorted and chunked delta files
+Value serializeChunkedDeltaFile(const Standalone<StringRef>& fileNameRef,
+                                const Standalone<GranuleDeltas>& deltas,
+                                const KeyRangeRef& fileRange,
+                                int chunkSize,
+                                Optional<CompressionFilter> compressFilter,
+                                Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx = {});
 
 ErrorOr<RangeResult> loadAndMaterializeBlobGranules(const Standalone<VectorRef<BlobGranuleChunkRef>>& files,
                                                     const KeyRangeRef& keyRange,
                                                     Version beginVersion,
                                                     Version readVersion,
-                                                    ReadBlobGranuleContext granuleContext);
+                                                    ReadBlobGranuleContext granuleContext,
+                                                    GranuleMaterializeStats& stats);
 
 RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
                                    KeyRangeRef keyRange,
diff --git a/fdbclient/include/fdbclient/BlobGranuleReader.actor.h b/fdbclient/include/fdbclient/BlobGranuleReader.actor.h
index a9008b03eb..395b76b26c 100644
--- a/fdbclient/include/fdbclient/BlobGranuleReader.actor.h
+++ b/fdbclient/include/fdbclient/BlobGranuleReader.actor.h
@@ -51,5 +51,7 @@ ACTOR Future<Void> readBlobGranules(BlobGranuleFileRequest request,
                                     Reference<BlobConnectionProvider> bstore,
                                     PromiseStream<RangeResult> results);
 
+bool isRangeFullyCovered(KeyRange range, Standalone<VectorRef<BlobGranuleChunkRef>> blobChunks);
+
 #include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbclient/include/fdbclient/BlobWorkerCommon.h b/fdbclient/include/fdbclient/BlobWorkerCommon.h
index 9912d4d2c7..9539db459b 100644
--- a/fdbclient/include/fdbclient/BlobWorkerCommon.h
+++ b/fdbclient/include/fdbclient/BlobWorkerCommon.h
@@ -30,7 +30,7 @@ struct BlobWorkerStats {
 	Counter deltaBytesWritten, snapshotBytesWritten;
 	Counter bytesReadFromFDBForInitialSnapshot;
 	Counter bytesReadFromS3ForCompaction;
-	Counter rangeAssignmentRequests, readRequests;
+	Counter rangeAssignmentRequests, readRequests, summaryReads;
 	Counter wrongShardServer;
 	Counter changeFeedInputBytes;
 	Counter readReqTotalFilesReturned;
@@ -41,16 +41,33 @@ struct BlobWorkerStats {
 	Counter readRequestsWithBegin;
 	Counter readRequestsCollapsed;
 	Counter flushGranuleReqs;
+	Counter compressionBytesRaw;
+	Counter compressionBytesFinal;
+	Counter fullRejections;
+	Counter forceFlushCleanups;
 
 	int numRangesAssigned;
 	int mutationBytesBuffered;
 	int activeReadRequests;
 	int granulesPendingSplitCheck;
+	Version minimumCFVersion;
+	Version cfVersionLag;
+	int notAtLatestChangeFeeds;
+	int64_t lastResidentMemory;
+	int64_t estimatedMaxResidentMemory;
+
+	Reference<FlowLock> initialSnapshotLock;
+	Reference<FlowLock> resnapshotLock;
+	Reference<FlowLock> deltaWritesLock;
 
 	Future<Void> logger;
 
 	// Current stats maintained for a given blob worker process
-	explicit BlobWorkerStats(UID id, double interval)
+	explicit BlobWorkerStats(UID id,
+	                         double interval,
+	                         Reference<FlowLock> initialSnapshotLock,
+	                         Reference<FlowLock> resnapshotLock,
+	                         Reference<FlowLock> deltaWritesLock)
 	  : cc("BlobWorkerStats", id.toString()),
 
 	    s3PutReqs("S3PutReqs", cc), s3GetReqs("S3GetReqs", cc), s3DeleteReqs("S3DeleteReqs", cc),
@@ -59,17 +76,32 @@ struct BlobWorkerStats {
 	    bytesReadFromFDBForInitialSnapshot("BytesReadFromFDBForInitialSnapshot", cc),
 	    bytesReadFromS3ForCompaction("BytesReadFromS3ForCompaction", cc),
 	    rangeAssignmentRequests("RangeAssignmentRequests", cc), readRequests("ReadRequests", cc),
-	    wrongShardServer("WrongShardServer", cc), changeFeedInputBytes("RangeFeedInputBytes", cc),
-	    readReqTotalFilesReturned("ReadReqTotalFilesReturned", cc),
+	    summaryReads("SummaryReads", cc), wrongShardServer("WrongShardServer", cc),
+	    changeFeedInputBytes("ChangeFeedInputBytes", cc), readReqTotalFilesReturned("ReadReqTotalFilesReturned", cc),
 	    readReqDeltaBytesReturned("ReadReqDeltaBytesReturned", cc), commitVersionChecks("CommitVersionChecks", cc),
 	    granuleUpdateErrors("GranuleUpdateErrors", cc), granuleRequestTimeouts("GranuleRequestTimeouts", cc),
 	    readRequestsWithBegin("ReadRequestsWithBegin", cc), readRequestsCollapsed("ReadRequestsCollapsed", cc),
-	    flushGranuleReqs("FlushGranuleReqs", cc), numRangesAssigned(0), mutationBytesBuffered(0), activeReadRequests(0),
-	    granulesPendingSplitCheck(0) {
+	    flushGranuleReqs("FlushGranuleReqs", cc), compressionBytesRaw("CompressionBytesRaw", cc),
+	    compressionBytesFinal("CompressionBytesFinal", cc), fullRejections("FullRejections", cc),
+	    forceFlushCleanups("ForceFlushCleanups", cc), numRangesAssigned(0), mutationBytesBuffered(0),
+	    activeReadRequests(0), granulesPendingSplitCheck(0), minimumCFVersion(0), cfVersionLag(0),
+	    notAtLatestChangeFeeds(0), lastResidentMemory(0), estimatedMaxResidentMemory(0),
+	    initialSnapshotLock(initialSnapshotLock), resnapshotLock(resnapshotLock), deltaWritesLock(deltaWritesLock) {
 		specialCounter(cc, "NumRangesAssigned", [this]() { return this->numRangesAssigned; });
 		specialCounter(cc, "MutationBytesBuffered", [this]() { return this->mutationBytesBuffered; });
 		specialCounter(cc, "ActiveReadRequests", [this]() { return this->activeReadRequests; });
 		specialCounter(cc, "GranulesPendingSplitCheck", [this]() { return this->granulesPendingSplitCheck; });
+		specialCounter(cc, "MinimumChangeFeedVersion", [this]() { return this->minimumCFVersion; });
+		specialCounter(cc, "CFVersionLag", [this]() { return this->cfVersionLag; });
+		specialCounter(cc, "NotAtLatestChangeFeeds", [this]() { return this->notAtLatestChangeFeeds; });
+		specialCounter(cc, "LastResidentMemory", [this]() { return this->lastResidentMemory; });
+		specialCounter(cc, "EstimatedMaxResidentMemory", [this]() { return this->estimatedMaxResidentMemory; });
+		specialCounter(cc, "InitialSnapshotsActive", [this]() { return this->initialSnapshotLock->activePermits(); });
+		specialCounter(cc, "InitialSnapshotsWaiting", [this]() { return this->initialSnapshotLock->waiters(); });
+		specialCounter(cc, "ReSnapshotsActive", [this]() { return this->resnapshotLock->activePermits(); });
+		specialCounter(cc, "ReSnapshotsWaiting", [this]() { return this->resnapshotLock->waiters(); });
+		specialCounter(cc, "DeltaFileWritesActive", [this]() { return this->deltaWritesLock->activePermits(); });
+		specialCounter(cc, "DeltaFileWritesWaiting", [this]() { return this->deltaWritesLock->waiters(); });
 
 		logger = traceCounters("BlobWorkerMetrics", id, interval, &cc, "BlobWorkerMetrics");
 	}
diff --git a/fdbclient/include/fdbclient/BlobWorkerInterface.h b/fdbclient/include/fdbclient/BlobWorkerInterface.h
index de370d248f..69d938300e 100644
--- a/fdbclient/include/fdbclient/BlobWorkerInterface.h
+++ b/fdbclient/include/fdbclient/BlobWorkerInterface.h
@@ -30,15 +30,15 @@
 
 struct BlobWorkerInterface {
 	constexpr static FileIdentifier file_identifier = 8358753;
-	// TODO: mimic what StorageServerInterface does with sequential endpoint IDs
 	RequestStream<ReplyPromise<Void>> waitFailure;
-	RequestStream<struct BlobGranuleFileRequest> blobGranuleFileRequest;
+	PublicRequestStream<struct BlobGranuleFileRequest> blobGranuleFileRequest;
 	RequestStream<struct AssignBlobRangeRequest> assignBlobRangeRequest;
 	RequestStream<struct RevokeBlobRangeRequest> revokeBlobRangeRequest;
 	RequestStream<struct GetGranuleAssignmentsRequest> granuleAssignmentsRequest;
 	RequestStream<struct GranuleStatusStreamRequest> granuleStatusStreamRequest;
 	RequestStream<struct HaltBlobWorkerRequest> haltBlobWorker;
 	RequestStream<struct FlushGranuleRequest> flushGranuleRequest;
+	RequestStream<struct MinBlobVersionRequest> minBlobVersionRequest;
 
 	struct LocalityData locality;
 	UID myId;
@@ -57,6 +57,7 @@ struct BlobWorkerInterface {
 		streams.push_back(granuleStatusStreamRequest.getReceiver());
 		streams.push_back(haltBlobWorker.getReceiver());
 		streams.push_back(flushGranuleRequest.getReceiver());
+		streams.push_back(minBlobVersionRequest.getReceiver());
 		FlowTransport::transport().addEndpoints(streams);
 	}
 	UID id() const { return myId; }
@@ -72,7 +73,7 @@ struct BlobWorkerInterface {
 		serializer(ar, myId, locality, waitFailure);
 		if (Archive::isDeserializing) {
 			blobGranuleFileRequest =
-			    RequestStream<struct BlobGranuleFileRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(1));
+			    PublicRequestStream<struct BlobGranuleFileRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(1));
 			assignBlobRangeRequest =
 			    RequestStream<struct AssignBlobRangeRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(2));
 			revokeBlobRangeRequest =
@@ -85,6 +86,8 @@ struct BlobWorkerInterface {
 			    RequestStream<struct HaltBlobWorkerRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(6));
 			flushGranuleRequest =
 			    RequestStream<struct FlushGranuleRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(7));
+			minBlobVersionRequest =
+			    RequestStream<struct MinBlobVersionRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(8));
 		}
 	}
 };
@@ -110,13 +113,16 @@ struct BlobGranuleFileRequest {
 	Version readVersion;
 	bool canCollapseBegin = true;
 	TenantInfo tenantInfo;
+	bool summarize = false;
 	ReplyPromise<BlobGranuleFileReply> reply;
 
 	BlobGranuleFileRequest() {}
 
+	bool verify() const { return tenantInfo.isAuthorized(); }
+
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, keyRange, beginVersion, readVersion, canCollapseBegin, tenantInfo, reply, arena);
+		serializer(ar, keyRange, beginVersion, readVersion, canCollapseBegin, tenantInfo, summarize, reply, arena);
 	}
 };
 
@@ -137,6 +143,28 @@ struct RevokeBlobRangeRequest {
 	}
 };
 
+struct MinBlobVersionReply {
+	constexpr static FileIdentifier file_identifier = 6857512;
+	Version version;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version);
+	}
+};
+
+struct MinBlobVersionRequest {
+	constexpr static FileIdentifier file_identifier = 4833278;
+	Version grv;
+	ReplyPromise<MinBlobVersionReply> reply;
+
+	MinBlobVersionRequest() {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, grv, reply);
+	}
+};
 /*
  * Continue: Blob worker should continue handling a granule that was evaluated for a split
  * Normal: Blob worker should open the granule and start processing it
@@ -172,6 +200,7 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply {
 	KeyRange granuleRange;
 	bool doSplit;
 	bool writeHotSplit;
+	bool initialSplitTooBig;
 	int64_t continueEpoch;
 	int64_t continueSeqno;
 	UID granuleID;
@@ -180,11 +209,13 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply {
 	bool mergeCandidate;
 	int64_t originalEpoch;
 	int64_t originalSeqno;
+	Optional<Key> proposedSplitKey;
 
 	GranuleStatusReply() {}
 	explicit GranuleStatusReply(KeyRange range,
 	                            bool doSplit,
 	                            bool writeHotSplit,
+	                            bool initialSplitTooBig,
 	                            int64_t continueEpoch,
 	                            int64_t continueSeqno,
 	                            UID granuleID,
@@ -193,11 +224,15 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply {
 	                            bool mergeCandidate,
 	                            int64_t originalEpoch,
 	                            int64_t originalSeqno)
-	  : granuleRange(range), doSplit(doSplit), writeHotSplit(writeHotSplit), continueEpoch(continueEpoch),
-	    continueSeqno(continueSeqno), granuleID(granuleID), startVersion(startVersion), blockedVersion(blockedVersion),
-	    mergeCandidate(mergeCandidate), originalEpoch(originalEpoch), originalSeqno(originalSeqno) {}
+	  : granuleRange(range), doSplit(doSplit), writeHotSplit(writeHotSplit), initialSplitTooBig(initialSplitTooBig),
+	    continueEpoch(continueEpoch), continueSeqno(continueSeqno), granuleID(granuleID), startVersion(startVersion),
+	    blockedVersion(blockedVersion), mergeCandidate(mergeCandidate), originalEpoch(originalEpoch),
+	    originalSeqno(originalSeqno) {}
 
-	int expectedSize() const { return sizeof(GranuleStatusReply) + granuleRange.expectedSize(); }
+	int expectedSize() const {
+		return sizeof(GranuleStatusReply) + granuleRange.expectedSize() +
+		       (proposedSplitKey.present() ? proposedSplitKey.get().expectedSize() : 0);
+	}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -207,6 +242,7 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply {
 		           granuleRange,
 		           doSplit,
 		           writeHotSplit,
+		           initialSplitTooBig,
 		           continueEpoch,
 		           continueSeqno,
 		           granuleID,
@@ -214,7 +250,8 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply {
 		           blockedVersion,
 		           mergeCandidate,
 		           originalEpoch,
-		           originalSeqno);
+		           originalSeqno,
+		           proposedSplitKey);
 	}
 };
 
diff --git a/fdbclient/include/fdbclient/ClientKnobs.h b/fdbclient/include/fdbclient/ClientKnobs.h
index 6e60b56cbe..61f0359539 100644
--- a/fdbclient/include/fdbclient/ClientKnobs.h
+++ b/fdbclient/include/fdbclient/ClientKnobs.h
@@ -39,10 +39,6 @@ public:
 
 	double FAILURE_MAX_DELAY;
 	double FAILURE_MIN_DELAY;
-	double FAILURE_TIMEOUT_DELAY;
-	double CLIENT_FAILURE_TIMEOUT_DELAY;
-	double FAILURE_EMERGENCY_DELAY;
-	double FAILURE_MAX_GENERATIONS;
 	double RECOVERY_DELAY_START_GENERATION;
 	double RECOVERY_DELAY_SECONDS_PER_GENERATION;
 	double MAX_GENERATIONS;
@@ -61,6 +57,7 @@ public:
 	double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
 	                                 // mostly wrong (e.g. dumping the database after a test)
 	double FUTURE_VERSION_RETRY_DELAY;
+	double GRV_ERROR_RETRY_DELAY;
 	double UNKNOWN_TENANT_RETRY_DELAY;
 	int REPLY_BYTE_LIMIT;
 	double DEFAULT_BACKOFF;
@@ -81,7 +78,7 @@ public:
 	int64_t CHANGE_FEED_CACHE_SIZE;
 	double CHANGE_FEED_POP_TIMEOUT;
 	int64_t CHANGE_FEED_STREAM_MIN_BYTES;
-	int64_t TENANT_PREFIX_SIZE_LIMIT;
+	double CHANGE_FEED_START_INTERVAL;
 
 	int MAX_BATCH_SIZE;
 	double GRV_BATCH_TIMEOUT;
@@ -110,7 +107,6 @@ public:
 	int RANGESTREAM_BUFFERED_FRAGMENTS_LIMIT;
 	bool QUARANTINE_TSS_ON_MISMATCH;
 	double CHANGE_FEED_EMPTY_BATCH_TIME;
-	bool SHARD_ENCODE_LOCATION_METADATA;
 
 	// KeyRangeMap
 	int KRM_GET_RANGE_LIMIT;
@@ -163,10 +159,8 @@ public:
 	double BACKUP_AGGREGATE_POLL_RATE;
 	double BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL;
 	int BACKUP_LOG_WRITE_BATCH_MAX_SIZE;
-	int BACKUP_LOG_ATOMIC_OPS_SIZE;
 	int BACKUP_MAX_LOG_RANGES;
 	int BACKUP_SIM_COPY_LOG_RANGES;
-	int BACKUP_OPERATION_COST_OVERHEAD;
 	int BACKUP_VERSION_DELAY;
 	int BACKUP_MAP_KEY_LOWER_LIMIT;
 	int BACKUP_MAP_KEY_UPPER_LIMIT;
@@ -206,6 +200,10 @@ public:
 	int32_t DEFAULT_AUTO_RESOLVERS;
 	int32_t DEFAULT_AUTO_LOGS;
 
+	double GLOBAL_CONFIG_REFRESH_BACKOFF;
+	double GLOBAL_CONFIG_REFRESH_MAX_BACKOFF;
+	double GLOBAL_CONFIG_REFRESH_TIMEOUT;
+
 	// Dynamic Knobs
 	double COMMIT_QUORUM_TIMEOUT;
 	double GET_GENERATION_QUORUM_TIMEOUT;
@@ -217,12 +215,7 @@ public:
 	int64_t CSI_SIZE_LIMIT;
 	double CSI_STATUS_DELAY;
 
-	int HTTP_SEND_SIZE;
-	int HTTP_READ_SIZE;
-	int HTTP_VERBOSE_LEVEL;
-	std::string HTTP_REQUEST_ID_HEADER;
 	bool HTTP_REQUEST_AWS_V4_HEADER; // setting this knob to true will enable AWS V4 style header.
-	bool HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT; // skip verify md5 checksum for 206 response
 	std::string BLOBSTORE_ENCRYPTION_TYPE;
 	int BLOBSTORE_CONNECT_TRIES;
 	int BLOBSTORE_CONNECT_TIMEOUT;
@@ -261,23 +254,21 @@ public:
 	int MAX_TRANSACTION_TAG_LENGTH;
 	int MAX_TAGS_PER_TRANSACTION;
 	int COMMIT_SAMPLE_COST; // The expectation of sampling is every COMMIT_SAMPLE_COST sample once
-	int WRITE_COST_BYTE_FACTOR;
 	int INCOMPLETE_SHARD_PLUS; // The size of (possible) incomplete shard when estimate clear range
 	double READ_TAG_SAMPLE_RATE; // Communicated to clients from cluster
 	double TAG_THROTTLE_SMOOTHING_WINDOW;
 	double TAG_THROTTLE_RECHECK_INTERVAL;
 	double TAG_THROTTLE_EXPIRATION_INTERVAL;
+	int64_t WRITE_COST_BYTE_FACTOR; // Used to round up the cost of write operations
+	int64_t READ_COST_BYTE_FACTOR; // Used to round up the cost of read operations
 
 	// busyness reporting
 	double BUSYNESS_SPIKE_START_THRESHOLD;
 	double BUSYNESS_SPIKE_SATURATED_THRESHOLD;
 
-	// multi-version client control
-	int MVC_CLIENTLIB_CHUNK_SIZE;
-	int MVC_CLIENTLIB_CHUNKS_PER_TRANSACTION;
-
 	// Blob Granules
 	int BG_MAX_GRANULE_PARALLELISM;
+	int BG_TOO_MANY_GRANULES;
 
 	// The coordinator key/value in storage server might be inconsistent to the value stored in the cluster file.
 	// This might happen when a recovery is happening together with a cluster controller coordinator key change.
@@ -286,6 +277,19 @@ public:
 	int CHANGE_QUORUM_BAD_STATE_RETRY_TIMES;
 	double CHANGE_QUORUM_BAD_STATE_RETRY_DELAY;
 
+	// Tenants and Metacluster
+	int MAX_TENANTS_PER_CLUSTER;
+	int TENANT_TOMBSTONE_CLEANUP_INTERVAL;
+	int MAX_DATA_CLUSTERS;
+	int REMOVE_CLUSTER_TENANT_BATCH_SIZE;
+	int METACLUSTER_ASSIGNMENT_CLUSTERS_TO_CHECK;
+	double METACLUSTER_ASSIGNMENT_FIRST_CHOICE_DELAY;
+	double METACLUSTER_ASSIGNMENT_AVAILABILITY_TIMEOUT;
+	int TENANT_ENTRY_CACHE_LIST_REFRESH_INTERVAL; // How often the TenantEntryCache is refreshed
+
+	// Encryption-at-rest
+	bool ENABLE_ENCRYPTION_CPU_TIME_LOGGING;
+
 	ClientKnobs(Randomize randomize);
 	void initialize(Randomize randomize);
 };
diff --git a/fdbclient/include/fdbclient/ClientVersion.h b/fdbclient/include/fdbclient/ClientVersion.h
index fe3068affc..c395a69cc2 100644
--- a/fdbclient/include/fdbclient/ClientVersion.h
+++ b/fdbclient/include/fdbclient/ClientVersion.h
@@ -37,7 +37,7 @@ struct ClientVersionRef {
 	ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion)
 	  : clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
 	ClientVersionRef(StringRef versionString) {
-		std::vector<StringRef> parts = versionString.splitAny(LiteralStringRef(","));
+		std::vector<StringRef> parts = versionString.splitAny(","_sr);
 		if (parts.size() != 3) {
 			initUnknown();
 			return;
@@ -48,9 +48,9 @@ struct ClientVersionRef {
 	}
 
 	void initUnknown() {
-		clientVersion = LiteralStringRef("Unknown");
-		sourceVersion = LiteralStringRef("Unknown");
-		protocolVersion = LiteralStringRef("Unknown");
+		clientVersion = "Unknown"_sr;
+		sourceVersion = "Unknown"_sr;
+		protocolVersion = "Unknown"_sr;
 	}
 
 	template <class Ar>
diff --git a/fdbclient/include/fdbclient/ClusterInterface.h b/fdbclient/include/fdbclient/ClusterInterface.h
index 14935f1700..a4e3da44f3 100644
--- a/fdbclient/include/fdbclient/ClusterInterface.h
+++ b/fdbclient/include/fdbclient/ClusterInterface.h
@@ -98,32 +98,44 @@ struct ClusterControllerClientInterface {
 	}
 };
 
-template <class T>
-struct ItemWithExamples {
-	T item;
-	int count;
-	std::vector<std::pair<NetworkAddress, Key>> examples;
-
-	ItemWithExamples() : item{}, count(0) {}
-	ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress, Key>> const& examples)
-	  : item(item), count(count), examples(examples) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, item, count, examples);
-	}
-};
-
 struct OpenDatabaseRequest {
 	constexpr static FileIdentifier file_identifier = 2799502;
 	// Sent by the native API to the cluster controller to open a database and track client
 	//   info changes.  Returns immediately if the current client info id is different from
 	//   knownClientInfoID; otherwise returns when it next changes (or perhaps after a long interval)
 
-	int clientCount;
-	std::vector<ItemWithExamples<Key>> issues;
-	std::vector<ItemWithExamples<Standalone<ClientVersionRef>>> supportedVersions;
-	std::vector<ItemWithExamples<Key>> maxProtocolSupported;
+	struct Samples {
+		int count;
+
+		// network address / trace log group
+		std::set<std::pair<NetworkAddress, Key>> samples;
+
+		Samples() : count(0), samples{} {}
+
+		template <typename Ar>
+		void serialize(Ar& ar) {
+			serializer(ar, count, samples);
+		}
+
+		// Merges a set of Samples into *this
+		Samples& operator+=(const Samples& other) {
+			count += other.count;
+			samples.insert(std::begin(other.samples), std::end(other.samples));
+
+			return *this;
+		}
+	};
+
+	int clientCount = 0;
+
+	// Maps issue to Samples
+	std::map<Key, Samples> issues;
+
+	// Maps ClientVersionRef to Samples
+	std::map<Standalone<ClientVersionRef>, Samples> supportedVersions;
+
+	// Maps max protocol to Samples
+	std::map<Key, Samples> maxProtocolSupported;
 
 	UID knownClientInfoID;
 	ReplyPromise<struct ClientDBInfo> reply;
diff --git a/fdbclient/include/fdbclient/CommitProxyInterface.h b/fdbclient/include/fdbclient/CommitProxyInterface.h
index 9a8095f808..1614aeacf0 100644
--- a/fdbclient/include/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/include/fdbclient/CommitProxyInterface.h
@@ -25,16 +25,17 @@
 #include <utility>
 #include <vector>
 
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/CommitTransaction.h"
-#include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/EncryptKeyProxyInterface.h"
+#include "fdbclient/FDBTypes.h"
 #include "fdbclient/GlobalConfig.h"
+#include "fdbclient/GrvProxyInterface.h"
+#include "fdbclient/StorageServerInterface.h"
+#include "fdbclient/TagThrottle.actor.h"
 #include "fdbclient/VersionVector.h"
 
 #include "fdbrpc/Stats.h"
 #include "fdbrpc/TimedRequest.h"
-#include "GrvProxyInterface.h"
 
 struct CommitProxyInterface {
 	constexpr static FileIdentifier file_identifier = 8954922;
@@ -117,8 +118,12 @@ struct ClientDBInfo {
 	Optional<Value> forward;
 	std::vector<VersionHistory> history;
 	UID clusterId;
+	bool isEncryptionEnabled = false;
+	Optional<EncryptKeyProxyInterface> encryptKeyProxy;
 
 	TenantMode tenantMode;
+	ClusterType clusterType = ClusterType::STANDALONE;
+	Optional<ClusterName> metaclusterName;
 
 	ClientDBInfo() {}
 
@@ -130,7 +135,18 @@ struct ClientDBInfo {
 		if constexpr (!is_fb_function<Archive>) {
 			ASSERT(ar.protocolVersion().isValid());
 		}
-		serializer(ar, grvProxies, commitProxies, id, forward, history, tenantMode, clusterId);
+		serializer(ar,
+		           grvProxies,
+		           commitProxies,
+		           id,
+		           forward,
+		           history,
+		           tenantMode,
+		           isEncryptionEnabled,
+		           encryptKeyProxy,
+		           clusterId,
+		           clusterType,
+		           metaclusterName);
 	}
 };
 
@@ -176,10 +192,12 @@ struct CommitTransactionRequest : TimedRequest {
 	CommitTransactionRequest() : CommitTransactionRequest(SpanContext()) {}
 	CommitTransactionRequest(SpanContext const& context) : spanContext(context), flags(0) {}
 
+	bool verify() const { return tenantInfo.isAuthorized(); }
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(
-		    ar, transaction, reply, arena, flags, debugID, commitCostEstimation, tagSet, spanContext, tenantInfo);
+		    ar, transaction, reply, flags, debugID, commitCostEstimation, tagSet, spanContext, tenantInfo, arena);
 	}
 };
 
@@ -281,6 +299,8 @@ struct GetReadVersionRequest : TimedRequest {
 		}
 	}
 
+	bool verify() const { return true; }
+
 	bool operator<(GetReadVersionRequest const& rhs) const { return priority < rhs.priority; }
 
 	template <class Ar>
@@ -319,7 +339,7 @@ struct GetKeyServerLocationsReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, results, resultsTssMapping, tenantEntry, arena, resultsTagMapping);
+		serializer(ar, results, resultsTssMapping, tenantEntry, resultsTagMapping, arena);
 	}
 };
 
@@ -327,7 +347,7 @@ struct GetKeyServerLocationsRequest {
 	constexpr static FileIdentifier file_identifier = 9144680;
 	Arena arena;
 	SpanContext spanContext;
-	Optional<TenantNameRef> tenant;
+	TenantInfo tenant;
 	KeyRef begin;
 	Optional<KeyRef> end;
 	int limit;
@@ -342,7 +362,7 @@ struct GetKeyServerLocationsRequest {
 
 	GetKeyServerLocationsRequest() : limit(0), reverse(false), minTenantVersion(latestVersion) {}
 	GetKeyServerLocationsRequest(SpanContext spanContext,
-	                             Optional<TenantNameRef> const& tenant,
+	                             TenantInfo const& tenant,
 	                             KeyRef const& begin,
 	                             Optional<KeyRef> const& end,
 	                             int limit,
@@ -352,6 +372,8 @@ struct GetKeyServerLocationsRequest {
 	  : arena(arena), spanContext(spanContext), tenant(tenant), begin(begin), end(end), limit(limit), reverse(reverse),
 	    minTenantVersion(minTenantVersion) {}
 
+	bool verify() const { return tenant.isAuthorized(); }
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, begin, end, limit, reverse, reply, spanContext, tenant, minTenantVersion, arena);
@@ -521,7 +543,7 @@ struct ProxySnapRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, snapPayload, snapUID, reply, arena, debugID);
+		serializer(ar, snapPayload, snapUID, reply, debugID, arena);
 	}
 };
 
@@ -552,4 +574,32 @@ struct ExclusionSafetyCheckRequest {
 	}
 };
 
+struct GlobalConfigRefreshReply {
+	constexpr static FileIdentifier file_identifier = 12680327;
+	Arena arena;
+	RangeResultRef result;
+
+	GlobalConfigRefreshReply() {}
+	GlobalConfigRefreshReply(Arena const& arena, RangeResultRef result) : arena(arena), result(result) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, result, arena);
+	}
+};
+
+struct GlobalConfigRefreshRequest {
+	constexpr static FileIdentifier file_identifier = 2828131;
+	Version lastKnown;
+	ReplyPromise<GlobalConfigRefreshReply> reply;
+
+	GlobalConfigRefreshRequest() {}
+	explicit GlobalConfigRefreshRequest(Version lastKnown) : lastKnown(lastKnown) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, lastKnown, reply);
+	}
+};
+
 #endif
diff --git a/fdbclient/include/fdbclient/CommitTransaction.h b/fdbclient/include/fdbclient/CommitTransaction.h
index f6757ac17e..7bc080b130 100644
--- a/fdbclient/include/fdbclient/CommitTransaction.h
+++ b/fdbclient/include/fdbclient/CommitTransaction.h
@@ -22,9 +22,13 @@
 #define FLOW_FDBCLIENT_COMMITTRANSACTION_H
 #pragma once
 
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/FDBTypes.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/Tracing.h"
+#include "flow/EncryptUtils.h"
+#include "flow/Knobs.h"
 
 // The versioned message has wire format : -1, version, messages
 static const int32_t VERSION_HEADER = -1;
@@ -79,7 +83,7 @@ struct MutationRef {
 		CompareAndClear,
 		Reserved_For_SpanContextMessage /* See fdbserver/SpanContextMessage.h */,
 		Reserved_For_OTELSpanContextMessage,
-		Reserved_For_EncryptedMutationMessage /* See fdbserver/EncryptedMutationMessage.actor.h */,
+		Encrypted, /* Represents an encrypted mutation and cannot be used directly before decrypting */
 		MAX_ATOMIC_OP
 	};
 	// This is stored this way for serialization purposes.
@@ -128,6 +132,80 @@ struct MutationRef {
 		}
 	}
 
+	// An encrypted mutation has type Encrypted, encryption header (which contains encryption metadata) as param1,
+	// and the payload as param2. It can be serialize/deserialize as normal mutation, but can only be used after
+	// decryption via decrypt().
+	bool isEncrypted() const { return type == Encrypted; }
+
+	const BlobCipherEncryptHeader* encryptionHeader() const {
+		ASSERT(isEncrypted());
+		return reinterpret_cast<const BlobCipherEncryptHeader*>(param1.begin());
+	}
+
+	MutationRef encrypt(const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys,
+	                    const EncryptCipherDomainId& domainId,
+	                    Arena& arena,
+	                    BlobCipherMetrics::UsageType usageType) const {
+		ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID);
+		auto textCipherItr = cipherKeys.find(domainId);
+		auto headerCipherItr = cipherKeys.find(ENCRYPT_HEADER_DOMAIN_ID);
+		ASSERT(textCipherItr != cipherKeys.end() && textCipherItr->second.isValid());
+		ASSERT(headerCipherItr != cipherKeys.end() && headerCipherItr->second.isValid());
+		uint8_t iv[AES_256_IV_LENGTH] = { 0 };
+		deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH);
+		BinaryWriter bw(AssumeVersion(ProtocolVersion::withEncryptionAtRest()));
+		bw << *this;
+		EncryptBlobCipherAes265Ctr cipher(
+		    textCipherItr->second,
+		    headerCipherItr->second,
+		    iv,
+		    AES_256_IV_LENGTH,
+		    getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
+		    usageType);
+		BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
+		StringRef headerRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader));
+		StringRef payload =
+		    cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef();
+		return MutationRef(Encrypted, headerRef, payload);
+	}
+
+	MutationRef encryptMetadata(const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys,
+	                            Arena& arena,
+	                            BlobCipherMetrics::UsageType usageType) const {
+		return encrypt(cipherKeys, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, arena, usageType);
+	}
+
+	MutationRef decrypt(TextAndHeaderCipherKeys cipherKeys,
+	                    Arena& arena,
+	                    BlobCipherMetrics::UsageType usageType,
+	                    StringRef* buf = nullptr) const {
+		const BlobCipherEncryptHeader* header = encryptionHeader();
+		DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header->iv, usageType);
+		StringRef plaintext = cipher.decrypt(param2.begin(), param2.size(), *header, arena)->toStringRef();
+		if (buf != nullptr) {
+			*buf = plaintext;
+		}
+		ArenaReader reader(arena, plaintext, AssumeVersion(ProtocolVersion::withEncryptionAtRest()));
+		MutationRef mutation;
+		reader >> mutation;
+		return mutation;
+	}
+
+	MutationRef decrypt(const std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>& cipherKeys,
+	                    Arena& arena,
+	                    BlobCipherMetrics::UsageType usageType,
+	                    StringRef* buf = nullptr) const {
+		const BlobCipherEncryptHeader* header = encryptionHeader();
+		auto textCipherItr = cipherKeys.find(header->cipherTextDetails);
+		auto headerCipherItr = cipherKeys.find(header->cipherHeaderDetails);
+		ASSERT(textCipherItr != cipherKeys.end() && textCipherItr->second.isValid());
+		ASSERT(headerCipherItr != cipherKeys.end() && headerCipherItr->second.isValid());
+		TextAndHeaderCipherKeys textAndHeaderKeys;
+		textAndHeaderKeys.cipherHeaderKey = headerCipherItr->second;
+		textAndHeaderKeys.cipherTextKey = textCipherItr->second;
+		return decrypt(textAndHeaderKeys, arena, usageType, buf);
+	}
+
 	// These masks define which mutation types have particular properties (they are used to implement
 	// isSingleKeyMutation() etc)
 	enum {
@@ -190,6 +268,11 @@ struct CommitTransactionRef {
 	VectorRef<KeyRangeRef> read_conflict_ranges;
 	VectorRef<KeyRangeRef> write_conflict_ranges;
 	VectorRef<MutationRef> mutations; // metadata mutations
+	// encryptedMutations should be a 1-1 corespondence with mutations field above. That is either
+	// encryptedMutations.size() == 0 or encryptedMutations.size() == mutations.size() and encryptedMutations[i] =
+	// mutations[i].encrypt(). Currently this field is not serialized so clients should NOT set this field during a
+	// usual commit path. It is currently only used during backup mutation log restores.
+	VectorRef<Optional<MutationRef>> encryptedMutations;
 	Version read_snapshot = 0;
 	bool report_conflicting_keys = false;
 	bool lock_aware = false; // set when metadata mutations are present
diff --git a/fdbclient/include/fdbclient/ConfigKnobs.h b/fdbclient/include/fdbclient/ConfigKnobs.h
index 536bca16f3..168e2fed16 100644
--- a/fdbclient/include/fdbclient/ConfigKnobs.h
+++ b/fdbclient/include/fdbclient/ConfigKnobs.h
@@ -25,6 +25,8 @@
 
 #include "fdbclient/FDBTypes.h"
 
+typedef uint64_t CoordinatorsHash;
+
 /*
  * KnobValueRefs are stored in the configuration database, and in local configuration files. They are created from
  * ParsedKnobValue objects, so it is assumed that the value type is correct for the corresponding knob name
diff --git a/fdbclient/include/fdbclient/ConfigTransactionInterface.h b/fdbclient/include/fdbclient/ConfigTransactionInterface.h
index 98b65e4c4b..dad60f2d04 100644
--- a/fdbclient/include/fdbclient/ConfigTransactionInterface.h
+++ b/fdbclient/include/fdbclient/ConfigTransactionInterface.h
@@ -65,16 +65,18 @@ struct ConfigTransactionGetGenerationReply {
 
 struct ConfigTransactionGetGenerationRequest {
 	static constexpr FileIdentifier file_identifier = 138941;
+	CoordinatorsHash coordinatorsHash{ 0 };
 	// A hint to catch up lagging nodes:
 	Optional<Version> lastSeenLiveVersion;
 	ReplyPromise<ConfigTransactionGetGenerationReply> reply;
 	ConfigTransactionGetGenerationRequest() = default;
-	explicit ConfigTransactionGetGenerationRequest(Optional<Version> const& lastSeenLiveVersion)
-	  : lastSeenLiveVersion(lastSeenLiveVersion) {}
+	explicit ConfigTransactionGetGenerationRequest(CoordinatorsHash coordinatorsHash,
+	                                               Optional<Version> const& lastSeenLiveVersion)
+	  : coordinatorsHash(coordinatorsHash), lastSeenLiveVersion(lastSeenLiveVersion) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, lastSeenLiveVersion, reply);
+		serializer(ar, coordinatorsHash, lastSeenLiveVersion, reply);
 	}
 };
 
@@ -92,39 +94,43 @@ struct ConfigTransactionGetReply {
 
 struct ConfigTransactionGetRequest {
 	static constexpr FileIdentifier file_identifier = 923040;
+	CoordinatorsHash coordinatorsHash{ 0 };
 	ConfigGeneration generation;
 	ConfigKey key;
 	ReplyPromise<ConfigTransactionGetReply> reply;
 
 	ConfigTransactionGetRequest() = default;
-	explicit ConfigTransactionGetRequest(ConfigGeneration generation, ConfigKey key)
-	  : generation(generation), key(key) {}
+	explicit ConfigTransactionGetRequest(CoordinatorsHash coordinatorsHash, ConfigGeneration generation, ConfigKey key)
+	  : coordinatorsHash(coordinatorsHash), generation(generation), key(key) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, generation, key, reply);
+		serializer(ar, coordinatorsHash, generation, key, reply);
 	}
 };
 
 struct ConfigTransactionCommitRequest {
 	static constexpr FileIdentifier file_identifier = 103841;
 	Arena arena;
+	CoordinatorsHash coordinatorsHash{ 0 };
 	ConfigGeneration generation{ ::invalidVersion, ::invalidVersion };
 	VectorRef<ConfigMutationRef> mutations;
 	ConfigCommitAnnotationRef annotation;
 	ReplyPromise<Void> reply;
 
 	ConfigTransactionCommitRequest() = default;
-	explicit ConfigTransactionCommitRequest(ConfigGeneration generation,
+	explicit ConfigTransactionCommitRequest(CoordinatorsHash coordinatorsHash,
+	                                        ConfigGeneration generation,
 	                                        VectorRef<ConfigMutationRef> mutations,
 	                                        ConfigCommitAnnotationRef annotation)
-	  : generation(generation), mutations(arena, mutations), annotation(arena, annotation) {}
+	  : coordinatorsHash(coordinatorsHash), generation(generation), mutations(arena, mutations),
+	    annotation(arena, annotation) {}
 
 	size_t expectedSize() const { return mutations.expectedSize() + annotation.expectedSize(); }
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, generation, mutations, annotation, reply);
+		serializer(ar, coordinatorsHash, generation, mutations, annotation, reply, arena);
 	}
 };
 
@@ -144,15 +150,17 @@ struct ConfigTransactionGetConfigClassesReply {
 
 struct ConfigTransactionGetConfigClassesRequest {
 	static constexpr FileIdentifier file_identifier = 7163400;
+	CoordinatorsHash coordinatorsHash{ 0 };
 	ConfigGeneration generation;
 	ReplyPromise<ConfigTransactionGetConfigClassesReply> reply;
 
 	ConfigTransactionGetConfigClassesRequest() = default;
-	explicit ConfigTransactionGetConfigClassesRequest(ConfigGeneration generation) : generation(generation) {}
+	explicit ConfigTransactionGetConfigClassesRequest(CoordinatorsHash coordinatorsHash, ConfigGeneration generation)
+	  : coordinatorsHash(coordinatorsHash), generation(generation) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, generation);
+		serializer(ar, coordinatorsHash, generation);
 	}
 };
 
@@ -171,17 +179,20 @@ struct ConfigTransactionGetKnobsReply {
 
 struct ConfigTransactionGetKnobsRequest {
 	static constexpr FileIdentifier file_identifier = 987410;
+	CoordinatorsHash coordinatorsHash{ 0 };
 	ConfigGeneration generation;
 	Optional<Key> configClass;
 	ReplyPromise<ConfigTransactionGetKnobsReply> reply;
 
 	ConfigTransactionGetKnobsRequest() = default;
-	explicit ConfigTransactionGetKnobsRequest(ConfigGeneration generation, Optional<Key> configClass)
-	  : generation(generation), configClass(configClass) {}
+	explicit ConfigTransactionGetKnobsRequest(CoordinatorsHash coordinatorsHash,
+	                                          ConfigGeneration generation,
+	                                          Optional<Key> configClass)
+	  : coordinatorsHash(coordinatorsHash), generation(generation), configClass(configClass) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, generation, configClass, reply);
+		serializer(ar, coordinatorsHash, generation, configClass, reply);
 	}
 };
 
diff --git a/fdbclient/include/fdbclient/ConsistencyScanInterface.actor.h b/fdbclient/include/fdbclient/ConsistencyScanInterface.actor.h
new file mode 100644
index 0000000000..7d6529ced0
--- /dev/null
+++ b/fdbclient/include/fdbclient/ConsistencyScanInterface.actor.h
@@ -0,0 +1,196 @@
+/*
+ * ConsistencyScanInterface.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_CONSISTENCYSCANINTERFACE_ACTOR_G_H)
+#define FDBCLIENT_CONSISTENCYSCANINTERFACE_ACTOR_G_H
+#include "fdbclient/ConsistencyScanInterface.actor.g.h"
+#elif !defined(FDBCLIENT_CONSISTENCYSCANINTERFACE_ACTOR_H)
+#define FDBCLIENT_CONSISTENCYSCANINTERFACE_ACTOR_H
+
+#include "fdbclient/CommitProxyInterface.h"
+#include "fdbclient/DatabaseConfiguration.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/RunTransaction.actor.h"
+#include "fdbrpc/fdbrpc.h"
+#include "fdbrpc/Locality.h"
+
+#include "flow/actorcompiler.h" // must be last include
+
+struct ConsistencyScanInterface {
+	constexpr static FileIdentifier file_identifier = 4983265;
+	RequestStream<ReplyPromise<Void>> waitFailure;
+	RequestStream<struct HaltConsistencyScanRequest> haltConsistencyScan;
+	struct LocalityData locality;
+	UID myId;
+
+	ConsistencyScanInterface() {}
+	explicit ConsistencyScanInterface(const struct LocalityData& l, UID id) : locality(l), myId(id) {}
+
+	void initEndpoints() {}
+	UID id() const { return myId; }
+	NetworkAddress address() const { return waitFailure.getEndpoint().getPrimaryAddress(); }
+	bool operator==(const ConsistencyScanInterface& r) const { return id() == r.id(); }
+	bool operator!=(const ConsistencyScanInterface& r) const { return !(*this == r); }
+
+	template <class Archive>
+	void serialize(Archive& ar) {
+		serializer(ar, waitFailure, haltConsistencyScan, locality, myId);
+	}
+};
+
+struct HaltConsistencyScanRequest {
+	constexpr static FileIdentifier file_identifier = 2323417;
+	UID requesterID;
+	ReplyPromise<Void> reply;
+
+	HaltConsistencyScanRequest() {}
+	explicit HaltConsistencyScanRequest(UID uid) : requesterID(uid) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, requesterID, reply);
+	}
+};
+
+// consistency scan configuration and metrics
+struct ConsistencyScanInfo {
+	constexpr static FileIdentifier file_identifier = 732125;
+	bool consistency_scan_enabled = false;
+	bool restart = false;
+	int64_t max_rate = 0;
+	int64_t target_interval = CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
+	int64_t bytes_read_prev_round = 0;
+	KeyRef progress_key = KeyRef();
+
+	// Round Metrics - one round of complete validation across all SSs
+	// Start and finish are in epoch seconds
+	double last_round_start = 0;
+	double last_round_finish = 0;
+	TimerSmoother smoothed_round_duration;
+	int finished_rounds = 0;
+
+	ConsistencyScanInfo() : smoothed_round_duration(20.0 * 60) {}
+	ConsistencyScanInfo(bool enabled, bool r, uint64_t rate, uint64_t interval)
+	  : consistency_scan_enabled(enabled), restart(r), max_rate(rate), target_interval(interval),
+	    smoothed_round_duration(20.0 * 60) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		double round_total;
+		if (!ar.isDeserializing) {
+			round_total = smoothed_round_duration.getTotal();
+		}
+		serializer(ar,
+		           consistency_scan_enabled,
+		           restart,
+		           max_rate,
+		           target_interval,
+		           bytes_read_prev_round,
+		           last_round_start,
+		           last_round_finish,
+		           round_total,
+		           finished_rounds);
+		if (ar.isDeserializing) {
+			smoothed_round_duration.reset(round_total);
+		}
+	}
+
+	static Future<Void> setInfo(Reference<ReadYourWritesTransaction> tr, ConsistencyScanInfo info) {
+		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+		tr->set(consistencyScanInfoKey, ObjectWriter::toValue(info, IncludeVersion()));
+		return Void();
+	}
+
+	static Future<Void> setInfo(Database cx, ConsistencyScanInfo info) {
+		return runRYWTransaction(
+		    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> { return setInfo(tr, info); });
+	}
+
+	static Future<Optional<Value>> getInfo(Reference<ReadYourWritesTransaction> tr) {
+		tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+		tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+		return tr->get(consistencyScanInfoKey);
+	}
+
+	static Future<Optional<Value>> getInfo(Database cx) {
+		return runRYWTransaction(
+		    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return getInfo(tr); });
+	}
+
+	StatusObject toJSON() const {
+		StatusObject result;
+		result["consistency_scan_enabled"] = consistency_scan_enabled;
+		result["restart"] = restart;
+		result["max_rate"] = max_rate;
+		result["target_interval"] = target_interval;
+		result["bytes_read_prev_round"] = bytes_read_prev_round;
+		result["last_round_start_datetime"] = epochsToGMTString(last_round_start);
+		result["last_round_finish_datetime"] = epochsToGMTString(last_round_finish);
+		result["last_round_start_timestamp"] = last_round_start;
+		result["last_round_finish_timestamp"] = last_round_finish;
+		result["smoothed_round_seconds"] = smoothed_round_duration.smoothTotal();
+		result["finished_rounds"] = finished_rounds;
+		return result;
+	}
+
+	std::string toString() const {
+		return format("consistency_scan_enabled = %d, restart =  %d, max_rate = %ld, target_interval = %ld",
+		              consistency_scan_enabled,
+		              restart,
+		              max_rate,
+		              target_interval);
+	}
+};
+
+ACTOR Future<Version> getVersion(Database cx);
+ACTOR Future<bool> getKeyServers(
+    Database cx,
+    Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise,
+    KeyRangeRef kr,
+    bool performQuiescentChecks);
+ACTOR Future<bool> getKeyLocations(Database cx,
+                                   std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> shards,
+                                   Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise,
+                                   bool performQuiescentChecks);
+ACTOR Future<bool> checkDataConsistency(Database cx,
+                                        VectorRef<KeyValueRef> keyLocations,
+                                        DatabaseConfiguration configuration,
+                                        std::map<UID, StorageServerInterface> tssMapping,
+                                        bool performQuiescentChecks,
+                                        bool performTSSCheck,
+                                        bool firstClient,
+                                        bool failureIsError,
+                                        int clientId,
+                                        int clientCount,
+                                        bool distributed,
+                                        bool shuffleShards,
+                                        int shardSampleFactor,
+                                        int64_t sharedRandomNumber,
+                                        int64_t repetitions,
+                                        int64_t* bytesReadInPreviousRound,
+                                        int restart,
+                                        int64_t maxRate,
+                                        int64_t targetInterval,
+                                        KeyRef progressKey);
+
+#include "flow/unactorcompiler.h"
+
+#endif // FDBCLIENT_CONSISTENCYSCANINTERFACE_H
\ No newline at end of file
diff --git a/fdbclient/include/fdbclient/CoordinationInterface.h b/fdbclient/include/fdbclient/CoordinationInterface.h
index 7ccaf9170a..c5d53d9ad7 100644
--- a/fdbclient/include/fdbclient/CoordinationInterface.h
+++ b/fdbclient/include/fdbclient/CoordinationInterface.h
@@ -63,6 +63,8 @@ struct ClientLeaderRegInterface {
 //  - There is no address present more than once
 class ClusterConnectionString {
 public:
+	constexpr static FileIdentifier file_identifier = 13602011;
+
 	ClusterConnectionString() {}
 	ClusterConnectionString(const std::string& connectionString);
 	ClusterConnectionString(const std::vector<NetworkAddress>& coordinators, Key key);
@@ -84,9 +86,22 @@ public:
 	std::vector<NetworkAddress> coords;
 	std::vector<Hostname> hostnames;
 
+	size_t getNumberOfCoordinators() const { return coords.size() + hostnames.size(); }
+
+	bool operator==(const ClusterConnectionString& other) const noexcept {
+		return key == other.key && keyDesc == other.keyDesc && coords == other.coords && hostnames == other.hostnames;
+	}
+	bool operator!=(const ClusterConnectionString& other) const noexcept { return !(*this == other); }
+
 private:
 	void parseConnString();
 	Key key, keyDesc;
+
+public:
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, coords, hostnames, key, keyDesc);
+	}
 };
 
 FDB_DECLARE_BOOLEAN_PARAM(ConnectionStringNeedsPersisted);
@@ -229,6 +244,8 @@ struct GetLeaderRequest {
 	GetLeaderRequest() {}
 	explicit GetLeaderRequest(Key key, UID kl) : key(key), knownLeader(kl) {}
 
+	bool verify() const { return true; }
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, key, knownLeader, reply);
@@ -249,6 +266,8 @@ struct OpenDatabaseCoordRequest {
 	std::vector<NetworkAddress> coordinators;
 	ReplyPromise<CachedSerialization<struct ClientDBInfo>> reply;
 
+	bool verify() const { return true; }
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar,
diff --git a/fdbclient/include/fdbclient/DatabaseConfiguration.h b/fdbclient/include/fdbclient/DatabaseConfiguration.h
index 363b48e4b6..f73cd99c9d 100644
--- a/fdbclient/include/fdbclient/DatabaseConfiguration.h
+++ b/fdbclient/include/fdbclient/DatabaseConfiguration.h
@@ -256,6 +256,8 @@ struct DatabaseConfiguration {
 	bool blobGranulesEnabled;
 	TenantMode tenantMode;
 
+	EncryptionAtRestMode encryptionAtRestMode;
+
 	// Excluded servers (no state should be here)
 	bool isExcludedServer(NetworkAddressList) const;
 	bool isExcludedLocality(const LocalityData& locality) const;
diff --git a/fdbclient/include/fdbclient/DatabaseContext.h b/fdbclient/include/fdbclient/DatabaseContext.h
index 27e499d607..8ec21c5472 100644
--- a/fdbclient/include/fdbclient/DatabaseContext.h
+++ b/fdbclient/include/fdbclient/DatabaseContext.h
@@ -21,10 +21,12 @@
 #ifndef DatabaseContext_h
 #define DatabaseContext_h
 #include "fdbclient/Notified.h"
+#include "flow/ApiVersion.h"
 #include "flow/FastAlloc.h"
 #include "flow/FastRef.h"
 #include "fdbclient/GlobalConfig.actor.h"
 #include "fdbclient/StorageServerInterface.h"
+#include "flow/IRandom.h"
 #include "flow/genericactors.actor.h"
 #include <vector>
 #include <unordered_map>
@@ -167,10 +169,11 @@ struct ChangeFeedStorageData : ReferenceCounted<ChangeFeedStorageData> {
 	Future<Void> updater;
 	NotifiedVersion version;
 	NotifiedVersion desired;
-	Promise<Void> destroyed;
 	UID interfToken;
+	DatabaseContext* context;
+	double created;
 
-	~ChangeFeedStorageData() { destroyed.send(Void()); }
+	~ChangeFeedStorageData();
 };
 
 struct ChangeFeedData : ReferenceCounted<ChangeFeedData> {
@@ -180,6 +183,8 @@ struct ChangeFeedData : ReferenceCounted<ChangeFeedData> {
 	Version getVersion();
 	Future<Void> whenAtLeast(Version version);
 
+	UID dbgid;
+	DatabaseContext* context;
 	NotifiedVersion lastReturnedVersion;
 	std::vector<Reference<ChangeFeedStorageData>> storageData;
 	AsyncVar<int> notAtLatest;
@@ -188,8 +193,10 @@ struct ChangeFeedData : ReferenceCounted<ChangeFeedData> {
 	Version endVersion = invalidVersion;
 	Version popVersion =
 	    invalidVersion; // like TLog pop version, set by SS and client can check it to see if they missed data
+	double created = 0;
 
-	ChangeFeedData() : notAtLatest(1) {}
+	explicit ChangeFeedData(DatabaseContext* context = nullptr);
+	~ChangeFeedData();
 };
 
 struct EndpointFailureInfo {
@@ -207,6 +214,16 @@ struct KeyRangeLocationInfo {
 	  : tenantEntry(tenantEntry), range(range), locations(locations) {}
 };
 
+struct OverlappingChangeFeedsInfo {
+	Arena arena;
+	VectorRef<OverlappingChangeFeedEntry> feeds;
+	// would prefer to use key range map but it complicates copy/move constructors
+	std::vector<std::pair<KeyRangeRef, Version>> feedMetadataVersions;
+
+	// for a feed that wasn't present, returns the metadata version it would have been fetched at.
+	Version getFeedMetadataVersion(const KeyRangeRef& feedRange) const;
+};
+
 class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAllocated<DatabaseContext>, NonCopyable {
 public:
 	static DatabaseContext* allocateOnForeignThread() {
@@ -221,7 +238,7 @@ public:
 	                       EnableLocalityLoadBalance,
 	                       TaskPriority taskID = TaskPriority::DefaultEndpoint,
 	                       LockAware = LockAware::False,
-	                       int apiVersion = Database::API_VERSION_LATEST,
+	                       int _apiVersion = ApiVersion::LATEST_VERSION,
 	                       IsSwitchable = IsSwitchable::False);
 
 	~DatabaseContext();
@@ -237,7 +254,7 @@ public:
 		                                           enableLocalityLoadBalance,
 		                                           lockAware,
 		                                           internal,
-		                                           apiVersion,
+		                                           apiVersion.version(),
 		                                           switchable,
 		                                           defaultTenant));
 		cx->globalConfig->init(Reference<AsyncVar<ClientDBInfo> const>(cx->clientInfo),
@@ -245,16 +262,16 @@ public:
 		return cx;
 	}
 
-	Optional<KeyRangeLocationInfo> getCachedLocation(const Optional<TenantName>& tenant,
+	Optional<KeyRangeLocationInfo> getCachedLocation(const Optional<TenantNameRef>& tenant,
 	                                                 const KeyRef&,
 	                                                 Reverse isBackward = Reverse::False);
-	bool getCachedLocations(const Optional<TenantName>& tenant,
+	bool getCachedLocations(const Optional<TenantNameRef>& tenant,
 	                        const KeyRangeRef&,
 	                        std::vector<KeyRangeLocationInfo>&,
 	                        int limit,
 	                        Reverse reverse);
 	void cacheTenant(const TenantName& tenant, const TenantMapEntry& tenantEntry);
-	Reference<LocationInfo> setCachedLocation(const Optional<TenantName>& tenant,
+	Reference<LocationInfo> setCachedLocation(const Optional<TenantNameRef>& tenant,
 	                                          const TenantMapEntry& tenantEntry,
 	                                          const KeyRangeRef&,
 	                                          const std::vector<struct StorageServerInterface>&);
@@ -328,7 +345,7 @@ public:
 		}
 	}
 
-	int apiVersionAtLeast(int minVersion) const { return apiVersion < 0 || apiVersion >= minVersion; }
+	int apiVersionAtLeast(int minVersion) const { return apiVersion.version() >= minVersion; }
 
 	Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a
 	                            // leader. The cluster file therefore is valid, but the database might be unavailable.
@@ -361,15 +378,21 @@ public:
 	                                 int replyBufferSize = -1,
 	                                 bool canReadPopped = true);
 
-	Future<std::vector<OverlappingChangeFeedEntry>> getOverlappingChangeFeeds(KeyRangeRef ranges, Version minVersion);
+	Future<OverlappingChangeFeedsInfo> getOverlappingChangeFeeds(KeyRangeRef ranges, Version minVersion);
 	Future<Void> popChangeFeedMutations(Key rangeID, Version version);
 
+	// BlobGranule API.
 	Future<Key> purgeBlobGranules(KeyRange keyRange,
 	                              Version purgeVersion,
 	                              Optional<TenantName> tenant,
 	                              bool force = false);
 	Future<Void> waitPurgeGranulesComplete(Key purgeKey);
 
+	Future<bool> blobbifyRange(KeyRange range);
+	Future<bool> unblobbifyRange(KeyRange range);
+	Future<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(KeyRange range, int rangeLimit);
+	Future<Version> verifyBlobRange(const KeyRange& range, Optional<Version> version);
+
 	// private:
 	explicit DatabaseContext(Reference<AsyncVar<Reference<IClusterConnectionRecord>>> connectionRecord,
 	                         Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
@@ -380,7 +403,7 @@ public:
 	                         EnableLocalityLoadBalance,
 	                         LockAware,
 	                         IsInternal = IsInternal::True,
-	                         int apiVersion = Database::API_VERSION_LATEST,
+	                         int _apiVersion = ApiVersion::LATEST_VERSION,
 	                         IsSwitchable = IsSwitchable::False,
 	                         Optional<TenantName> defaultTenant = Optional<TenantName>());
 
@@ -457,9 +480,12 @@ public:
 	std::unordered_map<UID, Reference<TSSMetrics>> tssMetrics;
 	// map from changeFeedId -> changeFeedRange
 	std::unordered_map<Key, KeyRange> changeFeedCache;
-	std::unordered_map<UID, Reference<ChangeFeedStorageData>> changeFeedUpdaters;
+	std::unordered_map<UID, ChangeFeedStorageData*> changeFeedUpdaters;
+	std::map<UID, ChangeFeedData*> notAtLatestChangeFeeds;
 
 	Reference<ChangeFeedStorageData> getStorageData(StorageServerInterface interf);
+	Version getMinimumChangeFeedVersion();
+	void setDesiredChangeFeedVersion(Version v);
 
 	// map from ssid -> ss tag
 	// @note this map allows the client to identify the latest commit versions
@@ -517,6 +543,19 @@ public:
 	Counter transactionsExpensiveClearCostEstCount;
 	Counter transactionGrvFullBatches;
 	Counter transactionGrvTimedOutBatches;
+	Counter transactionCommitVersionNotFoundForSS;
+	Counter bgReadInputBytes;
+	Counter bgReadOutputBytes;
+
+	// Change Feed metrics. Omit change feed metrics from logging if not used
+	bool usedAnyChangeFeeds;
+	CounterCollection ccFeed;
+	Counter feedStreamStarts;
+	Counter feedMergeStreamStarts;
+	Counter feedErrors;
+	Counter feedNonRetriableErrors;
+	Counter feedPops;
+	Counter feedPopsFallback;
 
 	ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit,
 	    bytesPerCommit, bgLatencies, bgGranulesPerRequest;
@@ -569,7 +608,7 @@ public:
 	Future<Void> statusLeaderMon;
 	double lastStatusFetch;
 
-	int apiVersion;
+	ApiVersion apiVersion;
 
 	int mvCacheInsertLocation;
 	std::vector<std::pair<Version, Optional<Value>>> metadataVersionCache;
@@ -638,14 +677,18 @@ private:
 
 // Similar to tr.onError(), but doesn't require a DatabaseContext.
 struct Backoff {
+	Backoff(double backoff = CLIENT_KNOBS->DEFAULT_BACKOFF, double maxBackoff = CLIENT_KNOBS->DEFAULT_MAX_BACKOFF)
+	  : backoff(backoff), maxBackoff(maxBackoff) {}
+
 	Future<Void> onError() {
 		double currentBackoff = backoff;
-		backoff = std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, CLIENT_KNOBS->DEFAULT_MAX_BACKOFF);
+		backoff = std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, maxBackoff);
 		return delay(currentBackoff * deterministicRandom()->random01());
 	}
 
 private:
-	double backoff = CLIENT_KNOBS->DEFAULT_BACKOFF;
+	double backoff;
+	double maxBackoff;
 };
 
 #endif
diff --git a/fdbserver/include/fdbserver/EncryptKeyProxyInterface.h b/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h
similarity index 95%
rename from fdbserver/include/fdbserver/EncryptKeyProxyInterface.h
rename to fdbclient/include/fdbclient/EncryptKeyProxyInterface.h
index 12178b11ab..5f4d56eb96 100644
--- a/fdbserver/include/fdbserver/EncryptKeyProxyInterface.h
+++ b/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h
@@ -132,7 +132,7 @@ struct EKPGetBaseCipherKeysByIdsReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, baseCipherDetails, numHits, error);
+		serializer(ar, baseCipherDetails, numHits, error, arena);
 	}
 };
 
@@ -144,10 +144,10 @@ struct EKPGetBaseCipherKeysRequestInfo {
 	EncryptCipherBaseKeyId baseCipherId;
 	// Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by
 	// {domainId, cipherBaseId} tuple
-	EncryptCipherDomainName domainName;
+	EncryptCipherDomainNameRef domainName;
 
 	EKPGetBaseCipherKeysRequestInfo()
-	  : domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {}
+	  : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
 	EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId,
 	                                const EncryptCipherBaseKeyId bCId,
 	                                StringRef name,
@@ -176,7 +176,7 @@ struct EKPGetBaseCipherKeysByIdsRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, baseCipherInfos, debugId, reply);
+		serializer(ar, baseCipherInfos, debugId, reply, arena);
 	}
 };
 
@@ -193,7 +193,7 @@ struct EKPGetLatestBaseCipherKeysReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, baseCipherDetails, numHits, error);
+		serializer(ar, baseCipherDetails, numHits, error, arena);
 	}
 };
 
@@ -203,9 +203,9 @@ struct EKPGetLatestCipherKeysRequestInfo {
 	EncryptCipherDomainId domainId;
 	// Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by
 	// {domainId, cipherBaseId} tuple
-	EncryptCipherDomainName domainName;
+	EncryptCipherDomainNameRef domainName;
 
-	EKPGetLatestCipherKeysRequestInfo() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {}
+	EKPGetLatestCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
 	EKPGetLatestCipherKeysRequestInfo(const EncryptCipherDomainId dId, StringRef name, Arena& arena)
 	  : domainId(dId), domainName(StringRef(arena, name)) {}
 
@@ -239,7 +239,7 @@ struct EKPGetLatestBaseCipherKeysRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, encryptDomainInfos, debugId, reply);
+		serializer(ar, encryptDomainInfos, debugId, reply, arena);
 	}
 };
 
diff --git a/fdbclient/include/fdbclient/EventTypes.actor.h b/fdbclient/include/fdbclient/EventTypes.actor.h
index 39a75e09dc..dc946ce42e 100644
--- a/fdbclient/include/fdbclient/EventTypes.actor.h
+++ b/fdbclient/include/fdbclient/EventTypes.actor.h
@@ -26,7 +26,7 @@
 #define FDBCLIENT_EVENTTYPES_ACTOR_G_H
 #include "fdbclient/EventTypes.actor.g.h"
 #elif !defined(FDBCLIENT_EVENTTYPES_ACTOR_H)
-#define FDBCLIENT_EVENTTYPESS_ACTOR_H
+#define FDBCLIENT_EVENTTYPES_ACTOR_H
 
 #include "flow/flow.h"
 #include "flow/TDMetric.actor.h"
diff --git a/fdbclient/include/fdbclient/FDBTypes.h b/fdbclient/include/fdbclient/FDBTypes.h
index 15cd4fc527..4da1d4f72b 100644
--- a/fdbclient/include/fdbclient/FDBTypes.h
+++ b/fdbclient/include/fdbclient/FDBTypes.h
@@ -41,6 +41,7 @@ typedef StringRef KeyRef;
 typedef StringRef ValueRef;
 typedef int64_t Generation;
 typedef UID SpanID;
+typedef uint64_t CoordinatorsHash;
 
 enum {
 	tagLocalitySpecial = -1, // tag with this locality means it is invalidTag (id=0), txsTag (id=1), or cacheTag (id=2)
@@ -331,6 +332,22 @@ struct KeyRangeRef {
 	bool empty() const { return begin == end; }
 	bool singleKeyRange() const { return equalsKeyAfter(begin, end); }
 
+	// Return true if it's fully covered by given range list. Note that ranges should be sorted
+	bool isCovered(std::vector<KeyRangeRef>& ranges) {
+		ASSERT(std::is_sorted(ranges.begin(), ranges.end(), KeyRangeRef::ArbitraryOrder()));
+		KeyRangeRef clone(begin, end);
+		for (auto r : ranges) {
+			if (begin < r.begin)
+				return false; // uncovered gap between clone.begin and r.begin
+			if (end <= r.end)
+				return true; // range is fully covered
+			if (end > r.begin)
+				// {clone.begin, r.end} is covered. need to check coverage for {r.end, clone.end}
+				clone = KeyRangeRef(r.end, clone.end);
+		}
+		return false;
+	}
+
 	Standalone<KeyRangeRef> withPrefix(const StringRef& prefix) const {
 		return KeyRangeRef(begin.withPrefix(prefix), end.withPrefix(prefix));
 	}
@@ -500,10 +517,36 @@ using KeySelector = Standalone<struct KeySelectorRef>;
 using RangeResult = Standalone<struct RangeResultRef>;
 using MappedRangeResult = Standalone<struct MappedRangeResultRef>;
 
+namespace std {
+template <>
+struct hash<KeyRangeRef> {
+	static constexpr std::hash<StringRef> hashFunc{};
+	std::size_t operator()(KeyRangeRef const& range) const {
+		std::size_t seed = 0;
+		boost::hash_combine(seed, hashFunc(range.begin));
+		boost::hash_combine(seed, hashFunc(range.end));
+		return seed;
+	}
+};
+} // namespace std
+
+namespace std {
+template <>
+struct hash<KeyRange> {
+	static constexpr std::hash<StringRef> hashFunc{};
+	std::size_t operator()(KeyRangeRef const& range) const {
+		std::size_t seed = 0;
+		boost::hash_combine(seed, hashFunc(range.begin));
+		boost::hash_combine(seed, hashFunc(range.end));
+		return seed;
+	}
+};
+} // namespace std
+
 enum { invalidVersion = -1, latestVersion = -2, MAX_VERSION = std::numeric_limits<int64_t>::max() };
 
 inline Key keyAfter(const KeyRef& key) {
-	if (key == LiteralStringRef("\xff\xff"))
+	if (key == "\xff\xff"_sr)
 		return key;
 
 	Standalone<StringRef> r;
@@ -516,7 +559,7 @@ inline Key keyAfter(const KeyRef& key) {
 	return r;
 }
 inline KeyRef keyAfter(const KeyRef& key, Arena& arena) {
-	if (key == LiteralStringRef("\xff\xff"))
+	if (key == "\xff\xff"_sr)
 		return key;
 	uint8_t* t = new (arena) uint8_t[key.size() + 1];
 	memcpy(t, key.begin(), key.size());
@@ -931,17 +974,17 @@ struct TLogVersion {
 	}
 
 	static ErrorOr<TLogVersion> FromStringRef(StringRef s) {
-		if (s == LiteralStringRef("2"))
+		if (s == "2"_sr)
 			return V2;
-		if (s == LiteralStringRef("3"))
+		if (s == "3"_sr)
 			return V3;
-		if (s == LiteralStringRef("4"))
+		if (s == "4"_sr)
 			return V4;
-		if (s == LiteralStringRef("5"))
+		if (s == "5"_sr)
 			return V5;
-		if (s == LiteralStringRef("6"))
+		if (s == "6"_sr)
 			return V6;
-		if (s == LiteralStringRef("7"))
+		if (s == "7"_sr)
 			return V7;
 		return default_error_or();
 	}
@@ -991,9 +1034,9 @@ struct TLogSpillType {
 	}
 
 	static ErrorOr<TLogSpillType> FromStringRef(StringRef s) {
-		if (s == LiteralStringRef("1"))
+		if (s == "1"_sr)
 			return VALUE;
-		if (s == LiteralStringRef("2"))
+		if (s == "2"_sr)
 			return REFERENCE;
 		return default_error_or();
 	}
@@ -1392,6 +1435,60 @@ struct TenantMode {
 	uint32_t mode;
 };
 
+struct EncryptionAtRestMode {
+	// These enumerated values are stored in the database configuration, so can NEVER be changed.  Only add new ones
+	// just before END.
+	enum Mode { DISABLED = 0, AES_256_CTR = 1, END = 2 };
+
+	EncryptionAtRestMode() : mode(DISABLED) {}
+	EncryptionAtRestMode(Mode mode) : mode(mode) {
+		if ((uint32_t)mode >= END) {
+			this->mode = DISABLED;
+		}
+	}
+	operator Mode() const { return Mode(mode); }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, mode);
+	}
+
+	std::string toString() const {
+		switch (mode) {
+		case DISABLED:
+			return "disabled";
+		case AES_256_CTR:
+			return "aes_256_ctr";
+		default:
+			ASSERT(false);
+		}
+		return "";
+	}
+
+	Value toValue() const { return ValueRef(format("%d", (int)mode)); }
+
+	static EncryptionAtRestMode fromValue(Optional<ValueRef> val) {
+		if (!val.present()) {
+			return DISABLED;
+		}
+
+		// A failed parsing returns 0 (DISABLED)
+		int num = atoi(val.get().toString().c_str());
+		if (num < 0 || num >= END) {
+			return DISABLED;
+		}
+
+		return static_cast<Mode>(num);
+	}
+
+	uint32_t mode;
+};
+
+typedef StringRef ClusterNameRef;
+typedef Standalone<ClusterNameRef> ClusterName;
+
+enum class ClusterType { STANDALONE, METACLUSTER_MANAGEMENT, METACLUSTER_DATA };
+
 struct GRVCacheSpace {
 	Version cachedReadVersion;
 	double lastGrvTime;
@@ -1413,7 +1510,7 @@ struct DatabaseSharedState {
 	std::atomic<int> refCount;
 
 	DatabaseSharedState()
-	  : protocolVersion(currentProtocolVersion), mutexLock(Mutex()), grvCacheSpace(GRVCacheSpace()), refCount(0) {}
+	  : protocolVersion(currentProtocolVersion()), mutexLock(Mutex()), grvCacheSpace(GRVCacheSpace()), refCount(0) {}
 };
 
 inline bool isValidPerpetualStorageWiggleLocality(std::string locality) {
@@ -1460,7 +1557,7 @@ struct StorageMetadataType {
 	bool wrongConfigured = false;
 
 	StorageMetadataType() : createdTime(0) {}
-	StorageMetadataType(uint64_t t, KeyValueStoreType storeType = KeyValueStoreType::END, bool wrongConfigured = false)
+	StorageMetadataType(double t, KeyValueStoreType storeType = KeyValueStoreType::END, bool wrongConfigured = false)
 	  : createdTime(t), storeType(storeType), wrongConfigured(wrongConfigured) {}
 
 	static double currentTime() { return g_network->timer(); }
@@ -1510,6 +1607,42 @@ struct StorageWiggleValue {
 	}
 };
 
+enum class ReadType {
+	EAGER,
+	FETCH,
+	LOW,
+	NORMAL,
+	HIGH,
+};
+
+FDB_DECLARE_BOOLEAN_PARAM(CacheResult);
+
+// store options for storage engine read
+// ReadType describes the usage and priority of the read
+// cacheResult determines whether the storage engine cache for this read
+// consistencyCheckStartVersion indicates the consistency check which began at this version
+// debugID helps to trace the path of the read
+struct ReadOptions {
+	ReadType type;
+	// Once CacheResult is serializable, change type from bool to CacheResult
+	bool cacheResult;
+	Optional<UID> debugID;
+	Optional<Version> consistencyCheckStartVersion;
+
+	ReadOptions() : type(ReadType::NORMAL), cacheResult(CacheResult::True){};
+
+	ReadOptions(Optional<UID> debugID,
+	            ReadType type = ReadType::NORMAL,
+	            CacheResult cache = CacheResult::False,
+	            Optional<Version> version = Optional<Version>())
+	  : type(type), cacheResult(cache), debugID(debugID), consistencyCheckStartVersion(version){};
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, type, cacheResult, debugID, consistencyCheckStartVersion);
+	}
+};
+
 // Can be used to identify types (e.g. IDatabase) that can be used to create transactions with a `createTransaction`
 // function
 template <typename, typename = void>
diff --git a/fdbclient/include/fdbclient/GenericManagementAPI.actor.h b/fdbclient/include/fdbclient/GenericManagementAPI.actor.h
index 8fe0d08fd2..21ebbd3a3c 100644
--- a/fdbclient/include/fdbclient/GenericManagementAPI.actor.h
+++ b/fdbclient/include/fdbclient/GenericManagementAPI.actor.h
@@ -39,6 +39,7 @@ the contents of the system key space.
 #include "fdbclient/Status.h"
 #include "fdbclient/Subspace.h"
 #include "fdbclient/DatabaseConfiguration.h"
+#include "fdbclient/Metacluster.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/SystemData.h"
 #include "flow/actorcompiler.h" // has to be last include
@@ -69,6 +70,8 @@ enum class ConfigurationResult {
 	SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL,
 	DATABASE_CREATED_WARN_ROCKSDB_EXPERIMENTAL,
 	DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL,
+	DATABASE_IS_REGISTERED,
+	ENCRYPTION_AT_REST_MODE_ALREADY_SET
 };
 
 enum class CoordinatorsResult {
@@ -246,7 +249,7 @@ Future<std::vector<ProcessData>> getWorkers(Reference<Tr> tr,
 // Accepts a full configuration in key/value format (from buildConfiguration)
 ACTOR template <class DB>
 Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string, std::string> m, bool force) {
-	state StringRef initIdKey = LiteralStringRef("\xff/init_id");
+	state StringRef initIdKey = "\xff/init_id"_sr;
 	state Reference<typename DB::TransactionT> tr = db->createTransaction();
 
 	if (!m.size()) {
@@ -272,6 +275,9 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 		if (!isCompleteConfiguration(m)) {
 			return ConfigurationResult::INCOMPLETE_CONFIGURATION;
 		}
+	} else if (m.count(encryptionAtRestModeConfKey.toString()) != 0) {
+		// Encryption data at-rest mode can be set only at the time of database creation
+		return ConfigurationResult::ENCRYPTION_AT_REST_MODE_ALREADY_SET;
 	}
 
 	state Future<Void> tooLong = delay(60);
@@ -475,6 +481,14 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 					           newConfig.storageServerStoreType == KeyValueStoreType::SSD_SHARDED_ROCKSDB) {
 						warnShardedRocksDBIsExperimental = true;
 					}
+
+					if (newConfig.tenantMode != oldConfig.tenantMode) {
+						Optional<MetaclusterRegistrationEntry> metaclusterRegistration =
+						    wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
+						if (metaclusterRegistration.present()) {
+							return ConfigurationResult::DATABASE_IS_REGISTERED;
+						}
+					}
 				}
 			}
 			if (creating) {
@@ -491,8 +505,8 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 				ASSERT(creating);
 				tr->atomicOp(databaseLockedKey,
 				             BinaryWriter::toValue(locked.get(), Unversioned())
-				                 .withPrefix(LiteralStringRef("0123456789"))
-				                 .withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
+				                 .withPrefix("0123456789"_sr)
+				                 .withSuffix("\x00\x00\x00\x00"_sr),
 				             MutationRef::SetVersionstampedValue);
 			}
 
@@ -636,7 +650,7 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db,
                                          std::vector<StringRef> const& modes,
                                          Optional<ConfigureAutoResult> const& conf,
                                          bool force) {
-	if (modes.size() && modes[0] == LiteralStringRef("auto") && conf.present()) {
+	if (modes.size() && modes[0] == "auto"_sr && conf.present()) {
 		return autoConfig(db, conf.get());
 	}
 
diff --git a/fdbserver/GetEncryptCipherKeys.actor.cpp b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h
similarity index 56%
rename from fdbserver/GetEncryptCipherKeys.actor.cpp
rename to fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h
index 6b77d24d03..0f93675a6a 100644
--- a/fdbserver/GetEncryptCipherKeys.actor.cpp
+++ b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h
@@ -1,5 +1,5 @@
 /*
- * GetEncryptCipherKeys.actor.cpp
+ * GetEncryptCipherKeys.actor.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -17,20 +17,31 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#pragma once
+#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H)
+#define FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H
+#include "fdbclient/GetEncryptCipherKeys.actor.g.h"
+#elif !defined(FDBCLIENT_GETCIPHERKEYS_ACTOR_H)
+#define FDBCLIENT_GETCIPHERKEYS_ACTOR_H
 
-#include "fdbserver/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/BlobCipher.h"
+#include "fdbclient/EncryptKeyProxyInterface.h"
+#include "fdbrpc/Stats.h"
+#include "flow/Knobs.h"
+#include "flow/IRandom.h"
 
-#include <boost/functional/hash.hpp>
+#include <unordered_map>
+#include <unordered_set>
 
-#include "flow/actorcompiler.h" // has to be last include
+#include "flow/actorcompiler.h" // This must be the last #include.
 
-namespace {
-
-Optional<UID> getEncryptKeyProxyId(const Reference<AsyncVar<ServerDBInfo> const>& db) {
-	return db->get().encryptKeyProxy.map<UID>([](EncryptKeyProxyInterface proxy) { return proxy.id(); });
+template <class T>
+Optional<UID> getEncryptKeyProxyId(const Reference<AsyncVar<T> const>& db) {
+	return db->get().encryptKeyProxy.template map<UID>([](EncryptKeyProxyInterface proxy) { return proxy.id(); });
 }
 
-ACTOR Future<Void> onEncryptKeyProxyChange(Reference<AsyncVar<ServerDBInfo> const> db) {
+ACTOR template <class T>
+Future<Void> onEncryptKeyProxyChange(Reference<AsyncVar<T> const> db) {
 	state Optional<UID> previousProxyId = getEncryptKeyProxyId(db);
 	state Optional<UID> currentProxyId;
 	loop {
@@ -46,9 +57,9 @@ ACTOR Future<Void> onEncryptKeyProxyChange(Reference<AsyncVar<ServerDBInfo> cons
 	return Void();
 }
 
-ACTOR Future<EKPGetLatestBaseCipherKeysReply> getUncachedLatestEncryptCipherKeys(
-    Reference<AsyncVar<ServerDBInfo> const> db,
-    EKPGetLatestBaseCipherKeysRequest request) {
+ACTOR template <class T>
+Future<EKPGetLatestBaseCipherKeysReply> getUncachedLatestEncryptCipherKeys(Reference<AsyncVar<T> const> db,
+                                                                           EKPGetLatestBaseCipherKeysRequest request) {
 	Optional<EncryptKeyProxyInterface> proxy = db->get().encryptKeyProxy;
 	if (!proxy.present()) {
 		// Wait for onEncryptKeyProxyChange.
@@ -73,11 +84,14 @@ ACTOR Future<EKPGetLatestBaseCipherKeysReply> getUncachedLatestEncryptCipherKeys
 	}
 }
 
-} // anonymous namespace
-
-ACTOR Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getLatestEncryptCipherKeys(
-    Reference<AsyncVar<ServerDBInfo> const> db,
-    std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains) {
+// Get latest cipher keys for given encryption domains. It tries to get the cipher keys from local cache.
+// In case of cache miss, it fetches the cipher keys from EncryptKeyProxy and put the result in the local cache
+// before return.
+ACTOR template <class T>
+Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getLatestEncryptCipherKeys(
+    Reference<AsyncVar<T> const> db,
+    std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains,
+    BlobCipherMetrics::UsageType usageType) {
 	state Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
 	state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
 	state EKPGetLatestBaseCipherKeysRequest request;
@@ -101,14 +115,19 @@ ACTOR Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>
 		return cipherKeys;
 	}
 	// Fetch any uncached cipher keys.
+	state double startTime = now();
 	loop choose {
 		when(EKPGetLatestBaseCipherKeysReply reply = wait(getUncachedLatestEncryptCipherKeys(db, request))) {
 			// Insert base cipher keys into cache and construct result.
 			for (const EKPBaseCipherDetails& details : reply.baseCipherDetails) {
 				EncryptCipherDomainId domainId = details.encryptDomainId;
 				if (domains.count(domainId) > 0 && cipherKeys.count(domainId) == 0) {
-					Reference<BlobCipherKey> cipherKey = cipherKeyCache->insertCipherKey(
-					    domainId, details.baseCipherId, details.baseCipherKey.begin(), details.baseCipherKey.size());
+					Reference<BlobCipherKey> cipherKey = cipherKeyCache->insertCipherKey(domainId,
+					                                                                     details.baseCipherId,
+					                                                                     details.baseCipherKey.begin(),
+					                                                                     details.baseCipherKey.size(),
+					                                                                     details.refreshAt,
+					                                                                     details.expireAt);
 					ASSERT(cipherKey.isValid());
 					cipherKeys[domainId] = cipherKey;
 				}
@@ -125,13 +144,30 @@ ACTOR Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>
 		// In case encryptKeyProxy has changed, retry the request.
 		when(wait(onEncryptKeyProxyChange(db))) {}
 	}
+	double elapsed = now() - startTime;
+	BlobCipherMetrics::getInstance()->getLatestCipherKeysLatency.addMeasurement(elapsed);
+	BlobCipherMetrics::counters(usageType).getLatestCipherKeysLatency.addMeasurement(elapsed);
 	return cipherKeys;
 }
 
-namespace {
+// Get latest cipher key for given a encryption domain. It tries to get the cipher key from the local cache.
+// In case of cache miss, it fetches the cipher key from EncryptKeyProxy and put the result in the local cache
+// before return.
+ACTOR template <class T>
+Future<Reference<BlobCipherKey>> getLatestEncryptCipherKey(Reference<AsyncVar<T> const> db,
+                                                           EncryptCipherDomainId domainId,
+                                                           EncryptCipherDomainName domainName,
+                                                           BlobCipherMetrics::UsageType usageType) {
+	std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains({ { domainId, domainName } });
+	std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKey =
+	    wait(getLatestEncryptCipherKeys(db, domains, usageType));
 
-ACTOR Future<EKPGetBaseCipherKeysByIdsReply> getUncachedEncryptCipherKeys(Reference<AsyncVar<ServerDBInfo> const> db,
-                                                                          EKPGetBaseCipherKeysByIdsRequest request) {
+	return cipherKey.at(domainId);
+}
+
+ACTOR template <class T>
+Future<EKPGetBaseCipherKeysByIdsReply> getUncachedEncryptCipherKeys(Reference<AsyncVar<T> const> db,
+                                                                    EKPGetBaseCipherKeysByIdsRequest request) {
 	Optional<EncryptKeyProxyInterface> proxy = db->get().encryptKeyProxy;
 	if (!proxy.present()) {
 		// Wait for onEncryptKeyProxyChange.
@@ -158,11 +194,14 @@ ACTOR Future<EKPGetBaseCipherKeysByIdsReply> getUncachedEncryptCipherKeys(Refere
 
 using BaseCipherIndex = std::pair<EncryptCipherDomainId, EncryptCipherBaseKeyId>;
 
-} // anonymous namespace
-
-ACTOR Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncryptCipherKeys(
-    Reference<AsyncVar<ServerDBInfo> const> db,
-    std::unordered_set<BlobCipherDetails> cipherDetails) {
+// Get cipher keys specified by the list of cipher details. It tries to get the cipher keys from local cache.
+// In case of cache miss, it fetches the cipher keys from EncryptKeyProxy and put the result in the local cache
+// before return.
+ACTOR template <class T>
+Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncryptCipherKeys(
+    Reference<AsyncVar<T> const> db,
+    std::unordered_set<BlobCipherDetails> cipherDetails,
+    BlobCipherMetrics::UsageType usageType) {
 	state Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
 	state std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> cipherKeys;
 	state std::unordered_set<BaseCipherIndex, boost::hash<BaseCipherIndex>> uncachedBaseCipherIds;
@@ -191,12 +230,13 @@ ACTOR Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> ge
 		    id.first /*domainId*/, id.second /*baseCipherId*/, StringRef() /*domainName*/, request.arena);
 	}
 	// Fetch any uncached cipher keys.
+	state double startTime = now();
 	loop choose {
 		when(EKPGetBaseCipherKeysByIdsReply reply = wait(getUncachedEncryptCipherKeys(db, request))) {
-			std::unordered_map<BaseCipherIndex, StringRef, boost::hash<BaseCipherIndex>> baseCipherKeys;
+			std::unordered_map<BaseCipherIndex, EKPBaseCipherDetails, boost::hash<BaseCipherIndex>> baseCipherKeys;
 			for (const EKPBaseCipherDetails& baseDetails : reply.baseCipherDetails) {
 				BaseCipherIndex baseIdx = std::make_pair(baseDetails.encryptDomainId, baseDetails.baseCipherId);
-				baseCipherKeys[baseIdx] = baseDetails.baseCipherKey;
+				baseCipherKeys[baseIdx] = baseDetails;
 			}
 			// Insert base cipher keys into cache and construct result.
 			for (const BlobCipherDetails& details : cipherDetails) {
@@ -213,9 +253,11 @@ ACTOR Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> ge
 				}
 				Reference<BlobCipherKey> cipherKey = cipherKeyCache->insertCipherKey(details.encryptDomainId,
 				                                                                     details.baseCipherId,
-				                                                                     itr->second.begin(),
-				                                                                     itr->second.size(),
-				                                                                     details.salt);
+				                                                                     itr->second.baseCipherKey.begin(),
+				                                                                     itr->second.baseCipherKey.size(),
+				                                                                     details.salt,
+				                                                                     itr->second.refreshAt,
+				                                                                     itr->second.expireAt);
 				ASSERT(cipherKey.isValid());
 				cipherKeys[details] = cipherKey;
 			}
@@ -224,30 +266,49 @@ ACTOR Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> ge
 		// In case encryptKeyProxy has changed, retry the request.
 		when(wait(onEncryptKeyProxyChange(db))) {}
 	}
+	double elapsed = now() - startTime;
+	BlobCipherMetrics::getInstance()->getCipherKeysLatency.addMeasurement(elapsed);
+	BlobCipherMetrics::counters(usageType).getCipherKeysLatency.addMeasurement(elapsed);
 	return cipherKeys;
 }
 
-ACTOR Future<TextAndHeaderCipherKeys> getLatestSystemEncryptCipherKeys(Reference<AsyncVar<ServerDBInfo> const> db) {
-	static std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains = {
-		{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME },
-		{ ENCRYPT_HEADER_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
-	};
+struct TextAndHeaderCipherKeys {
+	Reference<BlobCipherKey> cipherTextKey;
+	Reference<BlobCipherKey> cipherHeaderKey;
+};
+
+ACTOR template <class T>
+Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<AsyncVar<T> const> db,
+                                                                    EncryptCipherDomainId domainId,
+                                                                    EncryptCipherDomainName domainName,
+                                                                    BlobCipherMetrics::UsageType usageType) {
+	std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
+	domains[domainId] = domainName;
+	domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_ENCRYPT_HEADER_DOMAIN_NAME;
 	std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys =
-	    wait(getLatestEncryptCipherKeys(db, domains));
-	ASSERT(cipherKeys.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) > 0);
+	    wait(getLatestEncryptCipherKeys(db, domains, usageType));
+	ASSERT(cipherKeys.count(domainId) > 0);
 	ASSERT(cipherKeys.count(ENCRYPT_HEADER_DOMAIN_ID) > 0);
-	TextAndHeaderCipherKeys result{ cipherKeys.at(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID),
-		                            cipherKeys.at(ENCRYPT_HEADER_DOMAIN_ID) };
+	TextAndHeaderCipherKeys result{ cipherKeys.at(domainId), cipherKeys.at(ENCRYPT_HEADER_DOMAIN_ID) };
 	ASSERT(result.cipherTextKey.isValid());
 	ASSERT(result.cipherHeaderKey.isValid());
 	return result;
 }
 
-ACTOR Future<TextAndHeaderCipherKeys> getEncryptCipherKeys(Reference<AsyncVar<ServerDBInfo> const> db,
-                                                           BlobCipherEncryptHeader header) {
+template <class T>
+Future<TextAndHeaderCipherKeys> getLatestSystemEncryptCipherKeys(const Reference<AsyncVar<T> const>& db,
+                                                                 BlobCipherMetrics::UsageType usageType) {
+	return getLatestEncryptCipherKeysForDomain(
+	    db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME, usageType);
+}
+
+ACTOR template <class T>
+Future<TextAndHeaderCipherKeys> getEncryptCipherKeys(Reference<AsyncVar<T> const> db,
+                                                     BlobCipherEncryptHeader header,
+                                                     BlobCipherMetrics::UsageType usageType) {
 	std::unordered_set<BlobCipherDetails> cipherDetails{ header.cipherTextDetails, header.cipherHeaderDetails };
 	std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> cipherKeys =
-	    wait(getEncryptCipherKeys(db, cipherDetails));
+	    wait(getEncryptCipherKeys(db, cipherDetails, usageType));
 	ASSERT(cipherKeys.count(header.cipherTextDetails) > 0);
 	ASSERT(cipherKeys.count(header.cipherHeaderDetails) > 0);
 	TextAndHeaderCipherKeys result{ cipherKeys.at(header.cipherTextDetails),
@@ -256,3 +317,6 @@ ACTOR Future<TextAndHeaderCipherKeys> getEncryptCipherKeys(Reference<AsyncVar<Se
 	ASSERT(result.cipherHeaderKey.isValid());
 	return result;
 }
+
+#include "flow/unactorcompiler.h"
+#endif
\ No newline at end of file
diff --git a/fdbclient/include/fdbclient/GlobalConfig.actor.h b/fdbclient/include/fdbclient/GlobalConfig.actor.h
index b288e5b580..fb03106630 100644
--- a/fdbclient/include/fdbclient/GlobalConfig.actor.h
+++ b/fdbclient/include/fdbclient/GlobalConfig.actor.h
@@ -163,8 +163,7 @@ private:
 	// of the global configuration keyspace.
 	void erase(KeyRangeRef range);
 
-	ACTOR static Future<Void> migrate(GlobalConfig* self);
-	ACTOR static Future<Void> refresh(GlobalConfig* self);
+	ACTOR static Future<Void> refresh(GlobalConfig* self, Version lastKnown);
 	ACTOR static Future<Void> updater(GlobalConfig* self, const ClientDBInfo* dbInfo);
 
 	DatabaseContext* cx;
diff --git a/fdbclient/include/fdbclient/GrvProxyInterface.h b/fdbclient/include/fdbclient/GrvProxyInterface.h
index 4098a88d2c..5d1cec15b2 100644
--- a/fdbclient/include/fdbclient/GrvProxyInterface.h
+++ b/fdbclient/include/fdbclient/GrvProxyInterface.h
@@ -1,4 +1,3 @@
-
 /*
  * GrvProxyInterface.h
  *
@@ -26,8 +25,9 @@
 #include "fdbrpc/fdbrpc.h"
 #include "fdbclient/FDBTypes.h"
 
-// GrvProxy is proxy primarily specializing on serving GetReadVersion. It also serves health metrics since it
-// communicates with RateKeeper to gather health information of the cluster.
+// GrvProxy is proxy primarily specializing on serving GetReadVersion. It also
+// serves health metrics since it communicates with RateKeeper to gather health
+// information of the cluster, and handles proxied GlobalConfig requests.
 struct GrvProxyInterface {
 	constexpr static FileIdentifier file_identifier = 8743216;
 	enum { LocationAwareLoadBalance = 1 };
@@ -43,6 +43,7 @@ struct GrvProxyInterface {
 	//   committed)
 	RequestStream<ReplyPromise<Void>> waitFailure; // reports heartbeat to master.
 	RequestStream<struct GetHealthMetricsRequest> getHealthMetrics;
+	RequestStream<struct GlobalConfigRefreshRequest> refreshGlobalConfig;
 
 	UID id() const { return getConsistentReadVersion.getEndpoint().token; }
 	std::string toString() const { return id().shortString(); }
@@ -59,6 +60,8 @@ struct GrvProxyInterface {
 			    RequestStream<ReplyPromise<Void>>(getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(1));
 			getHealthMetrics = RequestStream<struct GetHealthMetricsRequest>(
 			    getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(2));
+			refreshGlobalConfig = RequestStream<struct GlobalConfigRefreshRequest>(
+			    getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(3));
 		}
 	}
 
@@ -67,6 +70,7 @@ struct GrvProxyInterface {
 		streams.push_back(getConsistentReadVersion.getReceiver(TaskPriority::ReadSocket));
 		streams.push_back(waitFailure.getReceiver());
 		streams.push_back(getHealthMetrics.getReceiver());
+		streams.push_back(refreshGlobalConfig.getReceiver());
 		FlowTransport::transport().addEndpoints(streams);
 	}
 };
diff --git a/fdbclient/include/fdbclient/HighContentionPrefixAllocator.actor.h b/fdbclient/include/fdbclient/HighContentionPrefixAllocator.actor.h
index 57e185cd78..20ec964436 100644
--- a/fdbclient/include/fdbclient/HighContentionPrefixAllocator.actor.h
+++ b/fdbclient/include/fdbclient/HighContentionPrefixAllocator.actor.h
@@ -134,7 +134,7 @@ private:
 
 				if (!candidateValueFuture.get().present()) {
 					tr->addWriteConflictRange(singleKeyRange(self->recent.get(candidate).key()));
-					return Tuple().append(candidate).pack();
+					return Tuple::makeTuple(candidate).pack();
 				}
 			}
 		}
diff --git a/fdbclient/include/fdbclient/IClientApi.h b/fdbclient/include/fdbclient/IClientApi.h
index 73e743d060..10d777a537 100644
--- a/fdbclient/include/fdbclient/IClientApi.h
+++ b/fdbclient/include/fdbclient/IClientApi.h
@@ -22,6 +22,7 @@
 #define FDBCLIENT_ICLIENTAPI_H
 #pragma once
 
+#include "fdbclient/BlobGranuleCommon.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/Tenant.h"
@@ -78,13 +79,30 @@ public:
 	virtual ThreadFuture<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRangeRef& range,
 	                                                                        int64_t chunkSize) = 0;
 
-	virtual ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRangeRef& keyRange) = 0;
+	virtual ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRangeRef& keyRange,
+	                                                                              int rowLimit) = 0;
 
 	virtual ThreadResult<RangeResult> readBlobGranules(const KeyRangeRef& keyRange,
 	                                                   Version beginVersion,
 	                                                   Optional<Version> readVersion,
 	                                                   ReadBlobGranuleContext granuleContext) = 0;
 
+	virtual ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesStart(
+	    const KeyRangeRef& keyRange,
+	    Version beginVersion,
+	    Optional<Version> readVersion,
+	    Version* readVersionOut) = 0;
+
+	virtual ThreadResult<RangeResult> readBlobGranulesFinish(
+	    ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> startFuture,
+	    const KeyRangeRef& keyRange,
+	    Version beginVersion,
+	    Version readVersion,
+	    ReadBlobGranuleContext granuleContext) = 0;
+
+	virtual ThreadFuture<Standalone<VectorRef<BlobGranuleSummaryRef>>>
+	summarizeBlobGranules(const KeyRangeRef& keyRange, Optional<Version> summaryVersion, int rangeLimit) = 0;
+
 	virtual void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) = 0;
 	virtual void set(const KeyRef& key, const ValueRef& value) = 0;
 	virtual void clear(const KeyRef& begin, const KeyRef& end) = 0;
@@ -172,6 +190,13 @@ public:
 	virtual ThreadFuture<Key> purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) = 0;
 	virtual ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) = 0;
 
+	virtual ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) = 0;
+	virtual ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) = 0;
+	virtual ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
+	                                                                              int rangeLimit) = 0;
+
+	virtual ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) = 0;
+
 	// Interface to manage shared state across multiple connections to the same Database
 	virtual ThreadFuture<DatabaseSharedState*> createSharedState() = 0;
 	virtual void setSharedState(DatabaseSharedState* p) = 0;
@@ -190,6 +215,7 @@ public:
 
 	virtual void selectApiVersion(int apiVersion) = 0;
 	virtual const char* getClientVersion() = 0;
+	virtual void useFutureProtocolVersion() = 0;
 
 	virtual void setNetworkOption(FDBNetworkOptions::Option option,
 	                              Optional<StringRef> value = Optional<StringRef>()) = 0;
diff --git a/fdbclient/include/fdbclient/IConfigTransaction.h b/fdbclient/include/fdbclient/IConfigTransaction.h
index 8f21679e27..8451e86f21 100644
--- a/fdbclient/include/fdbclient/IConfigTransaction.h
+++ b/fdbclient/include/fdbclient/IConfigTransaction.h
@@ -55,7 +55,7 @@ public:
 	Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(KeyRange const& range, int64_t chunkSize) override {
 		throw client_invalid_operation();
 	}
-	Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(KeyRange const& range) override {
+	Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(KeyRange const& range, int rowLimit) override {
 		throw client_invalid_operation();
 	}
 	Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranules(KeyRange const& range,
@@ -64,7 +64,13 @@ public:
 	                                                                    Version* readVersionOut) override {
 		throw client_invalid_operation();
 	}
+	Future<Standalone<VectorRef<BlobGranuleSummaryRef>>> summarizeBlobGranules(KeyRange const& range,
+	                                                                           Optional<Version> readVersion,
+	                                                                           int rangeLimit) override {
+		throw client_invalid_operation();
+	}
 	Future<int64_t> getEstimatedRangeSizeBytes(KeyRange const& keys) override { throw client_invalid_operation(); }
+	void addGranuleMaterializeStats(const GranuleMaterializeStats& stats) override { throw client_invalid_operation(); }
 	void addReadConflictRange(KeyRangeRef const& keys) override { throw client_invalid_operation(); }
 	void makeSelfConflicting() override { throw client_invalid_operation(); }
 	void atomicOp(KeyRef const& key, ValueRef const& operand, uint32_t operationType) override {
diff --git a/fdbclient/include/fdbclient/ISingleThreadTransaction.h b/fdbclient/include/fdbclient/ISingleThreadTransaction.h
index b44f58b464..71a0897693 100644
--- a/fdbclient/include/fdbclient/ISingleThreadTransaction.h
+++ b/fdbclient/include/fdbclient/ISingleThreadTransaction.h
@@ -80,11 +80,15 @@ public:
 	virtual Future<Standalone<VectorRef<const char*>>> getAddressesForKey(Key const& key) = 0;
 	virtual Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(KeyRange const& range, int64_t chunkSize) = 0;
 	virtual Future<int64_t> getEstimatedRangeSizeBytes(KeyRange const& keys) = 0;
-	virtual Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(KeyRange const& range) = 0;
+	virtual Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(KeyRange const& range, int rangeLimit) = 0;
 	virtual Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranules(KeyRange const& range,
 	                                                                            Version begin,
 	                                                                            Optional<Version> readVersion,
 	                                                                            Version* readVersionOut = nullptr) = 0;
+	virtual Future<Standalone<VectorRef<BlobGranuleSummaryRef>>> summarizeBlobGranules(KeyRange const& range,
+	                                                                                   Optional<Version> summaryVersion,
+	                                                                                   int rangeLimit) = 0;
+	virtual void addGranuleMaterializeStats(const GranuleMaterializeStats& stats) = 0;
 	virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
 	virtual void makeSelfConflicting() = 0;
 	virtual void atomicOp(KeyRef const& key, ValueRef const& operand, uint32_t operationType) = 0;
diff --git a/fdbclient/include/fdbclient/KeyBackedTypes.h b/fdbclient/include/fdbclient/KeyBackedTypes.h
index 98171671ee..a3fee57644 100644
--- a/fdbclient/include/fdbclient/KeyBackedTypes.h
+++ b/fdbclient/include/fdbclient/KeyBackedTypes.h
@@ -23,9 +23,10 @@
 #include <utility>
 #include <vector>
 
+#include "fdbclient/ClientBooleanParams.h"
+#include "fdbclient/CommitTransaction.h"
+#include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/GenericTransactionHelper.h"
-#include "fdbclient/IClientApi.h"
-#include "fdbclient/DatabaseContext.h"
 #include "fdbclient/Subspace.h"
 #include "flow/ObjectSerializer.h"
 #include "flow/genericactors.actor.h"
@@ -59,7 +60,7 @@ inline Tuple TupleCodec<Tuple>::unpack(Standalone<StringRef> const& val) {
 
 template <>
 inline Standalone<StringRef> TupleCodec<int64_t>::pack(int64_t const& val) {
-	return Tuple().append(val).pack();
+	return Tuple::makeTuple(val).pack();
 }
 template <>
 inline int64_t TupleCodec<int64_t>::unpack(Standalone<StringRef> const& val) {
@@ -68,7 +69,7 @@ inline int64_t TupleCodec<int64_t>::unpack(Standalone<StringRef> const& val) {
 
 template <>
 inline Standalone<StringRef> TupleCodec<bool>::pack(bool const& val) {
-	return Tuple().append(val ? 1 : 0).pack();
+	return Tuple::makeTuple(val ? 1 : 0).pack();
 }
 template <>
 inline bool TupleCodec<bool>::unpack(Standalone<StringRef> const& val) {
@@ -77,7 +78,7 @@ inline bool TupleCodec<bool>::unpack(Standalone<StringRef> const& val) {
 
 template <>
 inline Standalone<StringRef> TupleCodec<Standalone<StringRef>>::pack(Standalone<StringRef> const& val) {
-	return Tuple().append(val).pack();
+	return Tuple::makeTuple(val).pack();
 }
 template <>
 inline Standalone<StringRef> TupleCodec<Standalone<StringRef>>::unpack(Standalone<StringRef> const& val) {
@@ -96,7 +97,7 @@ inline UID TupleCodec<UID>::unpack(Standalone<StringRef> const& val) {
 // This is backward compatible with TupleCodec<Standalone<StringRef>>
 template <>
 inline Standalone<StringRef> TupleCodec<std::string>::pack(std::string const& val) {
-	return Tuple().append(StringRef(val)).pack();
+	return Tuple::makeTuple(val).pack();
 }
 template <>
 inline std::string TupleCodec<std::string>::unpack(Standalone<StringRef> const& val) {
@@ -143,7 +144,7 @@ struct TupleCodec<std::vector<T>> {
 
 template <>
 inline Standalone<StringRef> TupleCodec<KeyRange>::pack(KeyRange const& val) {
-	return Tuple().append(val.begin).append(val.end).pack();
+	return Tuple::makeTuple(val.begin, val.end).pack();
 }
 template <>
 inline KeyRange TupleCodec<KeyRange>::unpack(Standalone<StringRef> const& val) {
@@ -151,6 +152,23 @@ inline KeyRange TupleCodec<KeyRange>::unpack(Standalone<StringRef> const& val) {
 	return KeyRangeRef(t.getString(0), t.getString(1));
 }
 
+struct NullCodec {
+	static Standalone<StringRef> pack(Standalone<StringRef> val) { return val; }
+	static Standalone<StringRef> unpack(Standalone<StringRef> val) { return val; }
+};
+
+template <class T>
+struct BinaryCodec {
+	static Standalone<StringRef> pack(T val) { return BinaryWriter::toValue<T>(val, Unversioned()); }
+	static T unpack(Standalone<StringRef> val) { return BinaryReader::fromStringRef<T>(val, Unversioned()); }
+};
+
+template <typename ResultType>
+struct KeyBackedRangeResult {
+	std::vector<ResultType> results;
+	bool more;
+};
+
 // Convenient read/write access to a single value of type T stored at key
 // Even though 'this' is not actually mutated, methods that change the db key are not const.
 template <typename T, typename Codec = TupleCodec<T>>
@@ -230,7 +248,7 @@ public:
 
 	template <class Transaction>
 	typename std::enable_if<!is_transaction_creator<Transaction>, void>::type set(Transaction tr, T const& val) {
-		return tr->set(key, Codec::pack(val));
+		tr->set(key, Codec::pack(val));
 	}
 
 	template <class DB>
@@ -245,7 +263,7 @@ public:
 
 	template <class Transaction>
 	typename std::enable_if<!is_transaction_creator<Transaction>, void>::type clear(Transaction tr) {
-		return tr->clear(key);
+		tr->clear(key);
 	}
 
 	Key key;
@@ -279,17 +297,17 @@ public:
 
 	template <class Transaction>
 	void set(Transaction tr, T const& val) {
-		return tr->set(key, BinaryWriter::toValue<T>(val, Unversioned()));
+		tr->set(key, BinaryWriter::toValue<T>(val, Unversioned()));
 	}
 
 	template <class Transaction>
 	void atomicOp(Transaction tr, T const& val, MutationRef::Type type) {
-		return tr->atomicOp(key, BinaryWriter::toValue<T>(val, Unversioned()), type);
+		tr->atomicOp(key, BinaryWriter::toValue<T>(val, Unversioned()), type);
 	}
 
 	template <class Transaction>
 	void clear(Transaction tr) {
-		return tr->clear(key);
+		tr->clear(key);
 	}
 
 	Key key;
@@ -308,16 +326,16 @@ public:
 	typedef _KeyType KeyType;
 	typedef _ValueType ValueType;
 	typedef std::pair<KeyType, ValueType> PairType;
-	typedef std::vector<PairType> PairsType;
+	typedef KeyBackedRangeResult<PairType> RangeResultType;
 
 	// If end is not present one key past the end of the map is used.
 	template <class Transaction>
-	Future<PairsType> getRange(Transaction tr,
-	                           Optional<KeyType> const& begin,
-	                           Optional<KeyType> const& end,
-	                           int limit,
-	                           Snapshot snapshot = Snapshot::False,
-	                           Reverse reverse = Reverse::False) const {
+	Future<RangeResultType> getRange(Transaction tr,
+	                                 Optional<KeyType> const& begin,
+	                                 Optional<KeyType> const& end,
+	                                 int limit,
+	                                 Snapshot snapshot = Snapshot::False,
+	                                 Reverse reverse = Reverse::False) const {
 		Key prefix = subspace.begin; // 'this' could be invalid inside lambda
 
 		Key beginKey = begin.present() ? prefix.withSuffix(KeyCodec::pack(begin.get())) : subspace.begin;
@@ -326,16 +344,18 @@ public:
 		typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
 		    tr->getRange(KeyRangeRef(beginKey, endKey), GetRangeLimits(limit), snapshot, reverse);
 
-		return holdWhile(getRangeFuture,
-		                 map(safeThreadFutureToFuture(getRangeFuture), [prefix](RangeResult const& kvs) -> PairsType {
-			                 PairsType results;
-			                 for (int i = 0; i < kvs.size(); ++i) {
-				                 KeyType key = KeyCodec::unpack(kvs[i].key.removePrefix(prefix));
-				                 ValueType val = ValueCodec::unpack(kvs[i].value);
-				                 results.push_back(PairType(key, val));
-			                 }
-			                 return results;
-		                 }));
+		return holdWhile(
+		    getRangeFuture,
+		    map(safeThreadFutureToFuture(getRangeFuture), [prefix](RangeResult const& kvs) -> RangeResultType {
+			    RangeResultType rangeResult;
+			    for (int i = 0; i < kvs.size(); ++i) {
+				    KeyType key = KeyCodec::unpack(kvs[i].key.removePrefix(prefix));
+				    ValueType val = ValueCodec::unpack(kvs[i].value);
+				    rangeResult.results.push_back(PairType(key, val));
+			    }
+			    rangeResult.more = kvs.more;
+			    return rangeResult;
+		    }));
 	}
 
 	template <class Transaction>
@@ -351,6 +371,16 @@ public:
 		    }));
 	}
 
+	// Get key's value or defaultValue if it doesn't exist
+	template <class Transaction>
+	Future<ValueType> getD(Transaction tr,
+	                       KeyType const& key,
+	                       Snapshot snapshot = Snapshot::False,
+	                       ValueType defaultValue = ValueType()) const {
+		return map(get(tr, key, snapshot),
+		           [=](Optional<ValueType> val) -> ValueType { return val.orDefault(defaultValue); });
+	}
+
 	// Returns a Property that can be get/set that represents key's entry in this this.
 	KeyBackedProperty<ValueType> getProperty(KeyType const& key) const {
 		return subspace.begin.withSuffix(KeyCodec::pack(key));
@@ -365,20 +395,27 @@ public:
 		return k.expectedSize() + v.expectedSize();
 	}
 
+	template <class Transaction>
+	void atomicOp(Transaction tr, KeyType const& key, ValueType const& val, MutationRef::Type type) {
+		Key k = subspace.begin.withSuffix(KeyCodec::pack(key));
+		Value v = ValueCodec::pack(val);
+		tr->atomicOp(k, v, type);
+	}
+
 	template <class Transaction>
 	void erase(Transaction tr, KeyType const& key) {
-		return tr->clear(subspace.begin.withSuffix(KeyCodec::pack(key)));
+		tr->clear(subspace.begin.withSuffix(KeyCodec::pack(key)));
 	}
 
 	template <class Transaction>
 	void erase(Transaction tr, KeyType const& begin, KeyType const& end) {
-		return tr->clear(KeyRangeRef(subspace.begin.withSuffix(KeyCodec::pack(begin)),
-		                             subspace.begin.withSuffix(KeyCodec::pack(end))));
+		tr->clear(KeyRangeRef(subspace.begin.withSuffix(KeyCodec::pack(begin)),
+		                      subspace.begin.withSuffix(KeyCodec::pack(end))));
 	}
 
 	template <class Transaction>
 	void clear(Transaction tr) {
-		return tr->clear(subspace);
+		tr->clear(subspace);
 	}
 
 	KeyRange subspace;
@@ -463,7 +500,7 @@ public:
 
 	template <class Transaction>
 	typename std::enable_if<!is_transaction_creator<Transaction>, void>::type set(Transaction tr, T const& val) {
-		return tr->set(key, ObjectWriter::toValue(val, versionOptions));
+		tr->set(key, ObjectWriter::toValue(val, versionOptions));
 	}
 
 	template <class DB>
@@ -478,7 +515,7 @@ public:
 
 	template <class Transaction>
 	typename std::enable_if<!is_transaction_creator<Transaction>, void>::type clear(Transaction tr) {
-		return tr->clear(key);
+		tr->clear(key);
 	}
 
 	Key key;
@@ -497,15 +534,15 @@ public:
 	typedef _KeyType KeyType;
 	typedef _ValueType ValueType;
 	typedef std::pair<KeyType, ValueType> PairType;
-	typedef std::vector<PairType> PairsType;
+	typedef KeyBackedRangeResult<PairType> RangeResultType;
 
 	template <class Transaction>
-	Future<PairsType> getRange(Transaction tr,
-	                           Optional<KeyType> const& begin,
-	                           Optional<KeyType> const& end,
-	                           int limit,
-	                           Snapshot snapshot = Snapshot::False,
-	                           Reverse reverse = Reverse::False) const {
+	Future<RangeResultType> getRange(Transaction tr,
+	                                 Optional<KeyType> const& begin,
+	                                 Optional<KeyType> const& end,
+	                                 int limit,
+	                                 Snapshot snapshot = Snapshot::False,
+	                                 Reverse reverse = Reverse::False) const {
 		Key beginKey = begin.present() ? subspace.begin.withSuffix(KeyCodec::pack(begin.get())) : subspace.begin;
 		Key endKey = end.present() ? subspace.begin.withSuffix(KeyCodec::pack(end.get())) : subspace.end;
 
@@ -514,14 +551,15 @@ public:
 
 		return holdWhile(
 		    getRangeFuture,
-		    map(safeThreadFutureToFuture(getRangeFuture), [self = *this](RangeResult const& kvs) -> PairsType {
-			    PairsType results;
+		    map(safeThreadFutureToFuture(getRangeFuture), [self = *this](RangeResult const& kvs) -> RangeResultType {
+			    RangeResultType rangeResult;
 			    for (int i = 0; i < kvs.size(); ++i) {
 				    KeyType key = KeyCodec::unpack(kvs[i].key.removePrefix(self.subspace.begin));
 				    ValueType val = ObjectReader::fromStringRef<ValueType>(kvs[i].value, self.versionOptions);
-				    results.push_back(PairType(key, val));
+				    rangeResult.results.push_back(PairType(key, val));
 			    }
-			    return results;
+			    rangeResult.more = kvs.more;
+			    return rangeResult;
 		    }));
 	}
 
@@ -560,18 +598,18 @@ public:
 
 	template <class Transaction>
 	void erase(Transaction tr, KeyType const& key) {
-		return tr->clear(subspace.begin.withSuffix(KeyCodec::pack(key)));
+		tr->clear(subspace.begin.withSuffix(KeyCodec::pack(key)));
 	}
 
 	template <class Transaction>
 	void erase(Transaction tr, KeyType const& begin, KeyType const& end) {
-		return tr->clear(KeyRangeRef(subspace.begin.withSuffix(KeyCodec::pack(begin)),
-		                             subspace.begin.withSuffix(KeyCodec::pack(end))));
+		tr->clear(KeyRangeRef(subspace.begin.withSuffix(KeyCodec::pack(begin)),
+		                      subspace.begin.withSuffix(KeyCodec::pack(end))));
 	}
 
 	template <class Transaction>
 	void clear(Transaction tr) {
-		return tr->clear(subspace);
+		tr->clear(subspace);
 	}
 
 	KeyRange subspace;
@@ -584,15 +622,15 @@ public:
 	KeyBackedSet(KeyRef key) : subspace(prefixRange(key)) {}
 
 	typedef _ValueType ValueType;
-	typedef std::vector<ValueType> Values;
+	typedef KeyBackedRangeResult<ValueType> RangeResultType;
 
 	template <class Transaction>
-	Future<Values> getRange(Transaction tr,
-	                        Optional<ValueType> const& begin,
-	                        Optional<ValueType> const& end,
-	                        int limit,
-	                        Snapshot snapshot = Snapshot::False,
-	                        Reverse reverse = Reverse::False) const {
+	Future<RangeResultType> getRange(Transaction tr,
+	                                 Optional<ValueType> const& begin,
+	                                 Optional<ValueType> const& end,
+	                                 int limit,
+	                                 Snapshot snapshot = Snapshot::False,
+	                                 Reverse reverse = Reverse::False) const {
 		Key prefix = subspace.begin; // 'this' could be invalid inside lambda
 		Key beginKey = begin.present() ? prefix.withSuffix(Codec::pack(begin.get())) : subspace.begin;
 		Key endKey = end.present() ? prefix.withSuffix(Codec::pack(end.get())) : subspace.end;
@@ -600,14 +638,16 @@ public:
 		typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
 		    tr->getRange(KeyRangeRef(beginKey, endKey), GetRangeLimits(limit), snapshot, reverse);
 
-		return holdWhile(getRangeFuture,
-		                 map(safeThreadFutureToFuture(getRangeFuture), [prefix](RangeResult const& kvs) -> Values {
-			                 Values results;
-			                 for (int i = 0; i < kvs.size(); ++i) {
-				                 results.push_back(Codec::unpack(kvs[i].key.removePrefix(prefix)));
-			                 }
-			                 return results;
-		                 }));
+		return holdWhile(
+		    getRangeFuture,
+		    map(safeThreadFutureToFuture(getRangeFuture), [prefix](RangeResult const& kvs) -> RangeResultType {
+			    RangeResultType rangeResult;
+			    for (int i = 0; i < kvs.size(); ++i) {
+				    rangeResult.results.push_back(Codec::unpack(kvs[i].key.removePrefix(prefix)));
+			    }
+			    rangeResult.more = kvs.more;
+			    return rangeResult;
+		    }));
 	}
 
 	template <class Transaction>
@@ -630,18 +670,18 @@ public:
 
 	template <class Transaction>
 	void erase(Transaction tr, ValueType const& val) {
-		return tr->clear(subspace.begin.withSuffix(Codec::pack(val)));
+		tr->clear(subspace.begin.withSuffix(Codec::pack(val)));
 	}
 
 	template <class Transaction>
 	void erase(Transaction tr, ValueType const& begin, ValueType const& end) {
-		return tr->clear(
+		tr->clear(
 		    KeyRangeRef(subspace.begin.withSuffix(Codec::pack(begin)), subspace.begin.withSuffix(Codec::pack(end))));
 	}
 
 	template <class Transaction>
 	void clear(Transaction tr) {
-		return tr->clear(subspace);
+		tr->clear(subspace);
 	}
 
 	KeyRange subspace;
diff --git a/fdbclient/include/fdbclient/KeyRangeMap.h b/fdbclient/include/fdbclient/KeyRangeMap.h
index 88cce027a8..f88dc72dda 100644
--- a/fdbclient/include/fdbclient/KeyRangeMap.h
+++ b/fdbclient/include/fdbclient/KeyRangeMap.h
@@ -136,6 +136,16 @@ Future<RangeResult> krmGetRanges(Reference<ReadYourWritesTransaction> const& tr,
                                  KeyRange const& keys,
                                  int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
                                  int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
+Future<RangeResult> krmGetRangesUnaligned(Transaction* const& tr,
+                                          Key const& mapPrefix,
+                                          KeyRange const& keys,
+                                          int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+                                          int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
+Future<RangeResult> krmGetRangesUnaligned(Reference<ReadYourWritesTransaction> const& tr,
+                                          Key const& mapPrefix,
+                                          KeyRange const& keys,
+                                          int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+                                          int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
 void krmSetPreviouslyEmptyRange(Transaction* tr,
                                 const KeyRef& mapPrefix,
                                 const KeyRangeRef& keys,
@@ -162,7 +172,7 @@ Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& t
                                    KeyRange const& range,
                                    KeyRange const& maxRange,
                                    Value const& value);
-RangeResult krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, RangeResult kv);
+RangeResult krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, RangeResult kv, bool align = true);
 
 template <class Val, class Metric, class MetricFunc>
 std::vector<KeyRangeWith<Val>> KeyRangeMap<Val, Metric, MetricFunc>::getAffectedRangesAfterInsertion(
diff --git a/fdbclient/include/fdbclient/ManagementAPI.actor.h b/fdbclient/include/fdbclient/ManagementAPI.actor.h
index 63f56242f7..c0725324c8 100644
--- a/fdbclient/include/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/include/fdbclient/ManagementAPI.actor.h
@@ -57,7 +57,8 @@ struct IQuorumChange : ReferenceCounted<IQuorumChange> {
 // Change to use the given set of coordination servers
 ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
                                                                ClusterConnectionString* conn,
-                                                               std::string newName);
+                                                               std::string newName,
+                                                               bool disableConfigDB);
 ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change);
 Reference<IQuorumChange> autoQuorumChange(int desired = -1);
 Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other);
@@ -159,5 +160,9 @@ bool schemaMatch(json_spirit::mValue const& schema,
 // storage nodes
 ACTOR Future<Void> mgmtSnapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID);
 
+// Set and get the storage quota per tenant
+void setStorageQuota(Transaction& tr, StringRef tenantName, uint64_t quota);
+ACTOR Future<Optional<uint64_t>> getStorageQuota(Transaction* tr, StringRef tenantName);
+
 #include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbclient/include/fdbclient/Metacluster.h b/fdbclient/include/fdbclient/Metacluster.h
new file mode 100644
index 0000000000..7f07286ae4
--- /dev/null
+++ b/fdbclient/include/fdbclient/Metacluster.h
@@ -0,0 +1,183 @@
+/*
+ * Metacluster.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBCLIENT_METACLUSTER_H
+#define FDBCLIENT_METACLUSTER_H
+#include "fdbclient/CoordinationInterface.h"
+#include "json_spirit/json_spirit_value.h"
+#pragma once
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "flow/flat_buffers.h"
+
+struct ClusterUsage {
+	int numTenantGroups = 0;
+
+	ClusterUsage() = default;
+	ClusterUsage(int numTenantGroups) : numTenantGroups(numTenantGroups) {}
+
+	json_spirit::mObject toJson() const;
+
+	bool operator==(const ClusterUsage& other) const noexcept { return numTenantGroups == other.numTenantGroups; }
+	bool operator!=(const ClusterUsage& other) const noexcept { return !(*this == other); }
+	bool operator<(const ClusterUsage& other) const noexcept { return numTenantGroups < other.numTenantGroups; }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, numTenantGroups);
+	}
+};
+
+template <>
+struct Traceable<ClusterUsage> : std::true_type {
+	static std::string toString(const ClusterUsage& value) {
+		return format("NumTenantGroups: %d", value.numTenantGroups);
+	}
+};
+
+// Represents the various states that a data cluster could be in.
+//
+// READY - the data cluster is active
+// REMOVING - the data cluster is being removed and cannot have its configuration changed or any tenants created
+// RESTORING - the data cluster is being restored and cannot have its configuration changed or any tenants
+//             created/updated/deleted.
+enum class DataClusterState { READY, REMOVING, RESTORING };
+
+struct DataClusterEntry {
+	constexpr static FileIdentifier file_identifier = 929511;
+
+	static std::string clusterStateToString(DataClusterState clusterState);
+	static DataClusterState stringToClusterState(std::string stateStr);
+
+	UID id;
+	ClusterUsage capacity;
+	ClusterUsage allocated;
+
+	DataClusterState clusterState = DataClusterState::READY;
+
+	DataClusterEntry() = default;
+	DataClusterEntry(ClusterUsage capacity) : capacity(capacity) {}
+	DataClusterEntry(UID id, ClusterUsage capacity, ClusterUsage allocated)
+	  : id(id), capacity(capacity), allocated(allocated) {}
+
+	// Returns true if all configurable properties match
+	bool matchesConfiguration(DataClusterEntry const& other) const {
+		return id == other.id && capacity == other.capacity;
+	}
+
+	bool hasCapacity() const { return allocated < capacity; }
+
+	Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion()); }
+	static DataClusterEntry decode(ValueRef const& value) {
+		return ObjectReader::fromStringRef<DataClusterEntry>(value, IncludeVersion());
+	}
+
+	json_spirit::mObject toJson() const;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, id, capacity, allocated, clusterState);
+	}
+};
+
+struct MetaclusterRegistrationEntry {
+	constexpr static FileIdentifier file_identifier = 13448589;
+
+	ClusterType clusterType;
+
+	ClusterName metaclusterName;
+	ClusterName name;
+	UID metaclusterId;
+	UID id;
+
+	MetaclusterRegistrationEntry() = default;
+	MetaclusterRegistrationEntry(ClusterName metaclusterName, UID metaclusterId)
+	  : clusterType(ClusterType::METACLUSTER_MANAGEMENT), metaclusterName(metaclusterName), name(metaclusterName),
+	    metaclusterId(metaclusterId), id(metaclusterId) {}
+	MetaclusterRegistrationEntry(ClusterName metaclusterName, ClusterName name, UID metaclusterId, UID id)
+	  : clusterType(ClusterType::METACLUSTER_DATA), metaclusterName(metaclusterName), name(name),
+	    metaclusterId(metaclusterId), id(id) {
+		ASSERT(metaclusterName != name && metaclusterId != id);
+	}
+
+	// Returns true if this entry is associated with the same cluster as the passed in entry. If one entry is from the
+	// management cluster and the other is from a data cluster, this checks whether they are part of the same
+	// metacluster.
+	bool matches(MetaclusterRegistrationEntry const& other) const {
+		if (metaclusterName != other.metaclusterName || metaclusterId != other.metaclusterId) {
+			return false;
+		} else if (clusterType == ClusterType::METACLUSTER_DATA && other.clusterType == ClusterType::METACLUSTER_DATA &&
+		           (name != other.name || id != other.id)) {
+			return false;
+		}
+
+		return true;
+	}
+
+	MetaclusterRegistrationEntry toManagementClusterRegistration() const {
+		ASSERT(clusterType == ClusterType::METACLUSTER_DATA);
+		return MetaclusterRegistrationEntry(metaclusterName, metaclusterId);
+	}
+
+	MetaclusterRegistrationEntry toDataClusterRegistration(ClusterName name, UID id) const {
+		ASSERT(clusterType == ClusterType::METACLUSTER_MANAGEMENT);
+		return MetaclusterRegistrationEntry(metaclusterName, name, metaclusterId, id);
+	}
+
+	Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion()); }
+	static MetaclusterRegistrationEntry decode(ValueRef const& value) {
+		return ObjectReader::fromStringRef<MetaclusterRegistrationEntry>(value, IncludeVersion());
+	}
+	static Optional<MetaclusterRegistrationEntry> decode(Optional<Value> value) {
+		return value.map<MetaclusterRegistrationEntry>(
+		    [](ValueRef const& v) { return MetaclusterRegistrationEntry::decode(v); });
+	}
+
+	std::string toString() const {
+		if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+			return fmt::format(
+			    "metacluster name: {}, metacluster id: {}", printable(metaclusterName), metaclusterId.shortString());
+		} else {
+			return fmt::format("metacluster name: {}, metacluster id: {}, data cluster name: {}, data cluster id: {}",
+			                   printable(metaclusterName),
+			                   metaclusterId.shortString(),
+			                   printable(name),
+			                   id.shortString());
+		}
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, clusterType, metaclusterName, name, metaclusterId, id);
+	}
+};
+
+template <>
+struct Traceable<MetaclusterRegistrationEntry> : std::true_type {
+	static std::string toString(MetaclusterRegistrationEntry const& entry) { return entry.toString(); }
+};
+
+struct MetaclusterMetadata {
+	// Registration information for a metacluster, stored on both management and data clusters
+	static KeyBackedObjectProperty<MetaclusterRegistrationEntry, decltype(IncludeVersion())>& metaclusterRegistration();
+};
+
+#endif
diff --git a/fdbclient/include/fdbclient/MetaclusterManagement.actor.h b/fdbclient/include/fdbclient/MetaclusterManagement.actor.h
new file mode 100644
index 0000000000..b13257f64e
--- /dev/null
+++ b/fdbclient/include/fdbclient/MetaclusterManagement.actor.h
@@ -0,0 +1,2020 @@
+/*
+ * MetaclusterManagement.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#include "fdbclient/FDBOptions.g.h"
+#include "flow/IRandom.h"
+#include "flow/ThreadHelper.actor.h"
+#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_METACLUSTER_MANAGEMENT_ACTOR_G_H)
+#define FDBCLIENT_METACLUSTER_MANAGEMENT_ACTOR_G_H
+#include "fdbclient/MetaclusterManagement.actor.g.h"
+#elif !defined(FDBCLIENT_METACLUSTER_MANAGEMENT_ACTOR_H)
+#define FDBCLIENT_METACLUSTER_MANAGEMENT_ACTOR_H
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/GenericTransactionHelper.h"
+#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/Metacluster.h"
+#include "fdbclient/MultiVersionTransaction.h"
+#include "fdbclient/SystemData.h"
+#include "fdbclient/TenantManagement.actor.h"
+#include "fdbclient/VersionedMap.h"
+#include "flow/flat_buffers.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+// This file provides the interfaces to manage metacluster metadata.
+//
+// These transactions can operate on clusters at different versions, so care needs to be taken to update the metadata
+// according to the cluster version.
+//
+// Support is maintained in this file for the current and the previous protocol versions.
+
+struct DataClusterMetadata {
+	constexpr static FileIdentifier file_identifier = 5573993;
+
+	DataClusterEntry entry;
+	ClusterConnectionString connectionString;
+
+	DataClusterMetadata() = default;
+	DataClusterMetadata(DataClusterEntry const& entry, ClusterConnectionString const& connectionString)
+	  : entry(entry), connectionString(connectionString) {}
+
+	bool matchesConfiguration(DataClusterMetadata const& other) const {
+		return entry.matchesConfiguration(other.entry) && connectionString == other.connectionString;
+	}
+
+	Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion()); }
+	static DataClusterMetadata decode(ValueRef const& value) {
+		return ObjectReader::fromStringRef<DataClusterMetadata>(value, IncludeVersion());
+	}
+
+	json_spirit::mValue toJson() const {
+		json_spirit::mObject obj = entry.toJson();
+		obj["connection_string"] = connectionString.toString();
+		return obj;
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, connectionString, entry);
+	}
+};
+
+FDB_DECLARE_BOOLEAN_PARAM(AddNewTenants);
+FDB_DECLARE_BOOLEAN_PARAM(RemoveMissingTenants);
+
+namespace MetaclusterAPI {
+
+struct ManagementClusterMetadata {
+	struct ConnectionStringCodec {
+		static inline Standalone<StringRef> pack(ClusterConnectionString const& val) {
+			return StringRef(val.toString());
+		}
+		static inline ClusterConnectionString unpack(Standalone<StringRef> const& val) {
+			return ClusterConnectionString(val.toString());
+		}
+	};
+
+	static TenantMetadataSpecification& tenantMetadata();
+
+	// A map from cluster name to the metadata associated with a cluster
+	static KeyBackedObjectMap<ClusterName, DataClusterEntry, decltype(IncludeVersion())>& dataClusters();
+
+	// A map from cluster name to the connection string for the cluster
+	static KeyBackedMap<ClusterName, ClusterConnectionString, TupleCodec<ClusterName>, ConnectionStringCodec>
+	    dataClusterConnectionRecords;
+
+	// A set of non-full clusters where the key is the tuple (num tenant groups allocated, cluster name).
+	static KeyBackedSet<Tuple> clusterCapacityIndex;
+
+	// A map from cluster name to a count of tenants
+	static KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>> clusterTenantCount;
+
+	// A set of (cluster name, tenant name, tenant ID) tuples ordered by cluster
+	static KeyBackedSet<Tuple> clusterTenantIndex;
+
+	// A set of (cluster, tenant group name) tuples ordered by cluster
+	static KeyBackedSet<Tuple> clusterTenantGroupIndex;
+};
+
+ACTOR Future<Reference<IDatabase>> openDatabase(ClusterConnectionString connectionString);
+
+ACTOR template <class Transaction>
+Future<Optional<DataClusterMetadata>> tryGetClusterTransaction(Transaction tr, ClusterName name) {
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+
+	state Future<Void> metaclusterRegistrationCheck =
+	    TenantAPI::checkTenantMode(tr, ClusterType::METACLUSTER_MANAGEMENT);
+
+	state Future<Optional<DataClusterEntry>> clusterEntryFuture =
+	    ManagementClusterMetadata::dataClusters().get(tr, name);
+	state Future<Optional<ClusterConnectionString>> connectionRecordFuture =
+	    ManagementClusterMetadata::dataClusterConnectionRecords.get(tr, name);
+
+	wait(metaclusterRegistrationCheck);
+
+	state Optional<DataClusterEntry> clusterEntry = wait(clusterEntryFuture);
+	Optional<ClusterConnectionString> connectionString = wait(connectionRecordFuture);
+
+	if (clusterEntry.present()) {
+		ASSERT(connectionString.present());
+		return Optional<DataClusterMetadata>(DataClusterMetadata(clusterEntry.get(), connectionString.get()));
+	} else {
+		return Optional<DataClusterMetadata>();
+	}
+}
+
+ACTOR template <class DB>
+Future<Optional<DataClusterMetadata>> tryGetCluster(Reference<DB> db, ClusterName name) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			Optional<DataClusterMetadata> metadata = wait(tryGetClusterTransaction(tr, name));
+			return metadata;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+ACTOR template <class Transaction>
+Future<DataClusterMetadata> getClusterTransaction(Transaction tr, ClusterNameRef name) {
+	Optional<DataClusterMetadata> metadata = wait(tryGetClusterTransaction(tr, name));
+	if (!metadata.present()) {
+		throw cluster_not_found();
+	}
+
+	return metadata.get();
+}
+
+ACTOR template <class DB>
+Future<DataClusterMetadata> getCluster(Reference<DB> db, ClusterName name) {
+	Optional<DataClusterMetadata> metadata = wait(tryGetCluster(db, name));
+	if (!metadata.present()) {
+		throw cluster_not_found();
+	}
+
+	return metadata.get();
+}
+
+ACTOR template <class Transaction>
+Future<Reference<IDatabase>> getAndOpenDatabase(Transaction managementTr, ClusterName clusterName) {
+	DataClusterMetadata clusterMetadata = wait(getClusterTransaction(managementTr, clusterName));
+	Reference<IDatabase> db = wait(openDatabase(clusterMetadata.connectionString));
+	return db;
+}
+
+template <class DB>
+struct MetaclusterOperationContext {
+	Reference<DB> managementDb;
+	Reference<IDatabase> dataClusterDb;
+
+	Optional<ClusterName> clusterName;
+
+	Optional<MetaclusterRegistrationEntry> metaclusterRegistration;
+	Optional<DataClusterMetadata> dataClusterMetadata;
+
+	MetaclusterOperationContext(Reference<DB> managementDb, Optional<ClusterName> clusterName = {})
+	  : managementDb(managementDb), clusterName(clusterName) {}
+
+	// Run a transaction on the management cluster. This verifies that the cluster is a management cluster and matches
+	// the same metacluster that we've run any previous transactions on. If a clusterName is set, it also verifies that
+	// the specified cluster is present. Stores the metaclusterRegistration entry and, if a clusterName is set, the
+	// dataClusterMetadata and dataClusterDb in the context.
+	ACTOR template <class Function>
+	static Future<decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue())>
+	runManagementTransaction(MetaclusterOperationContext* self, Function func) {
+		state Reference<typename DB::TransactionT> tr = self->managementDb->createTransaction();
+		state bool clusterPresentAtStart = self->clusterName.present();
+		loop {
+			try {
+				// If this transaction is retrying and didn't have the cluster name set at the beginning, clear it out
+				// to be set again in the next iteration.
+				if (!clusterPresentAtStart) {
+					self->clearCluster();
+				}
+
+				// Get the data cluster metadata for the specified cluster, if present
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				state Future<Optional<DataClusterMetadata>> dataClusterMetadataFuture;
+				if (self->clusterName.present()) {
+					dataClusterMetadataFuture = tryGetClusterTransaction(tr, self->clusterName.get());
+				}
+
+				// Get the metacluster registration information
+				state Optional<MetaclusterRegistrationEntry> currentMetaclusterRegistration =
+				    wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
+
+				state Optional<DataClusterMetadata> currentDataClusterMetadata;
+				if (self->clusterName.present()) {
+					wait(store(currentDataClusterMetadata, dataClusterMetadataFuture));
+				}
+
+				// Check that this is a management cluster and is the same metacluster that any previous transactions
+				// have run on.
+				if (!currentMetaclusterRegistration.present() ||
+				    currentMetaclusterRegistration.get().clusterType != ClusterType::METACLUSTER_MANAGEMENT) {
+					throw invalid_metacluster_operation();
+				} else if (self->metaclusterRegistration.present() &&
+				           !self->metaclusterRegistration.get().matches(currentMetaclusterRegistration.get())) {
+					throw invalid_metacluster_operation();
+				}
+
+				// If a cluster was specified, check that the cluster metadata is present. If so, load it and store it
+				// in the context. Additionally, store the data cluster details in the local metacluster registration
+				// entry.
+				if (self->clusterName.present()) {
+					if (!currentDataClusterMetadata.present()) {
+						throw cluster_not_found();
+					} else {
+						currentMetaclusterRegistration = currentMetaclusterRegistration.get().toDataClusterRegistration(
+						    self->clusterName.get(), currentDataClusterMetadata.get().entry.id);
+					}
+				}
+
+				// Store the metacluster registration entry
+				if (!self->metaclusterRegistration.present()) {
+					self->metaclusterRegistration = currentMetaclusterRegistration;
+				}
+
+				// Check that our data cluster has the same ID as previous transactions. If so, then store the updated
+				// cluster metadata in the context and open a connection to the data DB.
+				if (self->dataClusterMetadata.present() &&
+				    self->dataClusterMetadata.get().entry.id != currentDataClusterMetadata.get().entry.id) {
+					throw cluster_not_found();
+				} else if (self->clusterName.present()) {
+					self->dataClusterMetadata = currentDataClusterMetadata;
+					if (!self->dataClusterDb) {
+						wait(
+						    store(self->dataClusterDb, openDatabase(self->dataClusterMetadata.get().connectionString)));
+					}
+				}
+
+				state decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue()) result =
+				    wait(func(tr));
+
+				wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
+				return result;
+			} catch (Error& e) {
+				wait(safeThreadFutureToFuture(tr->onError(e)));
+			}
+		}
+	}
+
+	template <class Function>
+	Future<decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue())>
+	runManagementTransaction(Function func) {
+		return runManagementTransaction(this, func);
+	}
+
+	// Runs a transaction on the data cluster. This requires that a cluster name be set and that a transaction has
+	// already been run on the management cluster to populate the needed metadata. This verifies that the data cluster
+	// has the expected ID and is part of the metacluster that previous transactions have run on.
+	ACTOR template <class Function>
+	static Future<decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue())>
+	runDataClusterTransaction(MetaclusterOperationContext* self, Function func) {
+		ASSERT(self->dataClusterDb);
+		ASSERT(self->dataClusterMetadata.present());
+		ASSERT(self->metaclusterRegistration.present() &&
+		       self->metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_DATA);
+
+		state Reference<ITransaction> tr = self->dataClusterDb->createTransaction();
+		loop {
+			try {
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+
+				state Optional<MetaclusterRegistrationEntry> currentMetaclusterRegistration =
+				    wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
+
+				// Check that this is the expected data cluster and is part of the right metacluster
+				if (!currentMetaclusterRegistration.present() ||
+				    currentMetaclusterRegistration.get().clusterType != ClusterType::METACLUSTER_DATA) {
+					throw invalid_metacluster_operation();
+				} else if (!self->metaclusterRegistration.get().matches(currentMetaclusterRegistration.get())) {
+					throw invalid_metacluster_operation();
+				}
+
+				state decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue()) result =
+				    wait(func(tr));
+
+				wait(safeThreadFutureToFuture(tr->commit()));
+				return result;
+			} catch (Error& e) {
+				wait(safeThreadFutureToFuture(tr->onError(e)));
+			}
+		}
+	}
+
+	template <class Function>
+	Future<decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue())>
+	runDataClusterTransaction(Function func) {
+		return runDataClusterTransaction(this, func);
+	}
+
+	ACTOR static Future<Void> updateClusterName(MetaclusterOperationContext* self,
+	                                            Reference<typename DB::TransactionT> tr) {
+		state DataClusterMetadata currentDataClusterMetadata = wait(getClusterTransaction(tr, self->clusterName.get()));
+
+		self->metaclusterRegistration = self->metaclusterRegistration.get().toDataClusterRegistration(
+		    self->clusterName.get(), currentDataClusterMetadata.entry.id);
+
+		self->dataClusterMetadata = currentDataClusterMetadata;
+		if (!self->dataClusterDb) {
+			wait(store(self->dataClusterDb, openDatabase(self->dataClusterMetadata.get().connectionString)));
+		}
+
+		return Void();
+	}
+
+	// Sets the cluster used in this context. This must be called from a management cluster transaction, and it
+	// will load the cluster metadata and connect to the cluster.
+	Future<Void> setCluster(Reference<typename DB::TransactionT> tr, ClusterName clusterName) {
+		ASSERT(!this->clusterName.present());
+		ASSERT(!dataClusterMetadata.present());
+		ASSERT(metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_MANAGEMENT);
+		this->clusterName = clusterName;
+		return updateClusterName(this, tr);
+	}
+
+	// Clears the chosen cluster for this context. This is useful if we are retrying a transaction that expects an
+	// uninitialized cluster.
+	void clearCluster() {
+		clusterName = {};
+		dataClusterMetadata = {};
+		dataClusterDb = {};
+		if (metaclusterRegistration.present() &&
+		    metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_DATA) {
+			metaclusterRegistration = metaclusterRegistration.get().toManagementClusterRegistration();
+		}
+	}
+};
+
+template <class Transaction>
+Future<Optional<TenantMapEntry>> tryGetTenantTransaction(Transaction tr, TenantName name) {
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+	return ManagementClusterMetadata::tenantMetadata().tenantMap.get(tr, name);
+}
+
+ACTOR template <class DB>
+Future<Optional<TenantMapEntry>> tryGetTenant(Reference<DB> db, TenantName name) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			Optional<TenantMapEntry> entry = wait(tryGetTenantTransaction(tr, name));
+			return entry;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+ACTOR template <class Transaction>
+Future<TenantMapEntry> getTenantTransaction(Transaction tr, TenantName name) {
+	Optional<TenantMapEntry> entry = wait(tryGetTenantTransaction(tr, name));
+	if (!entry.present()) {
+		throw tenant_not_found();
+	}
+
+	return entry.get();
+}
+
+ACTOR template <class DB>
+Future<TenantMapEntry> getTenant(Reference<DB> db, TenantName name) {
+	Optional<TenantMapEntry> entry = wait(tryGetTenant(db, name));
+	if (!entry.present()) {
+		throw tenant_not_found();
+	}
+
+	return entry.get();
+}
+
+ACTOR template <class Transaction>
+Future<Void> managementClusterCheckEmpty(Transaction tr) {
+	state Future<KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>>> tenantsFuture =
+	    TenantMetadata::tenantMap().getRange(tr, {}, {}, 1);
+	state typename transaction_future_type<Transaction, RangeResult>::type dbContentsFuture =
+	    tr->getRange(normalKeys, 1);
+
+	KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenants = wait(tenantsFuture);
+	if (!tenants.results.empty()) {
+		throw cluster_not_empty();
+	}
+
+	RangeResult dbContents = wait(safeThreadFutureToFuture(dbContentsFuture));
+	if (!dbContents.empty()) {
+		throw cluster_not_empty();
+	}
+
+	return Void();
+}
+
+ACTOR template <class DB>
+Future<Optional<std::string>> createMetacluster(Reference<DB> db, ClusterName name) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+	state Optional<UID> metaclusterUid;
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+
+			state Future<Optional<MetaclusterRegistrationEntry>> metaclusterRegistrationFuture =
+			    MetaclusterMetadata::metaclusterRegistration().get(tr);
+
+			wait(managementClusterCheckEmpty(tr));
+
+			Optional<MetaclusterRegistrationEntry> existingRegistration = wait(metaclusterRegistrationFuture);
+			if (existingRegistration.present()) {
+				if (metaclusterUid.present() && metaclusterUid.get() == existingRegistration.get().metaclusterId) {
+					return Optional<std::string>();
+				} else {
+					return format("cluster is already registered as a %s named `%s'",
+					              existingRegistration.get().clusterType == ClusterType::METACLUSTER_DATA
+					                  ? "data cluster"
+					                  : "metacluster",
+					              printable(existingRegistration.get().name).c_str());
+				}
+			}
+
+			if (!metaclusterUid.present()) {
+				metaclusterUid = deterministicRandom()->randomUniqueID();
+			}
+
+			MetaclusterMetadata::metaclusterRegistration().set(
+			    tr, MetaclusterRegistrationEntry(name, metaclusterUid.get()));
+
+			wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
+			break;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+
+	return Optional<std::string>();
+}
+
+ACTOR template <class DB>
+Future<Void> decommissionMetacluster(Reference<DB> db) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+	state bool firstTry = true;
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+
+			ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
+			if (clusterType != ClusterType::METACLUSTER_MANAGEMENT) {
+				if (firstTry) {
+					throw invalid_metacluster_operation();
+				} else {
+					return Void();
+				}
+			}
+
+			// Erase all metadata not associated with specific tenants prior to checking
+			// cluster emptiness
+			ManagementClusterMetadata::tenantMetadata().tenantCount.clear(tr);
+			ManagementClusterMetadata::tenantMetadata().lastTenantId.clear(tr);
+			ManagementClusterMetadata::tenantMetadata().tenantTombstones.clear(tr);
+			ManagementClusterMetadata::tenantMetadata().tombstoneCleanupData.clear(tr);
+
+			wait(managementClusterCheckEmpty(tr));
+			MetaclusterMetadata::metaclusterRegistration().clear(tr);
+
+			firstTry = false;
+			wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
+			break;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+
+	return Void();
+}
+
+template <class Transaction>
+void updateClusterCapacityIndex(Transaction tr,
+                                ClusterName name,
+                                DataClusterEntry const& previousEntry,
+                                DataClusterEntry const& updatedEntry) {
+	// Entries are put in the cluster capacity index ordered by how many items are already allocated to them
+	if (previousEntry.hasCapacity()) {
+		ManagementClusterMetadata::clusterCapacityIndex.erase(
+		    tr, Tuple::makeTuple(previousEntry.allocated.numTenantGroups, name));
+	}
+	if (updatedEntry.hasCapacity()) {
+		ManagementClusterMetadata::clusterCapacityIndex.insert(
+		    tr, Tuple::makeTuple(updatedEntry.allocated.numTenantGroups, name));
+	}
+}
+
+// This should only be called from a transaction that has already confirmed that the cluster entry
+// is present. The updatedEntry should use the existing entry and modify only those fields that need
+// to be changed.
+template <class Transaction>
+void updateClusterMetadata(Transaction tr,
+                           ClusterNameRef name,
+                           DataClusterMetadata const& previousMetadata,
+                           Optional<ClusterConnectionString> const& updatedConnectionString,
+                           Optional<DataClusterEntry> const& updatedEntry) {
+
+	if (updatedEntry.present()) {
+		if (previousMetadata.entry.clusterState == DataClusterState::REMOVING) {
+			throw cluster_removed();
+		}
+		ManagementClusterMetadata::dataClusters().set(tr, name, updatedEntry.get());
+		updateClusterCapacityIndex(tr, name, previousMetadata.entry, updatedEntry.get());
+	}
+	if (updatedConnectionString.present()) {
+		ManagementClusterMetadata::dataClusterConnectionRecords.set(tr, name, updatedConnectionString.get());
+	}
+}
+
+template <class DB>
+struct RegisterClusterImpl {
+	MetaclusterOperationContext<DB> ctx;
+
+	// Initialization parameters
+	ClusterName clusterName;
+	ClusterConnectionString connectionString;
+	DataClusterEntry clusterEntry;
+
+	RegisterClusterImpl(Reference<DB> managementDb,
+	                    ClusterName clusterName,
+	                    ClusterConnectionString connectionString,
+	                    DataClusterEntry clusterEntry)
+	  : ctx(managementDb), clusterName(clusterName), connectionString(connectionString), clusterEntry(clusterEntry) {}
+
+	// Check that cluster name is available
+	ACTOR static Future<Void> registrationPrecheck(RegisterClusterImpl* self, Reference<typename DB::TransactionT> tr) {
+		state Optional<DataClusterMetadata> dataClusterMetadata = wait(tryGetClusterTransaction(tr, self->clusterName));
+		if (dataClusterMetadata.present()) {
+			throw cluster_already_exists();
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> configureDataCluster(RegisterClusterImpl* self) {
+		state Reference<IDatabase> dataClusterDb = wait(openDatabase(self->connectionString));
+		state Reference<ITransaction> tr = dataClusterDb->createTransaction();
+		loop {
+			try {
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+
+				state Future<std::vector<std::pair<TenantName, TenantMapEntry>>> existingTenantsFuture =
+				    TenantAPI::listTenantsTransaction(tr, ""_sr, "\xff\xff"_sr, 1);
+				state ThreadFuture<RangeResult> existingDataFuture = tr->getRange(normalKeys, 1);
+
+				// Check whether this cluster has already been registered
+				state Optional<MetaclusterRegistrationEntry> existingRegistration =
+				    wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
+				if (existingRegistration.present()) {
+					if (existingRegistration.get().clusterType != ClusterType::METACLUSTER_DATA ||
+					    existingRegistration.get().name != self->clusterName ||
+					    !existingRegistration.get().matches(self->ctx.metaclusterRegistration.get())) {
+						throw cluster_already_registered();
+					} else {
+						// We already successfully registered the cluster with these details, so there's nothing to do
+						self->clusterEntry.id = existingRegistration.get().id;
+						return Void();
+					}
+				}
+
+				// Check for any existing data
+				std::vector<std::pair<TenantName, TenantMapEntry>> existingTenants =
+				    wait(safeThreadFutureToFuture(existingTenantsFuture));
+				if (!existingTenants.empty()) {
+					TraceEvent(SevWarn, "CannotRegisterClusterWithTenants").detail("ClusterName", self->clusterName);
+					throw cluster_not_empty();
+				}
+
+				RangeResult existingData = wait(safeThreadFutureToFuture(existingDataFuture));
+				if (!existingData.empty()) {
+					TraceEvent(SevWarn, "CannotRegisterClusterWithData").detail("ClusterName", self->clusterName);
+					throw cluster_not_empty();
+				}
+
+				self->clusterEntry.id = deterministicRandom()->randomUniqueID();
+				MetaclusterMetadata::metaclusterRegistration().set(
+				    tr,
+				    self->ctx.metaclusterRegistration.get().toDataClusterRegistration(self->clusterName,
+				                                                                      self->clusterEntry.id));
+
+				wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
+
+				TraceEvent("ConfiguredDataCluster")
+				    .detail("ClusterName", self->clusterName)
+				    .detail("ClusterID", self->clusterEntry.id)
+				    .detail("Capacity", self->clusterEntry.capacity)
+				    .detail("Version", tr->getCommittedVersion())
+				    .detail("ConnectionString", self->connectionString.toString());
+
+				return Void();
+			} catch (Error& e) {
+				wait(safeThreadFutureToFuture(tr->onError(e)));
+			}
+		}
+	}
+
+	// Store the cluster entry for the new cluster
+	ACTOR static Future<Void> registerInManagementCluster(RegisterClusterImpl* self,
+	                                                      Reference<typename DB::TransactionT> tr) {
+		state Optional<DataClusterMetadata> dataClusterMetadata = wait(tryGetClusterTransaction(tr, self->clusterName));
+		if (dataClusterMetadata.present() && !dataClusterMetadata.get().matchesConfiguration(
+		                                         DataClusterMetadata(self->clusterEntry, self->connectionString))) {
+			throw cluster_already_exists();
+		} else if (!dataClusterMetadata.present()) {
+			self->clusterEntry.allocated = ClusterUsage();
+
+			if (self->clusterEntry.hasCapacity()) {
+				ManagementClusterMetadata::clusterCapacityIndex.insert(
+				    tr, Tuple::makeTuple(self->clusterEntry.allocated.numTenantGroups, self->clusterName));
+			}
+			ManagementClusterMetadata::dataClusters().set(tr, self->clusterName, self->clusterEntry);
+			ManagementClusterMetadata::dataClusterConnectionRecords.set(tr, self->clusterName, self->connectionString);
+		}
+
+		TraceEvent("RegisteredDataCluster")
+		    .detail("ClusterName", self->clusterName)
+		    .detail("ClusterID", self->clusterEntry.id)
+		    .detail("Capacity", self->clusterEntry.capacity)
+		    .detail("Version", tr->getCommittedVersion())
+		    .detail("ConnectionString", self->connectionString.toString());
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> run(RegisterClusterImpl* self) {
+		wait(self->ctx.runManagementTransaction(
+		    [self = self](Reference<typename DB::TransactionT> tr) { return registrationPrecheck(self, tr); }));
+		// Don't use ctx to run this transaction because we have not set up the data cluster metadata on it and we don't
+		// have a metacluster registration on the data cluster
+		wait(configureDataCluster(self));
+		wait(self->ctx.runManagementTransaction(
+		    [self = self](Reference<typename DB::TransactionT> tr) { return registerInManagementCluster(self, tr); }));
+		return Void();
+	}
+	Future<Void> run() { return run(this); }
+};
+
+ACTOR template <class DB>
+Future<Void> registerCluster(Reference<DB> db,
+                             ClusterName name,
+                             ClusterConnectionString connectionString,
+                             DataClusterEntry entry) {
+	state RegisterClusterImpl<DB> impl(db, name, connectionString, entry);
+	wait(impl.run());
+	return Void();
+}
+
+ACTOR template <class DB>
+Future<Void> restoreCluster(Reference<DB> db,
+                            ClusterName name,
+                            std::string connectionString,
+                            DataClusterEntry entry,
+                            AddNewTenants addNewTenants,
+                            RemoveMissingTenants removeMissingTenants) {
+	// TODO: add implementation
+	wait(delay(0.0));
+	return Void();
+}
+
+template <class DB>
+struct RemoveClusterImpl {
+	MetaclusterOperationContext<DB> ctx;
+
+	// Initialization parameters
+	bool forceRemove;
+
+	// Parameters set in markClusterRemoving
+	Optional<int64_t> lastTenantId;
+
+	RemoveClusterImpl(Reference<DB> managementDb, ClusterName clusterName, bool forceRemove)
+	  : ctx(managementDb, clusterName), forceRemove(forceRemove) {}
+
+	// Returns false if the cluster is no longer present, or true if it is present and the removal should proceed.
+	ACTOR static Future<bool> markClusterRemoving(RemoveClusterImpl* self, Reference<typename DB::TransactionT> tr) {
+		if (!self->forceRemove && self->ctx.dataClusterMetadata.get().entry.allocated.numTenantGroups > 0) {
+			throw cluster_not_empty();
+		} else if (self->ctx.dataClusterMetadata.get().entry.clusterState != DataClusterState::REMOVING) {
+			// Mark the cluster in a removing state while we finish the remaining removal steps. This prevents new
+			// tenants from being assigned to it.
+			DataClusterEntry updatedEntry = self->ctx.dataClusterMetadata.get().entry;
+			updatedEntry.clusterState = DataClusterState::REMOVING;
+			updatedEntry.capacity.numTenantGroups = 0;
+
+			updateClusterMetadata(tr,
+			                      self->ctx.clusterName.get(),
+			                      self->ctx.dataClusterMetadata.get(),
+			                      Optional<ClusterConnectionString>(),
+			                      updatedEntry);
+		}
+
+		ManagementClusterMetadata::clusterCapacityIndex.erase(
+		    tr,
+		    Tuple::makeTuple(self->ctx.dataClusterMetadata.get().entry.allocated.numTenantGroups,
+		                     self->ctx.clusterName.get()));
+
+		// Get the last allocated tenant ID to be used on the detached data cluster
+		if (self->forceRemove) {
+			Optional<int64_t> lastId = wait(ManagementClusterMetadata::tenantMetadata().lastTenantId.get(tr));
+			self->lastTenantId = lastId;
+		}
+
+		TraceEvent("MarkedDataClusterRemoving")
+		    .detail("Name", self->ctx.clusterName.get())
+		    .detail("Version", tr->getCommittedVersion());
+
+		return true;
+	}
+
+	// Delete metacluster metadata from the data cluster
+	ACTOR static Future<Void> updateDataCluster(RemoveClusterImpl* self, Reference<ITransaction> tr) {
+		// Delete metacluster related metadata
+		MetaclusterMetadata::metaclusterRegistration().clear(tr);
+		TenantMetadata::tenantTombstones().clear(tr);
+		TenantMetadata::tombstoneCleanupData().clear(tr);
+
+		// If we are force removing a cluster, then it will potentially contain tenants that have IDs
+		// larger than the next tenant ID to be allocated on the cluster. To avoid collisions, we advance
+		// the ID so that it will be the larger of the current one on the data cluster and the management
+		// cluster.
+		if (self->lastTenantId.present()) {
+			Optional<int64_t> lastId = wait(TenantMetadata::lastTenantId().get(tr));
+			if (!lastId.present() || lastId.get() < self->lastTenantId.get()) {
+				TenantMetadata::lastTenantId().set(tr, self->lastTenantId.get());
+			}
+		}
+
+		TraceEvent("ReconfiguredDataCluster")
+		    .detail("Name", self->ctx.clusterName.get())
+		    .detail("Version", tr->getCommittedVersion());
+
+		return Void();
+	}
+
+	// Returns true if all tenants have been purged
+	ACTOR static Future<bool> purgeTenants(RemoveClusterImpl* self,
+	                                       Reference<typename DB::TransactionT> tr,
+	                                       std::pair<Tuple, Tuple> clusterTupleRange) {
+		ASSERT(self->ctx.dataClusterMetadata.get().entry.clusterState == DataClusterState::REMOVING);
+
+		// Get the list of tenants
+		state Future<KeyBackedRangeResult<Tuple>> tenantEntriesFuture =
+		    ManagementClusterMetadata::clusterTenantIndex.getRange(
+		        tr, clusterTupleRange.first, clusterTupleRange.second, CLIENT_KNOBS->REMOVE_CLUSTER_TENANT_BATCH_SIZE);
+
+		state KeyBackedRangeResult<Tuple> tenantEntries = wait(tenantEntriesFuture);
+
+		// Erase each tenant from the tenant map on the management cluster
+		for (Tuple entry : tenantEntries.results) {
+			ASSERT(entry.getString(0) == self->ctx.clusterName.get());
+			ManagementClusterMetadata::tenantMetadata().tenantMap.erase(tr, entry.getString(1));
+			ManagementClusterMetadata::tenantMetadata().tenantIdIndex.erase(tr, entry.getInt(2));
+		}
+
+		// Erase all of the tenants processed in this transaction from the cluster tenant index
+		if (!tenantEntries.results.empty()) {
+			ManagementClusterMetadata::clusterTenantIndex.erase(
+			    tr,
+			    clusterTupleRange.first,
+			    Tuple::makeTuple(self->ctx.clusterName.get(), keyAfter(tenantEntries.results.rbegin()->getString(1))));
+		}
+
+		ManagementClusterMetadata::tenantMetadata().tenantCount.atomicOp(
+		    tr, -tenantEntries.results.size(), MutationRef::AddValue);
+		ManagementClusterMetadata::clusterTenantCount.atomicOp(
+		    tr, self->ctx.clusterName.get(), -tenantEntries.results.size(), MutationRef::AddValue);
+
+		return !tenantEntries.more;
+	}
+
+	// Returns true if all tenant groups and the data cluster have been purged
+	ACTOR static Future<bool> purgeTenantGroupsAndDataCluster(RemoveClusterImpl* self,
+	                                                          Reference<typename DB::TransactionT> tr,
+	                                                          std::pair<Tuple, Tuple> clusterTupleRange) {
+		ASSERT(self->ctx.dataClusterMetadata.get().entry.clusterState == DataClusterState::REMOVING);
+
+		// Get the list of tenant groups
+		state Future<KeyBackedRangeResult<Tuple>> tenantGroupEntriesFuture =
+		    ManagementClusterMetadata::clusterTenantGroupIndex.getRange(
+		        tr, clusterTupleRange.first, clusterTupleRange.second, CLIENT_KNOBS->REMOVE_CLUSTER_TENANT_BATCH_SIZE);
+
+		// Erase each tenant group from the tenant group map and the tenant group tenant index
+		state KeyBackedRangeResult<Tuple> tenantGroupEntries = wait(tenantGroupEntriesFuture);
+		for (Tuple entry : tenantGroupEntries.results) {
+			ASSERT(entry.getString(0) == self->ctx.clusterName.get());
+			TenantGroupName tenantGroup = entry.getString(1);
+			ManagementClusterMetadata::tenantMetadata().tenantGroupTenantIndex.erase(
+			    tr, Tuple::makeTuple(tenantGroup), Tuple::makeTuple(keyAfter(tenantGroup)));
+			ManagementClusterMetadata::tenantMetadata().tenantGroupMap.erase(tr, tenantGroup);
+		}
+
+		if (!tenantGroupEntries.results.empty()) {
+			// Erase all of the tenant groups processed in this transaction from the cluster tenant group index
+			ManagementClusterMetadata::clusterTenantGroupIndex.erase(
+			    tr,
+			    clusterTupleRange.first,
+			    Tuple::makeTuple(self->ctx.clusterName.get(),
+			                     keyAfter(tenantGroupEntries.results.rbegin()->getString(1))));
+		}
+
+		// Erase the data cluster record from the management cluster if processing our last batch
+		if (!tenantGroupEntries.more) {
+			ManagementClusterMetadata::dataClusters().erase(tr, self->ctx.clusterName.get());
+			ManagementClusterMetadata::dataClusterConnectionRecords.erase(tr, self->ctx.clusterName.get());
+			ManagementClusterMetadata::clusterTenantCount.erase(tr, self->ctx.clusterName.get());
+		}
+
+		return !tenantGroupEntries.more;
+	}
+
+	// Remove all metadata associated with the data cluster from the management cluster
+	ACTOR static Future<Void> managementClusterPurgeDataCluster(RemoveClusterImpl* self) {
+		state std::pair<Tuple, Tuple> clusterTupleRange = std::make_pair(
+		    Tuple::makeTuple(self->ctx.clusterName.get()), Tuple::makeTuple(keyAfter(self->ctx.clusterName.get())));
+
+		// First remove all tenants associated with the data cluster from the management cluster
+		loop {
+			bool clearedAll = wait(self->ctx.runManagementTransaction(
+			    [self = self, clusterTupleRange = clusterTupleRange](Reference<typename DB::TransactionT> tr) {
+				    return purgeTenants(self, tr, clusterTupleRange);
+			    }));
+
+			if (clearedAll) {
+				break;
+			}
+		}
+
+		// Next remove all tenant groups associated with the data cluster from the management cluster
+		loop {
+			bool clearedAll = wait(self->ctx.runManagementTransaction(
+			    [self = self, clusterTupleRange = clusterTupleRange](Reference<typename DB::TransactionT> tr) {
+				    return purgeTenantGroupsAndDataCluster(self, tr, clusterTupleRange);
+			    }));
+			if (clearedAll) {
+				break;
+			}
+		}
+
+		TraceEvent("RemovedDataCluster").detail("Name", self->ctx.clusterName.get());
+		return Void();
+	}
+
+	ACTOR static Future<Void> run(RemoveClusterImpl* self) {
+		state bool clusterIsPresent;
+		try {
+			wait(store(clusterIsPresent,
+			           self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
+				           return markClusterRemoving(self, tr);
+			           })));
+		} catch (Error& e) {
+			// If the transaction retries after success or if we are trying a second time to remove the cluster, it will
+			// throw an error indicating that the removal has already started
+			if (e.code() == error_code_cluster_removed) {
+				clusterIsPresent = true;
+			} else {
+				throw;
+			}
+		}
+
+		if (clusterIsPresent) {
+			try {
+				wait(self->ctx.runDataClusterTransaction(
+				    [self = self](Reference<ITransaction> tr) { return updateDataCluster(self, tr); }));
+			} catch (Error& e) {
+				// If this transaction gets retried, the metacluster information may have already been erased.
+				if (e.code() != error_code_invalid_metacluster_operation) {
+					throw;
+				}
+			}
+
+			// This runs multiple transactions, so the run transaction calls are inside the function
+			try {
+				wait(managementClusterPurgeDataCluster(self));
+			} catch (Error& e) {
+				// If this transaction gets retried, the cluster may have already been deleted.
+				if (e.code() != error_code_cluster_not_found) {
+					throw;
+				}
+			}
+		}
+
+		return Void();
+	}
+	Future<Void> run() { return run(this); }
+};
+
+ACTOR template <class DB>
+Future<Void> removeCluster(Reference<DB> db, ClusterName name, bool forceRemove) {
+	state RemoveClusterImpl<DB> impl(db, name, forceRemove);
+	wait(impl.run());
+	return Void();
+}
+
+ACTOR template <class Transaction>
+Future<std::map<ClusterName, DataClusterMetadata>> listClustersTransaction(Transaction tr,
+                                                                           ClusterNameRef begin,
+                                                                           ClusterNameRef end,
+                                                                           int limit) {
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+
+	state Future<Void> tenantModeCheck = TenantAPI::checkTenantMode(tr, ClusterType::METACLUSTER_MANAGEMENT);
+
+	state Future<KeyBackedRangeResult<std::pair<ClusterName, DataClusterEntry>>> clusterEntriesFuture =
+	    ManagementClusterMetadata::dataClusters().getRange(tr, begin, end, limit);
+	state Future<KeyBackedRangeResult<std::pair<ClusterName, ClusterConnectionString>>> connectionStringFuture =
+	    ManagementClusterMetadata::dataClusterConnectionRecords.getRange(tr, begin, end, limit);
+
+	wait(tenantModeCheck);
+
+	state KeyBackedRangeResult<std::pair<ClusterName, DataClusterEntry>> clusterEntries =
+	    wait(safeThreadFutureToFuture(clusterEntriesFuture));
+	KeyBackedRangeResult<std::pair<ClusterName, ClusterConnectionString>> connectionStrings =
+	    wait(safeThreadFutureToFuture(connectionStringFuture));
+
+	ASSERT(clusterEntries.results.size() == connectionStrings.results.size());
+
+	std::map<ClusterName, DataClusterMetadata> clusters;
+	for (int i = 0; i < clusterEntries.results.size(); ++i) {
+		ASSERT(clusterEntries.results[i].first == connectionStrings.results[i].first);
+		clusters[clusterEntries.results[i].first] =
+		    DataClusterMetadata(clusterEntries.results[i].second, connectionStrings.results[i].second);
+	}
+
+	return clusters;
+}
+
+ACTOR template <class DB>
+Future<std::map<ClusterName, DataClusterMetadata>> listClusters(Reference<DB> db,
+                                                                ClusterName begin,
+                                                                ClusterName end,
+                                                                int limit) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			std::map<ClusterName, DataClusterMetadata> clusters = wait(listClustersTransaction(tr, begin, end, limit));
+
+			return clusters;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+template <class Transaction>
+void managementClusterAddTenantToGroup(Transaction tr,
+                                       TenantName tenantName,
+                                       TenantMapEntry tenantEntry,
+                                       DataClusterMetadata* clusterMetadata,
+                                       bool groupAlreadyExists) {
+	if (tenantEntry.tenantGroup.present()) {
+		if (tenantEntry.tenantGroup.get().startsWith("\xff"_sr)) {
+			throw invalid_tenant_group_name();
+		}
+
+		if (!groupAlreadyExists) {
+			ManagementClusterMetadata::tenantMetadata().tenantGroupMap.set(
+			    tr, tenantEntry.tenantGroup.get(), TenantGroupEntry(tenantEntry.assignedCluster));
+			ManagementClusterMetadata::clusterTenantGroupIndex.insert(
+			    tr, Tuple::makeTuple(tenantEntry.assignedCluster.get(), tenantEntry.tenantGroup.get()));
+		}
+		ManagementClusterMetadata::tenantMetadata().tenantGroupTenantIndex.insert(
+		    tr, Tuple::makeTuple(tenantEntry.tenantGroup.get(), tenantName));
+	}
+
+	if (!groupAlreadyExists) {
+		ASSERT(clusterMetadata->entry.hasCapacity());
+
+		DataClusterEntry updatedEntry = clusterMetadata->entry;
+		++updatedEntry.allocated.numTenantGroups;
+
+		updateClusterMetadata(
+		    tr, tenantEntry.assignedCluster.get(), *clusterMetadata, Optional<ClusterConnectionString>(), updatedEntry);
+
+		clusterMetadata->entry = updatedEntry;
+	}
+}
+
+ACTOR template <class Transaction>
+Future<Void> managementClusterRemoveTenantFromGroup(Transaction tr,
+                                                    TenantName tenantName,
+                                                    TenantMapEntry tenantEntry,
+                                                    DataClusterMetadata* clusterMetadata,
+                                                    bool isRenamePair = false) {
+	state bool updateClusterCapacity = !tenantEntry.tenantGroup.present() && !isRenamePair;
+	if (tenantEntry.tenantGroup.present()) {
+		ManagementClusterMetadata::tenantMetadata().tenantGroupTenantIndex.erase(
+		    tr, Tuple::makeTuple(tenantEntry.tenantGroup.get(), tenantName));
+
+		state KeyBackedSet<Tuple>::RangeResultType result =
+		    wait(ManagementClusterMetadata::tenantMetadata().tenantGroupTenantIndex.getRange(
+		        tr,
+		        Tuple::makeTuple(tenantEntry.tenantGroup.get()),
+		        Tuple::makeTuple(keyAfter(tenantEntry.tenantGroup.get())),
+		        1));
+
+		if (result.results.size() == 0) {
+			ManagementClusterMetadata::clusterTenantGroupIndex.erase(
+			    tr, Tuple::makeTuple(tenantEntry.assignedCluster.get(), tenantEntry.tenantGroup.get()));
+
+			ManagementClusterMetadata::tenantMetadata().tenantGroupMap.erase(tr, tenantEntry.tenantGroup.get());
+			updateClusterCapacity = true;
+		}
+	}
+
+	// Update the tenant group count information for the assigned cluster if this tenant group was erased so we
+	// can use the freed capacity.
+	if (updateClusterCapacity) {
+		DataClusterEntry updatedEntry = clusterMetadata->entry;
+		--updatedEntry.allocated.numTenantGroups;
+		updateClusterMetadata(
+		    tr, tenantEntry.assignedCluster.get(), *clusterMetadata, Optional<ClusterConnectionString>(), updatedEntry);
+
+		clusterMetadata->entry = updatedEntry;
+	}
+
+	return Void();
+}
+
+template <class DB>
+struct CreateTenantImpl {
+	MetaclusterOperationContext<DB> ctx;
+	bool preferAssignedCluster;
+
+	// Initialization parameters
+	TenantName tenantName;
+	TenantMapEntry tenantEntry;
+
+	// Parameter set if tenant creation permanently fails on the data cluster
+	Optional<int64_t> replaceExistingTenantId;
+
+	CreateTenantImpl(Reference<DB> managementDb,
+	                 bool preferAssignedCluster,
+	                 TenantName tenantName,
+	                 TenantMapEntry tenantEntry)
+	  : ctx(managementDb), preferAssignedCluster(preferAssignedCluster), tenantName(tenantName),
+	    tenantEntry(tenantEntry) {}
+
+	ACTOR static Future<ClusterName> checkClusterAvailability(Reference<IDatabase> dataClusterDb,
+	                                                          ClusterName clusterName) {
+		state Reference<ITransaction> tr = dataClusterDb->createTransaction();
+		loop {
+			try {
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->addWriteConflictRange(KeyRangeRef("\xff/metacluster/availability_check"_sr,
+				                                      "\xff/metacluster/availability_check\x00"_sr));
+				wait(safeThreadFutureToFuture(tr->commit()));
+				return clusterName;
+			} catch (Error& e) {
+				wait(safeThreadFutureToFuture(tr->onError(e)));
+			}
+		}
+	}
+
+	// Returns true if the tenant is already assigned and can proceed to the next step and false if it needs
+	// to be created. Throws an error if the tenant already exists and cannot be created.
+	ACTOR static Future<bool> checkForExistingTenant(CreateTenantImpl* self, Reference<typename DB::TransactionT> tr) {
+		// Check if the tenant already exists. If it's partially created and matches the parameters we
+		// specified, continue creating it. Otherwise, fail with an error.
+		state Optional<TenantMapEntry> existingEntry = wait(tryGetTenantTransaction(tr, self->tenantName));
+		if (existingEntry.present()) {
+			if (!existingEntry.get().matchesConfiguration(self->tenantEntry) ||
+			    existingEntry.get().tenantState != TenantState::REGISTERING) {
+				// The tenant already exists and is either completely created or has a different
+				// configuration
+				throw tenant_already_exists();
+			} else if (!self->replaceExistingTenantId.present() ||
+			           self->replaceExistingTenantId.get() != existingEntry.get().id) {
+				// The tenant creation has already started, so resume where we left off
+				ASSERT(existingEntry.get().assignedCluster.present());
+				if (self->preferAssignedCluster &&
+				    existingEntry.get().assignedCluster.get() != self->tenantEntry.assignedCluster.get()) {
+					TraceEvent("MetaclusterCreateTenantClusterMismatch")
+					    .detail("Preferred", self->tenantEntry.assignedCluster.get())
+					    .detail("Actual", existingEntry.get().assignedCluster.get());
+					throw invalid_tenant_configuration();
+				}
+				self->tenantEntry = existingEntry.get();
+				wait(self->ctx.setCluster(tr, existingEntry.get().assignedCluster.get()));
+				return true;
+			} else {
+				// The previous creation is permanently failed, so cleanup the tenant and create it again from scratch
+				// We don't need to remove it from the tenant map because we will overwrite the existing entry later in
+				// this transaction.
+				ManagementClusterMetadata::tenantMetadata().tenantIdIndex.erase(tr, existingEntry.get().id);
+				ManagementClusterMetadata::tenantMetadata().tenantCount.atomicOp(tr, -1, MutationRef::AddValue);
+				ManagementClusterMetadata::clusterTenantCount.atomicOp(
+				    tr, existingEntry.get().assignedCluster.get(), -1, MutationRef::AddValue);
+
+				ManagementClusterMetadata::clusterTenantIndex.erase(
+				    tr,
+				    Tuple::makeTuple(
+				        existingEntry.get().assignedCluster.get(), self->tenantName, existingEntry.get().id));
+
+				state DataClusterMetadata previousAssignedClusterMetadata =
+				    wait(getClusterTransaction(tr, existingEntry.get().assignedCluster.get()));
+
+				wait(managementClusterRemoveTenantFromGroup(
+				    tr, self->tenantName, existingEntry.get(), &previousAssignedClusterMetadata));
+			}
+		} else if (self->replaceExistingTenantId.present()) {
+			throw tenant_removed();
+		}
+
+		return false;
+	}
+
+	// Returns a pair with the name of the assigned cluster and whether the group was already assigned
+	ACTOR static Future<std::pair<ClusterName, bool>> assignTenant(CreateTenantImpl* self,
+	                                                               Reference<typename DB::TransactionT> tr) {
+		// If our tenant group is already assigned, then we just use that assignment
+		state Optional<TenantGroupEntry> groupEntry;
+		if (self->tenantEntry.tenantGroup.present()) {
+			Optional<TenantGroupEntry> _groupEntry =
+			    wait(ManagementClusterMetadata::tenantMetadata().tenantGroupMap.get(
+			        tr, self->tenantEntry.tenantGroup.get()));
+			groupEntry = _groupEntry;
+
+			if (groupEntry.present()) {
+				ASSERT(groupEntry.get().assignedCluster.present());
+				if (self->preferAssignedCluster &&
+				    groupEntry.get().assignedCluster.get() != self->tenantEntry.assignedCluster.get()) {
+					TraceEvent("MetaclusterCreateTenantGroupClusterMismatch")
+					    .detail("TenantGroupCluster", groupEntry.get().assignedCluster.get())
+					    .detail("SpecifiedCluster", self->tenantEntry.assignedCluster.get());
+					throw invalid_tenant_configuration();
+				}
+				return std::make_pair(groupEntry.get().assignedCluster.get(), true);
+			}
+		}
+
+		state std::vector<Future<Reference<IDatabase>>> dataClusterDbs;
+		state std::vector<ClusterName> dataClusterNames;
+		state std::vector<Future<ClusterName>> clusterAvailabilityChecks;
+		// Get a set of the most full clusters that still have capacity
+		// If preferred cluster is specified, look for that one.
+		if (self->preferAssignedCluster) {
+			DataClusterMetadata dataClusterMetadata =
+			    wait(getClusterTransaction(tr, self->tenantEntry.assignedCluster.get()));
+			if (!dataClusterMetadata.entry.hasCapacity()) {
+				throw cluster_no_capacity();
+			}
+			dataClusterNames.push_back(self->tenantEntry.assignedCluster.get());
+		} else {
+			state KeyBackedSet<Tuple>::RangeResultType availableClusters =
+			    wait(ManagementClusterMetadata::clusterCapacityIndex.getRange(
+			        tr,
+			        {},
+			        {},
+			        CLIENT_KNOBS->METACLUSTER_ASSIGNMENT_CLUSTERS_TO_CHECK,
+			        Snapshot::False,
+			        Reverse::True));
+			if (availableClusters.results.empty()) {
+				throw metacluster_no_capacity();
+			}
+			for (auto clusterTuple : availableClusters.results) {
+				dataClusterNames.push_back(clusterTuple.getString(1));
+			}
+		}
+		for (auto dataClusterName : dataClusterNames) {
+			dataClusterDbs.push_back(getAndOpenDatabase(tr, dataClusterName));
+		}
+		wait(waitForAll(dataClusterDbs));
+		// Check the availability of our set of clusters
+		for (int i = 0; i < dataClusterDbs.size(); ++i) {
+			clusterAvailabilityChecks.push_back(checkClusterAvailability(dataClusterDbs[i].get(), dataClusterNames[i]));
+		}
+
+		// Wait for a successful availability check from some cluster. We prefer the most full cluster, but if it
+		// doesn't return quickly we may choose another.
+		Optional<Void> clusterAvailabilityCheck = wait(timeout(
+		    success(clusterAvailabilityChecks[0]) || (delay(CLIENT_KNOBS->METACLUSTER_ASSIGNMENT_FIRST_CHOICE_DELAY) &&
+		                                              waitForAny(clusterAvailabilityChecks)),
+		    CLIENT_KNOBS->METACLUSTER_ASSIGNMENT_AVAILABILITY_TIMEOUT));
+
+		if (!clusterAvailabilityCheck.present()) {
+			// If no clusters were available for long enough, then we throw an error and try again
+			throw transaction_too_old();
+		}
+
+		// Get the first cluster that was available
+		state Optional<ClusterName> chosenCluster;
+		for (auto f : clusterAvailabilityChecks) {
+			if (f.isReady()) {
+				chosenCluster = f.get();
+				break;
+			}
+		}
+
+		ASSERT(chosenCluster.present());
+		return std::make_pair(chosenCluster.get(), false);
+	}
+
+	ACTOR static Future<Void> assignTenantAndStoreInManagementCluster(CreateTenantImpl* self,
+	                                                                  Reference<typename DB::TransactionT> tr) {
+		// If the tenant already exists, we either throw an error from this function or move on to the next phase
+		bool tenantExists = wait(checkForExistingTenant(self, tr));
+		if (tenantExists) {
+			return Void();
+		}
+
+		// Choose a cluster for the tenant
+		state std::pair<ClusterName, bool> assignment = wait(assignTenant(self, tr));
+		self->tenantEntry.assignedCluster = assignment.first;
+
+		// Update the context with the chosen cluster
+		state Future<Void> setClusterFuture = self->ctx.setCluster(tr, assignment.first);
+
+		// Create a tenant entry in the management cluster
+		Optional<int64_t> lastId = wait(ManagementClusterMetadata::tenantMetadata().lastTenantId.get(tr));
+		self->tenantEntry.setId(lastId.orDefault(-1) + 1);
+		ManagementClusterMetadata::tenantMetadata().lastTenantId.set(tr, self->tenantEntry.id);
+
+		self->tenantEntry.tenantState = TenantState::REGISTERING;
+		ManagementClusterMetadata::tenantMetadata().tenantMap.set(tr, self->tenantName, self->tenantEntry);
+		ManagementClusterMetadata::tenantMetadata().tenantIdIndex.set(tr, self->tenantEntry.id, self->tenantName);
+
+		ManagementClusterMetadata::tenantMetadata().tenantCount.atomicOp(tr, 1, MutationRef::AddValue);
+		ManagementClusterMetadata::clusterTenantCount.atomicOp(
+		    tr, self->tenantEntry.assignedCluster.get(), 1, MutationRef::AddValue);
+
+		int64_t clusterTenantCount = wait(ManagementClusterMetadata::clusterTenantCount.getD(
+		    tr, self->tenantEntry.assignedCluster.get(), Snapshot::False, 0));
+
+		if (clusterTenantCount > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER) {
+			throw cluster_no_capacity();
+		}
+
+		// Updated indexes to include the new tenant
+		ManagementClusterMetadata::clusterTenantIndex.insert(
+		    tr, Tuple::makeTuple(self->tenantEntry.assignedCluster.get(), self->tenantName, self->tenantEntry.id));
+
+		wait(setClusterFuture);
+
+		// If we are part of a tenant group that is assigned to a cluster being removed from the metacluster,
+		// then we fail with an error.
+		if (self->ctx.dataClusterMetadata.get().entry.clusterState == DataClusterState::REMOVING) {
+			throw cluster_removed();
+		}
+
+		managementClusterAddTenantToGroup(
+		    tr, self->tenantName, self->tenantEntry, &self->ctx.dataClusterMetadata.get(), assignment.second);
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> storeTenantInDataCluster(CreateTenantImpl* self, Reference<ITransaction> tr) {
+		std::pair<Optional<TenantMapEntry>, bool> dataClusterTenant = wait(
+		    TenantAPI::createTenantTransaction(tr, self->tenantName, self->tenantEntry, ClusterType::METACLUSTER_DATA));
+
+		// If the tenant map entry is empty, then we encountered a tombstone indicating that the tenant was
+		// simultaneously removed.
+		if (!dataClusterTenant.first.present()) {
+			throw tenant_removed();
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> markTenantReady(CreateTenantImpl* self, Reference<typename DB::TransactionT> tr) {
+		state Optional<TenantMapEntry> managementEntry = wait(tryGetTenantTransaction(tr, self->tenantName));
+		if (!managementEntry.present()) {
+			throw tenant_removed();
+		} else if (managementEntry.get().id != self->tenantEntry.id) {
+			throw tenant_already_exists();
+		}
+
+		if (managementEntry.get().tenantState == TenantState::REGISTERING) {
+			TenantMapEntry updatedEntry = managementEntry.get();
+			updatedEntry.tenantState = TenantState::READY;
+			ManagementClusterMetadata::tenantMetadata().tenantMap.set(tr, self->tenantName, updatedEntry);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> run(CreateTenantImpl* self) {
+		if (self->tenantName.startsWith("\xff"_sr)) {
+			throw invalid_tenant_name();
+		}
+
+		loop {
+			wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
+				return assignTenantAndStoreInManagementCluster(self, tr);
+			}));
+
+			self->replaceExistingTenantId = {};
+			try {
+				wait(self->ctx.runDataClusterTransaction(
+				    [self = self](Reference<ITransaction> tr) { return storeTenantInDataCluster(self, tr); }));
+
+				wait(self->ctx.runManagementTransaction(
+				    [self = self](Reference<typename DB::TransactionT> tr) { return markTenantReady(self, tr); }));
+
+				return Void();
+			} catch (Error& e) {
+				if (e.code() == error_code_tenant_creation_permanently_failed) {
+					// If the data cluster has permanently failed to create the tenant, then we can reassign it in
+					// the management cluster and start over
+					self->replaceExistingTenantId = self->tenantEntry.id;
+					self->ctx.clearCluster();
+				} else {
+					throw;
+				}
+			}
+		}
+	}
+	Future<Void> run() { return run(this); }
+};
+
+ACTOR template <class DB>
+Future<Void> createTenant(Reference<DB> db, TenantName name, TenantMapEntry tenantEntry) {
+	state CreateTenantImpl<DB> impl(db, tenantEntry.assignedCluster.present(), name, tenantEntry);
+	wait(impl.run());
+	return Void();
+}
+
+template <class DB>
+struct DeleteTenantImpl {
+	MetaclusterOperationContext<DB> ctx;
+
+	// Initialization parameters
+	TenantName tenantName;
+
+	// Parameters set in getAssignedLocation
+	int64_t tenantId;
+
+	// Parameters set in markTenantInRemovingState
+	Optional<TenantName> pairName;
+
+	DeleteTenantImpl(Reference<DB> managementDb, TenantName tenantName) : ctx(managementDb), tenantName(tenantName) {}
+
+	// Loads the cluster details for the cluster where the tenant is assigned.
+	// Returns true if the deletion is already in progress
+	ACTOR static Future<bool> getAssignedLocation(DeleteTenantImpl* self, Reference<typename DB::TransactionT> tr) {
+		state Optional<TenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, self->tenantName));
+
+		if (!tenantEntry.present()) {
+			throw tenant_not_found();
+		}
+
+		// Disallow removing the "new" name of a renamed tenant before it completes
+		if (tenantEntry.get().tenantState == TenantState::RENAMING_TO) {
+			throw tenant_not_found();
+		}
+
+		if (tenantEntry.get().tenantState == TenantState::REMOVING) {
+			if (tenantEntry.get().renamePair.present()) {
+				self->pairName = tenantEntry.get().renamePair.get();
+			}
+		}
+
+		self->tenantId = tenantEntry.get().id;
+		wait(self->ctx.setCluster(tr, tenantEntry.get().assignedCluster.get()));
+		return tenantEntry.get().tenantState == TenantState::REMOVING;
+	}
+
+	// Does an initial check if the tenant is empty. This is an optimization to prevent us marking a tenant
+	// in the deleted state while it has data, but it is still possible that data gets added to it after this
+	// point.
+	//
+	// SOMEDAY: should this also lock the tenant when locking is supported?
+	ACTOR static Future<Void> checkTenantEmpty(DeleteTenantImpl* self, Reference<ITransaction> tr) {
+		state Optional<TenantMapEntry> tenantEntry = wait(TenantAPI::tryGetTenantTransaction(tr, self->tenantName));
+		if (!tenantEntry.present() || tenantEntry.get().id != self->tenantId) {
+			// The tenant must have been removed simultaneously
+			return Void();
+		}
+
+		ThreadFuture<RangeResult> rangeFuture = tr->getRange(prefixRange(tenantEntry.get().prefix), 1);
+		RangeResult result = wait(safeThreadFutureToFuture(rangeFuture));
+		if (!result.empty()) {
+			throw tenant_not_empty();
+		}
+
+		return Void();
+	}
+
+	// Mark the tenant as being in a removing state on the management cluster
+	ACTOR static Future<Void> markTenantInRemovingState(DeleteTenantImpl* self,
+	                                                    Reference<typename DB::TransactionT> tr) {
+		state Optional<TenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, self->tenantName));
+
+		if (!tenantEntry.present() || tenantEntry.get().id != self->tenantId) {
+			throw tenant_not_found();
+		}
+
+		if (tenantEntry.get().tenantState != TenantState::REMOVING) {
+			// Disallow removing the "new" name of a renamed tenant before it completes
+			if (tenantEntry.get().tenantState == TenantState::RENAMING_TO) {
+				throw tenant_not_found();
+			}
+			state TenantMapEntry updatedEntry = tenantEntry.get();
+			// Check if we are deleting a tenant in the middle of a rename
+			if (updatedEntry.renamePair.present()) {
+				ASSERT(updatedEntry.tenantState == TenantState::RENAMING_FROM);
+				self->pairName = updatedEntry.renamePair.get();
+			}
+			updatedEntry.tenantState = TenantState::REMOVING;
+			ManagementClusterMetadata::tenantMetadata().tenantMap.set(tr, self->tenantName, updatedEntry);
+			// If this has a rename pair, also mark the other entry for deletion
+			if (self->pairName.present()) {
+				state Optional<TenantMapEntry> pairEntry = wait(tryGetTenantTransaction(tr, self->pairName.get()));
+				TenantMapEntry updatedPairEntry = pairEntry.get();
+				// Sanity check that our pair has us named as their partner
+				ASSERT(updatedPairEntry.renamePair.present());
+				ASSERT(updatedPairEntry.renamePair.get() == self->tenantName);
+				ASSERT(updatedPairEntry.id == self->tenantId);
+				CODE_PROBE(true, "marking pair tenant in removing state");
+				updatedPairEntry.tenantState = TenantState::REMOVING;
+				ManagementClusterMetadata::tenantMetadata().tenantMap.set(tr, self->pairName.get(), updatedPairEntry);
+			}
+		}
+
+		return Void();
+	}
+
+	// Delete the tenant and related metadata on the management cluster
+	ACTOR static Future<Void> deleteTenantFromManagementCluster(DeleteTenantImpl* self,
+	                                                            Reference<typename DB::TransactionT> tr,
+	                                                            bool pairDelete = false) {
+		// If pair is present, and this is not already a pair delete, call this function recursively
+		state Future<Void> pairFuture = Void();
+		if (!pairDelete && self->pairName.present()) {
+			CODE_PROBE(true, "deleting pair tenant from management cluster");
+			pairFuture = deleteTenantFromManagementCluster(self, tr, true);
+		}
+		state TenantName tenantName = pairDelete ? self->pairName.get() : self->tenantName;
+		state Optional<TenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, tenantName));
+
+		if (!tenantEntry.present() || tenantEntry.get().id != self->tenantId) {
+			return Void();
+		}
+
+		ASSERT(tenantEntry.get().tenantState == TenantState::REMOVING);
+
+		// Erase the tenant entry itself
+		ManagementClusterMetadata::tenantMetadata().tenantMap.erase(tr, tenantName);
+		ManagementClusterMetadata::tenantMetadata().tenantIdIndex.erase(tr, tenantEntry.get().id);
+
+		// This is idempotent because this function is only called if the tenant is in the map
+		ManagementClusterMetadata::tenantMetadata().tenantCount.atomicOp(tr, -1, MutationRef::AddValue);
+		ManagementClusterMetadata::clusterTenantCount.atomicOp(
+		    tr, tenantEntry.get().assignedCluster.get(), -1, MutationRef::AddValue);
+
+		// Remove the tenant from the cluster -> tenant index
+		ManagementClusterMetadata::clusterTenantIndex.erase(
+		    tr, Tuple::makeTuple(tenantEntry.get().assignedCluster.get(), tenantName, self->tenantId));
+
+		// Remove the tenant from its tenant group
+		wait(managementClusterRemoveTenantFromGroup(
+		    tr, tenantName, tenantEntry.get(), &self->ctx.dataClusterMetadata.get(), pairDelete));
+
+		wait(pairFuture);
+		return Void();
+	}
+
+	ACTOR static Future<Void> run(DeleteTenantImpl* self) {
+		// Get information about the tenant and where it is assigned
+		bool deletionInProgress = wait(self->ctx.runManagementTransaction(
+		    [self = self](Reference<typename DB::TransactionT> tr) { return getAssignedLocation(self, tr); }));
+
+		if (!deletionInProgress) {
+			wait(self->ctx.runDataClusterTransaction(
+			    [self = self](Reference<ITransaction> tr) { return checkTenantEmpty(self, tr); }));
+
+			wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
+				return markTenantInRemovingState(self, tr);
+			}));
+		}
+
+		// Delete tenant on the data cluster
+		wait(self->ctx.runDataClusterTransaction([self = self](Reference<ITransaction> tr) {
+			// If the removed tenant is being renamed, attempt to delete both the old and new names.
+			// At most one should be present with the given ID, and the other will be a no-op.
+			Future<Void> pairDelete = Void();
+			if (self->pairName.present()) {
+				CODE_PROBE(true, "deleting pair tenant from data cluster");
+				pairDelete = TenantAPI::deleteTenantTransaction(
+				    tr, self->pairName.get(), self->tenantId, ClusterType::METACLUSTER_DATA);
+			}
+			return pairDelete && TenantAPI::deleteTenantTransaction(
+			                         tr, self->tenantName, self->tenantId, ClusterType::METACLUSTER_DATA);
+		}));
+		wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
+			return deleteTenantFromManagementCluster(self, tr);
+		}));
+
+		return Void();
+	}
+	Future<Void> run() { return run(this); }
+};
+
+ACTOR template <class DB>
+Future<Void> deleteTenant(Reference<DB> db, TenantName name) {
+	state DeleteTenantImpl<DB> impl(db, name);
+	wait(impl.run());
+	return Void();
+}
+
+ACTOR template <class Transaction>
+Future<std::vector<std::pair<TenantName, TenantMapEntry>>> listTenantsTransaction(Transaction tr,
+                                                                                  TenantNameRef begin,
+                                                                                  TenantNameRef end,
+                                                                                  int limit) {
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+
+	KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> results =
+	    wait(ManagementClusterMetadata::tenantMetadata().tenantMap.getRange(tr, begin, end, limit));
+
+	return results.results;
+}
+
+ACTOR template <class DB>
+Future<std::vector<std::pair<TenantName, TenantMapEntry>>> listTenants(Reference<DB> db,
+                                                                       TenantName begin,
+                                                                       TenantName end,
+                                                                       int limit) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			std::vector<std::pair<TenantName, TenantMapEntry>> tenants =
+			    wait(listTenantsTransaction(tr, begin, end, limit));
+			return tenants;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+template <class DB>
+struct ConfigureTenantImpl {
+	MetaclusterOperationContext<DB> ctx;
+
+	// Initialization parameters
+	TenantName tenantName;
+	std::map<Standalone<StringRef>, Optional<Value>> configurationParameters;
+
+	// Parameters set in updateManagementCluster
+	TenantMapEntry updatedEntry;
+
+	ConfigureTenantImpl(Reference<DB> managementDb,
+	                    TenantName tenantName,
+	                    std::map<Standalone<StringRef>, Optional<Value>> configurationParameters)
+	  : ctx(managementDb), tenantName(tenantName), configurationParameters(configurationParameters) {}
+
+	// This verifies that the tenant group can be changed, and if so it updates all of the tenant group data
+	// structures. It does not update the TenantMapEntry stored in the tenant map.
+	ACTOR static Future<Void> updateTenantGroup(ConfigureTenantImpl* self,
+	                                            Reference<typename DB::TransactionT> tr,
+	                                            TenantMapEntry tenantEntry,
+	                                            Optional<TenantGroupName> desiredGroup) {
+
+		state TenantMapEntry entryWithUpdatedGroup = tenantEntry;
+		entryWithUpdatedGroup.tenantGroup = desiredGroup;
+
+		if (tenantEntry.tenantGroup == desiredGroup) {
+			return Void();
+		}
+
+		// Removing a tenant group is only possible if we have capacity for more groups on the current cluster
+		else if (!desiredGroup.present()) {
+			if (!self->ctx.dataClusterMetadata.get().entry.hasCapacity()) {
+				throw cluster_no_capacity();
+			}
+
+			wait(managementClusterRemoveTenantFromGroup(
+			    tr, self->tenantName, tenantEntry, &self->ctx.dataClusterMetadata.get()));
+			managementClusterAddTenantToGroup(
+			    tr, self->tenantName, entryWithUpdatedGroup, &self->ctx.dataClusterMetadata.get(), false);
+			return Void();
+		}
+
+		state Optional<TenantGroupEntry> tenantGroupEntry =
+		    wait(ManagementClusterMetadata::tenantMetadata().tenantGroupMap.get(tr, desiredGroup.get()));
+
+		// If we are creating a new tenant group, we need to have capacity on the current cluster
+		if (!tenantGroupEntry.present()) {
+			if (!self->ctx.dataClusterMetadata.get().entry.hasCapacity()) {
+				throw cluster_no_capacity();
+			}
+			wait(managementClusterRemoveTenantFromGroup(
+			    tr, self->tenantName, tenantEntry, &self->ctx.dataClusterMetadata.get()));
+			managementClusterAddTenantToGroup(
+			    tr, self->tenantName, entryWithUpdatedGroup, &self->ctx.dataClusterMetadata.get(), false);
+			return Void();
+		}
+
+		// Moves between groups in the same cluster are freely allowed
+		else if (tenantGroupEntry.get().assignedCluster == tenantEntry.assignedCluster) {
+			wait(managementClusterRemoveTenantFromGroup(
+			    tr, self->tenantName, tenantEntry, &self->ctx.dataClusterMetadata.get()));
+			managementClusterAddTenantToGroup(
+			    tr, self->tenantName, entryWithUpdatedGroup, &self->ctx.dataClusterMetadata.get(), true);
+			return Void();
+		}
+
+		// We don't currently support movement between groups on different clusters
+		else {
+			TraceEvent("TenantGroupChangeToDifferentCluster")
+			    .detail("Tenant", self->tenantName)
+			    .detail("OriginalGroup", tenantEntry.tenantGroup)
+			    .detail("DesiredGroup", desiredGroup)
+			    .detail("TenantAssignedCluster", tenantEntry.assignedCluster)
+			    .detail("DesiredGroupAssignedCluster", tenantGroupEntry.get().assignedCluster);
+
+			throw invalid_tenant_configuration();
+		}
+	}
+
+	// Updates the configuration in the management cluster and marks it as being in the UPDATING_CONFIGURATION state
+	ACTOR static Future<Void> updateManagementCluster(ConfigureTenantImpl* self,
+	                                                  Reference<typename DB::TransactionT> tr) {
+		state Optional<TenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, self->tenantName));
+
+		if (!tenantEntry.present()) {
+			throw tenant_not_found();
+		}
+
+		if (tenantEntry.get().tenantState != TenantState::READY &&
+		    tenantEntry.get().tenantState != TenantState::UPDATING_CONFIGURATION) {
+			throw invalid_tenant_state();
+		}
+
+		wait(self->ctx.setCluster(tr, tenantEntry.get().assignedCluster.get()));
+
+		self->updatedEntry = tenantEntry.get();
+		self->updatedEntry.tenantState = TenantState::UPDATING_CONFIGURATION;
+
+		state std::map<Standalone<StringRef>, Optional<Value>>::iterator configItr;
+		for (configItr = self->configurationParameters.begin(); configItr != self->configurationParameters.end();
+		     ++configItr) {
+			if (configItr->first == "tenant_group"_sr) {
+				wait(updateTenantGroup(self, tr, self->updatedEntry, configItr->second));
+			}
+			self->updatedEntry.configure(configItr->first, configItr->second);
+		}
+
+		++self->updatedEntry.configurationSequenceNum;
+		ManagementClusterMetadata::tenantMetadata().tenantMap.set(tr, self->tenantName, self->updatedEntry);
+
+		return Void();
+	}
+
+	// Updates the configuration in the data cluster
+	ACTOR static Future<Void> updateDataCluster(ConfigureTenantImpl* self, Reference<ITransaction> tr) {
+		state Optional<TenantMapEntry> tenantEntry = wait(TenantAPI::tryGetTenantTransaction(tr, self->tenantName));
+
+		if (!tenantEntry.present() || tenantEntry.get().id != self->updatedEntry.id ||
+		    tenantEntry.get().configurationSequenceNum >= self->updatedEntry.configurationSequenceNum) {
+			// If the tenant isn't in the metacluster, it must have been concurrently removed
+			return Void();
+		}
+
+		TenantMapEntry dataClusterEntry = self->updatedEntry;
+		dataClusterEntry.tenantState = TenantState::READY;
+		dataClusterEntry.assignedCluster = {};
+
+		wait(TenantAPI::configureTenantTransaction(tr, self->tenantName, tenantEntry.get(), dataClusterEntry));
+		return Void();
+	}
+
+	// Updates the tenant state in the management cluster to READY
+	ACTOR static Future<Void> markManagementTenantAsReady(ConfigureTenantImpl* self,
+	                                                      Reference<typename DB::TransactionT> tr) {
+		state Optional<TenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, self->tenantName));
+
+		if (!tenantEntry.present() || tenantEntry.get().id != self->updatedEntry.id ||
+		    tenantEntry.get().tenantState != TenantState::UPDATING_CONFIGURATION ||
+		    tenantEntry.get().configurationSequenceNum > self->updatedEntry.configurationSequenceNum) {
+			return Void();
+		}
+
+		tenantEntry.get().tenantState = TenantState::READY;
+		ManagementClusterMetadata::tenantMetadata().tenantMap.set(tr, self->tenantName, tenantEntry.get());
+		return Void();
+	}
+
+	ACTOR static Future<Void> run(ConfigureTenantImpl* self) {
+		wait(self->ctx.runManagementTransaction(
+		    [self = self](Reference<typename DB::TransactionT> tr) { return updateManagementCluster(self, tr); }));
+		wait(self->ctx.runDataClusterTransaction(
+		    [self = self](Reference<ITransaction> tr) { return updateDataCluster(self, tr); }));
+		wait(self->ctx.runManagementTransaction(
+		    [self = self](Reference<typename DB::TransactionT> tr) { return markManagementTenantAsReady(self, tr); }));
+
+		return Void();
+	}
+	Future<Void> run() { return run(this); }
+};
+
+ACTOR template <class DB>
+Future<Void> configureTenant(Reference<DB> db,
+                             TenantName name,
+                             std::map<Standalone<StringRef>, Optional<Value>> configurationParameters) {
+	state ConfigureTenantImpl<DB> impl(db, name, configurationParameters);
+	wait(impl.run());
+	return Void();
+}
+
+template <class DB>
+struct RenameTenantImpl {
+	MetaclusterOperationContext<DB> ctx;
+
+	// Initialization parameters
+	TenantName oldName;
+	TenantName newName;
+
+	// Parameters set in markTenantsInRenamingState
+	int64_t tenantId = -1;
+	int64_t configurationSequenceNum = -1;
+
+	RenameTenantImpl(Reference<DB> managementDb, TenantName oldName, TenantName newName)
+	  : ctx(managementDb), oldName(oldName), newName(newName) {}
+
+	// Delete the tenant and related metadata on the management cluster
+	ACTOR static Future<Void> deleteTenantFromManagementCluster(RenameTenantImpl* self,
+	                                                            Reference<typename DB::TransactionT> tr,
+	                                                            TenantMapEntry tenantEntry) {
+		// Erase the tenant entry itself
+		ManagementClusterMetadata::tenantMetadata().tenantMap.erase(tr, self->oldName);
+
+		// Remove old tenant from tenant count
+		ManagementClusterMetadata::tenantMetadata().tenantCount.atomicOp(tr, -1, MutationRef::AddValue);
+		ManagementClusterMetadata::clusterTenantCount.atomicOp(
+		    tr, tenantEntry.assignedCluster.get(), -1, MutationRef::AddValue);
+
+		// Clean up cluster based tenant indices and remove the old entry from its tenant group
+		// Remove the tenant from the cluster -> tenant index
+		ManagementClusterMetadata::clusterTenantIndex.erase(
+		    tr, Tuple::makeTuple(tenantEntry.assignedCluster.get(), self->oldName, self->tenantId));
+
+		// Remove the tenant from its tenant group
+		wait(managementClusterRemoveTenantFromGroup(
+		    tr, self->oldName, tenantEntry, &self->ctx.dataClusterMetadata.get(), true));
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> markTenantsInRenamingState(RenameTenantImpl* self,
+	                                                     Reference<typename DB::TransactionT> tr) {
+		state TenantMapEntry oldTenantEntry;
+		state Optional<TenantMapEntry> newTenantEntry;
+		wait(store(oldTenantEntry, getTenantTransaction(tr, self->oldName)) &&
+		     store(newTenantEntry, tryGetTenantTransaction(tr, self->newName)));
+
+		if (self->tenantId != -1 && oldTenantEntry.id != self->tenantId) {
+			// The tenant must have been removed simultaneously
+			CODE_PROBE(true, "Metacluster rename old tenant ID mismatch");
+			throw tenant_removed();
+		}
+
+		// If marked for deletion, abort the rename
+		if (oldTenantEntry.tenantState == TenantState::REMOVING) {
+			CODE_PROBE(true, "Metacluster rename candidates marked for deletion");
+			throw tenant_removed();
+		}
+
+		// If the new entry is present, we can only continue if this is a retry of the same rename
+		// To check this, verify both entries are in the correct state
+		// and have each other as pairs
+		if (newTenantEntry.present()) {
+			if (newTenantEntry.get().tenantState == TenantState::RENAMING_TO &&
+			    oldTenantEntry.tenantState == TenantState::RENAMING_FROM && newTenantEntry.get().renamePair.present() &&
+			    newTenantEntry.get().renamePair.get() == self->oldName && oldTenantEntry.renamePair.present() &&
+			    oldTenantEntry.renamePair.get() == self->newName) {
+				wait(self->ctx.setCluster(tr, oldTenantEntry.assignedCluster.get()));
+				self->tenantId = newTenantEntry.get().id;
+				self->configurationSequenceNum = newTenantEntry.get().configurationSequenceNum;
+				CODE_PROBE(true, "Metacluster rename retry in progress");
+				return Void();
+			} else {
+				CODE_PROBE(true, "Metacluster rename new name already exists");
+				throw tenant_already_exists();
+			};
+		} else {
+			if (self->tenantId == -1) {
+				self->tenantId = oldTenantEntry.id;
+			}
+			++oldTenantEntry.configurationSequenceNum;
+			self->configurationSequenceNum = oldTenantEntry.configurationSequenceNum;
+			wait(self->ctx.setCluster(tr, oldTenantEntry.assignedCluster.get()));
+			if (oldTenantEntry.tenantState != TenantState::READY) {
+				CODE_PROBE(true, "Metacluster unable to proceed with rename operation");
+				throw invalid_tenant_state();
+			}
+		}
+
+		// Check cluster capacity. If we would exceed the amount due to temporary extra tenants
+		// then we deny the rename request altogether.
+		int64_t clusterTenantCount = wait(ManagementClusterMetadata::clusterTenantCount.getD(
+		    tr, oldTenantEntry.assignedCluster.get(), Snapshot::False, 0));
+
+		if (clusterTenantCount + 1 > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER) {
+			throw cluster_no_capacity();
+		}
+
+		TenantMapEntry updatedOldEntry = oldTenantEntry;
+		TenantMapEntry updatedNewEntry(updatedOldEntry);
+		ASSERT(updatedOldEntry.configurationSequenceNum == self->configurationSequenceNum);
+		ASSERT(updatedNewEntry.configurationSequenceNum == self->configurationSequenceNum);
+		updatedOldEntry.tenantState = TenantState::RENAMING_FROM;
+		updatedNewEntry.tenantState = TenantState::RENAMING_TO;
+		updatedOldEntry.renamePair = self->newName;
+		updatedNewEntry.renamePair = self->oldName;
+
+		ManagementClusterMetadata::tenantMetadata().tenantMap.set(tr, self->oldName, updatedOldEntry);
+		ManagementClusterMetadata::tenantMetadata().tenantMap.set(tr, self->newName, updatedNewEntry);
+
+		// Add temporary tenant to tenantCount to prevent exceeding capacity during a rename
+		ManagementClusterMetadata::tenantMetadata().tenantCount.atomicOp(tr, 1, MutationRef::AddValue);
+		ManagementClusterMetadata::clusterTenantCount.atomicOp(
+		    tr, updatedNewEntry.assignedCluster.get(), 1, MutationRef::AddValue);
+
+		// Updated indexes to include the new tenant
+		ManagementClusterMetadata::clusterTenantIndex.insert(
+		    tr, Tuple::makeTuple(updatedNewEntry.assignedCluster.get(), self->newName, self->tenantId));
+
+		// Add new name to tenant group. It should already exist since the old name was part of it.
+		managementClusterAddTenantToGroup(
+		    tr, self->newName, updatedNewEntry, &self->ctx.dataClusterMetadata.get(), true);
+		return Void();
+	}
+
+	ACTOR static Future<Void> updateDataCluster(RenameTenantImpl* self, Reference<typename DB::TransactionT> tr) {
+		ASSERT(self->tenantId != -1);
+		ASSERT(self->configurationSequenceNum != -1);
+		wait(TenantAPI::renameTenantTransaction(tr,
+		                                        self->oldName,
+		                                        self->newName,
+		                                        self->tenantId,
+		                                        ClusterType::METACLUSTER_DATA,
+		                                        self->configurationSequenceNum));
+		return Void();
+	}
+
+	ACTOR static Future<Void> finishRenameFromManagementCluster(RenameTenantImpl* self,
+	                                                            Reference<typename DB::TransactionT> tr) {
+		state Optional<TenantMapEntry> oldTenantEntry;
+		state Optional<TenantMapEntry> newTenantEntry;
+		wait(store(oldTenantEntry, tryGetTenantTransaction(tr, self->oldName)) &&
+		     store(newTenantEntry, tryGetTenantTransaction(tr, self->newName)));
+
+		// Another (or several other) operations have already removed/changed the old entry
+		// Possible for the new entry to also have been tampered with,
+		// so it may or may not be present with or without the same id, which are all
+		// legal states. Assume the rename completed properly in this case
+		if (!oldTenantEntry.present() || oldTenantEntry.get().id != self->tenantId ||
+		    oldTenantEntry.get().configurationSequenceNum > self->configurationSequenceNum) {
+			CODE_PROBE(true,
+			           "Metacluster finished rename with missing entries, mismatched id, and/or mismatched "
+			           "configuration sequence.");
+			return Void();
+		}
+		if (oldTenantEntry.get().tenantState == TenantState::REMOVING) {
+			ASSERT(newTenantEntry.get().tenantState == TenantState::REMOVING);
+			throw tenant_removed();
+		}
+		ASSERT(newTenantEntry.present());
+		ASSERT(newTenantEntry.get().id == self->tenantId);
+
+		TenantMapEntry updatedOldEntry = oldTenantEntry.get();
+		TenantMapEntry updatedNewEntry = newTenantEntry.get();
+
+		// Only update if in the expected state
+		if (updatedNewEntry.tenantState == TenantState::RENAMING_TO) {
+			updatedNewEntry.tenantState = TenantState::READY;
+			updatedNewEntry.renamePair.reset();
+			ManagementClusterMetadata::tenantMetadata().tenantMap.set(tr, self->newName, updatedNewEntry);
+			ManagementClusterMetadata::tenantMetadata().tenantIdIndex.set(tr, self->tenantId, self->newName);
+		}
+
+		// We will remove the old entry from the management cluster
+		// This should still be the same old entry since the tenantId matches from the check above.
+		wait(deleteTenantFromManagementCluster(self, tr, updatedOldEntry));
+		return Void();
+	}
+
+	ACTOR static Future<Void> run(RenameTenantImpl* self) {
+		wait(self->ctx.runManagementTransaction(
+		    [self = self](Reference<typename DB::TransactionT> tr) { return markTenantsInRenamingState(self, tr); }));
+
+		// Rename tenant on the data cluster
+		try {
+			wait(self->ctx.runDataClusterTransaction(
+			    [self = self](Reference<ITransaction> tr) { return updateDataCluster(self, tr); }));
+		} catch (Error& e) {
+			// Since we track the tenant entries on the management cluster, these error codes should only appear
+			// on a retry of the transaction, typically caused by commit_unknown_result.
+			// Operating on the assumption that the first transaction completed successfully, we keep going
+			// so we can finish the rename on the management cluster.
+			if (e.code() == error_code_tenant_not_found || e.code() == error_code_tenant_already_exists) {
+				CODE_PROBE(true, "Metacluster rename ran into commit_unknown_result");
+			} else {
+				throw e;
+			}
+		}
+
+		wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
+			return finishRenameFromManagementCluster(self, tr);
+		}));
+		return Void();
+	}
+	Future<Void> run() { return run(this); }
+};
+
+ACTOR template <class DB>
+Future<Void> renameTenant(Reference<DB> db, TenantName oldName, TenantName newName) {
+	state RenameTenantImpl<DB> impl(db, oldName, newName);
+	wait(impl.run());
+	return Void();
+}
+
+template <class Transaction>
+Future<Optional<TenantGroupEntry>> tryGetTenantGroupTransaction(Transaction tr, TenantGroupName name) {
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+	return ManagementClusterMetadata::tenantMetadata().tenantGroupMap.get(tr, name);
+}
+
+ACTOR template <class DB>
+Future<Optional<TenantGroupEntry>> tryGetTenantGroup(Reference<DB> db, TenantGroupName name) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			Optional<TenantGroupEntry> entry = wait(tryGetTenantGroupTransaction(tr, name));
+			return entry;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+ACTOR template <class Transaction>
+Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroupsTransaction(Transaction tr,
+                                                                                              TenantGroupName begin,
+                                                                                              TenantGroupName end,
+                                                                                              int limit) {
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+
+	KeyBackedRangeResult<std::pair<TenantGroupName, TenantGroupEntry>> results =
+	    wait(ManagementClusterMetadata::tenantMetadata().tenantGroupMap.getRange(tr, begin, end, limit));
+
+	return results.results;
+}
+
+ACTOR template <class DB>
+Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroups(Reference<DB> db,
+                                                                                   TenantGroupName begin,
+                                                                                   TenantGroupName end,
+                                                                                   int limit) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups =
+			    wait(listTenantGroupsTransaction(tr, begin, end, limit));
+			return tenantGroups;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+} // namespace MetaclusterAPI
+
+#include "flow/unactorcompiler.h"
+#endif
\ No newline at end of file
diff --git a/fdbclient/include/fdbclient/MultiVersionAssignmentVars.h b/fdbclient/include/fdbclient/MultiVersionAssignmentVars.h
index 265ce0b3fe..888c8ec01a 100644
--- a/fdbclient/include/fdbclient/MultiVersionAssignmentVars.h
+++ b/fdbclient/include/fdbclient/MultiVersionAssignmentVars.h
@@ -28,16 +28,24 @@ template <class T>
 class AbortableSingleAssignmentVar final : public ThreadSingleAssignmentVar<T>, public ThreadCallback {
 public:
 	AbortableSingleAssignmentVar(ThreadFuture<T> future, ThreadFuture<Void> abortSignal)
-	  : future(future), abortSignal(abortSignal), hasBeenSet(false), callbacksCleared(false) {
+	  : future(future), abortSignal(abortSignal), hasBeenSet(false), callbacksCleared(true) {
 		int userParam;
 
 		ThreadSingleAssignmentVar<T>::addref();
 		ThreadSingleAssignmentVar<T>::addref();
 
-		// abortSignal comes first, because otherwise future could immediately call fire/error and attempt to remove
-		// this callback from abortSignal prematurely
 		abortSignal.callOrSetAsCallback(this, userParam, 0);
 		future.callOrSetAsCallback(this, userParam, 0);
+
+		// One of the signals could be already fired
+		// Make sure that the other is cancelled, and the references removed
+		lock.enter();
+		callbacksCleared = false;
+		bool hasBeenSet_ = hasBeenSet;
+		lock.leave();
+		if (hasBeenSet_) {
+			cancelCallbacks();
+		}
 	}
 
 	void cancel() override {
@@ -104,12 +112,28 @@ private:
 			callbacksCleared = true;
 			lock.leave();
 
-			future.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this
-			                           // callback gets destroyed
-			future.getPtr()->cancel();
+			bool notificationRequired = true;
+
+			if (future.clearCallback(this)) {
+				ThreadSingleAssignmentVar<T>::delref();
+			} else {
+				notificationRequired = false;
+				future.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this
+				                           // callback gets destroyed
+				future.getPtr()->cancel();
+			}
 
 			if (abortSignal.clearCallback(this)) {
 				ThreadSingleAssignmentVar<T>::delref();
+			} else {
+				notificationRequired = false;
+			}
+
+			if (notificationRequired) {
+				// The future has been cancelled before any of the signals were
+				// fired. Notify the futures about the cancellation
+				ASSERT(!hasBeenSet);
+				ThreadSingleAssignmentVar<T>::sendError(operation_cancelled());
 			}
 		} else {
 			lock.leave();
diff --git a/fdbclient/include/fdbclient/MultiVersionTransaction.h b/fdbclient/include/fdbclient/MultiVersionTransaction.h
index 4a59872c23..f74026d0ac 100644
--- a/fdbclient/include/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/include/fdbclient/MultiVersionTransaction.h
@@ -26,6 +26,7 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/IClientApi.h"
+#include "flow/ApiVersion.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/ThreadHelper.actor.h"
 
@@ -89,6 +90,14 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 		const void* endKey;
 		int endKeyLength;
 	} FDBKeyRange;
+
+	typedef struct granulesummary {
+		FDBKeyRange key_range;
+		int64_t snapshot_version;
+		int64_t snapshot_size;
+		int64_t delta_version;
+		int64_t delta_size;
+	} FDBGranuleSummary;
 #pragma pack(pop)
 
 	typedef struct readgranulecontext {
@@ -122,6 +131,8 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	// Network
 	fdb_error_t (*selectApiVersion)(int runtimeVersion, int headerVersion);
 	const char* (*getClientVersion)();
+	void (*useFutureProtocolVersion)();
+
 	fdb_error_t (*setNetworkOption)(FDBNetworkOption option, uint8_t const* value, int valueLength);
 	fdb_error_t (*setupNetwork)();
 	fdb_error_t (*runNetwork)();
@@ -169,6 +180,32 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	                                                uint8_t const* purge_key_name,
 	                                                int purge_key_name_length);
 
+	FDBFuture* (*databaseBlobbifyRange)(FDBDatabase* db,
+	                                    uint8_t const* begin_key_name,
+	                                    int begin_key_name_length,
+	                                    uint8_t const* end_key_name,
+	                                    int end_key_name_length);
+
+	FDBFuture* (*databaseUnblobbifyRange)(FDBDatabase* db,
+	                                      uint8_t const* begin_key_name,
+	                                      int begin_key_name_length,
+	                                      uint8_t const* end_key_name,
+	                                      int end_key_name_length);
+
+	FDBFuture* (*databaseListBlobbifiedRanges)(FDBDatabase* db,
+	                                           uint8_t const* begin_key_name,
+	                                           int begin_key_name_length,
+	                                           uint8_t const* end_key_name,
+	                                           int end_key_name_length,
+	                                           int rangeLimit);
+
+	FDBFuture* (*databaseVerifyBlobRange)(FDBDatabase* db,
+	                                      uint8_t const* begin_key_name,
+	                                      int begin_key_name_length,
+	                                      uint8_t const* end_key_name,
+	                                      int end_key_name_length,
+	                                      int64_t version);
+
 	// Tenant
 	fdb_error_t (*tenantCreateTransaction)(FDBTenant* tenant, FDBTransaction** outTransaction);
 
@@ -270,20 +307,47 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	                                             int end_key_name_length,
 	                                             int64_t chunkSize);
 
-	FDBFuture* (*transactionGetBlobGranuleRanges)(FDBTransaction* db,
+	FDBFuture* (*transactionGetBlobGranuleRanges)(FDBTransaction* tr,
 	                                              uint8_t const* begin_key_name,
 	                                              int begin_key_name_length,
 	                                              uint8_t const* end_key_name,
-	                                              int end_key_name_length);
+	                                              int end_key_name_length,
+	                                              int rangeLimit);
 
-	FDBResult* (*transactionReadBlobGranules)(FDBTransaction* db,
+	FDBResult* (*transactionReadBlobGranules)(FDBTransaction* tr,
 	                                          uint8_t const* begin_key_name,
 	                                          int begin_key_name_length,
 	                                          uint8_t const* end_key_name,
 	                                          int end_key_name_length,
 	                                          int64_t beginVersion,
-	                                          int64_t readVersion,
-	                                          FDBReadBlobGranuleContext granule_context);
+	                                          int64_t readVersion);
+
+	FDBFuture* (*transactionReadBlobGranulesStart)(FDBTransaction* tr,
+	                                               uint8_t const* begin_key_name,
+	                                               int begin_key_name_length,
+	                                               uint8_t const* end_key_name,
+	                                               int end_key_name_length,
+	                                               int64_t beginVersion,
+	                                               int64_t readVersion,
+	                                               int64_t* readVersionOut);
+
+	FDBResult* (*transactionReadBlobGranulesFinish)(FDBTransaction* tr,
+	                                                FDBFuture* startFuture,
+	                                                uint8_t const* begin_key_name,
+	                                                int begin_key_name_length,
+	                                                uint8_t const* end_key_name,
+	                                                int end_key_name_length,
+	                                                int64_t beginVersion,
+	                                                int64_t readVersion,
+	                                                FDBReadBlobGranuleContext* granule_context);
+
+	FDBFuture* (*transactionSummarizeBlobGranules)(FDBTransaction* tr,
+	                                               uint8_t const* begin_key_name,
+	                                               int begin_key_name_length,
+	                                               uint8_t const* end_key_name,
+	                                               int end_key_name_length,
+	                                               int64_t summaryVersion,
+	                                               int rangeLimit);
 
 	FDBFuture* (*transactionCommit)(FDBTransaction* tr);
 	fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction* tr, int64_t* outVersion);
@@ -304,7 +368,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	fdb_error_t (*futureGetDatabase)(FDBFuture* f, FDBDatabase** outDb);
 	fdb_error_t (*futureGetInt64)(FDBFuture* f, int64_t* outValue);
 	fdb_error_t (*futureGetUInt64)(FDBFuture* f, uint64_t* outValue);
-	fdb_error_t (*futureGetBool)(FDBFuture* f, bool* outValue);
+	fdb_error_t (*futureGetBool)(FDBFuture* f, fdb_bool_t* outValue);
 	fdb_error_t (*futureGetError)(FDBFuture* f);
 	fdb_error_t (*futureGetKey)(FDBFuture* f, uint8_t const** outKey, int* outKeyLength);
 	fdb_error_t (*futureGetValue)(FDBFuture* f, fdb_bool_t* outPresent, uint8_t const** outValue, int* outValueLength);
@@ -316,6 +380,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
 	                                            FDBMappedKeyValue const** outKVM,
 	                                            int* outCount,
 	                                            fdb_bool_t* outMore);
+	fdb_error_t (*futureGetGranuleSummaryArray)(FDBFuture* f, const FDBGranuleSummary** out_summaries, int* outCount);
 	fdb_error_t (*futureGetSharedState)(FDBFuture* f, DatabaseSharedState** outPtr);
 	fdb_error_t (*futureSetCallback)(FDBFuture* f, FDBCallback callback, void* callback_parameter);
 	void (*futureCancel)(FDBFuture* f);
@@ -374,13 +439,30 @@ public:
 	ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override;
 	ThreadFuture<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRangeRef& range,
 	                                                                int64_t chunkSize) override;
-	ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRangeRef& keyRange) override;
+	ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRangeRef& keyRange,
+	                                                                      int rangeLimit) override;
 
 	ThreadResult<RangeResult> readBlobGranules(const KeyRangeRef& keyRange,
 	                                           Version beginVersion,
 	                                           Optional<Version> readVersion,
 	                                           ReadBlobGranuleContext granule_context) override;
 
+	ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesStart(const KeyRangeRef& keyRange,
+	                                                                               Version beginVersion,
+	                                                                               Optional<Version> readVersion,
+	                                                                               Version* readVersionOut) override;
+
+	ThreadResult<RangeResult> readBlobGranulesFinish(
+	    ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> startFuture,
+	    const KeyRangeRef& keyRange,
+	    Version beginVersion,
+	    Version readVersion,
+	    ReadBlobGranuleContext granuleContext) override;
+
+	ThreadFuture<Standalone<VectorRef<BlobGranuleSummaryRef>>> summarizeBlobGranules(const KeyRangeRef& keyRange,
+	                                                                                 Optional<Version> summaryVersion,
+	                                                                                 int rangeLimit) override;
+
 	void addReadConflictRange(const KeyRangeRef& keys) override;
 
 	void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override;
@@ -474,6 +556,12 @@ public:
 	ThreadFuture<Key> purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) override;
 	ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
 
+	ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
+	ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
+	ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
+	                                                                      int rangeLimit) override;
+	ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
+
 	ThreadFuture<DatabaseSharedState*> createSharedState() override;
 	void setSharedState(DatabaseSharedState* p) override;
 
@@ -492,6 +580,7 @@ public:
 
 	void selectApiVersion(int apiVersion) override;
 	const char* getClientVersion() override;
+	void useFutureProtocolVersion() override;
 
 	void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
 	void setupNetwork() override;
@@ -571,13 +660,30 @@ public:
 
 	ThreadFuture<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRangeRef& range,
 	                                                                int64_t chunkSize) override;
-	ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRangeRef& keyRange) override;
+	ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRangeRef& keyRange,
+	                                                                      int rangeLimit) override;
 
 	ThreadResult<RangeResult> readBlobGranules(const KeyRangeRef& keyRange,
 	                                           Version beginVersion,
 	                                           Optional<Version> readVersion,
 	                                           ReadBlobGranuleContext granule_context) override;
 
+	ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesStart(const KeyRangeRef& keyRange,
+	                                                                               Version beginVersion,
+	                                                                               Optional<Version> readVersion,
+	                                                                               Version* readVersionOut) override;
+
+	ThreadResult<RangeResult> readBlobGranulesFinish(
+	    ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> startFuture,
+	    const KeyRangeRef& keyRange,
+	    Version beginVersion,
+	    Version readVersion,
+	    ReadBlobGranuleContext granuleContext) override;
+
+	ThreadFuture<Standalone<VectorRef<BlobGranuleSummaryRef>>> summarizeBlobGranules(const KeyRangeRef& keyRange,
+	                                                                                 Optional<Version> summaryVersion,
+	                                                                                 int rangeLimit) override;
+
 	void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override;
 	void set(const KeyRef& key, const ValueRef& value) override;
 	void clear(const KeyRef& begin, const KeyRef& end) override;
@@ -643,6 +749,9 @@ private:
 	template <class T>
 	ThreadResult<T> abortableTimeoutResult(ThreadFuture<Void> abortSignal);
 
+	template <class T>
+	ThreadResult<T> abortableResult(ThreadResult<T> result, ThreadFuture<Void> abortSignal);
+
 	TransactionInfo transaction;
 
 	TransactionInfo getTransaction();
@@ -650,15 +759,15 @@ private:
 	void setDefaultOptions(UniqueOrderedOptionList<FDBTransactionOptions> options);
 
 	std::vector<std::pair<FDBTransactionOptions::Option, Optional<Standalone<StringRef>>>> persistentOptions;
-
-	const Optional<TenantName> tenantName;
 };
 
 struct ClientDesc {
 	std::string const libPath;
 	bool const external;
+	bool const useFutureVersion;
 
-	ClientDesc(std::string libPath, bool external) : libPath(libPath), external(external) {}
+	ClientDesc(std::string libPath, bool external, bool useFutureVersion)
+	  : libPath(libPath), external(external), useFutureVersion(useFutureVersion) {}
 };
 
 struct ClientInfo : ClientDesc, ThreadSafeReferenceCounted<ClientInfo> {
@@ -667,17 +776,22 @@ struct ClientInfo : ClientDesc, ThreadSafeReferenceCounted<ClientInfo> {
 	IClientApi* api;
 	bool failed;
 	std::atomic_bool initialized;
+	int threadIndex;
 	std::vector<std::pair<void (*)(void*), void*>> threadCompletionHooks;
 
 	ClientInfo()
-	  : ClientDesc(std::string(), false), protocolVersion(0), api(nullptr), failed(true), initialized(false) {}
+	  : ClientDesc(std::string(), false, false), protocolVersion(0), api(nullptr), failed(true), initialized(false),
+	    threadIndex(0) {}
 	ClientInfo(IClientApi* api)
-	  : ClientDesc("internal", false), protocolVersion(0), api(api), failed(false), initialized(false) {}
-	ClientInfo(IClientApi* api, std::string libPath)
-	  : ClientDesc(libPath, true), protocolVersion(0), api(api), failed(false), initialized(false) {}
+	  : ClientDesc("internal", false, false), protocolVersion(0), api(api), failed(false), initialized(false),
+	    threadIndex(0) {}
+	ClientInfo(IClientApi* api, std::string libPath, bool useFutureVersion, int threadIndex)
+	  : ClientDesc(libPath, true, useFutureVersion), protocolVersion(0), api(api), failed(false), initialized(false),
+	    threadIndex(threadIndex) {}
 
 	void loadVersion();
 	bool canReplace(Reference<ClientInfo> other) const;
+	std::string getTraceFileIdentifier(const std::string& baseIdentifier);
 };
 
 class MultiVersionApi;
@@ -814,6 +928,12 @@ public:
 	ThreadFuture<Key> purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) override;
 	ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
 
+	ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
+	ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
+	ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
+	                                                                      int rangeLimit) override;
+	ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
+
 	ThreadFuture<DatabaseSharedState*> createSharedState() override;
 	void setSharedState(DatabaseSharedState* p) override;
 
@@ -921,6 +1041,7 @@ class MultiVersionApi : public IClientApi {
 public:
 	void selectApiVersion(int apiVersion) override;
 	const char* getClientVersion() override;
+	void useFutureProtocolVersion() override;
 
 	void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
 	void setupNetwork() override;
@@ -958,7 +1079,7 @@ public:
 	};
 	std::map<std::string, SharedStateInfo> clusterSharedStateMap;
 
-	static bool apiVersionAtLeast(int minVersion);
+	ApiVersion getApiVersion() { return apiVersion; }
 
 private:
 	MultiVersionApi();
@@ -967,7 +1088,7 @@ private:
 
 	void disableMultiVersionClientApi();
 	void setCallbacksOnExternalThreads();
-	void addExternalLibrary(std::string path);
+	void addExternalLibrary(std::string path, bool useFutureVersion);
 	void addExternalLibraryDirectory(std::string path);
 	// Return a vector of (pathname, unlink_on_close) pairs.  Makes threadCount - 1 copies of the library stored in
 	// path, and returns a vector of length threadCount.
@@ -983,13 +1104,16 @@ private:
 
 	bool networkStartSetup;
 	volatile bool networkSetup;
+	bool disableBypass;
 	volatile bool bypassMultiClientApi;
 	volatile bool externalClient;
-	int apiVersion;
+	ApiVersion apiVersion;
 
 	int nextThread = 0;
 	int threadCount;
 	std::string tmpDir;
+	bool traceShareBaseNameAmongThreads;
+	std::string traceFileIdentifier;
 
 	Mutex lock;
 	std::vector<std::pair<FDBNetworkOptions::Option, Optional<Standalone<StringRef>>>> options;
diff --git a/fdbclient/include/fdbclient/NativeAPI.actor.h b/fdbclient/include/fdbclient/NativeAPI.actor.h
index 3abab222bb..3931182ab0 100644
--- a/fdbclient/include/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/include/fdbclient/NativeAPI.actor.h
@@ -82,8 +82,6 @@ struct NetworkOptions {
 
 class Database {
 public:
-	enum { API_VERSION_LATEST = -1 };
-
 	// Creates a database object that represents a connection to a cluster
 	// This constructor uses a preallocated DatabaseContext that may have been created
 	// on another thread
@@ -98,6 +96,9 @@ public:
 	                               IsInternal internal = IsInternal::True,
 	                               LocalityData const& clientLocality = LocalityData());
 
+	static Database createSimulatedExtraDatabase(std::string connectionString,
+	                                             Optional<TenantName> defaultTenant = Optional<TenantName>());
+
 	Database() {} // an uninitialized database can be destructed or reassigned safely; that's it
 	void operator=(Database const& rhs) { db = rhs.db; }
 	Database(Database const& rhs) : db(rhs.db) {}
@@ -235,18 +236,27 @@ struct Watch : public ReferenceCounted<Watch>, NonCopyable {
 	void setWatch(Future<Void> watchFuture);
 };
 
+FDB_DECLARE_BOOLEAN_PARAM(AllowInvalidTenantID);
+
 struct TransactionState : ReferenceCounted<TransactionState> {
 	Database cx;
-	int64_t tenantId = TenantInfo::INVALID_TENANT;
+	Optional<Standalone<StringRef>> authToken;
 	Reference<TransactionLogInfo> trLogInfo;
 	TransactionOptions options;
+	Optional<ReadOptions> readOptions;
 
-	Optional<UID> debugID;
 	TaskPriority taskID;
 	SpanContext spanContext;
 	UseProvisionalProxies useProvisionalProxies = UseProvisionalProxies::False;
 	bool readVersionObtainedFromGrvProxy;
 
+	// Special flag to skip prepending tenant prefix to mutations and conflict ranges
+	// when a dummy, internal transaction gets commited. The sole purpose of commitDummyTransaction() is to
+	// resolve the state of earlier transaction that returned commit_unknown_result or request_maybe_delivered.
+	// Therefore, the dummy transaction can simply reuse one conflict range of the earlier commit, if it already has
+	// been prefixed.
+	bool skipApplyTenantPrefix = false;
+
 	int numErrors = 0;
 	double startTime = 0;
 	Promise<Standalone<StringRef>> versionstampPromise;
@@ -270,13 +280,23 @@ struct TransactionState : ReferenceCounted<TransactionState> {
 	                 Reference<TransactionLogInfo> trLogInfo);
 
 	Reference<TransactionState> cloneAndReset(Reference<TransactionLogInfo> newTrLogInfo, bool generateNewSpan) const;
-	TenantInfo getTenantInfo();
+	TenantInfo getTenantInfo(AllowInvalidTenantID allowInvalidId = AllowInvalidTenantID::False);
 
 	Optional<TenantName> const& tenant();
 	bool hasTenant() const;
 
+	int64_t tenantId() const { return tenantId_; }
+	void trySetTenantId(int64_t tenantId) {
+		if (tenantId_ == TenantInfo::INVALID_TENANT) {
+			tenantId_ = tenantId;
+		}
+	}
+
+	Future<Void> handleUnknownTenant();
+
 private:
 	Optional<TenantName> tenant_;
+	int64_t tenantId_ = TenantInfo::INVALID_TENANT;
 	bool tenantSet;
 };
 
@@ -346,19 +366,19 @@ private:
 public:
 	// A method for streaming data from the storage server that is more efficient than getRange when reading large
 	// amounts of data
-	[[nodiscard]] Future<Void> getRangeStream(const PromiseStream<Standalone<RangeResultRef>>& results,
+	[[nodiscard]] Future<Void> getRangeStream(PromiseStream<Standalone<RangeResultRef>>& results,
 	                                          const KeySelector& begin,
 	                                          const KeySelector& end,
 	                                          int limit,
 	                                          Snapshot = Snapshot::False,
 	                                          Reverse = Reverse::False);
-	[[nodiscard]] Future<Void> getRangeStream(const PromiseStream<Standalone<RangeResultRef>>& results,
+	[[nodiscard]] Future<Void> getRangeStream(PromiseStream<Standalone<RangeResultRef>>& results,
 	                                          const KeySelector& begin,
 	                                          const KeySelector& end,
 	                                          GetRangeLimits limits,
 	                                          Snapshot = Snapshot::False,
 	                                          Reverse = Reverse::False);
-	[[nodiscard]] Future<Void> getRangeStream(const PromiseStream<Standalone<RangeResultRef>>& results,
+	[[nodiscard]] Future<Void> getRangeStream(PromiseStream<Standalone<RangeResultRef>>& results,
 	                                          const KeyRange& keys,
 	                                          int limit,
 	                                          Snapshot snapshot = Snapshot::False,
@@ -370,7 +390,7 @@ public:
 		                      snapshot,
 		                      reverse);
 	}
-	[[nodiscard]] Future<Void> getRangeStream(const PromiseStream<Standalone<RangeResultRef>>& results,
+	[[nodiscard]] Future<Void> getRangeStream(PromiseStream<Standalone<RangeResultRef>>& results,
 	                                          const KeyRange& keys,
 	                                          GetRangeLimits limits,
 	                                          Snapshot snapshot = Snapshot::False,
@@ -396,12 +416,18 @@ public:
 	// The returned list would still be in form of [keys.begin, splitPoint1, splitPoint2, ... , keys.end]
 	Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(KeyRange const& keys, int64_t chunkSize);
 
-	Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRange& range);
+	Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRange& range, int rangeLimit);
 	Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranules(const KeyRange& range,
 	                                                                    Version begin,
 	                                                                    Optional<Version> readVersion,
 	                                                                    Version* readVersionOut = nullptr);
 
+	Future<Standalone<VectorRef<BlobGranuleSummaryRef>>> summarizeBlobGranules(const KeyRange& range,
+	                                                                           Optional<Version> summaryVersion,
+	                                                                           int rangeLimit);
+
+	void addGranuleMaterializeStats(const GranuleMaterializeStats& stats);
+
 	// If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key
 	void set(const KeyRef& key, const ValueRef& value, AddConflictRange = AddConflictRange::True);
 	void atomicOp(const KeyRef& key,
@@ -436,7 +462,13 @@ public:
 	void fullReset();
 	double getBackoff(int errCode);
 
-	void debugTransaction(UID dID) { trState->debugID = dID; }
+	void debugTransaction(UID dID) {
+		if (trState->readOptions.present()) {
+			trState->readOptions.get().debugID = dID;
+		} else {
+			trState->readOptions = ReadOptions(dID);
+		}
+	}
 	VersionVector getVersionVector() const;
 	SpanContext getSpanContext() const { return trState->spanContext; }
 
@@ -531,8 +563,9 @@ ACTOR Future<std::vector<CheckpointMetaData>> getCheckpointMetaData(Database cx,
 // Checks with Data Distributor that it is safe to mark all servers in exclusions as failed
 ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion> exclusions);
 
+// Round up to the nearest page size
 inline uint64_t getWriteOperationCost(uint64_t bytes) {
-	return bytes / std::max(1, CLIENT_KNOBS->WRITE_COST_BYTE_FACTOR) + 1;
+	return (bytes - 1) / CLIENT_KNOBS->WRITE_COST_BYTE_FACTOR + 1;
 }
 
 // Create a transaction to set the value of system key \xff/conf/perpetual_storage_wiggle. If enable == true, the value
diff --git a/fdbclient/include/fdbclient/PImpl.h b/fdbclient/include/fdbclient/PImpl.h
index 424761efe5..992753fa06 100644
--- a/fdbclient/include/fdbclient/PImpl.h
+++ b/fdbclient/include/fdbclient/PImpl.h
@@ -39,4 +39,6 @@ public:
 	T const& operator*() const { return *impl; }
 	T* operator->() { return impl.get(); }
 	T const* operator->() const { return impl.get(); }
+	T* get() { return impl.get(); }
+	T const* get() const { return impl.get(); }
 };
diff --git a/fdbclient/include/fdbclient/RESTClient.h b/fdbclient/include/fdbclient/RESTClient.h
index ba9719a9cc..cb8754ad9d 100644
--- a/fdbclient/include/fdbclient/RESTClient.h
+++ b/fdbclient/include/fdbclient/RESTClient.h
@@ -25,7 +25,7 @@
 #pragma once
 
 #include "fdbclient/JSONDoc.h"
-#include "fdbclient/HTTP.h"
+#include "fdbrpc/HTTP.h"
 #include "fdbclient/RESTUtils.h"
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
diff --git a/fdbclient/include/fdbclient/ReadYourWrites.h b/fdbclient/include/fdbclient/ReadYourWrites.h
index 89de979bc1..659f7768ae 100644
--- a/fdbclient/include/fdbclient/ReadYourWrites.h
+++ b/fdbclient/include/fdbclient/ReadYourWrites.h
@@ -20,6 +20,7 @@
 
 #ifndef FDBCLIENT_READYOURWRITES_H
 #define FDBCLIENT_READYOURWRITES_H
+#include "fdbclient/Status.h"
 #pragma once
 
 #include "fdbclient/NativeAPI.actor.h"
@@ -120,12 +121,17 @@ public:
 	Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override;
 	Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
 
-	Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRange& range) override;
+	Future<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRange& range, int rangeLimit) override;
 	Future<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranules(const KeyRange& range,
 	                                                                    Version begin,
 	                                                                    Optional<Version> readVersion,
 	                                                                    Version* readVersionOut) override;
 
+	Future<Standalone<VectorRef<BlobGranuleSummaryRef>>> summarizeBlobGranules(const KeyRange& range,
+	                                                                           Optional<Version> summaryVersion,
+	                                                                           int rangeLimit) override;
+	void addGranuleMaterializeStats(const GranuleMaterializeStats& stats) override;
+
 	void addReadConflictRange(KeyRangeRef const& keys) override;
 	void makeSelfConflicting() override { tr.makeSelfConflicting(); }
 
@@ -192,7 +198,17 @@ public:
 	KeyRangeMap<std::pair<bool, Optional<Value>>>& getSpecialKeySpaceWriteMap() { return specialKeySpaceWriteMap; }
 	bool readYourWritesDisabled() const { return options.readYourWritesDisabled; }
 	const Optional<std::string>& getSpecialKeySpaceErrorMsg() { return specialKeySpaceErrorMsg; }
-	void setSpecialKeySpaceErrorMsg(const std::string& msg) { specialKeySpaceErrorMsg = msg; }
+	void setSpecialKeySpaceErrorMsg(const std::string& msg) {
+		if (g_network && g_network->isSimulated()) {
+			try {
+				readJSONStrictly(msg);
+			} catch (Error& e) {
+				TraceEvent(SevError, "InvalidSpecialKeySpaceErrorMessage").error(e).detail("Message", msg);
+				ASSERT(false);
+			}
+		}
+		specialKeySpaceErrorMsg = msg;
+	}
 	Transaction& getTransaction() { return tr; }
 
 	Optional<TenantName> getTenant() { return tr.getTenant(); }
diff --git a/fdbclient/include/fdbclient/S3BlobStore.h b/fdbclient/include/fdbclient/S3BlobStore.h
index 8649060b09..a4eba100c9 100644
--- a/fdbclient/include/fdbclient/S3BlobStore.h
+++ b/fdbclient/include/fdbclient/S3BlobStore.h
@@ -26,7 +26,7 @@
 #include "flow/Net2Packet.h"
 #include "fdbclient/Knobs.h"
 #include "flow/IRateControl.h"
-#include "fdbclient/HTTP.h"
+#include "fdbrpc/HTTP.h"
 #include "fdbclient/JSONDoc.h"
 
 // Representation of all the things you need to connect to a blob store instance with some credentials.
diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h
index 1f4f6ee190..f4fc882ea3 100644
--- a/fdbclient/include/fdbclient/ServerKnobs.h
+++ b/fdbclient/include/fdbclient/ServerKnobs.h
@@ -50,7 +50,6 @@ public:
 	bool PEEK_USING_STREAMING;
 	double TLOG_TIMEOUT; // tlog OR commit proxy failure - master's reaction time
 	double TLOG_SLOW_REJOIN_WARN_TIMEOUT_SECS; // Warns if a tlog takes too long to rejoin
-	double RECOVERY_TLOG_SMART_QUORUM_DELAY; // smaller might be better for bug amplification
 	double TLOG_STORAGE_MIN_UPDATE_INTERVAL;
 	double BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL;
 	int DESIRED_TOTAL_BYTES;
@@ -58,10 +57,6 @@ public:
 	double UPDATE_DELAY;
 	int MAXIMUM_PEEK_BYTES;
 	int APPLY_MUTATION_BYTES;
-	int RECOVERY_DATA_BYTE_LIMIT;
-	int BUGGIFY_RECOVERY_DATA_LIMIT;
-	double LONG_TLOG_COMMIT_TIME;
-	int64_t LARGE_TLOG_COMMIT_BYTES;
 	double BUGGIFY_RECOVER_MEMORY_LIMIT;
 	double BUGGIFY_WORKER_REMOVED_MAX_LAG;
 	int64_t UPDATE_STORAGE_BYTE_LIMIT;
@@ -123,16 +118,16 @@ public:
 	double BG_REBALANCE_POLLING_INTERVAL;
 	double BG_REBALANCE_SWITCH_CHECK_INTERVAL;
 	double DD_QUEUE_LOGGING_INTERVAL;
+	double DD_QUEUE_COUNTER_REFRESH_INTERVAL;
+	double DD_QUEUE_COUNTER_MAX_LOG; // max number of servers for which trace events will be generated in each round of
+	                                 // DD_QUEUE_COUNTER_REFRESH_INTERVAL duration
+	bool DD_QUEUE_COUNTER_SUMMARIZE; // Enable summary of remaining servers when the number of servers with ongoing
+	                                 // relocations in the last minute exceeds DD_QUEUE_COUNTER_MAX_LOG
 	double RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	double RELOCATION_PARALLELISM_PER_DEST_SERVER;
 	int DD_QUEUE_MAX_KEY_SERVERS;
 	int DD_REBALANCE_PARALLELISM;
 	int DD_REBALANCE_RESET_AMOUNT;
-	double BG_DD_MAX_WAIT;
-	double BG_DD_MIN_WAIT;
-	double BG_DD_INCREASE_RATE;
-	double BG_DD_DECREASE_RATE;
-	double BG_DD_SATURATION_DELAY;
 	double INFLIGHT_PENALTY_HEALTHY;
 	double INFLIGHT_PENALTY_REDUNDANT;
 	double INFLIGHT_PENALTY_UNHEALTHY;
@@ -161,8 +156,15 @@ public:
 	int PRIORITY_TEAM_FAILED; // Priority when a server in the team is excluded as failed
 	int PRIORITY_TEAM_0_LEFT;
 	int PRIORITY_SPLIT_SHARD;
+	int PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD; // Priority when a physical shard is oversize or anonymous
 
 	// Data distribution
+	bool SHARD_ENCODE_LOCATION_METADATA; // If true, location metadata will contain shard ID.
+	bool ENABLE_DD_PHYSICAL_SHARD; // EXPERIMENTAL; If true, SHARD_ENCODE_LOCATION_METADATA must be true.
+	int64_t MAX_PHYSICAL_SHARD_BYTES;
+	double PHYSICAL_SHARD_METRICS_DELAY;
+	double ANONYMOUS_PHYSICAL_SHARD_TRANSITION_TIME;
+
 	double READ_REBALANCE_CPU_THRESHOLD; // read rebalance only happens if the source servers' CPU > threshold
 	int READ_REBALANCE_SRC_PARALLELISM; // the max count a server become a source server within a certain interval
 	int READ_REBALANCE_SHARD_TOPK; // top k shards from which to select randomly for read-rebalance
@@ -193,7 +195,6 @@ public:
 	double SERVER_LIST_DELAY;
 	double RECRUITMENT_IDLE_DELAY;
 	double STORAGE_RECRUITMENT_DELAY;
-	double BLOB_WORKER_RECRUITMENT_DELAY;
 	bool TSS_HACK_IDENTITY_MAPPING;
 	double TSS_RECRUITMENT_TIMEOUT;
 	double TSS_DD_CHECK_INTERVAL;
@@ -232,6 +233,8 @@ public:
 	int DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY;
 	int DD_STORAGE_WIGGLE_PAUSE_THRESHOLD; // How many unhealthy relocations are ongoing will pause storage wiggle
 	int DD_STORAGE_WIGGLE_STUCK_THRESHOLD; // How many times bestTeamStuck accumulate will pause storage wiggle
+	int64_t
+	    DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC; // Minimal age of a correct-configured server before it's chosen to be wiggled
 	bool DD_TENANT_AWARENESS_ENABLED;
 	int TENANT_CACHE_LIST_REFRESH_INTERVAL; // How often the TenantCache is refreshed
 
@@ -253,9 +256,8 @@ public:
 
 	// Run storage enginee on a child process on the same machine with storage process
 	bool REMOTE_KV_STORE;
-	// A delay to avoid race on file resources if the new kv store process started immediately after the previous kv
-	// store process died
-	double REMOTE_KV_STORE_INIT_DELAY;
+	// A delay to avoid race on file resources after seeing lock_file_failure
+	double REBOOT_KV_STORE_DELAY;
 	// max waiting time for the remote kv store to initialize
 	double REMOTE_KV_STORE_MAX_INIT_DURATION;
 
@@ -300,9 +302,13 @@ public:
 	int64_t REPLACE_CONTENTS_BYTES;
 
 	// KeyValueStoreRocksDB
+	bool ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES;
+	int ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE;
+	int ROCKSDB_READ_RANGE_ROW_LIMIT;
 	int ROCKSDB_BACKGROUND_PARALLELISM;
 	int ROCKSDB_READ_PARALLELISM;
 	int64_t ROCKSDB_MEMTABLE_BYTES;
+	bool ROCKSDB_LEVEL_STYLE_COMPACTION;
 	bool ROCKSDB_UNSAFE_AUTO_FSYNC;
 	int64_t ROCKSDB_PERIODIC_COMPACTION_SECONDS;
 	int ROCKSDB_PREFIX_LEN;
@@ -324,8 +330,10 @@ public:
 	int64_t ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC;
 	bool ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE;
 	std::string DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY;
+	bool ROCKSDB_DISABLE_AUTO_COMPACTIONS;
 	bool ROCKSDB_PERFCONTEXT_ENABLE; // Enable rocks perf context metrics. May cause performance overhead
 	double ROCKSDB_PERFCONTEXT_SAMPLE_RATE;
+	double ROCKSDB_METRICS_SAMPLE_INTERVAL;
 	int ROCKSDB_MAX_SUBCOMPACTIONS;
 	int64_t ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT;
 	int64_t ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
@@ -335,6 +343,12 @@ public:
 	int64_t ROCKSDB_COMPACTION_READAHEAD_SIZE;
 	int64_t ROCKSDB_BLOCK_SIZE;
 	bool ENABLE_SHARDED_ROCKSDB;
+	int64_t ROCKSDB_WRITE_BUFFER_SIZE;
+	int64_t ROCKSDB_CF_WRITE_BUFFER_SIZE;
+	int64_t ROCKSDB_MAX_TOTAL_WAL_SIZE;
+	int64_t ROCKSDB_MAX_BACKGROUND_JOBS;
+	int64_t ROCKSDB_DELETE_OBSOLETE_FILE_PERIOD;
+	double ROCKSDB_PHYSICAL_SHARD_CLEAN_UP_DELAY;
 
 	// Leader election
 	int MAX_NOTIFICATIONS;
@@ -399,6 +413,10 @@ public:
 	double RESET_MASTER_DELAY;
 	double RESET_RESOLVER_DELAY;
 
+	double GLOBAL_CONFIG_MIGRATE_TIMEOUT;
+	double GLOBAL_CONFIG_REFRESH_INTERVAL;
+	double GLOBAL_CONFIG_REFRESH_TIMEOUT;
+
 	// Master Server
 	double COMMIT_SLEEP_TIME;
 	double MIN_BALANCE_TIME;
@@ -443,6 +461,7 @@ public:
 	double ATTEMPT_RECRUITMENT_DELAY;
 	double WAIT_FOR_DISTRIBUTOR_JOIN_DELAY;
 	double WAIT_FOR_RATEKEEPER_JOIN_DELAY;
+	double WAIT_FOR_CONSISTENCYSCAN_JOIN_DELAY;
 	double WAIT_FOR_BLOB_MANAGER_JOIN_DELAY;
 	double WAIT_FOR_ENCRYPT_KEY_PROXY_JOIN_DELAY;
 	double WORKER_FAILURE_TIME;
@@ -456,6 +475,7 @@ public:
 	double CHECK_REMOTE_HEALTH_INTERVAL; // Remote DC health refresh interval.
 	double FORCE_RECOVERY_CHECK_DELAY;
 	double RATEKEEPER_FAILURE_TIME;
+	double CONSISTENCYSCAN_FAILURE_TIME;
 	double BLOB_MANAGER_FAILURE_TIME;
 	double REPLACE_INTERFACE_DELAY;
 	double REPLACE_INTERFACE_CHECK_DELAY;
@@ -542,6 +562,10 @@ public:
 	double RATEKEEPER_DEFAULT_LIMIT;
 	double RATEKEEPER_LIMIT_REASON_SAMPLE_RATE;
 	bool RATEKEEPER_PRINT_LIMIT_REASON;
+	double RATEKEEPER_MIN_RATE;
+	double RATEKEEPER_MAX_RATE;
+	double RATEKEEPER_BATCH_MIN_RATE;
+	double RATEKEEPER_BATCH_MAX_RATE;
 
 	int64_t TARGET_BYTES_PER_STORAGE_SERVER;
 	int64_t SPRING_BYTES_STORAGE_SERVER;
@@ -550,9 +574,12 @@ public:
 	int64_t SPRING_BYTES_STORAGE_SERVER_BATCH;
 	int64_t STORAGE_HARD_LIMIT_BYTES;
 	int64_t STORAGE_HARD_LIMIT_BYTES_OVERAGE;
+	int64_t STORAGE_HARD_LIMIT_BYTES_SPEED_UP_SIM;
+	int64_t STORAGE_HARD_LIMIT_BYTES_OVERAGE_SPEED_UP_SIM;
 	int64_t STORAGE_HARD_LIMIT_VERSION_OVERAGE;
 	int64_t STORAGE_DURABILITY_LAG_HARD_MAX;
 	int64_t STORAGE_DURABILITY_LAG_SOFT_MAX;
+	bool STORAGE_INCLUDE_FEED_STORAGE_QUEUE;
 
 	int64_t LOW_PRIORITY_STORAGE_QUEUE_BYTES;
 	int64_t LOW_PRIORITY_DURABILITY_LAG;
@@ -585,11 +612,14 @@ public:
 	// Use global tag throttling strategy. i.e. throttle based on the cluster-wide
 	// throughput for tags and their associated quotas.
 	bool GLOBAL_TAG_THROTTLING;
+	// Enforce tag throttling on proxies rather than on clients
+	bool ENFORCE_TAG_THROTTLING_ON_PROXIES;
 	// Minimum number of transactions per second that the global tag throttler must allow for each tag
 	double GLOBAL_TAG_THROTTLING_MIN_RATE;
 	// Used by global tag throttling counters
 	double GLOBAL_TAG_THROTTLING_FOLDING_TIME;
-	double GLOBAL_TAG_THROTTLING_TRACE_INTERVAL;
+	// Cost multiplier for writes (because write operations are more expensive than reads)
+	double GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO;
 
 	double MAX_TRANSACTIONS_PER_BYTE;
 
@@ -612,8 +642,19 @@ public:
 	double INITIAL_DURABILITY_LAG_MULTIPLIER;
 	double DURABILITY_LAG_REDUCTION_RATE;
 	double DURABILITY_LAG_INCREASE_RATE;
-
 	double STORAGE_SERVER_LIST_FETCH_TIMEOUT;
+	bool BW_THROTTLING_ENABLED;
+	double TARGET_BW_LAG;
+	double TARGET_BW_LAG_BATCH;
+	double TARGET_BW_LAG_UPDATE;
+	int MIN_BW_HISTORY;
+	double BW_ESTIMATION_INTERVAL;
+	double BW_LAG_INCREASE_AMOUNT;
+	double BW_LAG_DECREASE_AMOUNT;
+	double BW_FETCH_WORKERS_INTERVAL;
+	double BW_RW_LOGGING_INTERVAL;
+	double BW_MAX_BLOCKED_INTERVAL;
+	double BW_RK_SIM_QUIESCE_DELAY;
 
 	// disk snapshot
 	int64_t MAX_FORKED_PROCESS_OUTPUT;
@@ -652,12 +693,14 @@ public:
 	int STORAGE_LIMIT_BYTES;
 	int BUGGIFY_LIMIT_BYTES;
 	bool FETCH_USING_STREAMING;
+	bool FETCH_USING_BLOB;
 	int FETCH_BLOCK_BYTES;
 	int FETCH_KEYS_PARALLELISM_BYTES;
 	int FETCH_KEYS_PARALLELISM;
+	int FETCH_KEYS_PARALLELISM_FULL;
 	int FETCH_KEYS_LOWER_PRIORITY;
-	int FETCH_CHANGEFEED_PARALLELISM;
 	int SERVE_FETCH_CHECKPOINT_PARALLELISM;
+	int CHANGE_FEED_DISK_READS_PARALLELISM;
 	int BUGGIFY_BLOCK_BYTES;
 	int64_t STORAGE_RECOVERY_VERSION_LAG_LIMIT;
 	double STORAGE_DURABILITY_LAG_REJECT_THRESHOLD;
@@ -665,7 +708,6 @@ public:
 	int STORAGE_COMMIT_BYTES;
 	int STORAGE_FETCH_BYTES;
 	double STORAGE_COMMIT_INTERVAL;
-	double UPDATE_SHARD_VERSION_INTERVAL;
 	int BYTE_SAMPLING_FACTOR;
 	int BYTE_SAMPLING_OVERHEAD;
 	int MAX_STORAGE_SERVER_WATCH_BYTES;
@@ -674,7 +716,6 @@ public:
 	int BYTE_SAMPLE_LOAD_PARALLELISM;
 	double BYTE_SAMPLE_LOAD_DELAY;
 	double BYTE_SAMPLE_START_DELAY;
-	double UPDATE_STORAGE_PROCESS_STATS_INTERVAL;
 	double BEHIND_CHECK_DELAY;
 	int BEHIND_CHECK_COUNT;
 	int64_t BEHIND_CHECK_VERSIONS;
@@ -698,6 +739,7 @@ public:
 	int CHECKPOINT_TRANSFER_BLOCK_BYTES;
 	int QUICK_GET_KEY_VALUES_LIMIT;
 	int QUICK_GET_KEY_VALUES_LIMIT_BYTES;
+	int STORAGE_FEED_QUERY_HARD_LIMIT;
 
 	// Wait Failure
 	int MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS;
@@ -731,6 +773,12 @@ public:
 	bool WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER; // When enabled, the worker's health monitor also report any recent
 	                                                 // destroyed peers who are part of the transaction system to
 	                                                 // cluster controller.
+	bool STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT; // When enabled, storage server's worker will crash on io_timeout error;
+	                                          // this allows fdbmonitor to restart the worker and recreate the same SS.
+	                                          // When SS can be temporarily throttled by infrastructure, e.g, k8s,
+	                                          // Enabling this can reduce toil of manually restarting the SS.
+	                                          // Enable with caution: If io_timeout is caused by disk failure, we won't
+	                                          // want to restart the SS, which increases risk of data corruption.
 
 	// Test harness
 	double WORKER_POLL_DELAY;
@@ -744,7 +792,7 @@ public:
 
 	// Dynamic Knobs (implementation)
 	double COMPACTION_INTERVAL;
-	double UPDATE_NODE_TIMEOUT;
+	double BROADCASTER_SELF_UPDATE_DELAY;
 	double GET_COMMITTED_VERSION_TIMEOUT;
 	double GET_SNAPSHOT_AND_CHANGES_TIMEOUT;
 	double FETCH_CHANGES_TIMEOUT;
@@ -760,14 +808,6 @@ public:
 	bool DISABLE_DUPLICATE_LOG_WARNING;
 	double HISTOGRAM_REPORT_INTERVAL;
 
-	// IPager
-	int PAGER_RESERVED_PAGES;
-
-	// IndirectShadowPager
-	int FREE_PAGE_VACUUM_THRESHOLD;
-	int VACUUM_QUEUE_SIZE;
-	int VACUUM_BYTES_PER_SECOND;
-
 	// Timekeeper
 	int64_t TIME_KEEPER_DELAY;
 	int64_t TIME_KEEPER_MAX_ENTRIES;
@@ -790,11 +830,9 @@ public:
 	int64_t FASTRESTORE_ROLE_LOGGING_DELAY;
 	int64_t FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL; // How quickly to update process metrics for restore
 	int64_t FASTRESTORE_ATOMICOP_WEIGHT; // workload amplication factor for atomic op
-	int64_t FASTRESTORE_APPLYING_PARALLELISM; // number of outstanding txns writing to dest. DB
 	int64_t FASTRESTORE_MONITOR_LEADER_DELAY;
 	int64_t FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS;
 	bool FASTRESTORE_TRACK_REQUEST_LATENCY; // true to track reply latency of each request in a request batch
-	bool FASTRESTORE_TRACK_LOADER_SEND_REQUESTS; // track requests of load send mutations to appliers?
 	int64_t FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT; // threshold when pipelined actors should be delayed
 	int64_t FASTRESTORE_WAIT_FOR_MEMORY_LATENCY;
 	int64_t FASTRESTORE_HEARTBEAT_DELAY; // interval for master to ping loaders and appliers
@@ -863,6 +901,7 @@ public:
 	int SIM_KMS_MAX_KEYS;
 	int ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH;
 	bool ENABLE_TLOG_ENCRYPTION;
+	bool ENABLE_STORAGE_SERVER_ENCRYPTION; // Currently only Redwood engine supports encryption
 	bool ENABLE_BLOB_GRANULE_ENCRYPTION;
 
 	// Compression
@@ -876,14 +915,13 @@ public:
 	// FIXME: configure url with database configuration instead of knob eventually
 	std::string BG_URL;
 
-	// whether to use blobRangeKeys or tenants for blob granule range sources
-	std::string BG_RANGE_SOURCE;
 	// Whether to use knobs or EKP for blob metadata and credentials
 	std::string BG_METADATA_SOURCE;
 
 	int BG_SNAPSHOT_FILE_TARGET_BYTES;
-	int BG_SNAPSHOT_FILE_TARGET_CHUNKS;
+	int BG_SNAPSHOT_FILE_TARGET_CHUNK_BYTES;
 	int BG_DELTA_FILE_TARGET_BYTES;
+	int BG_DELTA_FILE_TARGET_CHUNK_BYTES;
 	int BG_DELTA_BYTES_BEFORE_COMPACT;
 	int BG_MAX_SPLIT_FANOUT;
 	int BG_MAX_MERGE_FANIN;
@@ -892,18 +930,29 @@ public:
 	int BG_CONSISTENCY_CHECK_TARGET_SPEED_KB;
 	bool BG_ENABLE_MERGING;
 	int BG_MERGE_CANDIDATE_THRESHOLD_SECONDS;
+	int BG_MERGE_CANDIDATE_DELAY_SECONDS;
+	int BG_KEY_TUPLE_TRUNCATE_OFFSET;
 
 	int BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM;
+	int BLOB_WORKER_RESNAPSHOT_PARALLELISM;
+	int BLOB_WORKER_DELTA_FILE_WRITE_PARALLELISM;
+
 	double BLOB_WORKER_TIMEOUT; // Blob Manager's reaction time to a blob worker failure
 	double BLOB_WORKER_REQUEST_TIMEOUT; // Blob Worker's server-side request timeout
 	double BLOB_WORKERLIST_FETCH_INTERVAL;
 	double BLOB_WORKER_BATCH_GRV_INTERVAL;
+	bool BLOB_WORKER_DO_REJECT_WHEN_FULL;
+	double BLOB_WORKER_REJECT_WHEN_FULL_THRESHOLD;
+	double BLOB_WORKER_FORCE_FLUSH_CLEANUP_DELAY;
 
 	double BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN;
 	double BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX;
 	double BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT;
+	int BLOB_MANAGER_CONCURRENT_MERGE_CHECKS;
 	double BGCC_TIMEOUT;
 	double BGCC_MIN_INTERVAL;
+	bool BLOB_MANIFEST_BACKUP;
+	bool BLOB_FULL_RESTORE_MODE;
 
 	// Blob metadata
 	int64_t BLOB_METADATA_CACHE_TTL;
diff --git a/fdbclient/include/fdbclient/SpecialKeySpace.actor.h b/fdbclient/include/fdbclient/SpecialKeySpace.actor.h
index 3a2c7f6b83..d2ce7f5cf9 100644
--- a/fdbclient/include/fdbclient/SpecialKeySpace.actor.h
+++ b/fdbclient/include/fdbclient/SpecialKeySpace.actor.h
@@ -60,6 +60,8 @@ public:
 	// TODO : give this function a more descriptive name
 	virtual bool isAsync() const { return false; }
 
+	virtual bool supportsTenants() const { return false; }
+
 	virtual ~SpecialKeyRangeReadImpl() {}
 
 protected:
@@ -125,7 +127,7 @@ public:
 	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
 	                             KeyRangeRef kr,
 	                             GetRangeLimits limitsHint,
-	                             Optional<RangeResult>* cache) const {
+	                             KeyRangeMap<Optional<RangeResult>>* cache) const {
 		return getRangeAsyncActor(this, ryw, kr, limitsHint, cache);
 	}
 
@@ -135,17 +137,18 @@ public:
 	                                                    ReadYourWritesTransaction* ryw,
 	                                                    KeyRangeRef kr,
 	                                                    GetRangeLimits limits,
-	                                                    Optional<RangeResult>* cache) {
+	                                                    KeyRangeMap<Optional<RangeResult>>* cache) {
 		ASSERT(skrAyncImpl->getKeyRange().contains(kr));
 		ASSERT(cache != nullptr);
-		if (!cache->present()) {
+		ASSERT(cache->rangeContaining(kr.begin) == cache->rangeContainingKeyBefore(kr.end));
+		if (!(*cache)[kr.begin].present()) {
 			// For simplicity, every time we need to cache, we read the whole range
 			// Although sometimes the range can be narrowed,
 			// there is not a general way to do it in complicated scenarios
 			RangeResult result_ = wait(skrAyncImpl->getRange(ryw, skrAyncImpl->getKeyRange(), limits));
-			*cache = result_;
+			cache->insert(skrAyncImpl->getKeyRange(), result_);
 		}
-		const auto& allResults = cache->get();
+		const auto& allResults = (*cache)[kr.begin].get();
 		int start = 0, end = allResults.size();
 		while (start < allResults.size() && allResults[start].key < kr.begin)
 			++start;
@@ -271,15 +274,23 @@ private:
 };
 
 // Used for SpecialKeySpaceCorrectnessWorkload
-class SKSCTestImpl : public SpecialKeyRangeRWImpl {
+class SKSCTestRWImpl : public SpecialKeyRangeRWImpl {
 public:
-	explicit SKSCTestImpl(KeyRangeRef kr);
+	explicit SKSCTestRWImpl(KeyRangeRef kr);
 	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
 	                             KeyRangeRef kr,
 	                             GetRangeLimits limitsHint) const override;
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 };
 
+class SKSCTestAsyncReadImpl : public SpecialKeyRangeAsyncImpl {
+public:
+	explicit SKSCTestAsyncReadImpl(KeyRangeRef kr);
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
+};
+
 // Use special key prefix "\xff\xff/transaction/conflicting_keys/<some_key>",
 // to retrieve keys which caused latest not_committed(conflicting with another transaction) error.
 // The returned key value pairs are interpreted as :
@@ -292,6 +303,7 @@ public:
 	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
 	                             KeyRangeRef kr,
 	                             GetRangeLimits limitsHint) const override;
+	bool supportsTenants() const override { return true; };
 };
 
 class ReadConflictRangeImpl : public SpecialKeyRangeReadImpl {
@@ -300,6 +312,7 @@ public:
 	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
 	                             KeyRangeRef kr,
 	                             GetRangeLimits limitsHint) const override;
+	bool supportsTenants() const override { return true; };
 };
 
 class WriteConflictRangeImpl : public SpecialKeyRangeReadImpl {
@@ -308,6 +321,7 @@ public:
 	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
 	                             KeyRangeRef kr,
 	                             GetRangeLimits limitsHint) const override;
+	bool supportsTenants() const override { return true; };
 };
 
 class DDStatsRangeImpl : public SpecialKeyRangeAsyncImpl {
@@ -539,5 +553,24 @@ public:
 	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
 };
 
+class WorkerInterfacesSpecialKeyImpl : public SpecialKeyRangeReadImpl {
+public:
+	explicit WorkerInterfacesSpecialKeyImpl(KeyRangeRef kr);
+
+	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
+	                             KeyRangeRef kr,
+	                             GetRangeLimits limitsHint) const override;
+};
+
+// If the underlying set of key-value pairs of a key space is not changing, then we expect repeating a read to give the
+// same result. Additionally, we can generate the expected result of any read if that read is reading a subrange. This
+// actor performs a read of an arbitrary subrange of [begin, end) and validates the results.
+ACTOR Future<Void> validateSpecialSubrangeRead(ReadYourWritesTransaction* ryw,
+                                               KeySelector begin,
+                                               KeySelector end,
+                                               GetRangeLimits limits,
+                                               Reverse reverse,
+                                               RangeResult result);
+
 #include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbclient/include/fdbclient/StorageServerInterface.h b/fdbclient/include/fdbclient/StorageServerInterface.h
index febbc1311b..6b3e64d0aa 100644
--- a/fdbclient/include/fdbclient/StorageServerInterface.h
+++ b/fdbclient/include/fdbclient/StorageServerInterface.h
@@ -22,15 +22,16 @@
 #define FDBCLIENT_STORAGESERVERINTERFACE_H
 #pragma once
 
-#include <ostream>
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/StorageCheckpoint.h"
+#include "fdbclient/StorageServerShard.h"
 #include "fdbrpc/Locality.h"
 #include "fdbrpc/QueueModel.h"
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/LoadBalance.actor.h"
 #include "fdbrpc/Stats.h"
 #include "fdbrpc/TimedRequest.h"
+#include "fdbrpc/TenantInfo.h"
 #include "fdbrpc/TSSComparison.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/TagThrottle.actor.h"
@@ -53,6 +54,34 @@ struct VersionReply {
 	}
 };
 
+// This struct is used by RK to forward the commit cost to SS, see discussion in #7258
+struct UpdateCommitCostRequest {
+	constexpr static FileIdentifier file_identifier = 4159439;
+
+	// Ratekeeper ID, it is only reasonable to compare postTime from the same Ratekeeper
+	UID ratekeeperID;
+
+	// The time the request being posted
+	double postTime;
+
+	double elapsed;
+	TransactionTag busiestTag;
+
+	// Properties that are defined in TransactionCommitCostEstimation
+	int opsSum;
+	uint64_t costSum;
+
+	uint64_t totalWriteCosts;
+	bool reported;
+
+	ReplyPromise<Void> reply;
+
+	template <typename Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, ratekeeperID, postTime, elapsed, busiestTag, opsSum, costSum, totalWriteCosts, reported, reply);
+	}
+};
+
 struct StorageServerInterface {
 	constexpr static FileIdentifier file_identifier = 15302073;
 	enum { BUSY_ALLOWED = 0, BUSY_FORCE = 1, BUSY_LOCAL = 2 };
@@ -84,13 +113,15 @@ struct StorageServerInterface {
 	RequestStream<struct ReadHotSubRangeRequest> getReadHotRanges;
 	RequestStream<struct SplitRangeRequest> getRangeSplitPoints;
 	PublicRequestStream<struct GetKeyValuesStreamRequest> getKeyValuesStream;
-	PublicRequestStream<struct ChangeFeedStreamRequest> changeFeedStream;
-	PublicRequestStream<struct OverlappingChangeFeedsRequest> overlappingChangeFeeds;
-	PublicRequestStream<struct ChangeFeedPopRequest> changeFeedPop;
-	PublicRequestStream<struct ChangeFeedVersionUpdateRequest> changeFeedVersionUpdate;
-	PublicRequestStream<struct GetCheckpointRequest> checkpoint;
-	PublicRequestStream<struct FetchCheckpointRequest> fetchCheckpoint;
-	PublicRequestStream<struct FetchCheckpointKeyValuesRequest> fetchCheckpointKeyValues;
+	RequestStream<struct ChangeFeedStreamRequest> changeFeedStream;
+	RequestStream<struct OverlappingChangeFeedsRequest> overlappingChangeFeeds;
+	RequestStream<struct ChangeFeedPopRequest> changeFeedPop;
+	RequestStream<struct ChangeFeedVersionUpdateRequest> changeFeedVersionUpdate;
+	RequestStream<struct GetCheckpointRequest> checkpoint;
+	RequestStream<struct FetchCheckpointRequest> fetchCheckpoint;
+	RequestStream<struct FetchCheckpointKeyValuesRequest> fetchCheckpointKeyValues;
+
+	RequestStream<struct UpdateCommitCostRequest> updateCommitCostRequest;
 
 private:
 	bool acceptingRequests;
@@ -149,19 +180,20 @@ public:
 				getMappedKeyValues = PublicRequestStream<struct GetMappedKeyValuesRequest>(
 				    getValue.getEndpoint().getAdjustedEndpoint(14));
 				changeFeedStream =
-				    PublicRequestStream<struct ChangeFeedStreamRequest>(getValue.getEndpoint().getAdjustedEndpoint(15));
-				overlappingChangeFeeds = PublicRequestStream<struct OverlappingChangeFeedsRequest>(
-				    getValue.getEndpoint().getAdjustedEndpoint(16));
+				    RequestStream<struct ChangeFeedStreamRequest>(getValue.getEndpoint().getAdjustedEndpoint(15));
+				overlappingChangeFeeds =
+				    RequestStream<struct OverlappingChangeFeedsRequest>(getValue.getEndpoint().getAdjustedEndpoint(16));
 				changeFeedPop =
-				    PublicRequestStream<struct ChangeFeedPopRequest>(getValue.getEndpoint().getAdjustedEndpoint(17));
-				changeFeedVersionUpdate = PublicRequestStream<struct ChangeFeedVersionUpdateRequest>(
+				    RequestStream<struct ChangeFeedPopRequest>(getValue.getEndpoint().getAdjustedEndpoint(17));
+				changeFeedVersionUpdate = RequestStream<struct ChangeFeedVersionUpdateRequest>(
 				    getValue.getEndpoint().getAdjustedEndpoint(18));
-				checkpoint =
-				    PublicRequestStream<struct GetCheckpointRequest>(getValue.getEndpoint().getAdjustedEndpoint(19));
+				checkpoint = RequestStream<struct GetCheckpointRequest>(getValue.getEndpoint().getAdjustedEndpoint(19));
 				fetchCheckpoint =
-				    PublicRequestStream<struct FetchCheckpointRequest>(getValue.getEndpoint().getAdjustedEndpoint(20));
-				fetchCheckpointKeyValues = PublicRequestStream<struct FetchCheckpointKeyValuesRequest>(
+				    RequestStream<struct FetchCheckpointRequest>(getValue.getEndpoint().getAdjustedEndpoint(20));
+				fetchCheckpointKeyValues = RequestStream<struct FetchCheckpointKeyValuesRequest>(
 				    getValue.getEndpoint().getAdjustedEndpoint(21));
+				updateCommitCostRequest =
+				    RequestStream<struct UpdateCommitCostRequest>(getValue.getEndpoint().getAdjustedEndpoint(22));
 			}
 		} else {
 			ASSERT(Ar::isDeserializing);
@@ -212,6 +244,7 @@ public:
 		streams.push_back(checkpoint.getReceiver());
 		streams.push_back(fetchCheckpoint.getReceiver());
 		streams.push_back(fetchCheckpointKeyValues.getReceiver());
+		streams.push_back(updateCommitCostRequest.getReceiver());
 		FlowTransport::transport().addEndpoints(streams);
 	}
 };
@@ -241,21 +274,6 @@ struct ServerCacheInfo {
 	}
 };
 
-struct TenantInfo {
-	static const int64_t INVALID_TENANT = -1;
-
-	Optional<TenantName> name;
-	int64_t tenantId;
-
-	TenantInfo() : tenantId(INVALID_TENANT) {}
-	TenantInfo(TenantName name, int64_t tenantId) : name(name), tenantId(tenantId) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, name, tenantId);
-	}
-};
-
 struct GetValueReply : public LoadBalancedReply {
 	constexpr static FileIdentifier file_identifier = 1378929;
 	Optional<Value> value;
@@ -277,26 +295,28 @@ struct GetValueRequest : TimedRequest {
 	Key key;
 	Version version;
 	Optional<TagSet> tags;
-	Optional<UID> debugID;
 	ReplyPromise<GetValueReply> reply;
+	Optional<ReadOptions> options;
 	VersionVector ssLatestCommitVersions; // includes the latest commit versions, as known
 	                                      // to this client, of all storage replicas that
 	                                      // serve the given key
-
 	GetValueRequest() {}
+
+	bool verify() const { return tenantInfo.isAuthorized(); }
+
 	GetValueRequest(SpanContext spanContext,
 	                const TenantInfo& tenantInfo,
 	                const Key& key,
 	                Version ver,
 	                Optional<TagSet> tags,
-	                Optional<UID> debugID,
+	                Optional<ReadOptions> options,
 	                VersionVector latestCommitVersions)
-	  : spanContext(spanContext), tenantInfo(tenantInfo), key(key), version(ver), tags(tags), debugID(debugID),
+	  : spanContext(spanContext), tenantInfo(tenantInfo), key(key), version(ver), tags(tags), options(options),
 	    ssLatestCommitVersions(latestCommitVersions) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, key, version, tags, debugID, reply, spanContext, tenantInfo, ssLatestCommitVersions);
+		serializer(ar, key, version, tags, reply, spanContext, tenantInfo, options, ssLatestCommitVersions);
 	}
 };
 
@@ -337,6 +357,8 @@ struct WatchValueRequest {
 	  : spanContext(spanContext), tenantInfo(tenantInfo), key(key), value(value), version(ver), tags(tags),
 	    debugID(debugID) {}
 
+	bool verify() const { return tenantInfo.isAuthorized(); }
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, key, value, version, tags, debugID, reply, spanContext, tenantInfo);
@@ -370,15 +392,16 @@ struct GetKeyValuesRequest : TimedRequest {
 	KeyRef mapper = KeyRef();
 	Version version; // or latestVersion
 	int limit, limitBytes;
-	bool isFetchKeys;
 	Optional<TagSet> tags;
-	Optional<UID> debugID;
+	Optional<ReadOptions> options;
 	ReplyPromise<GetKeyValuesReply> reply;
 	VersionVector ssLatestCommitVersions; // includes the latest commit versions, as known
 	                                      // to this client, of all storage replicas that
 	                                      // serve the given key
 
-	GetKeyValuesRequest() : isFetchKeys(false) {}
+	GetKeyValuesRequest() {}
+
+	bool verify() const { return tenantInfo.isAuthorized(); }
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -388,14 +411,13 @@ struct GetKeyValuesRequest : TimedRequest {
 		           version,
 		           limit,
 		           limitBytes,
-		           isFetchKeys,
 		           tags,
-		           debugID,
 		           reply,
 		           spanContext,
 		           tenantInfo,
-		           arena,
-		           ssLatestCommitVersions);
+		           options,
+		           ssLatestCommitVersions,
+		           arena);
 	}
 };
 
@@ -427,15 +449,17 @@ struct GetMappedKeyValuesRequest : TimedRequest {
 	Version version; // or latestVersion
 	int limit, limitBytes;
 	int matchIndex;
-	bool isFetchKeys;
 	Optional<TagSet> tags;
-	Optional<UID> debugID;
+	Optional<ReadOptions> options;
 	ReplyPromise<GetMappedKeyValuesReply> reply;
 	VersionVector ssLatestCommitVersions; // includes the latest commit versions, as known
 	                                      // to this client, of all storage replicas that
 	                                      // serve the given key range
 
-	GetMappedKeyValuesRequest() : isFetchKeys(false) {}
+	GetMappedKeyValuesRequest() {}
+
+	bool verify() const { return tenantInfo.isAuthorized(); }
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar,
@@ -445,15 +469,14 @@ struct GetMappedKeyValuesRequest : TimedRequest {
 		           version,
 		           limit,
 		           limitBytes,
-		           isFetchKeys,
 		           tags,
-		           debugID,
 		           reply,
 		           spanContext,
 		           tenantInfo,
-		           arena,
+		           options,
 		           ssLatestCommitVersions,
-		           matchIndex);
+		           matchIndex,
+		           arena);
 	}
 };
 
@@ -492,15 +515,16 @@ struct GetKeyValuesStreamRequest {
 	KeySelectorRef begin, end;
 	Version version; // or latestVersion
 	int limit, limitBytes;
-	bool isFetchKeys;
 	Optional<TagSet> tags;
-	Optional<UID> debugID;
+	Optional<ReadOptions> options;
 	ReplyPromiseStream<GetKeyValuesStreamReply> reply;
 	VersionVector ssLatestCommitVersions; // includes the latest commit versions, as known
 	                                      // to this client, of all storage replicas that
 	                                      // serve the given key range
 
-	GetKeyValuesStreamRequest() : isFetchKeys(false) {}
+	GetKeyValuesStreamRequest() {}
+
+	bool verify() const { return tenantInfo.isAuthorized(); }
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -510,14 +534,13 @@ struct GetKeyValuesStreamRequest {
 		           version,
 		           limit,
 		           limitBytes,
-		           isFetchKeys,
 		           tags,
-		           debugID,
 		           reply,
 		           spanContext,
 		           tenantInfo,
-		           arena,
-		           ssLatestCommitVersions);
+		           options,
+		           ssLatestCommitVersions,
+		           arena);
 	}
 };
 
@@ -543,27 +566,29 @@ struct GetKeyRequest : TimedRequest {
 	KeySelectorRef sel;
 	Version version; // or latestVersion
 	Optional<TagSet> tags;
-	Optional<UID> debugID;
 	ReplyPromise<GetKeyReply> reply;
+	Optional<ReadOptions> options;
 	VersionVector ssLatestCommitVersions; // includes the latest commit versions, as known
 	                                      // to this client, of all storage replicas that
 	                                      // serve the given key
 
 	GetKeyRequest() {}
 
+	bool verify() const { return tenantInfo.isAuthorized(); }
+
 	GetKeyRequest(SpanContext spanContext,
 	              TenantInfo tenantInfo,
 	              KeySelectorRef const& sel,
 	              Version version,
 	              Optional<TagSet> tags,
-	              Optional<UID> debugID,
+	              Optional<ReadOptions> options,
 	              VersionVector latestCommitVersions)
-	  : spanContext(spanContext), tenantInfo(tenantInfo), sel(sel), version(version), debugID(debugID),
+	  : spanContext(spanContext), tenantInfo(tenantInfo), sel(sel), version(version), tags(tags), options(options),
 	    ssLatestCommitVersions(latestCommitVersions) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, sel, version, tags, debugID, reply, spanContext, tenantInfo, arena, ssLatestCommitVersions);
+		serializer(ar, sel, version, tags, reply, spanContext, tenantInfo, options, ssLatestCommitVersions, arena);
 	}
 };
 
@@ -572,12 +597,13 @@ struct GetShardStateReply {
 
 	Version first;
 	Version second;
+	std::vector<StorageServerShard> shards;
 	GetShardStateReply() = default;
 	GetShardStateReply(Version first, Version second) : first(first), second(second) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, first, second);
+		serializer(ar, first, second, shards);
 	}
 };
 
@@ -587,13 +613,16 @@ struct GetShardStateRequest {
 
 	KeyRange keys;
 	int32_t mode;
+	bool includePhysicalShard;
 	ReplyPromise<GetShardStateReply> reply;
-	GetShardStateRequest() {}
-	GetShardStateRequest(KeyRange const& keys, waitMode mode) : keys(keys), mode(mode) {}
+	GetShardStateRequest() = default;
+	GetShardStateRequest(KeyRange const& keys, waitMode mode, bool includePhysicalShard)
+	  : keys(keys), mode(mode), includePhysicalShard(includePhysicalShard) {}
+	GetShardStateRequest(KeyRange const& keys, waitMode mode) : keys(keys), mode(mode), includePhysicalShard(false) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, keys, mode, reply);
+		serializer(ar, keys, mode, reply, includePhysicalShard);
 	}
 };
 
@@ -605,7 +634,6 @@ struct StorageMetrics {
 	int64_t bytesPerKSecond = 0; // network bandwidth (average over 10s)
 	int64_t iosPerKSecond = 0;
 	int64_t bytesReadPerKSecond = 0;
-	Optional<KeyRange> keys; // this metric belongs to which range
 
 	static const int64_t infinity = 1LL << 60;
 
@@ -730,7 +758,7 @@ struct SplitMetricsRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, keys, limits, used, estimated, isLastShard, reply, arena, minSplitBytes);
+		serializer(ar, keys, limits, used, estimated, isLastShard, reply, minSplitBytes, arena);
 	}
 };
 
@@ -966,39 +994,51 @@ struct FetchCheckpointKeyValuesRequest {
 };
 
 struct OverlappingChangeFeedEntry {
-	Key rangeId;
-	KeyRange range;
+	KeyRef feedId;
+	KeyRangeRef range;
 	Version emptyVersion;
 	Version stopVersion;
+	Version feedMetadataVersion;
 
 	bool operator==(const OverlappingChangeFeedEntry& r) const {
-		return rangeId == r.rangeId && range == r.range && emptyVersion == r.emptyVersion &&
-		       stopVersion == r.stopVersion;
+		return feedId == r.feedId && range == r.range && emptyVersion == r.emptyVersion &&
+		       stopVersion == r.stopVersion && feedMetadataVersion == r.feedMetadataVersion;
 	}
 
 	OverlappingChangeFeedEntry() {}
-	OverlappingChangeFeedEntry(Key const& rangeId, KeyRange const& range, Version emptyVersion, Version stopVersion)
-	  : rangeId(rangeId), range(range), emptyVersion(emptyVersion), stopVersion(stopVersion) {}
+	OverlappingChangeFeedEntry(KeyRef const& feedId,
+	                           KeyRangeRef const& range,
+	                           Version emptyVersion,
+	                           Version stopVersion,
+	                           Version feedMetadataVersion)
+	  : feedId(feedId), range(range), emptyVersion(emptyVersion), stopVersion(stopVersion),
+	    feedMetadataVersion(feedMetadataVersion) {}
+
+	OverlappingChangeFeedEntry(Arena& arena, const OverlappingChangeFeedEntry& rhs)
+	  : feedId(arena, rhs.feedId), range(arena, rhs.range), emptyVersion(rhs.emptyVersion),
+	    stopVersion(rhs.stopVersion), feedMetadataVersion(rhs.feedMetadataVersion) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, rangeId, range, emptyVersion, stopVersion);
+		serializer(ar, feedId, range, emptyVersion, stopVersion, feedMetadataVersion);
 	}
 };
 
 struct OverlappingChangeFeedsReply {
 	constexpr static FileIdentifier file_identifier = 11815134;
-	std::vector<OverlappingChangeFeedEntry> rangeIds;
+	VectorRef<OverlappingChangeFeedEntry> feeds;
 	bool cached;
 	Arena arena;
+	Version feedMetadataVersion;
 
-	OverlappingChangeFeedsReply() : cached(false) {}
-	explicit OverlappingChangeFeedsReply(std::vector<OverlappingChangeFeedEntry> const& rangeIds)
-	  : rangeIds(rangeIds), cached(false) {}
+	OverlappingChangeFeedsReply() : cached(false), feedMetadataVersion(invalidVersion) {}
+	explicit OverlappingChangeFeedsReply(VectorRef<OverlappingChangeFeedEntry> const& feeds,
+	                                     Version feedMetadataVersion)
+	  : feeds(feeds), cached(false), feedMetadataVersion(feedMetadataVersion) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, rangeIds, arena);
+		serializer(ar, feeds, feedMetadataVersion, arena);
 	}
 };
 
diff --git a/fdbclient/include/fdbclient/StorageServerShard.h b/fdbclient/include/fdbclient/StorageServerShard.h
new file mode 100644
index 0000000000..dce0d5a6fa
--- /dev/null
+++ b/fdbclient/include/fdbclient/StorageServerShard.h
@@ -0,0 +1,88 @@
+/*
+ * StorageServerShard.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBCLIENT_STORAGESERVERSHARD_H
+#define FDBCLIENT_STORAGESERVERSHARD_H
+#pragma once
+
+#include "fdbclient/FDBTypes.h"
+#include "flow/flow.h"
+
+// Represents a data shard on a storage server hosting a continuous keyrange.
+struct StorageServerShard {
+	constexpr static FileIdentifier file_identifier = 4028358;
+
+	enum ShardState {
+		NotAssigned = 0,
+		MovingIn = 1,
+		ReadWritePending = 2,
+		ReadWrite = 3,
+	};
+
+	StorageServerShard() = default;
+	StorageServerShard(KeyRange range,
+	                   Version version,
+	                   const uint64_t id,
+	                   const uint64_t desiredId,
+	                   ShardState shardState)
+	  : range(range), version(version), id(id), desiredId(desiredId), shardState(shardState) {}
+
+	static StorageServerShard notAssigned(KeyRange range, Version version = 0) {
+		return StorageServerShard(range, version, 0, 0, NotAssigned);
+	}
+
+	ShardState getShardState() const { return static_cast<ShardState>(this->shardState); };
+
+	void setShardState(const ShardState shardState) { this->shardState = static_cast<int8_t>(shardState); }
+
+	std::string getShardStateString() const {
+		const ShardState ss = getShardState();
+		switch (ss) {
+		case NotAssigned:
+			return "NotAssigned";
+		case MovingIn:
+			return "MovingIn";
+		case ReadWritePending:
+			return "ReadWritePending";
+		case ReadWrite:
+			return "ReadWrite";
+		}
+		return "InvalidState";
+	}
+
+	std::string toString() const {
+		return "StorageServerShard: [Range]: " + Traceable<KeyRangeRef>::toString(range) +
+		       " [Shard ID]: " + format("%016llx", this->id) + " [Version]: " + std::to_string(version) +
+		       " [State]: " + getShardStateString() + " [Desired Shard ID]: " + format("%016llx", this->desiredId);
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, range, version, id, desiredId, shardState);
+	}
+
+	KeyRange range;
+	Version version; // Shard creation version.
+	uint64_t id; // The actual shard ID.
+	uint64_t desiredId; // The intended shard ID.
+	int8_t shardState;
+};
+
+#endif
diff --git a/fdbclient/include/fdbclient/Subspace.h b/fdbclient/include/fdbclient/Subspace.h
index ef88fc3855..0cc85089d1 100644
--- a/fdbclient/include/fdbclient/Subspace.h
+++ b/fdbclient/include/fdbclient/Subspace.h
@@ -42,9 +42,7 @@ public:
 
 	template <class T>
 	Key pack(T const& item) const {
-		Tuple t;
-		t.append(item);
-		return pack(t);
+		return pack(Tuple::makeTuple(item));
 	}
 
 	Key pack(StringRef const& item, bool utf8 = false) const {
@@ -58,9 +56,7 @@ public:
 
 	template <class T>
 	Subspace get(T const& item) const {
-		Tuple t;
-		t.append(item);
-		return get(t);
+		return get(Tuple::makeTuple(item));
 	}
 
 	Subspace get(StringRef const& item, bool utf8 = false) const {
diff --git a/fdbclient/include/fdbclient/SystemData.h b/fdbclient/include/fdbclient/SystemData.h
index 370c1e1f29..068d1d9d37 100644
--- a/fdbclient/include/fdbclient/SystemData.h
+++ b/fdbclient/include/fdbclient/SystemData.h
@@ -28,7 +28,7 @@
 #include "fdbclient/BlobGranuleCommon.h"
 #include "fdbclient/BlobWorkerInterface.h" // TODO move the functions that depend on this out of here and into BlobWorkerInterface.h to remove this depdendency
 #include "fdbclient/StorageServerInterface.h"
-#include "Tenant.h"
+#include "fdbclient/Tenant.h"
 
 // Don't warn on constants being defined in this file.
 #pragma clang diagnostic push
@@ -163,6 +163,9 @@ extern const KeyRef cacheChangePrefix;
 const Key cacheChangeKeyFor(uint16_t idx);
 uint16_t cacheChangeKeyDecodeIndex(const KeyRef& key);
 
+// For persisting the consistency scan configuration and metrics
+extern const KeyRef consistencyScanInfoKey;
+
 // "\xff/tss/[[serverId]]" := "[[tssId]]"
 extern const KeyRangeRef tssMappingKeys;
 
@@ -273,6 +276,9 @@ extern const KeyRef perpetualStorageWiggleStatsPrefix;
 // Change the value of this key to anything and that will trigger detailed data distribution team info log.
 extern const KeyRef triggerDDTeamInfoPrintKey;
 
+// Encryption data at-rest config key
+extern const KeyRef encryptionAtRestModeConfKey;
+
 //	The differences between excluded and failed can be found in "command-line-interface.rst"
 //	and in the help message of the fdbcli command "exclude".
 
@@ -374,6 +380,12 @@ std::vector<std::pair<UID, Version>> decodeBackupStartedValue(const ValueRef& va
 // 1 = Send a signal to pause/already paused.
 extern const KeyRef backupPausedKey;
 
+//	"\xff/previousCoordinators" = "[[ClusterConnectionString]]"
+//	Set to the encoded structure of the cluster's previous set of coordinators.
+//	Changed when performing quorumChange.
+//	See "CoordinationInterface.h" struct ClusterConnectionString for more details
+extern const KeyRef previousCoordinatorsKey;
+
 //	"\xff/coordinators" = "[[ClusterConnectionString]]"
 //	Set to the encoded structure of the cluster's current set of coordinators.
 //	Changed when performing quorumChange.
@@ -594,6 +606,8 @@ const Value blobManagerEpochValueFor(int64_t epoch);
 int64_t decodeBlobManagerEpochValue(ValueRef const& value);
 
 // blob granule keys
+extern const StringRef blobRangeActive;
+extern const StringRef blobRangeInactive;
 
 extern const uint8_t BG_FILE_TYPE_DELTA;
 extern const uint8_t BG_FILE_TYPE_SNAPSHOT;
@@ -613,12 +627,16 @@ extern const KeyRangeRef blobGranuleSplitKeys;
 // \xff\x02/bgmerge/mergeGranuleId = [[BlobGranuleMergeState]]
 extern const KeyRangeRef blobGranuleMergeKeys;
 
+// \xff\x02/bgmergebounds/beginkey = [[BlobGranuleMergeBoundary]]
+extern const KeyRangeRef blobGranuleMergeBoundaryKeys;
+
 // \xff\x02/bgh/(beginKey,endKey,startVersion) = { granuleUID, [parentGranuleHistoryKeys] }
 extern const KeyRangeRef blobGranuleHistoryKeys;
 
 // \xff\x02/bgp/(start,end) = (version, force)
 extern const KeyRangeRef blobGranulePurgeKeys;
-extern const KeyRangeRef blobGranuleVersionKeys;
+// \xff\x02/bgpforce/(start) = {1|0} (key range map)
+extern const KeyRangeRef blobGranuleForcePurgedKeys;
 extern const KeyRef blobGranulePurgeChangeKey;
 
 const Key blobGranuleFileKeyFor(UID granuleID, Version fileVersion, uint8_t fileType);
@@ -658,11 +676,16 @@ std::pair<BlobGranuleSplitState, Version> decodeBlobGranuleSplitValue(ValueRef c
 
 const Value blobGranuleMergeValueFor(KeyRange mergeKeyRange,
                                      std::vector<UID> parentGranuleIDs,
-                                     std::vector<KeyRange> parentGranuleRanges,
+                                     std::vector<Key> parentGranuleRanges,
                                      std::vector<Version> parentGranuleStartVersions);
 // FIXME: probably just define object type for this?
-std::tuple<KeyRange, Version, std::vector<UID>, std::vector<KeyRange>, std::vector<Version>>
-decodeBlobGranuleMergeValue(ValueRef const& value);
+std::tuple<KeyRange, Version, std::vector<UID>, std::vector<Key>, std::vector<Version>> decodeBlobGranuleMergeValue(
+    ValueRef const& value);
+
+// BlobGranuleMergeBoundary.
+const Key blobGranuleMergeBoundaryKeyFor(const KeyRef& key);
+const Value blobGranuleMergeBoundaryValueFor(BlobGranuleMergeBoundary const& boundary);
+Standalone<BlobGranuleMergeBoundary> decodeBlobGranuleMergeBoundaryValue(const ValueRef& value);
 
 const Key blobGranuleHistoryKeyFor(KeyRangeRef const& range, Version version);
 std::pair<KeyRange, Version> decodeBlobGranuleHistoryKey(KeyRef const& key);
@@ -679,12 +702,11 @@ UID decodeBlobWorkerListKey(KeyRef const& key);
 const Value blobWorkerListValue(BlobWorkerInterface const& interface);
 BlobWorkerInterface decodeBlobWorkerListValue(ValueRef const& value);
 
-// State for the tenant map
-extern const KeyRangeRef tenantMapKeys;
-extern const KeyRef tenantMapPrefix;
-extern const KeyRef tenantMapPrivatePrefix;
-extern const KeyRef tenantLastIdKey;
-extern const KeyRef tenantDataPrefixKey;
+// Storage quota per tenant
+// "\xff/storageQuota/[[tenantName]]" := "[[quota]]"
+extern const KeyRangeRef storageQuotaKeys;
+extern const KeyRef storageQuotaPrefix;
+Key storageQuotaKey(StringRef tenantName);
 
 #pragma clang diagnostic pop
 
diff --git a/fdbclient/include/fdbclient/TagThrottle.actor.h b/fdbclient/include/fdbclient/TagThrottle.actor.h
index 020fcea568..cf4ef8fd16 100644
--- a/fdbclient/include/fdbclient/TagThrottle.actor.h
+++ b/fdbclient/include/fdbclient/TagThrottle.actor.h
@@ -264,9 +264,9 @@ Future<bool> getValidAutoEnabled(Reference<Tr> tr) {
 			tr->reset();
 			wait(delay(CLIENT_KNOBS->DEFAULT_BACKOFF));
 			continue;
-		} else if (value.get() == LiteralStringRef("1")) {
+		} else if (value.get() == "1"_sr) {
 			result = true;
-		} else if (value.get() == LiteralStringRef("0")) {
+		} else if (value.get() == "0"_sr) {
 			result = false;
 		} else {
 			TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue").detail("Value", value.get());
@@ -331,8 +331,7 @@ getThrottledTags(Reference<DB> db, int limit, ContainsRecommended containsRecomm
 
 template <class Tr>
 void signalThrottleChange(Reference<Tr> tr) {
-	tr->atomicOp(
-	    tagThrottleSignalKey, LiteralStringRef("XXXXXXXXXX\x00\x00\x00\x00"), MutationRef::SetVersionstampedValue);
+	tr->atomicOp(tagThrottleSignalKey, "XXXXXXXXXX\x00\x00\x00\x00"_sr, MutationRef::SetVersionstampedValue);
 }
 
 ACTOR template <class Tr>
@@ -583,9 +582,8 @@ Future<Void> enableAuto(Reference<DB> db, bool enabled) {
 			state typename DB::TransactionT::template FutureT<Optional<Value>> valueF =
 			    tr->get(tagThrottleAutoEnabledKey);
 			Optional<Value> value = wait(safeThreadFutureToFuture(valueF));
-			if (!value.present() || (enabled && value.get() != LiteralStringRef("1")) ||
-			    (!enabled && value.get() != LiteralStringRef("0"))) {
-				tr->set(tagThrottleAutoEnabledKey, LiteralStringRef(enabled ? "1" : "0"));
+			if (!value.present() || (enabled && value.get() != "1"_sr) || (!enabled && value.get() != "0"_sr)) {
+				tr->set(tagThrottleAutoEnabledKey, enabled ? "1"_sr : "0"_sr);
 				signalThrottleChange<typename DB::TransactionT>(tr);
 
 				wait(safeThreadFutureToFuture(tr->commit()));
@@ -599,10 +597,8 @@ Future<Void> enableAuto(Reference<DB> db, bool enabled) {
 
 class TagQuotaValue {
 public:
-	double reservedReadQuota{ 0.0 };
-	double totalReadQuota{ 0.0 };
-	double reservedWriteQuota{ 0.0 };
-	double totalWriteQuota{ 0.0 };
+	double reservedQuota{ 0.0 };
+	double totalQuota{ 0.0 };
 	bool isValid() const;
 	Value toValue() const;
 	static TagQuotaValue fromValue(ValueRef);
@@ -611,17 +607,10 @@ public:
 Key getTagQuotaKey(TransactionTagRef);
 
 template <class Tr>
-void setTagQuota(Reference<Tr> tr,
-                 TransactionTagRef tag,
-                 double reservedReadQuota,
-                 double totalReadQuota,
-                 double reservedWriteQuota,
-                 double totalWriteQuota) {
+void setTagQuota(Reference<Tr> tr, TransactionTagRef tag, double reservedQuota, double totalQuota) {
 	TagQuotaValue tagQuotaValue;
-	tagQuotaValue.reservedReadQuota = reservedReadQuota;
-	tagQuotaValue.totalReadQuota = totalReadQuota;
-	tagQuotaValue.reservedWriteQuota = reservedWriteQuota;
-	tagQuotaValue.totalWriteQuota = totalWriteQuota;
+	tagQuotaValue.reservedQuota = reservedQuota;
+	tagQuotaValue.totalQuota = totalQuota;
 	if (!tagQuotaValue.isValid()) {
 		throw invalid_throttle_quota_value();
 	}
diff --git a/fdbclient/include/fdbclient/TaskBucket.h b/fdbclient/include/fdbclient/TaskBucket.h
index b7e6091d0f..b0b7a2bc51 100644
--- a/fdbclient/include/fdbclient/TaskBucket.h
+++ b/fdbclient/include/fdbclient/TaskBucket.h
@@ -115,7 +115,7 @@ public:
 };
 
 struct ReservedTaskParams {
-	static TaskParam<Version> scheduledVersion() { return LiteralStringRef(__FUNCTION__); }
+	static TaskParam<Version> scheduledVersion() { return __FUNCTION__sr; }
 };
 
 class FutureBucket;
@@ -480,7 +480,8 @@ struct TaskFuncBase : IDispatched<TaskFuncBase, Standalone<StringRef>, std::func
 };
 #define REGISTER_TASKFUNC(TaskFunc) REGISTER_FACTORY(TaskFuncBase, TaskFunc, name)
 #define REGISTER_TASKFUNC_ALIAS(TaskFunc, Alias)                                                                       \
-	REGISTER_DISPATCHED_ALIAS(TaskFunc, Alias, TaskFunc::name, LiteralStringRef(#Alias))
+	REGISTER_DISPATCHED_ALIAS(                                                                                         \
+	    TaskFunc, Alias, TaskFunc::name, StringRef(reinterpret_cast<const uint8_t*>(#Alias), sizeof(#Alias) - 1))
 
 struct TaskCompletionKey {
 	Future<Key> get(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket);
diff --git a/fdbclient/include/fdbclient/Tenant.h b/fdbclient/include/fdbclient/Tenant.h
index baac9f4543..47d27a0f72 100644
--- a/fdbclient/include/fdbclient/Tenant.h
+++ b/fdbclient/include/fdbclient/Tenant.h
@@ -23,58 +23,191 @@
 #pragma once
 
 #include "fdbclient/FDBTypes.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/VersionedMap.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbrpc/TenantInfo.h"
+#include "flow/BooleanParam.h"
 #include "flow/flat_buffers.h"
 
 typedef StringRef TenantNameRef;
 typedef Standalone<TenantNameRef> TenantName;
+typedef StringRef TenantGroupNameRef;
+typedef Standalone<TenantGroupNameRef> TenantGroupName;
+
+// Represents the various states that a tenant could be in.
+// In a standalone cluster, a tenant should only ever be in the READY state.
+// In a metacluster, a tenant on the management cluster could be in the other states while changes are applied to the
+// data cluster.
+//
+// REGISTERING - the tenant has been created on the management cluster and is being created on the data cluster
+// READY - the tenant has been created on both clusters, is active, and is consistent between the two clusters
+// REMOVING - the tenant has been marked for removal and is being removed on the data cluster
+// UPDATING_CONFIGURATION - the tenant configuration has changed on the management cluster and is being applied to the
+//                          data cluster
+// RENAMING_FROM - the tenant is being renamed to a new name and is awaiting the rename to complete on the data cluster
+// RENAMING_TO - the tenant is being created as a rename from an existing tenant and is awaiting the rename to complete
+//               on the data cluster
+// ERROR - the tenant is in an error state
+//
+// A tenant in any configuration is allowed to be removed. Only tenants in the READY or UPDATING_CONFIGURATION phases
+// can have their configuration updated. A tenant must not exist or be in the REGISTERING phase to be created. To be
+// renamed, a tenant must be in the READY or RENAMING_FROM state. In the latter case, the rename destination must match
+// the original rename attempt.
+//
+// If an operation fails and the tenant is left in a non-ready state, re-running the same operation is legal. If
+// successful, the tenant will return to the READY state.
+enum class TenantState { REGISTERING, READY, REMOVING, UPDATING_CONFIGURATION, RENAMING_FROM, RENAMING_TO, ERROR };
+
+// Represents the lock state the tenant could be in.
+// Can be used in conjunction with the other tenant states above.
+enum class TenantLockState { UNLOCKED, READ_ONLY, LOCKED };
+
+constexpr int TENANT_PREFIX_SIZE = sizeof(int64_t);
+
+FDB_DECLARE_BOOLEAN_PARAM(EnforceValidTenantId);
 
 struct TenantMapEntry {
 	constexpr static FileIdentifier file_identifier = 12247338;
 
 	static Key idToPrefix(int64_t id);
-	static int64_t prefixToId(KeyRef prefix);
+	static int64_t prefixToId(KeyRef prefix, EnforceValidTenantId enforceTenantId = EnforceValidTenantId::True);
 
-	int64_t id;
+	static std::string tenantStateToString(TenantState tenantState);
+	static TenantState stringToTenantState(std::string stateStr);
+
+	static std::string tenantLockStateToString(TenantLockState tenantState);
+	static TenantLockState stringToTenantLockState(std::string stateStr);
+
+	int64_t id = -1;
 	Key prefix;
+	TenantState tenantState = TenantState::READY;
+	TenantLockState tenantLockState = TenantLockState::UNLOCKED;
+	Optional<TenantGroupName> tenantGroup;
+	bool encrypted = false;
+	Optional<ClusterName> assignedCluster;
+	int64_t configurationSequenceNum = 0;
+	Optional<TenantName> renamePair;
 
-	constexpr static int ROOT_PREFIX_SIZE = sizeof(id);
+	// Can be set to an error string if the tenant is in the ERROR state
+	std::string error;
 
-private:
-	void initPrefix(KeyRef subspace);
+	constexpr static int PREFIX_SIZE = sizeof(id);
 
-public:
 	TenantMapEntry();
-	TenantMapEntry(int64_t id, KeyRef subspace);
+	TenantMapEntry(int64_t id, TenantState tenantState, bool encrypted);
+	TenantMapEntry(int64_t id, TenantState tenantState, Optional<TenantGroupName> tenantGroup, bool encrypted);
 
-	Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion(ProtocolVersion::withTenants())); }
+	void setId(int64_t id);
+	std::string toJson() const;
 
+	bool matchesConfiguration(TenantMapEntry const& other) const;
+	void configure(Standalone<StringRef> parameter, Optional<Value> value);
+
+	Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion()); }
 	static TenantMapEntry decode(ValueRef const& value) {
-		TenantMapEntry entry;
-		ObjectReader reader(value.begin(), IncludeVersion(ProtocolVersion::withTenants()));
-		reader.deserialize(entry);
-		return entry;
+		return ObjectReader::fromStringRef<TenantMapEntry>(value, IncludeVersion());
 	}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		KeyRef subspace;
-		if (ar.isDeserializing) {
-			serializer(ar, id, subspace);
+		serializer(ar,
+		           id,
+		           tenantState,
+		           tenantLockState,
+		           tenantGroup,
+		           encrypted,
+		           assignedCluster,
+		           configurationSequenceNum,
+		           renamePair,
+		           error);
+		if constexpr (Ar::isDeserializing) {
 			if (id >= 0) {
-				initPrefix(subspace);
+				prefix = idToPrefix(id);
 			}
-		} else {
-			ASSERT(prefix.size() >= 8 || (prefix.empty() && id == -1));
-			if (!prefix.empty()) {
-				subspace = prefix.substr(0, prefix.size() - 8);
-			}
-			serializer(ar, id, subspace);
+			ASSERT(tenantState >= TenantState::REGISTERING && tenantState <= TenantState::ERROR);
 		}
 	}
 };
 
-typedef VersionedMap<TenantName, TenantMapEntry> TenantMap;
-typedef VersionedMap<Key, TenantName> TenantPrefixIndex;
+struct TenantGroupEntry {
+	constexpr static FileIdentifier file_identifier = 10764222;
 
-#endif
\ No newline at end of file
+	Optional<ClusterName> assignedCluster;
+
+	TenantGroupEntry() = default;
+	TenantGroupEntry(Optional<ClusterName> assignedCluster) : assignedCluster(assignedCluster) {}
+
+	json_spirit::mObject toJson() const;
+
+	Value encode() { return ObjectWriter::toValue(*this, IncludeVersion()); }
+	static TenantGroupEntry decode(ValueRef const& value) {
+		return ObjectReader::fromStringRef<TenantGroupEntry>(value, IncludeVersion());
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, assignedCluster);
+	}
+};
+
+struct TenantTombstoneCleanupData {
+	constexpr static FileIdentifier file_identifier = 3291339;
+
+	// All tombstones have been erased up to and including this id.
+	// We should not generate new tombstones at IDs equal to or older than this.
+	int64_t tombstonesErasedThrough = -1;
+
+	// The version at which we will next erase tombstones.
+	Version nextTombstoneEraseVersion = invalidVersion;
+
+	// When we reach the nextTombstoneEraseVersion, we will erase tombstones up through this ID.
+	int64_t nextTombstoneEraseId = -1;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, tombstonesErasedThrough, nextTombstoneEraseVersion, nextTombstoneEraseId);
+	}
+};
+
+struct TenantMetadataSpecification {
+	Key subspace;
+
+	KeyBackedObjectMap<TenantName, TenantMapEntry, decltype(IncludeVersion()), NullCodec> tenantMap;
+	KeyBackedMap<int64_t, TenantName> tenantIdIndex;
+	KeyBackedProperty<int64_t> lastTenantId;
+	KeyBackedBinaryValue<int64_t> tenantCount;
+	KeyBackedSet<int64_t> tenantTombstones;
+	KeyBackedObjectProperty<TenantTombstoneCleanupData, decltype(IncludeVersion())> tombstoneCleanupData;
+	KeyBackedSet<Tuple> tenantGroupTenantIndex;
+	KeyBackedObjectMap<TenantGroupName, TenantGroupEntry, decltype(IncludeVersion()), NullCodec> tenantGroupMap;
+
+	TenantMetadataSpecification(KeyRef prefix)
+	  : subspace(prefix.withSuffix("tenant/"_sr)), tenantMap(subspace.withSuffix("map/"_sr), IncludeVersion()),
+	    tenantIdIndex(subspace.withSuffix("idIndex/"_sr)), lastTenantId(subspace.withSuffix("lastId"_sr)),
+	    tenantCount(subspace.withSuffix("count"_sr)), tenantTombstones(subspace.withSuffix("tombstones/"_sr)),
+	    tombstoneCleanupData(subspace.withSuffix("tombstoneCleanup"_sr), IncludeVersion()),
+	    tenantGroupTenantIndex(subspace.withSuffix("tenantGroup/tenantIndex/"_sr)),
+	    tenantGroupMap(subspace.withSuffix("tenantGroup/map/"_sr), IncludeVersion()) {}
+};
+
+struct TenantMetadata {
+	static TenantMetadataSpecification& instance();
+
+	static inline auto& subspace() { return instance().subspace; }
+	static inline auto& tenantMap() { return instance().tenantMap; }
+	static inline auto& tenantIdIndex() { return instance().tenantIdIndex; }
+	static inline auto& lastTenantId() { return instance().lastTenantId; }
+	static inline auto& tenantCount() { return instance().tenantCount; }
+	static inline auto& tenantTombstones() { return instance().tenantTombstones; }
+	static inline auto& tombstoneCleanupData() { return instance().tombstoneCleanupData; }
+	static inline auto& tenantGroupTenantIndex() { return instance().tenantGroupTenantIndex; }
+	static inline auto& tenantGroupMap() { return instance().tenantGroupMap; }
+
+	static Key tenantMapPrivatePrefix();
+};
+
+typedef VersionedMap<TenantName, TenantMapEntry> TenantMap;
+class TenantPrefixIndex : public VersionedMap<Key, TenantName>, public ReferenceCounted<TenantPrefixIndex> {};
+
+#endif
diff --git a/fdbclient/include/fdbclient/TenantEntryCache.actor.h b/fdbclient/include/fdbclient/TenantEntryCache.actor.h
new file mode 100644
index 0000000000..cd35c5a985
--- /dev/null
+++ b/fdbclient/include/fdbclient/TenantEntryCache.actor.h
@@ -0,0 +1,390 @@
+/*
+ * TenantEntryCache.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_TENANTENTRYCACHE_ACTOR_G_H)
+#define FDBCLIENT_TENANTENTRYCACHE_ACTOR_G_H
+#include "fdbclient/TenantEntryCache.actor.g.h"
+#elif !defined(FDBCLIENT_TENANTENTRYCACHE_ACTOR_H)
+#define FDBCLIENT_TENANTENTRYCACHE_ACTOR_H
+
+#pragma once
+
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/RunTransaction.actor.h"
+#include "fdbclient/Tenant.h"
+#include "fdbclient/TenantManagement.actor.h"
+#include "fdbclient/Knobs.h"
+#include "fdbrpc/TenantName.h"
+#include "flow/IndexedSet.h"
+
+#include <functional>
+#include <unordered_map>
+
+#include "flow/actorcompiler.h" // has to be last include
+
+using TenantNameEntryPair = std::pair<TenantName, TenantMapEntry>;
+using TenantNameEntryPairVec = std::vector<TenantNameEntryPair>;
+
+enum class TenantEntryCacheRefreshReason { INIT = 1, PERIODIC_TASK = 2, CACHE_MISS = 3, REMOVE_ENTRY = 4 };
+enum class TenantEntryCacheRefreshMode { PERIODIC_TASK = 1, NONE = 2 };
+
+template <class T>
+struct TenantEntryCachePayload {
+	TenantName name;
+	TenantMapEntry entry;
+	// Custom client payload
+	T payload;
+};
+
+template <class T>
+using TenantEntryCachePayloadFunc = std::function<TenantEntryCachePayload<T>(const TenantName&, const TenantMapEntry&)>;
+
+// In-memory cache for TenantEntryMap objects. It supports three indices:
+// 1. Lookup by 'TenantId'
+// 2. Lookup by 'TenantPrefix'
+// 3. Lookup by 'TenantName'
+//
+// TODO:
+// ----
+// The cache allows user to construct the 'cached object' by supplying a callback. The cache implements a periodic
+// refresh mechanism, polling underlying database for updates (add/remove tenants), in future we might want to implement
+// database range-watch to monitor such updates
+
+template <class T>
+class TenantEntryCache : public ReferenceCounted<TenantEntryCache<T>>, NonCopyable {
+private:
+	UID uid;
+	Database db;
+	TenantEntryCachePayloadFunc<T> createPayloadFunc;
+	TenantEntryCacheRefreshMode refreshMode;
+
+	Future<Void> refresher;
+	Map<int64_t, TenantEntryCachePayload<T>> mapByTenantId;
+	Map<TenantName, TenantEntryCachePayload<T>> mapByTenantName;
+
+	CounterCollection metrics;
+	Counter hits;
+	Counter misses;
+	Counter refreshByCacheInit;
+	Counter refreshByCacheMiss;
+	Counter numRefreshes;
+
+	ACTOR static Future<TenantNameEntryPairVec> getTenantList(Reference<ReadYourWritesTransaction> tr) {
+		tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+		tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+
+		KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenantList =
+		    wait(TenantMetadata::tenantMap().getRange(
+		        tr, Optional<TenantName>(), Optional<TenantName>(), CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1));
+		ASSERT(tenantList.results.size() <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER && !tenantList.more);
+
+		TraceEvent(SevDebug, "TenantEntryCacheGetTenantList").detail("Count", tenantList.results.size());
+
+		return tenantList.results;
+	}
+
+	static void updateCacheRefreshMetrics(TenantEntryCache<T>* cache, TenantEntryCacheRefreshReason reason) {
+		if (reason == TenantEntryCacheRefreshReason::INIT) {
+			cache->refreshByCacheInit += 1;
+		} else if (reason == TenantEntryCacheRefreshReason::CACHE_MISS) {
+			cache->refreshByCacheMiss += 1;
+		}
+
+		cache->numRefreshes += 1;
+	}
+
+	ACTOR static Future<Void> refreshImpl(TenantEntryCache<T>* cache, TenantEntryCacheRefreshReason reason) {
+		TraceEvent(SevDebug, "TenantEntryCacheRefreshStart", cache->id()).detail("Reason", static_cast<int>(reason));
+
+		state Reference<ReadYourWritesTransaction> tr = cache->getDatabase()->createTransaction();
+		loop {
+			try {
+				state TenantNameEntryPairVec tenantList = wait(getTenantList(tr));
+
+				// Refresh cache entries reflecting the latest database state
+				cache->clear();
+				for (auto& tenant : tenantList) {
+					cache->put(tenant);
+				}
+
+				updateCacheRefreshMetrics(cache, reason);
+				break;
+			} catch (Error& e) {
+				if (e.code() != error_code_actor_cancelled) {
+					TraceEvent(SevInfo, "TenantEntryCacheRefreshError", cache->id())
+					    .errorUnsuppressed(e)
+					    .suppressFor(1.0);
+				}
+				wait(tr->onError(e));
+			}
+		}
+
+		TraceEvent(SevDebug, "TenantEntryCacheRefreshEnd", cache->id()).detail("Reason", static_cast<int>(reason));
+
+		return Void();
+	}
+
+	ACTOR static Future<Optional<TenantEntryCachePayload<T>>> getByIdImpl(TenantEntryCache<T>* cache,
+	                                                                      int64_t tenantId) {
+		Optional<TenantEntryCachePayload<T>> ret = cache->lookupById(tenantId);
+		if (ret.present()) {
+			cache->hits += 1;
+			return ret;
+		}
+
+		TraceEvent(SevInfo, "TenantEntryCacheGetByIdRefresh").detail("TenantId", tenantId);
+
+		// Entry not found. Refresh cacheEntries by scanning underlying KeyRange.
+		// TODO: Cache will implement a "KeyRange" watch, monitoring notification when a new entry gets added or any
+		// existing entry gets updated within the KeyRange of interest. Hence, misses would be very rare
+		wait(refreshImpl(cache, TenantEntryCacheRefreshReason::CACHE_MISS));
+
+		cache->misses += 1;
+		return cache->lookupById(tenantId);
+	}
+
+	ACTOR static Future<Optional<TenantEntryCachePayload<T>>> getByNameImpl(TenantEntryCache<T>* cache,
+	                                                                        TenantName name) {
+		Optional<TenantEntryCachePayload<T>> ret = cache->lookupByName(name);
+		if (ret.present()) {
+			cache->hits += 1;
+			return ret;
+		}
+
+		TraceEvent("TenantEntryCacheGetByNameRefresh").detail("TenantName", name);
+
+		// Entry not found. Refresh cacheEntries by scanning underlying KeyRange.
+		// TODO: Cache will implement a "KeyRange" watch, monitoring notification when a new entry gets added or any
+		// existing entry gets updated within the KeyRange of interest. Hence, misses would be very rare
+		wait(refreshImpl(cache, TenantEntryCacheRefreshReason::CACHE_MISS));
+
+		cache->misses += 1;
+		return cache->lookupByName(name);
+	}
+
+	Optional<TenantEntryCachePayload<T>> lookupById(int64_t tenantId) {
+		Optional<TenantEntryCachePayload<T>> ret;
+		auto itr = mapByTenantId.find(tenantId);
+		if (itr == mapByTenantId.end()) {
+			return ret;
+		}
+
+		return itr->value;
+	}
+
+	Optional<TenantEntryCachePayload<T>> lookupByName(TenantName name) {
+		Optional<TenantEntryCachePayload<T>> ret;
+		auto itr = mapByTenantName.find(name);
+		if (itr == mapByTenantName.end()) {
+			return ret;
+		}
+
+		return itr->value;
+	}
+
+	Future<Void> refresh(TenantEntryCacheRefreshReason reason) { return refreshImpl(this, reason); }
+
+	static TenantEntryCachePayload<Void> defaultCreatePayload(const TenantName& name, const TenantMapEntry& entry) {
+		TenantEntryCachePayload<Void> payload;
+		payload.name = name;
+		payload.entry = entry;
+
+		return payload;
+	}
+
+	Future<Void> removeEntryInt(Optional<int64_t> tenantId,
+	                            Optional<KeyRef> tenantPrefix,
+	                            Optional<TenantName> tenantName,
+	                            bool refreshCache) {
+		typename Map<int64_t, TenantEntryCachePayload<T>>::iterator itrId;
+		typename Map<TenantName, TenantEntryCachePayload<T>>::iterator itrName;
+
+		if (tenantId.present() || tenantPrefix.present()) {
+			// Ensure either tenantId OR tenantPrefix is valid (but not both)
+			ASSERT(tenantId.present() != tenantPrefix.present());
+			ASSERT(!tenantName.present());
+
+			int64_t tId = tenantId.present() ? tenantId.get() : TenantMapEntry::prefixToId(tenantPrefix.get());
+			TraceEvent("TenantEntryCacheRemoveEntry").detail("Id", tId);
+			itrId = mapByTenantId.find(tId);
+			if (itrId == mapByTenantId.end()) {
+				return Void();
+			}
+			// Ensure byId and byName cache are in-sync
+			itrName = mapByTenantName.find(itrId->value.name);
+			ASSERT(itrName != mapByTenantName.end());
+		} else if (tenantName.present()) {
+			ASSERT(!tenantId.present() && !tenantPrefix.present());
+
+			TraceEvent("TenantEntryCacheRemoveEntry").detail("Name", tenantName.get());
+			itrName = mapByTenantName.find(tenantName.get());
+			if (itrName == mapByTenantName.end()) {
+				return Void();
+			}
+			// Ensure byId and byName cache are in-sync
+			itrId = mapByTenantId.find(itrName->value.entry.id);
+			ASSERT(itrId != mapByTenantId.end());
+		} else {
+			// Invalid input, one of: tenantId, tenantPrefix or tenantName needs to be valid.
+			throw operation_failed();
+		}
+
+		ASSERT(itrId != mapByTenantId.end() && itrName != mapByTenantName.end());
+
+		TraceEvent("TenantEntryCacheRemoveEntry")
+		    .detail("Id", itrId->key)
+		    .detail("Prefix", itrId->value.entry.prefix)
+		    .detail("Name", itrName->key);
+
+		mapByTenantId.erase(itrId);
+		mapByTenantName.erase(itrName);
+
+		if (refreshCache) {
+			return refreshImpl(this, TenantEntryCacheRefreshReason::REMOVE_ENTRY);
+		}
+
+		return Void();
+	}
+
+public:
+	TenantEntryCache(Database db)
+	  : uid(deterministicRandom()->randomUniqueID()), db(db), createPayloadFunc(defaultCreatePayload),
+	    refreshMode(TenantEntryCacheRefreshMode::PERIODIC_TASK), metrics("TenantEntryCacheMetrics", uid.toString()),
+	    hits("TenantEntryCacheHits", metrics), misses("TenantEntryCacheMisses", metrics),
+	    refreshByCacheInit("TenantEntryCacheRefreshInit", metrics),
+	    refreshByCacheMiss("TenantEntryCacheRefreshMiss", metrics),
+	    numRefreshes("TenantEntryCacheNumRefreshes", metrics) {
+		TraceEvent("TenantEntryCacheCreatedDefaultFunc", uid);
+	}
+
+	TenantEntryCache(Database db, TenantEntryCachePayloadFunc<T> fn)
+	  : uid(deterministicRandom()->randomUniqueID()), db(db), createPayloadFunc(fn),
+	    refreshMode(TenantEntryCacheRefreshMode::PERIODIC_TASK), metrics("TenantEntryCacheMetrics", uid.toString()),
+	    hits("TenantEntryCacheHits", metrics), misses("TenantEntryCacheMisses", metrics),
+	    refreshByCacheInit("TenantEntryCacheRefreshInit", metrics),
+	    refreshByCacheMiss("TenantEntryCacheRefreshMiss", metrics),
+	    numRefreshes("TenantEntryCacheNumRefreshes", metrics) {
+		TraceEvent("TenantEntryCacheCreated", uid);
+	}
+
+	TenantEntryCache(Database db, UID id, TenantEntryCachePayloadFunc<T> fn)
+	  : uid(id), db(db), createPayloadFunc(fn), refreshMode(TenantEntryCacheRefreshMode::PERIODIC_TASK),
+	    metrics("TenantEntryCacheMetrics", uid.toString()), hits("TenantEntryCacheHits", metrics),
+	    misses("TenantEntryCacheMisses", metrics), refreshByCacheInit("TenantEntryCacheRefreshInit", metrics),
+	    refreshByCacheMiss("TenantEntryCacheRefreshMiss", metrics),
+	    numRefreshes("TenantEntryCacheNumRefreshes", metrics) {
+		TraceEvent("TenantEntryCacheCreated", uid);
+	}
+
+	TenantEntryCache(Database db, UID id, TenantEntryCachePayloadFunc<T> fn, TenantEntryCacheRefreshMode mode)
+	  : uid(id), db(db), createPayloadFunc(fn), refreshMode(mode), metrics("TenantEntryCacheMetrics", uid.toString()),
+	    hits("TenantEntryCacheHits", metrics), misses("TenantEntryCacheMisses", metrics),
+	    refreshByCacheInit("TenantEntryCacheRefreshInit", metrics),
+	    refreshByCacheMiss("TenantEntryCacheRefreshMiss", metrics),
+	    numRefreshes("TenantEntryCacheNumRefreshes", metrics) {
+		TraceEvent("TenantEntryCacheCreated", uid);
+	}
+
+	Future<Void> init() {
+		TraceEvent("TenantEntryCacheInit", uid);
+
+		Future<Void> f = refreshImpl(this, TenantEntryCacheRefreshReason::INIT);
+
+		// Launch reaper task to periodically refresh cache by scanning database KeyRange
+		TenantEntryCacheRefreshReason reason = TenantEntryCacheRefreshReason::PERIODIC_TASK;
+		if (refreshMode == TenantEntryCacheRefreshMode::PERIODIC_TASK) {
+			refresher = recurringAsync([&, reason]() { return refresh(reason); },
+			                           CLIENT_KNOBS->TENANT_ENTRY_CACHE_LIST_REFRESH_INTERVAL, /* interval */
+			                           true, /* absoluteIntervalDelay */
+			                           CLIENT_KNOBS->TENANT_ENTRY_CACHE_LIST_REFRESH_INTERVAL, /* intialDelay */
+			                           TaskPriority::Worker);
+		}
+
+		return f;
+	}
+
+	Database getDatabase() const { return db; }
+	UID id() const { return uid; }
+
+	void clear() {
+		mapByTenantId.clear();
+		mapByTenantName.clear();
+	}
+
+	Future<Void> removeEntryById(int64_t tenantId, bool refreshCache = false) {
+		return removeEntryInt(tenantId, Optional<KeyRef>(), Optional<TenantName>(), refreshCache);
+	}
+	Future<Void> removeEntryByPrefix(KeyRef tenantPrefix, bool refreshCache = false) {
+		return removeEntryInt(Optional<int64_t>(), tenantPrefix, Optional<TenantName>(), refreshCache);
+	}
+	Future<Void> removeEntryByName(TenantName tenantName, bool refreshCache = false) {
+		return removeEntryInt(Optional<int64_t>(), Optional<KeyRef>(), tenantName, refreshCache);
+	}
+
+	void put(const TenantNameEntryPair& pair) {
+		TenantEntryCachePayload<T> payload = createPayloadFunc(pair.first, pair.second);
+		auto idItr = mapByTenantId.find(pair.second.id);
+		auto nameItr = mapByTenantName.find(pair.first);
+
+		Optional<TenantName> existingName;
+		Optional<int64_t> existingId;
+		if (nameItr != mapByTenantName.end()) {
+			existingId = nameItr->value.entry.id;
+			mapByTenantId.erase(nameItr->value.entry.id);
+		}
+		if (idItr != mapByTenantId.end()) {
+			existingName = idItr->value.name;
+			mapByTenantName.erase(idItr->value.name);
+		}
+
+		mapByTenantId[pair.second.id] = payload;
+		mapByTenantName[pair.first] = payload;
+
+		TraceEvent("TenantEntryCachePut")
+		    .detail("TenantName", pair.first)
+		    .detail("TenantNameExisting", existingName)
+		    .detail("TenantID", pair.second.id)
+		    .detail("TenantIDExisting", existingId)
+		    .detail("TenantPrefix", pair.second.prefix);
+
+		CODE_PROBE(idItr == mapByTenantId.end() && nameItr == mapByTenantName.end(), "TenantCache new entry");
+		CODE_PROBE(idItr != mapByTenantId.end() && nameItr == mapByTenantName.end(), "TenantCache entry name updated");
+		CODE_PROBE(idItr == mapByTenantId.end() && nameItr != mapByTenantName.end(), "TenantCache entry id updated");
+		CODE_PROBE(idItr != mapByTenantId.end() && nameItr != mapByTenantName.end(),
+		           "TenantCache entry id and name updated");
+	}
+
+	Future<Optional<TenantEntryCachePayload<T>>> getById(int64_t tenantId) { return getByIdImpl(this, tenantId); }
+	Future<Optional<TenantEntryCachePayload<T>>> getByPrefix(KeyRef prefix) {
+		int64_t id = TenantMapEntry::prefixToId(prefix);
+		return getByIdImpl(this, id);
+	}
+	Future<Optional<TenantEntryCachePayload<T>>> getByName(TenantName name) { return getByNameImpl(this, name); }
+
+	// Counter access APIs
+	Counter::Value numCacheRefreshes() const { return numRefreshes.getValue(); }
+	Counter::Value numRefreshByMisses() const { return refreshByCacheMiss.getValue(); }
+	Counter::Value numRefreshByInit() const { return refreshByCacheInit.getValue(); }
+};
+
+#include "flow/unactorcompiler.h"
+#endif // FDBCLIENT_TENANTENTRYCACHE_ACTOR_H
\ No newline at end of file
diff --git a/fdbclient/include/fdbclient/TenantManagement.actor.h b/fdbclient/include/fdbclient/TenantManagement.actor.h
index 680edccc05..6e91c8fb90 100644
--- a/fdbclient/include/fdbclient/TenantManagement.actor.h
+++ b/fdbclient/include/fdbclient/TenantManagement.actor.h
@@ -19,7 +19,9 @@
  */
 
 #pragma once
+#include "fdbclient/ClientBooleanParams.h"
 #include "flow/IRandom.h"
+#include "flow/ThreadHelper.actor.h"
 #if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_TENANT_MANAGEMENT_ACTOR_G_H)
 #define FDBCLIENT_TENANT_MANAGEMENT_ACTOR_G_H
 #include "fdbclient/TenantManagement.actor.g.h"
@@ -29,19 +31,16 @@
 #include <string>
 #include <map>
 #include "fdbclient/GenericTransactionHelper.h"
+#include "fdbclient/Metacluster.h"
 #include "fdbclient/SystemData.h"
 #include "flow/actorcompiler.h" // has to be last include
 
 namespace TenantAPI {
-ACTOR template <class Transaction>
+
+template <class Transaction>
 Future<Optional<TenantMapEntry>> tryGetTenantTransaction(Transaction tr, TenantName name) {
-	state Key tenantMapKey = name.withPrefix(tenantMapPrefix);
-
 	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
-
-	state typename transaction_future_type<Transaction, Optional<Value>>::type tenantFuture = tr->get(tenantMapKey);
-	Optional<Value> val = wait(safeThreadFutureToFuture(tenantFuture));
-	return val.map<TenantMapEntry>([](Optional<Value> v) { return TenantMapEntry::decode(v.get()); });
+	return TenantMetadata::tenantMap().get(tr, name);
 }
 
 ACTOR template <class DB>
@@ -80,67 +79,133 @@ Future<TenantMapEntry> getTenant(Reference<DB> db, TenantName name) {
 	return entry.get();
 }
 
-// Creates a tenant with the given name. If the tenant already exists, an empty optional will be returned.
-// The caller must enforce that the tenant ID be unique from all current and past tenants, and it must also be unique
-// from all other tenants created in the same transaction.
 ACTOR template <class Transaction>
-Future<std::pair<TenantMapEntry, bool>> createTenantTransaction(Transaction tr, TenantNameRef name, int64_t tenantId) {
-	state Key tenantMapKey = name.withPrefix(tenantMapPrefix);
+Future<ClusterType> getClusterType(Transaction tr) {
+	Optional<MetaclusterRegistrationEntry> metaclusterRegistration =
+	    wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
+
+	return metaclusterRegistration.present() ? metaclusterRegistration.get().clusterType : ClusterType::STANDALONE;
+}
+
+ACTOR template <class Transaction>
+Future<Void> checkTenantMode(Transaction tr, ClusterType expectedClusterType) {
+	state typename transaction_future_type<Transaction, Optional<Value>>::type tenantModeFuture =
+	    tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr));
+
+	state ClusterType actualClusterType = wait(getClusterType(tr));
+	Optional<Value> tenantModeValue = wait(safeThreadFutureToFuture(tenantModeFuture));
+
+	TenantMode tenantMode = TenantMode::fromValue(tenantModeValue.castTo<ValueRef>());
+	if (actualClusterType != expectedClusterType) {
+		throw invalid_metacluster_operation();
+	} else if (actualClusterType == ClusterType::STANDALONE && tenantMode == TenantMode::DISABLED) {
+		throw tenants_disabled();
+	}
+
+	return Void();
+}
+
+TenantMode tenantModeForClusterType(ClusterType clusterType, TenantMode tenantMode);
+
+// Returns true if the specified ID has already been deleted and false if not. If the ID is old enough
+// that we no longer keep tombstones for it, an error is thrown.
+ACTOR template <class Transaction>
+Future<bool> checkTombstone(Transaction tr, int64_t id) {
+	state Future<bool> tombstoneFuture = TenantMetadata::tenantTombstones().exists(tr, id);
+
+	// If we are trying to create a tenant older than the oldest tombstones we still maintain, then we fail it
+	// with an error.
+	Optional<TenantTombstoneCleanupData> tombstoneCleanupData = wait(TenantMetadata::tombstoneCleanupData().get(tr));
+	if (tombstoneCleanupData.present() && tombstoneCleanupData.get().tombstonesErasedThrough >= id) {
+		throw tenant_creation_permanently_failed();
+	}
+
+	state bool hasTombstone = wait(tombstoneFuture);
+	return hasTombstone;
+}
+
+// Creates a tenant with the given name. If the tenant already exists, the boolean return parameter will be false
+// and the existing entry will be returned. If the tenant cannot be created, then the optional will be empty.
+ACTOR template <class Transaction>
+Future<std::pair<Optional<TenantMapEntry>, bool>> createTenantTransaction(
+    Transaction tr,
+    TenantNameRef name,
+    TenantMapEntry tenantEntry,
+    ClusterType clusterType = ClusterType::STANDALONE) {
+
+	ASSERT(clusterType != ClusterType::METACLUSTER_MANAGEMENT);
+	ASSERT(tenantEntry.id >= 0);
 
 	if (name.startsWith("\xff"_sr)) {
 		throw invalid_tenant_name();
 	}
+	if (tenantEntry.tenantGroup.present() && tenantEntry.tenantGroup.get().startsWith("\xff"_sr)) {
+		throw invalid_tenant_group_name();
+	}
 
 	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
 
-	state Future<Optional<TenantMapEntry>> tenantEntryFuture = tryGetTenantTransaction(tr, name);
-	state typename transaction_future_type<Transaction, Optional<Value>>::type tenantDataPrefixFuture =
-	    tr->get(tenantDataPrefixKey);
-	state typename transaction_future_type<Transaction, Optional<Value>>::type tenantModeFuture =
-	    tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr));
-
-	Optional<Value> tenantMode = wait(safeThreadFutureToFuture(tenantModeFuture));
-
-	if (!tenantMode.present() || tenantMode.get() == StringRef(format("%d", TenantMode::DISABLED))) {
-		throw tenants_disabled();
+	state Future<Optional<TenantMapEntry>> existingEntryFuture = tryGetTenantTransaction(tr, name);
+	state Future<Void> tenantModeCheck = checkTenantMode(tr, clusterType);
+	state Future<bool> tombstoneFuture =
+	    (clusterType == ClusterType::STANDALONE) ? false : checkTombstone(tr, tenantEntry.id);
+	state Future<Optional<TenantGroupEntry>> existingTenantGroupEntryFuture;
+	if (tenantEntry.tenantGroup.present()) {
+		existingTenantGroupEntryFuture = TenantMetadata::tenantGroupMap().get(tr, tenantEntry.tenantGroup.get());
 	}
 
-	Optional<TenantMapEntry> tenantEntry = wait(tenantEntryFuture);
-	if (tenantEntry.present()) {
-		return std::make_pair(tenantEntry.get(), false);
+	wait(tenantModeCheck);
+	Optional<TenantMapEntry> existingEntry = wait(existingEntryFuture);
+	if (existingEntry.present()) {
+		return std::make_pair(existingEntry.get(), false);
 	}
 
-	Optional<Value> tenantDataPrefix = wait(safeThreadFutureToFuture(tenantDataPrefixFuture));
-	if (tenantDataPrefix.present() &&
-	    tenantDataPrefix.get().size() + TenantMapEntry::ROOT_PREFIX_SIZE > CLIENT_KNOBS->TENANT_PREFIX_SIZE_LIMIT) {
-		TraceEvent(SevWarnAlways, "TenantPrefixTooLarge")
-		    .detail("TenantSubspace", tenantDataPrefix.get())
-		    .detail("TenantSubspaceLength", tenantDataPrefix.get().size())
-		    .detail("RootPrefixLength", TenantMapEntry::ROOT_PREFIX_SIZE)
-		    .detail("MaxTenantPrefixSize", CLIENT_KNOBS->TENANT_PREFIX_SIZE_LIMIT);
-
-		throw client_invalid_operation();
+	state bool hasTombstone = wait(tombstoneFuture);
+	if (hasTombstone) {
+		return std::make_pair(Optional<TenantMapEntry>(), false);
 	}
 
-	state TenantMapEntry newTenant(tenantId, tenantDataPrefix.present() ? (KeyRef)tenantDataPrefix.get() : ""_sr);
-
 	state typename transaction_future_type<Transaction, RangeResult>::type prefixRangeFuture =
-	    tr->getRange(prefixRange(newTenant.prefix), 1);
+	    tr->getRange(prefixRange(tenantEntry.prefix), 1);
+
 	RangeResult contents = wait(safeThreadFutureToFuture(prefixRangeFuture));
 	if (!contents.empty()) {
 		throw tenant_prefix_allocator_conflict();
 	}
 
-	tr->set(tenantMapKey, newTenant.encode());
+	tenantEntry.tenantState = TenantState::READY;
+	tenantEntry.assignedCluster = Optional<ClusterName>();
 
-	return std::make_pair(newTenant, true);
+	TenantMetadata::tenantMap().set(tr, name, tenantEntry);
+	TenantMetadata::tenantIdIndex().set(tr, tenantEntry.id, name);
+
+	if (tenantEntry.tenantGroup.present()) {
+		TenantMetadata::tenantGroupTenantIndex().insert(tr, Tuple::makeTuple(tenantEntry.tenantGroup.get(), name));
+
+		// Create the tenant group associated with this tenant if it doesn't already exist
+		Optional<TenantGroupEntry> existingTenantGroup = wait(existingTenantGroupEntryFuture);
+		if (!existingTenantGroup.present()) {
+			TenantMetadata::tenantGroupMap().set(tr, tenantEntry.tenantGroup.get(), TenantGroupEntry());
+		}
+	}
+
+	// This is idempotent because we only add an entry to the tenant map if it isn't already there
+	TenantMetadata::tenantCount().atomicOp(tr, 1, MutationRef::AddValue);
+
+	// Read the tenant count after incrementing the counter so that simultaneous attempts to create
+	// tenants in the same transaction are properly reflected.
+	int64_t tenantCount = wait(TenantMetadata::tenantCount().getD(tr, Snapshot::False, 0));
+	if (tenantCount > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER) {
+		throw cluster_no_capacity();
+	}
+
+	return std::make_pair(tenantEntry, true);
 }
 
 ACTOR template <class Transaction>
 Future<int64_t> getNextTenantId(Transaction tr) {
-	state typename transaction_future_type<Transaction, Optional<Value>>::type lastIdFuture = tr->get(tenantLastIdKey);
-	Optional<Value> lastIdVal = wait(safeThreadFutureToFuture(lastIdFuture));
-	int64_t tenantId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) + 1 : 0;
+	Optional<int64_t> lastId = wait(TenantMetadata::lastTenantId().get(tr));
+	int64_t tenantId = lastId.orDefault(-1) + 1;
 	if (BUGGIFY) {
 		tenantId += deterministicRandom()->randomSkewedUInt32(1, 1e9);
 	}
@@ -148,37 +213,56 @@ Future<int64_t> getNextTenantId(Transaction tr) {
 }
 
 ACTOR template <class DB>
-Future<TenantMapEntry> createTenant(Reference<DB> db, TenantName name) {
+Future<Optional<TenantMapEntry>> createTenant(Reference<DB> db,
+                                              TenantName name,
+                                              TenantMapEntry tenantEntry = TenantMapEntry(),
+                                              ClusterType clusterType = ClusterType::STANDALONE) {
 	state Reference<typename DB::TransactionT> tr = db->createTransaction();
 
-	state bool firstTry = true;
+	state bool checkExistence = clusterType != ClusterType::METACLUSTER_DATA;
+	state bool generateTenantId = tenantEntry.id < 0;
+
+	ASSERT(clusterType == ClusterType::STANDALONE || !generateTenantId);
+
 	loop {
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			state Future<int64_t> tenantIdFuture = getNextTenantId(tr);
+			state Future<int64_t> tenantIdFuture;
+			if (generateTenantId) {
+				tenantIdFuture = getNextTenantId(tr);
+			}
 
-			if (firstTry) {
+			if (checkExistence) {
 				Optional<TenantMapEntry> entry = wait(tryGetTenantTransaction(tr, name));
 				if (entry.present()) {
 					throw tenant_already_exists();
 				}
 
-				firstTry = false;
+				checkExistence = false;
 			}
 
-			int64_t tenantId = wait(tenantIdFuture);
-			tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(tenantId));
-			state std::pair<TenantMapEntry, bool> newTenant = wait(createTenantTransaction(tr, name, tenantId));
+			if (generateTenantId) {
+				int64_t tenantId = wait(tenantIdFuture);
+				tenantEntry.setId(tenantId);
+				TenantMetadata::lastTenantId().set(tr, tenantId);
+			}
 
-			wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
+			state std::pair<Optional<TenantMapEntry>, bool> newTenant =
+			    wait(createTenantTransaction(tr, name, tenantEntry, clusterType));
 
-			TraceEvent("CreatedTenant")
-			    .detail("Tenant", name)
-			    .detail("TenantId", newTenant.first.id)
-			    .detail("Prefix", newTenant.first.prefix)
-			    .detail("Version", tr->getCommittedVersion());
+			if (newTenant.second) {
+				ASSERT(newTenant.first.present());
+				wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
+
+				TraceEvent("CreatedTenant")
+				    .detail("Tenant", name)
+				    .detail("TenantId", newTenant.first.get().id)
+				    .detail("Prefix", newTenant.first.get().prefix)
+				    .detail("TenantGroup", tenantEntry.tenantGroup)
+				    .detail("Version", tr->getCommittedVersion());
+			}
 
 			return newTenant.first;
 		} catch (Error& e) {
@@ -188,48 +272,132 @@ Future<TenantMapEntry> createTenant(Reference<DB> db, TenantName name) {
 }
 
 ACTOR template <class Transaction>
-Future<Void> deleteTenantTransaction(Transaction tr, TenantNameRef name) {
-	state Key tenantMapKey = name.withPrefix(tenantMapPrefix);
+Future<Void> markTenantTombstones(Transaction tr, int64_t tenantId) {
+	// In data clusters, we store a tombstone
+	state Future<KeyBackedRangeResult<int64_t>> latestTombstoneFuture =
+	    TenantMetadata::tenantTombstones().getRange(tr, {}, {}, 1, Snapshot::False, Reverse::True);
+	state Optional<TenantTombstoneCleanupData> cleanupData = wait(TenantMetadata::tombstoneCleanupData().get(tr));
+	state Version transactionReadVersion = wait(safeThreadFutureToFuture(tr->getReadVersion()));
+
+	// If it has been long enough since we last cleaned up the tenant tombstones, we do that first
+	if (!cleanupData.present() || cleanupData.get().nextTombstoneEraseVersion <= transactionReadVersion) {
+		state int64_t deleteThroughId = cleanupData.present() ? cleanupData.get().nextTombstoneEraseId : -1;
+		// Delete all tombstones up through the one currently marked in the cleanup data
+		if (deleteThroughId >= 0) {
+			TenantMetadata::tenantTombstones().erase(tr, 0, deleteThroughId + 1);
+		}
+
+		KeyBackedRangeResult<int64_t> latestTombstone = wait(latestTombstoneFuture);
+		int64_t nextDeleteThroughId = std::max(deleteThroughId, tenantId);
+		if (!latestTombstone.results.empty()) {
+			nextDeleteThroughId = std::max(nextDeleteThroughId, latestTombstone.results[0]);
+		}
+
+		// The next cleanup will happen at or after TENANT_TOMBSTONE_CLEANUP_INTERVAL seconds have elapsed and
+		// will clean up tombstones through the most recently allocated ID.
+		TenantTombstoneCleanupData updatedCleanupData;
+		updatedCleanupData.tombstonesErasedThrough = deleteThroughId;
+		updatedCleanupData.nextTombstoneEraseId = nextDeleteThroughId;
+		updatedCleanupData.nextTombstoneEraseVersion =
+		    transactionReadVersion +
+		    CLIENT_KNOBS->TENANT_TOMBSTONE_CLEANUP_INTERVAL * CLIENT_KNOBS->VERSIONS_PER_SECOND;
+
+		TenantMetadata::tombstoneCleanupData().set(tr, updatedCleanupData);
+
+		// If the tenant being deleted is within the tombstone window, record the tombstone
+		if (tenantId > updatedCleanupData.tombstonesErasedThrough) {
+			TenantMetadata::tenantTombstones().insert(tr, tenantId);
+		}
+	} else if (tenantId > cleanupData.get().tombstonesErasedThrough) {
+		// If the tenant being deleted is within the tombstone window, record the tombstone
+		TenantMetadata::tenantTombstones().insert(tr, tenantId);
+	}
+	return Void();
+}
+
+// Deletes the tenant with the given name. If tenantId is specified, the tenant being deleted must also have the same
+// ID. If no matching tenant is found, this function returns without deleting anything. This behavior allows the
+// function to be used idempotently: if the transaction is retried after having succeeded, it will see that the tenant
+// is absent (or optionally created with a new ID) and do nothing.
+ACTOR template <class Transaction>
+Future<Void> deleteTenantTransaction(Transaction tr,
+                                     TenantNameRef name,
+                                     Optional<int64_t> tenantId = Optional<int64_t>(),
+                                     ClusterType clusterType = ClusterType::STANDALONE) {
+	ASSERT(clusterType == ClusterType::STANDALONE || tenantId.present());
+	ASSERT(clusterType != ClusterType::METACLUSTER_MANAGEMENT);
 
 	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
 
-	state Optional<TenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, name));
-	if (!tenantEntry.present()) {
-		return Void();
+	state Future<Optional<TenantMapEntry>> tenantEntryFuture = tryGetTenantTransaction(tr, name);
+	wait(checkTenantMode(tr, clusterType));
+
+	state Optional<TenantMapEntry> tenantEntry = wait(tenantEntryFuture);
+	if (tenantEntry.present() && (!tenantId.present() || tenantEntry.get().id == tenantId.get())) {
+		state typename transaction_future_type<Transaction, RangeResult>::type prefixRangeFuture =
+		    tr->getRange(prefixRange(tenantEntry.get().prefix), 1);
+
+		RangeResult contents = wait(safeThreadFutureToFuture(prefixRangeFuture));
+		if (!contents.empty()) {
+			throw tenant_not_empty();
+		}
+
+		// This is idempotent because we only erase an entry from the tenant map if it is present
+		TenantMetadata::tenantMap().erase(tr, name);
+		TenantMetadata::tenantIdIndex().erase(tr, tenantEntry.get().id);
+		TenantMetadata::tenantCount().atomicOp(tr, -1, MutationRef::AddValue);
+
+		if (tenantEntry.get().tenantGroup.present()) {
+			TenantMetadata::tenantGroupTenantIndex().erase(tr,
+			                                               Tuple::makeTuple(tenantEntry.get().tenantGroup.get(), name));
+			KeyBackedSet<Tuple>::RangeResultType tenantsInGroup =
+			    wait(TenantMetadata::tenantGroupTenantIndex().getRange(
+			        tr,
+			        Tuple::makeTuple(tenantEntry.get().tenantGroup.get()),
+			        Tuple::makeTuple(keyAfter(tenantEntry.get().tenantGroup.get())),
+			        2));
+			if (tenantsInGroup.results.empty() ||
+			    (tenantsInGroup.results.size() == 1 && tenantsInGroup.results[0].getString(1) == name)) {
+				TenantMetadata::tenantGroupMap().erase(tr, tenantEntry.get().tenantGroup.get());
+			}
+		}
 	}
 
-	state typename transaction_future_type<Transaction, RangeResult>::type prefixRangeFuture =
-	    tr->getRange(prefixRange(tenantEntry.get().prefix), 1);
-	RangeResult contents = wait(safeThreadFutureToFuture(prefixRangeFuture));
-	if (!contents.empty()) {
-		throw tenant_not_empty();
+	if (clusterType == ClusterType::METACLUSTER_DATA) {
+		wait(markTenantTombstones(tr, tenantId.get()));
 	}
 
-	tr->clear(tenantMapKey);
-
 	return Void();
 }
 
+// Deletes the tenant with the given name. If tenantId is specified, the tenant being deleted must also have the same
+// ID.
 ACTOR template <class DB>
-Future<Void> deleteTenant(Reference<DB> db, TenantName name) {
+Future<Void> deleteTenant(Reference<DB> db,
+                          TenantName name,
+                          Optional<int64_t> tenantId = Optional<int64_t>(),
+                          ClusterType clusterType = ClusterType::STANDALONE) {
 	state Reference<typename DB::TransactionT> tr = db->createTransaction();
 
-	state bool firstTry = true;
+	state bool checkExistence = clusterType == ClusterType::STANDALONE;
 	loop {
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			if (firstTry) {
-				Optional<TenantMapEntry> entry = wait(tryGetTenantTransaction(tr, name));
-				if (!entry.present()) {
-					throw tenant_not_found();
+			if (checkExistence) {
+				TenantMapEntry entry = wait(getTenantTransaction(tr, name));
+
+				// If an ID wasn't specified, use the current ID. This way we cannot inadvertently delete
+				// multiple tenants if this transaction retries.
+				if (!tenantId.present()) {
+					tenantId = entry.id;
 				}
 
-				firstTry = false;
+				checkExistence = false;
 			}
 
-			wait(deleteTenantTransaction(tr, name));
+			wait(deleteTenantTransaction(tr, name, tenantId, clusterType));
 			wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
 
 			TraceEvent("DeletedTenant").detail("Tenant", name).detail("Version", tr->getCommittedVersion());
@@ -240,39 +408,84 @@ Future<Void> deleteTenant(Reference<DB> db, TenantName name) {
 	}
 }
 
+// This should only be called from a transaction that has already confirmed that the tenant entry
+// is present. The tenantEntry should start with the existing entry and modify only those fields that need
+// to be changed. This must only be called on a non-management cluster.
 ACTOR template <class Transaction>
-Future<std::map<TenantName, TenantMapEntry>> listTenantsTransaction(Transaction tr,
-                                                                    TenantNameRef begin,
-                                                                    TenantNameRef end,
-                                                                    int limit) {
-	state KeyRange range = KeyRangeRef(begin, end).withPrefix(tenantMapPrefix);
+Future<Void> configureTenantTransaction(Transaction tr,
+                                        TenantNameRef tenantName,
+                                        TenantMapEntry originalEntry,
+                                        TenantMapEntry updatedTenantEntry) {
+	ASSERT(updatedTenantEntry.id == originalEntry.id);
 
 	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+	TenantMetadata::tenantMap().set(tr, tenantName, updatedTenantEntry);
 
-	state typename transaction_future_type<Transaction, RangeResult>::type listFuture =
-	    tr->getRange(firstGreaterOrEqual(range.begin), firstGreaterOrEqual(range.end), limit);
-	RangeResult results = wait(safeThreadFutureToFuture(listFuture));
+	// If the tenant group was changed, we need to update the tenant group metadata structures
+	if (originalEntry.tenantGroup != updatedTenantEntry.tenantGroup) {
+		if (updatedTenantEntry.tenantGroup.present() && updatedTenantEntry.tenantGroup.get().startsWith("\xff"_sr)) {
+			throw invalid_tenant_group_name();
+		}
+		if (originalEntry.tenantGroup.present()) {
+			// Remove this tenant from the original tenant group index
+			TenantMetadata::tenantGroupTenantIndex().erase(
+			    tr, Tuple::makeTuple(originalEntry.tenantGroup.get(), tenantName));
 
-	std::map<TenantName, TenantMapEntry> tenants;
-	for (auto kv : results) {
-		tenants[kv.key.removePrefix(tenantMapPrefix)] = TenantMapEntry::decode(kv.value);
+			// Check if the original tenant group is now empty. If so, remove the tenant group.
+			KeyBackedSet<Tuple>::RangeResultType tenants = wait(TenantMetadata::tenantGroupTenantIndex().getRange(
+			    tr,
+			    Tuple::makeTuple(originalEntry.tenantGroup.get()),
+			    Tuple::makeTuple(keyAfter(originalEntry.tenantGroup.get())),
+			    2));
+
+			if (tenants.results.empty() ||
+			    (tenants.results.size() == 1 && tenants.results[0].getString(1) == tenantName)) {
+				TenantMetadata::tenantGroupMap().erase(tr, originalEntry.tenantGroup.get());
+			}
+		}
+		if (updatedTenantEntry.tenantGroup.present()) {
+			// If this is creating a new tenant group, add it to the tenant group map
+			Optional<TenantGroupEntry> entry =
+			    wait(TenantMetadata::tenantGroupMap().get(tr, updatedTenantEntry.tenantGroup.get()));
+			if (!entry.present()) {
+				TenantMetadata::tenantGroupMap().set(tr, updatedTenantEntry.tenantGroup.get(), TenantGroupEntry());
+			}
+
+			// Insert this tenant in the tenant group index
+			TenantMetadata::tenantGroupTenantIndex().insert(
+			    tr, Tuple::makeTuple(updatedTenantEntry.tenantGroup.get(), tenantName));
+		}
 	}
 
-	return tenants;
+	return Void();
+}
+
+ACTOR template <class Transaction>
+Future<std::vector<std::pair<TenantName, TenantMapEntry>>> listTenantsTransaction(Transaction tr,
+                                                                                  TenantName begin,
+                                                                                  TenantName end,
+                                                                                  int limit) {
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+
+	KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> results =
+	    wait(TenantMetadata::tenantMap().getRange(tr, begin, end, limit));
+
+	return results.results;
 }
 
 ACTOR template <class DB>
-Future<std::map<TenantName, TenantMapEntry>> listTenants(Reference<DB> db,
-                                                         TenantName begin,
-                                                         TenantName end,
-                                                         int limit) {
+Future<std::vector<std::pair<TenantName, TenantMapEntry>>> listTenants(Reference<DB> db,
+                                                                       TenantName begin,
+                                                                       TenantName end,
+                                                                       int limit) {
 	state Reference<typename DB::TransactionT> tr = db->createTransaction();
 
 	loop {
 		try {
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
-			std::map<TenantName, TenantMapEntry> tenants = wait(listTenantsTransaction(tr, begin, end, limit));
+			std::vector<std::pair<TenantName, TenantMapEntry>> tenants =
+			    wait(listTenantsTransaction(tr, begin, end, limit));
 			return tenants;
 		} catch (Error& e) {
 			wait(safeThreadFutureToFuture(tr->onError(e)));
@@ -280,12 +493,60 @@ Future<std::map<TenantName, TenantMapEntry>> listTenants(Reference<DB> db,
 	}
 }
 
-ACTOR template <class DB>
-Future<Void> renameTenant(Reference<DB> db, TenantName oldName, TenantName newName) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+ACTOR template <class Transaction>
+Future<Void> renameTenantTransaction(Transaction tr,
+                                     TenantName oldName,
+                                     TenantName newName,
+                                     Optional<int64_t> tenantId = Optional<int64_t>(),
+                                     ClusterType clusterType = ClusterType::STANDALONE,
+                                     Optional<int64_t> configureSequenceNum = Optional<int64_t>()) {
+	ASSERT(clusterType == ClusterType::STANDALONE || (tenantId.present() && configureSequenceNum.present()));
+	ASSERT(clusterType != ClusterType::METACLUSTER_MANAGEMENT);
+	wait(checkTenantMode(tr, clusterType));
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+	state Optional<TenantMapEntry> oldEntry;
+	state Optional<TenantMapEntry> newEntry;
+	wait(store(oldEntry, tryGetTenantTransaction(tr, oldName)) &&
+	     store(newEntry, tryGetTenantTransaction(tr, newName)));
+	if (!oldEntry.present() || (tenantId.present() && tenantId.get() != oldEntry.get().id)) {
+		throw tenant_not_found();
+	}
+	if (newEntry.present()) {
+		throw tenant_already_exists();
+	}
+	if (configureSequenceNum.present()) {
+		if (oldEntry.get().configurationSequenceNum >= configureSequenceNum.get()) {
+			return Void();
+		}
+		oldEntry.get().configurationSequenceNum = configureSequenceNum.get();
+	}
+	TenantMetadata::tenantMap().erase(tr, oldName);
+	TenantMetadata::tenantMap().set(tr, newName, oldEntry.get());
+	TenantMetadata::tenantIdIndex().set(tr, oldEntry.get().id, newName);
+
+	// Update the tenant group index to reflect the new tenant name
+	if (oldEntry.get().tenantGroup.present()) {
+		TenantMetadata::tenantGroupTenantIndex().erase(tr, Tuple::makeTuple(oldEntry.get().tenantGroup.get(), oldName));
+		TenantMetadata::tenantGroupTenantIndex().insert(tr,
+		                                                Tuple::makeTuple(oldEntry.get().tenantGroup.get(), newName));
+	}
+
+	if (clusterType == ClusterType::METACLUSTER_DATA) {
+		wait(markTenantTombstones(tr, tenantId.get()));
+	}
+
+	return Void();
+}
+
+ACTOR template <class DB>
+Future<Void> renameTenant(Reference<DB> db,
+                          TenantName oldName,
+                          TenantName newName,
+                          Optional<int64_t> tenantId = Optional<int64_t>(),
+                          ClusterType clusterType = ClusterType::STANDALONE) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+	ASSERT(clusterType == ClusterType::STANDALONE || tenantId.present());
 
-	state Key oldNameKey = oldName.withPrefix(tenantMapPrefix);
-	state Key newNameKey = newName.withPrefix(tenantMapPrefix);
 	state bool firstTry = true;
 	state int64_t id;
 	loop {
@@ -328,9 +589,8 @@ Future<Void> renameTenant(Reference<DB> db, TenantName oldName, TenantName newNa
 					throw tenant_not_found();
 				}
 			}
-			tr->clear(oldNameKey);
-			tr->set(newNameKey, oldEntry.get().encode());
-			wait(safeThreadFutureToFuture(tr->commit()));
+			wait(renameTenantTransaction(tr, oldName, newName, tenantId, clusterType));
+			wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
 			TraceEvent("RenameTenantSuccess").detail("OldName", oldName).detail("NewName", newName);
 			return Void();
 		} catch (Error& e) {
@@ -338,6 +598,62 @@ Future<Void> renameTenant(Reference<DB> db, TenantName oldName, TenantName newNa
 		}
 	}
 }
+
+template <class Transaction>
+Future<Optional<TenantGroupEntry>> tryGetTenantGroupTransaction(Transaction tr, TenantGroupName name) {
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+	return TenantMetadata::tenantGroupMap().get(tr, name);
+}
+
+ACTOR template <class DB>
+Future<Optional<TenantGroupEntry>> tryGetTenantGroup(Reference<DB> db, TenantGroupName name) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			Optional<TenantGroupEntry> entry = wait(tryGetTenantGroupTransaction(tr, name));
+			return entry;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+ACTOR template <class Transaction>
+Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroupsTransaction(Transaction tr,
+                                                                                              TenantGroupName begin,
+                                                                                              TenantGroupName end,
+                                                                                              int limit) {
+	tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+
+	KeyBackedRangeResult<std::pair<TenantGroupName, TenantGroupEntry>> results =
+	    wait(TenantMetadata::tenantGroupMap().getRange(tr, begin, end, limit));
+
+	return results.results;
+}
+
+ACTOR template <class DB>
+Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroups(Reference<DB> db,
+                                                                                   TenantGroupName begin,
+                                                                                   TenantGroupName end,
+                                                                                   int limit) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups =
+			    wait(listTenantGroupsTransaction(tr, begin, end, limit));
+			return tenantGroups;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
 } // namespace TenantAPI
 
 #include "flow/unactorcompiler.h"
diff --git a/fdbclient/include/fdbclient/TenantSpecialKeys.actor.h b/fdbclient/include/fdbclient/TenantSpecialKeys.actor.h
index 73362deb45..cf190fc77d 100644
--- a/fdbclient/include/fdbclient/TenantSpecialKeys.actor.h
+++ b/fdbclient/include/fdbclient/TenantSpecialKeys.actor.h
@@ -31,12 +31,11 @@
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/SpecialKeySpace.actor.h"
 #include "fdbclient/TenantManagement.actor.h"
-#include "fdbclient/libb64/encode.h"
+#include "fdbclient/Tuple.h"
 #include "flow/Arena.h"
 #include "flow/UnitTest.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-template <bool HasSubRanges = true>
 class TenantRangeImpl : public SpecialKeyRangeRWImpl {
 private:
 	static KeyRangeRef removePrefix(KeyRangeRef range, KeyRef prefix, KeyRef defaultEnd) {
@@ -53,15 +52,14 @@ private:
 
 	static KeyRef withTenantMapPrefix(KeyRef key, Arena& ar) {
 		int keySize = SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin.size() +
-		              TenantRangeImpl::submoduleRange.begin.size() + TenantRangeImpl::mapSubRange.begin.size() +
-		              key.size();
+		              submoduleRange.begin.size() + mapSubRange.begin.size() + key.size();
 
 		KeyRef prefixedKey = makeString(keySize, ar);
 		uint8_t* mutableKey = mutateString(prefixedKey);
 
 		mutableKey = SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin.copyTo(mutableKey);
-		mutableKey = TenantRangeImpl::submoduleRange.begin.copyTo(mutableKey);
-		mutableKey = TenantRangeImpl::mapSubRange.begin.copyTo(mutableKey);
+		mutableKey = submoduleRange.begin.copyTo(mutableKey);
+		mutableKey = mapSubRange.begin.copyTo(mutableKey);
 
 		key.copyTo(mutableKey);
 		return prefixedKey;
@@ -71,27 +69,12 @@ private:
 	                                        KeyRangeRef kr,
 	                                        RangeResult* results,
 	                                        GetRangeLimits limitsHint) {
-		std::map<TenantName, TenantMapEntry> tenants =
+		std::vector<std::pair<TenantName, TenantMapEntry>> tenants =
 		    wait(TenantAPI::listTenantsTransaction(&ryw->getTransaction(), kr.begin, kr.end, limitsHint.rows));
 
 		for (auto tenant : tenants) {
-			json_spirit::mObject tenantEntry;
-			tenantEntry["id"] = tenant.second.id;
-			if (ryw->getDatabase()->apiVersionAtLeast(720)) {
-				json_spirit::mObject prefixObject;
-				std::string encodedPrefix = base64::encoder::from_string(tenant.second.prefix.toString());
-				// Remove trailing newline
-				encodedPrefix.resize(encodedPrefix.size() - 1);
-
-				prefixObject["base64"] = encodedPrefix;
-				prefixObject["printable"] = printable(tenant.second.prefix);
-				tenantEntry["prefix"] = prefixObject;
-			} else {
-				// This is not a standard encoding in JSON, and some libraries may not be able to easily decode it
-				tenantEntry["prefix"] = tenant.second.prefix.toString();
-			}
-			std::string tenantEntryString = json_spirit::write_string(json_spirit::mValue(tenantEntry));
-			ValueRef tenantEntryBytes(results->arena(), tenantEntryString);
+			std::string jsonString = tenant.second.toJson();
+			ValueRef tenantEntryBytes(results->arena(), jsonString);
 			results->push_back(results->arena(),
 			                   KeyValueRef(withTenantMapPrefix(tenant.first, results->arena()), tenantEntryBytes));
 		}
@@ -120,25 +103,108 @@ private:
 		return results;
 	}
 
-	ACTOR static Future<Void> createTenants(ReadYourWritesTransaction* ryw, std::vector<TenantNameRef> tenants) {
-		int64_t _nextId = wait(TenantAPI::getNextTenantId(&ryw->getTransaction()));
-		int64_t nextId = _nextId;
+	// Returns true if the tenant was created, false if it already existed
+	ACTOR static Future<bool> createTenant(
+	    ReadYourWritesTransaction* ryw,
+	    TenantNameRef tenantName,
+	    std::vector<std::pair<Standalone<StringRef>, Optional<Value>>> configMutations,
+	    int64_t tenantId,
+	    std::map<TenantGroupName, int>* tenantGroupNetTenantDelta) {
+		state TenantMapEntry tenantEntry;
+		tenantEntry.setId(tenantId);
+		tenantEntry.encrypted = ryw->getTransactionState()->cx->clientInfo->get().isEncryptionEnabled;
 
-		std::vector<Future<Void>> createFutures;
-		for (auto tenant : tenants) {
-			createFutures.push_back(
-			    success(TenantAPI::createTenantTransaction(&ryw->getTransaction(), tenant, nextId++)));
+		for (auto const& [name, value] : configMutations) {
+			tenantEntry.configure(name, value);
 		}
 
-		ryw->getTransaction().set(tenantLastIdKey, TenantMapEntry::idToPrefix(nextId - 1));
+		if (tenantEntry.tenantGroup.present()) {
+			(*tenantGroupNetTenantDelta)[tenantEntry.tenantGroup.get()]++;
+		}
+
+		std::pair<Optional<TenantMapEntry>, bool> entry =
+		    wait(TenantAPI::createTenantTransaction(&ryw->getTransaction(), tenantName, tenantEntry));
+
+		return entry.second;
+	}
+
+	ACTOR static Future<Void> createTenants(
+	    ReadYourWritesTransaction* ryw,
+	    std::map<TenantName, std::vector<std::pair<Standalone<StringRef>, Optional<Value>>>> tenants,
+	    std::map<TenantGroupName, int>* tenantGroupNetTenantDelta) {
+		state Future<int64_t> tenantCountFuture =
+		    TenantMetadata::tenantCount().getD(&ryw->getTransaction(), Snapshot::False, 0);
+		int64_t _nextId = wait(TenantAPI::getNextTenantId(&ryw->getTransaction()));
+		state int64_t nextId = _nextId;
+
+		state std::vector<Future<bool>> createFutures;
+		for (auto const& [tenant, config] : tenants) {
+			createFutures.push_back(createTenant(ryw, tenant, config, nextId++, tenantGroupNetTenantDelta));
+		}
+
+		TenantMetadata::lastTenantId().set(&ryw->getTransaction(), nextId - 1);
 		wait(waitForAll(createFutures));
+
+		state int numCreatedTenants = 0;
+		for (auto f : createFutures) {
+			if (f.get()) {
+				++numCreatedTenants;
+			}
+		}
+
+		// Check the tenant count here rather than rely on the createTenantTransaction check because we don't have RYW
+		int64_t tenantCount = wait(tenantCountFuture);
+		if (tenantCount + numCreatedTenants > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER) {
+			throw cluster_no_capacity();
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> changeTenantConfig(
+	    ReadYourWritesTransaction* ryw,
+	    TenantName tenantName,
+	    std::vector<std::pair<Standalone<StringRef>, Optional<Value>>> configEntries,
+	    std::map<TenantGroupName, int>* tenantGroupNetTenantDelta) {
+		TenantMapEntry originalEntry = wait(TenantAPI::getTenantTransaction(&ryw->getTransaction(), tenantName));
+		TenantMapEntry updatedEntry = originalEntry;
+		for (auto const& [name, value] : configEntries) {
+			updatedEntry.configure(name, value);
+		}
+
+		if (originalEntry.tenantGroup != updatedEntry.tenantGroup) {
+			if (originalEntry.tenantGroup.present()) {
+				(*tenantGroupNetTenantDelta)[originalEntry.tenantGroup.get()]--;
+			}
+			if (updatedEntry.tenantGroup.present()) {
+				(*tenantGroupNetTenantDelta)[updatedEntry.tenantGroup.get()]++;
+			}
+		}
+
+		wait(TenantAPI::configureTenantTransaction(&ryw->getTransaction(), tenantName, originalEntry, updatedEntry));
+		return Void();
+	}
+
+	ACTOR static Future<Void> deleteSingleTenant(ReadYourWritesTransaction* ryw,
+	                                             TenantName tenantName,
+	                                             std::map<TenantGroupName, int>* tenantGroupNetTenantDelta) {
+		state Optional<TenantMapEntry> tenantEntry =
+		    wait(TenantAPI::tryGetTenantTransaction(&ryw->getTransaction(), tenantName));
+		if (tenantEntry.present()) {
+			wait(TenantAPI::deleteTenantTransaction(&ryw->getTransaction(), tenantName));
+			if (tenantEntry.get().tenantGroup.present()) {
+				(*tenantGroupNetTenantDelta)[tenantEntry.get().tenantGroup.get()]--;
+			}
+		}
+
 		return Void();
 	}
 
 	ACTOR static Future<Void> deleteTenantRange(ReadYourWritesTransaction* ryw,
 	                                            TenantName beginTenant,
-	                                            TenantName endTenant) {
-		state std::map<TenantName, TenantMapEntry> tenants = wait(
+	                                            TenantName endTenant,
+	                                            std::map<TenantGroupName, int>* tenantGroupNetTenantDelta) {
+		state std::vector<std::pair<TenantName, TenantMapEntry>> tenants = wait(
 		    TenantAPI::listTenantsTransaction(&ryw->getTransaction(), beginTenant, endTenant, CLIENT_KNOBS->TOO_MANY));
 
 		if (tenants.size() == CLIENT_KNOBS->TOO_MANY) {
@@ -153,15 +219,41 @@ private:
 		std::vector<Future<Void>> deleteFutures;
 		for (auto tenant : tenants) {
 			deleteFutures.push_back(TenantAPI::deleteTenantTransaction(&ryw->getTransaction(), tenant.first));
+			if (tenant.second.tenantGroup.present()) {
+				(*tenantGroupNetTenantDelta)[tenant.second.tenantGroup.get()]--;
+			}
 		}
+
 		wait(waitForAll(deleteFutures));
+		return Void();
+	}
+
+	// Check if the number of tenants in the tenant group is equal to the net reduction in the number of tenants.
+	// If it is, then we can delete the tenant group.
+	ACTOR static Future<Void> checkAndRemoveTenantGroup(ReadYourWritesTransaction* ryw,
+	                                                    TenantGroupName tenantGroup,
+	                                                    int tenantDelta) {
+		ASSERT(tenantDelta < 0);
+		state int removedTenants = -tenantDelta;
+		KeyBackedSet<Tuple>::RangeResultType tenantsInGroup =
+		    wait(TenantMetadata::tenantGroupTenantIndex().getRange(&ryw->getTransaction(),
+		                                                           Tuple::makeTuple(tenantGroup),
+		                                                           Tuple::makeTuple(keyAfter(tenantGroup)),
+		                                                           removedTenants + 1));
+
+		ASSERT(tenantsInGroup.results.size() >= removedTenants);
+		if (tenantsInGroup.results.size() == removedTenants) {
+			TenantMetadata::tenantGroupMap().erase(&ryw->getTransaction(), tenantGroup);
+		}
 
 		return Void();
 	}
 
 public:
-	const static KeyRangeRef submoduleRange;
-	const static KeyRangeRef mapSubRange;
+	const inline static KeyRangeRef submoduleRange = KeyRangeRef("tenant/"_sr, "tenant0"_sr);
+	const inline static KeyRangeRef mapSubRange = KeyRangeRef("map/"_sr, "map0"_sr);
+	const inline static KeyRangeRef configureSubRange = KeyRangeRef("configure/"_sr, "configure0"_sr);
+	const inline static KeyRangeRef renameSubRange = KeyRangeRef("rename/"_sr, "rename0"_sr);
 
 	explicit TenantRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
@@ -171,18 +263,33 @@ public:
 		return getTenantRange(ryw, kr, limitsHint);
 	}
 
-	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override {
-		auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
-		std::vector<Future<Void>> tenantManagementFutures;
+	ACTOR static Future<Optional<std::string>> commitImpl(TenantRangeImpl* self, ReadYourWritesTransaction* ryw) {
+		state std::vector<Future<Void>> tenantManagementFutures;
 
-		std::vector<std::pair<KeyRangeRef, Optional<Value>>> mapMutations;
+		// This map is an ugly workaround to the fact that we cannot use RYW in these transactions.
+		// It tracks the net change to the number of tenants in a tenant group, and at the end we can compare
+		// that with how many tenants the tenant group started with. If we removed all of the tenants, then we
+		// delete the tenant group.
+		//
+		// SOMEDAY: enable RYW support in special keys and remove this complexity.
+		state std::map<TenantGroupName, int> tenantGroupNetTenantDelta;
+
+		state KeyRangeMap<std::pair<bool, Optional<Value>>>::Ranges ranges =
+		    ryw->getSpecialKeySpaceWriteMap().containedRanges(self->range);
+
+		state std::vector<std::pair<KeyRangeRef, Optional<Value>>> mapMutations;
+		state std::map<TenantName, std::vector<std::pair<Standalone<StringRef>, Optional<Value>>>> configMutations;
+		state std::set<TenantName> renameSet;
+		state std::vector<std::pair<TenantName, TenantName>> renameMutations;
+
+		tenantManagementFutures.push_back(TenantAPI::checkTenantMode(&ryw->getTransaction(), ClusterType::STANDALONE));
 
 		for (auto range : ranges) {
 			if (!range.value().first) {
 				continue;
 			}
 
-			KeyRangeRef adjustedRange =
+			state KeyRangeRef adjustedRange =
 			    range.range()
 			        .removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
 			        .removePrefix(submoduleRange.begin);
@@ -191,31 +298,104 @@ public:
 				adjustedRange = mapSubRange & adjustedRange;
 				adjustedRange = removePrefix(adjustedRange, mapSubRange.begin, "\xff"_sr);
 				mapMutations.push_back(std::make_pair(adjustedRange, range.value().second));
+			} else if (configureSubRange.intersects(adjustedRange) && adjustedRange.singleKeyRange()) {
+				StringRef configTupleStr = adjustedRange.begin.removePrefix(configureSubRange.begin);
+				try {
+					Tuple tuple = Tuple::unpack(configTupleStr);
+					if (tuple.size() != 2) {
+						throw invalid_tuple_index();
+					}
+					configMutations[tuple.getString(0)].push_back(
+					    std::make_pair(tuple.getString(1), range.value().second));
+				} catch (Error& e) {
+					TraceEvent(SevWarn, "InvalidTenantConfigurationKey").error(e).detail("Key", adjustedRange.begin);
+					ryw->setSpecialKeySpaceErrorMsg(ManagementAPIError::toJsonString(
+					    false, "configure tenant", "invalid tenant configuration key"));
+					throw special_keys_api_failure();
+				}
+			} else if (renameSubRange.intersects(adjustedRange)) {
+				StringRef oldName = adjustedRange.begin.removePrefix(renameSubRange.begin);
+				StringRef newName = range.value().second.get();
+				// Do not allow overlapping renames in the same commit
+				// e.g. A->B + B->C, D->D
+				if (renameSet.count(oldName) || renameSet.count(newName) || oldName == newName) {
+					ryw->setSpecialKeySpaceErrorMsg(
+					    ManagementAPIError::toJsonString(false, "rename tenant", "tenant rename conflict"));
+					throw special_keys_api_failure();
+				}
+				renameSet.insert(oldName);
+				renameSet.insert(newName);
+				renameMutations.push_back(std::make_pair(oldName, newName));
 			}
 		}
 
-		std::vector<TenantNameRef> tenantsToCreate;
+		std::map<TenantName, std::vector<std::pair<Standalone<StringRef>, Optional<Value>>>> tenantsToCreate;
 		for (auto mapMutation : mapMutations) {
 			TenantNameRef tenantName = mapMutation.first.begin;
+			auto set_iter = renameSet.lower_bound(tenantName);
+			if (set_iter != renameSet.end() && mapMutation.first.contains(*set_iter)) {
+				ryw->setSpecialKeySpaceErrorMsg(
+				    ManagementAPIError::toJsonString(false, "rename tenant", "tenant rename conflict"));
+				throw special_keys_api_failure();
+			}
 			if (mapMutation.second.present()) {
-				tenantsToCreate.push_back(tenantName);
+				std::vector<std::pair<Standalone<StringRef>, Optional<Value>>> createMutations;
+				auto itr = configMutations.find(tenantName);
+				if (itr != configMutations.end()) {
+					createMutations = itr->second;
+					configMutations.erase(itr);
+				}
+				tenantsToCreate[tenantName] = createMutations;
 			} else {
 				// For a single key clear, just issue the delete
 				if (mapMutation.first.singleKeyRange()) {
-					tenantManagementFutures.push_back(
-					    TenantAPI::deleteTenantTransaction(&ryw->getTransaction(), tenantName));
+					tenantManagementFutures.push_back(deleteSingleTenant(ryw, tenantName, &tenantGroupNetTenantDelta));
+
+					// Configuration changes made to a deleted tenant are discarded
+					configMutations.erase(tenantName);
 				} else {
-					tenantManagementFutures.push_back(deleteTenantRange(ryw, tenantName, mapMutation.first.end));
+					tenantManagementFutures.push_back(
+					    deleteTenantRange(ryw, tenantName, mapMutation.first.end, &tenantGroupNetTenantDelta));
+
+					// Configuration changes made to a deleted tenant are discarded
+					configMutations.erase(configMutations.lower_bound(tenantName),
+					                      configMutations.lower_bound(mapMutation.first.end));
 				}
 			}
 		}
 
 		if (!tenantsToCreate.empty()) {
-			tenantManagementFutures.push_back(createTenants(ryw, tenantsToCreate));
+			tenantManagementFutures.push_back(createTenants(ryw, tenantsToCreate, &tenantGroupNetTenantDelta));
+		}
+		for (auto configMutation : configMutations) {
+			if (renameSet.count(configMutation.first)) {
+				ryw->setSpecialKeySpaceErrorMsg(
+				    ManagementAPIError::toJsonString(false, "rename tenant", "tenant rename conflict"));
+				throw special_keys_api_failure();
+			}
+			tenantManagementFutures.push_back(
+			    changeTenantConfig(ryw, configMutation.first, configMutation.second, &tenantGroupNetTenantDelta));
 		}
 
-		return tag(waitForAll(tenantManagementFutures), Optional<std::string>());
+		for (auto renameMutation : renameMutations) {
+			tenantManagementFutures.push_back(TenantAPI::renameTenantTransaction(
+			    &ryw->getTransaction(), renameMutation.first, renameMutation.second));
+		}
+
+		wait(waitForAll(tenantManagementFutures));
+
+		state std::vector<Future<Void>> tenantGroupUpdateFutures;
+		for (auto [tenantGroup, count] : tenantGroupNetTenantDelta) {
+			if (count < 0) {
+				tenantGroupUpdateFutures.push_back(checkAndRemoveTenantGroup(ryw, tenantGroup, count));
+			}
+		}
+
+		wait(waitForAll(tenantGroupUpdateFutures));
+		return Optional<std::string>();
 	}
+
+	Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override { return commitImpl(this, ryw); }
 };
 
 #include "flow/unactorcompiler.h"
diff --git a/fdbclient/include/fdbclient/ThreadSafeTransaction.h b/fdbclient/include/fdbclient/ThreadSafeTransaction.h
index 875664ea76..0d7c6f608d 100644
--- a/fdbclient/include/fdbclient/ThreadSafeTransaction.h
+++ b/fdbclient/include/fdbclient/ThreadSafeTransaction.h
@@ -20,6 +20,7 @@
 
 #ifndef FDBCLIENT_THREADSAFETRANSACTION_H
 #define FDBCLIENT_THREADSAFETRANSACTION_H
+#include "flow/ApiVersion.h"
 #include "flow/ProtocolVersion.h"
 #pragma once
 
@@ -62,6 +63,13 @@ public:
 	ThreadFuture<Key> purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) override;
 	ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
 
+	ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
+	ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
+	ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
+	                                                                      int rangeLimit) override;
+
+	ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
+
 	ThreadFuture<DatabaseSharedState*> createSharedState() override;
 	void setSharedState(DatabaseSharedState* p) override;
 
@@ -72,7 +80,8 @@ private:
 	DatabaseContext* db;
 
 public: // Internal use only
-	ThreadSafeDatabase(Reference<IClusterConnectionRecord> connectionRecord, int apiVersion);
+	enum class ConnectionRecordType { FILE, CONNECTION_STRING };
+	ThreadSafeDatabase(ConnectionRecordType connectionRecordType, std::string connectionRecord, int apiVersion);
 	ThreadSafeDatabase(DatabaseContext* db) : db(db) {}
 	DatabaseContext* unsafeGetPtr() const { return db; }
 };
@@ -148,13 +157,30 @@ public:
 	ThreadFuture<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRangeRef& range,
 	                                                                int64_t chunkSize) override;
 
-	ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRangeRef& keyRange) override;
+	ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> getBlobGranuleRanges(const KeyRangeRef& keyRange,
+	                                                                      int rangeLimit) override;
 
 	ThreadResult<RangeResult> readBlobGranules(const KeyRangeRef& keyRange,
 	                                           Version beginVersion,
 	                                           Optional<Version> readVersion,
 	                                           ReadBlobGranuleContext granuleContext) override;
 
+	ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> readBlobGranulesStart(const KeyRangeRef& keyRange,
+	                                                                               Version beginVersion,
+	                                                                               Optional<Version> readVersion,
+	                                                                               Version* readVersionOut) override;
+
+	ThreadResult<RangeResult> readBlobGranulesFinish(
+	    ThreadFuture<Standalone<VectorRef<BlobGranuleChunkRef>>> startFuture,
+	    const KeyRangeRef& keyRange,
+	    Version beginVersion,
+	    Version readVersion,
+	    ReadBlobGranuleContext granuleContext) override;
+
+	ThreadFuture<Standalone<VectorRef<BlobGranuleSummaryRef>>> summarizeBlobGranules(const KeyRangeRef& keyRange,
+	                                                                                 Optional<Version> summaryVersion,
+	                                                                                 int rangeLimit) override;
+
 	void addReadConflictRange(const KeyRangeRef& keys) override;
 	void makeSelfConflicting();
 
@@ -205,6 +231,7 @@ class ThreadSafeApi : public IClientApi, ThreadSafeReferenceCounted<ThreadSafeAp
 public:
 	void selectApiVersion(int apiVersion) override;
 	const char* getClientVersion() override;
+	void useFutureProtocolVersion() override;
 
 	void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
 	void setupNetwork() override;
@@ -220,8 +247,8 @@ private:
 	friend IClientApi* getLocalClientAPI();
 	ThreadSafeApi();
 
-	int apiVersion;
-	const std::string clientVersion;
+	ApiVersion apiVersion;
+	std::string clientVersion;
 	uint64_t transportId;
 
 	Mutex lock;
diff --git a/fdbclient/include/fdbclient/Tracing.h b/fdbclient/include/fdbclient/Tracing.h
index 30fa210e3f..789b346dfd 100644
--- a/fdbclient/include/fdbclient/Tracing.h
+++ b/fdbclient/include/fdbclient/Tracing.h
@@ -50,8 +50,7 @@ struct SpanContext {
 	SpanContext() : traceID(UID()), spanID(0), m_Flags(TraceFlags::unsampled) {}
 	SpanContext(UID traceID, uint64_t spanID, TraceFlags flags) : traceID(traceID), spanID(spanID), m_Flags(flags) {}
 	SpanContext(UID traceID, uint64_t spanID) : traceID(traceID), spanID(spanID), m_Flags(TraceFlags::unsampled) {}
-	SpanContext(Arena arena, const SpanContext& span)
-	  : traceID(span.traceID), spanID(span.spanID), m_Flags(span.m_Flags) {}
+	SpanContext(const SpanContext& span) = default;
 	bool isSampled() const { return (m_Flags & TraceFlags::sampled) == TraceFlags::sampled; }
 	std::string toString() const { return format("%016llx%016llx%016llx", traceID.first(), traceID.second(), spanID); };
 	bool isValid() const { return traceID.first() != 0 && traceID.second() != 0 && spanID != 0; }
@@ -62,6 +61,9 @@ struct SpanContext {
 	}
 };
 
+template <>
+struct flow_ref<SpanContext> : std::false_type {};
+
 // Span
 //
 // Span is a tracing implementation which, for the most part, complies with the W3C Trace Context specification
@@ -155,7 +157,7 @@ public:
 	// We've determined for initial tracing release, spans with only a location will not be traced.
 	// Generally these are for background processes, some are called infrequently, while others may be high volume.
 	// TODO: review and address in subsequent PRs.
-	Span(const Location& location) : location(location), begin(g_network->now()) {}
+	explicit Span(const Location& location) : Span(location, SpanContext()) {}
 
 	Span(const Span&) = delete;
 	Span(Span&& o) {
diff --git a/fdbclient/include/fdbclient/Tuple.h b/fdbclient/include/fdbclient/Tuple.h
index 4c52c1ebc6..bf997e4309 100644
--- a/fdbclient/include/fdbclient/Tuple.h
+++ b/fdbclient/include/fdbclient/Tuple.h
@@ -28,6 +28,19 @@
 #include "fdbclient/Versionstamp.h"
 
 struct Tuple {
+	struct UnicodeStr {
+		StringRef str;
+		explicit UnicodeStr(StringRef str) : str(str) {}
+	};
+
+	struct UserTypeStr {
+		uint8_t code;
+		Standalone<StringRef> str;
+		UserTypeStr(uint8_t code, StringRef str) : code(code), str(str) {}
+
+		bool operator==(const UserTypeStr& other) const { return (code == other.code && str == other.str); }
+	};
+
 	Tuple() {}
 
 	// Tuple parsing normally does not care of the final value is a numeric type and is incomplete.
@@ -35,35 +48,44 @@ struct Tuple {
 	// Note that strings can't be incomplete because they are parsed such that the end of the packed
 	// byte string is considered the end of the string in lieu of a specific end.
 	static Tuple unpack(StringRef const& str, bool exclude_incomplete = false);
+	static Tuple unpackUserType(StringRef const& str, bool exclude_incomplete = false);
 
 	Tuple& append(Tuple const& tuple);
 
 	// the str needs to be a Tuple encoded string.
 	Tuple& appendRaw(StringRef const& str);
 	Tuple& append(StringRef const& str, bool utf8 = false);
+	Tuple& append(UnicodeStr const& str);
+	Tuple& append(int32_t);
 	Tuple& append(int64_t);
-	// There are some ambiguous append calls in fdbclient, so to make it easier
-	// to add append for floats and doubles, name them differently for now.
-	Tuple& appendBool(bool);
-	Tuple& appendFloat(float);
-	Tuple& appendDouble(double);
+	Tuple& append(bool);
+	Tuple& append(float);
+	Tuple& append(double);
+	Tuple& append(std::nullptr_t);
 	Tuple& appendNull();
-	Tuple& appendVersionstamp(Versionstamp const&);
+	Tuple& append(Versionstamp const&);
+	Tuple& append(UserTypeStr const&);
 
-	StringRef pack() const { return StringRef(data.begin(), data.size()); }
+	Standalone<StringRef> pack() const {
+		return Standalone<StringRef>(StringRef(data.begin(), data.size()), data.arena());
+	}
 
 	template <typename T>
 	Tuple& operator<<(T const& t) {
 		return append(t);
 	}
 
-	enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, VERSIONSTAMP };
+	enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, VERSIONSTAMP, USER_TYPE };
+
+	bool isUserType(uint8_t code) const;
 
 	// this is number of elements, not length of data
 	size_t size() const { return offsets.size(); }
 	void reserve(size_t cap) { offsets.reserve(cap); }
 	void clear() {
-		data.clear();
+		// Make a new Standalone to use different memory so that
+		// previously returned objects from pack() are valid.
+		data = Standalone<VectorRef<uint8_t>>();
 		offsets.clear();
 	}
 	// Return a Tuple encoded raw string.
@@ -75,6 +97,7 @@ struct Tuple {
 	bool getBool(size_t index) const;
 	float getFloat(size_t index) const;
 	double getDouble(size_t index) const;
+	Tuple::UserTypeStr getUserType(size_t index) const;
 
 	KeyRange range(Tuple const& tuple = Tuple()) const;
 
@@ -84,8 +107,20 @@ struct Tuple {
 	Standalone<VectorRef<uint8_t>> getData() { return data; }
 	Standalone<StringRef> getDataAsStandalone() { return Standalone<StringRef>(pack(), data.arena()); }
 
+	// Create a tuple from a parameter pack
+	template <class... Types>
+	static Tuple makeTuple(Types&&... args) {
+		Tuple t;
+
+		// Use a fold expression to append each argument using the << operator.
+		// https://en.cppreference.com/w/cpp/language/fold
+		(t << ... << std::forward<Types>(args));
+
+		return t;
+	}
+
 private:
-	Tuple(const StringRef& data, bool exclude_incomplete = false);
+	Tuple(const StringRef& data, bool exclude_incomplete = false, bool exclude_user_type = false);
 	Standalone<VectorRef<uint8_t>> data;
 	std::vector<size_t> offsets;
 };
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index 57ae180a07..8df1bcc150 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -57,6 +57,8 @@ description is not currently required but encouraged.
     <Option name="trace_file_identifier" code="36"
             paramType="String" paramDescription="The identifier that will be part of all trace file names"
             description="Once provided, this string will be used to replace the port/PID in the log file names." />
+    <Option name="trace_share_among_client_threads" code="37"
+            description="Use the same base trace file name for all client threads as it did before version 7.2. The current default behavior is to use distinct trace file names for client threads by including their version and thread index." />
     <Option name="trace_partial_file_suffix" code="39"
             paramType="String" paramDescription="Append this suffix to partially written log files. When a log file is complete, it is renamed to remove the suffix. No separator is added between the file and the suffix. If you want to add a file extension, you should include the separator - e.g. '.tmp' instead of 'tmp' to add the 'tmp' extension."
             description="Set file suffix for partially written log files." />
@@ -115,12 +117,17 @@ description is not currently required but encouraged.
     <Option name="client_threads_per_version" code="65"
             paramType="Int" paramDescription="Number of client threads to be spawned.  Each cluster will be serviced by a single client thread."
             description="Spawns multiple worker threads for each version of the client that is loaded.  Setting this to a number greater than one implies disable_local_client." />
+    <Option name="future_version_client_library" code="66"
+            paramType="String" paramDescription="path to client library"
+            description="Adds an external client library to be used with a future version protocol. This option can be used testing purposes only!" />
     <Option name="disable_client_statistics_logging" code="70"
             description="Disables logging of client statistics, such as sampled transaction activity." />
     <Option name="enable_slow_task_profiling" code="71"
             description="Deprecated" />
     <Option name="enable_run_loop_profiling" code="71"
             description="Enables debugging feature to perform run loop profiling. Requires trace logging to be enabled. WARNING: this feature is not recommended for use in production." />
+    <Option name="disable_client_bypass" code="72"
+            description="Prevents the multi-version client API from being disabled, even if no external clients are configured. This option is required to use GRV caching."/>
     <Option name="client_buggify_enable" code="80"
             description="Enable client buggify - will make requests randomly fail (intended for client testing)" />
     <Option name="client_buggify_disable" code="81"
@@ -298,10 +305,13 @@ description is not currently required but encouraged.
     <Option name="bypass_unreadable" code="1100"
                 description="Allows ``get`` operations to read from sections of keyspace that have become unreadable because of versionstamp operations. These reads will view versionstamp operations as if they were set operations that did not fill in the versionstamp." />            
     <Option name="use_grv_cache" code="1101"
-            description="Allows this transaction to use cached GRV from the database context. Defaults to off. Upon first usage, starts a background updater to periodically update the cache to avoid stale read versions." />
+            description="Allows this transaction to use cached GRV from the database context. Defaults to off. Upon first usage, starts a background updater to periodically update the cache to avoid stale read versions. The disable_client_bypass option must also be set." />
     <Option name="skip_grv_cache" code="1102"
             description="Specifically instruct this transaction to NOT use cached GRV. Primarily used for the read version cache's background updater to avoid attempting to read a cached entry in specific situations."
             hidden="true"/>
+    <Option name="authorization_token" code="2000"
+            description="Attach given authorization token to the transaction such that subsequent tenant-aware requests are authorized"
+            paramType="String" paramDescription="A JSON Web Token authorized to access data belonging to one or more tenants, indicated by 'tenants' claim of the token's payload."/>
   </Scope>
 
   <!-- The enumeration values matter - do not change them without
diff --git a/fdbkubernetesmonitor/api/config.go b/fdbkubernetesmonitor/api/config.go
index 04d3500157..07839ccf37 100644
--- a/fdbkubernetesmonitor/api/config.go
+++ b/fdbkubernetesmonitor/api/config.go
@@ -21,8 +21,10 @@ package api
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"strconv"
+	"strings"
 )
 
 // ProcessConfiguration models the configuration for starting a FoundationDB
@@ -63,8 +65,11 @@ type Argument struct {
 	Multiplier int `json:"multiplier,omitempty"`
 
 	// Offset provides an offset to add to the process number for ProcessNumber
-	// type argujments.
+	// type arguments.
 	Offset int `json:"offset,omitempty"`
+
+	// IPFamily provides the family to use for IPList type arguments.
+	IPFamily int `json:"ipFamily,omitempty"`
 }
 
 // ArgumentType defines the types for arguments.
@@ -84,6 +89,10 @@ const (
 	// ProcessNumberArgumentType defines an argument that is calculated using
 	// the number of the process in the process list.
 	ProcessNumberArgumentType = "ProcessNumber"
+
+	// IPListArgumentType defines an argument that is a comma-separated list of
+	// IP addresses, provided through an environment variable.
+	IPListArgumentType = "IPList"
 )
 
 // GenerateArgument processes an argument and generates its string
@@ -111,24 +120,52 @@ func (argument Argument) GenerateArgument(processNumber int, env map[string]stri
 		}
 		number = number + argument.Offset
 		return strconv.Itoa(number), nil
-	case EnvironmentArgumentType:
-		var value string
-		var present bool
-		if env != nil {
-			value, present = env[argument.Source]
-		}
-		if !present {
-			value, present = os.LookupEnv(argument.Source)
-		}
-		if !present {
-			return "", fmt.Errorf("Missing environment variable %s", argument.Source)
-		}
-		return value, nil
+	case EnvironmentArgumentType, IPListArgumentType:
+		return argument.LookupEnv(env)
 	default:
 		return "", fmt.Errorf("unsupported argument type %s", argument.ArgumentType)
 	}
 }
 
+// LookupEnv looks up the value for an argument from the environment.
+func (argument Argument) LookupEnv(env map[string]string) (string, error) {
+	var value string
+	var present bool
+	if env != nil {
+		value, present = env[argument.Source]
+	}
+	if !present {
+		value, present = os.LookupEnv(argument.Source)
+	}
+	if !present {
+		return "", fmt.Errorf("missing environment variable %s", argument.Source)
+	}
+
+	if argument.ArgumentType == IPListArgumentType {
+		ips := strings.Split(value, ",")
+		for _, ipString := range ips {
+			ip := net.ParseIP(ipString)
+			if ip == nil {
+				continue
+			}
+			switch argument.IPFamily {
+			case 4:
+				if ip.To4() != nil {
+					return ipString, nil
+				}
+			case 6:
+				if ip.To16() != nil && ip.To4() == nil {
+					return ipString, nil
+				}
+			default:
+				return "", fmt.Errorf("unsupported IP family %d", argument.IPFamily)
+			}
+		}
+		return "", fmt.Errorf("could not find IP with family %d", argument.IPFamily)
+	}
+	return value, nil
+}
+
 // GenerateArguments interprets the arguments in the process configuration and
 // generates a command invocation.
 func (configuration *ProcessConfiguration) GenerateArguments(processNumber int, env map[string]string) ([]string, error) {
diff --git a/fdbkubernetesmonitor/api/config_test.go b/fdbkubernetesmonitor/api/config_test.go
index adaf49457d..5f80e75f35 100644
--- a/fdbkubernetesmonitor/api/config_test.go
+++ b/fdbkubernetesmonitor/api/config_test.go
@@ -120,10 +120,216 @@ func TestGeneratingArgumentForEnvironmentVariable(t *testing.T) {
 		t.Fail()
 		return
 	}
-	expectedError := "Missing environment variable FDB_ZONE_ID"
+	expectedError := "missing environment variable FDB_ZONE_ID"
 	if err.Error() != expectedError {
 		t.Logf("Expected error %s, but got error %s", expectedError, err)
 		t.Fail()
 		return
 	}
 }
+
+func TestGeneratingArgumentForIPList(t *testing.T) {
+	argument := Argument{ArgumentType: IPListArgumentType, Source: "FDB_PUBLIC_IP", IPFamily: 4}
+
+	result, err := argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "127.0.0.1" {
+		t.Logf("Expected result 127.0.0.1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "127.0.0.1" {
+		t.Logf("Expected result 127.0.0.1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	argument.IPFamily = 6
+
+	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "bad,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	_, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
+	if err == nil {
+		t.Logf("Expected error, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError := "could not find IP with family 6"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
+		t.Fail()
+		return
+	}
+
+	argument.IPFamily = 5
+
+	_, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
+	if err == nil {
+		t.Logf("Expected error, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError = "unsupported IP family 5"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
+		t.Fail()
+		return
+	}
+}
+
+func TestLookupEnvForEnvironmentVariable(t *testing.T) {
+	argument := Argument{ArgumentType: EnvironmentArgumentType, Source: "FDB_ZONE_ID"}
+
+	result, err := argument.LookupEnv(map[string]string{"FDB_ZONE_ID": "zone1", "FDB_MACHINE_ID": "machine1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "zone1" {
+		t.Logf("Expected result zone1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	_, err = argument.LookupEnv(map[string]string{"FDB_MACHINE_ID": "machine1"})
+	if err == nil {
+		t.Logf("Expected error result, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError := "missing environment variable FDB_ZONE_ID"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err)
+		t.Fail()
+		return
+	}
+}
+
+func TestLookupEnvForIPList(t *testing.T) {
+	argument := Argument{ArgumentType: IPListArgumentType, Source: "FDB_PUBLIC_IP", IPFamily: 4}
+
+	result, err := argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "127.0.0.1" {
+		t.Logf("Expected result 127.0.0.1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "127.0.0.1" {
+		t.Logf("Expected result 127.0.0.1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	argument.IPFamily = 6
+
+	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "bad,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	_, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
+	if err == nil {
+		t.Logf("Expected error, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError := "could not find IP with family 6"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
+		t.Fail()
+		return
+	}
+
+	argument.IPFamily = 5
+
+	_, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
+	if err == nil {
+		t.Logf("Expected error, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError = "unsupported IP family 5"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
+		t.Fail()
+		return
+	}
+}
diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index cefd492dc7..ea5e5d0d72 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -96,13 +96,16 @@ func CreatePodClient(logger logr.Logger) (*PodClient, error) {
 
 // retrieveEnvironmentVariables extracts the environment variables we have for
 // an argument into a map.
-func retrieveEnvironmentVariables(argument api.Argument, target map[string]string) {
+func retrieveEnvironmentVariables(monitor *Monitor, argument api.Argument, target map[string]string) {
 	if argument.Source != "" {
-		target[argument.Source] = os.Getenv(argument.Source)
+		value, err := argument.LookupEnv(monitor.CustomEnvironment)
+		if err == nil {
+			target[argument.Source] = value
+		}
 	}
 	if argument.Values != nil {
 		for _, childArgument := range argument.Values {
-			retrieveEnvironmentVariables(childArgument, target)
+			retrieveEnvironmentVariables(monitor, childArgument, target)
 		}
 	}
 }
@@ -112,7 +115,7 @@ func retrieveEnvironmentVariables(argument api.Argument, target map[string]strin
 func (client *PodClient) UpdateAnnotations(monitor *Monitor) error {
 	environment := make(map[string]string)
 	for _, argument := range monitor.ActiveConfiguration.Arguments {
-		retrieveEnvironmentVariables(argument, environment)
+		retrieveEnvironmentVariables(monitor, argument, environment)
 	}
 	environment["BINARY_DIR"] = path.Dir(monitor.ActiveConfiguration.BinaryPath)
 	jsonEnvironment, err := json.Marshal(environment)
diff --git a/fdbrpc/AsyncFileCached.actor.cpp b/fdbrpc/AsyncFileCached.actor.cpp
index 93bf9cfc70..1d56974094 100644
--- a/fdbrpc/AsyncFileCached.actor.cpp
+++ b/fdbrpc/AsyncFileCached.actor.cpp
@@ -193,14 +193,14 @@ Future<Void> AsyncFileCached::changeFileSize(int64_t size) {
 	prevLength = size;
 
 	if (offsetInPage) {
-		TEST(true); // Truncating to the middle of a page
+		CODE_PROBE(true, "Truncating to the middle of a page");
 		auto p = pages.find(pageOffset);
 		if (p != pages.end()) {
 			auto f = p->second->flush();
 			if (!f.isReady() || f.isError())
 				actors.push_back(f);
 		} else {
-			TEST(true); // Truncating to the middle of a page that isn't in cache
+			CODE_PROBE(true, "Truncating to the middle of a page that isn't in cache");
 		}
 
 		pageOffset += pageCache->pageSize;
diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp
index bf1c44da59..ec37f79414 100644
--- a/fdbrpc/AsyncFileEncrypted.actor.cpp
+++ b/fdbrpc/AsyncFileEncrypted.actor.cpp
@@ -258,7 +258,7 @@ Optional<Standalone<StringRef>> AsyncFileEncrypted::RandomCache::get(uint32_t bl
 TEST_CASE("fdbrpc/AsyncFileEncrypted") {
 	state const int bytes = FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * deterministicRandom()->randomInt(0, 1000);
 	state std::vector<unsigned char> writeBuffer(bytes, 0);
-	generateRandomData(&writeBuffer.front(), bytes);
+	deterministicRandom()->randomBytes(&writeBuffer.front(), bytes);
 	state std::vector<unsigned char> readBuffer(bytes, 0);
 	ASSERT(g_network->isSimulated());
 	StreamCipherKey::initializeGlobalRandomTestKey();
diff --git a/fdbrpc/AsyncFileNonDurable.actor.cpp b/fdbrpc/AsyncFileNonDurable.actor.cpp
index f7b41c5ae2..559c7cfce3 100644
--- a/fdbrpc/AsyncFileNonDurable.actor.cpp
+++ b/fdbrpc/AsyncFileNonDurable.actor.cpp
@@ -24,7 +24,7 @@
 std::map<std::string, Future<Void>> AsyncFileNonDurable::filesBeingDeleted;
 
 ACTOR Future<Void> sendOnProcess(ISimulator::ProcessInfo* process, Promise<Void> promise, TaskPriority taskID) {
-	wait(g_simulator.onProcess(process, taskID));
+	wait(g_simulator->onProcess(process, taskID));
 	promise.send(Void());
 	return Void();
 }
@@ -33,7 +33,7 @@ ACTOR Future<Void> sendErrorOnProcess(ISimulator::ProcessInfo* process,
                                       Promise<Void> promise,
                                       Error e,
                                       TaskPriority taskID) {
-	wait(g_simulator.onProcess(process, taskID));
+	wait(g_simulator->onProcess(process, taskID));
 	promise.sendError(e);
 	return Void();
 }
diff --git a/fdbrpc/Base64UrlDecode.cpp b/fdbrpc/Base64UrlDecode.cpp
index 073060d566..644fd4412d 100644
--- a/fdbrpc/Base64UrlDecode.cpp
+++ b/fdbrpc/Base64UrlDecode.cpp
@@ -117,6 +117,9 @@ Optional<StringRef> decode(Arena& arena, StringRef base64UrlStr) {
 	}
 	auto out = new (arena) uint8_t[decodedLen];
 	auto actualLen = decode(base64UrlStr.begin(), base64UrlStr.size(), out);
+	if (actualLen == -1) {
+		return {};
+	}
 	ASSERT_EQ(decodedLen, actualLen);
 	return StringRef(out, decodedLen);
 }
diff --git a/fdbrpc/CMakeLists.txt b/fdbrpc/CMakeLists.txt
index 0a892af5d7..48ece5eedc 100644
--- a/fdbrpc/CMakeLists.txt
+++ b/fdbrpc/CMakeLists.txt
@@ -61,10 +61,10 @@ if(${COROUTINE_IMPL} STREQUAL libcoro)
 endif()
 
 target_include_directories(fdbrpc PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include" PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/libeio)
-target_link_libraries(fdbrpc PUBLIC flow PRIVATE rapidjson)
+target_link_libraries(fdbrpc PUBLIC flow libb64 md5 PRIVATE rapidjson)
 
 target_include_directories(fdbrpc_sampling PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include" PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/libeio)
-target_link_libraries(fdbrpc_sampling PUBLIC flow_sampling PRIVATE rapidjson)
+target_link_libraries(fdbrpc_sampling PUBLIC flow_sampling libb64 md5 PRIVATE rapidjson)
 
 if(${COROUTINE_IMPL} STREQUAL libcoro)
   target_link_libraries(fdbrpc PUBLIC coro)
@@ -80,3 +80,5 @@ target_compile_definitions(fdbrpc_sampling PRIVATE -DENABLE_SAMPLING)
 if(WIN32)
   add_dependencies(fdbrpc_sampling_actors fdbrpc_actors)
 endif()
+
+add_subdirectory(tests)
diff --git a/fdbrpc/FlowTests.actor.cpp b/fdbrpc/FlowTests.actor.cpp
index c3a18f9b33..e2715fd45b 100644
--- a/fdbrpc/FlowTests.actor.cpp
+++ b/fdbrpc/FlowTests.actor.cpp
@@ -48,7 +48,7 @@ TEST_CASE("/flow/actorcompiler/lineNumbers") {
 		}
 		break;
 	}
-	ASSERT(LiteralStringRef(__FILE__).endsWith(LiteralStringRef("FlowTests.actor.cpp")));
+	ASSERT(__FILE__sr.endsWith("FlowTests.actor.cpp"_sr));
 	return Void();
 }
 
@@ -69,7 +69,7 @@ TEST_CASE("/flow/buggifiedDelay") {
 		});
 		wait(f1 && f2);
 		if (last == 1) {
-			TEST(true); // Delays can become ready out of order
+			CODE_PROBE(true, "Delays can become ready out of order");
 			return Void();
 		}
 	}
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 2dddf2ab48..8bb476efc9 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -28,12 +28,16 @@
 #include <memcheck.h>
 #endif
 
-#include "fdbrpc/TenantInfo.h"
+#include <boost/unordered_map.hpp>
+
+#include "fdbrpc/TokenSign.h"
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/HealthMonitor.h"
+#include "fdbrpc/JsonWebKeySet.h"
 #include "fdbrpc/genericactors.actor.h"
 #include "fdbrpc/IPAllowList.h"
+#include "fdbrpc/TokenCache.h"
 #include "fdbrpc/simulator.h"
 #include "flow/ActorCollection.h"
 #include "flow/Error.h"
@@ -41,14 +45,21 @@
 #include "flow/Net2Packet.h"
 #include "flow/TDMetric.actor.h"
 #include "flow/ObjectSerializer.h"
+#include "flow/Platform.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/UnitTest.h"
+#include "flow/WatchFile.actor.h"
 #define XXH_INLINE_ALL
 #include "flow/xxhash.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-static NetworkAddressList g_currentDeliveryPeerAddress = NetworkAddressList();
-static Future<Void> g_currentDeliveryPeerDisconnect;
+namespace {
+
+NetworkAddressList g_currentDeliveryPeerAddress = NetworkAddressList();
+bool g_currentDeliverPeerAddressTrusted = false;
+Future<Void> g_currentDeliveryPeerDisconnect;
+
+} // namespace
 
 constexpr int PACKET_LEN_WIDTH = sizeof(uint32_t);
 const uint64_t TOKEN_STREAM_FLAG = 1;
@@ -239,31 +250,6 @@ struct PingReceiver final : NetworkMessageReceiver {
 	bool isPublic() const override { return true; }
 };
 
-struct TenantAuthorizer final : NetworkMessageReceiver {
-	TenantAuthorizer(EndpointMap& endpoints) {
-		endpoints.insertWellKnown(this, Endpoint::wellKnownToken(WLTOKEN_AUTH_TENANT), TaskPriority::ReadSocket);
-	}
-	void receive(ArenaObjectReader& reader) override {
-		AuthorizationRequest req;
-		try {
-			reader.deserialize(req);
-			// TODO: verify that token is valid
-			AuthorizedTenants& auth = reader.variable<AuthorizedTenants>("AuthorizedTenants");
-			for (auto const& t : req.tenants) {
-				auth.authorizedTenants.insert(TenantInfoRef(auth.arena, t));
-			}
-			req.reply.send(Void());
-		} catch (Error& e) {
-			if (e.code() == error_code_permission_denied) {
-				req.reply.sendError(e);
-			} else {
-				throw;
-			}
-		}
-	}
-	bool isPublic() const override { return true; }
-};
-
 struct UnauthorizedEndpointReceiver final : NetworkMessageReceiver {
 	UnauthorizedEndpointReceiver(EndpointMap& endpoints) {
 		endpoints.insertWellKnown(
@@ -313,12 +299,12 @@ public:
 	~TransportData();
 
 	void initMetrics() {
-		bytesSent.init(LiteralStringRef("Net2.BytesSent"));
-		countPacketsReceived.init(LiteralStringRef("Net2.CountPacketsReceived"));
-		countPacketsGenerated.init(LiteralStringRef("Net2.CountPacketsGenerated"));
-		countConnEstablished.init(LiteralStringRef("Net2.CountConnEstablished"));
-		countConnClosedWithError.init(LiteralStringRef("Net2.CountConnClosedWithError"));
-		countConnClosedWithoutError.init(LiteralStringRef("Net2.CountConnClosedWithoutError"));
+		bytesSent.init("Net2.BytesSent"_sr);
+		countPacketsReceived.init("Net2.CountPacketsReceived"_sr);
+		countPacketsGenerated.init("Net2.CountPacketsGenerated"_sr);
+		countConnEstablished.init("Net2.CountConnEstablished"_sr);
+		countConnClosedWithError.init("Net2.CountConnClosedWithError"_sr);
+		countConnClosedWithoutError.init("Net2.CountConnClosedWithoutError"_sr);
 	}
 
 	Reference<struct Peer> getPeer(NetworkAddress const& address);
@@ -326,6 +312,7 @@ public:
 
 	// Returns true if given network address 'address' is one of the address we are listening on.
 	bool isLocalAddress(const NetworkAddress& address) const;
+	void applyPublicKeySet(StringRef jwkSetString);
 
 	NetworkAddressCachedString localAddresses;
 	std::vector<Future<Void>> listeners;
@@ -339,7 +326,6 @@ public:
 	EndpointMap endpoints;
 	EndpointNotFoundReceiver endpointNotFoundReceiver{ endpoints };
 	PingReceiver pingReceiver{ endpoints };
-	TenantAuthorizer tenantReceiver{ endpoints };
 	UnauthorizedEndpointReceiver unauthorizedEndpointReceiver{ endpoints };
 
 	Int64MetricHandle bytesSent;
@@ -356,10 +342,12 @@ public:
 	double lastIncompatibleMessage;
 	uint64_t transportId;
 	IPAllowList allowList;
-	std::shared_ptr<ContextVariableMap> localCVM = std::make_shared<ContextVariableMap>(); // for local delivery
 
 	Future<Void> multiVersionCleanup;
 	Future<Void> pingLogger;
+	Future<Void> publicKeyFileWatch;
+
+	std::unordered_map<Standalone<StringRef>, PublicKey> publicKeys;
 };
 
 ACTOR Future<Void> pingLatencyLogger(TransportData* self) {
@@ -615,8 +603,8 @@ ACTOR Future<Void> connectionWriter(Reference<Peer> self, Reference<IConnection>
 				break;
 			}
 
-			TEST(true); // We didn't write everything, so apparently the write buffer is full.  Wait for it to be
-			            // nonfull.
+			CODE_PROBE(
+			    true, "We didn't write everything, so apparently the write buffer is full.  Wait for it to be nonfull");
 			wait(conn->onWritable());
 			wait(yield(TaskPriority::WriteSocket));
 		}
@@ -926,10 +914,20 @@ void Peer::prependConnectPacket() {
 	pkt.protocolVersion.addObjectSerializerFlag();
 	pkt.connectionId = transport->transportId;
 
-	PacketBuffer* pb_first = PacketBuffer::create();
+	PacketBuffer *pb_first = PacketBuffer::create(), *pb_end = nullptr;
 	PacketWriter wr(pb_first, nullptr, Unversioned());
 	pkt.serialize(wr);
-	unsent.prependWriteBuffer(pb_first, wr.finish());
+	pb_end = wr.finish();
+#if VALGRIND
+	SendBuffer* checkbuf = pb_first;
+	while (checkbuf) {
+		int size = checkbuf->bytes_written;
+		const uint8_t* data = checkbuf->data();
+		VALGRIND_CHECK_MEM_IS_DEFINED(data, size);
+		checkbuf = checkbuf->next;
+	}
+#endif
+	unsent.prependWriteBuffer(pb_first, pb_end);
 }
 
 void Peer::discardUnreliablePackets() {
@@ -965,7 +963,7 @@ void Peer::onIncomingConnection(Reference<Peer> self, Reference<IConnection> con
 		    .detail("FromAddr", conn->getPeerAddress())
 		    .detail("CanonicalAddr", destination)
 		    .detail("IsPublic", destination.isPublic())
-		    .detail("Trusted", self->transport->allowList(conn->getPeerAddress().ip));
+		    .detail("Trusted", self->transport->allowList(conn->getPeerAddress().ip) && conn->hasTrustedPeer());
 
 		connect.cancel();
 		prependConnectPacket();
@@ -1013,8 +1011,7 @@ ACTOR static void deliver(TransportData* self,
                           TaskPriority priority,
                           ArenaReader reader,
                           NetworkAddress peerAddress,
-                          Reference<AuthorizedTenants> authorizedTenants,
-                          std::shared_ptr<ContextVariableMap> cvm,
+                          bool isTrustedPeer,
                           InReadSocket inReadSocket,
                           Future<Void> disconnect) {
 	// We want to run the task at the right priority. If the priority is higher than the current priority (which is
@@ -1029,22 +1026,26 @@ ACTOR static void deliver(TransportData* self,
 	}
 
 	auto receiver = self->endpoints.get(destination.token);
-	if (receiver && (authorizedTenants->trusted || receiver->isPublic())) {
+	if (receiver && (isTrustedPeer || receiver->isPublic())) {
 		if (!checkCompatible(receiver->peerCompatibilityPolicy(), reader.protocolVersion())) {
 			return;
 		}
 		try {
+			ASSERT(g_currentDeliveryPeerAddress == NetworkAddressList());
+			ASSERT(!g_currentDeliverPeerAddressTrusted);
 			g_currentDeliveryPeerAddress = destination.addresses;
+			g_currentDeliverPeerAddressTrusted = isTrustedPeer;
 			g_currentDeliveryPeerDisconnect = disconnect;
 			StringRef data = reader.arenaReadAll();
 			ASSERT(data.size() > 8);
 			ArenaObjectReader objReader(reader.arena(), reader.arenaReadAll(), AssumeVersion(reader.protocolVersion()));
-			objReader.setContextVariableMap(cvm);
 			receiver->receive(objReader);
-			g_currentDeliveryPeerAddress = { NetworkAddress() };
+			g_currentDeliveryPeerAddress = NetworkAddressList();
+			g_currentDeliverPeerAddressTrusted = false;
 			g_currentDeliveryPeerDisconnect = Future<Void>();
 		} catch (Error& e) {
-			g_currentDeliveryPeerAddress = { NetworkAddress() };
+			g_currentDeliveryPeerAddress = NetworkAddressList();
+			g_currentDeliverPeerAddressTrusted = false;
 			g_currentDeliveryPeerDisconnect = Future<Void>();
 			TraceEvent(SevError, "ReceiverError")
 			    .error(e)
@@ -1092,8 +1093,7 @@ static void scanPackets(TransportData* transport,
                         const uint8_t* e,
                         Arena& arena,
                         NetworkAddress const& peerAddress,
-                        Reference<AuthorizedTenants> const& authorizedTenants,
-                        std::shared_ptr<ContextVariableMap> cvm,
+                        bool isTrustedPeer,
                         ProtocolVersion peerProtocolVersion,
                         Future<Void> disconnect,
                         IsStableConnection isStableConnection) {
@@ -1136,9 +1136,10 @@ static void scanPackets(TransportData* transport,
 		if (checksumEnabled) {
 			bool isBuggifyEnabled = false;
 			if (g_network->isSimulated() && !isStableConnection &&
-			    g_network->now() - g_simulator.lastConnectionFailure > g_simulator.connectionFailuresDisableDuration &&
+			    g_network->now() - g_simulator->lastConnectionFailure >
+			        g_simulator->connectionFailuresDisableDuration &&
 			    BUGGIFY_WITH_PROB(0.0001)) {
-				g_simulator.lastConnectionFailure = g_network->now();
+				g_simulator->lastConnectionFailure = g_network->now();
 				isBuggifyEnabled = true;
 				TraceEvent(SevInfo, "BitsFlip").log();
 				int flipBits = 32 - (int)floor(log2(deterministicRandom()->randomUInt32()));
@@ -1215,8 +1216,7 @@ static void scanPackets(TransportData* transport,
 			        priority,
 			        std::move(reader),
 			        peerAddress,
-			        authorizedTenants,
-			        cvm,
+			        isTrustedPeer,
 			        InReadSocket::True,
 			        disconnect);
 		}
@@ -1263,14 +1263,9 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 	state bool incompatiblePeerCounted = false;
 	state NetworkAddress peerAddress;
 	state ProtocolVersion peerProtocolVersion;
-	state Reference<AuthorizedTenants> authorizedTenants = makeReference<AuthorizedTenants>();
-	state std::shared_ptr<ContextVariableMap> cvm = std::make_shared<ContextVariableMap>();
+	state bool trusted = transport->allowList(conn->getPeerAddress().ip) && conn->hasTrustedPeer();
 	peerAddress = conn->getPeerAddress();
-	authorizedTenants->trusted = transport->allowList(conn->getPeerAddress().ip);
-	(*cvm)["AuthorizedTenants"] = &authorizedTenants;
-	(*cvm)["PeerAddress"] = &peerAddress;
 
-	authorizedTenants->trusted = transport->allowList(peerAddress.ip);
 	if (!peer) {
 		ASSERT(!peerAddress.isPublic());
 	}
@@ -1420,8 +1415,7 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 						            unprocessed_end,
 						            arena,
 						            peerAddress,
-						            authorizedTenants,
-						            cvm,
+						            trusted,
 						            peerProtocolVersion,
 						            peer->disconnect.getFuture(),
 						            IsStableConnection(g_network->isSimulated() && conn->isStableConnection()));
@@ -1462,7 +1456,7 @@ ACTOR static Future<Void> connectionIncoming(TransportData* self, Reference<ICon
 			}
 			when(Reference<Peer> p = wait(onConnected.getFuture())) { p->onIncomingConnection(p, conn, reader); }
 			when(wait(delayJittered(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
-				TEST(true); // Incoming connection timed out
+				CODE_PROBE(true, "Incoming connection timed out");
 				throw timed_out();
 			}
 		}
@@ -1487,6 +1481,7 @@ ACTOR static Future<Void> listen(TransportData* self, NetworkAddress listenAddr)
 		TraceEvent(SevInfo, "UpdatingListenAddress")
 		    .detail("AssignedListenAddress", listener->getListenAddress().toString());
 		self->localAddresses.setNetworkAddress(listener->getListenAddress());
+		setTraceLocalAddress(listener->getListenAddress());
 	}
 	state uint64_t connectionCount = 0;
 	try {
@@ -1540,6 +1535,27 @@ bool TransportData::isLocalAddress(const NetworkAddress& address) const {
 	        address == localAddresses.getAddressList().secondaryAddress.get());
 }
 
+void TransportData::applyPublicKeySet(StringRef jwkSetString) {
+	auto jwks = JsonWebKeySet::parse(jwkSetString, {});
+	if (!jwks.present())
+		throw pkey_decode_error();
+	const auto& keySet = jwks.get().keys;
+	publicKeys.clear();
+	int numPrivateKeys = 0;
+	for (auto [keyName, key] : keySet) {
+		// ignore private keys
+		if (key.isPublic()) {
+			publicKeys[keyName] = key.getPublic();
+		} else {
+			numPrivateKeys++;
+		}
+	}
+	TraceEvent(SevInfo, "AuthzPublicKeySetApply").detail("NumPublicKeys", publicKeys.size());
+	if (numPrivateKeys > 0) {
+		TraceEvent(SevWarnAlways, "AuthzPublicKeySetContainsPrivateKeys").detail("NumPrivateKeys", numPrivateKeys);
+	}
+}
+
 ACTOR static Future<Void> multiVersionCleanupWorker(TransportData* self) {
 	loop {
 		wait(delay(FLOW_KNOBS->CONNECTION_CLEANUP_DELAY));
@@ -1572,6 +1588,11 @@ ACTOR static Future<Void> multiVersionCleanupWorker(TransportData* self) {
 FlowTransport::FlowTransport(uint64_t transportId, int maxWellKnownEndpoints, IPAllowList const* allowList)
   : self(new TransportData(transportId, maxWellKnownEndpoints, allowList)) {
 	self->multiVersionCleanup = multiVersionCleanupWorker(self);
+	if (g_network->isSimulated()) {
+		for (auto const& p : g_simulator->authKeys) {
+			self->publicKeys.emplace(p.first, p.second.toPublic());
+		}
+	}
 }
 
 FlowTransport::~FlowTransport() {
@@ -1703,7 +1724,7 @@ void FlowTransport::addWellKnownEndpoint(Endpoint& endpoint, NetworkMessageRecei
 }
 
 static void sendLocal(TransportData* self, ISerializeSource const& what, const Endpoint& destination) {
-	TEST(true); // "Loopback" delivery
+	CODE_PROBE(true, "\"Loopback\" delivery");
 	// SOMEDAY: Would it be better to avoid (de)serialization by doing this check in flow?
 
 	Standalone<StringRef> copy;
@@ -1717,15 +1738,12 @@ static void sendLocal(TransportData* self, ISerializeSource const& what, const E
 	ASSERT(copy.size() > 0);
 	TaskPriority priority = self->endpoints.getPriority(destination.token);
 	if (priority != TaskPriority::UnknownEndpoint || (destination.token.first() & TOKEN_STREAM_FLAG) != 0) {
-		Reference<AuthorizedTenants> authorizedTenants = makeReference<AuthorizedTenants>();
-		authorizedTenants->trusted = true;
 		deliver(self,
 		        destination,
 		        priority,
-		        ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion)),
+		        ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion())),
 		        NetworkAddress(),
-		        authorizedTenants,
-		        self->localCVM,
+		        true,
 		        InReadSocket::False,
 		        Never());
 	}
@@ -1742,7 +1760,7 @@ static ReliablePacket* sendPacket(TransportData* self,
 	// If there isn't an open connection, a public address, or the peer isn't compatible, we can't send
 	if (!peer || (peer->outgoingConnectionIdle && !destination.getPrimaryAddress().isPublic()) ||
 	    (!peer->compatible && destination.token != Endpoint::wellKnownToken(WLTOKEN_PING_PACKET))) {
-		TEST(true); // Can't send to private address without a compatible open connection
+		CODE_PROBE(true, "Can't send to private address without a compatible open connection");
 		return nullptr;
 	}
 
@@ -1936,6 +1954,7 @@ void FlowTransport::createInstance(bool isClient,
                                    uint64_t transportId,
                                    int maxWellKnownEndpoints,
                                    IPAllowList const* allowList) {
+	TokenCache::createInstance();
 	g_network->setGlobal(INetwork::enFlowTransport,
 	                     (flowGlobalType) new FlowTransport(transportId, maxWellKnownEndpoints, allowList));
 	g_network->setGlobal(INetwork::enNetworkAddressFunc, (flowGlobalType)&FlowTransport::getGlobalLocalAddress);
@@ -1947,3 +1966,90 @@ void FlowTransport::createInstance(bool isClient,
 HealthMonitor* FlowTransport::healthMonitor() {
 	return &self->healthMonitor;
 }
+
+Optional<PublicKey> FlowTransport::getPublicKeyByName(StringRef name) const {
+	auto iter = self->publicKeys.find(name);
+	if (iter != self->publicKeys.end()) {
+		return iter->second;
+	}
+	return {};
+}
+
+NetworkAddress FlowTransport::currentDeliveryPeerAddress() const {
+	return g_currentDeliveryPeerAddress.address;
+}
+
+bool FlowTransport::currentDeliveryPeerIsTrusted() const {
+	return g_currentDeliverPeerAddressTrusted;
+}
+
+void FlowTransport::addPublicKey(StringRef name, PublicKey key) {
+	self->publicKeys[name] = key;
+}
+
+void FlowTransport::removePublicKey(StringRef name) {
+	self->publicKeys.erase(name);
+}
+
+void FlowTransport::removeAllPublicKeys() {
+	self->publicKeys.clear();
+}
+
+void FlowTransport::loadPublicKeyFile(const std::string& filePath) {
+	if (!fileExists(filePath)) {
+		throw file_not_found();
+	}
+	int64_t const len = fileSize(filePath);
+	if (len <= 0) {
+		TraceEvent(SevWarn, "AuthzPublicKeySetEmpty").detail("Path", filePath);
+	} else if (len > FLOW_KNOBS->PUBLIC_KEY_FILE_MAX_SIZE) {
+		throw file_too_large();
+	} else {
+		auto json = readFileBytes(filePath, len);
+		self->applyPublicKeySet(StringRef(json));
+	}
+}
+
+ACTOR static Future<Void> watchPublicKeyJwksFile(std::string filePath, TransportData* self) {
+	state AsyncTrigger fileChanged;
+	state Future<Void> fileWatch;
+	state unsigned errorCount = 0; // error since watch start or last successful refresh
+
+	// Make sure this watch setup does not break due to async file system initialization not having been called
+	loop {
+		if (IAsyncFileSystem::filesystem())
+			break;
+		wait(delay(1.0));
+	}
+	const int& intervalSeconds = FLOW_KNOBS->PUBLIC_KEY_FILE_REFRESH_INTERVAL_SECONDS;
+	fileWatch = watchFileForChanges(filePath, &fileChanged, &intervalSeconds, "AuthzPublicKeySetRefreshStatError");
+	loop {
+		try {
+			wait(fileChanged.onTrigger());
+			state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(
+			    filePath, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0));
+			state int64_t filesize = wait(file->size());
+			state std::string json(filesize, '\0');
+			if (filesize > FLOW_KNOBS->PUBLIC_KEY_FILE_MAX_SIZE)
+				throw file_too_large();
+			if (filesize <= 0) {
+				TraceEvent(SevWarn, "AuthzPublicKeySetEmpty").suppressFor(60);
+				continue;
+			}
+			wait(success(file->read(&json[0], filesize, 0)));
+			self->applyPublicKeySet(StringRef(json));
+			errorCount = 0;
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				throw;
+			}
+			// parse/read error
+			errorCount++;
+			TraceEvent(SevWarn, "AuthzPublicKeySetRefreshError").error(e).detail("ErrorCount", errorCount);
+		}
+	}
+}
+
+void FlowTransport::watchPublicKeyFile(const std::string& publicKeyFilePath) {
+	self->publicKeyFileWatch = watchPublicKeyJwksFile(publicKeyFilePath, self);
+}
diff --git a/fdbclient/HTTP.actor.cpp b/fdbrpc/HTTP.actor.cpp
similarity index 74%
rename from fdbclient/HTTP.actor.cpp
rename to fdbrpc/HTTP.actor.cpp
index 6cecf7fd39..cf0bf6e157 100644
--- a/fdbclient/HTTP.actor.cpp
+++ b/fdbrpc/HTTP.actor.cpp
@@ -18,12 +18,11 @@
  * limitations under the License.
  */
 
-#include "fdbclient/HTTP.h"
+#include "fdbrpc/HTTP.h"
 
-#include "fdbclient/md5/md5.h"
-#include "fdbclient/ClientKnobs.h"
-#include "fdbclient/libb64/encode.h"
-#include "fdbclient/Knobs.h"
+#include "md5/md5.h"
+#include "libb64/encode.h"
+#include "flow/Knobs.h"
 #include <cctype>
 
 #include "flow/actorcompiler.h" // has to be last include
@@ -101,14 +100,14 @@ PacketBuffer* writeRequestHeader(std::string const& verb,
 	writer.serializeBytes(verb);
 	writer.serializeBytes(" ", 1);
 	writer.serializeBytes(resource);
-	writer.serializeBytes(LiteralStringRef(" HTTP/1.1\r\n"));
+	writer.serializeBytes(" HTTP/1.1\r\n"_sr);
 	for (auto h : headers) {
 		writer.serializeBytes(h.first);
-		writer.serializeBytes(LiteralStringRef(": "));
+		writer.serializeBytes(": "_sr);
 		writer.serializeBytes(h.second);
-		writer.serializeBytes(LiteralStringRef("\r\n"));
+		writer.serializeBytes("\r\n"_sr);
 	}
-	writer.serializeBytes(LiteralStringRef("\r\n"));
+	writer.serializeBytes("\r\n"_sr);
 	return writer.finish();
 }
 
@@ -153,7 +152,7 @@ ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn,
 		// Next search will start at the current end of the buffer - delim size + 1
 		if (sPos >= lookBack)
 			sPos -= lookBack;
-		wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
+		wait(success(read_into_string(conn, buf, FLOW_KNOBS->HTTP_READ_SIZE)));
 	}
 }
 
@@ -161,7 +160,7 @@ ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn,
 ACTOR Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string* buf, size_t pos) {
 	state int stop_size = pos + len;
 	while (buf->size() < stop_size)
-		wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
+		wait(success(read_into_string(conn, buf, FLOW_KNOBS->HTTP_READ_SIZE)));
 	return Void();
 }
 
@@ -329,7 +328,7 @@ ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<ICo
 
 	// If there is actual response content, check the MD5 sum against the Content-MD5 response header
 	if (r->content.size() > 0) {
-		if (r->code == 206 && CLIENT_KNOBS->HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT) {
+		if (r->code == 206 && FLOW_KNOBS->HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT) {
 			return Void();
 		}
 
@@ -368,7 +367,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 	// There is no standard http request id header field, so either a global default can be set via a knob
 	// or it can be set per-request with the requestIDHeader argument (which overrides the default)
 	if (requestIDHeader.empty()) {
-		requestIDHeader = CLIENT_KNOBS->HTTP_REQUEST_ID_HEADER;
+		requestIDHeader = FLOW_KNOBS->HTTP_REQUEST_ID_HEADER;
 	}
 
 	state bool earlyResponse = false;
@@ -400,19 +399,19 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 		// Prepend headers to content packer buffer chain
 		pContent->prependWriteBuffer(pFirst, pLast);
 
-		if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1)
+		if (FLOW_KNOBS->HTTP_VERBOSE_LEVEL > 1)
 			printf("[%s] HTTP starting %s %s ContentLen:%d\n",
 			       conn->getDebugID().toString().c_str(),
 			       verb.c_str(),
 			       resource.c_str(),
 			       contentLen);
-		if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
+		if (FLOW_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
 			for (auto h : headers)
 				printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str());
 		}
 
 		state Reference<HTTP::Response> r(new HTTP::Response());
-		state Future<Void> responseReading = r->read(conn, verb == "HEAD" || verb == "DELETE");
+		state Future<Void> responseReading = r->read(conn, verb == "HEAD" || verb == "DELETE" || verb == "CONNECT");
 
 		send_start = timer();
 
@@ -427,7 +426,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 				break;
 			}
 
-			state int trySend = CLIENT_KNOBS->HTTP_SEND_SIZE;
+			state int trySend = FLOW_KNOBS->HTTP_SEND_SIZE;
 			wait(sendRate->getAllowance(trySend));
 			int len = conn->write(pContent->getUnsent(), trySend);
 			if (pSent != nullptr)
@@ -481,7 +480,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 			}
 		}
 
-		if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
+		if (FLOW_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
 			printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
 			       conn->getDebugID().toString().c_str(),
 			       (err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
@@ -494,7 +493,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 			       total_sent,
 			       (int)r->contentLen);
 		}
-		if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
+		if (FLOW_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
 			printf("[%s] HTTP RESPONSE:  %s %s\n%s\n",
 			       conn->getDebugID().toString().c_str(),
 			       verb.c_str(),
@@ -510,7 +509,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 	} catch (Error& e) {
 		double elapsed = timer() - send_start;
 		// A bad_request_id error would have already been logged in verbose mode before err is thrown above.
-		if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
+		if (FLOW_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
 			printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n",
 			       conn->getDebugID().toString().c_str(),
 			       e.name(),
@@ -526,4 +525,135 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 	}
 }
 
+ACTOR Future<Void> sendProxyConnectRequest(Reference<IConnection> conn,
+                                           std::string remoteHost,
+                                           std::string remoteService) {
+	state Headers headers;
+	headers["Host"] = remoteHost + ":" + remoteService;
+	headers["Accept"] = "application/xml";
+	headers["Proxy-Connection"] = "Keep-Alive";
+	state int requestTimeout = 60;
+	state int maxTries = FLOW_KNOBS->RESTCLIENT_CONNECT_TRIES;
+	state int thisTry = 1;
+	state double nextRetryDelay = 2.0;
+	state Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
+	state int64_t bytes_sent = 0;
+
+	loop {
+		state Optional<Error> err;
+		state Reference<Response> r;
+
+		try {
+			Reference<Response> _r = wait(timeoutError(doRequest(conn,
+			                                                     "CONNECT",
+			                                                     remoteHost + ":" + remoteService,
+			                                                     headers,
+			                                                     nullptr,
+			                                                     0,
+			                                                     sendReceiveRate,
+			                                                     &bytes_sent,
+			                                                     sendReceiveRate),
+			                                           requestTimeout));
+			r = _r;
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled)
+				throw;
+			err = e;
+		}
+
+		// If err is not present then r is valid.
+		// If r->code is in successCodes then record the successful request and return r.
+		if (!err.present() && r->code == 200) {
+			return Void();
+		}
+
+		// All errors in err are potentially retryable as well as certain HTTP response codes...
+		bool retryable = err.present() || r->code == 500 || r->code == 502 || r->code == 503 || r->code == 429;
+
+		// But only if our previous attempt was not the last allowable try.
+		retryable = retryable && (thisTry < maxTries);
+
+		TraceEvent event(SevWarn, retryable ? "ProxyConnectCommandFailedRetryable" : "ProxyConnectCommandFailed");
+
+		// Attach err to trace event if present, otherwise extract some stuff from the response
+		if (err.present()) {
+			event.errorUnsuppressed(err.get());
+		}
+		event.suppressFor(60);
+		if (!err.present()) {
+			event.detail("ResponseCode", r->code);
+		}
+
+		event.detail("ThisTry", thisTry);
+
+		// If r is not valid or not code 429 then increment the try count.  429's will not count against the attempt
+		// limit.
+		if (!r || r->code != 429)
+			++thisTry;
+
+		// We will wait delay seconds before the next retry, start with nextRetryDelay.
+		double delay = nextRetryDelay;
+		// Double but limit the *next* nextRetryDelay.
+		nextRetryDelay = std::min(nextRetryDelay * 2, 60.0);
+
+		if (retryable) {
+			// If r is valid then obey the Retry-After response header if present.
+			if (r) {
+				auto iRetryAfter = r->headers.find("Retry-After");
+				if (iRetryAfter != r->headers.end()) {
+					event.detail("RetryAfterHeader", iRetryAfter->second);
+					char* pEnd;
+					double retryAfter = strtod(iRetryAfter->second.c_str(), &pEnd);
+					if (*pEnd) // If there were other characters then don't trust the parsed value, use a probably safe
+					           // value of 5 minutes.
+						retryAfter = 300;
+					// Update delay
+					delay = std::max(delay, retryAfter);
+				}
+			}
+
+			// Log the delay then wait.
+			event.detail("RetryDelay", delay);
+			wait(::delay(delay));
+		} else {
+			// We can't retry, so throw something.
+
+			// This error code means the authentication header was not accepted, likely the account or key is wrong.
+			if (r && r->code == 406)
+				throw http_not_accepted();
+
+			if (r && r->code == 401)
+				throw http_auth_failed();
+
+			throw connection_failed();
+		}
+	}
+}
+
+ACTOR Future<Reference<IConnection>> proxyConnectImpl(std::string remoteHost,
+                                                      std::string remoteService,
+                                                      std::string proxyHost,
+                                                      std::string proxyService) {
+	state NetworkAddress remoteEndpoint =
+	    wait(map(INetworkConnections::net()->resolveTCPEndpoint(remoteHost, remoteService),
+	             [=](std::vector<NetworkAddress> const& addresses) -> NetworkAddress {
+		             NetworkAddress addr = addresses[deterministicRandom()->randomInt(0, addresses.size())];
+		             addr.fromHostname = true;
+		             addr.flags = NetworkAddress::FLAG_TLS;
+		             return addr;
+	             }));
+	state Reference<IConnection> connection = wait(INetworkConnections::net()->connect(proxyHost, proxyService));
+	wait(sendProxyConnectRequest(connection, remoteHost, remoteService));
+	boost::asio::ip::tcp::socket socket = std::move(connection->getSocket());
+	Reference<IConnection> remoteConnection = wait(INetworkConnections::net()->connect(remoteEndpoint, &socket));
+	return remoteConnection;
+}
+
+Future<Reference<IConnection>> proxyConnect(const std::string& remoteHost,
+                                            const std::string& remoteService,
+                                            const std::string& proxyHost,
+                                            const std::string& proxyService) {
+	return proxyConnectImpl(remoteHost, remoteService, proxyHost, proxyService);
+}
+
 } // namespace HTTP
diff --git a/fdbrpc/JsonWebKeySet.cpp b/fdbrpc/JsonWebKeySet.cpp
index a83fd7051e..bf78799bd4 100644
--- a/fdbrpc/JsonWebKeySet.cpp
+++ b/fdbrpc/JsonWebKeySet.cpp
@@ -830,11 +830,18 @@ TEST_CASE("/fdbrpc/JsonWebKeySet/EC/PrivateKey") {
 }
 
 TEST_CASE("/fdbrpc/JsonWebKeySet/RSA/PublicKey") {
-	testPublicKey(&mkcert::makeRsa2048Bit);
+	testPublicKey(&mkcert::makeRsa4096Bit);
 	return Void();
 }
 
 TEST_CASE("/fdbrpc/JsonWebKeySet/RSA/PrivateKey") {
-	testPrivateKey(&mkcert::makeRsa2048Bit);
+	testPrivateKey(&mkcert::makeRsa4096Bit);
+	return Void();
+}
+
+TEST_CASE("/fdbrpc/JsonWebKeySet/Empty") {
+	auto keyset = JsonWebKeySet::parse("{\"keys\":[]}"_sr, {});
+	ASSERT(keyset.present());
+	ASSERT(keyset.get().keys.empty());
 	return Void();
 }
diff --git a/fdbrpc/Locality.cpp b/fdbrpc/Locality.cpp
index bd4c3a9461..724b539ffb 100644
--- a/fdbrpc/Locality.cpp
+++ b/fdbrpc/Locality.cpp
@@ -21,13 +21,13 @@
 #include "fdbrpc/Locality.h"
 
 const UID LocalityData::UNSET_ID = UID(0x0ccb4e0feddb5583, 0x010f6b77d9d10ece);
-const StringRef LocalityData::keyProcessId = LiteralStringRef("processid");
-const StringRef LocalityData::keyZoneId = LiteralStringRef("zoneid");
-const StringRef LocalityData::keyDcId = LiteralStringRef("dcid");
-const StringRef LocalityData::keyMachineId = LiteralStringRef("machineid");
-const StringRef LocalityData::keyDataHallId = LiteralStringRef("data_hall");
-const StringRef LocalityData::ExcludeLocalityKeyMachineIdPrefix = LiteralStringRef("locality_machineid:");
-const StringRef LocalityData::ExcludeLocalityPrefix = LiteralStringRef("locality_");
+const StringRef LocalityData::keyProcessId = "processid"_sr;
+const StringRef LocalityData::keyZoneId = "zoneid"_sr;
+const StringRef LocalityData::keyDcId = "dcid"_sr;
+const StringRef LocalityData::keyMachineId = "machineid"_sr;
+const StringRef LocalityData::keyDataHallId = "data_hall"_sr;
+const StringRef LocalityData::ExcludeLocalityKeyMachineIdPrefix = "locality_machineid:"_sr;
+const StringRef LocalityData::ExcludeLocalityPrefix = "locality_"_sr;
 
 ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const {
 	switch (role) {
@@ -240,6 +240,24 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const
 		default:
 			return ProcessClass::WorstFit;
 		}
+	case ProcessClass::ConsistencyScan:
+		switch (_class) {
+		case ProcessClass::ConsistencyScanClass:
+			return ProcessClass::BestFit;
+		case ProcessClass::StatelessClass:
+			return ProcessClass::GoodFit;
+		case ProcessClass::UnsetClass:
+			return ProcessClass::UnsetFit;
+		case ProcessClass::MasterClass:
+			return ProcessClass::OkayFit;
+		case ProcessClass::CoordinatorClass:
+		case ProcessClass::TesterClass:
+		case ProcessClass::StorageCacheClass:
+		case ProcessClass::BlobWorkerClass:
+			return ProcessClass::NeverAssign;
+		default:
+			return ProcessClass::WorstFit;
+		}
 	case ProcessClass::BlobManager:
 		switch (_class) {
 		case ProcessClass::BlobManagerClass:
diff --git a/fdbrpc/ReplicationUtils.cpp b/fdbrpc/ReplicationUtils.cpp
index cd84eea44f..bd55708614 100644
--- a/fdbrpc/ReplicationUtils.cpp
+++ b/fdbrpc/ReplicationUtils.cpp
@@ -493,10 +493,10 @@ Reference<LocalitySet> createTestLocalityMap(std::vector<repTestType>& indexes,
 					serverValue = dcLoop + szLoop * 10 + rackLoop * 100 + slotLoop * 1000;
 					slotText = format(".%d", slotLoop);
 					LocalityData data;
-					data.set(LiteralStringRef("dc"), StringRef(dcText));
-					data.set(LiteralStringRef("sz"), StringRef(dcText + szText));
-					data.set(LiteralStringRef("rack"), StringRef(dcText + szText + rackText));
-					data.set(LiteralStringRef("zoneid"), StringRef(dcText + szText + rackText + slotText));
+					data.set("dc"_sr, StringRef(dcText));
+					data.set("sz"_sr, StringRef(dcText + szText));
+					data.set("rack"_sr, StringRef(dcText + szText + rackText));
+					data.set("zoneid"_sr, StringRef(dcText + szText + rackText + slotText));
 					for (int independentLoop = 0; independentLoop < independentItems; independentLoop++) {
 						independentName = format("indiv%02d", independentLoop + 1);
 						for (int totalLoop = 0; totalLoop < independentTotal; totalLoop++) {
@@ -520,10 +520,10 @@ Reference<LocalitySet> createTestLocalityMap(std::vector<repTestType>& indexes,
 					serverValue = (dcLoop + 2) + szLoop * 10 + rackLoop * 100 + slotLoop * 1000;
 					slotText = format(".%d", slotLoop);
 					LocalityData data;
-					data.set(LiteralStringRef("dc"), StringRef(dcText));
-					data.set(LiteralStringRef("az"), StringRef(dcText + szText));
-					data.set(LiteralStringRef("rack"), StringRef(dcText + szText + rackText));
-					data.set(LiteralStringRef("zoneid"), StringRef(dcText + szText + rackText + slotText));
+					data.set("dc"_sr, StringRef(dcText));
+					data.set("az"_sr, StringRef(dcText + szText));
+					data.set("rack"_sr, StringRef(dcText + szText + rackText));
+					data.set("zoneid"_sr, StringRef(dcText + szText + rackText + slotText));
 					for (int independentLoop = 0; independentLoop < independentItems; independentLoop++) {
 						independentName = format("indiv%02d", independentLoop);
 						for (int totalLoop = 0; totalLoop < independentTotal; totalLoop++) {
diff --git a/fdbrpc/SimExternalConnection.actor.cpp b/fdbrpc/SimExternalConnection.actor.cpp
index 440dda5ffc..95bf3e03c1 100644
--- a/fdbrpc/SimExternalConnection.actor.cpp
+++ b/fdbrpc/SimExternalConnection.actor.cpp
@@ -125,6 +125,10 @@ NetworkAddress SimExternalConnection::getPeerAddress() const {
 	}
 }
 
+bool SimExternalConnection::hasTrustedPeer() const {
+	return true;
+}
+
 UID SimExternalConnection::getDebugID() const {
 	return dbgid;
 }
diff --git a/fdbrpc/TokenCache.actor.cpp b/fdbrpc/TokenCache.actor.cpp
new file mode 100644
index 0000000000..afa1fecf10
--- /dev/null
+++ b/fdbrpc/TokenCache.actor.cpp
@@ -0,0 +1,385 @@
+#include "fdbrpc/FlowTransport.h"
+#include "fdbrpc/TokenCache.h"
+#include "fdbrpc/TokenSign.h"
+#include "fdbrpc/TenantInfo.h"
+#include "flow/MkCert.h"
+#include "flow/ScopeExit.h"
+#include "flow/UnitTest.h"
+#include "flow/network.h"
+
+#include <boost/unordered_map.hpp>
+
+#include <fmt/format.h>
+#include <list>
+#include <deque>
+
+#include "flow/actorcompiler.h" // has to be last include
+
+template <class Key, class Value>
+class LRUCache {
+public:
+	using key_type = Key;
+	using list_type = std::list<key_type>;
+	using mapped_type = Value;
+	using map_type = boost::unordered_map<key_type, std::pair<mapped_type, typename list_type::iterator>>;
+	using size_type = unsigned;
+
+	explicit LRUCache(size_type capacity) : _capacity(capacity) { _map.reserve(capacity); }
+
+	size_type size() const { return _map.size(); }
+	size_type capacity() const { return _capacity; }
+	bool empty() const { return _map.empty(); }
+
+	Optional<mapped_type*> get(key_type const& key) {
+		auto i = _map.find(key);
+		if (i == _map.end()) {
+			return Optional<mapped_type*>();
+		}
+		auto j = i->second.second;
+		if (j != _list.begin()) {
+			_list.erase(j);
+			_list.push_front(i->first);
+			i->second.second = _list.begin();
+		}
+		return &i->second.first;
+	}
+
+	template <class K, class V>
+	mapped_type* insert(K&& key, V&& value) {
+		auto iter = _map.find(key);
+		if (iter != _map.end()) {
+			return &iter->second.first;
+		}
+		if (size() == capacity()) {
+			auto i = --_list.end();
+			_map.erase(*i);
+			_list.erase(i);
+		}
+		_list.push_front(std::forward<K>(key));
+		std::tie(iter, std::ignore) =
+		    _map.insert(std::make_pair(*_list.begin(), std::make_pair(std::forward<V>(value), _list.begin())));
+		return &iter->second.first;
+	}
+
+private:
+	const size_type _capacity;
+	map_type _map;
+	list_type _list;
+};
+
+TEST_CASE("/fdbrpc/authz/LRUCache") {
+	auto& rng = *deterministicRandom();
+	{
+		// test very small LRU cache
+		LRUCache<int, StringRef> cache(rng.randomInt(2, 10));
+		for (int i = 0; i < 200; ++i) {
+			cache.insert(i, "val"_sr);
+			if (i >= cache.capacity()) {
+				for (auto j = 0; j <= i - cache.capacity(); j++)
+					ASSERT(!cache.get(j).present());
+				// ordering is important so as not to disrupt the LRU order
+				for (auto j = i - cache.capacity() + 1; j <= i; j++)
+					ASSERT(cache.get(j).present());
+			}
+		}
+	}
+	{
+		// Test larger cache
+		LRUCache<int, StringRef> cache(1000);
+		for (auto i = 0; i < 1000; ++i) {
+			cache.insert(i, "value"_sr);
+		}
+		cache.insert(1000, "value"_sr); // should evict 0
+		ASSERT(!cache.get(0).present());
+	}
+	{
+		// memory test -- this is what the boost implementation didn't do correctly
+		LRUCache<StringRef, Standalone<StringRef>> cache(10);
+		std::deque<std::string> cachedStrings;
+		std::deque<std::string> evictedStrings;
+		for (int i = 0; i < 10; ++i) {
+			auto str = rng.randomAlphaNumeric(rng.randomInt(100, 1024));
+			Standalone<StringRef> sref(str);
+			cache.insert(sref, sref);
+			cachedStrings.push_back(str);
+		}
+		for (int i = 0; i < 10; ++i) {
+			Standalone<StringRef> existingStr(cachedStrings.back());
+			auto cachedStr = cache.get(existingStr);
+			ASSERT(cachedStr.present());
+			ASSERT(*cachedStr.get() == existingStr);
+			if (!evictedStrings.empty()) {
+				Standalone<StringRef> nonexisting(evictedStrings.at(rng.randomInt(0, evictedStrings.size())));
+				ASSERT(!cache.get(nonexisting).present());
+			}
+			auto str = rng.randomAlphaNumeric(rng.randomInt(100, 1024));
+			Standalone<StringRef> sref(str);
+			evictedStrings.push_back(cachedStrings.front());
+			cachedStrings.pop_front();
+			cachedStrings.push_back(str);
+			cache.insert(sref, sref);
+		}
+	}
+	return Void();
+}
+
+struct TokenCacheImpl {
+	struct CacheEntry {
+		Arena arena;
+		VectorRef<TenantNameRef> tenants;
+		double expirationTime = 0.0;
+	};
+
+	LRUCache<StringRef, CacheEntry> cache;
+	TokenCacheImpl() : cache(FLOW_KNOBS->TOKEN_CACHE_SIZE) {}
+
+	bool validate(TenantNameRef tenant, StringRef token);
+	bool validateAndAdd(double currentTime, StringRef token, NetworkAddress const& peer);
+};
+
+TokenCache::TokenCache() : impl(new TokenCacheImpl()) {}
+TokenCache::~TokenCache() {
+	delete impl;
+}
+
+void TokenCache::createInstance() {
+	g_network->setGlobal(INetwork::enTokenCache, new TokenCache());
+}
+
+TokenCache& TokenCache::instance() {
+	return *reinterpret_cast<TokenCache*>(g_network->global(INetwork::enTokenCache));
+}
+
+bool TokenCache::validate(TenantNameRef name, StringRef token) {
+	return impl->validate(name, token);
+}
+
+#define TRACE_INVALID_PARSED_TOKEN(reason, token)                                                                      \
+	TraceEvent(SevWarn, "InvalidToken")                                                                                \
+	    .detail("From", peer)                                                                                          \
+	    .detail("Reason", reason)                                                                                      \
+	    .detail("CurrentTime", currentTime)                                                                            \
+	    .detail("Token", token.toStringRef(arena).toStringView())
+
+bool TokenCacheImpl::validateAndAdd(double currentTime, StringRef token, NetworkAddress const& peer) {
+	Arena arena;
+	authz::jwt::TokenRef t;
+	if (!authz::jwt::parseToken(arena, t, token)) {
+		CODE_PROBE(true, "Token can't be parsed");
+		TraceEvent(SevWarn, "InvalidToken")
+		    .detail("From", peer)
+		    .detail("Reason", "ParseError")
+		    .detail("Token", token.toString());
+		return false;
+	}
+	auto key = FlowTransport::transport().getPublicKeyByName(t.keyId);
+	if (!key.present()) {
+		CODE_PROBE(true, "Token referencing non-existing key");
+		TRACE_INVALID_PARSED_TOKEN("UnknownKey", t);
+		return false;
+	} else if (!t.issuedAtUnixTime.present()) {
+		CODE_PROBE(true, "Token has no issued-at field");
+		TRACE_INVALID_PARSED_TOKEN("NoIssuedAt", t);
+		return false;
+	} else if (!t.expiresAtUnixTime.present()) {
+		CODE_PROBE(true, "Token has no expiration time");
+		TRACE_INVALID_PARSED_TOKEN("NoExpirationTime", t);
+		return false;
+	} else if (double(t.expiresAtUnixTime.get()) <= currentTime) {
+		CODE_PROBE(true, "Expired token");
+		TRACE_INVALID_PARSED_TOKEN("Expired", t);
+		return false;
+	} else if (!t.notBeforeUnixTime.present()) {
+		CODE_PROBE(true, "Token has no not-before field");
+		TRACE_INVALID_PARSED_TOKEN("NoNotBefore", t);
+		return false;
+	} else if (double(t.notBeforeUnixTime.get()) > currentTime) {
+		CODE_PROBE(true, "Tokens not-before is in the future");
+		TRACE_INVALID_PARSED_TOKEN("TokenNotYetValid", t);
+		return false;
+	} else if (!t.tenants.present()) {
+		CODE_PROBE(true, "Token with no tenants");
+		TRACE_INVALID_PARSED_TOKEN("NoTenants", t);
+		return false;
+	} else if (!authz::jwt::verifyToken(token, key.get())) {
+		CODE_PROBE(true, "Token with invalid signature");
+		TRACE_INVALID_PARSED_TOKEN("InvalidSignature", t);
+		return false;
+	} else {
+		CacheEntry c;
+		c.expirationTime = t.expiresAtUnixTime.get();
+		c.tenants.reserve(c.arena, t.tenants.get().size());
+		for (auto tenant : t.tenants.get()) {
+			c.tenants.push_back_deep(c.arena, tenant);
+		}
+		cache.insert(StringRef(c.arena, token), c);
+		return true;
+	}
+}
+
+bool TokenCacheImpl::validate(TenantNameRef name, StringRef token) {
+	NetworkAddress peer = FlowTransport::transport().currentDeliveryPeerAddress();
+	auto cachedEntry = cache.get(token);
+	double currentTime = g_network->timer();
+
+	if (!cachedEntry.present()) {
+		if (validateAndAdd(currentTime, token, peer)) {
+			cachedEntry = cache.get(token);
+		} else {
+			return false;
+		}
+	}
+
+	ASSERT(cachedEntry.present());
+
+	auto& entry = cachedEntry.get();
+	if (entry->expirationTime < currentTime) {
+		CODE_PROBE(true, "Found expired token in cache");
+		TraceEvent(SevWarn, "InvalidToken").detail("From", peer).detail("Reason", "ExpiredInCache");
+		return false;
+	}
+	bool tenantFound = false;
+	for (auto const& t : entry->tenants) {
+		if (t == name) {
+			tenantFound = true;
+			break;
+		}
+	}
+	if (!tenantFound) {
+		CODE_PROBE(true, "Valid token doesn't reference tenant");
+		TraceEvent(SevWarn, "TenantTokenMismatch").detail("From", peer).detail("Tenant", name.toString());
+		return false;
+	}
+	return true;
+}
+
+namespace authz::jwt {
+extern TokenRef makeRandomTokenSpec(Arena&, IRandom&, authz::Algorithm);
+}
+
+TEST_CASE("/fdbrpc/authz/TokenCache/BadTokens") {
+	std::pair<void (*)(Arena&, IRandom&, authz::jwt::TokenRef&), char const*> badMutations[]{
+		{
+		    [](Arena&, IRandom&, authz::jwt::TokenRef&) { FlowTransport::transport().removeAllPublicKeys(); },
+		    "NoKeyWithSuchName",
+		},
+		{
+		    [](Arena&, IRandom&, authz::jwt::TokenRef& token) { token.expiresAtUnixTime.reset(); },
+		    "NoExpirationTime",
+		},
+		{
+		    [](Arena&, IRandom& rng, authz::jwt::TokenRef& token) {
+		        token.expiresAtUnixTime = std::max<double>(g_network->timer() - 10 - rng.random01() * 50, 0);
+		    },
+		    "ExpiredToken",
+		},
+		{
+		    [](Arena&, IRandom&, authz::jwt::TokenRef& token) { token.notBeforeUnixTime.reset(); },
+		    "NoNotBefore",
+		},
+		{
+		    [](Arena&, IRandom& rng, authz::jwt::TokenRef& token) {
+		        token.notBeforeUnixTime = g_network->timer() + 10 + rng.random01() * 50;
+		    },
+		    "TokenNotYetValid",
+		},
+		{
+		    [](Arena&, IRandom&, authz::jwt::TokenRef& token) { token.issuedAtUnixTime.reset(); },
+		    "NoIssuedAt",
+		},
+		{
+		    [](Arena& arena, IRandom&, authz::jwt::TokenRef& token) { token.tenants.reset(); },
+		    "NoTenants",
+		},
+		{
+		    [](Arena& arena, IRandom&, authz::jwt::TokenRef& token) {
+		        StringRef* newTenants = new (arena) StringRef[1];
+		        *newTenants = token.tenants.get()[0].substr(1);
+		        token.tenants = VectorRef<StringRef>(newTenants, 1);
+		    },
+		    "UnmatchedTenant",
+		},
+	};
+	auto const pubKeyName = "somePublicKey"_sr;
+	auto privateKey = mkcert::makeEcP256();
+	auto const numBadMutations = sizeof(badMutations) / sizeof(badMutations[0]);
+	for (auto repeat = 0; repeat < 50; repeat++) {
+		auto arena = Arena();
+		auto& rng = *deterministicRandom();
+		auto validTokenSpec = authz::jwt::makeRandomTokenSpec(arena, rng, authz::Algorithm::ES256);
+		validTokenSpec.keyId = pubKeyName;
+		for (auto i = 0; i <= numBadMutations; i++) {
+			FlowTransport::transport().addPublicKey(pubKeyName, privateKey.toPublic());
+			auto publicKeyClearGuard =
+			    ScopeExit([pubKeyName]() { FlowTransport::transport().removePublicKey(pubKeyName); });
+			auto signedToken = StringRef();
+			auto tmpArena = Arena();
+			if (i < numBadMutations) {
+				auto [mutationFn, mutationDesc] = badMutations[i];
+				auto mutatedTokenSpec = validTokenSpec;
+				mutationFn(tmpArena, rng, mutatedTokenSpec);
+				signedToken = authz::jwt::signToken(tmpArena, mutatedTokenSpec, privateKey);
+				if (TokenCache::instance().validate(validTokenSpec.tenants.get()[0], signedToken)) {
+					fmt::print("Unexpected successful validation at mutation {}, token spec: {}\n",
+					           mutationDesc,
+					           mutatedTokenSpec.toStringRef(tmpArena).toStringView());
+					ASSERT(false);
+				}
+			} else {
+				// squeeze in a bad signature case that does not fit into mutation interface
+				signedToken = authz::jwt::signToken(tmpArena, validTokenSpec, privateKey);
+				signedToken = signedToken.substr(0, signedToken.size() - 1);
+				if (TokenCache::instance().validate(validTokenSpec.tenants.get()[0], signedToken)) {
+					fmt::print("Unexpected successful validation with a token with truncated signature part\n");
+					ASSERT(false);
+				}
+			}
+		}
+	}
+	if (TokenCache::instance().validate("TenantNameDontMatterHere"_sr, StringRef())) {
+		fmt::print("Unexpected successful validation of ill-formed token (no signature part)\n");
+		ASSERT(false);
+	}
+	if (TokenCache::instance().validate("TenantNameDontMatterHere"_sr, "1111.22"_sr)) {
+		fmt::print("Unexpected successful validation of ill-formed token (no signature part)\n");
+		ASSERT(false);
+	}
+	if (TokenCache::instance().validate("TenantNameDontMatterHere2"_sr, "////.////.////"_sr)) {
+		fmt::print("Unexpected successful validation of unparseable token\n");
+		ASSERT(false);
+	}
+	fmt::print("TEST OK\n");
+	return Void();
+}
+
+TEST_CASE("/fdbrpc/authz/TokenCache/GoodTokens") {
+	// Don't repeat because token expiry is at seconds granularity and sleeps are costly in unit tests
+	state Arena arena;
+	state PrivateKey privateKey = mkcert::makeEcP256();
+	state StringRef pubKeyName = "somePublicKey"_sr;
+	state ScopeExit<std::function<void()>> publicKeyClearGuard(
+	    [pubKeyName = pubKeyName]() { FlowTransport::transport().removePublicKey(pubKeyName); });
+	state authz::jwt::TokenRef tokenSpec =
+	    authz::jwt::makeRandomTokenSpec(arena, *deterministicRandom(), authz::Algorithm::ES256);
+	state StringRef signedToken;
+	FlowTransport::transport().addPublicKey(pubKeyName, privateKey.toPublic());
+	tokenSpec.expiresAtUnixTime = g_network->timer() + 2.0;
+	tokenSpec.keyId = pubKeyName;
+	signedToken = authz::jwt::signToken(arena, tokenSpec, privateKey);
+	if (!TokenCache::instance().validate(tokenSpec.tenants.get()[0], signedToken)) {
+		fmt::print("Unexpected failed token validation, token spec: {}, now: {}\n",
+		           tokenSpec.toStringRef(arena).toStringView(),
+		           g_network->timer());
+		ASSERT(false);
+	}
+	wait(delay(3.5));
+	if (TokenCache::instance().validate(tokenSpec.tenants.get()[0], signedToken)) {
+		fmt::print(
+		    "Unexpected successful token validation after supposedly expiring in cache, token spec: {}, now: {}\n",
+		    tokenSpec.toStringRef(arena).toStringView(),
+		    g_network->timer());
+		ASSERT(false);
+	}
+	fmt::print("TEST OK\n");
+	return Void();
+}
diff --git a/fdbrpc/TokenSign.cpp b/fdbrpc/TokenSign.cpp
index 26892c17bc..fd5826f872 100644
--- a/fdbrpc/TokenSign.cpp
+++ b/fdbrpc/TokenSign.cpp
@@ -22,6 +22,7 @@
 #include "flow/network.h"
 #include "flow/serialize.h"
 #include "flow/Arena.h"
+#include "flow/AutoCPointer.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/MkCert.h"
@@ -30,6 +31,8 @@
 #include "flow/Trace.h"
 #include "flow/UnitTest.h"
 #include <fmt/format.h>
+#include <cmath>
+#include <iterator>
 #include <string_view>
 #include <type_traits>
 #include <utility>
@@ -50,12 +53,15 @@
 namespace {
 
 // test-only constants for generating random tenant/key names
+constexpr int MinIssuerNameLen = 16;
 constexpr int MaxIssuerNameLenPlus1 = 25;
+constexpr int MinTenantNameLen = 8;
 constexpr int MaxTenantNameLenPlus1 = 17;
+constexpr int MinKeyNameLen = 10;
 constexpr int MaxKeyNameLenPlus1 = 21;
 
-StringRef genRandomAlphanumStringRef(Arena& arena, IRandom& rng, int maxLenPlusOne) {
-	const auto len = rng.randomInt(1, maxLenPlusOne);
+StringRef genRandomAlphanumStringRef(Arena& arena, IRandom& rng, int minLen, int maxLenPlusOne) {
+	const auto len = rng.randomInt(minLen, maxLenPlusOne);
 	auto strRaw = new (arena) uint8_t[len];
 	for (auto i = 0; i < len; i++)
 		strRaw[i] = (uint8_t)rng.randomAlphaNumeric();
@@ -86,6 +92,51 @@ bool checkSignAlgorithm(PKeyAlgorithm algo, PrivateKey key) {
 	}
 }
 
+Optional<StringRef> convertEs256P1363ToDer(Arena& arena, StringRef p1363) {
+	const int SIGLEN = p1363.size();
+	const int HALF_SIGLEN = SIGLEN / 2;
+	auto r = AutoCPointer(BN_bin2bn(p1363.begin(), HALF_SIGLEN, nullptr), &::BN_free);
+	auto s = AutoCPointer(BN_bin2bn(p1363.begin() + HALF_SIGLEN, HALF_SIGLEN, nullptr), &::BN_free);
+	if (!r || !s)
+		return {};
+	auto sig = AutoCPointer(::ECDSA_SIG_new(), &ECDSA_SIG_free);
+	if (!sig)
+		return {};
+	::ECDSA_SIG_set0(sig, r.release(), s.release());
+	auto const derLen = ::i2d_ECDSA_SIG(sig, nullptr);
+	if (derLen < 0)
+		return {};
+	auto buf = new (arena) uint8_t[derLen];
+	auto bufPtr = buf;
+	::i2d_ECDSA_SIG(sig, &bufPtr);
+	return StringRef(buf, derLen);
+}
+
+Optional<StringRef> convertEs256DerToP1363(Arena& arena, StringRef der) {
+	uint8_t const* derPtr = der.begin();
+	auto sig = AutoCPointer(::d2i_ECDSA_SIG(nullptr, &derPtr, der.size()), &::ECDSA_SIG_free);
+	if (!sig) {
+		return {};
+	}
+	// ES256-specific constant. Adapt as needed
+	constexpr const int SIGLEN = 64;
+	constexpr const int HALF_SIGLEN = SIGLEN / 2;
+	auto buf = new (arena) uint8_t[SIGLEN];
+	::memset(buf, 0, SIGLEN);
+	auto bufr = buf;
+	auto bufs = bufr + HALF_SIGLEN;
+	auto r = std::add_pointer_t<BIGNUM const>();
+	auto s = std::add_pointer_t<BIGNUM const>();
+	ECDSA_SIG_get0(sig, &r, &s);
+	auto const lenr = BN_num_bytes(r);
+	auto const lens = BN_num_bytes(s);
+	if (lenr > HALF_SIGLEN || lens > HALF_SIGLEN)
+		return {};
+	BN_bn2bin(r, bufr + (HALF_SIGLEN - lenr));
+	BN_bn2bin(s, bufs + (HALF_SIGLEN - lens));
+	return StringRef(buf, SIGLEN);
+}
+
 } // namespace
 
 namespace authz {
@@ -129,11 +180,7 @@ SignedTokenRef signToken(Arena& arena, TokenRef token, StringRef keyName, Privat
 	auto writer = ObjectWriter([&arena](size_t len) { return new (arena) uint8_t[len]; }, IncludeVersion());
 	writer.serialize(token);
 	auto tokenStr = writer.toStringRef();
-	auto [signAlgo, digest] = getMethod(Algorithm::ES256);
-	if (!checkSignAlgorithm(signAlgo, privateKey)) {
-		throw digital_signature_ops_error();
-	}
-	auto sig = privateKey.sign(arena, tokenStr, *digest);
+	auto sig = privateKey.sign(arena, tokenStr, *::EVP_sha256());
 	ret.token = tokenStr;
 	ret.signature = sig;
 	ret.keyName = StringRef(arena, keyName);
@@ -141,10 +188,7 @@ SignedTokenRef signToken(Arena& arena, TokenRef token, StringRef keyName, Privat
 }
 
 bool verifyToken(SignedTokenRef signedToken, PublicKey publicKey) {
-	auto [keyAlg, digest] = getMethod(Algorithm::ES256);
-	if (!checkVerifyAlgorithm(keyAlg, publicKey))
-		return false;
-	return publicKey.verify(signedToken.token, signedToken.signature, *digest);
+	return publicKey.verify(signedToken.token, signedToken.signature, *::EVP_sha256());
 }
 
 TokenRef makeRandomTokenSpec(Arena& arena, IRandom& rng) {
@@ -152,7 +196,7 @@ TokenRef makeRandomTokenSpec(Arena& arena, IRandom& rng) {
 	token.expiresAt = timer_monotonic() * (0.5 + rng.random01());
 	const auto numTenants = rng.randomInt(1, 3);
 	for (auto i = 0; i < numTenants; i++) {
-		token.tenants.push_back(arena, genRandomAlphanumStringRef(arena, rng, MaxTenantNameLenPlus1));
+		token.tenants.push_back(arena, genRandomAlphanumStringRef(arena, rng, MinTenantNameLen, MaxTenantNameLenPlus1));
 	}
 	return token;
 }
@@ -161,6 +205,43 @@ TokenRef makeRandomTokenSpec(Arena& arena, IRandom& rng) {
 
 namespace authz::jwt {
 
+template <class FieldType, size_t NameLen>
+void appendField(fmt::memory_buffer& b, char const (&name)[NameLen], Optional<FieldType> const& field) {
+	if (!field.present())
+		return;
+	auto const& f = field.get();
+	auto bi = std::back_inserter(b);
+	if constexpr (std::is_same_v<FieldType, VectorRef<StringRef>>) {
+		fmt::format_to(bi, " {}=[", name);
+		for (auto i = 0; i < f.size(); i++) {
+			if (i)
+				fmt::format_to(bi, ",");
+			fmt::format_to(bi, f[i].toStringView());
+		}
+		fmt::format_to(bi, "]");
+	} else if constexpr (std::is_same_v<FieldType, StringRef>) {
+		fmt::format_to(bi, " {}={}", name, f.toStringView());
+	} else {
+		fmt::format_to(bi, " {}={}", name, f);
+	}
+}
+
+StringRef TokenRef::toStringRef(Arena& arena) {
+	auto buf = fmt::memory_buffer();
+	fmt::format_to(std::back_inserter(buf), "alg={} kid={}", getAlgorithmName(algorithm), keyId.toStringView());
+	appendField(buf, "iss", issuer);
+	appendField(buf, "sub", subject);
+	appendField(buf, "aud", audience);
+	appendField(buf, "iat", issuedAtUnixTime);
+	appendField(buf, "exp", expiresAtUnixTime);
+	appendField(buf, "nbf", notBeforeUnixTime);
+	appendField(buf, "jti", tokenId);
+	appendField(buf, "tenants", tenants);
+	auto str = new (arena) uint8_t[buf.size()];
+	memcpy(str, buf.data(), buf.size());
+	return StringRef(str, buf.size());
+}
+
 template <class FieldType, class Writer>
 void putField(Optional<FieldType> const& field, Writer& wr, const char* fieldName) {
 	if (!field.present())
@@ -192,9 +273,12 @@ StringRef makeTokenPart(Arena& arena, TokenRef tokenSpec) {
 	header.StartObject();
 	header.Key("typ");
 	header.String("JWT");
-	header.Key("alg");
 	auto algo = getAlgorithmName(tokenSpec.algorithm);
+	header.Key("alg");
 	header.String(algo.data(), algo.size());
+	auto kid = tokenSpec.keyId.toStringView();
+	header.Key("kid");
+	header.String(kid.data(), kid.size());
 	header.EndObject();
 	payload.StartObject();
 	putField(tokenSpec.issuer, payload, "iss");
@@ -203,7 +287,6 @@ StringRef makeTokenPart(Arena& arena, TokenRef tokenSpec) {
 	putField(tokenSpec.issuedAtUnixTime, payload, "iat");
 	putField(tokenSpec.expiresAtUnixTime, payload, "exp");
 	putField(tokenSpec.notBeforeUnixTime, payload, "nbf");
-	putField(tokenSpec.keyId, payload, "kid");
 	putField(tokenSpec.tokenId, payload, "jti");
 	putField(tokenSpec.tenants, payload, "tenants");
 	payload.EndObject();
@@ -228,6 +311,17 @@ StringRef signToken(Arena& arena, TokenRef tokenSpec, PrivateKey privateKey) {
 		throw digital_signature_ops_error();
 	}
 	auto plainSig = privateKey.sign(tmpArena, tokenPart, *digest);
+	if (tokenSpec.algorithm == Algorithm::ES256) {
+		// Need to convert ASN.1/DER signature to IEEE-P1363
+		auto convertedSig = convertEs256DerToP1363(tmpArena, plainSig);
+		if (!convertedSig.present()) {
+			auto tmpArena = Arena();
+			TraceEvent(SevWarn, "TokenSigConversionFailure")
+			    .detail("TokenSpec", tokenSpec.toStringRef(tmpArena).toString());
+			throw digital_signature_ops_error();
+		}
+		plainSig = convertedSig.get();
+	}
 	auto const sigPartLen = base64url::encodedLength(plainSig.size());
 	auto const totalLen = tokenPart.size() + 1 + sigPartLen;
 	auto out = new (arena) uint8_t[totalLen];
@@ -240,7 +334,7 @@ StringRef signToken(Arena& arena, TokenRef tokenSpec, PrivateKey privateKey) {
 	return StringRef(out, totalLen);
 }
 
-bool parseHeaderPart(TokenRef& token, StringRef b64urlHeader) {
+bool parseHeaderPart(Arena& arena, TokenRef& token, StringRef b64urlHeader) {
 	auto tmpArena = Arena();
 	auto optHeader = base64url::decode(tmpArena, b64urlHeader);
 	if (!optHeader.present())
@@ -256,24 +350,30 @@ bool parseHeaderPart(TokenRef& token, StringRef b64urlHeader) {
 		    .detail("Offset", d.GetErrorOffset());
 		return false;
 	}
-	auto algItr = d.FindMember("alg");
+	if (!d.IsObject())
+		return false;
 	auto typItr = d.FindMember("typ");
-	if (d.IsObject() && algItr != d.MemberEnd() && typItr != d.MemberEnd()) {
-		auto const& alg = algItr->value;
-		auto const& typ = typItr->value;
-		if (alg.IsString() && typ.IsString()) {
-			auto algValue = StringRef(reinterpret_cast<const uint8_t*>(alg.GetString()), alg.GetStringLength());
-			auto algType = algorithmFromString(algValue);
-			if (algType == Algorithm::UNKNOWN)
-				return false;
-			token.algorithm = algType;
-			auto typValue = StringRef(reinterpret_cast<const uint8_t*>(typ.GetString()), typ.GetStringLength());
-			if (typValue != "JWT"_sr)
-				return false;
-			return true;
-		}
-	}
-	return false;
+	if (typItr == d.MemberEnd() || !typItr->value.IsString())
+		return false;
+	auto algItr = d.FindMember("alg");
+	if (algItr == d.MemberEnd() || !algItr->value.IsString())
+		return false;
+	auto kidItr = d.FindMember("kid");
+	if (kidItr == d.MemberEnd() || !kidItr->value.IsString())
+		return false;
+	auto const& typ = typItr->value;
+	auto const& alg = algItr->value;
+	auto const& kid = kidItr->value;
+	auto typValue = StringRef(reinterpret_cast<const uint8_t*>(typ.GetString()), typ.GetStringLength());
+	if (typValue != "JWT"_sr)
+		return false;
+	auto algValue = StringRef(reinterpret_cast<const uint8_t*>(alg.GetString()), alg.GetStringLength());
+	auto algType = algorithmFromString(algValue);
+	if (algType == Algorithm::UNKNOWN)
+		return false;
+	token.algorithm = algType;
+	token.keyId = StringRef(arena, reinterpret_cast<const uint8_t*>(kid.GetString()), kid.GetStringLength());
+	return true;
 }
 
 template <class FieldType>
@@ -289,9 +389,9 @@ bool parseField(Arena& arena, Optional<FieldType>& out, const rapidjson::Documen
 			return false;
 		out = StringRef(arena, reinterpret_cast<const uint8_t*>(field.GetString()), field.GetStringLength());
 	} else if constexpr (std::is_same_v<FieldType, uint64_t>) {
-		if (!field.IsUint64())
+		if (!field.IsNumber())
 			return false;
-		out = field.GetUint64();
+		out = static_cast<uint64_t>(field.GetDouble());
 	} else {
 		if (!field.IsArray())
 			return false;
@@ -343,8 +443,6 @@ bool parsePayloadPart(Arena& arena, TokenRef& token, StringRef b64urlPayload) {
 		return false;
 	if (!parseField(arena, token.notBeforeUnixTime, d, "nbf"))
 		return false;
-	if (!parseField(arena, token.keyId, d, "kid"))
-		return false;
 	if (!parseField(arena, token.tenants, d, "tenants"))
 		return false;
 	return true;
@@ -358,13 +456,19 @@ bool parseSignaturePart(Arena& arena, TokenRef& token, StringRef b64urlSignature
 	return true;
 }
 
+StringRef signaturePart(StringRef token) {
+	token.eat("."_sr);
+	token.eat("."_sr);
+	return token;
+}
+
 bool parseToken(Arena& arena, TokenRef& token, StringRef signedToken) {
 	auto b64urlHeader = signedToken.eat("."_sr);
 	auto b64urlPayload = signedToken.eat("."_sr);
 	auto b64urlSignature = signedToken;
 	if (b64urlHeader.empty() || b64urlPayload.empty() || b64urlSignature.empty())
 		return false;
-	if (!parseHeaderPart(token, b64urlHeader))
+	if (!parseHeaderPart(arena, token, b64urlHeader))
 		return false;
 	if (!parsePayloadPart(arena, token, b64urlPayload))
 		return false;
@@ -387,36 +491,40 @@ bool verifyToken(StringRef signedToken, PublicKey publicKey) {
 		return false;
 	auto sig = optSig.get();
 	auto parsedToken = TokenRef();
-	if (!parseHeaderPart(parsedToken, b64urlHeader))
+	if (!parseHeaderPart(arena, parsedToken, b64urlHeader))
 		return false;
 	auto [verifyAlgo, digest] = getMethod(parsedToken.algorithm);
 	if (!checkVerifyAlgorithm(verifyAlgo, publicKey))
 		return false;
+	if (parsedToken.algorithm == Algorithm::ES256) {
+		// Need to convert IEEE-P1363 signature to ASN.1/DER
+		auto convertedSig = convertEs256P1363ToDer(arena, sig);
+		if (!convertedSig.present())
+			return false;
+		sig = convertedSig.get();
+	}
 	return publicKey.verify(b64urlTokenPart, sig, *digest);
 }
 
 TokenRef makeRandomTokenSpec(Arena& arena, IRandom& rng, Algorithm alg) {
-	if (alg != Algorithm::ES256) {
-		throw unsupported_operation();
-	}
 	auto ret = TokenRef{};
 	ret.algorithm = alg;
-	ret.issuer = genRandomAlphanumStringRef(arena, rng, MaxIssuerNameLenPlus1);
-	ret.subject = genRandomAlphanumStringRef(arena, rng, MaxIssuerNameLenPlus1);
-	ret.tokenId = genRandomAlphanumStringRef(arena, rng, 31);
+	ret.keyId = genRandomAlphanumStringRef(arena, rng, MinKeyNameLen, MaxKeyNameLenPlus1);
+	ret.issuer = genRandomAlphanumStringRef(arena, rng, MinIssuerNameLen, MaxIssuerNameLenPlus1);
+	ret.subject = genRandomAlphanumStringRef(arena, rng, MinIssuerNameLen, MaxIssuerNameLenPlus1);
+	ret.tokenId = genRandomAlphanumStringRef(arena, rng, 16, 31);
 	auto numAudience = rng.randomInt(1, 5);
 	auto aud = new (arena) StringRef[numAudience];
 	for (auto i = 0; i < numAudience; i++)
-		aud[i] = genRandomAlphanumStringRef(arena, rng, MaxTenantNameLenPlus1);
+		aud[i] = genRandomAlphanumStringRef(arena, rng, MinTenantNameLen, MaxTenantNameLenPlus1);
 	ret.audience = VectorRef<StringRef>(aud, numAudience);
-	ret.issuedAtUnixTime = timer_int() / 1'000'000'000ul;
-	ret.notBeforeUnixTime = timer_int() / 1'000'000'000ul;
+	ret.issuedAtUnixTime = g_network->timer();
+	ret.notBeforeUnixTime = ret.issuedAtUnixTime.get();
 	ret.expiresAtUnixTime = ret.issuedAtUnixTime.get() + rng.randomInt(360, 1080 + 1);
-	ret.keyId = genRandomAlphanumStringRef(arena, rng, MaxKeyNameLenPlus1);
 	auto numTenants = rng.randomInt(1, 3);
 	auto tenants = new (arena) StringRef[numTenants];
 	for (auto i = 0; i < numTenants; i++)
-		tenants[i] = genRandomAlphanumStringRef(arena, rng, MaxTenantNameLenPlus1);
+		tenants[i] = genRandomAlphanumStringRef(arena, rng, MinTenantNameLen, MaxTenantNameLenPlus1);
 	ret.tenants = VectorRef<StringRef>(tenants, numTenants);
 	return ret;
 }
@@ -432,12 +540,13 @@ TEST_CASE("/fdbrpc/TokenSign/FlatBuffer") {
 		auto privateKey = mkcert::makeEcP256();
 		auto& rng = *deterministicRandom();
 		auto tokenSpec = authz::flatbuffers::makeRandomTokenSpec(arena, rng);
-		auto keyName = genRandomAlphanumStringRef(arena, rng, MaxKeyNameLenPlus1);
+		auto keyName = genRandomAlphanumStringRef(arena, rng, MinKeyNameLen, MaxKeyNameLenPlus1);
 		auto signedToken = authz::flatbuffers::signToken(arena, tokenSpec, keyName, privateKey);
 		const auto verifyExpectOk = authz::flatbuffers::verifyToken(signedToken, privateKey.toPublic());
 		ASSERT(verifyExpectOk);
 		// try tampering with signed token by adding one more tenant
-		tokenSpec.tenants.push_back(arena, genRandomAlphanumStringRef(arena, rng, MaxTenantNameLenPlus1));
+		tokenSpec.tenants.push_back(arena,
+		                            genRandomAlphanumStringRef(arena, rng, MinTenantNameLen, MaxTenantNameLenPlus1));
 		auto writer = ObjectWriter([&arena](size_t len) { return new (arena) uint8_t[len]; }, IncludeVersion());
 		writer.serialize(tokenSpec);
 		signedToken.token = writer.toStringRef();
@@ -481,7 +590,8 @@ TEST_CASE("/fdbrpc/TokenSign/JWT") {
 			ASSERT(optSig.get() == parsedToken.signature);
 		}
 		// try tampering with signed token by adding one more tenant
-		tokenSpec.tenants.get().push_back(arena, genRandomAlphanumStringRef(arena, rng, MaxTenantNameLenPlus1));
+		tokenSpec.tenants.get().push_back(
+		    arena, genRandomAlphanumStringRef(arena, rng, MinTenantNameLen, MaxTenantNameLenPlus1));
 		auto tamperedTokenPart = makeTokenPart(arena, tokenSpec);
 		auto tamperedTokenString = fmt::format("{}.{}", tamperedTokenPart.toString(), signaturePart.toString());
 		const auto verifyExpectFail = authz::jwt::verifyToken(StringRef(tamperedTokenString), privateKey.toPublic());
@@ -491,52 +601,100 @@ TEST_CASE("/fdbrpc/TokenSign/JWT") {
 	return Void();
 }
 
-TEST_CASE("/fdbrpc/TokenSign/bench") {
-	constexpr auto repeat = 5;
-	constexpr auto numSamples = 10000;
-	auto keys = std::vector<PrivateKey>(numSamples);
-	auto pubKeys = std::vector<PublicKey>(numSamples);
-	for (auto i = 0; i < numSamples; i++) {
-		keys[i] = mkcert::makeEcP256();
-		pubKeys[i] = keys[i].toPublic();
-	}
-	fmt::print("{} keys generated\n", numSamples);
-	auto& rng = *deterministicRandom();
+TEST_CASE("/fdbrpc/TokenSign/JWT/ToStringRef") {
+	auto t = authz::jwt::TokenRef();
+	t.algorithm = authz::Algorithm::ES256;
+	t.issuer = "issuer"_sr;
+	t.subject = "subject"_sr;
+	StringRef aud[3]{ "aud1"_sr, "aud2"_sr, "aud3"_sr };
+	t.audience = VectorRef<StringRef>(aud, 3);
+	t.issuedAtUnixTime = 123ul;
+	t.expiresAtUnixTime = 456ul;
+	t.notBeforeUnixTime = 789ul;
+	t.keyId = "keyId"_sr;
+	t.tokenId = "tokenId"_sr;
+	StringRef tenants[2]{ "tenant1"_sr, "tenant2"_sr };
+	t.tenants = VectorRef<StringRef>(tenants, 2);
 	auto arena = Arena();
-	auto jwts = new (arena) StringRef[numSamples];
-	auto fbs = new (arena) StringRef[numSamples];
-	{
-		auto tmpArena = Arena();
-		for (auto i = 0; i < numSamples; i++) {
-			auto jwtSpec = authz::jwt::makeRandomTokenSpec(tmpArena, rng, authz::Algorithm::ES256);
-			jwts[i] = authz::jwt::signToken(arena, jwtSpec, keys[i]);
-			auto fbSpec = authz::flatbuffers::makeRandomTokenSpec(tmpArena, rng);
-			auto fbToken = authz::flatbuffers::signToken(tmpArena, fbSpec, "defaultKey"_sr, keys[i]);
-			auto wr = ObjectWriter([&arena](size_t len) { return new (arena) uint8_t[len]; }, Unversioned());
-			wr.serialize(fbToken);
-			fbs[i] = wr.toStringRef();
-		}
+	auto tokenStr = t.toStringRef(arena);
+	auto tokenStrExpected =
+	    "alg=ES256 kid=keyId iss=issuer sub=subject aud=[aud1,aud2,aud3] iat=123 exp=456 nbf=789 jti=tokenId tenants=[tenant1,tenant2]"_sr;
+	if (tokenStr != tokenStrExpected) {
+		fmt::print("Expected: {}\nGot     : {}\n", tokenStrExpected.toStringView(), tokenStr.toStringView());
+		ASSERT(false);
+	} else {
+		fmt::print("TEST OK\n");
 	}
-	fmt::print("{} FB/JWT tokens generated\n", numSamples);
-	auto jwtBegin = timer_monotonic();
-	for (auto rep = 0; rep < repeat; rep++) {
-		for (auto i = 0; i < numSamples; i++) {
-			auto verifyOk = authz::jwt::verifyToken(jwts[i], pubKeys[i]);
-			ASSERT(verifyOk);
-		}
-	}
-	auto jwtEnd = timer_monotonic();
-	fmt::print("JWT:         {:.2f} OPS\n", repeat * numSamples / (jwtEnd - jwtBegin));
-	auto fbBegin = timer_monotonic();
-	for (auto rep = 0; rep < repeat; rep++) {
-		for (auto i = 0; i < numSamples; i++) {
-			auto signedToken =
-			    ObjectReader::fromStringRef<Standalone<authz::flatbuffers::SignedTokenRef>>(fbs[i], Unversioned());
-			auto verifyOk = authz::flatbuffers::verifyToken(signedToken, pubKeys[i]);
-			ASSERT(verifyOk);
-		}
-	}
-	auto fbEnd = timer_monotonic();
-	fmt::print("FlatBuffers: {:.2f} OPS\n", repeat * numSamples / (fbEnd - fbBegin));
 	return Void();
 }
+
+// This unit test takes too long to run in RandomUnitTests.toml
+// FIXME: Move this to benchmark to flowbench
+/*
+TEST_CASE("/fdbrpc/TokenSign/bench") {
+    auto keyTypes = std::array<StringRef, 2>{ "EC"_sr, "RSA"_sr };
+    for (auto kty : keyTypes) {
+        constexpr auto repeat = 5;
+        constexpr auto numSamples = 10000;
+        fmt::print("=== {} keys case\n", kty.toString());
+        auto key = kty == "EC"_sr ? mkcert::makeEcP256() : mkcert::makeRsa4096Bit();
+        auto pubKey = key.toPublic();
+        auto& rng = *deterministicRandom();
+        auto arena = Arena();
+        auto jwtSpecs = new (arena) authz::jwt::TokenRef[numSamples];
+        auto fbSpecs = new (arena) authz::flatbuffers::TokenRef[numSamples];
+        auto jwts = new (arena) StringRef[numSamples];
+        auto fbs = new (arena) StringRef[numSamples];
+        for (auto i = 0; i < numSamples; i++) {
+            jwtSpecs[i] = authz::jwt::makeRandomTokenSpec(
+                arena, rng, kty == "EC"_sr ? authz::Algorithm::ES256 : authz::Algorithm::RS256);
+            fbSpecs[i] = authz::flatbuffers::makeRandomTokenSpec(arena, rng);
+        }
+        {
+            auto const jwtSignBegin = timer_monotonic();
+            for (auto i = 0; i < numSamples; i++) {
+                jwts[i] = authz::jwt::signToken(arena, jwtSpecs[i], key);
+            }
+            auto const jwtSignEnd = timer_monotonic();
+            fmt::print("JWT Sign   :         {:.2f} OPS\n", numSamples / (jwtSignEnd - jwtSignBegin));
+        }
+        {
+            auto const jwtVerifyBegin = timer_monotonic();
+            for (auto rep = 0; rep < repeat; rep++) {
+                for (auto i = 0; i < numSamples; i++) {
+                    auto verifyOk = authz::jwt::verifyToken(jwts[i], pubKey);
+                    ASSERT(verifyOk);
+                }
+            }
+            auto const jwtVerifyEnd = timer_monotonic();
+            fmt::print("JWT Verify :         {:.2f} OPS\n", repeat * numSamples / (jwtVerifyEnd - jwtVerifyBegin));
+        }
+        {
+            auto tmpArena = Arena();
+            auto const fbSignBegin = timer_monotonic();
+            for (auto i = 0; i < numSamples; i++) {
+                auto fbToken = authz::flatbuffers::signToken(tmpArena, fbSpecs[i], "defaultKey"_sr, key);
+                auto wr = ObjectWriter([&arena](size_t len) { return new (arena) uint8_t[len]; }, Unversioned());
+                wr.serialize(fbToken);
+                fbs[i] = wr.toStringRef();
+            }
+            auto const fbSignEnd = timer_monotonic();
+            fmt::print("FlatBuffers Sign   : {:.2f} OPS\n", numSamples / (fbSignEnd - fbSignBegin));
+        }
+        {
+            auto const fbVerifyBegin = timer_monotonic();
+            for (auto rep = 0; rep < repeat; rep++) {
+                for (auto i = 0; i < numSamples; i++) {
+                    auto signedToken = ObjectReader::fromStringRef<Standalone<authz::flatbuffers::SignedTokenRef>>(
+                        fbs[i], Unversioned());
+                    auto verifyOk = authz::flatbuffers::verifyToken(signedToken, pubKey);
+                    ASSERT(verifyOk);
+                }
+            }
+            auto const fbVerifyEnd = timer_monotonic();
+            fmt::print("FlatBuffers Verify : {:.2f} OPS\n", repeat * numSamples / (fbVerifyEnd - fbVerifyBegin));
+        }
+    }
+    return Void();
+}
+*/
diff --git a/fdbrpc/actorFuzz.py b/fdbrpc/actorFuzz.py
index dc83b7dbaa..0b6ca8f99e 100755
--- a/fdbrpc/actorFuzz.py
+++ b/fdbrpc/actorFuzz.py
@@ -439,7 +439,7 @@ testCaseCount = 30
 outputFile = open("ActorFuzz.actor.cpp", "wt")
 print(header, file=outputFile)
 print('// THIS FILE WAS GENERATED BY actorFuzz.py; DO NOT MODIFY IT DIRECTLY\n', file=outputFile)
-print('#include "ActorFuzz.h"\n', file=outputFile)
+print('#include "fdbrpc/ActorFuzz.h"\n', file=outputFile)
 print('#ifndef WIN32\n', file=outputFile)
 
 actors = [randomActor(i) for i in range(testCaseCount)]
diff --git a/fdbrpc/dsltest.actor.cpp b/fdbrpc/dsltest.actor.cpp
index fa33301145..b1e557e028 100644
--- a/fdbrpc/dsltest.actor.cpp
+++ b/fdbrpc/dsltest.actor.cpp
@@ -643,9 +643,9 @@ void arenaTest() {
 	{
 		Arena arena;
 		VectorRef<StringRef> test;
-		test.push_back(arena, StringRef(arena, LiteralStringRef("Hello")));
-		test.push_back(arena, StringRef(arena, LiteralStringRef(", ")));
-		test.push_back(arena, StringRef(arena, LiteralStringRef("World!")));
+		test.push_back(arena, StringRef(arena, "Hello"_sr));
+		test.push_back(arena, StringRef(arena, ", "_sr));
+		test.push_back(arena, StringRef(arena, "World!"_sr));
 
 		for (auto i = test.begin(); i != test.end(); ++i)
 			for (auto j = i->begin(); j != i->end(); ++j)
@@ -1200,8 +1200,8 @@ void dsltest() {
 	actorTest1(true);
 	actorTest2(true);
 	actorTest3(true);
-	// if (g_network == &g_simulator)
-	// g_simulator.run( actorTest4(true) );
+	// if (g_network == g_simulator)
+	// g_simulator->run( actorTest4(true) );
 	actorTest5();
 	actorTest6();
 	actorTest7();
diff --git a/fdbrpc/genericactors.actor.cpp b/fdbrpc/genericactors.actor.cpp
index 8eb9ba49b1..887c68990b 100644
--- a/fdbrpc/genericactors.actor.cpp
+++ b/fdbrpc/genericactors.actor.cpp
@@ -27,8 +27,8 @@
 ACTOR Future<Void> disableConnectionFailuresAfter(double time, std::string context) {
 	if (g_network->isSimulated()) {
 		wait(delayUntil(time));
-		g_simulator.connectionFailuresDisableDuration = 1e6;
-		g_simulator.speedUpSimulation = true;
+		g_simulator->connectionFailuresDisableDuration = 1e6;
+		g_simulator->speedUpSimulation = true;
 		TraceEvent(SevWarnAlways, ("DisableConnectionFailures_" + context).c_str());
 	}
 	return Void();
diff --git a/fdbrpc/include/fdbrpc/AsyncFileCached.actor.h b/fdbrpc/include/fdbrpc/AsyncFileCached.actor.h
index 4ea481fdfb..75a6d5f26b 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileCached.actor.h
+++ b/fdbrpc/include/fdbrpc/AsyncFileCached.actor.h
@@ -73,7 +73,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 	explicit EvictablePageCache(int pageSize, int64_t maxSize)
 	  : pageSize(pageSize), maxPages(maxSize / pageSize),
 	    cacheEvictionType(evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY)) {
-		cacheEvictions.init(LiteralStringRef("EvictablePageCache.CacheEvictions"));
+		cacheEvictions.init("EvictablePageCache.CacheEvictions"_sr);
 	}
 
 	void allocate(EvictablePage* page) {
@@ -303,25 +303,25 @@ private:
 	  : filename(filename), uncached(uncached), length(length), prevLength(length), pageCache(pageCache),
 	    currentTruncate(Void()), currentTruncateSize(0), rateControl(nullptr) {
 		if (!g_network->isSimulated()) {
-			countFileCacheWrites.init(LiteralStringRef("AsyncFile.CountFileCacheWrites"), filename);
-			countFileCacheReads.init(LiteralStringRef("AsyncFile.CountFileCacheReads"), filename);
-			countFileCacheWritesBlocked.init(LiteralStringRef("AsyncFile.CountFileCacheWritesBlocked"), filename);
-			countFileCacheReadsBlocked.init(LiteralStringRef("AsyncFile.CountFileCacheReadsBlocked"), filename);
-			countFileCachePageReadsHit.init(LiteralStringRef("AsyncFile.CountFileCachePageReadsHit"), filename);
-			countFileCachePageReadsMissed.init(LiteralStringRef("AsyncFile.CountFileCachePageReadsMissed"), filename);
-			countFileCachePageReadsMerged.init(LiteralStringRef("AsyncFile.CountFileCachePageReadsMerged"), filename);
-			countFileCacheFinds.init(LiteralStringRef("AsyncFile.CountFileCacheFinds"), filename);
-			countFileCacheReadBytes.init(LiteralStringRef("AsyncFile.CountFileCacheReadBytes"), filename);
+			countFileCacheWrites.init("AsyncFile.CountFileCacheWrites"_sr, filename);
+			countFileCacheReads.init("AsyncFile.CountFileCacheReads"_sr, filename);
+			countFileCacheWritesBlocked.init("AsyncFile.CountFileCacheWritesBlocked"_sr, filename);
+			countFileCacheReadsBlocked.init("AsyncFile.CountFileCacheReadsBlocked"_sr, filename);
+			countFileCachePageReadsHit.init("AsyncFile.CountFileCachePageReadsHit"_sr, filename);
+			countFileCachePageReadsMissed.init("AsyncFile.CountFileCachePageReadsMissed"_sr, filename);
+			countFileCachePageReadsMerged.init("AsyncFile.CountFileCachePageReadsMerged"_sr, filename);
+			countFileCacheFinds.init("AsyncFile.CountFileCacheFinds"_sr, filename);
+			countFileCacheReadBytes.init("AsyncFile.CountFileCacheReadBytes"_sr, filename);
 
-			countCacheWrites.init(LiteralStringRef("AsyncFile.CountCacheWrites"));
-			countCacheReads.init(LiteralStringRef("AsyncFile.CountCacheReads"));
-			countCacheWritesBlocked.init(LiteralStringRef("AsyncFile.CountCacheWritesBlocked"));
-			countCacheReadsBlocked.init(LiteralStringRef("AsyncFile.CountCacheReadsBlocked"));
-			countCachePageReadsHit.init(LiteralStringRef("AsyncFile.CountCachePageReadsHit"));
-			countCachePageReadsMissed.init(LiteralStringRef("AsyncFile.CountCachePageReadsMissed"));
-			countCachePageReadsMerged.init(LiteralStringRef("AsyncFile.CountCachePageReadsMerged"));
-			countCacheFinds.init(LiteralStringRef("AsyncFile.CountCacheFinds"));
-			countCacheReadBytes.init(LiteralStringRef("AsyncFile.CountCacheReadBytes"));
+			countCacheWrites.init("AsyncFile.CountCacheWrites"_sr);
+			countCacheReads.init("AsyncFile.CountCacheReads"_sr);
+			countCacheWritesBlocked.init("AsyncFile.CountCacheWritesBlocked"_sr);
+			countCacheReadsBlocked.init("AsyncFile.CountCacheReadsBlocked"_sr);
+			countCachePageReadsHit.init("AsyncFile.CountCachePageReadsHit"_sr);
+			countCachePageReadsMissed.init("AsyncFile.CountCachePageReadsMissed"_sr);
+			countCachePageReadsMerged.init("AsyncFile.CountCachePageReadsMerged"_sr);
+			countCacheFinds.init("AsyncFile.CountCacheFinds"_sr);
+			countCacheReadBytes.init("AsyncFile.CountCacheReadBytes"_sr);
 		}
 	}
 
diff --git a/fdbrpc/include/fdbrpc/AsyncFileEIO.actor.h b/fdbrpc/include/fdbrpc/AsyncFileEIO.actor.h
index 1a4c37ed20..b3f2889450 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileEIO.actor.h
+++ b/fdbrpc/include/fdbrpc/AsyncFileEIO.actor.h
@@ -107,8 +107,8 @@ public:
 		    .detail("Flags", flags);
 
 		if ((flags & OPEN_LOCK) && !lock_fd(r->result)) {
-			TraceEvent(SevError, "UnableToLockFile").detail("Filename", filename).GetLastError();
-			throw io_error();
+			TraceEvent(SevWarn, "UnableToLockFile").detail("Filename", filename).GetLastError();
+			throw lock_file_failure();
 		}
 
 		return Reference<IAsyncFile>(new AsyncFileEIO(r->result, flags, filename));
@@ -116,7 +116,7 @@ public:
 	static Future<Void> deleteFile(std::string filename, bool mustBeDurable) {
 		::deleteFile(filename);
 		if (mustBeDurable) {
-			TEST(true); // deleteFile and fsync parent dir
+			CODE_PROBE(true, "deleteFile and fsync parent dir");
 			return async_fsync_parent(filename);
 		} else
 			return Void();
@@ -279,11 +279,11 @@ private:
 	AsyncFileEIO(int fd, int flags, std::string const& filename)
 	  : fd(fd), flags(flags), err(new ErrorInfo), filename(filename) {
 		if (!g_network->isSimulated()) {
-			countFileLogicalWrites.init(LiteralStringRef("AsyncFile.CountFileLogicalWrites"), filename);
-			countFileLogicalReads.init(LiteralStringRef("AsyncFile.CountFileLogicalReads"), filename);
+			countFileLogicalWrites.init("AsyncFile.CountFileLogicalWrites"_sr, filename);
+			countFileLogicalReads.init("AsyncFile.CountFileLogicalReads"_sr, filename);
 
-			countLogicalWrites.init(LiteralStringRef("AsyncFile.CountLogicalWrites"));
-			countLogicalReads.init(LiteralStringRef("AsyncFile.CountLogicalReads"));
+			countLogicalWrites.init("AsyncFile.CountLogicalWrites"_sr);
+			countLogicalReads.init("AsyncFile.CountLogicalReads"_sr);
 		}
 	}
 
diff --git a/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h b/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h
index dd1f1b014d..40a84e6d9e 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h
+++ b/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h
@@ -38,7 +38,7 @@
 #include <sys/syscall.h>
 #include "fdbrpc/linux_kaio.h"
 #include "flow/Knobs.h"
-#include "flow/Histogram.h"
+#include "fdbrpc/Stats.h"
 #include "flow/UnitTest.h"
 #include "crc32/crc32c.h"
 #include "flow/genericactors.actor.h"
@@ -48,14 +48,6 @@
 // /data/v7/fdb/
 #define KAIO_LOGGING 0
 
-struct AsyncFileKAIOMetrics {
-	Reference<Histogram> readLatencyDist;
-	Reference<Histogram> writeLatencyDist;
-	Reference<Histogram> syncLatencyDist;
-} g_asyncFileKAIOMetrics;
-
-Future<Void> g_asyncFileKAIOHistogramLogger;
-
 DESCR struct SlowAioSubmit {
 	int64_t submitDuration; // ns
 	int64_t truncateDuration; // ns
@@ -66,6 +58,25 @@ DESCR struct SlowAioSubmit {
 
 class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFileKAIO> {
 public:
+	struct AsyncFileKAIOMetrics {
+		LatencySample readLatencySample = { "AsyncFileKAIOReadLatency",
+			                                UID(),
+			                                FLOW_KNOBS->KAIO_LATENCY_LOGGING_INTERVAL,
+			                                FLOW_KNOBS->KAIO_LATENCY_SAMPLE_SIZE };
+		LatencySample writeLatencySample = { "AsyncFileKAIOWriteLatency",
+			                                 UID(),
+			                                 FLOW_KNOBS->KAIO_LATENCY_LOGGING_INTERVAL,
+			                                 FLOW_KNOBS->KAIO_LATENCY_SAMPLE_SIZE };
+		LatencySample syncLatencySample = { "AsyncFileKAIOSyncLatency",
+			                                UID(),
+			                                FLOW_KNOBS->KAIO_LATENCY_LOGGING_INTERVAL,
+			                                FLOW_KNOBS->KAIO_LATENCY_SAMPLE_SIZE };
+	};
+
+	static AsyncFileKAIOMetrics& getMetrics() {
+		static AsyncFileKAIOMetrics metrics;
+		return metrics;
+	}
 
 #if KAIO_LOGGING
 private:
@@ -162,8 +173,8 @@ public:
 			       // by l_whence and l_start through to the end of file, no matter how large the file grows."
 			lockDesc.l_pid = 0;
 			if (fcntl(fd, F_SETLK, &lockDesc) == -1) {
-				TraceEvent(SevError, "UnableToLockFile").detail("Filename", filename).GetLastError();
-				return io_error();
+				TraceEvent(SevWarn, "UnableToLockFile").detail("Filename", filename).GetLastError();
+				return lock_file_failure();
 			}
 		}
 
@@ -180,12 +191,12 @@ public:
 	static void init(Reference<IEventFD> ev, double ioTimeout) {
 		ASSERT(!FLOW_KNOBS->DISABLE_POSIX_KERNEL_AIO);
 		if (!g_network->isSimulated()) {
-			ctx.countAIOSubmit.init(LiteralStringRef("AsyncFile.CountAIOSubmit"));
-			ctx.countAIOCollect.init(LiteralStringRef("AsyncFile.CountAIOCollect"));
-			ctx.submitMetric.init(LiteralStringRef("AsyncFile.Submit"));
-			ctx.countPreSubmitTruncate.init(LiteralStringRef("AsyncFile.CountPreAIOSubmitTruncate"));
-			ctx.preSubmitTruncateBytes.init(LiteralStringRef("AsyncFile.PreAIOSubmitTruncateBytes"));
-			ctx.slowAioSubmitMetric.init(LiteralStringRef("AsyncFile.SlowAIOSubmit"));
+			ctx.countAIOSubmit.init("AsyncFile.CountAIOSubmit"_sr);
+			ctx.countAIOCollect.init("AsyncFile.CountAIOCollect"_sr);
+			ctx.submitMetric.init("AsyncFile.Submit"_sr);
+			ctx.countPreSubmitTruncate.init("AsyncFile.CountPreAIOSubmitTruncate"_sr);
+			ctx.preSubmitTruncateBytes.init("AsyncFile.PreAIOSubmitTruncateBytes"_sr);
+			ctx.slowAioSubmitMetric.init("AsyncFile.SlowAIOSubmit"_sr);
 		}
 
 		int rc = io_setup(FLOW_KNOBS->MAX_OUTSTANDING, &ctx.iocx);
@@ -353,7 +364,7 @@ public:
 #endif
 
 		KAIOLogEvent(logFile, id, OpLogEntry::SYNC, OpLogEntry::START);
-		double start_time = now();
+		double start_time = timer();
 
 		Future<Void> fsync = throwErrorIfFailed(
 		    Reference<AsyncFileKAIO>::addRef(this),
@@ -365,7 +376,7 @@ public:
 
 		fsync = map(fsync, [=](Void r) mutable {
 			KAIOLogEvent(logFile, id, OpLogEntry::SYNC, OpLogEntry::COMPLETE);
-			g_asyncFileKAIOMetrics.syncLatencyDist->sampleSeconds(now() - start_time);
+			getMetrics().syncLatencySample.addMeasurement(timer() - start_time);
 			return r;
 		});
 
@@ -404,6 +415,7 @@ public:
 			int64_t previousTruncateBytes = ctx.preSubmitTruncateBytes;
 			int64_t largestTruncate = 0;
 
+			double start = timer();
 			for (int i = 0; i < n; i++) {
 				auto io = ctx.queue.top();
 
@@ -411,7 +423,7 @@ public:
 
 				ctx.queue.pop();
 				toStart[i] = io;
-				io->startTime = now();
+				io->startTime = start;
 
 				if (ctx.ioTimeout > 0) {
 					ctx.appendToRequestList(io);
@@ -636,27 +648,17 @@ private:
 	  : failed(false), fd(fd), flags(flags), filename(filename) {
 		ASSERT(!FLOW_KNOBS->DISABLE_POSIX_KERNEL_AIO);
 		if (!g_network->isSimulated()) {
-			countFileLogicalWrites.init(LiteralStringRef("AsyncFile.CountFileLogicalWrites"), filename);
-			countFileLogicalReads.init(LiteralStringRef("AsyncFile.CountFileLogicalReads"), filename);
-			countLogicalWrites.init(LiteralStringRef("AsyncFile.CountLogicalWrites"));
-			countLogicalReads.init(LiteralStringRef("AsyncFile.CountLogicalReads"));
-			if (!g_asyncFileKAIOHistogramLogger.isValid()) {
-				auto& metrics = g_asyncFileKAIOMetrics;
-				metrics.readLatencyDist = Reference<Histogram>(new Histogram(
-				    Reference<HistogramRegistry>(), "AsyncFileKAIO", "ReadLatency", Histogram::Unit::microseconds));
-				metrics.writeLatencyDist = Reference<Histogram>(new Histogram(
-				    Reference<HistogramRegistry>(), "AsyncFileKAIO", "WriteLatency", Histogram::Unit::microseconds));
-				metrics.syncLatencyDist = Reference<Histogram>(new Histogram(
-				    Reference<HistogramRegistry>(), "AsyncFileKAIO", "SyncLatency", Histogram::Unit::microseconds));
-				g_asyncFileKAIOHistogramLogger = histogramLogger(FLOW_KNOBS->DISK_METRIC_LOGGING_INTERVAL);
-			}
+			countFileLogicalWrites.init("AsyncFile.CountFileLogicalWrites"_sr, filename);
+			countFileLogicalReads.init("AsyncFile.CountFileLogicalReads"_sr, filename);
+			countLogicalWrites.init("AsyncFile.CountLogicalWrites"_sr);
+			countLogicalReads.init("AsyncFile.CountLogicalReads"_sr);
 		}
 
 #if KAIO_LOGGING
 		logFile = nullptr;
 		// TODO:  Don't do this hacky investigation-specific thing
 		StringRef fname(filename);
-		if (fname.endsWith(LiteralStringRef(".sqlite")) || fname.endsWith(LiteralStringRef(".sqlite-wal"))) {
+		if (fname.endsWith(".sqlite"_sr) || fname.endsWith(".sqlite-wal"_sr)) {
 			std::string logFileName = basename(filename);
 			while (logFileName.find("/") != std::string::npos)
 				logFileName = logFileName.substr(logFileName.find("/") + 1);
@@ -735,6 +737,8 @@ private:
 					break;
 			}
 
+			double currentTime = timer();
+
 			++ctx.countAIOCollect;
 			// printf("io_getevents: collected %d/%d in %f us (%d queued)\n", n, ctx.outstanding, (timer()-before)*1e6,
 			// ctx.queue.size());
@@ -753,7 +757,6 @@ private:
 			ctx.outstanding -= n;
 
 			if (ctx.ioTimeout > 0) {
-				double currentTime = now();
 				while (ctx.submittedRequestList && currentTime - ctx.submittedRequestList->startTime > ctx.ioTimeout) {
 					ctx.submittedRequestList->timeout(ctx.timeoutWarnOnly);
 					ctx.removeFromRequestList(ctx.submittedRequestList);
@@ -769,13 +772,12 @@ private:
 					ctx.removeFromRequestList(iob);
 				}
 
-				auto& metrics = g_asyncFileKAIOMetrics;
 				switch (iob->aio_lio_opcode) {
 				case IO_CMD_PREAD:
-					metrics.readLatencyDist->sampleSeconds(now() - iob->startTime);
+					getMetrics().readLatencySample.addMeasurement(currentTime - iob->startTime);
 					break;
 				case IO_CMD_PWRITE:
-					metrics.writeLatencyDist->sampleSeconds(now() - iob->startTime);
+					getMetrics().writeLatencySample.addMeasurement(currentTime - iob->startTime);
 					break;
 				}
 
@@ -783,19 +785,6 @@ private:
 			}
 		}
 	}
-
-	ACTOR static Future<Void> histogramLogger(double interval) {
-		state double currentTime;
-		loop {
-			currentTime = now();
-			wait(delay(interval));
-			double elapsed = now() - currentTime;
-			auto& metrics = g_asyncFileKAIOMetrics;
-			metrics.readLatencyDist->writeToLog(elapsed);
-			metrics.writeLatencyDist->writeToLog(elapsed);
-			metrics.syncLatencyDist->writeToLog(elapsed);
-		}
-	}
 };
 
 #if KAIO_LOGGING
diff --git a/fdbrpc/include/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/include/fdbrpc/AsyncFileNonDurable.actor.h
index 071839bc15..c01e7f27df 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileNonDurable.actor.h
+++ b/fdbrpc/include/fdbrpc/AsyncFileNonDurable.actor.h
@@ -48,7 +48,7 @@ ACTOR Future<Void> sendErrorOnProcess(ISimulator::ProcessInfo* process,
 ACTOR template <class T>
 Future<T> sendErrorOnShutdown(Future<T> in) {
 	choose {
-		when(wait(success(g_simulator.getCurrentProcess()->shutdownSignal.getFuture()))) {
+		when(wait(success(g_simulator->getCurrentProcess()->shutdownSignal.getFuture()))) {
 			throw io_error().asInjectedFault();
 		}
 		when(T rep = wait(in)) { return rep; }
@@ -64,14 +64,14 @@ public:
 	explicit AsyncFileDetachable(Reference<IAsyncFile> file) : file(file) { shutdown = doShutdown(this); }
 
 	ACTOR Future<Void> doShutdown(AsyncFileDetachable* self) {
-		wait(success(g_simulator.getCurrentProcess()->shutdownSignal.getFuture()));
+		wait(success(g_simulator->getCurrentProcess()->shutdownSignal.getFuture()));
 		self->file = Reference<IAsyncFile>();
 		return Void();
 	}
 
 	ACTOR static Future<Reference<IAsyncFile>> open(Future<Reference<IAsyncFile>> wrappedFile) {
 		choose {
-			when(wait(success(g_simulator.getCurrentProcess()->shutdownSignal.getFuture()))) {
+			when(wait(success(g_simulator->getCurrentProcess()->shutdownSignal.getFuture()))) {
 				throw io_error().asInjectedFault();
 			}
 			when(Reference<IAsyncFile> f = wait(wrappedFile)) { return makeReference<AsyncFileDetachable>(f); }
@@ -82,31 +82,31 @@ public:
 	void delref() override { ReferenceCounted<AsyncFileDetachable>::delref(); }
 
 	Future<int> read(void* data, int length, int64_t offset) override {
-		if (!file.getPtr() || g_simulator.getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
 			return io_error().asInjectedFault();
 		return sendErrorOnShutdown(file->read(data, length, offset));
 	}
 
 	Future<Void> write(void const* data, int length, int64_t offset) override {
-		if (!file.getPtr() || g_simulator.getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
 			return io_error().asInjectedFault();
 		return sendErrorOnShutdown(file->write(data, length, offset));
 	}
 
 	Future<Void> truncate(int64_t size) override {
-		if (!file.getPtr() || g_simulator.getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
 			return io_error().asInjectedFault();
 		return sendErrorOnShutdown(file->truncate(size));
 	}
 
 	Future<Void> sync() override {
-		if (!file.getPtr() || g_simulator.getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
 			return io_error().asInjectedFault();
 		return sendErrorOnShutdown(file->sync());
 	}
 
 	Future<int64_t> size() const override {
-		if (!file.getPtr() || g_simulator.getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
 			return io_error().asInjectedFault();
 		return sendErrorOnShutdown(file->size());
 	}
@@ -214,12 +214,12 @@ public:
 	                                                Future<Reference<IAsyncFile>> wrappedFile,
 	                                                Reference<DiskParameters> diskParameters,
 	                                                bool aio) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
 		state TaskPriority currentTaskID = g_network->getCurrentTask();
 		state Future<Void> shutdown = success(currentProcess->shutdownSignal.getFuture());
 
-		//TraceEvent("AsyncFileNonDurableOpenBegin").detail("Filename", filename).detail("Addr", g_simulator.getCurrentProcess()->address);
-		wait(g_simulator.onMachine(currentProcess));
+		//TraceEvent("AsyncFileNonDurableOpenBegin").detail("Filename", filename).detail("Addr", g_simulator->getCurrentProcess()->address);
+		wait(g_simulator->onMachine(currentProcess));
 		try {
 			wait(success(wrappedFile) || shutdown);
 
@@ -237,7 +237,7 @@ public:
 				//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete2").detail("Filename", filename);
 				if (shutdown.isReady())
 					throw io_error().asInjectedFault();
-				wait(g_simulator.onProcess(currentProcess, currentTaskID));
+				wait(g_simulator->onProcess(currentProcess, currentTaskID));
 			}
 
 			state Reference<AsyncFileNonDurable> nonDurableFile(
@@ -252,7 +252,7 @@ public:
 
 			//TraceEvent("AsyncFileNonDurableOpenComplete").detail("Filename", filename);
 
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 
 			return nonDurableFile;
 		} catch (Error& e) {
@@ -260,8 +260,8 @@ public:
 			std::string currentFilename =
 			    (wrappedFile.isReady() && !wrappedFile.isError()) ? wrappedFile.get()->getFilename() : actualFilename;
 			currentProcess->machine->openFiles.erase(currentFilename);
-			//TraceEvent("AsyncFileNonDurableOpenError").errorUnsuppressed(e).detail("Filename", filename).detail("Address", currentProcess->address).detail("Addr", g_simulator.getCurrentProcess()->address);
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			//TraceEvent("AsyncFileNonDurableOpenError").errorUnsuppressed(e).detail("Filename", filename).detail("Address", currentProcess->address).detail("Addr", g_simulator->getCurrentProcess()->address);
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 			throw err;
 		}
 	}
@@ -290,7 +290,7 @@ public:
 
 	// Removes a file from the openFiles map
 	static void removeOpenFile(std::string filename, AsyncFileNonDurable* file) {
-		auto& openFiles = g_simulator.getCurrentProcess()->machine->openFiles;
+		auto& openFiles = g_simulator->getCurrentProcess()->machine->openFiles;
 
 		auto iter = openFiles.find(filename);
 
@@ -360,7 +360,7 @@ public:
 	//(e.g. to simulate power failure)
 	Future<Void> kill() {
 		TraceEvent("AsyncFileNonDurable_Kill", id).detail("Filename", filename);
-		TEST(true); // AsyncFileNonDurable was killed
+		CODE_PROBE(true, "AsyncFileNonDurable was killed");
 		return sync(this, false);
 	}
 
@@ -404,7 +404,7 @@ private:
 			TraceEvent("AsyncFileNonDurable_KilledFileOperation", self->id)
 			    .detail("In", context)
 			    .detail("Filename", self->filename);
-			TEST(true); // AsyncFileNonDurable operation killed
+			CODE_PROBE(true, "AsyncFileNonDurable operation killed");
 			throw io_error().asInjectedFault();
 		}
 
@@ -425,24 +425,24 @@ private:
 
 		debugFileCheck("AsyncFileNonDurableRead", self->filename, data, offset, length);
 
-		// if(g_simulator.getCurrentProcess()->rebooting)
+		// if(g_simulator->getCurrentProcess()->rebooting)
 		//TraceEvent("AsyncFileNonDurable_ReadEnd", self->id).detail("Filename", self->filename);
 
 		return readFuture.get();
 	}
 
 	ACTOR Future<int> read(AsyncFileNonDurable* self, void* data, int length, int64_t offset) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
 		state TaskPriority currentTaskID = g_network->getCurrentTask();
-		wait(g_simulator.onMachine(currentProcess));
+		wait(g_simulator->onMachine(currentProcess));
 
 		try {
 			state int rep = wait(self->onRead(self, data, length, offset));
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 			return rep;
 		} catch (Error& e) {
 			state Error err = e;
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 			throw err;
 		}
 	}
@@ -457,12 +457,12 @@ private:
 	                         int length,
 	                         int64_t offset) {
 		state Standalone<StringRef> dataCopy(StringRef((uint8_t*)data, length));
-		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
 		state TaskPriority currentTaskID = g_network->getCurrentTask();
-		wait(g_simulator.onMachine(currentProcess));
+		wait(g_simulator->onMachine(currentProcess));
 
 		state double delayDuration =
-		    g_simulator.speedUpSimulation ? 0.0001 : (deterministicRandom()->random01() * self->maxWriteDelay);
+		    g_simulator->speedUpSimulation ? 0.0001 : (deterministicRandom()->random01() * self->maxWriteDelay);
 
 		state Future<bool> startSyncFuture = self->startSyncPromise.getFuture();
 
@@ -475,7 +475,7 @@ private:
 			    self->getModificationsAndInsert(offset, length, true, writeEnded);
 			self->minSizeAfterPendingModifications = std::max(self->minSizeAfterPendingModifications, offset + length);
 
-			if (BUGGIFY_WITH_PROB(0.001) && !g_simulator.speedUpSimulation)
+			if (BUGGIFY_WITH_PROB(0.001) && !g_simulator->speedUpSimulation)
 				priorModifications.push_back(
 				    delay(deterministicRandom()->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY) ||
 				    self->killed.getFuture());
@@ -603,13 +603,13 @@ private:
 					    .detail("HasGarbage", garbage)
 					    .detail("Side", side)
 					    .detail("Filename", self->filename);
-					TEST(true); // AsyncFileNonDurable bad write
+					CODE_PROBE(true, "AsyncFileNonDurable bad write");
 				} else {
 					TraceEvent("AsyncFileNonDurable_DroppedWrite", self->id)
 					    .detail("Offset", offset + writeOffset + pageOffset)
 					    .detail("Length", sectorLength)
 					    .detail("Filename", self->filename);
-					TEST(true); // AsyncFileNonDurable dropped write
+					CODE_PROBE(true, "AsyncFileNonDurable dropped write");
 				}
 
 				pageOffset += sectorLength;
@@ -629,12 +629,12 @@ private:
 	                            Promise<Void> truncateStarted,
 	                            Future<Future<Void>> ownFuture,
 	                            int64_t size) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
 		state TaskPriority currentTaskID = g_network->getCurrentTask();
-		wait(g_simulator.onMachine(currentProcess));
+		wait(g_simulator->onMachine(currentProcess));
 
 		state double delayDuration =
-		    g_simulator.speedUpSimulation ? 0.0001 : (deterministicRandom()->random01() * self->maxWriteDelay);
+		    g_simulator->speedUpSimulation ? 0.0001 : (deterministicRandom()->random01() * self->maxWriteDelay);
 		state Future<bool> startSyncFuture = self->startSyncPromise.getFuture();
 
 		try {
@@ -689,7 +689,7 @@ private:
 			wait(self->file->truncate(size));
 		else {
 			TraceEvent("AsyncFileNonDurable_DroppedTruncate", self->id).detail("Size", size);
-			TEST(true); // AsyncFileNonDurable dropped truncate
+			CODE_PROBE(true, "AsyncFileNonDurable dropped truncate");
 		}
 
 		return Void();
@@ -753,7 +753,7 @@ private:
 			// temporary file and then renamed to the correct location once sync is called.  By not calling sync, we
 			// simulate a failure to fsync the directory storing the file
 			if (self->hasBeenSynced && writeDurable && deterministicRandom()->random01() < 0.5) {
-				TEST(true); // AsyncFileNonDurable kill was durable and synced
+				CODE_PROBE(true, "AsyncFileNonDurable kill was durable and synced");
 				wait(success(errorOr(self->file->sync())));
 			}
 
@@ -773,18 +773,18 @@ private:
 	}
 
 	ACTOR Future<Void> sync(AsyncFileNonDurable* self, bool durable) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
 		state TaskPriority currentTaskID = g_network->getCurrentTask();
-		wait(g_simulator.onMachine(currentProcess));
+		wait(g_simulator->onMachine(currentProcess));
 
 		try {
 			wait(self->onSync(self, durable));
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 
 			return Void();
 		} catch (Error& e) {
 			state Error err = e;
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 			throw err;
 		}
 	}
@@ -806,32 +806,33 @@ private:
 	}
 
 	ACTOR static Future<int64_t> size(AsyncFileNonDurable const* self) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
 		state TaskPriority currentTaskID = g_network->getCurrentTask();
 
-		wait(g_simulator.onMachine(currentProcess));
+		wait(g_simulator->onMachine(currentProcess));
 
 		try {
 			state int64_t rep = wait(onSize(self));
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 
 			return rep;
 		} catch (Error& e) {
 			state Error err = e;
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 			throw err;
 		}
 	}
 
 	// Finishes all outstanding actors on an AsyncFileNonDurable and then deletes it
 	ACTOR Future<Void> closeFile(AsyncFileNonDurable* self) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
 		state TaskPriority currentTaskID = g_network->getCurrentTask();
 		state std::string filename = self->filename;
 
-		g_simulator.getMachineByNetworkAddress(self->openedAddress)->deletingOrClosingFiles.insert(self->getFilename());
+		g_simulator->getMachineByNetworkAddress(self->openedAddress)
+		    ->deletingOrClosingFiles.insert(self->getFilename());
 
-		wait(g_simulator.onMachine(currentProcess));
+		wait(g_simulator->onMachine(currentProcess));
 		try {
 			// Make sure all writes have gone through.
 			Promise<bool> startSyncPromise = self->startSyncPromise;
@@ -854,8 +855,8 @@ private:
 				wait(self->killComplete.getFuture());
 
 			// Remove this file from the filesBeingDeleted map so that new files can be created with this filename
-			g_simulator.getMachineByNetworkAddress(self->openedAddress)->closingFiles.erase(self->getFilename());
-			g_simulator.getMachineByNetworkAddress(self->openedAddress)
+			g_simulator->getMachineByNetworkAddress(self->openedAddress)->closingFiles.erase(self->getFilename());
+			g_simulator->getMachineByNetworkAddress(self->openedAddress)
 			    ->deletingOrClosingFiles.erase(self->getFilename());
 			AsyncFileNonDurable::filesBeingDeleted.erase(self->filename);
 			//TraceEvent("AsyncFileNonDurable_FinishDelete", self->id).detail("Filename", self->filename);
diff --git a/fdbrpc/include/fdbrpc/FlowTransport.h b/fdbrpc/include/fdbrpc/FlowTransport.h
index b6c12fa170..af0a7a8b75 100644
--- a/fdbrpc/include/fdbrpc/FlowTransport.h
+++ b/fdbrpc/include/fdbrpc/FlowTransport.h
@@ -20,25 +20,21 @@
 
 #ifndef FLOW_TRANSPORT_H
 #define FLOW_TRANSPORT_H
-#include "flow/Arena.h"
 #pragma once
 
 #include <algorithm>
+
+#include "fdbrpc/ContinuousSample.h"
 #include "fdbrpc/HealthMonitor.h"
 #include "flow/genericactors.actor.h"
 #include "flow/network.h"
 #include "flow/FileIdentifier.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/Net2Packet.h"
-#include "fdbrpc/ContinuousSample.h"
+#include "flow/Arena.h"
+#include "flow/PKey.h"
 
-enum {
-	WLTOKEN_ENDPOINT_NOT_FOUND = 0,
-	WLTOKEN_PING_PACKET,
-	WLTOKEN_AUTH_TENANT,
-	WLTOKEN_UNAUTHORIZED_ENDPOINT,
-	WLTOKEN_FIRST_AVAILABLE
-};
+enum { WLTOKEN_ENDPOINT_NOT_FOUND = 0, WLTOKEN_PING_PACKET, WLTOKEN_UNAUTHORIZED_ENDPOINT, WLTOKEN_FIRST_AVAILABLE };
 
 #pragma pack(push, 4)
 class Endpoint {
@@ -191,7 +187,7 @@ struct Peer : public ReferenceCounted<Peer> {
 
 class IPAllowList;
 
-class FlowTransport {
+class FlowTransport : NonCopyable {
 public:
 	FlowTransport(uint64_t transportId, int maxWellKnownEndpoints, IPAllowList const* allowList);
 	~FlowTransport();
@@ -293,6 +289,21 @@ public:
 
 	HealthMonitor* healthMonitor();
 
+	bool currentDeliveryPeerIsTrusted() const;
+	NetworkAddress currentDeliveryPeerAddress() const;
+
+	Optional<PublicKey> getPublicKeyByName(StringRef name) const;
+	// Adds or replaces a public key
+	void addPublicKey(StringRef name, PublicKey key);
+	void removePublicKey(StringRef name);
+	void removeAllPublicKeys();
+
+	// Synchronously load and apply JWKS (RFC 7517) public key file with which to verify authorization tokens.
+	void loadPublicKeyFile(const std::string& publicKeyFilePath);
+
+	// Periodically read JWKS (RFC 7517) public key file to refresh public key set.
+	void watchPublicKeyFile(const std::string& publicKeyFilePath);
+
 private:
 	class TransportData* self;
 };
diff --git a/fdbclient/include/fdbclient/HTTP.h b/fdbrpc/include/fdbrpc/HTTP.h
similarity index 90%
rename from fdbclient/include/fdbclient/HTTP.h
rename to fdbrpc/include/fdbrpc/HTTP.h
index a629e9ecd6..ad6b649d19 100644
--- a/fdbclient/include/fdbclient/HTTP.h
+++ b/fdbrpc/include/fdbrpc/HTTP.h
@@ -68,6 +68,12 @@ Future<Reference<Response>> doRequest(Reference<IConnection> const& conn,
                                       Reference<IRateControl> const& recvRate,
                                       const std::string& requestHeader = std::string());
 
+// Connect to proxy, send CONNECT command, and connect to the remote host.
+Future<Reference<IConnection>> proxyConnect(const std::string& remoteHost,
+                                            const std::string& remoteService,
+                                            const std::string& proxyHost,
+                                            const std::string& proxyService);
+
 constexpr int HTTP_STATUS_CODE_OK = 200;
 constexpr int HTTP_STATUS_CODE_CREATED = 201;
 constexpr int HTTP_STATUS_CODE_ACCEPTED = 202;
diff --git a/fdbrpc/include/fdbrpc/LoadBalance.actor.h b/fdbrpc/include/fdbrpc/LoadBalance.actor.h
index 7b742b2798..3eb2133c21 100644
--- a/fdbrpc/include/fdbrpc/LoadBalance.actor.h
+++ b/fdbrpc/include/fdbrpc/LoadBalance.actor.h
@@ -144,9 +144,9 @@ Future<Void> tssComparison(Req req,
 			tssData.metrics->recordLatency(req, srcEndTime - startTime, tssEndTime - startTime);
 
 			if (!TSS_doCompare(src.get(), tss.get().get())) {
-				TEST(true); // TSS Mismatch
+				CODE_PROBE(true, "TSS Mismatch");
 				state TraceEvent mismatchEvent(
-				    (g_network->isSimulated() && g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations)
+				    (g_network->isSimulated() && g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations)
 				        ? SevWarnAlways
 				        : SevError,
 				    TSS_mismatchTraceName(req));
@@ -154,7 +154,7 @@ Future<Void> tssComparison(Req req,
 				mismatchEvent.detail("TSSID", tssData.tssId);
 
 				if (FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS && ssTeam->size() > 1) {
-					TEST(true); // checking TSS mismatch against rest of storage team
+					CODE_PROBE(true, "checking TSS mismatch against rest of storage team");
 
 					// if there is more than 1 SS in the team, attempt to verify that the other SS servers have the same
 					// data
@@ -199,9 +199,9 @@ Future<Void> tssComparison(Req req,
 				if (tssData.metrics->shouldRecordDetailedMismatch()) {
 					TSS_traceMismatch(mismatchEvent, req, src.get(), tss.get().get());
 
-					TEST(FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Full TSS Mismatch
-					TEST(!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Partial TSS Mismatch and storing
-					                                                         // the rest in FDB
+					CODE_PROBE(FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL, "Tracing Full TSS Mismatch");
+					CODE_PROBE(!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL,
+					           "Tracing Partial TSS Mismatch and storing the rest in FDB");
 
 					if (!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL) {
 						mismatchEvent.disable();
@@ -210,7 +210,7 @@ Future<Void> tssComparison(Req req,
 
 						// record a summarized trace event instead
 						TraceEvent summaryEvent((g_network->isSimulated() &&
-						                         g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations)
+						                         g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations)
 						                            ? SevWarnAlways
 						                            : SevError,
 						                        TSS_mismatchTraceName(req));
@@ -272,7 +272,7 @@ struct RequestData : NonCopyable {
 			Optional<TSSEndpointData> tssData = model->getTssData(stream->getEndpoint().token.first());
 
 			if (tssData.present()) {
-				TEST(true); // duplicating request to TSS
+				CODE_PROBE(true, "duplicating request to TSS");
 				resetReply(request);
 				// FIXME: optimize to avoid creating new netNotifiedQueue for each message
 				RequestStream<Request, P> tssRequestStream(tssData.get().endpoint);
@@ -490,6 +490,12 @@ Future<REPLY_TYPE(Request)> loadBalance(
 				// server count is within "LOAD_BALANCE_MAX_BAD_OPTIONS". We
 				// do not need to consider any remote servers.
 				break;
+			} else if (badServers == alternatives->countBest() && i == badServers) {
+				TraceEvent("AllLocalAlternativesFailed")
+				    .suppressFor(1.0)
+				    .detail("Alternatives", alternatives->description())
+				    .detail("Total", alternatives->size())
+				    .detail("Best", alternatives->countBest());
 			}
 
 			RequestStream<Request, P> const* thisStream = &alternatives->get(i, channel);
@@ -591,6 +597,7 @@ Future<REPLY_TYPE(Request)> loadBalance(
 		// nextAlt. This logic matters only if model == nullptr. Otherwise, the
 		// bestAlt and nextAlt have been decided.
 		state RequestStream<Request, P> const* stream = nullptr;
+		state LBDistance::Type distance;
 		for (int alternativeNum = 0; alternativeNum < alternatives->size(); alternativeNum++) {
 			int useAlt = nextAlt;
 			if (nextAlt == startAlt)
@@ -599,6 +606,7 @@ Future<REPLY_TYPE(Request)> loadBalance(
 				useAlt = (nextAlt + alternatives->size() - 1) % alternatives->size();
 
 			stream = &alternatives->get(useAlt, channel);
+			distance = alternatives->getDistance(useAlt);
 			if (!IFailureMonitor::failureMonitor().getState(stream->getEndpoint()).failed &&
 			    (!firstRequestEndpoint.present() || stream->getEndpoint().token.first() != firstRequestEndpoint.get()))
 				break;
@@ -606,6 +614,7 @@ Future<REPLY_TYPE(Request)> loadBalance(
 			if (nextAlt == startAlt)
 				triedAllOptions = TriedAllOptions::True;
 			stream = nullptr;
+			distance = LBDistance::DISTANT;
 		}
 
 		if (!stream && !firstRequestData.isValid()) {
@@ -641,6 +650,18 @@ Future<REPLY_TYPE(Request)> loadBalance(
 			firstRequestEndpoint = Optional<uint64_t>();
 		} else if (firstRequestData.isValid()) {
 			// Issue a second request, the first one is taking a long time.
+			if (distance == LBDistance::DISTANT) {
+				TraceEvent("LBDistant2nd")
+				    .suppressFor(0.1)
+				    .detail("Distance", (int)distance)
+				    .detail("BackOff", backoff)
+				    .detail("TriedAllOptions", triedAllOptions)
+				    .detail("Alternatives", alternatives->description())
+				    .detail("Token", stream->getEndpoint().token)
+				    .detail("Total", alternatives->size())
+				    .detail("Best", alternatives->countBest())
+				    .detail("Attempts", numAttempts);
+			}
 			secondRequestData.startRequest(backoff, triedAllOptions, stream, request, model, alternatives, channel);
 
 			loop choose {
@@ -668,6 +689,18 @@ Future<REPLY_TYPE(Request)> loadBalance(
 			}
 		} else {
 			// Issue a request, if it takes too long to get a reply, go around the loop
+			if (distance == LBDistance::DISTANT) {
+				TraceEvent("LBDistant")
+				    .suppressFor(0.1)
+				    .detail("Distance", (int)distance)
+				    .detail("BackOff", backoff)
+				    .detail("TriedAllOptions", triedAllOptions)
+				    .detail("Alternatives", alternatives->description())
+				    .detail("Token", stream->getEndpoint().token)
+				    .detail("Total", alternatives->size())
+				    .detail("Best", alternatives->countBest())
+				    .detail("Attempts", numAttempts);
+			}
 			firstRequestData.startRequest(backoff, triedAllOptions, stream, request, model, alternatives, channel);
 			firstRequestEndpoint = stream->getEndpoint().token.first();
 
diff --git a/fdbrpc/include/fdbrpc/Locality.h b/fdbrpc/include/fdbrpc/Locality.h
index 3e8d07f6f4..20269b6d05 100644
--- a/fdbrpc/include/fdbrpc/Locality.h
+++ b/fdbrpc/include/fdbrpc/Locality.h
@@ -43,6 +43,7 @@ struct ProcessClass {
 		DataDistributorClass,
 		CoordinatorClass,
 		RatekeeperClass,
+		ConsistencyScanClass,
 		StorageCacheClass,
 		BackupClass,
 		GrvProxyClass,
@@ -72,6 +73,7 @@ struct ProcessClass {
 		ClusterController,
 		DataDistributor,
 		Ratekeeper,
+		ConsistencyScan,
 		BlobManager,
 		BlobWorker,
 		StorageCache,
@@ -110,6 +112,7 @@ public:
 		else if (s=="data_distributor") _class = DataDistributorClass;
 		else if (s=="coordinator") _class = CoordinatorClass;
 		else if (s=="ratekeeper") _class = RatekeeperClass;
+		else if (s=="consistency_scan") _class = ConsistencyScanClass;
 		else if (s=="blob_manager") _class = BlobManagerClass;
 		else if (s=="blob_worker") _class = BlobWorkerClass;
 		else if (s=="storage_cache") _class = StorageCacheClass;
@@ -140,6 +143,7 @@ public:
 		else if (classStr=="data_distributor") _class = DataDistributorClass;
 		else if (classStr=="coordinator") _class = CoordinatorClass;
 		else if (classStr=="ratekeeper") _class = RatekeeperClass;
+		else if (classStr=="consistency_scan") _class = ConsistencyScanClass;
 		else if (classStr=="blob_manager") _class = BlobManagerClass;
 		else if (classStr=="blob_worker") _class = BlobWorkerClass;
 		else if (classStr=="storage_cache") _class = StorageCacheClass;
@@ -180,6 +184,7 @@ public:
 			case DataDistributorClass: return "data_distributor";
 			case CoordinatorClass: return "coordinator";
 			case RatekeeperClass: return "ratekeeper";
+			case ConsistencyScanClass: return "consistency_scan";
 			case BlobManagerClass: return "blob_manager";
 			case BlobWorkerClass: return "blob_worker";
 			case StorageCacheClass: return "storage_cache";
diff --git a/fdbrpc/include/fdbrpc/MultiInterface.h b/fdbrpc/include/fdbrpc/MultiInterface.h
index 4f15dbf087..85fa195206 100644
--- a/fdbrpc/include/fdbrpc/MultiInterface.h
+++ b/fdbrpc/include/fdbrpc/MultiInterface.h
@@ -226,6 +226,7 @@ public:
 	}
 
 	T const& getInterface(int index) { return alternatives[index]->interf; }
+	LBDistance::Type getDistance(int index) const { return (LBDistance::Type)alternatives[index]->distance; }
 	UID getId(int index) const { return alternatives[index]->interf.id(); }
 	bool hasInterface(UID id) const {
 		for (const auto& ref : alternatives) {
diff --git a/fdbrpc/include/fdbrpc/RangeMap.h b/fdbrpc/include/fdbrpc/RangeMap.h
index 09f5163b74..8280c0766c 100644
--- a/fdbrpc/include/fdbrpc/RangeMap.h
+++ b/fdbrpc/include/fdbrpc/RangeMap.h
@@ -139,7 +139,8 @@ public:
 		pair_type endPair(endKey, Val());
 		map.insert(endPair, true, mf(endPair));
 	}
-	Val const& operator[](const Key& k) { return rangeContaining(k).value(); }
+	Val const& operator[](const Key& k) const { return rangeContaining(k).value(); }
+	Val& operator[](const Key& k) { return rangeContaining(k).value(); }
 
 	Ranges ranges() { return Ranges(iterator(map.begin()), iterator(map.lastItem())); }
 	ConstRanges ranges() const { return ConstRanges(const_iterator(map.begin()), const_iterator(map.lastItem())); }
@@ -216,6 +217,19 @@ public:
 
 	Future<Void> clearAsync() { return map.clearAsync(); }
 
+	Metric getMetric(const_iterator x) const { return map.getMetric(x); }
+	Metric getMetric(iterator x) const { return getMetric(const_iterator{ x }); }
+
+	Metric sumTo(const_iterator to) const { return map.sumTo(to); }
+	Metric sumTo(iterator to) const { return sumTo(const_iterator{ to }); }
+
+	Metric sumRange(const_iterator begin, const_iterator end) const { return map.sumRange(begin, end); }
+	Metric sumRange(iterator begin, iterator end) const { return map.sumRange(begin, end); }
+	template <class KeyCompatible>
+	Metric sumRange(const KeyCompatible& begin, const KeyCompatible& end) const {
+		return map.sumRange(begin, end);
+	}
+
 protected:
 	Map<Key, Val, pair_type, Metric> map;
 	const MetricFunc mf;
diff --git a/fdbrpc/include/fdbrpc/ReplicationPolicy.h b/fdbrpc/include/fdbrpc/ReplicationPolicy.h
index acb0550a68..20e1638c81 100644
--- a/fdbrpc/include/fdbrpc/ReplicationPolicy.h
+++ b/fdbrpc/include/fdbrpc/ReplicationPolicy.h
@@ -254,19 +254,19 @@ void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy) {
 		StringRef name;
 		serializer(ar, name);
 
-		if (name == LiteralStringRef("One")) {
+		if (name == "One"_sr) {
 			PolicyOne* pointer = new PolicyOne();
 			pointer->serialize(ar);
 			policy = Reference<IReplicationPolicy>(pointer);
-		} else if (name == LiteralStringRef("Across")) {
+		} else if (name == "Across"_sr) {
 			PolicyAcross* pointer = new PolicyAcross(0, "", Reference<IReplicationPolicy>());
 			pointer->serialize(ar);
 			policy = Reference<IReplicationPolicy>(pointer);
-		} else if (name == LiteralStringRef("And")) {
+		} else if (name == "And"_sr) {
 			PolicyAnd* pointer = new PolicyAnd{};
 			pointer->serialize(ar);
 			policy = Reference<IReplicationPolicy>(pointer);
-		} else if (name == LiteralStringRef("None")) {
+		} else if (name == "None"_sr) {
 			policy = Reference<IReplicationPolicy>();
 		} else {
 			TraceEvent(SevError, "SerializingInvalidPolicyType").detail("PolicyName", name);
diff --git a/fdbrpc/include/fdbrpc/SimExternalConnection.h b/fdbrpc/include/fdbrpc/SimExternalConnection.h
index 3726e6d3fa..82a585f50e 100644
--- a/fdbrpc/include/fdbrpc/SimExternalConnection.h
+++ b/fdbrpc/include/fdbrpc/SimExternalConnection.h
@@ -47,7 +47,9 @@ public:
 	int read(uint8_t* begin, uint8_t* end) override;
 	int write(SendBuffer const* buffer, int limit) override;
 	NetworkAddress getPeerAddress() const override;
+	bool hasTrustedPeer() const override;
 	UID getDebugID() const override;
+	boost::asio::ip::tcp::socket& getSocket() override { return socket; }
 	static Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string& host,
 	                                                              const std::string& service,
 	                                                              DNSCache* dnsCache);
diff --git a/fdbrpc/include/fdbrpc/TenantInfo.h b/fdbrpc/include/fdbrpc/TenantInfo.h
index 3efec71845..f05412d7d4 100644
--- a/fdbrpc/include/fdbrpc/TenantInfo.h
+++ b/fdbrpc/include/fdbrpc/TenantInfo.h
@@ -21,52 +21,62 @@
 #pragma once
 #ifndef FDBRPC_TENANTINFO_H_
 #define FDBRPC_TENANTINFO_H_
+#include "fdbrpc/TenantName.h"
+#include "fdbrpc/TokenSign.h"
+#include "fdbrpc/TokenCache.h"
+#include "fdbrpc/FlowTransport.h"
 #include "flow/Arena.h"
-#include "fdbrpc/fdbrpc.h"
-#include <set>
+#include "flow/Knobs.h"
 
-struct TenantInfoRef {
-	TenantInfoRef() {}
-	TenantInfoRef(Arena& p, StringRef toCopy) : tenantName(StringRef(p, toCopy)) {}
-	TenantInfoRef(Arena& p, TenantInfoRef toCopy)
-	  : tenantName(toCopy.tenantName.present() ? Optional<StringRef>(StringRef(p, toCopy.tenantName.get()))
-	                                           : Optional<StringRef>()) {}
-	// Empty tenant name means that the peer is trusted
-	Optional<StringRef> tenantName;
+struct TenantInfo {
+	static constexpr const int64_t INVALID_TENANT = -1;
 
-	bool operator<(TenantInfoRef const& other) const {
-		if (!other.tenantName.present()) {
-			return false;
-		}
-		if (!tenantName.present()) {
-			return true;
-		}
-		return tenantName.get() < other.tenantName.get();
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, tenantName);
-	}
-};
-
-struct AuthorizedTenants : ReferenceCounted<AuthorizedTenants> {
 	Arena arena;
-	std::set<TenantInfoRef> authorizedTenants;
+	Optional<TenantNameRef> name;
+	Optional<StringRef> token;
+	int64_t tenantId;
+	// this field is not serialized and instead set by FlowTransport during
+	// deserialization. This field indicates whether the client is trusted.
+	// Untrusted clients are generally expected to set a TenantName
 	bool trusted = false;
+	// Is set during deserialization. It will be set to true if the tenant
+	// name is set and the client is authorized to use this tenant.
+	bool tenantAuthorized = false;
+
+	// Helper function for most endpoints that read/write data. This returns true iff
+	// the client is either a) a trusted peer or b) is accessing keyspace belonging to a tenant,
+	// for which it has a valid authorization token.
+	// NOTE: In a cluster where TenantMode is OPTIONAL or DISABLED, tenant name may be unset.
+	//       In such case, the request containing such TenantInfo is valid iff the requesting peer is trusted.
+	bool isAuthorized() const { return trusted || tenantAuthorized; }
+
+	TenantInfo() : tenantId(INVALID_TENANT) {}
+	TenantInfo(Optional<TenantName> const& tenantName, Optional<Standalone<StringRef>> const& token, int64_t tenantId)
+	  : tenantId(tenantId) {
+		if (tenantName.present()) {
+			arena.dependsOn(tenantName.get().arena());
+			name = tenantName.get();
+		}
+		if (token.present()) {
+			arena.dependsOn(token.get().arena());
+			this->token = token.get();
+		}
+	}
 };
 
-// TODO: receive and validate token instead
-struct AuthorizationRequest {
-	constexpr static FileIdentifier file_identifier = 11499331;
-
-	Arena arena;
-	VectorRef<TenantInfoRef> tenants;
-	ReplyPromise<Void> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, tenants, reply, arena);
+template <>
+struct serializable_traits<TenantInfo> : std::true_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, TenantInfo& v) {
+		serializer(ar, v.name, v.tenantId, v.token, v.arena);
+		if constexpr (Archiver::isDeserializing) {
+			bool tenantAuthorized = FLOW_KNOBS->ALLOW_TOKENLESS_TENANT_ACCESS;
+			if (!tenantAuthorized && v.name.present() && v.token.present()) {
+				tenantAuthorized = TokenCache::instance().validate(v.name.get(), v.token.get());
+			}
+			v.trusted = FlowTransport::transport().currentDeliveryPeerIsTrusted();
+			v.tenantAuthorized = tenantAuthorized;
+		}
 	}
 };
 
diff --git a/fdbrpc/include/fdbrpc/TenantName.h b/fdbrpc/include/fdbrpc/TenantName.h
new file mode 100644
index 0000000000..c71b07edb2
--- /dev/null
+++ b/fdbrpc/include/fdbrpc/TenantName.h
@@ -0,0 +1,27 @@
+/*
+ * TenantName.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#ifndef FDBRPC_TENANTNAME_H
+#define FDBRPC_TENANTNAME_H
+#include "flow/Arena.h"
+typedef StringRef TenantNameRef;
+typedef Standalone<TenantNameRef> TenantName;
+#endif // FDBRPC_TENANTNAME_H
diff --git a/fdbrpc/include/fdbrpc/TokenCache.h b/fdbrpc/include/fdbrpc/TokenCache.h
new file mode 100644
index 0000000000..e7e15d53e3
--- /dev/null
+++ b/fdbrpc/include/fdbrpc/TokenCache.h
@@ -0,0 +1,37 @@
+/*
+ * TokenCache.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TOKENCACHE_H_
+#define TOKENCACHE_H_
+#include "fdbrpc/TenantName.h"
+#include "flow/Arena.h"
+
+class TokenCache : NonCopyable {
+	struct TokenCacheImpl* impl;
+	TokenCache();
+
+public:
+	~TokenCache();
+	static void createInstance();
+	static TokenCache& instance();
+	bool validate(TenantNameRef tenant, StringRef token);
+};
+
+#endif // TOKENCACHE_H_
diff --git a/fdbrpc/include/fdbrpc/TokenSign.h b/fdbrpc/include/fdbrpc/TokenSign.h
index 10abbfd990..13217f16cd 100644
--- a/fdbrpc/include/fdbrpc/TokenSign.h
+++ b/fdbrpc/include/fdbrpc/TokenSign.h
@@ -26,6 +26,7 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/FileIdentifier.h"
+#include "fdbrpc/TenantInfo.h"
 #include "flow/PKey.h"
 
 namespace authz {
@@ -63,6 +64,8 @@ struct SignedTokenRef {
 	void serialize(Ar& ar) {
 		serializer(ar, token, keyName, signature);
 	}
+
+	int expectedSize() const { return token.size() + keyName.size() + signature.size(); }
 };
 
 SignedTokenRef signToken(Arena& arena, TokenRef token, StringRef keyName, PrivateKey privateKey);
@@ -82,6 +85,7 @@ namespace authz::jwt {
 struct TokenRef {
 	// header part ("typ": "JWT" implicitly enforced)
 	Algorithm algorithm; // alg
+	StringRef keyId; // kid
 	// payload part
 	Optional<StringRef> issuer; // iss
 	Optional<StringRef> subject; // sub
@@ -89,11 +93,13 @@ struct TokenRef {
 	Optional<uint64_t> issuedAtUnixTime; // iat
 	Optional<uint64_t> expiresAtUnixTime; // exp
 	Optional<uint64_t> notBeforeUnixTime; // nbf
-	Optional<StringRef> keyId; // kid
 	Optional<StringRef> tokenId; // jti
 	Optional<VectorRef<StringRef>> tenants; // tenants
 	// signature part
 	StringRef signature;
+
+	// print each non-signature field in non-JSON, human-readable format e.g. for trace
+	StringRef toStringRef(Arena& arena);
 };
 
 // Make plain JSON token string with fields (except signature) from passed spec
@@ -107,7 +113,7 @@ StringRef signToken(Arena& arena, TokenRef tokenSpec, PrivateKey privateKey);
 
 // Parse passed b64url-encoded header part and materialize its contents into tokenOut,
 // using memory allocated from arena
-bool parseHeaderPart(TokenRef& tokenOut, StringRef b64urlHeaderIn);
+bool parseHeaderPart(Arena& arena, TokenRef& tokenOut, StringRef b64urlHeaderIn);
 
 // Parse passed b64url-encoded payload part and materialize its contents into tokenOut,
 // using memory allocated from arena
@@ -117,6 +123,9 @@ bool parsePayloadPart(Arena& arena, TokenRef& tokenOut, StringRef b64urlPayloadI
 // using memory allocated from arena
 bool parseSignaturePart(Arena& arena, TokenRef& tokenOut, StringRef b64urlSignatureIn);
 
+// Returns the base64 encoded signature of the token
+StringRef signaturePart(StringRef token);
+
 // Parse passed token string and materialize its contents into tokenOut,
 // using memory allocated from arena
 // Return whether the signed token string is well-formed
diff --git a/fdbrpc/include/fdbrpc/WellKnownEndpoints.h b/fdbrpc/include/fdbrpc/WellKnownEndpoints.h
index e42d13e443..b5d4beb17b 100644
--- a/fdbrpc/include/fdbrpc/WellKnownEndpoints.h
+++ b/fdbrpc/include/fdbrpc/WellKnownEndpoints.h
@@ -28,27 +28,28 @@
  * All well-known endpoints of FDB must be listed here to guarantee their uniqueness
  */
 enum WellKnownEndpoints {
-	WLTOKEN_CLIENTLEADERREG_GETLEADER = WLTOKEN_FIRST_AVAILABLE, // 4
-	WLTOKEN_CLIENTLEADERREG_OPENDATABASE, // 5
-	WLTOKEN_LEADERELECTIONREG_CANDIDACY, // 6
-	WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT, // 7
-	WLTOKEN_LEADERELECTIONREG_LEADERHEARTBEAT, // 8
-	WLTOKEN_LEADERELECTIONREG_FORWARD, // 9
+	WLTOKEN_CLIENTLEADERREG_GETLEADER = WLTOKEN_FIRST_AVAILABLE, // 3
+	WLTOKEN_CLIENTLEADERREG_OPENDATABASE, // 4
+	WLTOKEN_LEADERELECTIONREG_CANDIDACY, // 5
+	WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT, // 6
+	WLTOKEN_LEADERELECTIONREG_LEADERHEARTBEAT, // 7
+	WLTOKEN_LEADERELECTIONREG_FORWARD, // 8
+	WLTOKEN_GENERATIONREG_READ, // 9
 	WLTOKEN_PROTOCOL_INFO, // 10 : the value of this endpoint should be stable and not change.
-	WLTOKEN_GENERATIONREG_READ, // 11
-	WLTOKEN_GENERATIONREG_WRITE, // 12
-	WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE, // 13
-	WLTOKEN_CONFIGTXN_GETGENERATION, // 14
-	WLTOKEN_CONFIGTXN_GET, // 15
-	WLTOKEN_CONFIGTXN_GETCLASSES, // 16
-	WLTOKEN_CONFIGTXN_GETKNOBS, // 17
-	WLTOKEN_CONFIGTXN_COMMIT, // 18
-	WLTOKEN_CONFIGFOLLOWER_GETSNAPSHOTANDCHANGES, // 19
-	WLTOKEN_CONFIGFOLLOWER_GETCHANGES, // 20
-	WLTOKEN_CONFIGFOLLOWER_COMPACT, // 21
-	WLTOKEN_CONFIGFOLLOWER_ROLLFORWARD, // 22
-	WLTOKEN_CONFIGFOLLOWER_GETCOMMITTEDVERSION, // 23
-	WLTOKEN_PROCESS, // 24
+	WLTOKEN_GENERATIONREG_WRITE, // 11
+	WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE, // 12
+	WLTOKEN_CONFIGTXN_GETGENERATION, // 13
+	WLTOKEN_CONFIGTXN_GET, // 14
+	WLTOKEN_CONFIGTXN_GETCLASSES, // 15
+	WLTOKEN_CONFIGTXN_GETKNOBS, // 16
+	WLTOKEN_CONFIGTXN_COMMIT, // 17
+	WLTOKEN_CONFIGFOLLOWER_GETSNAPSHOTANDCHANGES, // 18
+	WLTOKEN_CONFIGFOLLOWER_GETCHANGES, // 19
+	WLTOKEN_CONFIGFOLLOWER_COMPACT, // 20
+	WLTOKEN_CONFIGFOLLOWER_ROLLFORWARD, // 21
+	WLTOKEN_CONFIGFOLLOWER_GETCOMMITTEDVERSION, // 22
+	WLTOKEN_PROCESS, // 23
+	WLTOKEN_CONFIGFOLLOWER_LOCK, // 24
 	WLTOKEN_RESERVED_COUNT // 25
 };
 
diff --git a/fdbrpc/include/fdbrpc/fdbrpc.h b/fdbrpc/include/fdbrpc/fdbrpc.h
index bec32b6696..63cf55c374 100644
--- a/fdbrpc/include/fdbrpc/fdbrpc.h
+++ b/fdbrpc/include/fdbrpc/fdbrpc.h
@@ -648,10 +648,29 @@ struct serializable_traits<ReplyPromiseStream<T>> : std::true_type {
 	}
 };
 
+template <class T, class = int>
+struct HasReply_t : std::false_type {};
+
+template <class T>
+struct HasReply_t<T, decltype((void)T::reply, 0)> : std::true_type {};
+
+template <class T>
+constexpr bool HasReply = HasReply_t<T>::value;
+
+template <class T, class = int>
+struct HasVerify_t : std::false_type {};
+
+template <class T>
+struct HasVerify_t<T, decltype(void(std::declval<T>().verify()), 0)> : std::true_type {};
+
+template <class T>
+constexpr bool HasVerify = HasVerify_t<T>::value;
+
 template <class T, bool IsPublic>
 struct NetNotifiedQueue final : NotifiedQueue<T>, FlowReceiver, FastAllocated<NetNotifiedQueue<T, IsPublic>> {
 	using FastAllocated<NetNotifiedQueue<T, IsPublic>>::operator new;
 	using FastAllocated<NetNotifiedQueue<T, IsPublic>>::operator delete;
+	static_assert(!IsPublic || HasVerify<T>, "Public request stream objects need to implement bool T::verify()");
 
 	NetNotifiedQueue(int futures, int promises) : NotifiedQueue<T>(futures, promises) {}
 	NetNotifiedQueue(int futures, int promises, const Endpoint& remoteEndpoint)
@@ -662,7 +681,17 @@ struct NetNotifiedQueue final : NotifiedQueue<T>, FlowReceiver, FastAllocated<Ne
 		this->addPromiseRef();
 		T message;
 		reader.deserialize(message);
-		this->send(std::move(message));
+		if constexpr (IsPublic) {
+			if (!message.verify()) {
+				if constexpr (HasReply<T>) {
+					message.reply.sendError(permission_denied());
+				}
+			} else {
+				this->send(std::move(message));
+			}
+		} else {
+			this->send(std::move(message));
+		}
 		this->delPromiseRef();
 	}
 	bool isStream() const override { return true; }
@@ -782,8 +811,13 @@ public:
 			Future<Void> disc =
 			    makeDependent<T>(IFailureMonitor::failureMonitor()).onDisconnectOrFailure(getEndpoint());
 			auto& p = getReplyPromiseStream(value);
-			if (disc.isReady()) {
-				p.sendError(request_maybe_delivered());
+			// FIXME: buggify only in simulation/not during speed up simulation?
+			if (disc.isReady() || BUGGIFY_WITH_PROB(0.01)) {
+				if (disc.isReady() && IFailureMonitor::failureMonitor().knownUnauthorized(getEndpoint())) {
+					p.sendError(unauthorized_attempt());
+				} else {
+					p.sendError(request_maybe_delivered());
+				}
 			} else {
 				Reference<Peer> peer =
 				    FlowTransport::transport().sendUnreliable(SerializeSource<T>(value), getEndpoint(), true);
diff --git a/fdbrpc/include/fdbrpc/genericactors.actor.h b/fdbrpc/include/fdbrpc/genericactors.actor.h
index de253069fc..f36a6271fb 100644
--- a/fdbrpc/include/fdbrpc/genericactors.actor.h
+++ b/fdbrpc/include/fdbrpc/genericactors.actor.h
@@ -47,7 +47,7 @@ Future<REPLY_TYPE(Req)> retryBrokenPromise(RequestStream<Req, P> to, Req request
 				throw;
 			resetReply(request);
 			wait(delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-			TEST(true); // retryBrokenPromise
+			CODE_PROBE(true, "retryBrokenPromise");
 		}
 	}
 }
@@ -67,7 +67,7 @@ Future<REPLY_TYPE(Req)> retryBrokenPromise(RequestStream<Req, P> to, Req request
 				throw;
 			resetReply(request);
 			wait(delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY, taskID));
-			TEST(true); // retryBrokenPromise with taskID
+			CODE_PROBE(true, "retryBrokenPromise with taskID");
 		}
 	}
 }
diff --git a/fdbrpc/include/fdbrpc/simulator.h b/fdbrpc/include/fdbrpc/simulator.h
index 67155490cd..19b7d94dc3 100644
--- a/fdbrpc/include/fdbrpc/simulator.h
+++ b/fdbrpc/include/fdbrpc/simulator.h
@@ -20,19 +20,23 @@
 
 #ifndef FLOW_SIMULATOR_H
 #define FLOW_SIMULATOR_H
-#include "flow/ProtocolVersion.h"
+#pragma once
 #include <algorithm>
 #include <string>
-#pragma once
+#include <random>
+#include <limits>
 
 #include "flow/flow.h"
 #include "flow/Histogram.h"
+#include "flow/ProtocolVersion.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/Locality.h"
 #include "flow/IAsyncFile.h"
 #include "flow/TDMetric.actor.h"
-#include <random>
+#include "fdbrpc/FailureMonitor.h"
+#include "fdbrpc/Locality.h"
 #include "fdbrpc/ReplicationPolicy.h"
+#include "fdbrpc/TokenSign.h"
 
 enum ClogMode { ClogDefault, ClogAll, ClogSend, ClogReceive };
 
@@ -59,6 +63,25 @@ public:
 	enum TSSMode { Disabled, EnabledNormal, EnabledAddDelay, EnabledDropMutations };
 
 	enum class BackupAgentType { NoBackupAgents, WaitForType, BackupToFile, BackupToDB };
+	enum class ExtraDatabaseMode { Disabled, LocalOrSingle, Single, Local, Multiple };
+
+	static ExtraDatabaseMode stringToExtraDatabaseMode(std::string databaseMode) {
+		if (databaseMode == "Disabled") {
+			return ExtraDatabaseMode::Disabled;
+		} else if (databaseMode == "LocalOrSingle") {
+			return ExtraDatabaseMode::LocalOrSingle;
+		} else if (databaseMode == "Single") {
+			return ExtraDatabaseMode::Single;
+		} else if (databaseMode == "Local") {
+			return ExtraDatabaseMode::Local;
+		} else if (databaseMode == "Multiple") {
+			return ExtraDatabaseMode::Multiple;
+		} else {
+			TraceEvent(SevError, "UnknownExtraDatabaseMode").detail("DatabaseMode", databaseMode);
+			ASSERT(false);
+			throw internal_error();
+		}
+	};
 
 	// Subclasses may subclass ProcessInfo as well
 	struct MachineInfo;
@@ -163,6 +186,8 @@ public:
 				return false;
 			case ProcessClass::RatekeeperClass:
 				return false;
+			case ProcessClass::ConsistencyScanClass:
+				return false;
 			case ProcessClass::BlobManagerClass:
 				return false;
 			case ProcessClass::StorageCacheClass:
@@ -401,7 +426,7 @@ public:
 		allSwapsDisabled = false;
 	}
 	bool canSwapToMachine(Optional<Standalone<StringRef>> zoneId) const {
-		return swapsDisabled.count(zoneId) == 0 && !allSwapsDisabled && !extraDB;
+		return swapsDisabled.count(zoneId) == 0 && !allSwapsDisabled && extraDatabases.empty();
 	}
 	void enableSwapsToAll() {
 		swapsDisabled.clear();
@@ -426,9 +451,15 @@ public:
 	int physicalDatacenters;
 	int processesPerMachine;
 	int listenersPerProcess;
+
+	// We won't kill machines in this set, but we might reboot
+	// them.  This is a conservative mechanism to prevent the
+	// simulator from killing off important processes and rendering
+	// the cluster unrecoverable, e.g. a quorum of coordinators.
 	std::set<NetworkAddress> protectedAddresses;
+
 	std::map<NetworkAddress, ProcessInfo*> currentlyRebootingProcesses;
-	class ClusterConnectionString* extraDB = nullptr;
+	std::vector<std::string> extraDatabases;
 	Reference<IReplicationPolicy> storagePolicy;
 	Reference<IReplicationPolicy> tLogPolicy;
 	int32_t tLogWriteAntiQuorum;
@@ -451,6 +482,7 @@ public:
 	TSSMode tssMode;
 	std::map<NetworkAddress, bool> corruptWorkerMap;
 	ConfigDBType configDBType;
+	bool blobGranulesEnabled;
 
 	// Used by workloads that perform reconfigurations
 	int testerCount;
@@ -469,6 +501,10 @@ public:
 	bool setDiffProtocol; // true if a process with a different protocol version has been started
 
 	bool allowStorageMigrationTypeChange = false;
+	double injectTargetedSSRestartTime = std::numeric_limits<double>::max();
+	double injectSSDelayTime = std::numeric_limits<double>::max();
+
+	std::unordered_map<Standalone<StringRef>, PrivateKey> authKeys;
 
 	flowGlobalType global(int id) const final { return getCurrentProcess()->global(id); };
 	void setGlobal(size_t id, flowGlobalType v) final { getCurrentProcess()->setGlobal(id, v); };
@@ -507,9 +543,7 @@ private:
 	bool allSwapsDisabled;
 };
 
-// Quickly make existing code work that expects g_simulator to be of class type (not a pointer)
-extern ISimulator* g_pSimulator;
-#define g_simulator (*g_pSimulator)
+extern ISimulator* g_simulator;
 
 void startNewSimulator(bool printSimTime);
 
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 98bc972b57..b000d53042 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -22,6 +22,7 @@
 #include <memory>
 #include <string>
 
+#include "flow/MkCert.h"
 #include "fmt/format.h"
 #include "fdbrpc/simulator.h"
 #include "flow/Arena.h"
@@ -54,31 +55,40 @@
 #include "flow/FaultInjection.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-ISimulator* g_pSimulator = nullptr;
+ISimulator* g_simulator = nullptr;
 thread_local ISimulator::ProcessInfo* ISimulator::currentProcess = nullptr;
 
 ISimulator::ISimulator()
   : desiredCoordinators(1), physicalDatacenters(1), processesPerMachine(0), listenersPerProcess(1), usableRegions(1),
-    allowLogSetKills(true), tssMode(TSSMode::Disabled), isStopped(false), lastConnectionFailure(0),
-    connectionFailuresDisableDuration(0), speedUpSimulation(false), backupAgents(BackupAgentType::WaitForType),
-    drAgents(BackupAgentType::WaitForType), allSwapsDisabled(false) {}
+    allowLogSetKills(true), tssMode(TSSMode::Disabled), configDBType(ConfigDBType::DISABLED), isStopped(false),
+    lastConnectionFailure(0), connectionFailuresDisableDuration(0), speedUpSimulation(false),
+    backupAgents(BackupAgentType::WaitForType), drAgents(BackupAgentType::WaitForType), allSwapsDisabled(false) {}
 ISimulator::~ISimulator() = default;
 
 bool simulator_should_inject_fault(const char* context, const char* file, int line, int error_code) {
 	if (!g_network->isSimulated() || !faultInjectionActivated)
 		return false;
 
-	auto p = g_simulator.getCurrentProcess();
+	auto p = g_simulator->getCurrentProcess();
 
 	if (p->fault_injection_p2 && deterministicRandom()->random01() < p->fault_injection_p2 &&
-	    !g_simulator.speedUpSimulation) {
+	    !g_simulator->speedUpSimulation) {
 		uint32_t h1 = line + (p->fault_injection_r >> 32);
 
 		if (h1 < p->fault_injection_p1 * std::numeric_limits<uint32_t>::max()) {
-			TEST(true); // A fault was injected
-			TEST(error_code == error_code_io_timeout); // An io timeout was injected
-			TEST(error_code == error_code_io_error); // An io error was injected
-			TEST(error_code == error_code_platform_error); // A platform error was injected.
+			CODE_PROBE(true, "A fault was injected", probe::assert::simOnly, probe::context::sim2);
+			CODE_PROBE(error_code == error_code_io_timeout,
+			           "An io timeout was injected",
+			           probe::assert::simOnly,
+			           probe::context::sim2);
+			CODE_PROBE(error_code == error_code_io_error,
+			           "An io error was injected",
+			           probe::assert::simOnly,
+			           probe::context::sim2);
+			CODE_PROBE(error_code == error_code_platform_error,
+			           "A platform error was injected.",
+			           probe::assert::simOnly,
+			           probe::context::sim2);
 			TraceEvent(SevWarn, "FaultInjected")
 			    .detail("Context", context)
 			    .detail("File", file)
@@ -144,13 +154,13 @@ struct SimClogging {
 
 		double tnow = now();
 		double t = tnow + (stableConnection ? 0.1 : 1.0) * halfLatency();
-		if (!g_simulator.speedUpSimulation && !stableConnection)
+		if (!g_simulator->speedUpSimulation && !stableConnection)
 			t += clogPairLatency[pair];
 
-		if (!g_simulator.speedUpSimulation && !stableConnection && clogPairUntil.count(pair))
+		if (!g_simulator->speedUpSimulation && !stableConnection && clogPairUntil.count(pair))
 			t = std::max(t, clogPairUntil[pair]);
 
-		if (!g_simulator.speedUpSimulation && !stableConnection && clogRecvUntil.count(to.ip))
+		if (!g_simulator->speedUpSimulation && !stableConnection && clogRecvUntil.count(to.ip))
 			t = std::max(t, clogRecvUntil[to.ip]);
 
 		return t - tnow;
@@ -182,7 +192,7 @@ private:
 	double halfLatency() const {
 		double a = deterministicRandom()->random01();
 		const double pFast = 0.999;
-		if (a <= pFast || g_simulator.speedUpSimulation) {
+		if (a <= pFast || g_simulator->speedUpSimulation) {
 			a = a / pFast;
 			return 0.5 * (FLOW_KNOBS->MIN_NETWORK_LATENCY * (1 - a) +
 			              FLOW_KNOBS->FAST_NETWORK_LATENCY / pFast * a); // 0.5ms average
@@ -198,7 +208,7 @@ SimClogging g_clogging;
 
 struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 	Sim2Conn(ISimulator::ProcessInfo* process)
-	  : opened(false), closedByCaller(false), stableConnection(false), process(process),
+	  : opened(false), closedByCaller(false), stableConnection(false), trustedPeer(true), process(process),
 	    dbgid(deterministicRandom()->randomUniqueID()), stopReceive(Never()) {
 		pipes = sender(this) && receiver(this);
 	}
@@ -249,6 +259,8 @@ struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 
 	bool isPeerGone() const { return !peer || peerProcess->failed; }
 
+	bool hasTrustedPeer() const override { return trustedPeer; }
+
 	bool isStableConnection() const override { return stableConnection; }
 
 	void peerClosed() {
@@ -315,7 +327,9 @@ struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 	NetworkAddress getPeerAddress() const override { return peerEndpoint; }
 	UID getDebugID() const override { return dbgid; }
 
-	bool opened, closedByCaller, stableConnection;
+	boost::asio::ip::tcp::socket& getSocket() override { throw operation_failed(); }
+
+	bool opened, closedByCaller, stableConnection, trustedPeer;
 
 private:
 	ISimulator::ProcessInfo *process, *peerProcess;
@@ -349,7 +363,7 @@ private:
 	ACTOR static Future<Void> sender(Sim2Conn* self) {
 		loop {
 			wait(self->writtenBytes.onChange()); // takes place on peer!
-			ASSERT(g_simulator.getCurrentProcess() == self->peerProcess);
+			ASSERT(g_simulator->getCurrentProcess() == self->peerProcess);
 			wait(delay(.002 * deterministicRandom()->random01()));
 			self->sentBytes.set(self->writtenBytes.get()); // or possibly just some sometimes...
 		}
@@ -357,41 +371,41 @@ private:
 	ACTOR static Future<Void> receiver(Sim2Conn* self) {
 		loop {
 			if (self->sentBytes.get() != self->receivedBytes.get())
-				wait(g_simulator.onProcess(self->peerProcess));
+				wait(g_simulator->onProcess(self->peerProcess));
 			while (self->sentBytes.get() == self->receivedBytes.get())
 				wait(self->sentBytes.onChange());
-			ASSERT(g_simulator.getCurrentProcess() == self->peerProcess);
+			ASSERT(g_simulator->getCurrentProcess() == self->peerProcess);
 			state int64_t pos =
 			    deterministicRandom()->random01() < .5
 			        ? self->sentBytes.get()
 			        : deterministicRandom()->randomInt64(self->receivedBytes.get(), self->sentBytes.get() + 1);
 			wait(delay(g_clogging.getSendDelay(
 			    self->process->address, self->peerProcess->address, self->isStableConnection())));
-			wait(g_simulator.onProcess(self->process));
-			ASSERT(g_simulator.getCurrentProcess() == self->process);
+			wait(g_simulator->onProcess(self->process));
+			ASSERT(g_simulator->getCurrentProcess() == self->process);
 			wait(delay(g_clogging.getRecvDelay(
 			    self->process->address, self->peerProcess->address, self->isStableConnection())));
-			ASSERT(g_simulator.getCurrentProcess() == self->process);
+			ASSERT(g_simulator->getCurrentProcess() == self->process);
 			if (self->stopReceive.isReady()) {
 				wait(Future<Void>(Never()));
 			}
 			self->receivedBytes.set(pos);
 			wait(Future<Void>(Void())); // Prior notification can delete self and cancel this actor
-			ASSERT(g_simulator.getCurrentProcess() == self->process);
+			ASSERT(g_simulator->getCurrentProcess() == self->process);
 		}
 	}
 	ACTOR static Future<Void> whenReadable(Sim2Conn* self) {
 		try {
 			loop {
 				if (self->readBytes.get() != self->receivedBytes.get()) {
-					ASSERT(g_simulator.getCurrentProcess() == self->process);
+					ASSERT(g_simulator->getCurrentProcess() == self->process);
 					return Void();
 				}
 				wait(self->receivedBytes.onChange());
 				self->rollRandomClose();
 			}
 		} catch (Error& e) {
-			ASSERT(g_simulator.getCurrentProcess() == self->process);
+			ASSERT(g_simulator->getCurrentProcess() == self->process);
 			throw;
 		}
 	}
@@ -401,20 +415,20 @@ private:
 				if (!self->peer)
 					return Void();
 				if (self->peer->availableSendBufferForPeer() > 0) {
-					ASSERT(g_simulator.getCurrentProcess() == self->process);
+					ASSERT(g_simulator->getCurrentProcess() == self->process);
 					return Void();
 				}
 				try {
 					wait(self->peer->receivedBytes.onChange());
-					ASSERT(g_simulator.getCurrentProcess() == self->peerProcess);
+					ASSERT(g_simulator->getCurrentProcess() == self->peerProcess);
 				} catch (Error& e) {
 					if (e.code() != error_code_broken_promise)
 						throw;
 				}
-				wait(g_simulator.onProcess(self->process));
+				wait(g_simulator->onProcess(self->process));
 			}
 		} catch (Error& e) {
-			ASSERT(g_simulator.getCurrentProcess() == self->process);
+			ASSERT(g_simulator->getCurrentProcess() == self->process);
 			throw;
 		}
 	}
@@ -422,11 +436,11 @@ private:
 	void rollRandomClose() {
 		// make sure connections between parenta and their childs are not closed
 		if (!stableConnection &&
-		    now() - g_simulator.lastConnectionFailure > g_simulator.connectionFailuresDisableDuration &&
+		    now() - g_simulator->lastConnectionFailure > g_simulator->connectionFailuresDisableDuration &&
 		    deterministicRandom()->random01() < .00001) {
-			g_simulator.lastConnectionFailure = now();
+			g_simulator->lastConnectionFailure = now();
 			double a = deterministicRandom()->random01(), b = deterministicRandom()->random01();
-			TEST(true); // Simulated connection failure
+			CODE_PROBE(true, "Simulated connection failure", probe::context::sim2, probe::assert::simOnly);
 			TraceEvent("ConnectionFailure", dbgid)
 			    .detail("MyAddr", process->address)
 			    .detail("PeerAddr", peerProcess->address)
@@ -446,7 +460,7 @@ private:
 	}
 
 	ACTOR static Future<Void> trackLeakedConnection(Sim2Conn* self) {
-		wait(g_simulator.onProcess(self->process));
+		wait(g_simulator->onProcess(self->process));
 		if (self->process->address.isPublic()) {
 			wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_IDLE_TIMEOUT * FLOW_KNOBS->CONNECTION_MONITOR_IDLE_TIMEOUT * 1.5 +
 			           FLOW_KNOBS->CONNECTION_MONITOR_LOOP_TIME * 2.1 + FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT));
@@ -503,25 +517,25 @@ public:
 	    int mode,
 	    Reference<DiskParameters> diskParameters = makeReference<DiskParameters>(25000, 150000000),
 	    bool delayOnWrite = true) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
 		state TaskPriority currentTaskID = g_network->getCurrentTask();
 
-		if (++openCount >= 3000) {
+		if (++openCount >= 6000) {
 			TraceEvent(SevError, "TooManyFiles").log();
 			ASSERT(false);
 		}
 
-		if (openCount == 2000) {
+		if (openCount == 4000) {
 			TraceEvent(SevWarnAlways, "DisableConnectionFailures_TooManyFiles").log();
-			g_simulator.speedUpSimulation = true;
-			g_simulator.connectionFailuresDisableDuration = 1e6;
+			g_simulator->speedUpSimulation = true;
+			g_simulator->connectionFailuresDisableDuration = 1e6;
 		}
 
 		// Filesystems on average these days seem to start to have limits of around 255 characters for a
 		// filename.  We add ".part" below, so we need to stay under 250.
 		ASSERT(basename(filename).size() < 250);
 
-		wait(g_simulator.onMachine(currentProcess));
+		wait(g_simulator->onMachine(currentProcess));
 		try {
 			wait(delay(FLOW_KNOBS->MIN_OPEN_TIME +
 			           deterministicRandom()->random01() * (FLOW_KNOBS->MAX_OPEN_TIME - FLOW_KNOBS->MIN_OPEN_TIME)));
@@ -547,11 +561,11 @@ public:
 			platform::makeTemporary(open_filename.c_str());
 			SimpleFile* simpleFile = new SimpleFile(h, diskParameters, delayOnWrite, filename, open_filename, flags);
 			state Reference<IAsyncFile> file = Reference<IAsyncFile>(simpleFile);
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 			return file;
 		} catch (Error& e) {
 			state Error err = e;
-			wait(g_simulator.onProcess(currentProcess, currentTaskID));
+			wait(g_simulator->onProcess(currentProcess, currentTaskID));
 			throw err;
 		}
 	}
@@ -765,7 +779,7 @@ private:
 
 		if (self->flags & OPEN_ATOMIC_WRITE_AND_CREATE) {
 			self->flags &= ~OPEN_ATOMIC_WRITE_AND_CREATE;
-			auto& machineCache = g_simulator.getCurrentProcess()->machine->openFiles;
+			auto& machineCache = g_simulator->getCurrentProcess()->machine->openFiles;
 			std::string sourceFilename = self->filename + ".part";
 
 			if (machineCache.count(sourceFilename)) {
@@ -849,7 +863,7 @@ private:
 	PromiseStream<Reference<IConnection>> nextConnection;
 
 	ACTOR static void incoming(Reference<Sim2Listener> self, double seconds, Reference<IConnection> conn) {
-		wait(g_simulator.onProcess(self->process));
+		wait(g_simulator->onProcess(self->process));
 		wait(delay(seconds));
 		if (((Sim2Conn*)conn.getPtr())->isPeerGone() && deterministicRandom()->random01() < 0.5)
 			return;
@@ -867,7 +881,7 @@ private:
 	NetworkAddress address;
 };
 
-#define g_sim2 ((Sim2&)g_simulator)
+#define g_sim2 ((Sim2&)(*g_simulator))
 
 class Sim2 final : public ISimulator, public INetworkConnections {
 public:
@@ -939,8 +953,9 @@ public:
 	TaskPriority getCurrentTask() const override { return currentTaskID; }
 	void setCurrentTask(TaskPriority taskID) override { currentTaskID = taskID; }
 	// Sets the taskID/priority of the current task, without yielding
-	Future<Reference<IConnection>> connect(NetworkAddress toAddr, const std::string& host) override {
-		ASSERT(host.empty());
+	Future<Reference<IConnection>> connect(NetworkAddress toAddr,
+	                                       boost::asio::ip::tcp::socket* existingSocket = nullptr) override {
+		ASSERT(existingSocket == nullptr);
 		if (!addressMap.count(toAddr)) {
 			return waitForProcessAndConnect(toAddr, this);
 		}
@@ -966,7 +981,7 @@ public:
 		return onConnect(::delay(0.5 * deterministicRandom()->random01()), myc);
 	}
 
-	Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr, const std::string& host) override {
+	Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr) override {
 		return SimExternalConnection::connect(toAddr);
 	}
 
@@ -1086,7 +1101,7 @@ public:
 
 		SimThreadArgs(THREAD_FUNC_RETURN (*func)(void*), void* arg) : func(func), arg(arg) {
 			ASSERT(g_network->isSimulated());
-			currentProcess = g_simulator.getCurrentProcess();
+			currentProcess = g_simulator->getCurrentProcess();
 		}
 	};
 
@@ -1160,38 +1175,38 @@ public:
 		// This is a _rudimentary_ simulation of the untrustworthiness of non-durable deletes and the possibility of
 		// rebooting during a durable one.  It isn't perfect: for example, on real filesystems testing
 		// for the existence of a non-durably deleted file BEFORE a reboot will show that it apparently doesn't exist.
-		if (g_simulator.getCurrentProcess()->machine->openFiles.count(filename)) {
-			g_simulator.getCurrentProcess()->machine->openFiles.erase(filename);
-			g_simulator.getCurrentProcess()->machine->deletingOrClosingFiles.insert(filename);
+		if (g_simulator->getCurrentProcess()->machine->openFiles.count(filename)) {
+			g_simulator->getCurrentProcess()->machine->openFiles.erase(filename);
+			g_simulator->getCurrentProcess()->machine->deletingOrClosingFiles.insert(filename);
 		}
 		if (mustBeDurable || deterministicRandom()->random01() < 0.5) {
-			state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+			state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
 			state TaskPriority currentTaskID = g_network->getCurrentTask();
 			TraceEvent(SevDebug, "Sim2DeleteFileImpl")
 			    .detail("CurrentProcess", currentProcess->toString())
 			    .detail("Filename", filename)
 			    .detail("Durable", mustBeDurable);
-			wait(g_simulator.onMachine(currentProcess));
+			wait(g_simulator->onMachine(currentProcess));
 			try {
 				wait(::delay(0.05 * deterministicRandom()->random01()));
 				if (!currentProcess->rebooting) {
 					auto f = IAsyncFileSystem::filesystem(self->net2)->deleteFile(filename, false);
 					ASSERT(f.isReady());
 					wait(::delay(0.05 * deterministicRandom()->random01()));
-					TEST(true); // Simulated durable delete
+					CODE_PROBE(true, "Simulated durable delete", probe::context::sim2, probe::assert::simOnly);
 				}
-				wait(g_simulator.onProcess(currentProcess, currentTaskID));
+				wait(g_simulator->onProcess(currentProcess, currentTaskID));
 				return Void();
 			} catch (Error& e) {
 				state Error err = e;
-				wait(g_simulator.onProcess(currentProcess, currentTaskID));
+				wait(g_simulator->onProcess(currentProcess, currentTaskID));
 				throw err;
 			}
 		} else {
 			TraceEvent(SevDebug, "Sim2DeleteFileImplNonDurable")
 			    .detail("Filename", filename)
 			    .detail("Durable", mustBeDurable);
-			TEST(true); // Simulated non-durable delete
+			CODE_PROBE(true, "Simulated non-durable delete", probe::context::sim2, probe::assert::simOnly);
 			return Void();
 		}
 	}
@@ -1278,8 +1293,8 @@ public:
 		m->machine = &machine;
 		machine.processes.push_back(m);
 		currentlyRebootingProcesses.erase(addresses.address);
-		m->excluded = g_simulator.isExcluded(NetworkAddress(ip, port, true, false));
-		m->cleared = g_simulator.isCleared(addresses.address);
+		m->excluded = g_simulator->isExcluded(NetworkAddress(ip, port, true, false));
+		m->cleared = g_simulator->isCleared(addresses.address);
 		m->protocolVersion = protocol;
 
 		m->setGlobal(enTDMetrics, (flowGlobalType)&m->tdmetrics);
@@ -1352,6 +1367,45 @@ public:
 		return primaryTLogsDead || primaryProcessesDead.validate(storagePolicy);
 	}
 
+	// The following function will determine if a machine can be remove in case when it has a blob worker
+	bool canKillMachineWithBlobWorkers(Optional<Standalone<StringRef>> machineId, KillType kt, KillType* ktFinal) {
+		// Allow if no blob workers, or it's a reboot(without removing the machine)
+		if (!blobGranulesEnabled && kt >= RebootAndDelete) {
+			return true;
+		}
+
+		// Allow if the machine doesn't support blob worker
+		MachineInfo& currentMachine = machines[machineId];
+		bool hasBlobWorker = false;
+		for (auto processInfo : currentMachine.processes) {
+			if (processInfo->startingClass == ProcessClass::BlobWorkerClass) {
+				hasBlobWorker = true;
+				break;
+			}
+		}
+		if (!hasBlobWorker)
+			return true;
+
+		// Count # remaining support blob workers in current dc
+		auto currentDcId = currentMachine.machineProcess->locality.dcId();
+		int nLeft = 0;
+		for (auto processInfo : getAllProcesses()) {
+			if (currentDcId != processInfo->locality.dcId() || // skip other dc
+			    processInfo->startingClass != ProcessClass::BlobWorkerClass || // skip non blob workers
+			    processInfo->locality.machineId() == machineId) { // skip current machine
+				continue;
+			}
+			nLeft++; // alive blob workers after killing machineId
+		}
+
+		// Ensure there is at least 1 remaining blob workers after removing current machine
+		if (nLeft <= 1) {
+			*ktFinal = RebootAndDelete; // reboot and delete data, but keep this machine
+			return false;
+		}
+		return true;
+	}
+
 	// The following function will determine if the specified configuration of available and dead processes can allow
 	// the cluster to survive
 	bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses,
@@ -1587,10 +1641,20 @@ public:
 		killProcess_internal(p, KillInstantly);
 	}
 	void killProcess_internal(ProcessInfo* machine, KillType kt) {
-		TEST(true); // Simulated machine was killed with any kill type
-		TEST(kt == KillInstantly); // Simulated machine was killed instantly
-		TEST(kt == InjectFaults); // Simulated machine was killed with faults
-		TEST(kt == FailDisk); // Simulated machine was killed with a failed disk
+		CODE_PROBE(
+		    true, "Simulated machine was killed with any kill type", probe::context::sim2, probe::assert::simOnly);
+		CODE_PROBE(kt == KillInstantly,
+		           "Simulated machine was killed instantly",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
+		CODE_PROBE(kt == InjectFaults,
+		           "Simulated machine was killed with faults",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
+		CODE_PROBE(kt == FailDisk,
+		           "Simulated machine was killed with a failed disk",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
 
 		if (kt == KillInstantly) {
 			TraceEvent(SevWarn, "FailMachine")
@@ -1663,15 +1727,20 @@ public:
 	}
 	void killProcess(ProcessInfo* machine, KillType kt) override {
 		TraceEvent("AttemptingKillProcess").detail("ProcessInfo", machine->toString());
-		if (kt < RebootAndDelete) {
+		// Refuse to kill a protected process.
+		if (kt < RebootAndDelete && protectedAddresses.count(machine->address) == 0) {
 			killProcess_internal(machine, kt);
 		}
 	}
 	void killInterface(NetworkAddress address, KillType kt) override {
 		if (kt < RebootAndDelete) {
 			std::vector<ProcessInfo*>& processes = machines[addressMap[address]->locality.machineId()].processes;
-			for (int i = 0; i < processes.size(); i++)
-				killProcess_internal(processes[i], kt);
+			for (auto& process : processes) {
+				// Refuse to kill a protected process.
+				if (protectedAddresses.count(process->address) == 0) {
+					killProcess_internal(process, kt);
+				}
+			}
 		}
 	}
 	bool killZone(Optional<Standalone<StringRef>> zoneId, KillType kt, bool forceKill, KillType* ktFinal) override {
@@ -1715,9 +1784,10 @@ public:
 	                 KillType* ktFinal) override {
 		auto ktOrig = kt;
 
-		TEST(true); // Trying to killing a machine
-		TEST(kt == KillInstantly); // Trying to kill instantly
-		TEST(kt == InjectFaults); // Trying to kill by injecting faults
+		CODE_PROBE(true, "Trying to killing a machine", probe::context::sim2, probe::assert::simOnly);
+		CODE_PROBE(kt == KillInstantly, "Trying to kill instantly", probe::context::sim2, probe::assert::simOnly);
+		CODE_PROBE(
+		    kt == InjectFaults, "Trying to kill by injecting faults", probe::context::sim2, probe::assert::simOnly);
 
 		if (speedUpSimulation && !forceKill) {
 			TraceEvent(SevWarn, "AbortedKill")
@@ -1756,6 +1826,14 @@ public:
 		// Check if machine can be removed, if requested
 		if (!forceKill && ((kt == KillInstantly) || (kt == InjectFaults) || (kt == FailDisk) ||
 		                   (kt == RebootAndDelete) || (kt == RebootProcessAndDelete))) {
+
+			if (!canKillMachineWithBlobWorkers(machineId, kt, &kt)) {
+				TraceEvent("CanKillMachineWithBlobWorkers")
+				    .detail("MachineId", machineId)
+				    .detail("KillType", kt)
+				    .detail("OrigKillType", ktOrig);
+			}
+
 			std::vector<ProcessInfo*> processesLeft, processesDead;
 			int protectedWorker = 0, unavailable = 0, excluded = 0, cleared = 0;
 
@@ -1851,11 +1929,17 @@ public:
 			}
 		}
 
-		TEST(originalKt != kt); // Kill type was changed from requested to reboot.
+		CODE_PROBE(originalKt != kt,
+		           "Kill type was changed from requested to reboot.",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
 
 		// Check if any processes on machine are rebooting
 		if (processesOnMachine != processesPerMachine && kt >= RebootAndDelete) {
-			TEST(true); // Attempted reboot, but the target did not have all of its processes running
+			CODE_PROBE(true,
+			           "Attempted reboot, but the target did not have all of its processes running",
+			           probe::context::sim2,
+			           probe::assert::simOnly);
 			TraceEvent(SevWarn, "AbortedKill")
 			    .detail("KillType", kt)
 			    .detail("MachineId", machineId)
@@ -1870,7 +1954,10 @@ public:
 
 		// Check if any processes on machine are rebooting
 		if (processesOnMachine != processesPerMachine) {
-			TEST(true); // Attempted reboot and kill, but the target did not have all of its processes running
+			CODE_PROBE(true,
+			           "Attempted reboot and kill, but the target did not have all of its processes running",
+			           probe::context::sim2,
+			           probe::assert::simOnly);
 			TraceEvent(SevWarn, "AbortedKill")
 			    .detail("KillType", kt)
 			    .detail("MachineId", machineId)
@@ -1920,10 +2007,12 @@ public:
 			}
 		}
 
-		TEST(kt == RebootAndDelete); // Resulted in a reboot and delete
-		TEST(kt == Reboot); // Resulted in a reboot
-		TEST(kt == KillInstantly); // Resulted in an instant kill
-		TEST(kt == InjectFaults); // Resulted in a kill by injecting faults
+		CODE_PROBE(
+		    kt == RebootAndDelete, "Resulted in a reboot and delete", probe::context::sim2, probe::assert::simOnly);
+		CODE_PROBE(kt == Reboot, "Resulted in a reboot", probe::context::sim2, probe::assert::simOnly);
+		CODE_PROBE(kt == KillInstantly, "Resulted in an instant kill", probe::context::sim2, probe::assert::simOnly);
+		CODE_PROBE(
+		    kt == InjectFaults, "Resulted in a kill by injecting faults", probe::context::sim2, probe::assert::simOnly);
 
 		if (ktFinal)
 			*ktFinal = kt;
@@ -2037,13 +2126,32 @@ public:
 		    .detail("KillTypeMin", ktMin)
 		    .detail("KilledDC", kt == ktMin);
 
-		TEST(kt != ktMin); // DataCenter kill was rejected by killMachine
-		TEST((kt == ktMin) && (kt == RebootAndDelete)); // Datacenter kill Resulted in a reboot and delete
-		TEST((kt == ktMin) && (kt == Reboot)); // Datacenter kill Resulted in a reboot
-		TEST((kt == ktMin) && (kt == KillInstantly)); // Datacenter kill Resulted in an instant kill
-		TEST((kt == ktMin) && (kt == InjectFaults)); // Datacenter kill Resulted in a kill by injecting faults
-		TEST((kt == ktMin) && (kt != ktOrig)); // Datacenter Kill request was downgraded
-		TEST((kt == ktMin) && (kt == ktOrig)); // Datacenter kill - Requested kill was done
+		CODE_PROBE(
+		    kt != ktMin, "DataCenter kill was rejected by killMachine", probe::context::sim2, probe::assert::simOnly);
+		CODE_PROBE((kt == ktMin) && (kt == RebootAndDelete),
+		           "Datacenter kill Resulted in a reboot and delete",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
+		CODE_PROBE((kt == ktMin) && (kt == Reboot),
+		           "Datacenter kill Resulted in a reboot",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
+		CODE_PROBE((kt == ktMin) && (kt == KillInstantly),
+		           "Datacenter kill Resulted in an instant kill",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
+		CODE_PROBE((kt == ktMin) && (kt == InjectFaults),
+		           "Datacenter kill Resulted in a kill by injecting faults",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
+		CODE_PROBE((kt == ktMin) && (kt != ktOrig),
+		           "Datacenter Kill request was downgraded",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
+		CODE_PROBE((kt == ktMin) && (kt == ktOrig),
+		           "Datacenter kill - Requested kill was done",
+		           probe::context::sim2,
+		           probe::assert::simOnly);
 
 		if (ktFinal)
 			*ktFinal = ktMin;
@@ -2125,6 +2233,9 @@ public:
 		                    this,
 		                    "",
 		                    "");
+		// create a key pair for AuthZ testing
+		auto key = mkcert::makeEcP256();
+		authKeys.insert(std::make_pair(Standalone<StringRef>("DefaultKey"_sr), key));
 		g_network = net2 = newNet2(TLSConfig(), false, true);
 		g_network->addStopCallback(Net2FileSystem::stop);
 		Net2FileSystem::newFileSystem();
@@ -2276,7 +2387,7 @@ class UDPSimSocket : public IUDPSocket, ReferenceCounted<UDPSimSocket> {
 	NetworkAddress _localAddress;
 	bool randomDropPacket() {
 		auto res = deterministicRandom()->random01() < .000001;
-		TEST(res); // UDP packet drop
+		CODE_PROBE(res, "UDP packet drop", probe::context::sim2, probe::assert::simOnly);
 		return res;
 	}
 
@@ -2319,8 +2430,8 @@ class UDPSimSocket : public IUDPSocket, ReferenceCounted<UDPSimSocket> {
 
 public:
 	UDPSimSocket(NetworkAddress const& localAddress, Optional<NetworkAddress> const& peerAddress)
-	  : id(deterministicRandom()->randomUniqueID()), process(g_simulator.getCurrentProcess()), peerAddress(peerAddress),
-	    actors(false), _localAddress(localAddress) {
+	  : id(deterministicRandom()->randomUniqueID()), process(g_simulator->getCurrentProcess()),
+	    peerAddress(peerAddress), actors(false), _localAddress(localAddress) {
 		g_sim2.addressMap.emplace(_localAddress, process);
 		ASSERT(process->boundUDPSockets.find(localAddress) == process->boundUDPSockets.end());
 		process->boundUDPSockets.emplace(localAddress, this);
@@ -2423,7 +2534,7 @@ public:
 
 Future<Reference<IUDPSocket>> Sim2::createUDPSocket(NetworkAddress toAddr) {
 	NetworkAddress localAddress;
-	auto process = g_simulator.getCurrentProcess();
+	auto process = g_simulator->getCurrentProcess();
 	if (process->address.ip.isV6()) {
 		IPAddress::IPAddressStore store = process->address.ip.toV6();
 		uint16_t* ipParts = (uint16_t*)store.data();
@@ -2441,7 +2552,7 @@ Future<Reference<IUDPSocket>> Sim2::createUDPSocket(NetworkAddress toAddr) {
 
 Future<Reference<IUDPSocket>> Sim2::createUDPSocket(bool isV6) {
 	NetworkAddress localAddress;
-	auto process = g_simulator.getCurrentProcess();
+	auto process = g_simulator->getCurrentProcess();
 	if (process->address.ip.isV6() == isV6) {
 		localAddress = process->address;
 	} else {
@@ -2463,8 +2574,8 @@ Future<Reference<IUDPSocket>> Sim2::createUDPSocket(bool isV6) {
 
 void startNewSimulator(bool printSimTime) {
 	ASSERT(!g_network);
-	g_network = g_pSimulator = new Sim2(printSimTime);
-	g_simulator.connectionFailuresDisableDuration = deterministicRandom()->random01() < 0.5 ? 0 : 1e6;
+	g_network = g_simulator = new Sim2(printSimTime);
+	g_simulator->connectionFailuresDisableDuration = deterministicRandom()->random01() < 0.5 ? 0 : 1e6;
 }
 
 ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
@@ -2485,12 +2596,20 @@ ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
 		ASSERT(kt == ISimulator::RebootProcess || kt == ISimulator::Reboot || kt == ISimulator::RebootAndDelete ||
 		       kt == ISimulator::RebootProcessAndDelete);
 
-		TEST(kt == ISimulator::RebootProcess); // Simulated process rebooted
-		TEST(kt == ISimulator::Reboot); // Simulated machine rebooted
-		TEST(kt == ISimulator::RebootAndDelete); // Simulated machine rebooted with data and coordination state deletion
-		TEST(
-		    kt ==
-		    ISimulator::RebootProcessAndDelete); // Simulated process rebooted with data and coordination state deletion
+		CODE_PROBE(kt == ISimulator::RebootProcess,
+		           "Simulated process rebooted",
+		           probe::assert::simOnly,
+		           probe::context::sim2);
+		CODE_PROBE(
+		    kt == ISimulator::Reboot, "Simulated machine rebooted", probe::assert::simOnly, probe::context::sim2);
+		CODE_PROBE(kt == ISimulator::RebootAndDelete,
+		           "Simulated machine rebooted with data and coordination state deletion",
+		           probe::assert::simOnly,
+		           probe::context::sim2);
+		CODE_PROBE(kt == ISimulator::RebootProcessAndDelete,
+		           "Simulated process rebooted with data and coordination state deletion",
+		           probe::assert::simOnly,
+		           probe::context::sim2);
 
 		if (p->rebooting || !p->isReliable()) {
 			TraceEvent(SevDebug, "DoRebootFailed")
@@ -2518,7 +2637,7 @@ ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
 		p->rebooting = true;
 		if ((kt == ISimulator::RebootAndDelete) || (kt == ISimulator::RebootProcessAndDelete)) {
 			p->cleared = true;
-			g_simulator.clearAddress(p->address);
+			g_simulator->clearAddress(p->address);
 		}
 		p->shutdownSignal.send(kt);
 	} catch (Error& e) {
@@ -2530,10 +2649,10 @@ ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
 
 // Simulates delays for performing operations on disk
 Future<Void> waitUntilDiskReady(Reference<DiskParameters> diskParameters, int64_t size, bool sync) {
-	if (g_simulator.getCurrentProcess()->failedDisk) {
+	if (g_simulator->getCurrentProcess()->failedDisk) {
 		return Never();
 	}
-	if (g_simulator.connectionFailuresDisableDuration > 1e4)
+	if (g_simulator->connectionFailuresDisableDuration > 1e4)
 		return delay(0.0001);
 
 	if (diskParameters->nextOperation < now())
@@ -2581,14 +2700,14 @@ int sf_open(const char* filename, int flags, int convFlags, int mode) {
 Future<Reference<class IAsyncFile>> Sim2FileSystem::open(const std::string& filename, int64_t flags, int64_t mode) {
 	ASSERT((flags & IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE) || !(flags & IAsyncFile::OPEN_CREATE) ||
 	       StringRef(filename).endsWith(
-	           LiteralStringRef(".fdb-lock"))); // We don't use "ordinary" non-atomic file creation right now except for
-	                                            // folder locking, and we don't have code to simulate its unsafeness.
+	           ".fdb-lock"_sr)); // We don't use "ordinary" non-atomic file creation right now except for
+	                             // folder locking, and we don't have code to simulate its unsafeness.
 
 	if ((flags & IAsyncFile::OPEN_EXCLUSIVE))
 		ASSERT(flags & IAsyncFile::OPEN_CREATE);
 
 	if (flags & IAsyncFile::OPEN_UNCACHED) {
-		auto& machineCache = g_simulator.getCurrentProcess()->machine->openFiles;
+		auto& machineCache = g_simulator->getCurrentProcess()->machine->openFiles;
 		std::string actualFilename = filename;
 		if (flags & IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE) {
 			actualFilename = filename + ".part";
diff --git a/fdbrpc/sim_validation.cpp b/fdbrpc/sim_validation.cpp
index 061df1386e..6fff16afd3 100644
--- a/fdbrpc/sim_validation.cpp
+++ b/fdbrpc/sim_validation.cpp
@@ -51,13 +51,13 @@ void debug_advanceVersion(UID id, int64_t version, const char* suffix) {
 }
 
 void debug_advanceMinCommittedVersion(UID id, int64_t version) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return;
 	debug_advanceVersion(id, version, "min");
 }
 
 void debug_advanceMaxCommittedVersion(UID id, int64_t version) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return;
 	debug_advanceVersion(id, version, "max");
 }
@@ -67,7 +67,7 @@ bool debug_checkPartRestoredVersion(UID id,
                                     std::string context,
                                     std::string minormax,
                                     Severity sev = SevError) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return false;
 	if (disabledMachines.count(id))
 		return false;
@@ -88,33 +88,33 @@ bool debug_checkPartRestoredVersion(UID id,
 }
 
 bool debug_checkRestoredVersion(UID id, int64_t version, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return false;
 	return debug_checkPartRestoredVersion(id, version, context, "min", sev) ||
 	       debug_checkPartRestoredVersion(id, version, context, "max", sev);
 }
 
 void debug_removeVersions(UID id) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return;
 	validationData.erase(id.toString() + "min");
 	validationData.erase(id.toString() + "max");
 }
 
 bool debug_versionsExist(UID id) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return false;
 	return validationData.count(id.toString() + "min") != 0 || validationData.count(id.toString() + "max") != 0;
 }
 
 bool debug_checkMinRestoredVersion(UID id, int64_t version, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return false;
 	return debug_checkPartRestoredVersion(id, version, context, "min", sev);
 }
 
 bool debug_checkMaxRestoredVersion(UID id, int64_t version, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return false;
 	return debug_checkPartRestoredVersion(id, version, context, "max", sev);
 }
@@ -129,13 +129,13 @@ void debug_setCheckRelocationDuration(bool check) {
 	checkRelocationDuration = check;
 }
 void debug_advanceVersionTimestamp(int64_t version, double t) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return;
 	timedVersionsValidationData[version] = t;
 }
 
 bool debug_checkVersionTime(int64_t version, double t, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || g_simulator.extraDB)
+	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
 		return false;
 	if (!timedVersionsValidationData.count(version)) {
 		TraceEvent(SevWarn, (context + "UnknownTime").c_str())
diff --git a/fdbrpc/tests/AuthzTlsTest.actor.cpp b/fdbrpc/tests/AuthzTlsTest.actor.cpp
new file mode 100644
index 0000000000..54dedb05a7
--- /dev/null
+++ b/fdbrpc/tests/AuthzTlsTest.actor.cpp
@@ -0,0 +1,513 @@
+/*
+ * AuthzTlsTest.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _WIN32
+#include <algorithm>
+#include <cstring>
+#include <cstdlib>
+#include <ctime>
+#include <fmt/format.h>
+#include <limits>
+#include <unistd.h>
+#include <string_view>
+#include <signal.h>
+#include <sys/wait.h>
+#include <type_traits>
+#include "flow/Arena.h"
+#include "flow/Error.h"
+#include "flow/MkCert.h"
+#include "flow/ScopeExit.h"
+#include "flow/TLSConfig.actor.h"
+#include "fdbrpc/fdbrpc.h"
+#include "fdbrpc/FlowTransport.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+std::FILE* outp = stdout;
+
+enum ChainLength : int {
+	NO_TLS = std::numeric_limits<int>::min(),
+};
+
+template <class... Args>
+void logRaw(Args&&... args) {
+	auto buf = fmt::memory_buffer{};
+	fmt::format_to(std::back_inserter(buf), std::forward<Args>(args)...);
+	fmt::print(outp, "{}", std::string_view(buf.data(), buf.size()));
+}
+
+template <class... Args>
+void logWithPrefix(const char* prefix, Args&&... args) {
+	auto buf = fmt::memory_buffer{};
+	fmt::format_to(std::back_inserter(buf), std::forward<Args>(args)...);
+	fmt::print(outp, "{}{}\n", prefix, std::string_view(buf.data(), buf.size()));
+}
+
+template <class... Args>
+void logc(Args&&... args) {
+	logWithPrefix("[CLIENT] ", std::forward<Args>(args)...);
+}
+
+template <class... Args>
+void logs(Args&&... args) {
+	logWithPrefix("[SERVER] ", std::forward<Args>(args)...);
+}
+
+template <class... Args>
+void logm(Args&&... args) {
+	logWithPrefix("[ MAIN ] ", std::forward<Args>(args)...);
+}
+
+std::string drainPipe(int pipeFd) {
+	int readRc = 0;
+	std::string ret;
+	char buf[PIPE_BUF];
+	while ((readRc = ::read(pipeFd, buf, PIPE_BUF)) > 0) {
+		ret.append(buf, readRc);
+	}
+	if (readRc != 0) {
+		logm("Unexpected error draining pipe: {}", strerror(errno));
+		throw std::runtime_error("pipe read error");
+	}
+	return ret;
+}
+
+struct TLSCreds {
+	bool noTls = false;
+	std::string certBytes;
+	std::string keyBytes;
+	std::string caBytes;
+};
+
+TLSCreds makeCreds(ChainLength chainLen, mkcert::ESide side) {
+	if (chainLen == 0 || chainLen == NO_TLS) {
+		return TLSCreds{ chainLen == NO_TLS, "", "", "" };
+	}
+	auto arena = Arena();
+	auto ret = TLSCreds{};
+	auto specs = mkcert::makeCertChainSpec(arena, std::labs(chainLen), side);
+	if (chainLen < 0) {
+		specs[0].offsetNotBefore = -60l * 60 * 24 * 365;
+		specs[0].offsetNotAfter = -10l; // cert that expired 10 seconds ago
+	}
+	auto chain = mkcert::makeCertChain(arena, specs, {} /* create root CA cert from spec*/);
+	if (chain.size() == 1) {
+		ret.certBytes = concatCertChain(arena, chain).toString();
+	} else {
+		auto nonRootChain = chain;
+		nonRootChain.pop_back();
+		ret.certBytes = concatCertChain(arena, nonRootChain).toString();
+	}
+	ret.caBytes = chain.back().certPem.toString();
+	ret.keyBytes = chain.front().privateKeyPem.toString();
+	return ret;
+}
+
+enum class Result : int {
+	ERROR = 0,
+	TRUSTED,
+	UNTRUSTED,
+	TIMEOUT,
+};
+
+template <>
+struct fmt::formatter<Result> {
+	constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) { return ctx.begin(); }
+
+	template <class FormatContext>
+	auto format(const Result& r, FormatContext& ctx) -> decltype(ctx.out()) {
+		if (r == Result::TRUSTED)
+			return fmt::format_to(ctx.out(), "TRUSTED");
+		else if (r == Result::UNTRUSTED)
+			return fmt::format_to(ctx.out(), "UNTRUSTED");
+		else if (r == Result::TIMEOUT)
+			return fmt::format_to(ctx.out(), "TIMEOUT");
+		else
+			return fmt::format_to(ctx.out(), "ERROR");
+	}
+};
+
+template <>
+struct fmt::formatter<ChainLength> {
+	constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) { return ctx.begin(); }
+
+	template <class FormatContext>
+	auto format(ChainLength value, FormatContext& ctx) -> decltype(ctx.out()) {
+		if (value == NO_TLS)
+			return fmt::format_to(ctx.out(), "NO_TLS");
+		else
+			return fmt::format_to(ctx.out(), "{}", static_cast<std::underlying_type_t<ChainLength>>(value));
+	}
+};
+
+template <>
+struct fmt::formatter<std::vector<std::pair<ChainLength, ChainLength>>> {
+	constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) { return ctx.begin(); }
+
+	template <class FormatContext>
+	auto format(const std::vector<std::pair<ChainLength, ChainLength>>& entries, FormatContext& ctx)
+	    -> decltype(ctx.out()) {
+		fmt::format_to(ctx.out(), "[");
+		bool first = true;
+		for (const auto& entry : entries) {
+			fmt::format_to(ctx.out(), "{}{{ {}, {} }}", (first ? "" : ", "), entry.first, entry.second);
+			first = false;
+		}
+		return fmt::format_to(ctx.out(), "]");
+	}
+};
+
+ACTOR template <class T>
+Future<T> stopNetworkAfter(Future<T> what) {
+	T t = wait(what);
+	g_network->stop();
+	return t;
+}
+
+// Reflective struct containing information about the requester from a server PoV
+struct SessionInfo {
+	constexpr static FileIdentifier file_identifier = 1578312;
+	bool isPeerTrusted = false;
+	NetworkAddress peerAddress;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, isPeerTrusted, peerAddress);
+	}
+};
+
+struct SessionProbeRequest {
+	constexpr static FileIdentifier file_identifier = 1559713;
+	ReplyPromise<SessionInfo> reply{ PeerCompatibilityPolicy{ RequirePeer::AtLeast,
+		                                                      ProtocolVersion::withStableInterfaces() } };
+
+	bool verify() const { return true; }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, reply);
+	}
+};
+
+struct SessionProbeReceiver final : NetworkMessageReceiver {
+	SessionProbeReceiver() {}
+	void receive(ArenaObjectReader& reader) override {
+		SessionProbeRequest req;
+		reader.deserialize(req);
+		SessionInfo res;
+		res.isPeerTrusted = FlowTransport::transport().currentDeliveryPeerIsTrusted();
+		res.peerAddress = FlowTransport::transport().currentDeliveryPeerAddress();
+		req.reply.send(res);
+	}
+	PeerCompatibilityPolicy peerCompatibilityPolicy() const override {
+		return PeerCompatibilityPolicy{ RequirePeer::AtLeast, ProtocolVersion::withStableInterfaces() };
+	}
+	bool isPublic() const override { return true; }
+};
+
+void runServer(const Endpoint& endpoint, int addrPipe, int completionPipe) {
+	auto realAddr = FlowTransport::transport().getLocalAddresses().address;
+	logs("Listening at {}", realAddr.toString());
+	logs("Endpoint token is {}", endpoint.token.toString());
+	static_assert(std::is_trivially_destructible_v<NetworkAddress>,
+	              "NetworkAddress cannot be directly put on wire; need proper (de-)serialization");
+	// below writes/reads would block, but this is good enough for a test.
+	if (sizeof(realAddr) != ::write(addrPipe, &realAddr, sizeof(realAddr))) {
+		logs("Failed to write server addr to pipe: {}", strerror(errno));
+		return;
+	}
+	if (sizeof(endpoint.token) != ::write(addrPipe, &endpoint.token, sizeof(endpoint.token))) {
+		logs("Failed to write server endpoint to pipe: {}", strerror(errno));
+		return;
+	}
+	auto done = false;
+	if (sizeof(done) != ::read(completionPipe, &done, sizeof(done))) {
+		logs("Failed to read completion flag from pipe: {}", strerror(errno));
+		return;
+	}
+	return;
+}
+
+ACTOR Future<Void> waitAndPrintResponse(Future<SessionInfo> response, Result* rc) {
+	try {
+		SessionInfo info = wait(response);
+		logc("Probe response: trusted={} peerAddress={}", info.isPeerTrusted, info.peerAddress.toString());
+		*rc = info.isPeerTrusted ? Result::TRUSTED : Result::UNTRUSTED;
+	} catch (Error& err) {
+		if (err.code() != error_code_operation_cancelled) {
+			logc("Unexpected error: {}", err.what());
+			*rc = Result::ERROR;
+		} else {
+			logc("Timed out");
+			*rc = Result::TIMEOUT;
+		}
+	}
+	return Void();
+}
+
+template <bool IsServer>
+int runHost(TLSCreds creds, int addrPipe, int completionPipe, Result expect) {
+	auto tlsConfig = TLSConfig(IsServer ? TLSEndpointType::SERVER : TLSEndpointType::CLIENT);
+	bool const noTls = creds.noTls;
+	if (!noTls) {
+		tlsConfig.setCertificateBytes(creds.certBytes);
+		tlsConfig.setCABytes(creds.caBytes);
+		tlsConfig.setKeyBytes(creds.keyBytes);
+	}
+	g_network = newNet2(tlsConfig);
+	openTraceFile(NetworkAddress(),
+	              10 << 20,
+	              10 << 20,
+	              ".",
+	              IsServer ? "authz_tls_unittest_server" : "authz_tls_unittest_client");
+	FlowTransport::createInstance(!IsServer, 1, WLTOKEN_RESERVED_COUNT);
+	auto& transport = FlowTransport::transport();
+	if constexpr (IsServer) {
+		auto addr = NetworkAddress::parse(noTls ? "127.0.0.1:0" : "127.0.0.1:0:tls");
+		auto endpoint = Endpoint();
+		auto receiver = SessionProbeReceiver();
+		auto listenFuture = transport.bind(addr, addr);
+		transport.addEndpoint(endpoint, &receiver, TaskPriority::ReadSocket);
+		auto thread = std::thread([]() {
+			g_network->run();
+			flushTraceFileVoid();
+		});
+		runServer(endpoint, addrPipe, completionPipe);
+		auto cleanupGuard = ScopeExit([&thread]() {
+			g_network->stop();
+			thread.join();
+		});
+		return 0;
+	} else {
+		auto dest = Endpoint();
+		auto& serverAddr = dest.addresses.address;
+		if (sizeof(serverAddr) != ::read(addrPipe, &serverAddr, sizeof(serverAddr))) {
+			logc("Failed to read server addr from pipe: {}", strerror(errno));
+			return 1;
+		}
+		if (noTls)
+			serverAddr.flags &= ~NetworkAddress::FLAG_TLS;
+		else
+			serverAddr.flags |= NetworkAddress::FLAG_TLS;
+		auto& token = dest.token;
+		if (sizeof(token) != ::read(addrPipe, &token, sizeof(token))) {
+			logc("Failed to read server endpoint token from pipe: {}", strerror(errno));
+			return 2;
+		}
+		logc("Server address is {}{}", serverAddr.toString(), noTls ? " (TLS suffix removed)" : "");
+		logc("Server endpoint token is {}", token.toString());
+		auto sessionProbeReq = SessionProbeRequest{};
+		transport.sendUnreliable(SerializeSource(sessionProbeReq), dest, true /*openConnection*/);
+		logc("Request is sent");
+		auto rc = 0;
+		auto result = Result::ERROR;
+		{
+			auto timeout = delay(expect == Result::TIMEOUT ? 0.5 : 5);
+			auto complete = waitAndPrintResponse(sessionProbeReq.reply.getFuture(), &result);
+			auto f = stopNetworkAfter(complete || timeout);
+			g_network->run();
+		}
+		auto done = true;
+		if (sizeof(done) != ::write(completionPipe, &done, sizeof(done))) {
+			logc("Failed to signal server to terminate: {}", strerror(errno));
+			rc = 4;
+		}
+		if (rc == 0) {
+			if (expect != result) {
+				logc("Test failed: expected {}, got {}", expect, result);
+				rc = 5;
+			} else {
+				logc("Response OK: got {} as expected", result);
+			}
+		}
+		return rc;
+	}
+}
+
+Result getExpectedResult(ChainLength serverChainLen, ChainLength clientChainLen) {
+	auto expect = Result::ERROR;
+	if (serverChainLen > 0) {
+		if (clientChainLen == NO_TLS || clientChainLen < 0) {
+			expect = Result::TIMEOUT;
+		} else if (clientChainLen > 0) {
+			expect = Result::TRUSTED;
+		} else if (clientChainLen == 0) {
+			expect = Result::UNTRUSTED;
+		}
+	} else if (serverChainLen == NO_TLS && clientChainLen == NO_TLS) {
+		expect = Result::TRUSTED;
+	} else {
+		expect = Result::TIMEOUT;
+	}
+	return expect;
+}
+
+bool waitPid(pid_t subProcPid, const char* procName) {
+	auto status = int{};
+	auto pid = ::waitpid(subProcPid, &status, 0);
+	if (pid < 0) {
+		logm("{} subprocess waitpid() failed with {}", procName, strerror(errno));
+		return false;
+	} else {
+		if (status != 0) {
+			logm("{} subprocess had error: rc={}", procName, status);
+			return false;
+		} else {
+			logm("{} subprocess waitpid() OK", procName);
+			return true;
+		}
+	}
+}
+
+int runTlsTest(ChainLength serverChainLen, ChainLength clientChainLen) {
+	logm("==== BEGIN TESTCASE ====");
+	auto const expect = getExpectedResult(serverChainLen, clientChainLen);
+	using namespace std::literals::string_literals;
+	logm("Cert chain length: server={} client={}", serverChainLen, clientChainLen);
+	auto arena = Arena();
+	auto serverCreds = makeCreds(serverChainLen, mkcert::ESide::Server);
+	auto clientCreds = makeCreds(clientChainLen, mkcert::ESide::Client);
+	// make server and client trust each other
+	std::swap(serverCreds.caBytes, clientCreds.caBytes);
+	auto clientPid = pid_t{};
+	auto serverPid = pid_t{};
+	int addrPipe[2], completionPipe[2], serverStdoutPipe[2], clientStdoutPipe[2];
+	if (::pipe(addrPipe) || ::pipe(completionPipe) || ::pipe(serverStdoutPipe) || ::pipe(clientStdoutPipe)) {
+		logm("Pipe open failed: {}", strerror(errno));
+		return 1;
+	}
+	auto ok = true;
+	{
+		serverPid = fork();
+		if (serverPid == -1) {
+			logm("fork() for server subprocess failed: {}", strerror(errno));
+			return 1;
+		} else if (serverPid == 0) {
+			// server subprocess
+			::close(addrPipe[0]); // close address-in pipe (server writes its own address for client)
+			::close(
+			    completionPipe[1]); // close completion-flag-out pipe (server awaits/reads completion flag from client)
+			::close(clientStdoutPipe[0]);
+			::close(clientStdoutPipe[1]);
+			::close(serverStdoutPipe[0]);
+			auto pipeCleanup = ScopeExit([&addrPipe, &completionPipe]() {
+				::close(addrPipe[1]);
+				::close(completionPipe[0]);
+			});
+			if (-1 == ::dup2(serverStdoutPipe[1], STDOUT_FILENO)) {
+				logs("Failed to redirect server stdout to pipe: {}", strerror(errno));
+				::close(serverStdoutPipe[1]);
+				return 1;
+			}
+			_exit(runHost<true>(std::move(serverCreds), addrPipe[1], completionPipe[0], expect));
+		}
+		auto serverProcCleanup = ScopeExit([&ok, serverPid]() {
+			if (!waitPid(serverPid, "Server"))
+				ok = false;
+		});
+		clientPid = fork();
+		if (clientPid == -1) {
+			logm("fork() for client subprocess failed: {}", strerror(errno));
+			return 1;
+		} else if (clientPid == 0) {
+			::close(addrPipe[1]);
+			::close(completionPipe[0]);
+			::close(serverStdoutPipe[0]);
+			::close(serverStdoutPipe[1]);
+			::close(clientStdoutPipe[0]);
+			auto pipeCleanup = ScopeExit([&addrPipe, &completionPipe]() {
+				::close(addrPipe[0]);
+				::close(completionPipe[1]);
+			});
+			if (-1 == ::dup2(clientStdoutPipe[1], STDOUT_FILENO)) {
+				logs("Failed to redirect client stdout to pipe: {}", strerror(errno));
+				::close(clientStdoutPipe[1]);
+				return 1;
+			}
+			_exit(runHost<false>(std::move(clientCreds), addrPipe[0], completionPipe[1], expect));
+		}
+		auto clientProcCleanup = ScopeExit([&ok, clientPid]() {
+			if (!waitPid(clientPid, "Client"))
+				ok = false;
+		});
+	}
+	// main process
+	::close(addrPipe[0]);
+	::close(addrPipe[1]);
+	::close(completionPipe[0]);
+	::close(completionPipe[1]);
+	::close(serverStdoutPipe[1]);
+	::close(clientStdoutPipe[1]);
+	auto pipeCleanup = ScopeExit([&]() {
+		::close(serverStdoutPipe[0]);
+		::close(clientStdoutPipe[0]);
+	});
+	std::string const clientStdout = drainPipe(clientStdoutPipe[0]);
+	logm("/// Begin Client STDOUT ///");
+	logRaw(clientStdout);
+	logm("/// End Client STDOUT ///");
+	std::string const serverStdout = drainPipe(serverStdoutPipe[0]);
+	logm("/// Begin Server STDOUT ///");
+	logRaw(serverStdout);
+	logm("/// End Server STDOUT ///");
+	logm(ok ? "OK" : "FAILED");
+	return !ok;
+}
+
+int main(int argc, char** argv) {
+	unsigned seed = std::time(nullptr);
+	if (argc > 1)
+		seed = std::stoul(argv[1]);
+	std::srand(seed);
+	logm("Seed: {}", seed);
+	auto categoryToValue = [](int category) -> ChainLength {
+		if (category == 2 || category == -2) {
+			return static_cast<ChainLength>(category + std::rand() % 3);
+		} else {
+			return static_cast<ChainLength>(category);
+		}
+	};
+	std::vector<std::pair<ChainLength, ChainLength>> inputs;
+	std::vector<int> categories{ 0, NO_TLS, 1, -1, 2, -2 };
+	for (auto lhs : categories) {
+		for (auto rhs : categories) {
+			auto input = std::pair(categoryToValue(lhs), categoryToValue(rhs));
+			inputs.push_back(input);
+		}
+	}
+	std::vector<std::pair<ChainLength, ChainLength>> failed;
+	for (auto input : inputs) {
+		auto [serverChainLen, clientChainLen] = input;
+		if (runTlsTest(serverChainLen, clientChainLen))
+			failed.push_back({ serverChainLen, clientChainLen });
+	}
+	if (!failed.empty()) {
+		logm("Test Failed: {}/{} cases: {}", failed.size(), inputs.size(), failed);
+		return 1;
+	} else {
+		logm("Test OK: {}/{} cases passed", inputs.size(), inputs.size());
+		return 0;
+	}
+}
+#else // _WIN32
+
+int main() {
+	return 0;
+}
+#endif // _WIN32
diff --git a/fdbrpc/tests/CMakeLists.txt b/fdbrpc/tests/CMakeLists.txt
new file mode 100644
index 0000000000..d01bed6f17
--- /dev/null
+++ b/fdbrpc/tests/CMakeLists.txt
@@ -0,0 +1,9 @@
+if(NOT WIN32)
+  add_flow_target(EXECUTABLE NAME authz_tls_unittest SRCS AuthzTlsTest.actor.cpp)
+  target_link_libraries(authz_tls_unittest PRIVATE flow fdbrpc fmt::fmt)
+  if(NOT OPEN_FOR_IDE)
+    add_test(NAME authorization_tls_unittest
+      COMMAND $<TARGET_FILE:authz_tls_unittest>)
+    set_tests_properties(authorization_tls_unittest PROPERTIES TIMEOUT 120)
+  endif()
+endif()
diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp
index b02f091179..bcea30254d 100644
--- a/fdbserver/ApplyMetadataMutation.cpp
+++ b/fdbserver/ApplyMetadataMutation.cpp
@@ -20,11 +20,12 @@
 
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/KeyBackedTypes.h" // for key backed map codecs for tss mapping
+#include "fdbclient/Metacluster.h"
 #include "fdbclient/MutationList.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/SystemData.h"
 #include "fdbserver/ApplyMetadataMutation.h"
-#include "fdbserver/EncryptedMutationMessage.h"
+#include "fdbserver/EncryptionOpsUtils.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "fdbserver/LogProtocolMessage.h"
 #include "fdbserver/LogSystem.h"
@@ -58,9 +59,10 @@ public:
 	                           const UID& dbgid_,
 	                           Arena& arena_,
 	                           const VectorRef<MutationRef>& mutations_,
-	                           IKeyValueStore* txnStateStore_)
+	                           IKeyValueStore* txnStateStore_,
+	                           Reference<AsyncVar<ServerDBInfo> const> db)
 	  : spanContext(spanContext_), dbgid(dbgid_), arena(arena_), mutations(mutations_), txnStateStore(txnStateStore_),
-	    confChange(dummyConfChange) {}
+	    confChange(dummyConfChange), dbInfo(db) {}
 
 	ApplyMetadataMutationsImpl(const SpanContext& spanContext_,
 	                           Arena& arena_,
@@ -82,16 +84,17 @@ public:
 	    commit(proxyCommitData_.commit), cx(proxyCommitData_.cx), committedVersion(&proxyCommitData_.committedVersion),
 	    storageCache(&proxyCommitData_.storageCache), tag_popped(&proxyCommitData_.tag_popped),
 	    tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap),
-	    initialCommit(initialCommit_) {}
+	    tenantIdIndex(&proxyCommitData_.tenantIdIndex), initialCommit(initialCommit_), dbInfo(proxyCommitData_.db) {}
 
 	ApplyMetadataMutationsImpl(const SpanContext& spanContext_,
 	                           ResolverData& resolverData_,
-	                           const VectorRef<MutationRef>& mutations_)
+	                           const VectorRef<MutationRef>& mutations_,
+	                           Reference<AsyncVar<ServerDBInfo> const> db)
 	  : spanContext(spanContext_), dbgid(resolverData_.dbgid), arena(resolverData_.arena), mutations(mutations_),
 	    txnStateStore(resolverData_.txnStateStore), toCommit(resolverData_.toCommit),
 	    confChange(resolverData_.confChanges), logSystem(resolverData_.logSystem), popVersion(resolverData_.popVersion),
 	    keyInfo(resolverData_.keyInfo), storageCache(resolverData_.storageCache),
-	    initialCommit(resolverData_.initialCommit), forResolver(true) {}
+	    initialCommit(resolverData_.initialCommit), forResolver(true), dbInfo(db) {}
 
 private:
 	// The following variables are incoming parameters
@@ -131,6 +134,7 @@ private:
 	std::unordered_map<UID, StorageServerInterface>* tssMapping = nullptr;
 
 	std::map<TenantName, TenantMapEntry>* tenantMap = nullptr;
+	std::unordered_map<int64_t, TenantName>* tenantIdIndex = nullptr;
 
 	// true if the mutations were already written to the txnStateStore as part of recovery
 	bool initialCommit = false;
@@ -138,6 +142,8 @@ private:
 	// true if called from Resolver
 	bool forResolver = false;
 
+	Reference<AsyncVar<ServerDBInfo> const> dbInfo;
+
 private:
 	// The following variables are used internally
 
@@ -158,12 +164,12 @@ private:
 
 private:
 	void writeMutation(const MutationRef& m) {
-		if (forResolver || !SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION) {
+		if (forResolver || !isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION, dbInfo->get().client)) {
 			toCommit->writeTypedMessage(m);
 		} else {
 			ASSERT(cipherKeys != nullptr);
 			Arena arena;
-			toCommit->writeTypedMessage(EncryptedMutationMessage::encryptMetadata(arena, *cipherKeys, m));
+			toCommit->writeTypedMessage(m.encryptMetadata(*cipherKeys, arena, BlobCipherMetrics::TLOG));
 		}
 	}
 
@@ -322,7 +328,8 @@ private:
 	}
 
 	void checkSetConfigKeys(MutationRef m) {
-		if (!m.param1.startsWith(configKeysPrefix) && m.param1 != coordinatorsKey) {
+		if (!m.param1.startsWith(configKeysPrefix) && m.param1 != coordinatorsKey &&
+		    m.param1 != previousCoordinatorsKey) {
 			return;
 		}
 		if (Optional<StringRef>(m.param2) !=
@@ -338,7 +345,8 @@ private:
 				TraceEvent("MutationRequiresRestart", dbgid)
 				    .detail("M", m)
 				    .detail("PrevValue", t.orDefault("(none)"_sr))
-				    .detail("ToCommit", toCommit != nullptr);
+				    .detail("ToCommit", toCommit != nullptr)
+				    .detail("InitialCommit", initialCommit);
 				confChange = true;
 			}
 		}
@@ -624,7 +632,7 @@ private:
 		if (!initialCommit)
 			txnStateStore->set(KeyValueRef(m.param1, m.param2));
 		confChange = true;
-		TEST(true); // Recovering at a higher version.
+		CODE_PROBE(true, "Recovering at a higher version.");
 	}
 
 	void checkSetVersionEpochKey(MutationRef m) {
@@ -636,7 +644,7 @@ private:
 		if (!initialCommit)
 			txnStateStore->set(KeyValueRef(m.param1, m.param2));
 		confChange = true;
-		TEST(true); // Setting version epoch
+		CODE_PROBE(true, "Setting version epoch");
 	}
 
 	void checkSetWriteRecoverKey(MutationRef m) {
@@ -646,18 +654,27 @@ private:
 		TraceEvent("WriteRecoveryKeySet", dbgid).log();
 		if (!initialCommit)
 			txnStateStore->set(KeyValueRef(m.param1, m.param2));
-		TEST(true); // Snapshot created, setting writeRecoveryKey in txnStateStore
+		CODE_PROBE(true, "Snapshot created, setting writeRecoveryKey in txnStateStore");
 	}
 
 	void checkSetTenantMapPrefix(MutationRef m) {
-		if (m.param1.startsWith(tenantMapPrefix)) {
+		KeyRef prefix = TenantMetadata::tenantMap().subspace.begin;
+		if (m.param1.startsWith(prefix)) {
+			TenantName tenantName = m.param1.removePrefix(prefix);
+			TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2);
+
 			if (tenantMap) {
 				ASSERT(version != invalidVersion);
-				TenantName tenantName = m.param1.removePrefix(tenantMapPrefix);
-				TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2);
 
-				TraceEvent("CommitProxyInsertTenant", dbgid).detail("Tenant", tenantName).detail("Version", version);
+				TraceEvent("CommitProxyInsertTenant", dbgid)
+				    .detail("Tenant", tenantName)
+				    .detail("Id", tenantEntry.id)
+				    .detail("Version", version);
+
 				(*tenantMap)[tenantName] = tenantEntry;
+				if (tenantIdIndex) {
+					(*tenantIdIndex)[tenantEntry.id] = tenantName;
+				}
 			}
 
 			if (!initialCommit) {
@@ -680,7 +697,31 @@ private:
 				writeMutation(privatized);
 			}
 
-			TEST(true); // Tenant added to map
+			CODE_PROBE(true, "Tenant added to map");
+		}
+	}
+
+	void checkSetMetaclusterRegistration(MutationRef m) {
+		if (m.param1 == MetaclusterMetadata::metaclusterRegistration().key) {
+			MetaclusterRegistrationEntry entry = MetaclusterRegistrationEntry::decode(m.param2);
+
+			TraceEvent("SetMetaclusterRegistration", dbgid)
+			    .detail("ClusterType", entry.clusterType)
+			    .detail("MetaclusterID", entry.metaclusterId)
+			    .detail("MetaclusterName", entry.metaclusterName)
+			    .detail("ClusterID", entry.id)
+			    .detail("ClusterName", entry.name);
+
+			Optional<Value> value = txnStateStore->readValue(MetaclusterMetadata::metaclusterRegistration().key).get();
+			if (!initialCommit) {
+				txnStateStore->set(KeyValueRef(m.param1, m.param2));
+			}
+
+			if (!value.present() || value.get() != m.param2) {
+				confChange = true;
+			}
+
+			CODE_PROBE(true, "Metacluster registration set");
 		}
 	}
 
@@ -940,6 +981,15 @@ private:
 						break;
 					}
 				}
+				auto& systemBackupRanges = getSystemBackupRanges();
+				for (auto r = systemBackupRanges.begin(); !foundKey && r != systemBackupRanges.end(); ++r) {
+					for (auto& it : vecBackupKeys->intersectingRanges(*r)) {
+						if (it.value().count(logDestination) > 0) {
+							foundKey = true;
+							break;
+						}
+					}
+				}
 				if (!foundKey) {
 					auto logRanges = vecBackupKeys->modify(singleKeyRange(metadataVersionKey));
 					for (auto logRange : logRanges) {
@@ -1028,13 +1078,14 @@ private:
 	}
 
 	void checkClearTenantMapPrefix(KeyRangeRef range) {
-		if (tenantMapKeys.intersects(range)) {
+		KeyRangeRef subspace = TenantMetadata::tenantMap().subspace;
+		if (subspace.intersects(range)) {
 			if (tenantMap) {
 				ASSERT(version != invalidVersion);
 
-				StringRef startTenant = std::max(range.begin, tenantMapPrefix).removePrefix(tenantMapPrefix);
-				StringRef endTenant = (range.end.startsWith(tenantMapPrefix) ? range.end : tenantMapKeys.end)
-				                          .removePrefix(tenantMapPrefix);
+				StringRef startTenant = std::max(range.begin, subspace.begin).removePrefix(subspace.begin);
+				StringRef endTenant =
+				    range.end.startsWith(subspace.begin) ? range.end.removePrefix(subspace.begin) : "\xff\xff"_sr;
 
 				TraceEvent("CommitProxyEraseTenants", dbgid)
 				    .detail("BeginTenant", startTenant)
@@ -1043,6 +1094,17 @@ private:
 
 				auto startItr = tenantMap->lower_bound(startTenant);
 				auto endItr = tenantMap->lower_bound(endTenant);
+
+				if (tenantIdIndex) {
+					// Iterate over iterator-range and remove entries from TenantIdName map
+					// TODO: O(n) operation, optimize cpu
+					auto itr = startItr;
+					while (itr != endItr) {
+						tenantIdIndex->erase(itr->second.id);
+						itr++;
+					}
+				}
+
 				tenantMap->erase(startItr, endItr);
 			}
 
@@ -1063,12 +1125,33 @@ private:
 
 				MutationRef privatized;
 				privatized.type = MutationRef::ClearRange;
-				privatized.param1 = range.begin.withPrefix(systemKeys.begin, arena);
-				privatized.param2 = range.end.withPrefix(systemKeys.begin, arena);
+				privatized.param1 = systemKeys.begin.withSuffix(std::max(range.begin, subspace.begin), arena);
+				if (range.end < subspace.end) {
+					privatized.param2 = systemKeys.begin.withSuffix(range.end, arena);
+				} else {
+					privatized.param2 = systemKeys.begin.withSuffix(subspace.begin).withSuffix("\xff\xff"_sr, arena);
+				}
 				writeMutation(privatized);
 			}
 
-			TEST(true); // Tenant cleared from map
+			CODE_PROBE(true, "Tenant cleared from map");
+		}
+	}
+
+	void checkClearMetaclusterRegistration(KeyRangeRef range) {
+		if (range.contains(MetaclusterMetadata::metaclusterRegistration().key)) {
+			TraceEvent("ClearMetaclusterRegistration", dbgid);
+
+			Optional<Value> value = txnStateStore->readValue(MetaclusterMetadata::metaclusterRegistration().key).get();
+			if (!initialCommit) {
+				txnStateStore->clear(singleKeyRange(MetaclusterMetadata::metaclusterRegistration().key));
+			}
+
+			if (value.present()) {
+				confChange = true;
+			}
+
+			CODE_PROBE(true, "Metacluster registration cleared");
 		}
 	}
 
@@ -1076,6 +1159,9 @@ private:
 		if (initialCommit) {
 			return;
 		}
+		if (range.contains(previousCoordinatorsKey)) {
+			txnStateStore->clear(singleKeyRange(previousCoordinatorsKey));
+		}
 		if (range.contains(coordinatorsKey)) {
 			txnStateStore->clear(singleKeyRange(coordinatorsKey));
 		}
@@ -1194,6 +1280,7 @@ public:
 				checkSetMinRequiredCommitVersionKey(m);
 				checkSetVersionEpochKey(m);
 				checkSetTenantMapPrefix(m);
+				checkSetMetaclusterRegistration(m);
 				checkSetOtherKeys(m);
 			} else if (m.type == MutationRef::ClearRange && isSystemKey(m.param2)) {
 				KeyRangeRef range(m.param1, m.param2);
@@ -1211,6 +1298,7 @@ public:
 				checkClearTssQuarantineKeys(m, range);
 				checkClearVersionEpochKeys(m, range);
 				checkClearTenantMapPrefix(range);
+				checkClearMetaclusterRegistration(range);
 				checkClearMiscRangeKeys(range);
 			}
 		}
@@ -1243,7 +1331,6 @@ void applyMetadataMutations(SpanContext const& spanContext,
                             Version version,
                             Version popVersion,
                             bool initialCommit) {
-
 	ApplyMetadataMutationsImpl(spanContext,
 	                           arena,
 	                           mutations,
@@ -1260,14 +1347,16 @@ void applyMetadataMutations(SpanContext const& spanContext,
 
 void applyMetadataMutations(SpanContext const& spanContext,
                             ResolverData& resolverData,
-                            const VectorRef<MutationRef>& mutations) {
-	ApplyMetadataMutationsImpl(spanContext, resolverData, mutations).apply();
+                            const VectorRef<MutationRef>& mutations,
+                            Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	ApplyMetadataMutationsImpl(spanContext, resolverData, mutations, dbInfo).apply();
 }
 
 void applyMetadataMutations(SpanContext const& spanContext,
                             const UID& dbgid,
                             Arena& arena,
                             const VectorRef<MutationRef>& mutations,
-                            IKeyValueStore* txnStateStore) {
-	ApplyMetadataMutationsImpl(spanContext, dbgid, arena, mutations, txnStateStore).apply();
+                            IKeyValueStore* txnStateStore,
+                            Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	ApplyMetadataMutationsImpl(spanContext, dbgid, arena, mutations, txnStateStore, dbInfo).apply();
 }
diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 1055cae265..e5719dd007 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -185,7 +185,7 @@ TEST_CASE("/BackupProgress/Unfinished") {
 	const Tag tag1(tagLocalityLogRouter, 0);
 	epochInfos.insert({ epoch1, ILogSystem::EpochTagsVersionsInfo(1, begin1, end1) });
 	BackupProgress progress(UID(0, 0), epochInfos);
-	progress.setBackupStartedValue(Optional<Value>(LiteralStringRef("1")));
+	progress.setBackupStartedValue(Optional<Value>("1"_sr));
 
 	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> unfinished = progress.getUnfinishedBackup();
 
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 7c77a47c6f..67fc20aff4 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -20,13 +20,13 @@
 
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/SystemData.h"
 #include "fdbserver/BackupInterface.h"
 #include "fdbserver/BackupProgress.actor.h"
-#include "fdbserver/EncryptedMutationMessage.h"
-#include "fdbserver/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/LogProtocolMessage.h"
 #include "fdbserver/LogSystem.h"
@@ -54,10 +54,10 @@ struct VersionedMessage {
 	Version getVersion() const { return version.version; }
 	uint32_t getSubVersion() const { return version.sub; }
 
-	// Returns true if the message is a mutation that should be backuped, i.e.,
-	// either key is not in system key space or is not a metadataVersionKey.
-	bool isBackupMessage(MutationRef* m,
-	                     const std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>& cipherKeys) {
+	// Returns true if the message is a mutation that could be backed up (normal keys, system key backup ranges, or the
+	// metadata version key)
+	bool isCandidateBackupMessage(MutationRef* m,
+	                              const std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>& cipherKeys) {
 		for (Tag tag : tags) {
 			if (tag.locality == tagLocalitySpecial || tag.locality == tagLocalityTxs) {
 				return false; // skip Txs mutations
@@ -72,27 +72,42 @@ struct VersionedMessage {
 		if (reader.protocolVersion().hasSpanContext() && SpanContextMessage::isNextIn(reader))
 			return false;
 		if (reader.protocolVersion().hasOTELSpanContext() && OTELSpanContextMessage::isNextIn(reader)) {
-			TEST(true); // Returning false for OTELSpanContextMessage
+			CODE_PROBE(true, "Returning false for OTELSpanContextMessage");
 			return false;
 		}
-		if (EncryptedMutationMessage::isNextIn(reader)) {
+		reader >> *m;
+		if (m->isEncrypted()) {
 			// In case the mutation is encrypted, get the decrypted mutation and also update message to point to
 			// the decrypted mutation.
 			// We use dedicated arena for decrypt buffer, as the other arena is used to count towards backup lock bytes.
-			*m = EncryptedMutationMessage::decrypt(reader, decryptArena, cipherKeys, &message);
-		} else {
-			reader >> *m;
+			*m = m->decrypt(cipherKeys, decryptArena, BlobCipherMetrics::BACKUP, &message);
+		}
+
+		// Return true if the mutation intersects any legal backup ranges
+		if (normalKeys.contains(m->param1) || m->param1 == metadataVersionKey) {
+			return true;
+		} else if (m->type != MutationRef::Type::ClearRange) {
+			return systemBackupMutationMask().rangeContaining(m->param1).value();
+		} else {
+			for (auto& r : systemBackupMutationMask().intersectingRanges(KeyRangeRef(m->param1, m->param2))) {
+				if (r->value()) {
+					return true;
+				}
+			}
+
+			return false;
 		}
-		return normalKeys.contains(m->param1) || m->param1 == metadataVersionKey;
 	}
 
 	void collectCipherDetailIfEncrypted(std::unordered_set<BlobCipherDetails>& cipherDetails) {
-		ArenaReader reader(arena, message, AssumeVersion(g_network->protocolVersion()));
-		if (EncryptedMutationMessage::isNextIn(reader)) {
-			EncryptedMutationMessage emm;
-			reader >> emm;
-			cipherDetails.insert(emm.header.cipherTextDetails);
-			cipherDetails.insert(emm.header.cipherHeaderDetails);
+		ASSERT(!message.empty());
+		if (*message.begin() == MutationRef::Encrypted) {
+			ArenaReader reader(arena, message, AssumeVersion(ProtocolVersion::withEncryptionAtRest()));
+			MutationRef m;
+			reader >> m;
+			const BlobCipherEncryptHeader* header = m.encryptionHeader();
+			cipherDetails.insert(header->cipherTextDetails);
+			cipherDetails.insert(header->cipherHeaderDetails);
 		}
 	}
 };
@@ -453,20 +468,30 @@ struct BackupData {
 	ACTOR static Future<Version> _getMinKnownCommittedVersion(BackupData* self) {
 		state Span span("BA:GetMinCommittedVersion"_loc);
 		loop {
-			GetReadVersionRequest request(span.context,
-			                              0,
-			                              TransactionPriority::DEFAULT,
-			                              invalidVersion,
-			                              GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
-			choose {
-				when(wait(self->cx->onProxiesChanged())) {}
-				when(GetReadVersionReply reply =
-				         wait(basicLoadBalance(self->cx->getGrvProxies(UseProvisionalProxies::False),
-				                               &GrvProxyInterface::getConsistentReadVersion,
-				                               request,
-				                               self->cx->taskID))) {
-					self->cx->ssVersionVectorCache.applyDelta(reply.ssVersionVectorDelta);
-					return reply.version;
+			try {
+				GetReadVersionRequest request(span.context,
+				                              0,
+				                              TransactionPriority::DEFAULT,
+				                              invalidVersion,
+				                              GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
+				choose {
+					when(wait(self->cx->onProxiesChanged())) {}
+					when(GetReadVersionReply reply =
+					         wait(basicLoadBalance(self->cx->getGrvProxies(UseProvisionalProxies::False),
+					                               &GrvProxyInterface::getConsistentReadVersion,
+					                               request,
+					                               self->cx->taskID))) {
+						self->cx->ssVersionVectorCache.applyDelta(reply.ssVersionVectorDelta);
+						return reply.version;
+					}
+				}
+			} catch (Error& e) {
+				if (e.code() == error_code_batch_transaction_throttled ||
+				    e.code() == error_code_grv_proxy_memory_limit_exceeded) {
+					// GRV Proxy returns an error
+					wait(delayJittered(CLIENT_KNOBS->GRV_ERROR_RETRY_DELAY));
+				} else {
+					throw;
 				}
 			}
 		}
@@ -770,7 +795,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self,
 	// Fetch cipher keys if any of the messages are encrypted.
 	if (!cipherDetails.empty()) {
 		std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> getCipherKeysResult =
-		    wait(getEncryptCipherKeys(self->db, cipherDetails));
+		    wait(getEncryptCipherKeys(self->db, cipherDetails, BlobCipherMetrics::BLOB_GRANULE));
 		cipherKeys = getCipherKeysResult;
 	}
 
@@ -778,7 +803,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self,
 	for (idx = 0; idx < numMsg; idx++) {
 		auto& message = self->messages[idx];
 		MutationRef m;
-		if (!message.isBackupMessage(&m, cipherKeys))
+		if (!message.isCandidateBackupMessage(&m, cipherKeys))
 			continue;
 
 		DEBUG_MUTATION("addMutation", message.version.version, m)
@@ -1058,7 +1083,7 @@ ACTOR static Future<Void> monitorWorkerPause(BackupData* self) {
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
 			Optional<Value> value = wait(tr->get(backupPausedKey));
-			bool paused = value.present() && value.get() == LiteralStringRef("1");
+			bool paused = value.present() && value.get() == "1"_sr;
 			if (self->paused.get() != paused) {
 				TraceEvent(paused ? "BackupWorkerPaused" : "BackupWorkerResumed", self->myId).log();
 				self->paused.set(paused);
diff --git a/fdbserver/BlobGranuleServerCommon.actor.cpp b/fdbserver/BlobGranuleServerCommon.actor.cpp
index d948c16c5f..750eb1592d 100644
--- a/fdbserver/BlobGranuleServerCommon.actor.cpp
+++ b/fdbserver/BlobGranuleServerCommon.actor.cpp
@@ -30,6 +30,18 @@
 #include "flow/UnitTest.h"
 #include "flow/actorcompiler.h" // has to be last include
 
+// serialize change feed key as UID bytes, to use 16 bytes on disk
+Key granuleIDToCFKey(UID granuleID) {
+	BinaryWriter wr(Unversioned());
+	wr << granuleID;
+	return wr.toValue();
+}
+
+// parse change feed key back to UID, to be human-readable
+UID cfKeyToGranuleID(Key cfKey) {
+	return BinaryReader::fromStringRef<UID>(cfKey, Unversioned());
+}
+
 // Gets the latest granule history node for range that was persisted
 ACTOR Future<Optional<GranuleHistory>> getLatestGranuleHistory(Transaction* tr, KeyRange range) {
 	state KeyRange historyRange = blobGranuleHistoryKeyRangeFor(range);
@@ -100,6 +112,7 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Database cx, UID granuleID) {
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID));
 			return files;
 		} catch (Error& e) {
@@ -108,6 +121,20 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Database cx, UID granuleID) {
 	}
 }
 
+ACTOR Future<ForcedPurgeState> getForcePurgedState(Transaction* tr, KeyRange keyRange) {
+	// because map is coalesced, if the result returns more than 1, they must be alternating
+	RangeResult values =
+	    wait(krmGetRanges(tr, blobGranuleForcePurgedKeys.begin, keyRange, 3, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+
+	ASSERT(!values.empty());
+	if (values.size() > 2) {
+		ASSERT(values[0].value != values[1].value);
+		return ForcedPurgeState::SomePurged;
+	} else {
+		return values[0].value == "1"_sr ? ForcedPurgeState::AllPurged : ForcedPurgeState::NonePurged;
+	}
+}
+
 // Normally a beginVersion != 0 means the caller wants all mutations between beginVersion and readVersion, instead of
 // the latest snapshot before readVersion + deltas after the snapshot. When canCollapse is set, the beginVersion is
 // essentially just an optimization hint. The caller is still concerned with reconstructing rows at readVersion, it just
@@ -116,14 +143,19 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Database cx, UID granuleID) {
 // key range, the granule may have a snapshot file at version X, where beginVersion < X <= readVersion. In this case, if
 // the number of bytes in delta files between beginVersion and X is larger than the snapshot file at version X, it is
 // strictly more efficient (in terms of files and bytes read) to just use the snapshot file at version X instead.
+//
+// To assist BlobGranule file (snapshot and/or delta) file encryption, the routine while populating snapshot and/or
+// delta files, constructs BlobFilePointerRef->cipherKeysMeta field. Approach avoids this method to be defined as an
+// ACTOR, as fetching desired EncryptionKey may potentially involve reaching out to EncryptKeyProxy or external KMS.
 void GranuleFiles::getFiles(Version beginVersion,
                             Version readVersion,
                             bool canCollapse,
                             BlobGranuleChunkRef& chunk,
                             Arena& replyArena,
-                            int64_t& deltaBytesCounter) const {
+                            int64_t& deltaBytesCounter,
+                            bool summarize) const {
 	BlobFileIndex dummyIndex; // for searching
-
+	ASSERT(!snapshotFiles.empty());
 	// if beginVersion == 0 or we can collapse, find the latest snapshot <= readVersion
 	auto snapshotF = snapshotFiles.end();
 	if (beginVersion == 0 || canCollapse) {
@@ -171,32 +203,43 @@ void GranuleFiles::getFiles(Version beginVersion,
 	Version lastIncluded = invalidVersion;
 	if (snapshotF != snapshotFiles.end()) {
 		chunk.snapshotVersion = snapshotF->version;
-		chunk.snapshotFile = BlobFilePointerRef(replyArena,
-		                                        snapshotF->filename,
-		                                        snapshotF->offset,
-		                                        snapshotF->length,
-		                                        snapshotF->fullFileLength,
-		                                        snapshotF->cipherKeysMeta);
+		chunk.snapshotFile =
+		    BlobFilePointerRef(replyArena,
+		                       summarize ? "" : snapshotF->filename,
+		                       snapshotF->offset,
+		                       snapshotF->length,
+		                       snapshotF->fullFileLength,
+		                       summarize ? Optional<BlobGranuleCipherKeysMeta>() : snapshotF->cipherKeysMeta);
 		lastIncluded = chunk.snapshotVersion;
 	} else {
 		chunk.snapshotVersion = invalidVersion;
 	}
 
 	while (deltaF != deltaFiles.end() && deltaF->version < readVersion) {
-		chunk.deltaFiles.emplace_back_deep(
-		    replyArena, deltaF->filename, deltaF->offset, deltaF->length, deltaF->fullFileLength);
+		chunk.deltaFiles.emplace_back_deep(replyArena,
+		                                   summarize ? "" : deltaF->filename,
+		                                   deltaF->offset,
+		                                   deltaF->length,
+		                                   deltaF->fullFileLength,
+		                                   summarize ? Optional<BlobGranuleCipherKeysMeta>() : deltaF->cipherKeysMeta);
 		deltaBytesCounter += deltaF->length;
 		ASSERT(lastIncluded < deltaF->version);
 		lastIncluded = deltaF->version;
 		deltaF++;
 	}
 	// include last delta file that passes readVersion, if it exists
-	if (deltaF != deltaFiles.end() && lastIncluded < readVersion) {
-		chunk.deltaFiles.emplace_back_deep(
-		    replyArena, deltaF->filename, deltaF->offset, deltaF->length, deltaF->fullFileLength);
+	if (deltaF != deltaFiles.end() &&
+	    ((!summarize && lastIncluded < readVersion) || (summarize && deltaF->version == readVersion))) {
+		chunk.deltaFiles.emplace_back_deep(replyArena,
+		                                   deltaF->filename,
+		                                   deltaF->offset,
+		                                   deltaF->length,
+		                                   deltaF->fullFileLength,
+		                                   deltaF->cipherKeysMeta);
 		deltaBytesCounter += deltaF->length;
 		lastIncluded = deltaF->version;
 	}
+	chunk.includedVersion = lastIncluded;
 }
 
 static std::string makeTestFileName(Version v) {
@@ -220,7 +263,7 @@ static void checkFiles(const GranuleFiles& f,
 	Arena a;
 	BlobGranuleChunkRef chunk;
 	int64_t deltaBytes = 0;
-	f.getFiles(beginVersion, readVersion, canCollapse, chunk, a, deltaBytes);
+	f.getFiles(beginVersion, readVersion, canCollapse, chunk, a, deltaBytes, false);
 	fmt::print("results({0}, {1}, {2}):\nEXPECTED:\n    snapshot={3}\n    deltas ({4}):\n",
 	           beginVersion,
 	           readVersion,
@@ -364,6 +407,49 @@ TEST_CASE("/blobgranule/server/common/granulefiles") {
 	return Void();
 }
 
+static void checkSummary(const GranuleFiles& f,
+                         Version summaryVersion,
+                         Version expectedSnapshotVersion,
+                         int64_t expectedSnapshotSize,
+                         Version expectedDeltaVersion,
+                         Version expectedDeltaSize) {
+	Arena fileArena, summaryArena;
+	BlobGranuleChunkRef chunk;
+	int64_t deltaBytes = 0;
+	f.getFiles(0, summaryVersion, true, chunk, fileArena, deltaBytes, true);
+
+	BlobGranuleSummaryRef summary = summarizeGranuleChunk(summaryArena, chunk);
+
+	ASSERT(expectedSnapshotVersion == summary.snapshotVersion);
+	ASSERT(expectedSnapshotSize == summary.snapshotSize);
+	ASSERT(expectedDeltaVersion == summary.deltaVersion);
+	ASSERT(expectedDeltaSize == summary.deltaSize);
+	ASSERT(deltaBytes == expectedDeltaSize);
+}
+
+/*
+ * This should technically be in client unit tests but we don't have a unit test there
+ * Files:
+ * S @ 100 (10 bytes)
+ * D @ 150 (5 bytes)
+ * D @ 200 (6 bytes)
+ */
+TEST_CASE("/blobgranule/server/common/granulesummary") {
+	GranuleFiles files;
+	files.snapshotFiles.push_back(makeTestFile(100, 10));
+	files.deltaFiles.push_back(makeTestFile(150, 5));
+	files.deltaFiles.push_back(makeTestFile(200, 6));
+
+	checkSummary(files, 100, 100, 10, 100, 0);
+	checkSummary(files, 149, 100, 10, 100, 0);
+	checkSummary(files, 150, 100, 10, 150, 5);
+	checkSummary(files, 199, 100, 10, 150, 5);
+	checkSummary(files, 200, 100, 10, 200, 11);
+	checkSummary(files, 700, 100, 10, 200, 11);
+
+	return Void();
+}
+
 // FIXME: if credentials can expire, refresh periodically
 ACTOR Future<Void> loadBlobMetadataForTenants(BGTenantMap* self, std::vector<TenantMapEntry> tenantMapEntries) {
 	ASSERT(SERVER_KNOBS->BG_METADATA_SOURCE == "tenant");
diff --git a/fdbserver/BlobGranuleValidation.actor.cpp b/fdbserver/BlobGranuleValidation.actor.cpp
index 6bc4d4626b..56f72ac63c 100644
--- a/fdbserver/BlobGranuleValidation.actor.cpp
+++ b/fdbserver/BlobGranuleValidation.actor.cpp
@@ -115,23 +115,34 @@ bool compareFDBAndBlob(RangeResult fdb,
 			Optional<KeyValueRef> lastCorrect;
 			for (int i = 0; i < std::max(fdb.size(), blob.first.size()); i++) {
 				if (i >= fdb.size() || i >= blob.first.size() || fdb[i] != blob.first[i]) {
+					TraceEvent ev("GranuleMismatchInfo");
+					ev.detail("Idx", i);
 					printf("  Found mismatch at %d.\n", i);
 					if (lastCorrect.present()) {
 						printf("    last correct: %s=%s\n",
 						       lastCorrect.get().key.printable().c_str(),
 						       lastCorrect.get().value.printable().c_str());
+						ev.detail("LastCorrectKey", lastCorrect.get().key);
 					}
 					if (i < fdb.size()) {
 						printf("    FDB: %s=%s\n", fdb[i].key.printable().c_str(), fdb[i].value.printable().c_str());
+						ev.detail("FDBKey", fdb[i].key);
 					} else {
 						printf("    FDB: <missing>\n");
+						ev.detail("FDBKey", "Missing");
 					}
 					if (i < blob.first.size()) {
 						printf("    BLB: %s=%s\n",
 						       blob.first[i].key.printable().c_str(),
 						       blob.first[i].value.printable().c_str());
+						ev.detail("BlobKey", blob.first[i].key);
 					} else {
 						printf("    BLB: <missing>\n");
+						ev.detail("BlobKey", "Missing");
+					}
+					if (i < fdb.size() && i < blob.first.size() && fdb[i].key == blob.first[i].key) {
+						// value mismatch
+						ev.detail("FDBValue", fdb[i].value).detail("BlobValue", blob.first[i].value);
 					}
 					printf("\n");
 					break;
@@ -143,26 +154,181 @@ bool compareFDBAndBlob(RangeResult fdb,
 				}
 			}
 
-			printf("Chunks:\n");
-			for (auto& chunk : blob.second) {
-				printf("[%s - %s)\n", chunk.keyRange.begin.printable().c_str(), chunk.keyRange.end.printable().c_str());
-
-				printf("  SnapshotFile:\n    %s\n",
-				       chunk.snapshotFile.present() ? chunk.snapshotFile.get().toString().c_str() : "<none>");
-				printf("  DeltaFiles:\n");
-				for (auto& df : chunk.deltaFiles) {
-					printf("    %s\n", df.toString().c_str());
-				}
-				printf("  Deltas: (%d)", chunk.newDeltas.size());
-				if (chunk.newDeltas.size() > 0) {
-					fmt::print(" with version [{0} - {1}]",
-					           chunk.newDeltas[0].version,
-					           chunk.newDeltas[chunk.newDeltas.size() - 1].version);
-				}
-				fmt::print("  IncludedVersion: {}\n", chunk.includedVersion);
-			}
-			printf("\n");
+			printGranuleChunks(blob.second);
 		}
 	}
 	return correct;
-}
\ No newline at end of file
+}
+
+void printGranuleChunks(const Standalone<VectorRef<BlobGranuleChunkRef>>& chunks) {
+	printf("Chunks:\n");
+	for (auto& chunk : chunks) {
+		printf("[%s - %s)\n", chunk.keyRange.begin.printable().c_str(), chunk.keyRange.end.printable().c_str());
+
+		printf("  SnapshotFile:\n    %s\n",
+		       chunk.snapshotFile.present() ? chunk.snapshotFile.get().toString().c_str() : "<none>");
+		printf("  DeltaFiles:\n");
+		for (auto& df : chunk.deltaFiles) {
+			printf("    %s\n", df.toString().c_str());
+		}
+		printf("  Deltas: (%d)", chunk.newDeltas.size());
+		if (chunk.newDeltas.size() > 0) {
+			fmt::print(" with version [{0} - {1}]",
+			           chunk.newDeltas[0].version,
+			           chunk.newDeltas[chunk.newDeltas.size() - 1].version);
+		}
+		fmt::print("  IncludedVersion: {}\n", chunk.includedVersion);
+	}
+	printf("\n");
+}
+
+ACTOR Future<Void> clearAndAwaitMerge(Database cx, KeyRange range) {
+	// clear key range and check whether it is merged or not, repeatedly
+	state Transaction tr(cx);
+	state int reClearCount = 1;
+	state int reClearInterval = 1; // do quadratic backoff on clear rate, b/c large keys can keep it not write-cold
+	loop {
+		try {
+			Standalone<VectorRef<KeyRangeRef>> ranges = wait(tr.getBlobGranuleRanges(range, 2));
+			if (ranges.size() == 1) {
+				return Void();
+			}
+			CODE_PROBE(true, "ClearAndAwaitMerge doing clear");
+			reClearCount--;
+			if (reClearCount <= 0) {
+				tr.clear(range);
+				wait(tr.commit());
+				fmt::print("ClearAndAwaitMerge cleared [{0} - {1}) @ {2}\n",
+				           range.begin.printable(),
+				           range.end.printable(),
+				           tr.getCommittedVersion());
+				reClearCount = reClearInterval;
+				reClearInterval++;
+			}
+			wait(delay(30.0)); // sleep a bit before checking on merge again
+			tr.reset();
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+ACTOR Future<Standalone<VectorRef<BlobGranuleSummaryRef>>> getSummaries(Database cx,
+                                                                        KeyRange range,
+                                                                        Version summaryVersion,
+                                                                        Optional<TenantName> tenantName) {
+	state Transaction tr(cx, tenantName);
+	loop {
+		try {
+			Standalone<VectorRef<BlobGranuleSummaryRef>> summaries =
+			    wait(tr.summarizeBlobGranules(range, summaryVersion, 1000000));
+
+			// do some basic validation
+			ASSERT(!summaries.empty());
+			ASSERT(summaries.front().keyRange.begin == range.begin);
+			ASSERT(summaries.back().keyRange.end == range.end);
+
+			for (int i = 0; i < summaries.size() - 1; i++) {
+				ASSERT(summaries[i].keyRange.end == summaries[i + 1].keyRange.begin);
+			}
+
+			return summaries;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+ACTOR Future<Void> validateGranuleSummaries(Database cx,
+                                            KeyRange range,
+                                            Optional<TenantName> tenantName,
+                                            Promise<Void> testComplete) {
+	state Arena lastSummaryArena;
+	state KeyRangeMap<Optional<BlobGranuleSummaryRef>> lastSummary;
+	state Version lastSummaryVersion = invalidVersion;
+	state Transaction tr(cx, tenantName);
+	state int successCount = 0;
+	try {
+		loop {
+			// get grv and get latest summaries
+			state Version nextSummaryVersion;
+			tr.reset();
+			loop {
+				try {
+					wait(store(nextSummaryVersion, tr.getReadVersion()));
+					ASSERT(nextSummaryVersion >= lastSummaryVersion);
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+
+			state Standalone<VectorRef<BlobGranuleSummaryRef>> nextSummary;
+			try {
+				wait(store(nextSummary, getSummaries(cx, range, nextSummaryVersion, tenantName)));
+			} catch (Error& e) {
+				if (e.code() == error_code_blob_granule_transaction_too_old) {
+					ASSERT(lastSummaryVersion == invalidVersion);
+
+					wait(delay(1.0));
+					continue;
+				} else {
+					throw e;
+				}
+			}
+
+			if (lastSummaryVersion != invalidVersion) {
+				CODE_PROBE(true, "comparing multiple summaries");
+				// diff with last summary ranges to ensure versions never decreased for any range
+				for (auto& it : nextSummary) {
+					auto lastSummaries = lastSummary.intersectingRanges(it.keyRange);
+					for (auto& itLast : lastSummaries) {
+
+						if (!itLast.cvalue().present()) {
+							ASSERT(lastSummaryVersion == invalidVersion);
+							continue;
+						}
+						auto& last = itLast.cvalue().get();
+
+						ASSERT(it.snapshotVersion >= last.snapshotVersion);
+						// same invariant isn't always true for delta version because of force flushing around granule
+						// merges
+						if (it.keyRange == itLast.range()) {
+							ASSERT(it.deltaVersion >= last.deltaVersion);
+							if (it.snapshotVersion == last.snapshotVersion) {
+								ASSERT(it.snapshotSize == last.snapshotSize);
+							}
+							if (it.snapshotVersion == last.snapshotVersion && it.deltaVersion == last.deltaVersion) {
+								ASSERT(it.snapshotSize == last.snapshotSize);
+								ASSERT(it.deltaSize == last.deltaSize);
+							} else if (it.snapshotVersion == last.snapshotVersion) {
+								ASSERT(it.deltaSize > last.deltaSize);
+							}
+							break;
+						}
+					}
+				}
+
+				if (!testComplete.canBeSet()) {
+					return Void();
+				}
+			}
+
+			successCount++;
+
+			lastSummaryArena = nextSummary.arena();
+			lastSummaryVersion = nextSummaryVersion;
+			lastSummary.insert(range, {});
+			for (auto& it : nextSummary) {
+				lastSummary.insert(it.keyRange, it);
+			}
+
+			wait(delayJittered(deterministicRandom()->randomInt(1, 10)));
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_operation_cancelled) {
+			TraceEvent(SevError, "UnexpectedErrorValidateGranuleSummaries").error(e);
+		}
+		throw e;
+	}
+}
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 605b95e91e..a6b064fe52 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -25,6 +25,8 @@
 #include <vector>
 #include <unordered_map>
 
+#include "fdbclient/ServerKnobs.h"
+#include "fdbrpc/simulator.h"
 #include "fmt/format.h"
 #include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbclient/BlobGranuleCommon.h"
@@ -34,6 +36,7 @@
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/SystemData.h"
+#include "fdbclient/Tuple.h"
 #include "fdbserver/BlobManagerInterface.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/BlobGranuleValidation.actor.h"
@@ -51,6 +54,7 @@
  */
 
 #define BM_DEBUG false
+#define BM_PURGE_DEBUG false
 
 void handleClientBlobRange(KeyRangeMap<bool>* knownBlobRanges,
                            Arena& ar,
@@ -127,7 +131,7 @@ void updateClientBlobRanges(KeyRangeMap<bool>* knownBlobRanges,
 				}
 				break;
 			}
-			bool active = dbBlobRanges[i].value == LiteralStringRef("1");
+			bool active = dbBlobRanges[i].value == blobRangeActive;
 			if (active) {
 				if (BM_DEBUG) {
 					fmt::print("BM sees client range [{0} - {1})\n",
@@ -191,6 +195,7 @@ struct RangeAssignment {
 	bool isAssign;
 	KeyRange keyRange;
 	Optional<UID> worker;
+	Optional<std::pair<UID, Error>> previousFailure;
 
 	// I tried doing this with a union and it was just kind of messy
 	Optional<RangeAssignmentData> assign;
@@ -256,6 +261,7 @@ struct BlobManagerStats {
 
 	Counter granuleSplits;
 	Counter granuleWriteHotSplits;
+	Counter granuleMerges;
 	Counter ccGranulesChecked;
 	Counter ccRowsChecked;
 	Counter ccBytesChecked;
@@ -267,20 +273,76 @@ struct BlobManagerStats {
 	Counter granulesPartiallyPurged;
 	Counter filesPurged;
 	Future<Void> logger;
+	int64_t activeMerges;
+	int64_t blockedAssignments;
 
 	// Current stats maintained for a given blob worker process
-	explicit BlobManagerStats(UID id, double interval, std::unordered_map<UID, BlobWorkerInterface>* workers)
+	explicit BlobManagerStats(UID id,
+	                          double interval,
+	                          int64_t epoch,
+	                          std::unordered_map<UID, BlobWorkerInterface>* workers,
+	                          std::unordered_map<Key, bool>* mergeHardBoundaries,
+	                          std::unordered_map<Key, BlobGranuleMergeBoundary>* mergeBoundaries)
 	  : cc("BlobManagerStats", id.toString()), granuleSplits("GranuleSplits", cc),
-	    granuleWriteHotSplits("GranuleWriteHotSplits", cc), ccGranulesChecked("CCGranulesChecked", cc),
-	    ccRowsChecked("CCRowsChecked", cc), ccBytesChecked("CCBytesChecked", cc), ccMismatches("CCMismatches", cc),
-	    ccTimeouts("CCTimeouts", cc), ccErrors("CCErrors", cc), purgesProcessed("PurgesProcessed", cc),
+	    granuleWriteHotSplits("GranuleWriteHotSplits", cc), granuleMerges("GranuleMerges", cc),
+	    ccGranulesChecked("CCGranulesChecked", cc), ccRowsChecked("CCRowsChecked", cc),
+	    ccBytesChecked("CCBytesChecked", cc), ccMismatches("CCMismatches", cc), ccTimeouts("CCTimeouts", cc),
+	    ccErrors("CCErrors", cc), purgesProcessed("PurgesProcessed", cc),
 	    granulesFullyPurged("GranulesFullyPurged", cc), granulesPartiallyPurged("GranulesPartiallyPurged", cc),
-	    filesPurged("FilesPurged", cc) {
+	    filesPurged("FilesPurged", cc), activeMerges(0), blockedAssignments(0) {
 		specialCounter(cc, "WorkerCount", [workers]() { return workers->size(); });
+		specialCounter(cc, "Epoch", [epoch]() { return epoch; });
+		specialCounter(cc, "ActiveMerges", [this]() { return this->activeMerges; });
+		specialCounter(cc, "HardBoundaries", [mergeHardBoundaries]() { return mergeHardBoundaries->size(); });
+		specialCounter(cc, "SoftBoundaries", [mergeBoundaries]() { return mergeBoundaries->size(); });
+		specialCounter(cc, "BlockedAssignments", [this]() { return this->blockedAssignments; });
 		logger = traceCounters("BlobManagerMetrics", id, interval, &cc, "BlobManagerMetrics");
 	}
 };
 
+enum MergeCandidateState {
+	MergeCandidateUnknown,
+	MergeCandidateCannotMerge,
+	MergeCandidateCanMerge,
+	MergeCandidateMerging
+};
+
+// The current merge algorithm, skipping just granules that will be merge-eligible on the next pass, but not
+// their neighbors, is optimal for guaranteeing merges to make progress where possible, with decently
+// optimal but not globally optimal merge behavior.
+// Alternative algorithms include not doing a two-pass consideration at all and immediately considering
+// all merge candidates, which guarantees the most progress but pretty much guarantees undesirably
+// suboptimal merge decisions, because of the time variance of granules becoming merge candidates. Or,
+// also skipping adjacent eligible granules in addition to the one that will be eligible next pass,
+// which ensures optimally large merges in a future pass, but adds decent delay to doing the merge. Or,
+// smarter considering of merge candidates adjacent to the one that will be eligible next pass
+// (depending on whether potential future merges with adjacent ones could include this candidate), which
+// would be the best of both worlds, but would add a decent amount of code complexity.
+struct MergeCandidateInfo {
+	MergeCandidateState st;
+	UID granuleID;
+	Version startVersion;
+	// This is if this candidate has been seen by the merge checker before.
+	bool seen;
+
+	MergeCandidateInfo() : st(MergeCandidateUnknown), startVersion(invalidVersion), seen(false) {}
+
+	MergeCandidateInfo(MergeCandidateState st) : st(st), startVersion(invalidVersion), seen(false) {
+		ASSERT(st != MergeCandidateCanMerge);
+	}
+	MergeCandidateInfo(UID granuleID, Version startVersion)
+	  : st(MergeCandidateCanMerge), granuleID(granuleID), startVersion(startVersion), seen(false) {}
+
+	bool canMerge() const { return st == MergeCandidateCanMerge; }
+
+	bool mergeEligible() const { return st == MergeCandidateCanMerge && seen; }
+};
+
+struct BlobGranuleSplitPoints {
+	Standalone<VectorRef<KeyRef>> keys;
+	std::unordered_map<Key, BlobGranuleMergeBoundary> boundaries;
+};
+
 struct BlobManagerData : NonCopyable, ReferenceCounted<BlobManagerData> {
 	UID id;
 	Database db;
@@ -301,10 +363,16 @@ struct BlobManagerData : NonCopyable, ReferenceCounted<BlobManagerData> {
 	KeyRangeMap<BoundaryEvaluation> boundaryEvaluations;
 	KeyRangeMap<bool> knownBlobRanges;
 	BGTenantMap tenantData;
-	KeyRangeMap<Optional<std::pair<UID, Version>>> mergeCandidates; // granule range to granule id + start version.
+	KeyRangeMap<MergeCandidateInfo> mergeCandidates; // granule range to granule id + start version.
 	KeyRangeMap<Version> activeGranuleMerges; // range map of active granule merges, because range in boundaryEval
 	                                          // doesn't correspond to merge range. invalidVersion is no merge,
 	                                          // 0 is no merge version determined yet
+	// TODO: consider switching to an iterator approach.
+	std::unordered_map<Key, bool> mergeHardBoundaries;
+	std::unordered_map<Key, BlobGranuleMergeBoundary> mergeBoundaries;
+	CoalescedKeyRangeMap<bool> forcePurgingRanges;
+
+	FlowLock concurrentMergeChecks;
 
 	AsyncTrigger startRecruiting;
 	Debouncer restartRecruiting;
@@ -312,18 +380,25 @@ struct BlobManagerData : NonCopyable, ReferenceCounted<BlobManagerData> {
 	AsyncVar<int> recruitingStream;
 	Promise<Void> foundBlobWorkers;
 	Promise<Void> doneRecovering;
+	Promise<Void> loadedClientRanges;
 
-	int64_t epoch = -1;
+	int64_t epoch;
 	int64_t seqNo = 1;
 
 	Promise<Void> iAmReplaced;
 
-	BlobManagerData(UID id, Reference<AsyncVar<ServerDBInfo> const> dbInfo, Database db, Optional<Key> dcId)
-	  : id(id), db(db), dcId(dcId), stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &workersById),
+	BlobManagerData(UID id,
+	                Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+	                Database db,
+	                Optional<Key> dcId,
+	                int64_t epoch)
+	  : id(id), db(db), dcId(dcId),
+	    stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, epoch, &workersById, &mergeHardBoundaries, &mergeBoundaries),
 	    knownBlobRanges(false, normalKeys.end), tenantData(BGTenantMap(dbInfo)),
-	    mergeCandidates(Optional<std::pair<UID, Version>>(), normalKeys.end),
-	    activeGranuleMerges(invalidVersion, normalKeys.end), restartRecruiting(SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY),
-	    recruitingStream(0) {}
+	    mergeCandidates(MergeCandidateInfo(MergeCandidateUnknown), normalKeys.end),
+	    activeGranuleMerges(invalidVersion, normalKeys.end), forcePurgingRanges(false, normalKeys.end),
+	    concurrentMergeChecks(SERVER_KNOBS->BLOB_MANAGER_CONCURRENT_MERGE_CHECKS),
+	    restartRecruiting(SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY), recruitingStream(0), epoch(epoch) {}
 
 	// only initialize blob store if actually needed
 	void initBStore() {
@@ -347,6 +422,7 @@ struct BlobManagerData : NonCopyable, ReferenceCounted<BlobManagerData> {
 		}
 		return false;
 	}
+
 	Version activeMergeVersion(const KeyRangeRef& range) {
 		auto ranges = activeGranuleMerges.intersectingRanges(range);
 		Version v = invalidVersion;
@@ -355,12 +431,159 @@ struct BlobManagerData : NonCopyable, ReferenceCounted<BlobManagerData> {
 		}
 		return v;
 	}
+
+	// FIXME: is it possible for merge/split/re-merge to call this with same range but a different granule id or
+	// startVersion? Unlikely but could cause weird history problems
+	void setMergeCandidate(const KeyRangeRef& range, UID granuleID, Version startVersion) {
+		// if this granule is not an active granule, it can't be merged
+		auto gIt = workerAssignments.rangeContaining(range.begin);
+		if (gIt->begin() != range.begin || gIt->end() != range.end) {
+			CODE_PROBE(true, "non-active granule reported merge eligible, ignoring");
+			if (BM_DEBUG) {
+				fmt::print(
+				    "BM {0} Ignoring Merge Candidate [{1} - {2}): range mismatch with active granule [{3} - {4})\n",
+				    epoch,
+				    range.begin.printable(),
+				    range.end.printable(),
+				    gIt->begin().printable(),
+				    gIt->end().printable());
+			}
+			return;
+		}
+		// Want this to be idempotent. If a granule was already reported as merge-eligible, we want to use the existing
+		// merge and mergeNow state.
+		auto it = mergeCandidates.rangeContaining(range.begin);
+
+		if (it->begin() == range.begin && it.end() == range.end) {
+			if (it->cvalue().st != MergeCandidateCanMerge) {
+				// same range, just update
+				it->value() = MergeCandidateInfo(granuleID, startVersion);
+			} else {
+				// else no-op, but validate data
+				ASSERT(granuleID == it->cvalue().granuleID);
+				ASSERT(startVersion == it->cvalue().startVersion);
+			}
+		} else if (it->cvalue().st != MergeCandidateMerging) {
+			mergeCandidates.insert(range, MergeCandidateInfo(granuleID, startVersion));
+		}
+	}
+
+	void setMergeCandidate(const KeyRangeRef& range, MergeCandidateState st) {
+		ASSERT(st != MergeCandidateCanMerge);
+		mergeCandidates.insert(range, MergeCandidateInfo(st));
+	}
+
+	void clearMergeCandidate(const KeyRangeRef& range) { setMergeCandidate(range, MergeCandidateCannotMerge); }
+
+	bool isForcePurging(const KeyRangeRef& range) {
+		auto ranges = forcePurgingRanges.intersectingRanges(range);
+		for (auto& it : ranges) {
+			if (it.value()) {
+				return true;
+			}
+		}
+		return false;
+	}
 };
 
-ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<BlobManagerData> bmData,
-                                                       KeyRange range,
-                                                       bool writeHot,
-                                                       bool initialSplit) {
+// Helper function for alignKeys().
+// This attempts to do truncation and compares with the last key in splitPoints.keys.
+static void alignKeyBoundary(Reference<BlobManagerData> bmData,
+                             Reference<GranuleTenantData> tenantData,
+                             KeyRef key,
+                             int offset,
+                             BlobGranuleSplitPoints& splitPoints) {
+	Standalone<VectorRef<KeyRef>>& keys = splitPoints.keys;
+	std::unordered_map<Key, BlobGranuleMergeBoundary>& boundaries = splitPoints.boundaries;
+	KeyRef alignedKey = key;
+	Tuple t, t2;
+
+	if (!offset) {
+		keys.push_back_deep(keys.arena(), alignedKey);
+		return;
+	}
+
+	// If this is tenant aware code.
+	if (tenantData.isValid()) {
+		alignedKey = alignedKey.removePrefix(tenantData->entry.prefix);
+	}
+	try {
+		t = Tuple::unpackUserType(alignedKey, true);
+		if (t.size() > offset) {
+			t2 = t.subTuple(0, t.size() - offset);
+			alignedKey = t2.pack();
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_invalid_tuple_data_type) {
+			throw;
+		}
+	}
+	if (tenantData.isValid()) {
+		alignedKey = alignedKey.withPrefix(tenantData->entry.prefix, keys.arena());
+	}
+
+	// Only add the alignedKey if it's larger than the last key. If it's the same, drop the split.
+	if (alignedKey <= keys.back()) {
+		// Set split boundary.
+		BlobGranuleMergeBoundary boundary = { /*buddy=*/true };
+		boundaries[key] = boundary;
+		keys.push_back_deep(keys.arena(), key);
+	} else {
+		keys.push_back_deep(keys.arena(), alignedKey);
+	}
+}
+
+ACTOR Future<BlobGranuleSplitPoints> alignKeys(Reference<BlobManagerData> bmData,
+                                               KeyRange granuleRange,
+                                               Standalone<VectorRef<KeyRef>> splits) {
+	state BlobGranuleSplitPoints splitPoints;
+
+	state int offset = SERVER_KNOBS->BG_KEY_TUPLE_TRUNCATE_OFFSET;
+	if (offset <= 0) {
+		splitPoints.keys = splits;
+		return splitPoints;
+	}
+
+	splitPoints.keys.push_back_deep(splitPoints.keys.arena(), splits.front());
+
+	state Transaction tr = Transaction(bmData->db);
+	state int idx = 1;
+	state Reference<GranuleTenantData> tenantData = bmData->tenantData.getDataForGranule(granuleRange);
+	while (SERVER_KNOBS->BG_METADATA_SOURCE == "tenant" && !tenantData.isValid()) {
+		// this is a bit of a hack, but if we know this range is supposed to have a tenant, and it doesn't, just wait
+		wait(delay(1.0));
+		tenantData = bmData->tenantData.getDataForGranule(granuleRange);
+	}
+	for (; idx < splits.size() - 1; idx++) {
+		loop {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			try {
+				// Get the next full key in the granule.
+				RangeResult nextKeyRes = wait(
+				    tr.getRange(firstGreaterOrEqual(splits[idx]), lastLessThan(splits[idx + 1]), GetRangeLimits(1)));
+				if (nextKeyRes.size() == 0) {
+					break;
+				}
+
+				alignKeyBoundary(bmData, tenantData, nextKeyRes[0].key, offset, splitPoints);
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	splitPoints.keys.push_back_deep(splitPoints.keys.arena(), splits.back());
+
+	return splitPoints;
+}
+
+ACTOR Future<BlobGranuleSplitPoints> splitRange(Reference<BlobManagerData> bmData,
+                                                KeyRange range,
+                                                bool writeHot,
+                                                bool initialSplit) {
+	state BlobGranuleSplitPoints splitPoints;
 	try {
 		if (BM_DEBUG) {
 			fmt::print("Splitting new range [{0} - {1}): {2}\n",
@@ -393,7 +616,7 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<BlobManagerData
 		if (writeHot) {
 			splitThreshold /= 3;
 		}
-		TEST(writeHot); // Change feed write hot split
+		CODE_PROBE(writeHot, "Change feed write hot split");
 		if (estimated.bytes > splitThreshold) {
 			// only split on bytes and write rate
 			state StorageMetrics splitMetrics;
@@ -424,19 +647,26 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<BlobManagerData
 				}
 			}
 
-			ASSERT(keys.size() >= 2);
-			ASSERT(keys.front() == range.begin);
-			ASSERT(keys.back() == range.end);
-			return keys;
+			// We only need to align the keys if there is a proposed split.
+			if (keys.size() > 2) {
+				BlobGranuleSplitPoints _splitPoints = wait(alignKeys(bmData, range, keys));
+				splitPoints = _splitPoints;
+			} else {
+				splitPoints.keys = keys;
+			}
+
+			ASSERT(splitPoints.keys.size() >= 2);
+			ASSERT(splitPoints.keys.front() == range.begin);
+			ASSERT(splitPoints.keys.back() == range.end);
+			return splitPoints;
 		} else {
-			TEST(writeHot); // Not splitting write-hot because granules would be too small
+			CODE_PROBE(writeHot, "Not splitting write-hot because granules would be too small");
 			if (BM_DEBUG) {
 				printf("Not splitting range\n");
 			}
-			Standalone<VectorRef<KeyRef>> keys;
-			keys.push_back_deep(keys.arena(), range.begin);
-			keys.push_back_deep(keys.arena(), range.end);
-			return keys;
+			splitPoints.keys.push_back_deep(splitPoints.keys.arena(), range.begin);
+			splitPoints.keys.push_back_deep(splitPoints.keys.arena(), range.end);
+			return splitPoints;
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_operation_cancelled) {
@@ -458,31 +688,70 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> splitRange(Reference<BlobManagerData
 
 // Picks a worker with the fewest number of already assigned ranges.
 // If there is a tie, picks one such worker at random.
-ACTOR Future<UID> pickWorkerForAssign(Reference<BlobManagerData> bmData) {
+ACTOR Future<UID> pickWorkerForAssign(Reference<BlobManagerData> bmData,
+                                      Optional<std::pair<UID, Error>> previousFailure) {
 	// wait until there are BWs to pick from
-	while (bmData->workerStats.size() == 0) {
-		TEST(true); // BM wants to assign range, but no workers available
-		if (BM_DEBUG) {
-			fmt::print("BM {0} waiting for blob workers before assigning granules\n", bmData->epoch);
+	loop {
+		state bool wasZeroWorkers = false;
+		while (bmData->workerStats.size() == 0) {
+			wasZeroWorkers = true;
+			CODE_PROBE(true, "BM wants to assign range, but no workers available");
+			if (BM_DEBUG) {
+				fmt::print("BM {0} waiting for blob workers before assigning granules\n", bmData->epoch);
+			}
+			bmData->restartRecruiting.trigger();
+			wait(bmData->recruitingStream.onChange() || bmData->foundBlobWorkers.getFuture());
 		}
-		bmData->restartRecruiting.trigger();
-		wait(bmData->recruitingStream.onChange() || bmData->foundBlobWorkers.getFuture());
-		// FIXME: may want to have some buffer here so zero-worker recruiting case doesn't assign every single pending
-		// range to the first worker recruited
+		if (wasZeroWorkers) {
+			// Add a bit of delay. If we were at zero workers, don't immediately assign all granules to the first worker
+			// we recruit
+			wait(delay(0.1));
+		}
+		if (bmData->workerStats.size() != 0) {
+			break;
+		}
+		// if in the post-zero workers delay, we went back down to zero workers, re-loop
 	}
 
 	int minGranulesAssigned = INT_MAX;
 	std::vector<UID> eligibleWorkers;
 
+	// because lowest number of granules worker(s) might not exactly have the lowest memory for various reasons, if we
+	// got blob_worker_full as the error last time, sometimes just pick a random worker that wasn't the last one we
+	// tried
+	if (bmData->workerStats.size() >= 2 && previousFailure.present() &&
+	    previousFailure.get().second.code() == error_code_blob_worker_full && deterministicRandom()->coinflip()) {
+		CODE_PROBE(true, "randomly picking worker due to blob_worker_full");
+		eligibleWorkers.reserve(bmData->workerStats.size());
+		for (auto& it : bmData->workerStats) {
+			if (it.first != previousFailure.get().first) {
+				eligibleWorkers.push_back(it.first);
+			}
+		}
+		ASSERT(!eligibleWorkers.empty());
+		int randomIdx = deterministicRandom()->randomInt(0, eligibleWorkers.size());
+		if (BM_DEBUG) {
+			fmt::print("picked worker {0} randomly since previous attempt got blob_worker_full\n",
+			           eligibleWorkers[randomIdx].toString().substr(0, 5));
+		}
+
+		return eligibleWorkers[randomIdx];
+	}
+
 	for (auto const& worker : bmData->workerStats) {
 		UID currId = worker.first;
 		int granulesAssigned = worker.second.numGranulesAssigned;
 
-		if (granulesAssigned < minGranulesAssigned) {
-			eligibleWorkers.resize(0);
-			minGranulesAssigned = granulesAssigned;
-			eligibleWorkers.emplace_back(currId);
-		} else if (granulesAssigned == minGranulesAssigned) {
+		// if previous attempt failed and that worker is still present, ignore it
+		if (bmData->workerStats.size() >= 2 && previousFailure.present() && previousFailure.get().first == currId) {
+			continue;
+		}
+
+		if (granulesAssigned <= minGranulesAssigned) {
+			if (granulesAssigned < minGranulesAssigned) {
+				eligibleWorkers.clear();
+				minGranulesAssigned = granulesAssigned;
+			}
 			eligibleWorkers.emplace_back(currId);
 		}
 	}
@@ -492,7 +761,7 @@ ACTOR Future<UID> pickWorkerForAssign(Reference<BlobManagerData> bmData) {
 	int idx = deterministicRandom()->randomInt(0, eligibleWorkers.size());
 	if (BM_DEBUG) {
 		fmt::print("picked worker {0}, which has a minimal number ({1}) of granules assigned\n",
-		           eligibleWorkers[idx].toString(),
+		           eligibleWorkers[idx].toString().substr(0, 5),
 		           minGranulesAssigned);
 	}
 
@@ -507,31 +776,49 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
                                      Optional<UID> workerID,
                                      int64_t epoch,
                                      int64_t seqNo) {
+	state bool blockedWaitingForWorker = false;
 	// WorkerId is set, except in case of assigning to any worker. Then we pick the worker to assign to in here
-
-	// inject delay into range assignments
-	if (BUGGIFY_WITH_PROB(0.05)) {
-		wait(delay(deterministicRandom()->random01()));
-	} else {
-		// otherwise, do delay(0) to ensure rest of code in calling handleRangeAssign runs, before this function can
-		// recursively call handleRangeAssign on error
-		wait(delay(0.0));
-	}
-
-	if (!workerID.present()) {
-		ASSERT(assignment.isAssign && assignment.assign.get().type != AssignRequestType::Continue);
-		UID _workerId = wait(pickWorkerForAssign(bmData));
-		if (BM_DEBUG) {
-			fmt::print("Chose BW {0} for seqno {1} in BM {2}\n", _workerId.toString(), seqNo, bmData->epoch);
+	try {
+		// inject delay into range assignments
+		if (BUGGIFY_WITH_PROB(0.05)) {
+			wait(delay(deterministicRandom()->random01()));
+		} else {
+			// otherwise, do delay(0) to ensure rest of code in calling handleRangeAssign runs, before this function can
+			// recursively call handleRangeAssign on error
+			wait(delay(0.0));
 		}
-		workerID = _workerId;
-		// We don't have to check for races with an overlapping assignment because it would insert over us in the actor
-		// map, cancelling this actor before it got here
-		bmData->workerAssignments.insert(assignment.keyRange, workerID.get());
+		if (!workerID.present()) {
+			ASSERT(assignment.isAssign && assignment.assign.get().type != AssignRequestType::Continue);
 
-		if (bmData->workerStats.count(workerID.get())) {
-			bmData->workerStats[workerID.get()].numGranulesAssigned += 1;
+			blockedWaitingForWorker = true;
+			if (!assignment.previousFailure.present()) {
+				// if not already blocked, now blocked
+				++bmData->stats.blockedAssignments;
+			}
+
+			UID _workerId = wait(pickWorkerForAssign(bmData, assignment.previousFailure));
+			if (BM_DEBUG) {
+				fmt::print("Chose BW {0} for seqno {1} in BM {2}\n", _workerId.toString(), seqNo, bmData->epoch);
+			}
+			workerID = _workerId;
+			// We don't have to check for races with an overlapping assignment because it would insert over us in the
+			// actor map, cancelling this actor before it got here
+			bmData->workerAssignments.insert(assignment.keyRange, workerID.get());
+
+			if (bmData->workerStats.count(workerID.get())) {
+				bmData->workerStats[workerID.get()].numGranulesAssigned += 1;
+			}
+
+			if (!assignment.previousFailure.present()) {
+				// if only blocked waiting for worker, now not blocked
+				--bmData->stats.blockedAssignments;
+			}
 		}
+	} catch (Error& e) {
+		if (assignment.previousFailure.present() || blockedWaitingForWorker) {
+			--bmData->stats.blockedAssignments;
+		}
+		throw e;
 	}
 
 	if (BM_DEBUG) {
@@ -562,6 +849,12 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 				throw no_more_servers();
 			}
 			wait(bmData->workersById[workerID.get()].assignBlobRangeRequest.getReply(req));
+			if (assignment.previousFailure.present()) {
+				// previous assign failed and this one succeeded
+				--bmData->stats.blockedAssignments;
+			}
+
+			return Void();
 		} else {
 			ASSERT(!assignment.assign.present());
 			ASSERT(assignment.revoke.present());
@@ -581,6 +874,11 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			}
 		}
 	} catch (Error& e) {
+		if (assignment.previousFailure.present()) {
+			// previous assign failed, consider it unblocked if it's not a retriable error
+			--bmData->stats.blockedAssignments;
+		}
+		state Error e2 = e;
 		if (e.code() == error_code_operation_cancelled) {
 			throw;
 		}
@@ -608,7 +906,8 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			return Void();
 		}
 
-		if (e.code() != error_code_broken_promise && e.code() != error_code_no_more_servers) {
+		if (e.code() != error_code_broken_promise && e.code() != error_code_no_more_servers &&
+		    e.code() != error_code_blob_worker_full) {
 			TraceEvent(SevWarn, "BlobManagerUnexpectedErrorDoRangeAssignment", bmData->id)
 			    .error(e)
 			    .detail("Epoch", bmData->epoch);
@@ -619,14 +918,38 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			throw;
 		}
 
-		TEST(true); // BM retrying range assign
+		// this assign failed and we will retry, consider it blocked until it successfully retries
+		if (assignment.isAssign) {
+			++bmData->stats.blockedAssignments;
+		}
+
+		if (e.code() == error_code_blob_worker_full) {
+			CODE_PROBE(true, "blob worker too full");
+			ASSERT(assignment.isAssign);
+			try {
+				if (assignment.previousFailure.present() &&
+				    assignment.previousFailure.get().second.code() == error_code_blob_worker_full) {
+					// if previous assignment also failed due to blob_worker_full, multiple workers are full, so wait
+					// even longer
+					CODE_PROBE(true, "multiple blob workers too full");
+					wait(delayJittered(10.0));
+				} else {
+					wait(delayJittered(1.0)); // wait a bit before retrying
+				}
+			} catch (Error& e) {
+				--bmData->stats.blockedAssignments;
+				throw;
+			}
+		}
+
+		CODE_PROBE(true, "BM retrying range assign");
 
 		// We use reliable delivery (getReply), so the broken_promise means the worker is dead, and we may need to retry
 		// somewhere else
 		if (assignment.isAssign) {
 			if (BM_DEBUG) {
 				fmt::print("BM got error {0} assigning range [{1} - {2}) to worker {3}, requeueing\n",
-				           e.name(),
+				           e2.name(),
 				           assignment.keyRange.begin.printable(),
 				           assignment.keyRange.end.printable(),
 				           workerID.get().toString());
@@ -647,8 +970,9 @@ ACTOR Future<Void> doRangeAssignment(Reference<BlobManagerData> bmData,
 			ASSERT(assignment.assign.present());
 			assignment.assign.get().type = AssignRequestType::Normal;
 			assignment.worker.reset();
+			std::pair<UID, Error> failure = { workerID.get(), e2 };
+			assignment.previousFailure = failure;
 			handleRangeAssign(bmData, assignment);
-			// FIXME: improvement would be to add history of failed workers to assignment so it can try other ones first
 		} else {
 			if (BM_DEBUG) {
 				fmt::print("BM got error revoking range [{0} - {1}) from worker",
@@ -683,7 +1007,7 @@ static bool handleRangeIsAssign(Reference<BlobManagerData> bmData, RangeAssignme
 		if (assignment.assign.get().type == AssignRequestType::Continue) {
 			ASSERT(assignment.worker.present());
 			if (i.range() != assignment.keyRange || i.cvalue() != assignment.worker.get()) {
-				TEST(true); // BM assignment out of date
+				CODE_PROBE(true, "BM assignment out of date");
 				if (BM_DEBUG) {
 					fmt::print("Out of date re-assign for ({0}, {1}). Assignment must have changed while "
 					           "checking split.\n  Reassign: [{2} - {3}): {4}\n  Existing: [{5} - {6}): {7}\n",
@@ -703,6 +1027,11 @@ static bool handleRangeIsAssign(Reference<BlobManagerData> bmData, RangeAssignme
 	}
 	ASSERT(count == 1);
 
+	bool forcePurging = bmData->isForcePurging(assignment.keyRange);
+
+	if (forcePurging && assignment.previousFailure.present()) {
+		--bmData->stats.blockedAssignments;
+	}
 	if (assignment.worker.present() && assignment.worker.get().isValid()) {
 		if (BM_DEBUG) {
 			fmt::print("BW {0} already chosen for seqno {1} in BM {2}\n",
@@ -720,8 +1049,10 @@ static bool handleRangeIsAssign(Reference<BlobManagerData> bmData, RangeAssignme
 			// assignsInProgress
 			bmData->addActor.send(doRangeAssignment(bmData, assignment, workerId, bmData->epoch, seqNo));
 		} else {
-			bmData->assignsInProgress.insert(assignment.keyRange,
-			                                 doRangeAssignment(bmData, assignment, workerId, bmData->epoch, seqNo));
+			if (!forcePurging) {
+				bmData->assignsInProgress.insert(assignment.keyRange,
+				                                 doRangeAssignment(bmData, assignment, workerId, bmData->epoch, seqNo));
+			}
 			if (bmData->workerStats.count(workerId)) {
 				bmData->workerStats[workerId].numGranulesAssigned += 1;
 			}
@@ -730,8 +1061,10 @@ static bool handleRangeIsAssign(Reference<BlobManagerData> bmData, RangeAssignme
 		// Ensure the key boundaries are updated before we pick a worker
 		bmData->workerAssignments.insert(assignment.keyRange, UID());
 		ASSERT(assignment.assign.get().type != AssignRequestType::Continue);
-		bmData->assignsInProgress.insert(assignment.keyRange,
-		                                 doRangeAssignment(bmData, assignment, Optional<UID>(), bmData->epoch, seqNo));
+		if (!forcePurging) {
+			bmData->assignsInProgress.insert(
+			    assignment.keyRange, doRangeAssignment(bmData, assignment, Optional<UID>(), bmData->epoch, seqNo));
+		}
 	}
 	return true;
 }
@@ -752,8 +1085,31 @@ static bool handleRangeIsRevoke(Reference<BlobManagerData> bmData, RangeAssignme
 		auto currentAssignments = bmData->workerAssignments.intersectingRanges(assignment.keyRange);
 		for (auto& it : currentAssignments) {
 			// ensure range doesn't truncate existing ranges
-			ASSERT(it.begin() >= assignment.keyRange.begin);
-			ASSERT(it.end() <= assignment.keyRange.end);
+			if (it.begin() < assignment.keyRange.begin || it.end() > assignment.keyRange.end) {
+				// the only case where this is ok is on startup when a BM is revoking old granules after reading
+				// knownBlobRanges and seeing that some are no longer present.
+				auto knownRanges = bmData->knownBlobRanges.intersectingRanges(it.range());
+				bool inKnownBlobRanges = false;
+				for (auto& r : knownRanges) {
+					if (r.value()) {
+						inKnownBlobRanges = true;
+						break;
+					}
+				}
+				bool forcePurging = bmData->isForcePurging(it.range());
+				if (it.cvalue() != UID() || (inKnownBlobRanges && !forcePurging)) {
+					fmt::print("Assignment [{0} - {1}): {2} truncates range [{3} - {4}) ({5}, {6})\n",
+					           assignment.keyRange.begin.printable(),
+					           assignment.keyRange.end.printable(),
+					           it.cvalue().toString().substr(0, 5),
+					           it.begin().printable(),
+					           it.end().printable(),
+					           inKnownBlobRanges,
+					           forcePurging);
+					// assert on condition again to make assertion failure better than "false"
+					ASSERT(it.cvalue() == UID() && (!inKnownBlobRanges || forcePurging));
+				}
+			}
 
 			// It is fine for multiple disjoint sub-ranges to have the same sequence number since they were part
 			// of the same logical change
@@ -784,7 +1140,7 @@ static bool handleRangeAssign(Reference<BlobManagerData> bmData, RangeAssignment
 	}
 }
 
-ACTOR Future<Void> checkManagerLock(Reference<ReadYourWritesTransaction> tr, Reference<BlobManagerData> bmData) {
+ACTOR Future<Void> checkManagerLock(Transaction* tr, Reference<BlobManagerData> bmData) {
 	Optional<Value> currentLockValue = wait(tr->get(blobManagerEpochKey));
 	ASSERT(currentLockValue.present());
 	int64_t currentEpoch = decodeBlobManagerEpochValue(currentLockValue.get());
@@ -807,30 +1163,51 @@ ACTOR Future<Void> checkManagerLock(Reference<ReadYourWritesTransaction> tr, Ref
 	return Void();
 }
 
-ACTOR Future<Void> writeInitialGranuleMapping(Reference<BlobManagerData> bmData,
-                                              Standalone<VectorRef<KeyRef>> boundaries) {
+ACTOR Future<Void> checkManagerLock(Reference<ReadYourWritesTransaction> tr, Reference<BlobManagerData> bmData) {
+	wait(checkManagerLock(&(tr->getTransaction()), bmData));
+	return Void();
+}
+
+ACTOR Future<Void> writeInitialGranuleMapping(Reference<BlobManagerData> bmData, BlobGranuleSplitPoints splitPoints) {
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
 	// don't do too many in one transaction
 	state int i = 0;
 	state int transactionChunkSize = BUGGIFY ? deterministicRandom()->randomInt(2, 5) : 1000;
-	while (i < boundaries.size() - 1) {
-		TEST(i > 0); // multiple transactions for large granule split
+	while (i < splitPoints.keys.size() - 1) {
+		CODE_PROBE(i > 0, "multiple transactions for large granule split");
 		tr->reset();
 		state int j = 0;
 		loop {
 			try {
-				tr->setOption(FDBTransactionOptions::Option::PRIORITY_SYSTEM_IMMEDIATE);
-				tr->setOption(FDBTransactionOptions::Option::ACCESS_SYSTEM_KEYS);
+				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				wait(checkManagerLock(tr, bmData));
-				while (i + j < boundaries.size() - 1 && j < transactionChunkSize) {
-					// set to empty UID - no worker assigned yet
-					wait(krmSetRange(tr,
-					                 blobGranuleMappingKeys.begin,
-					                 KeyRangeRef(boundaries[i + j], boundaries[i + j + 1]),
-					                 blobGranuleMappingValueFor(UID())));
-					j++;
+				// Instead of doing a krmSetRange for each granule, because it does a read-modify-write, we do one
+				// krmSetRange for the whole batch, and then just individual sets for each intermediate boundary This
+				// does one read per transaction instead of N serial reads per transaction
+				state int endIdx = std::min(i + transactionChunkSize, (int)(splitPoints.keys.size() - 1));
+				wait(krmSetRange(tr,
+				                 blobGranuleMappingKeys.begin,
+				                 KeyRangeRef(splitPoints.keys[i], splitPoints.keys[endIdx]),
+				                 blobGranuleMappingValueFor(UID())));
+				for (j = 0; i + j < endIdx; j++) {
+					if (splitPoints.boundaries.count(splitPoints.keys[i + j])) {
+						tr->set(blobGranuleMergeBoundaryKeyFor(splitPoints.keys[i + j]),
+						        blobGranuleMergeBoundaryValueFor(splitPoints.boundaries[splitPoints.keys[i + j]]));
+					}
+					tr->set(splitPoints.keys[i + j].withPrefix(blobGranuleMappingKeys.begin),
+					        blobGranuleMappingValueFor(UID()));
 				}
 				wait(tr->commit());
+
+				// Update BlobGranuleMergeBoundary in-memory state.
+				for (int k = i; k < i + j; k++) {
+					KeyRef beginKey = splitPoints.keys[k];
+					if (splitPoints.boundaries.count(beginKey)) {
+						bmData->mergeBoundaries[beginKey] = splitPoints.boundaries[beginKey];
+					}
+				}
 				break;
 			} catch (Error& e) {
 				wait(tr->onError(e));
@@ -842,23 +1219,44 @@ ACTOR Future<Void> writeInitialGranuleMapping(Reference<BlobManagerData> bmData,
 	return Void();
 }
 
+ACTOR Future<Void> loadTenantMap(Reference<ReadYourWritesTransaction> tr, Reference<BlobManagerData> bmData) {
+	state KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenantResults;
+	wait(store(tenantResults,
+	           TenantMetadata::tenantMap().getRange(
+	               tr, Optional<TenantName>(), Optional<TenantName>(), CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)));
+	ASSERT(tenantResults.results.size() <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER && !tenantResults.more);
+
+	bmData->tenantData.addTenants(tenantResults.results);
+
+	return Void();
+}
+
+ACTOR Future<Void> monitorTenants(Reference<BlobManagerData> bmData) {
+	loop {
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
+		loop {
+			try {
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+				wait(loadTenantMap(tr, bmData));
+
+				state Future<Void> watchChange = tr->watch(TenantMetadata::lastTenantId().key);
+				wait(tr->commit());
+				wait(watchChange);
+				tr->reset();
+			} catch (Error& e) {
+				wait(tr->onError(e));
+			}
+		}
+	}
+}
+
 // FIXME: better way to load tenant mapping?
 ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 	state Optional<Value> lastChangeKeyValue;
-	state Key changeKey;
 	state bool needToCoalesce = bmData->epoch > 1;
-	state std::unordered_map<int64_t, TenantMapEntry> knownTenantCache;
-
-	if (SERVER_KNOBS->BG_RANGE_SOURCE == "tenant") {
-		changeKey = tenantLastIdKey;
-	} else if (SERVER_KNOBS->BG_RANGE_SOURCE == "blobRangeKeys") {
-		changeKey = blobRangeChangeKey;
-	} else {
-		ASSERT_WE_THINK(false);
-		// wait to prevent spin looping
-		wait(delay(600.0));
-		throw internal_error();
-	}
+	state bool firstLoad = true;
 
 	loop {
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
@@ -870,70 +1268,36 @@ ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 				// read change key at this point along with data
-				state Optional<Value> ckvBegin = wait(tr->get(changeKey));
+				state Optional<Value> ckvBegin = wait(tr->get(blobRangeChangeKey));
 
-				// TODO why is there separate arena?
 				state Arena ar;
-				state RangeResult results;
-				if (SERVER_KNOBS->BG_RANGE_SOURCE == "blobRangeKeys") {
-					wait(store(results,
-					           krmGetRanges(tr,
-					                        blobRangeKeys.begin,
-					                        KeyRange(normalKeys),
-					                        CLIENT_KNOBS->TOO_MANY,
-					                        GetRangeLimits::BYTE_LIMIT_UNLIMITED)));
-					ASSERT_WE_THINK(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
-					if (results.more || results.size() >= CLIENT_KNOBS->TOO_MANY) {
-						TraceEvent(SevError, "BlobManagerTooManyClientRanges", bmData->id)
-						    .detail("Epoch", bmData->epoch)
-						    .detail("ClientRanges", results.size() - 1);
-						wait(delay(600));
-						if (bmData->iAmReplaced.canBeSet()) {
-							bmData->iAmReplaced.sendError(internal_error());
-						}
-						throw internal_error();
-					}
-
-					ar.dependsOn(results.arena());
-				} else {
-					state RangeResult tenantResults;
-					wait(store(tenantResults, tr->getRange(tenantMapKeys, CLIENT_KNOBS->TOO_MANY)));
-					ASSERT_WE_THINK(!tenantResults.more && tenantResults.size() < CLIENT_KNOBS->TOO_MANY);
-					if (tenantResults.more || tenantResults.size() >= CLIENT_KNOBS->TOO_MANY) {
-						TraceEvent(SevError, "BlobManagerTooManyTenants", bmData->id)
-						    .detail("Epoch", bmData->epoch)
-						    .detail("TenantCount", tenantResults.size());
-						wait(delay(600));
-						if (bmData->iAmReplaced.canBeSet()) {
-							bmData->iAmReplaced.sendError(internal_error());
-						}
-						throw internal_error();
-					}
-
-					std::vector<std::pair<TenantName, TenantMapEntry>> tenants;
-					std::vector<Key> prefixes;
-					for (auto& it : tenantResults) {
-						TenantNameRef tenantName = it.key.removePrefix(tenantMapPrefix);
-						TenantMapEntry entry = TenantMapEntry::decode(it.value);
-						tenants.push_back(std::pair(tenantName, entry));
-						prefixes.push_back(entry.prefix);
-					}
-					bmData->tenantData.addTenants(tenants);
-
-					// make this look like knownBlobRanges
-					std::sort(prefixes.begin(), prefixes.end());
-					for (auto& p : prefixes) {
-						if (!results.empty()) {
-							ASSERT(results.back().key < p);
-						}
-						results.push_back_deep(results.arena(), KeyValueRef(p, LiteralStringRef("1")));
-						results.push_back_deep(results.arena(),
-						                       KeyValueRef(p.withSuffix(normalKeys.end), LiteralStringRef("0")));
+				state RangeResult results = wait(krmGetRanges(tr,
+				                                              blobRangeKeys.begin,
+				                                              KeyRange(normalKeys),
+				                                              CLIENT_KNOBS->TOO_MANY,
+				                                              GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+				ASSERT_WE_THINK(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
+				if (results.more || results.size() >= CLIENT_KNOBS->TOO_MANY) {
+					TraceEvent(SevError, "BlobManagerTooManyClientRanges", bmData->id)
+					    .detail("Epoch", bmData->epoch)
+					    .detail("ClientRanges", results.size() - 1);
+					wait(delay(600));
+					if (bmData->iAmReplaced.canBeSet()) {
+						bmData->iAmReplaced.sendError(internal_error());
 					}
+					throw internal_error();
 				}
 
+				// TODO better way to do this!
+				bmData->mergeHardBoundaries.clear();
+				for (auto& it : results) {
+					bmData->mergeHardBoundaries[it.key] = true;
+				}
+				ar.dependsOn(results.arena());
+
 				VectorRef<KeyRangeRef> rangesToAdd;
 				VectorRef<KeyRangeRef> rangesToRemove;
 				updateClientBlobRanges(&bmData->knownBlobRanges, results, ar, &rangesToAdd, &rangesToRemove);
@@ -946,11 +1310,29 @@ ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 					needToCoalesce = false;
 
 					for (int i = 0; i < results.size() - 1; i++) {
-						bool active = results[i].value == LiteralStringRef("1");
+						bool active = results[i].value == blobRangeActive;
 						bmData->knownBlobRanges.insert(KeyRangeRef(results[i].key, results[i + 1].key), active);
 					}
 				}
 
+				state std::vector<Future<BlobGranuleSplitPoints>> splitFutures;
+				// Divide new ranges up into equal chunks by using SS byte sample
+				for (KeyRangeRef range : rangesToAdd) {
+					TraceEvent("ClientBlobRangeAdded", bmData->id).detail("Range", range);
+					// add client range as known "granule" until we determine initial split, in case a purge or
+					// unblobbify comes in before we finish splitting
+
+					// TODO can remove validation eventually
+					auto r = bmData->workerAssignments.intersectingRanges(range);
+					for (auto& it : r) {
+						ASSERT(it.cvalue() == UID());
+					}
+					bmData->workerAssignments.insert(range, UID());
+
+					// start initial split for range
+					splitFutures.push_back(splitRange(bmData, range, false, true));
+				}
+
 				for (KeyRangeRef range : rangesToRemove) {
 					TraceEvent("ClientBlobRangeRemoved", bmData->id).detail("Range", range);
 					if (BM_DEBUG) {
@@ -965,28 +1347,26 @@ ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 					handleRangeAssign(bmData, ra);
 				}
 
-				state std::vector<Future<Standalone<VectorRef<KeyRef>>>> splitFutures;
-				// Divide new ranges up into equal chunks by using SS byte sample
-				for (KeyRangeRef range : rangesToAdd) {
-					TraceEvent("ClientBlobRangeAdded", bmData->id).detail("Range", range);
-					splitFutures.push_back(splitRange(bmData, range, false, true));
+				if (firstLoad) {
+					bmData->loadedClientRanges.send(Void());
+					firstLoad = false;
 				}
 
 				for (auto f : splitFutures) {
-					state Standalone<VectorRef<KeyRef>> splits = wait(f);
+					state BlobGranuleSplitPoints splitPoints = wait(f);
 					if (BM_DEBUG) {
 						fmt::print("Split client range [{0} - {1}) into {2} ranges:\n",
-						           splits[0].printable(),
-						           splits[splits.size() - 1].printable(),
-						           splits.size() - 1);
+						           splitPoints.keys[0].printable(),
+						           splitPoints.keys[splitPoints.keys.size() - 1].printable(),
+						           splitPoints.keys.size() - 1);
 					}
 
 					// Write to DB BEFORE sending assign requests, so that if manager dies before/during, new manager
 					// picks up the same ranges
-					wait(writeInitialGranuleMapping(bmData, splits));
+					wait(writeInitialGranuleMapping(bmData, splitPoints));
 
-					for (int i = 0; i < splits.size() - 1; i++) {
-						KeyRange range = KeyRange(KeyRangeRef(splits[i], splits[i + 1]));
+					for (int i = 0; i < splitPoints.keys.size() - 1; i++) {
+						KeyRange range = KeyRange(KeyRangeRef(splitPoints.keys[i], splitPoints.keys[i + 1]));
 						// only add the client range if this is the first BM or it's not already assigned
 						if (BM_DEBUG) {
 							fmt::print(
@@ -1008,12 +1388,13 @@ ACTOR Future<Void> monitorClientRanges(Reference<BlobManagerData> bmData) {
 				tr->reset();
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				state Future<Void> watchFuture;
 
-				Optional<Value> ckvEnd = wait(tr->get(changeKey));
+				Optional<Value> ckvEnd = wait(tr->get(blobRangeChangeKey));
 
 				if (ckvEnd == lastChangeKeyValue) {
-					watchFuture = tr->watch(changeKey); // watch for change in key
+					watchFuture = tr->watch(blobRangeChangeKey); // watch for change in key
 					wait(tr->commit());
 					if (BM_DEBUG) {
 						printf("Blob manager done processing client ranges, awaiting update\n");
@@ -1060,6 +1441,249 @@ static void downsampleSplit(const Standalone<VectorRef<KeyRef>>& splits,
 	}
 }
 
+ACTOR Future<Void> reevaluateInitialSplit(Reference<BlobManagerData> bmData,
+                                          UID currentWorkerId,
+                                          KeyRange granuleRange,
+                                          UID granuleID,
+                                          int64_t epoch,
+                                          int64_t seqno,
+                                          Key proposedSplitKey) {
+	CODE_PROBE(true, "BM re-evaluating initial split too big");
+	if (BM_DEBUG) {
+		fmt::print("BM {0} re-evaluating initial split [{1} - {2}) too big from {3} @ ({4}, {5})\n",
+		           bmData->epoch,
+		           granuleRange.begin.printable(),
+		           granuleRange.end.printable(),
+		           currentWorkerId.toString().substr(0, 5),
+		           epoch,
+		           seqno);
+		fmt::print("Proposed split (2):\n");
+		fmt::print("    {0}\n", granuleRange.begin.printable());
+		fmt::print("    {0}\n", proposedSplitKey.printable());
+		fmt::print("    {0}\n", granuleRange.end.printable());
+	}
+	TraceEvent("BMCheckInitialSplitTooBig", bmData->id)
+	    .detail("Epoch", bmData->epoch)
+	    .detail("Granule", granuleRange)
+	    .detail("ProposedSplitKey", proposedSplitKey);
+	// calculate new split targets speculatively assuming split is too large and current worker still owns it
+	ASSERT(granuleRange.begin < proposedSplitKey);
+	ASSERT(proposedSplitKey < granuleRange.end);
+	state Future<BlobGranuleSplitPoints> fSplitFirst =
+	    splitRange(bmData, KeyRangeRef(granuleRange.begin, proposedSplitKey), false, true);
+	state Future<BlobGranuleSplitPoints> fSplitSecond =
+	    splitRange(bmData, KeyRangeRef(proposedSplitKey, granuleRange.end), false, true);
+
+	state Standalone<VectorRef<KeyRef>> newRanges;
+
+	BlobGranuleSplitPoints splitFirst = wait(fSplitFirst);
+	ASSERT(splitFirst.keys.size() >= 2);
+	ASSERT(splitFirst.keys.front() == granuleRange.begin);
+	ASSERT(splitFirst.keys.back() == proposedSplitKey);
+	for (int i = 0; i < splitFirst.keys.size(); i++) {
+		newRanges.push_back_deep(newRanges.arena(), splitFirst.keys[i]);
+	}
+
+	BlobGranuleSplitPoints splitSecond = wait(fSplitSecond);
+	ASSERT(splitSecond.keys.size() >= 2);
+	ASSERT(splitSecond.keys.front() == proposedSplitKey);
+	ASSERT(splitSecond.keys.back() == granuleRange.end);
+	// i=1 to skip proposedSplitKey, since above already added it
+	for (int i = 1; i < splitSecond.keys.size(); i++) {
+		newRanges.push_back_deep(newRanges.arena(), splitSecond.keys[i]);
+	}
+
+	if (BM_DEBUG) {
+		fmt::print("Re-evaluated split ({0}):\n", newRanges.size());
+		for (auto& it : newRanges) {
+			fmt::print("    {0}\n", it.printable());
+		}
+	}
+
+	// redo key alignment on full set of split points
+	// FIXME: only need to align propsedSplitKey in the middle
+	state BlobGranuleSplitPoints finalSplit = wait(alignKeys(bmData, granuleRange, newRanges));
+
+	ASSERT(finalSplit.keys.size() > 2);
+
+	if (BM_DEBUG) {
+		fmt::print("Aligned split ({0}):\n", finalSplit.keys.size());
+		for (auto& it : finalSplit.keys) {
+			fmt::print("    {0}{1}\n", it.printable(), finalSplit.boundaries.count(it) ? " *" : "");
+		}
+	}
+
+	// Check lock to see if lock is still the specified epoch and seqno, and there are no files for the granule.
+	// If either of these are false, some other worker now has the granule. if there are files, it already succeeded at
+	// a split. if not, and it fails too, it will retry and get back here
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
+	state Key lockKey = blobGranuleLockKeyFor(granuleRange);
+	state bool retried = false;
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			// make sure we're still manager when this transaction gets committed
+			wait(checkManagerLock(tr, bmData));
+
+			ForcedPurgeState purgeState = wait(getForcePurgedState(&tr->getTransaction(), granuleRange));
+			if (purgeState != ForcedPurgeState::NonePurged) {
+				CODE_PROBE(true, "Initial Split Re-evaluate stopped because of force purge");
+				TraceEvent("GranuleSplitReEvalCancelledForcePurge", bmData->id)
+				    .detail("Epoch", bmData->epoch)
+				    .detail("GranuleRange", granuleRange);
+
+				// destroy already created change feed from worker so it doesn't leak
+				wait(updateChangeFeed(&tr->getTransaction(),
+				                      granuleIDToCFKey(granuleID),
+				                      ChangeFeedStatus::CHANGE_FEED_DESTROY,
+				                      granuleRange));
+
+				wait(tr->commit());
+
+				return Void();
+			}
+
+			// this adds a read conflict range, so if another granule concurrently commits a file, we will retry and see
+			// that
+			KeyRange range = blobGranuleFileKeyRangeFor(granuleID);
+			RangeResult granuleFiles = wait(tr->getRange(range, 1));
+			if (!granuleFiles.empty()) {
+				CODE_PROBE(true, "split too big was eventually solved by another worker");
+				if (BM_DEBUG) {
+					fmt::print("BM {0} re-evaluating initial split [{1} - {2}) too big: solved by another worker\n",
+					           bmData->epoch,
+					           granuleRange.begin.printable(),
+					           granuleRange.end.printable());
+				}
+				return Void();
+			}
+
+			Optional<Value> prevLockValue = wait(tr->get(lockKey));
+			ASSERT(prevLockValue.present());
+			std::tuple<int64_t, int64_t, UID> prevOwner = decodeBlobGranuleLockValue(prevLockValue.get());
+			int64_t prevOwnerEpoch = std::get<0>(prevOwner);
+			int64_t prevOwnerSeqno = std::get<1>(prevOwner);
+			UID prevGranuleID = std::get<2>(prevOwner);
+			if (prevOwnerEpoch != epoch || prevOwnerSeqno != seqno || prevGranuleID != granuleID) {
+				if (retried && prevOwnerEpoch == bmData->epoch && prevGranuleID == granuleID &&
+				    prevOwnerSeqno == std::numeric_limits<int64_t>::max()) {
+					// owner didn't change, last iteration of this transaction just succeeded but threw an error.
+					CODE_PROBE(true, "split too big adjustment succeeded after retry");
+					break;
+				}
+				CODE_PROBE(true, "split too big was since moved to another worker");
+				if (BM_DEBUG) {
+					fmt::print("BM {0} re-evaluating initial split [{1} - {2}) too big: moved to another worker\n",
+					           bmData->epoch,
+					           granuleRange.begin.printable(),
+					           granuleRange.end.printable());
+					fmt::print("Epoch: Prev {0}, Cur {1}\n", prevOwnerEpoch, epoch);
+					fmt::print("Seqno: Prev {0}, Cur {1}\n", prevOwnerSeqno, seqno);
+					fmt::print("GranuleID: Prev {0}, Cur {1}\n",
+					           prevGranuleID.toString().substr(0, 6),
+					           granuleID.toString().substr(0, 6));
+				}
+				return Void();
+			}
+
+			if (prevOwnerEpoch > bmData->epoch) {
+				if (BM_DEBUG) {
+					fmt::print("BM {0} found a higher epoch {1} for granule lock of [{2} - {3})\n",
+					           bmData->epoch,
+					           prevOwnerEpoch,
+					           granuleRange.begin.printable(),
+					           granuleRange.end.printable());
+				}
+
+				if (bmData->iAmReplaced.canBeSet()) {
+					bmData->iAmReplaced.send(Void());
+				}
+				return Void();
+			}
+
+			// The lock check above *should* handle this, but just be sure, also make sure that this granule wasn't
+			// already split in the granule mapping
+			RangeResult existingRanges = wait(
+			    krmGetRanges(tr, blobGranuleMappingKeys.begin, granuleRange, 3, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+			if (existingRanges.size() > 2 || existingRanges.more) {
+				CODE_PROBE(true, "split too big was already re-split");
+				if (BM_DEBUG) {
+					fmt::print("BM {0} re-evaluating initial split [{1} - {2}) too big: already split\n",
+					           bmData->epoch,
+					           granuleRange.begin.printable(),
+					           granuleRange.end.printable());
+					for (auto& it : existingRanges) {
+						fmt::print("  {0}\n", it.key.printable());
+					}
+				}
+				return Void();
+			}
+
+			// Set lock to max value for this manager, so other reassignments can't race with this transaction
+			// and existing owner can't modify it further.
+			tr->set(lockKey, blobGranuleLockValueFor(bmData->epoch, std::numeric_limits<int64_t>::max(), granuleID));
+
+			// set new ranges
+			state int i;
+			for (i = 0; i < finalSplit.keys.size() - 1; i++) {
+				wait(krmSetRange(tr,
+				                 blobGranuleMappingKeys.begin,
+				                 KeyRangeRef(finalSplit.keys[i], finalSplit.keys[i + 1]),
+				                 blobGranuleMappingValueFor(UID())));
+				if (finalSplit.boundaries.count(finalSplit.keys[i])) {
+					tr->set(blobGranuleMergeBoundaryKeyFor(finalSplit.keys[i]),
+					        blobGranuleMergeBoundaryValueFor(finalSplit.boundaries[finalSplit.keys[i]]));
+				}
+			}
+
+			// Need to destroy the old change feed for the no longer needed feed, otherwise it will leak
+			// This has to be a non-ryw transaction for the change feed destroy mutations to propagate properly
+			// TODO: fix this better! (privatize change feed key clear)
+			wait(updateChangeFeed(&tr->getTransaction(),
+			                      granuleIDToCFKey(granuleID),
+			                      ChangeFeedStatus::CHANGE_FEED_DESTROY,
+			                      granuleRange));
+
+			retried = true;
+			wait(tr->commit());
+			break;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+
+	// transaction committed, send updated range assignments. Even if there is only one range still, we need to revoke
+	// it and re-assign it to cancel the old granule and retry
+	CODE_PROBE(true, "BM successfully changed initial split too big");
+	RangeAssignment raRevoke;
+	raRevoke.isAssign = false;
+	raRevoke.keyRange = granuleRange;
+	raRevoke.revoke = RangeRevokeData(false); // not a dispose
+	handleRangeAssign(bmData, raRevoke);
+
+	for (int i = 0; i < finalSplit.keys.size() - 1; i++) {
+		// reassign new range and do handover of previous range
+		RangeAssignment raAssignSplit;
+		raAssignSplit.isAssign = true;
+		raAssignSplit.keyRange = KeyRangeRef(finalSplit.keys[i], finalSplit.keys[i + 1]);
+		raAssignSplit.assign = RangeAssignmentData();
+		// don't care who this range gets assigned to
+		handleRangeAssign(bmData, raAssignSplit);
+	}
+
+	if (BM_DEBUG) {
+		fmt::print("BM {0} Re-splitting initial range [{1} - {2}) into {3} granules done\n",
+		           bmData->epoch,
+		           granuleRange.begin.printable(),
+		           granuleRange.end.printable(),
+		           finalSplit.keys.size() - 1);
+	}
+
+	return Void();
+}
+
 ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
                                    UID currentWorkerId,
                                    KeyRange granuleRange,
@@ -1068,15 +1692,17 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
                                    bool writeHot,
                                    int64_t originalEpoch,
                                    int64_t originalSeqno) {
+	if (bmData->isForcePurging(granuleRange)) {
+		// ignore
+		return Void();
+	}
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
-	state Standalone<VectorRef<KeyRef>> newRanges;
 
 	// first get ranges to split
-	Standalone<VectorRef<KeyRef>> _newRanges = wait(splitRange(bmData, granuleRange, writeHot, false));
-	newRanges = _newRanges;
+	state BlobGranuleSplitPoints splitPoints = wait(splitRange(bmData, granuleRange, writeHot, false));
 
-	ASSERT(newRanges.size() >= 2);
-	if (newRanges.size() == 2) {
+	ASSERT(splitPoints.keys.size() >= 2);
+	if (splitPoints.keys.size() == 2) {
 		// not large enough to split, just reassign back to worker
 		if (BM_DEBUG) {
 			fmt::print("Not splitting existing range [{0} - {1}). Continuing assignment to {2}\n",
@@ -1108,38 +1734,42 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 	}
 
 	// Enforce max split fanout for performance reasons. This mainly happens when a blob worker is behind.
-	if (newRanges.size() >=
+	if (splitPoints.keys.size() >=
 	    SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 2) { // +2 because this is boundaries, so N keys would have N+1 bounaries.
-		TEST(true); // downsampling granule split because fanout too high
+		CODE_PROBE(true, "downsampling granule split because fanout too high");
 		Standalone<VectorRef<KeyRef>> coalescedRanges;
-		coalescedRanges.arena().dependsOn(newRanges.arena());
-		coalescedRanges.push_back(coalescedRanges.arena(), newRanges.front());
+		coalescedRanges.arena().dependsOn(splitPoints.keys.arena());
+		coalescedRanges.push_back(coalescedRanges.arena(), splitPoints.keys.front());
 
 		// since we include start + end boundaries here, only need maxSplitFanout-1 split boundaries to produce
 		// maxSplitFanout granules
-		downsampleSplit(newRanges, coalescedRanges, 1, newRanges.size() - 1, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT - 1);
+		downsampleSplit(
+		    splitPoints.keys, coalescedRanges, 1, splitPoints.keys.size() - 1, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT - 1);
 
-		coalescedRanges.push_back(coalescedRanges.arena(), newRanges.back());
+		coalescedRanges.push_back(coalescedRanges.arena(), splitPoints.keys.back());
 		ASSERT(coalescedRanges.size() == SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
 		if (BM_DEBUG) {
-			fmt::print("Downsampled split from {0} -> {1} granules\n",
-			           newRanges.size() - 1,
+			fmt::print("Downsampled split [{0} - {1}) from {2} -> {3} granules\n",
+			           granuleRange.begin.printable(),
+			           granuleRange.end.printable(),
+			           splitPoints.keys.size() - 1,
 			           SERVER_KNOBS->BG_MAX_SPLIT_FANOUT);
 		}
 
-		newRanges = coalescedRanges;
-		ASSERT(newRanges.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
+		// TODO probably do something better here?
+		wait(store(splitPoints, alignKeys(bmData, granuleRange, coalescedRanges)));
+		ASSERT(splitPoints.keys.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1);
 	}
 
-	ASSERT(granuleRange.begin == newRanges.front());
-	ASSERT(granuleRange.end == newRanges.back());
+	ASSERT(granuleRange.begin == splitPoints.keys.front());
+	ASSERT(granuleRange.end == splitPoints.keys.back());
 
 	// Have to make set of granule ids deterministic across retries to not end up with extra UIDs in the split
 	// state, which could cause recovery to fail and resources to not be cleaned up.
 	// This entire transaction must be idempotent across retries for all splitting state
 	state std::vector<UID> newGranuleIDs;
-	newGranuleIDs.reserve(newRanges.size() - 1);
-	for (int i = 0; i < newRanges.size() - 1; i++) {
+	newGranuleIDs.reserve(splitPoints.keys.size() - 1);
+	for (int i = 0; i < splitPoints.keys.size() - 1; i++) {
 		newGranuleIDs.push_back(deterministicRandom()->randomUniqueID());
 	}
 
@@ -1147,11 +1777,12 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		fmt::print("Splitting range [{0} - {1}) into {2} granules:\n",
 		           granuleRange.begin.printable(),
 		           granuleRange.end.printable(),
-		           newRanges.size() - 1);
-		for (int i = 0; i < newRanges.size(); i++) {
-			fmt::print("    {}:{}\n",
+		           splitPoints.keys.size() - 1);
+		for (int i = 0; i < splitPoints.keys.size(); i++) {
+			fmt::print("    {0}:{1}{2}\n",
 			           (i < newGranuleIDs.size() ? newGranuleIDs[i] : UID()).toString().substr(0, 6).c_str(),
-			           newRanges[i].printable());
+			           splitPoints.keys[i].printable(),
+			           splitPoints.boundaries.count(splitPoints.keys[i]) ? " *" : "");
 		}
 	}
 
@@ -1162,12 +1793,21 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 	loop {
 		try {
 			tr->reset();
-			tr->setOption(FDBTransactionOptions::Option::PRIORITY_SYSTEM_IMMEDIATE);
-			tr->setOption(FDBTransactionOptions::Option::ACCESS_SYSTEM_KEYS);
-			ASSERT(newRanges.size() > 2);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			ASSERT(splitPoints.keys.size() > 2);
 
 			// make sure we're still manager when this transaction gets committed
 			wait(checkManagerLock(tr, bmData));
+			ForcedPurgeState purgeState = wait(getForcePurgedState(&tr->getTransaction(), granuleRange));
+			if (purgeState != ForcedPurgeState::NonePurged) {
+				CODE_PROBE(true, "Split stopped because of force purge");
+				TraceEvent("GranuleSplitCancelledForcePurge", bmData->id)
+				    .detail("Epoch", bmData->epoch)
+				    .detail("GranuleRange", granuleRange);
+				return Void();
+			}
 
 			// TODO can do this + lock in parallel
 			// Read splitState to see if anything was committed instead of reading granule mapping because we don't want
@@ -1184,7 +1824,7 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 			if (!existingState.empty()) {
 				// Something was previously committed, we must go with that decision.
 				// Read its boundaries and override our planned split boundaries
-				TEST(true); // Overriding split ranges with existing ones from DB
+				CODE_PROBE(true, "Overriding split ranges with existing ones from DB");
 				RangeResult existingBoundaries =
 				    wait(tr->getRange(KeyRangeRef(granuleRange.begin.withPrefix(blobGranuleMappingKeys.begin),
 				                                  keyAfter(granuleRange.end).withPrefix(blobGranuleMappingKeys.begin)),
@@ -1192,20 +1832,23 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				// +2 because this is boundaries and existingState was granules, and to ensure it doesn't set more
 				ASSERT(!existingBoundaries.more);
 				ASSERT(existingBoundaries.size() == existingState.size() + 1);
-				newRanges.clear();
-				newRanges.arena().dependsOn(existingBoundaries.arena());
+				splitPoints.keys.clear();
+				splitPoints.keys.arena().dependsOn(existingBoundaries.arena());
 				for (auto& it : existingBoundaries) {
-					newRanges.push_back(newRanges.arena(), it.key.removePrefix(blobGranuleMappingKeys.begin));
+					splitPoints.keys.push_back(splitPoints.keys.arena(),
+					                           it.key.removePrefix(blobGranuleMappingKeys.begin));
 				}
-				ASSERT(newRanges.front() == granuleRange.begin);
-				ASSERT(newRanges.back() == granuleRange.end);
+				// We don't care about splitPoints.boundaries as they are already persisted.
+				splitPoints.boundaries.clear();
+				ASSERT(splitPoints.keys.front() == granuleRange.begin);
+				ASSERT(splitPoints.keys.back() == granuleRange.end);
 				if (BM_DEBUG) {
 					fmt::print("Replaced old range splits for [{0} - {1}) with {2}:\n",
 					           granuleRange.begin.printable(),
 					           granuleRange.end.printable(),
-					           newRanges.size() - 1);
-					for (int i = 0; i < newRanges.size(); i++) {
-						fmt::print("    {}\n", newRanges[i].printable());
+					           splitPoints.keys.size() - 1);
+					for (int i = 0; i < splitPoints.keys.size(); i++) {
+						fmt::print("    {}\n", splitPoints.keys[i].printable());
 					}
 				}
 				break;
@@ -1260,20 +1903,30 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 
 			// set up splits in granule mapping, but point each part to the old owner (until they get reassigned)
 			state int i;
-			for (i = 0; i < newRanges.size() - 1; i++) {
+			for (i = 0; i < splitPoints.keys.size() - 1; i++) {
+				KeyRangeRef splitRange(splitPoints.keys[i], splitPoints.keys[i + 1]);
+
+				// Record split state.
 				Key splitKey = blobGranuleSplitKeyFor(granuleID, newGranuleIDs[i]);
 
 				tr->atomicOp(splitKey,
 				             blobGranuleSplitValueFor(BlobGranuleSplitState::Initialized),
 				             MutationRef::SetVersionstampedValue);
 
-				KeyRangeRef splitRange(newRanges[i], newRanges[i + 1]);
+				// Update BlobGranuleMergeBoundary.
+				if (splitPoints.boundaries.count(splitRange.begin)) {
+					tr->set(blobGranuleMergeBoundaryKeyFor(splitRange.begin),
+					        blobGranuleMergeBoundaryValueFor(splitPoints.boundaries[splitRange.begin]));
+				}
+
+				// History.
 				Key historyKey = blobGranuleHistoryKeyFor(splitRange, splitVersion);
 
 				Standalone<BlobGranuleHistoryValue> historyValue;
 				historyValue.granuleID = newGranuleIDs[i];
-				historyValue.parentGranules.push_back(historyValue.arena(),
-				                                      std::pair(granuleRange, granuleStartVersion));
+				historyValue.parentBoundaries.push_back(historyValue.arena(), granuleRange.begin);
+				historyValue.parentBoundaries.push_back(historyValue.arena(), granuleRange.end);
+				historyValue.parentVersions.push_back(historyValue.arena(), granuleStartVersion);
 
 				tr->set(historyKey, blobGranuleHistoryValueFor(historyValue));
 
@@ -1282,11 +1935,17 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 				// restarts
 				wait(krmSetRange(tr,
 				                 blobGranuleMappingKeys.begin,
-				                 KeyRangeRef(newRanges[i], newRanges[i + 1]),
+				                 KeyRangeRef(splitPoints.keys[i], splitPoints.keys[i + 1]),
 				                 blobGranuleMappingValueFor(currentWorkerId)));
 			}
 
 			wait(tr->commit());
+
+			// Update BlobGranuleMergeBoundary in-memory state.
+			for (auto it = splitPoints.boundaries.begin(); it != splitPoints.boundaries.end(); it++) {
+				bmData->mergeBoundaries[it->first] = it->second;
+			}
+
 			break;
 		} catch (Error& e) {
 			if (e.code() == error_code_operation_cancelled) {
@@ -1309,7 +1968,7 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3} done, sending assignments:\n",
 		           granuleRange.begin.printable(),
 		           granuleRange.end.printable(),
-		           newRanges.size() - 1,
+		           splitPoints.keys.size() - 1,
 		           splitVersion);
 	}
 
@@ -1328,11 +1987,11 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 	raRevoke.revoke = RangeRevokeData(false); // not a dispose
 	handleRangeAssign(bmData, raRevoke);
 
-	for (int i = 0; i < newRanges.size() - 1; i++) {
+	for (int i = 0; i < splitPoints.keys.size() - 1; i++) {
 		// reassign new range and do handover of previous range
 		RangeAssignment raAssignSplit;
 		raAssignSplit.isAssign = true;
-		raAssignSplit.keyRange = KeyRangeRef(newRanges[i], newRanges[i + 1]);
+		raAssignSplit.keyRange = KeyRangeRef(splitPoints.keys[i], splitPoints.keys[i + 1]);
 		raAssignSplit.assign = RangeAssignmentData();
 		// don't care who this range gets assigned to
 		handleRangeAssign(bmData, raAssignSplit);
@@ -1342,7 +2001,7 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 		fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3} got assignments processed\n",
 		           granuleRange.begin.printable(),
 		           granuleRange.end.printable(),
-		           newRanges.size() - 1,
+		           splitPoints.keys.size() - 1,
 		           splitVersion);
 	}
 
@@ -1357,7 +2016,10 @@ ACTOR Future<Void> maybeSplitRange(Reference<BlobManagerData> bmData,
 // read mapping from db to handle any in flight granules or other issues
 // Forces all granules in the specified key range to flush data to blob up to the specified version. This is required
 // for executing a merge.
-ACTOR Future<Void> forceGranuleFlush(Reference<BlobManagerData> bmData, KeyRange keyRange, Version version) {
+ACTOR Future<bool> forceGranuleFlush(Reference<BlobManagerData> bmData,
+                                     UID mergeGranuleID,
+                                     KeyRange keyRange,
+                                     Version version) {
 	state Transaction tr(bmData->db);
 	state KeyRange currentRange = keyRange;
 
@@ -1368,10 +2030,27 @@ ACTOR Future<Void> forceGranuleFlush(Reference<BlobManagerData> bmData, KeyRange
 
 	loop {
 		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 		if (currentRange.begin == currentRange.end) {
 			break;
 		}
 		try {
+			ForcedPurgeState purgeState = wait(getForcePurgedState(&tr, keyRange));
+			if (purgeState != ForcedPurgeState::NonePurged) {
+				CODE_PROBE(true, "Granule flush stopped because of force purge");
+				TraceEvent("GranuleFlushCancelledForcePurge", bmData->id)
+				    .detail("Epoch", bmData->epoch)
+				    .detail("KeyRange", keyRange);
+
+				// destroy already created change feed from earlier so it doesn't leak
+				wait(updateChangeFeed(
+				    &tr, granuleIDToCFKey(mergeGranuleID), ChangeFeedStatus::CHANGE_FEED_DESTROY, keyRange));
+
+				wait(tr.commit());
+				return false;
+			}
+
 			// TODO KNOB
 			state RangeResult blobGranuleMapping = wait(krmGetRanges(
 			    &tr, blobGranuleMappingKeys.begin, currentRange, 64, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
@@ -1386,6 +2065,7 @@ ACTOR Future<Void> forceGranuleFlush(Reference<BlobManagerData> bmData, KeyRange
 						           blobGranuleMapping[i].key.printable(),
 						           blobGranuleMapping[i + 1].key.printable());
 					}
+					// range isn't force purged because of above check, so flush was for invalid range
 					throw blob_granule_transaction_too_old();
 				}
 
@@ -1396,6 +2076,7 @@ ACTOR Future<Void> forceGranuleFlush(Reference<BlobManagerData> bmData, KeyRange
 						           blobGranuleMapping[i].key.printable(),
 						           blobGranuleMapping[i + 1].key.printable());
 					}
+					// range isn't force purged because of above check, so flush was for invalid range
 					throw blob_granule_transaction_too_old();
 				}
 
@@ -1451,7 +2132,8 @@ ACTOR Future<Void> forceGranuleFlush(Reference<BlobManagerData> bmData, KeyRange
 						break;
 					} else {
 						if (BM_DEBUG) {
-							fmt::print("ERROR: Error flushing range [{0} - {1}): {2}!\n",
+							fmt::print("ERROR: BM {0} Error flushing range [{1} - {2}): {3}!\n",
+							           bmData->epoch,
 							           blobGranuleMapping[j].key.printable(),
 							           blobGranuleMapping[j + 1].key.printable(),
 							           e.name());
@@ -1482,7 +2164,7 @@ ACTOR Future<Void> forceGranuleFlush(Reference<BlobManagerData> bmData, KeyRange
 		           version);
 	}
 
-	return Void();
+	return true;
 }
 
 // Persist the merge intent for this merge in the database. Once this transaction commits, the merge is in progress. It
@@ -1490,7 +2172,7 @@ ACTOR Future<Void> forceGranuleFlush(Reference<BlobManagerData> bmData, KeyRange
 ACTOR Future<std::pair<UID, Version>> persistMergeGranulesStart(Reference<BlobManagerData> bmData,
                                                                 KeyRange mergeRange,
                                                                 std::vector<UID> parentGranuleIDs,
-                                                                std::vector<KeyRange> parentGranuleRanges,
+                                                                std::vector<Key> parentGranuleRanges,
                                                                 std::vector<Version> parentGranuleStartVersions) {
 	state UID mergeGranuleID = deterministicRandom()->randomUniqueID();
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
@@ -1498,9 +2180,34 @@ ACTOR Future<std::pair<UID, Version>> persistMergeGranulesStart(Reference<BlobMa
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			wait(checkManagerLock(tr, bmData));
 
+			ForcedPurgeState purgeState = wait(getForcePurgedState(&tr->getTransaction(), mergeRange));
+			if (purgeState != ForcedPurgeState::NonePurged) {
+				CODE_PROBE(true, "Merge start stopped because of force purge");
+				TraceEvent("GranuleMergeStartCancelledForcePurge", bmData->id)
+				    .detail("Epoch", bmData->epoch)
+				    .detail("GranuleRange", mergeRange);
+
+				// destroy already created change feed from earlier so it doesn't leak
+				wait(updateChangeFeed(&tr->getTransaction(),
+				                      granuleIDToCFKey(mergeGranuleID),
+				                      ChangeFeedStatus::CHANGE_FEED_DESTROY,
+				                      mergeRange));
+
+				wait(tr->commit());
+
+				bmData->activeGranuleMerges.insert(mergeRange, invalidVersion);
+				bmData->activeGranuleMerges.coalesce(mergeRange.begin);
+
+				// TODO better error?
+				return std::pair(UID(), invalidVersion);
+			}
+			// FIXME: extra safeguard: check that granuleID of active lock == parentGranuleID for each parent, abort
+			// merge if so
+
 			tr->atomicOp(
 			    blobGranuleMergeKeyFor(mergeGranuleID),
 			    blobGranuleMergeValueFor(mergeRange, parentGranuleIDs, parentGranuleRanges, parentGranuleStartVersions),
@@ -1540,15 +2247,14 @@ ACTOR Future<std::pair<UID, Version>> persistMergeGranulesStart(Reference<BlobMa
 	}
 }
 
-// FIXME: why not just make parentGranuleRanges vector of N+1 keys?
 // Persists the merge being complete in the database by clearing the merge intent. Once this transaction commits, the
 // merge is considered completed.
-ACTOR Future<Void> persistMergeGranulesDone(Reference<BlobManagerData> bmData,
+ACTOR Future<bool> persistMergeGranulesDone(Reference<BlobManagerData> bmData,
                                             UID mergeGranuleID,
                                             KeyRange mergeRange,
                                             Version mergeVersion,
                                             std::vector<UID> parentGranuleIDs,
-                                            std::vector<KeyRange> parentGranuleRanges,
+                                            std::vector<Key> parentGranuleRanges,
                                             std::vector<Version> parentGranuleStartVersions) {
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
 	// pick worker that has part of old range, it will soon get overridden anyway
@@ -1561,7 +2267,7 @@ ACTOR Future<Void> persistMergeGranulesDone(Reference<BlobManagerData> bmData,
 		}
 	}
 	if (tmpWorkerId == UID()) {
-		TEST(true); // All workers dead right now
+		CODE_PROBE(true, "All workers dead right now");
 		while (bmData->workersById.empty()) {
 			wait(bmData->recruitingStream.onChange() || bmData->foundBlobWorkers.getFuture());
 		}
@@ -1571,21 +2277,51 @@ ACTOR Future<Void> persistMergeGranulesDone(Reference<BlobManagerData> bmData,
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			wait(checkManagerLock(tr, bmData));
 
+			ForcedPurgeState purgeState = wait(getForcePurgedState(&tr->getTransaction(), mergeRange));
+			if (purgeState != ForcedPurgeState::NonePurged) {
+				CODE_PROBE(true, "Merge finish stopped because of force purge");
+				TraceEvent("GranuleMergeCancelledForcePurge", bmData->id)
+				    .detail("Epoch", bmData->epoch)
+				    .detail("GranuleRange", mergeRange);
+
+				// destroy already created change feed from earlier so it doesn't leak
+				wait(updateChangeFeed(&tr->getTransaction(),
+				                      granuleIDToCFKey(mergeGranuleID),
+				                      ChangeFeedStatus::CHANGE_FEED_DESTROY,
+				                      mergeRange));
+
+				// TODO could also delete history entry here
+
+				wait(tr->commit());
+
+				bmData->activeGranuleMerges.insert(mergeRange, invalidVersion);
+				bmData->activeGranuleMerges.coalesce(mergeRange.begin);
+
+				return false;
+			}
+
 			tr->clear(blobGranuleMergeKeyFor(mergeGranuleID));
 
 			state int parentIdx;
 			// TODO: could parallelize these
 			for (parentIdx = 0; parentIdx < parentGranuleIDs.size(); parentIdx++) {
-				state Key lockKey = blobGranuleLockKeyFor(parentGranuleRanges[parentIdx]);
+				KeyRange parentRange(KeyRangeRef(parentGranuleRanges[parentIdx], parentGranuleRanges[parentIdx + 1]));
+				state Key lockKey = blobGranuleLockKeyFor(parentRange);
 				state Future<Optional<Value>> oldLockFuture = tr->get(lockKey);
 
-				wait(updateChangeFeed(tr,
+				// Clear existing merge boundaries.
+				tr->clear(blobGranuleMergeBoundaryKeyFor(parentRange.begin));
+
+				// This has to be a non-ryw transaction for the change feed destroy mutations to propagate properly
+				// TODO: fix this better! (privatize change feed key clear)
+				wait(updateChangeFeed(&tr->getTransaction(),
 				                      granuleIDToCFKey(parentGranuleIDs[parentIdx]),
 				                      ChangeFeedStatus::CHANGE_FEED_DESTROY,
-				                      parentGranuleRanges[parentIdx]));
+				                      parentRange));
 				if (BM_DEBUG) {
 					fmt::print("Granule merge destroying CF {0} ({1})!\n",
 					           parentGranuleIDs[parentIdx].shortString().substr(0, 6),
@@ -1604,6 +2340,9 @@ ACTOR Future<Void> persistMergeGranulesDone(Reference<BlobManagerData> bmData,
 				    blobGranuleLockValueFor(bmData->epoch, std::numeric_limits<int64_t>::max(), std::get<2>(prevLock)));
 			}
 
+			tr->clear(KeyRangeRef(keyAfter(blobGranuleMergeBoundaryKeyFor(mergeRange.begin)),
+			                      blobGranuleMergeBoundaryKeyFor(mergeRange.end)));
+
 			// either set initial lock value, or re-set it if it was set to (epoch, <max>) in a previous split
 			int64_t seqNo = bmData->seqNo++;
 			tr->set(blobGranuleLockKeyFor(mergeRange), blobGranuleLockValueFor(bmData->epoch, seqNo, mergeGranuleID));
@@ -1614,15 +2353,21 @@ ACTOR Future<Void> persistMergeGranulesDone(Reference<BlobManagerData> bmData,
 			Standalone<BlobGranuleHistoryValue> historyValue;
 			historyValue.granuleID = mergeGranuleID;
 			for (parentIdx = 0; parentIdx < parentGranuleIDs.size(); parentIdx++) {
-				historyValue.parentGranules.push_back(
-				    historyValue.arena(),
-				    std::pair(parentGranuleRanges[parentIdx], parentGranuleStartVersions[parentIdx]));
+				historyValue.parentBoundaries.push_back(historyValue.arena(), parentGranuleRanges[parentIdx]);
+				historyValue.parentVersions.push_back(historyValue.arena(), parentGranuleStartVersions[parentIdx]);
 			}
+			historyValue.parentBoundaries.push_back(historyValue.arena(), parentGranuleRanges.back());
 
 			tr->set(historyKey, blobGranuleHistoryValueFor(historyValue));
 
 			wait(krmSetRange(tr, blobGranuleMappingKeys.begin, mergeRange, blobGranuleMappingValueFor(tmpWorkerId)));
 			wait(tr->commit());
+
+			// Update in-memory mergeBoundaries map.
+			for (parentIdx = 1; parentIdx < parentGranuleIDs.size(); parentIdx++) {
+				bmData->mergeBoundaries.erase(parentGranuleRanges[parentIdx]);
+			}
+
 			if (BM_DEBUG) {
 				fmt::print("Granule merge intent cleared [{0} - {1}): {2} @ {3} (cv={4})\n",
 				           mergeRange.begin.printable(),
@@ -1631,8 +2376,8 @@ ACTOR Future<Void> persistMergeGranulesDone(Reference<BlobManagerData> bmData,
 				           mergeVersion,
 				           tr->getCommittedVersion());
 			}
-			TEST(true); // Granule merge complete
-			return Void();
+			CODE_PROBE(true, "Granule merge complete");
+			return true;
 		} catch (Error& e) {
 			wait(tr->onError(e));
 		}
@@ -1645,24 +2390,45 @@ ACTOR Future<Void> finishMergeGranules(Reference<BlobManagerData> bmData,
                                        KeyRange mergeRange,
                                        Version mergeVersion,
                                        std::vector<UID> parentGranuleIDs,
-                                       std::vector<KeyRange> parentGranuleRanges,
+                                       std::vector<Key> parentGranuleRanges,
                                        std::vector<Version> parentGranuleStartVersions) {
+	++bmData->stats.activeMerges;
 
-	// wait for BM to be fully recovered before starting actual merges
+	// wait for BM to be fully recovered and have loaded hard boundaries before starting actual merges
 	wait(bmData->doneRecovering.getFuture());
+	wait(bmData->loadedClientRanges.getFuture());
 	wait(delay(0));
 
+	// Assert that none of the subsequent granules are hard boundaries.
+	if (g_network->isSimulated()) {
+		for (int i = 1; i < parentGranuleRanges.size() - 1; i++) {
+			ASSERT(!bmData->mergeHardBoundaries.count(parentGranuleRanges[i]));
+		}
+	}
+
 	// force granules to persist state up to mergeVersion
-	wait(forceGranuleFlush(bmData, mergeRange, mergeVersion));
+	bool successFlush = wait(forceGranuleFlush(bmData, mergeGranuleID, mergeRange, mergeVersion));
+	if (!successFlush) {
+		bmData->activeGranuleMerges.insert(mergeRange, invalidVersion);
+		bmData->activeGranuleMerges.coalesce(mergeRange.begin);
+		--bmData->stats.activeMerges;
+		return Void();
+	}
 
 	// update state and clear merge intent
-	wait(persistMergeGranulesDone(bmData,
-	                              mergeGranuleID,
-	                              mergeRange,
-	                              mergeVersion,
-	                              parentGranuleIDs,
-	                              parentGranuleRanges,
-	                              parentGranuleStartVersions));
+	bool successFinish = wait(persistMergeGranulesDone(bmData,
+	                                                   mergeGranuleID,
+	                                                   mergeRange,
+	                                                   mergeVersion,
+	                                                   parentGranuleIDs,
+	                                                   parentGranuleRanges,
+	                                                   parentGranuleStartVersions));
+	if (!successFinish) {
+		bmData->activeGranuleMerges.insert(mergeRange, invalidVersion);
+		bmData->activeGranuleMerges.coalesce(mergeRange.begin);
+		--bmData->stats.activeMerges;
+		return Void();
+	}
 
 	int64_t seqnoForEval = bmData->seqNo;
 
@@ -1684,305 +2450,233 @@ ACTOR Future<Void> finishMergeGranules(Reference<BlobManagerData> bmData,
 
 	bmData->boundaryEvaluations.insert(mergeRange,
 	                                   BoundaryEvaluation(bmData->epoch, seqnoForEval, BoundaryEvalType::MERGE, 0, 0));
+	bmData->setMergeCandidate(mergeRange, MergeCandidateMerging);
+
+	--bmData->stats.activeMerges;
 
 	return Void();
 }
 
-// Make a decision on whether to merge this granule with surrounding ones.
-ACTOR Future<Void> maybeMergeRange(Reference<BlobManagerData> bmData,
-                                   UID granuleID,
-                                   KeyRange granuleRange,
-                                   Version granuleStartVersion) {
-	state std::deque<std::tuple<UID, KeyRange, Version>> beforeCandidates, afterCandidates;
+ACTOR Future<Void> doMerge(Reference<BlobManagerData> bmData,
+                           KeyRange mergeRange,
+                           std::vector<std::tuple<UID, KeyRange, Version>> toMerge) {
+	// switch to format persist merge wants
+	state std::vector<UID> ids;
+	state std::vector<Key> ranges;
+	state std::vector<Version> startVersions;
+	for (auto& it : toMerge) {
+		ids.push_back(std::get<0>(it));
+		ranges.push_back(std::get<1>(it).begin);
+		startVersions.push_back(std::get<2>(it));
+	}
+	ranges.push_back(std::get<1>(toMerge.back()).end);
+
+	++bmData->stats.granuleMerges;
+
+	try {
+		std::pair<UID, Version> persistMerge =
+		    wait(persistMergeGranulesStart(bmData, mergeRange, ids, ranges, startVersions));
+		if (persistMerge.second == invalidVersion) {
+			// cancelled because of force purge
+
+			return Void();
+		}
+		wait(finishMergeGranules(
+		    bmData, persistMerge.first, mergeRange, persistMerge.second, ids, ranges, startVersions));
+		return Void();
+	} catch (Error& e) {
+		if (e.code() == error_code_operation_cancelled || e.code() == error_code_blob_manager_replaced) {
+			throw;
+		}
+		TraceEvent(SevError, "UnexpectedErrorGranuleMerge").error(e).detail("Range", mergeRange);
+		throw e;
+	}
+}
+
+// Needs to not be an actor to run synchronously for the race checking.
+// Technically this could just be the first part of doMerge, but this guarantees no waits happen for the checks before
+// the logic starts
+static void attemptStartMerge(Reference<BlobManagerData> bmData,
+                              const std::vector<std::tuple<UID, KeyRange, Version>>& toMerge) {
+	if (toMerge.size() < 2) {
+		return;
+	}
+	// TODO REMOVE validation eventually
+	for (int i = 0; i < toMerge.size() - 1; i++) {
+		ASSERT(std::get<1>(toMerge[i]).end == std::get<1>(toMerge[i + 1]).begin);
+	}
+	KeyRange mergeRange(KeyRangeRef(std::get<1>(toMerge.front()).begin, std::get<1>(toMerge.back()).end));
+	// merge/merge races should not be possible because granuleMergeChecker should only start attemptMerges() for
+	// disjoint ranges, and merge candidate is not updated if it is already in the state MergeCandidateMerging
+	ASSERT(!bmData->isMergeActive(mergeRange));
+	// Check to avoid races where a split eval came in while merge was evaluating. This also effectively checks
+	// boundaryEvals because they're both updated before maybeSplitRange is called. This handles split/merge races.
+	auto reCheckMergeCandidates = bmData->mergeCandidates.intersectingRanges(mergeRange);
+	for (auto it : reCheckMergeCandidates) {
+		if (!it->cvalue().mergeEligible()) {
+			CODE_PROBE(true, " granule no longer merge candidate after checking metrics, aborting merge");
+			return;
+		}
+	}
+
+	if (bmData->isForcePurging(mergeRange)) {
+		// ignore
+		return;
+	}
 
 	if (BM_DEBUG) {
-		fmt::print("BM {0} maybe merge [{1} - {2}): Start\n",
+		fmt::print("BM {0} Starting merge of [{1} - {2}) ({3})\n",
 		           bmData->epoch,
-		           granuleRange.begin.printable(),
-		           granuleRange.end.printable());
+		           mergeRange.begin.printable(),
+		           mergeRange.end.printable(),
+		           toMerge.size());
 	}
+	CODE_PROBE(true, "Doing granule merge");
+	bmData->activeGranuleMerges.insert(mergeRange, 0);
+	bmData->setMergeCandidate(mergeRange, MergeCandidateMerging);
+	// Now, after setting activeGranuleMerges, we have committed to doing the merge, so any subsequent split eval for
+	// any of the ranges will be ignored. This handles merge/split races.
+	bmData->addActor.send(doMerge(bmData, mergeRange, toMerge));
+}
 
-	// look for candidates to the left
-	if (granuleRange.begin != normalKeys.begin) {
-		auto rangeBefore = bmData->mergeCandidates.rangeContainingKeyBefore(granuleRange.begin);
-		while (rangeBefore.cvalue().present() && beforeCandidates.size() < SERVER_KNOBS->BG_MAX_MERGE_FANIN - 1) {
-			// if it is a merge candidate, add it to the list
-			beforeCandidates.push_front(
-			    std::tuple(rangeBefore.cvalue().get().first, rangeBefore.range(), rangeBefore.cvalue().get().second));
+// Greedily merges any consecutive 2+ granules in a row that are mergeable
+ACTOR Future<Void> attemptMerges(Reference<BlobManagerData> bmData,
+                                 std::vector<std::tuple<UID, KeyRange, Version>> candidates) {
+	ASSERT(candidates.size() >= 2);
 
-			if (BM_DEBUG) {
-				fmt::print("BM {0} maybe merge [{1} - {2}):   Before candidate [{3} - {4})\n",
-				           bmData->epoch,
-				           granuleRange.begin.printable(),
-				           granuleRange.end.printable(),
-				           rangeBefore.begin().printable(),
-				           rangeBefore.end().printable());
-			}
-
-			ASSERT(rangeBefore.begin() >= normalKeys.begin);
-			if (rangeBefore.begin() == normalKeys.begin) {
-				break;
-			} else {
-				--rangeBefore;
-			}
-		}
+	// TODO REMOVE validation eventually
+	for (int i = 0; i < candidates.size() - 1; i++) {
+		ASSERT(std::get<1>(candidates[i]).end == std::get<1>(candidates[i + 1]).begin);
 	}
+	CODE_PROBE(true, "Candidate ranges to merge");
+	wait(bmData->concurrentMergeChecks.take());
+	state FlowLock::Releaser holdingLock(bmData->concurrentMergeChecks);
 
-	// look for candidates to right
-	if (granuleRange.end != normalKeys.end) {
-		auto rangeAfter = bmData->mergeCandidates.rangeContaining(granuleRange.end);
-		while (rangeAfter.cvalue().present() && afterCandidates.size() < SERVER_KNOBS->BG_MAX_MERGE_FANIN - 1) {
-			// if it is a merge candidate, add it to the list
-			afterCandidates.push_back(
-			    std::tuple(rangeAfter.cvalue().get().first, rangeAfter.range(), rangeAfter.cvalue().get().second));
-			if (BM_DEBUG) {
-				fmt::print("BM {0} maybe merge [{1} - {2}):   After candidate [{3} - {4})\n",
-				           bmData->epoch,
-				           granuleRange.begin.printable(),
-				           granuleRange.end.printable(),
-				           rangeAfter.begin().printable(),
-				           rangeAfter.end().printable());
-			}
-			ASSERT(rangeAfter.end() <= normalKeys.end);
-			if (rangeAfter.end() == normalKeys.end) {
-				break;
-			} else {
-				++rangeAfter;
-			}
-		}
-	}
-
-	if (beforeCandidates.empty() && afterCandidates.empty()) {
-		TEST(true); // no consecutive merge candidates
-		if (BM_DEBUG) {
-			fmt::print("BM {0} maybe merge [{1} - {2}): No merge candidates\n",
-			           bmData->epoch,
-			           granuleRange.begin.printable(),
-			           granuleRange.end.printable());
-		}
-		return Void();
-	}
-
-	TEST(true); // consecutive granule merge candidates
-
-	if (BM_DEBUG) {
-		fmt::print("BM {0} maybe merge [{1} - {2}): Checking metrics for {3} candidates ({4} - {5})\n",
-		           bmData->epoch,
-		           granuleRange.begin.printable(),
-		           granuleRange.end.printable(),
-		           beforeCandidates.size() + afterCandidates.size() + 1,
-		           beforeCandidates.size(),
-		           afterCandidates.size());
-	}
-
-	// get metrics for current granule to see if it is still mergeable
-	StorageMetrics targetGranuleMetrics = wait(bmData->db->getStorageMetrics(granuleRange, CLIENT_KNOBS->TOO_MANY));
-	if (targetGranuleMetrics.bytesPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC ||
-	    targetGranuleMetrics.bytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES) {
-		TEST(true); // granule merge candidate no longer mergeable
-		return Void();
-	}
-
-	// best set of granules to merge
-	state std::vector<UID> bestGranuleIDs;
-	state std::vector<KeyRange> bestGranuleRanges;
-	state std::vector<Version> bestGranuleStartVersions;
-	state KeyRange bestGranuleRange;
-
-	// current set of granules being evaluated
-	state std::deque<std::tuple<UID, KeyRange, Version, int64_t>> windowGranules;
-	state int64_t windowBytes = targetGranuleMetrics.bytes;
-	windowGranules.push_back(std::tuple(granuleID, granuleRange, granuleStartVersion, windowBytes));
-
-	// first walk backwards through before candidates until combined granule would be too large to merge, or we hit a
-	// granule that has too high bytesPerKSec and isn't mergeable
+	// start merging any set of 2+ consecutive granules that can be merged
+	state int64_t currentBytes = 0;
+	// large keys can cause a large number of granules in the merge to exceed the maximum value size
+	state int currentKeySumBytes = 0;
+	state std::vector<std::tuple<UID, KeyRange, Version>> currentCandidates;
 	state int i;
-	for (i = beforeCandidates.size() - 1; i >= 0; i--) {
-		if (BM_DEBUG) {
-			fmt::print("BM {0} maybe merge [{1} - {2}): Checking before candidate [{3} - {4})\n",
-			           bmData->epoch,
-			           granuleRange.begin.printable(),
-			           granuleRange.end.printable(),
-			           std::get<1>(beforeCandidates[i]).begin.printable(),
-			           std::get<1>(beforeCandidates[i]).end.printable());
+	for (i = 0; i < candidates.size(); i++) {
+		StorageMetrics metrics =
+		    wait(bmData->db->getStorageMetrics(std::get<1>(candidates[i]), CLIENT_KNOBS->TOO_MANY));
+
+		if (metrics.bytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES ||
+		    metrics.bytesPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
+			// This granule cannot be merged with any neighbors.
+			// If current candidates up to here can be merged, merge them and skip over this one
+			attemptStartMerge(bmData, currentCandidates);
+			currentCandidates.clear();
+			currentBytes = 0;
+			currentKeySumBytes = 0;
+			continue;
 		}
-		StorageMetrics beforeMetrics =
-		    wait(bmData->db->getStorageMetrics(std::get<1>(beforeCandidates[i]), CLIENT_KNOBS->TOO_MANY));
-		if (windowBytes + beforeMetrics.bytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES ||
-		    beforeMetrics.bytesPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
-			break;
+
+		// if the current window is already at the maximum merge size, or adding this granule would push the window over
+		// the edge, merge the existing candidates if possible
+		ASSERT(currentCandidates.size() <= SERVER_KNOBS->BG_MAX_MERGE_FANIN);
+		if (currentCandidates.size() == SERVER_KNOBS->BG_MAX_MERGE_FANIN ||
+		    currentBytes + metrics.bytes > SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES ||
+		    currentKeySumBytes >= CLIENT_KNOBS->VALUE_SIZE_LIMIT / 2) {
+			ASSERT(currentBytes <= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES);
+			CODE_PROBE(currentKeySumBytes >= CLIENT_KNOBS->VALUE_SIZE_LIMIT / 2, "merge early because of key size");
+			attemptStartMerge(bmData, currentCandidates);
+			currentCandidates.clear();
+			currentBytes = 0;
+			currentKeySumBytes = 0;
 		}
-		if (BM_DEBUG) {
-			fmt::print("BM {0} maybe merge [{1} - {2}): Before Candidate [{3} - {4}): {5} bytes\n",
-			           bmData->epoch,
-			           granuleRange.begin.printable(),
-			           granuleRange.end.printable(),
-			           std::get<1>(beforeCandidates[i]).begin.printable(),
-			           std::get<1>(beforeCandidates[i]).end.printable(),
-			           beforeMetrics.bytes);
+
+		// add this granule to the window
+		if (currentCandidates.empty()) {
+			currentKeySumBytes += std::get<1>(candidates[i]).begin.size();
 		}
-		windowBytes += beforeMetrics.bytes;
-		windowGranules.push_front(std::tuple(std::get<0>(beforeCandidates[i]),
-		                                     std::get<1>(beforeCandidates[i]),
-		                                     std::get<2>(beforeCandidates[i]),
-		                                     beforeMetrics.bytes));
+		currentKeySumBytes += std::get<1>(candidates[i]).end.size();
+		currentCandidates.push_back(candidates[i]);
 	}
 
-	// set first window as the best range
-	bestGranuleRange = KeyRangeRef(std::get<1>(windowGranules.front()).begin, std::get<1>(windowGranules.back()).end);
-	for (auto& it : windowGranules) {
-		bestGranuleIDs.push_back(std::get<0>(it));
-		bestGranuleRanges.push_back(std::get<1>(it));
-		bestGranuleStartVersions.push_back(std::get<2>(it));
-	}
+	attemptStartMerge(bmData, currentCandidates);
 
-	// Do moving window algorithm where we add the next afterCandidate to the merge window, and then remove the tail end
-	// of beforeCandidates until we are down to a mergeable granule
-	for (i = 0; i < afterCandidates.size(); i++) {
-		if (BM_DEBUG) {
-			fmt::print("BM {0} maybe merge [{1} - {2}): Checking after candidate [{3} - {4})\n",
-			           bmData->epoch,
-			           granuleRange.begin.printable(),
-			           granuleRange.end.printable(),
-			           std::get<1>(afterCandidates[i]).begin.printable(),
-			           std::get<1>(afterCandidates[i]).end.printable());
-		}
-		// include this granule in the window
-		StorageMetrics afterMetrics =
-		    wait(bmData->db->getStorageMetrics(std::get<1>(afterCandidates[i]), CLIENT_KNOBS->TOO_MANY));
-		if (afterMetrics.bytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES ||
-		    afterMetrics.bytesPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
-			break;
-		}
-		if (BM_DEBUG) {
-			fmt::print("BM {0} maybe merge [{1} - {2}): After Candidate [{3} - {4}): {5} bytes\n",
-			           bmData->epoch,
-			           granuleRange.begin.printable(),
-			           granuleRange.end.printable(),
-			           std::get<1>(afterCandidates[i]).begin.printable(),
-			           std::get<1>(afterCandidates[i]).end.printable(),
-			           afterMetrics.bytes);
-		}
-		windowBytes += afterMetrics.bytes;
-		windowGranules.push_back(std::tuple(std::get<0>(afterCandidates[i]),
-		                                    std::get<1>(afterCandidates[i]),
-		                                    std::get<2>(afterCandidates[i]),
-		                                    afterMetrics.bytes));
+	return Void();
+}
 
-		// slide the window forward back down to mergeable size
-		while (windowBytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES) {
-			if (BM_DEBUG) {
-				fmt::print("BM {0} maybe merge [{1} - {2}): window bytes {3} >= target {4}\n",
-				           bmData->epoch,
-				           granuleRange.begin.printable(),
-				           granuleRange.end.printable(),
-				           windowBytes,
-				           SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES);
-			}
-			ASSERT(!windowGranules.empty());
-			if (std::get<0>(windowGranules.front()) == granuleID) {
-				// merge must include target granule
-				break;
-			}
-			if (BM_DEBUG) {
-				fmt::print(
-				    "BM {0} maybe merge [{1} - {2}):   After Candidate [{3} - {4}) popping [{5} - {6}): {7} bytes\n",
-				    bmData->epoch,
-				    granuleRange.begin.printable(),
-				    granuleRange.end.printable(),
-				    std::get<1>(afterCandidates[i]).begin.printable(),
-				    std::get<1>(afterCandidates[i]).end.printable(),
-				    std::get<1>(windowGranules.front()).begin.printable(),
-				    std::get<1>(windowGranules.front()).end.printable(),
-				    std::get<3>(windowGranules.front()));
-			}
-			windowBytes -= std::get<3>(windowGranules.front());
-			windowGranules.pop_front();
-		}
+// Uses single-pass algorithm to identify mergeable sections of granules.
+// To ensure each granule waits to see whether all of its neighbors are merge-eligible before merging it, a newly
+// merge-eligible granule will be ignored on the first pass
+ACTOR Future<Void> granuleMergeChecker(Reference<BlobManagerData> bmData) {
+	// wait for BM data to have loaded hard boundaries before starting
+	wait(bmData->loadedClientRanges.getFuture());
+	// initial sleep
+	wait(delayJittered(SERVER_KNOBS->BG_MERGE_CANDIDATE_DELAY_SECONDS));
+	// TODO could optimize to not check if there are no new merge-eligible granules and none in merge pending state
+	loop {
 
-		// compare this candidate window to previous best
-		if (windowBytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES) {
-			break;
-		} else if (windowGranules.size() > bestGranuleIDs.size()) {
-			if (BM_DEBUG) {
-				fmt::print("BM {0} maybe merge [{1} - {2}): new best granules {3}\n",
-				           bmData->epoch,
-				           granuleRange.begin.printable(),
-				           granuleRange.end.printable(),
-				           windowGranules.size());
-			}
-			bestGranuleRange =
-			    KeyRangeRef(std::get<1>(windowGranules.front()).begin, std::get<1>(windowGranules.back()).end);
-			bestGranuleIDs.clear();
-			bestGranuleRanges.clear();
-			bestGranuleStartVersions.clear();
-			for (auto& it : windowGranules) {
-				bestGranuleIDs.push_back(std::get<0>(it));
-				bestGranuleRanges.push_back(std::get<1>(it));
-				bestGranuleStartVersions.push_back(std::get<2>(it));
-			}
+		double sleepTime = SERVER_KNOBS->BG_MERGE_CANDIDATE_DELAY_SECONDS;
+		// Check more frequently if speedUpSimulation is set. This may
+		if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
+			sleepTime = std::min(5.0, sleepTime);
 		}
-	}
+		// start delay at the start of the loop, to account for time spend in calculation
+		state Future<Void> intervalDelay = delayJittered(sleepTime);
 
-	TEST(bestGranuleIDs.size() == 1); // Cannot combine merge candidates into mergeable granule
-	TEST(bestGranuleIDs.size() > 1); // Granule ready for merge!
+		// go over granule states, and start a findMergeableGranules for each sub-range of mergeable granules
+		// FIXME: avoid SlowTask by breaking this up periodically
 
-	if (bestGranuleIDs.size() > 1) {
-		if (BM_DEBUG) {
-			fmt::print("BM {0} maybe merge [{1} - {2}): Found {3} consecutive granules in range [{4} - {5}):\n",
-			           bmData->epoch,
-			           granuleRange.begin.printable(),
-			           granuleRange.end.printable(),
-			           bestGranuleIDs.size(),
-			           bestGranuleRange.begin.printable(),
-			           bestGranuleRange.end.printable());
-		}
-		// This code block must execute withou a wait for the lock checks (isMergeActive, mergeCandidates) to not
-		// deadlock and to avoid merge-merge races.
-		if ((!g_network->isSimulated() || !g_simulator.speedUpSimulation) && !bmData->isMergeActive(bestGranuleRange)) {
-			// check to avoid races where a split eval came in while merge was evaluating
-			auto reCheckMergeCandidates = bmData->mergeCandidates.intersectingRanges(bestGranuleRange);
-			bool mergeStillOk = true;
-			for (auto it : reCheckMergeCandidates) {
-				if (!it->cvalue().present()) {
-					TEST(true); // granule no longer merge candidate after checking metrics, because of split eval
-					mergeStillOk = false;
-					break;
+		// Break it up into parallel chunks. This makes it possible to process large ranges, but does mean the merges
+		// can be slightly suboptimal at boundaries. Use relatively large chunks to minimize the impact of this.
+		int maxRangeSize = SERVER_KNOBS->BG_MAX_MERGE_FANIN * 10;
+
+		state std::vector<Future<Void>> mergeChecks;
+		auto allRanges = bmData->mergeCandidates.ranges();
+		std::vector<std::tuple<UID, KeyRange, Version>> currentCandidates;
+		auto& mergeBoundaries = bmData->mergeBoundaries;
+		for (auto& it : allRanges) {
+			// Conditions:
+			//   1. Next range is not eligible.
+			//   2. Hit the maximum in a merge evaluation window.
+			//   3. Hit a hard merge boundary meaning we should not merge across them.
+			if (!it->cvalue().mergeEligible() || currentCandidates.size() == maxRangeSize ||
+			    bmData->mergeHardBoundaries.count(it->range().begin)) {
+				if (currentCandidates.size() >= 2) {
+					mergeChecks.push_back(attemptMerges(bmData, currentCandidates));
+				}
+				currentCandidates.clear();
+			}
+
+			// Soft boundaries.
+			// We scan all ranges including non-eligible ranges which trigger non-consecutive merge flushes.
+			if (!currentCandidates.empty()) {
+				KeyRangeRef lastRange = std::get<1>(currentCandidates.back());
+				KeyRangeRef curRange = it->range();
+				ASSERT(lastRange.end == curRange.begin);
+				// Conditions:
+				//   1. Start a new soft merge range.
+				//   2. End a soft merge range.
+				if ((!mergeBoundaries.count(curRange.begin) && mergeBoundaries.count(curRange.end)) ||
+				    (mergeBoundaries.count(lastRange.begin) && !mergeBoundaries.count(lastRange.end))) {
+					if (currentCandidates.size() >= 2) {
+						mergeChecks.push_back(attemptMerges(bmData, currentCandidates));
+					}
+					currentCandidates.clear();
 				}
 			}
 
-			if (mergeStillOk) {
-				fmt::print("BM {0} maybe merge [{1} - {2}): Starting merge of [{3} - {4}) ({5})\n",
-				           bmData->epoch,
-				           granuleRange.begin.printable(),
-				           granuleRange.end.printable(),
-				           bestGranuleRange.begin.printable(),
-				           bestGranuleRange.end.printable(),
-				           bestGranuleIDs.size());
-				TEST(true); // Doing granule merge!
-				bmData->activeGranuleMerges.insert(bestGranuleRange, 0);
-				bmData->mergeCandidates.insert(bestGranuleRange, Optional<std::pair<UID, Version>>());
-				state std::pair<UID, Version> persistMerge = wait(persistMergeGranulesStart(
-				    bmData, bestGranuleRange, bestGranuleIDs, bestGranuleRanges, bestGranuleStartVersions));
-				wait(finishMergeGranules(bmData,
-				                         persistMerge.first,
-				                         bestGranuleRange,
-				                         persistMerge.second,
-				                         bestGranuleIDs,
-				                         bestGranuleRanges,
-				                         bestGranuleStartVersions));
+			if (it->cvalue().mergeEligible()) {
+				currentCandidates.push_back(std::tuple(it->cvalue().granuleID, it->range(), it->cvalue().startVersion));
+			} else if (it->cvalue().canMerge()) {
+				// set flag so this can get merged on the next pass
+				it->value().seen = true;
 			}
 		}
-	} else {
-		if (BM_DEBUG) {
-			fmt::print("BM {0} maybe merge [{1} - {2}): No mergeable granules after checking metrics\n",
-			           bmData->epoch,
-			           granuleRange.begin.printable(),
-			           granuleRange.end.printable());
+		if (currentCandidates.size() >= 2) {
+			mergeChecks.push_back(attemptMerges(bmData, currentCandidates));
 		}
-	}
 
-	return Void();
+		CODE_PROBE(mergeChecks.size() > 1, "parallel merge checks");
+		wait(waitForAll(mergeChecks));
+		// if the calculation took longer than the desired interval, still wait a bit
+		wait(intervalDelay && delay(5.0));
+	}
 }
 
 ACTOR Future<Void> deregisterBlobWorker(Reference<BlobManagerData> bmData, BlobWorkerInterface interf) {
@@ -1990,6 +2684,7 @@ ACTOR Future<Void> deregisterBlobWorker(Reference<BlobManagerData> bmData, BlobW
 	loop {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		try {
 			wait(checkManagerLock(tr, bmData));
 			Key blobWorkerListKey = blobWorkerListKeyFor(interf.id());
@@ -2130,7 +2825,9 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 					           rep.continueSeqno,
 					           bwInterf.id().toString(),
 					           rep.doSplit ? "split" : (rep.mergeCandidate ? "merge" : ""),
-					           rep.mergeCandidate ? "" : (rep.writeHotSplit ? "hot" : "normal"));
+					           rep.mergeCandidate
+					               ? ""
+					               : (rep.writeHotSplit ? "hot" : (rep.initialSplitTooBig ? "toobig" : "normal")));
 				}
 
 				ASSERT(rep.doSplit || rep.mergeCandidate);
@@ -2173,7 +2870,7 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 						}
 						ignore = true;
 					} else if (newEval < lastBoundaryEval.cvalue()) {
-						TEST(true); // BM got out-of-date split request
+						CODE_PROBE(true, "BM got out-of-date split request");
 						if (BM_DEBUG) {
 							fmt::print("BM {0} ignoring status from BW {1} for granule [{2} - {3}) {4} since it "
 							           "already processed [{5} - {6}) {7}.\n",
@@ -2249,9 +2946,9 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 					// suddenly gets a burst of writes after a decision to merge is made
 					if (inProgressMergeVersion != invalidVersion) {
 						if (rep.blockedVersion < inProgressMergeVersion) {
-							TEST(true); // merge blocking re-snapshot
+							CODE_PROBE(true, "merge blocking re-snapshot");
 							if (BM_DEBUG) {
-								fmt::print("DBG: BM {0} MERGE @ {1} blocking re-snapshot [{2} - {3}) @ {4}, "
+								fmt::print("BM {0} MERGE @ {1} blocking re-snapshot [{2} - {3}) @ {4}, "
 								           "continuing snapshot\n",
 								           bmData->epoch,
 								           inProgressMergeVersion,
@@ -2296,47 +2993,46 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 							           rep.granuleRange.end.printable().c_str(),
 							           newEval.toString());
 						}
-						newEval.inProgress = maybeSplitRange(bmData,
-						                                     bwInterf.id(),
-						                                     rep.granuleRange,
-						                                     rep.granuleID,
-						                                     rep.startVersion,
-						                                     rep.writeHotSplit,
-						                                     rep.originalEpoch,
-						                                     rep.originalSeqno);
+						if (rep.initialSplitTooBig) {
+							ASSERT(rep.proposedSplitKey.present());
+							newEval.inProgress = reevaluateInitialSplit(bmData,
+							                                            bwInterf.id(),
+							                                            rep.granuleRange,
+							                                            rep.granuleID,
+							                                            rep.originalEpoch,
+							                                            rep.originalSeqno,
+							                                            rep.proposedSplitKey.get());
+						} else {
+							newEval.inProgress = maybeSplitRange(bmData,
+							                                     bwInterf.id(),
+							                                     rep.granuleRange,
+							                                     rep.granuleID,
+							                                     rep.startVersion,
+							                                     rep.writeHotSplit,
+							                                     rep.originalEpoch,
+							                                     rep.originalSeqno);
+						}
 						bmData->boundaryEvaluations.insert(rep.granuleRange, newEval);
 					}
 
 					// clear merge candidates for range, if not already merging
 					if (clearMergeCandidate) {
-						bmData->mergeCandidates.insert(rep.granuleRange, Optional<std::pair<UID, Version>>());
+						bmData->clearMergeCandidate(rep.granuleRange);
 					}
 				}
 				if (rep.mergeCandidate && !ignore) {
+					// mark granule as merge candidate
 					ASSERT(!rep.doSplit);
-					// TODO: do we need any sort of validation that this is coming from the worker that currently owns
-					// the granule?
-					if (existingInProgress.present()) {
-						// TODO LOG?
-					} else {
-						if (BM_DEBUG) {
-							fmt::print("Manager {0} evaluating [{1} - {2}) {3}\n",
-							           bmData->epoch,
-							           rep.granuleRange.begin.printable().c_str(),
-							           rep.granuleRange.end.printable().c_str(),
-							           newEval.toString());
-						}
-						if (!bmData->isMergeActive(rep.granuleRange)) {
-							ASSERT(rep.mergeCandidate);
-							TEST(true); // Granule merge candidate
-							bmData->mergeCandidates.insert(rep.granuleRange,
-							                               std::pair(rep.granuleID, rep.startVersion));
-							newEval.inProgress =
-							    maybeMergeRange(bmData, rep.granuleID, rep.granuleRange, rep.startVersion);
-							// still update epoch/seqno even if not doing a merge eval
-							bmData->boundaryEvaluations.insert(rep.granuleRange, newEval);
-						}
+					CODE_PROBE(true, "Granule merge candidate");
+					if (BM_DEBUG) {
+						fmt::print("Manager {0} merge candidate granule [{1} - {2}) {3}\n",
+						           bmData->epoch,
+						           rep.granuleRange.begin.printable().c_str(),
+						           rep.granuleRange.end.printable().c_str(),
+						           newEval.toString());
 					}
+					bmData->boundaryEvaluations.insert(rep.granuleRange, newEval);
+					bmData->setMergeCandidate(rep.granuleRange, rep.granuleID, rep.startVersion);
 				}
 			}
 		} catch (Error& e) {
@@ -2363,7 +3059,7 @@ ACTOR Future<Void> monitorBlobWorkerStatus(Reference<BlobManagerData> bmData, Bl
 			// if it is permanent, the failure monitor will eventually trip.
 			ASSERT(e.code() != error_code_end_of_stream);
 			if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_connection_failed) {
-				TEST(true); // BM retrying BW monitoring
+				CODE_PROBE(true, "BM retrying BW monitoring");
 				wait(delay(backoff));
 				backoff = std::min(backoff * SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT,
 				                   SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX);
@@ -2501,7 +3197,7 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 		if (oldEpoch > newEpoch || (oldEpoch == newEpoch && oldSeqno > newSeqno)) {
 			newer.push_back(std::pair(old.range(), std::tuple(oldWorker, oldEpoch, oldSeqno)));
 			if (old.range() != newRange) {
-				TEST(true); // BM Recovery: BWs disagree on range boundaries
+				CODE_PROBE(true, "BM Recovery: BWs disagree on range boundaries");
 				anyConflicts = true;
 			}
 		} else {
@@ -2511,7 +3207,7 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 				ASSERT(oldEpoch != newEpoch || oldSeqno != newSeqno);
 			}
 			if (newEpoch == std::numeric_limits<int64_t>::max() && (oldWorker != newId || old.range() != newRange)) {
-				TEST(true); // BM Recovery: DB disagrees with workers
+				CODE_PROBE(true, "BM Recovery: DB disagrees with workers");
 				// new one is from DB (source of truth on boundaries) and existing mapping disagrees on boundary or
 				// assignment, do explicit revoke and re-assign to converge
 				anyConflicts = true;
@@ -2535,7 +3231,7 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 					std::get<0>(old.value()) = UID();
 				}
 				if (outOfDate.empty() || outOfDate.back() != std::pair(oldWorker, KeyRange(old.range()))) {
-					TEST(true); // BM Recovery: Two workers claim ownership of same granule
+					CODE_PROBE(true, "BM Recovery: Two workers claim ownership of same granule");
 					outOfDate.push_back(std::pair(oldWorker, old.range()));
 				}
 			}
@@ -2562,8 +3258,68 @@ static void addAssignment(KeyRangeMap<std::tuple<UID, int64_t, int64_t>>& map,
 	}
 }
 
-ACTOR Future<Void> resumeActiveMerges(Reference<BlobManagerData> bmData) {
+// essentially just error handling for resumed merge, since doMerge does it for new merge
+ACTOR Future<Void> resumeMerge(Future<Void> finishMergeFuture, KeyRange mergeRange) {
+	try {
+		wait(finishMergeFuture);
+		return Void();
+	} catch (Error& e) {
+		if (e.code() == error_code_operation_cancelled || e.code() == error_code_blob_manager_replaced) {
+			throw;
+		}
+		TraceEvent(SevError, "UnexpectedErrorResumeGranuleMerge").error(e).detail("Range", mergeRange);
+		throw e;
+	}
+}
+
+ACTOR Future<Void> loadForcePurgedRanges(Reference<BlobManagerData> bmData) {
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
+	state Key beginKey = blobGranuleForcePurgedKeys.begin;
+	state int rowLimit = BUGGIFY ? deterministicRandom()->randomInt(2, 10) : 10000;
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+
+			// using the krm functions can produce incorrect behavior here as it does weird stuff with beginKey
+			KeyRange nextRange(KeyRangeRef(beginKey, blobGranuleForcePurgedKeys.end));
+			state GetRangeLimits limits(rowLimit, GetRangeLimits::BYTE_LIMIT_UNLIMITED);
+			limits.minRows = 2;
+			RangeResult results = wait(tr->getRange(nextRange, limits));
+
+			// Add the mappings to our in memory key range map
+			for (int rangeIdx = 0; rangeIdx < results.size() - 1; rangeIdx++) {
+				if (results[rangeIdx].value == "1"_sr) {
+					Key rangeStartKey = results[rangeIdx].key.removePrefix(blobGranuleForcePurgedKeys.begin);
+					Key rangeEndKey = results[rangeIdx + 1].key.removePrefix(blobGranuleForcePurgedKeys.begin);
+					// note: if the old owner is dead, we handle this in rangeAssigner
+					bmData->forcePurgingRanges.insert(KeyRangeRef(rangeStartKey, rangeEndKey), true);
+				}
+			}
+
+			if (!results.more || results.size() <= 1) {
+				break;
+			}
+
+			// re-read last key to get range that starts there
+			beginKey = results.back().key;
+		} catch (Error& e) {
+			if (BM_DEBUG) {
+				fmt::print(
+				    "BM {0} got error reading force purge ranges during recovery: {1}\n", bmData->epoch, e.name());
+			}
+			wait(tr->onError(e));
+		}
+	}
+
+	return Void();
+}
+
+ACTOR Future<Void> resumeActiveMerges(Reference<BlobManagerData> bmData, Future<Void> loadForcePurgedRanges) {
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
+	wait(loadForcePurgedRanges); // load these first, to make sure to avoid resuming any merges that are being force
+	                             // purged
 
 	// FIXME: use range stream instead
 	state int rowLimit = BUGGIFY ? deterministicRandom()->randomInt(1, 10) : 10000;
@@ -2572,14 +3328,17 @@ ACTOR Future<Void> resumeActiveMerges(Reference<BlobManagerData> bmData) {
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			RangeResult result = wait(tr->getRange(currentRange, rowLimit));
+			state bool anyMore = result.more;
+			state std::vector<Future<Void>> cleanupForcePurged;
 			for (auto& it : result) {
-				TEST(true); // Blob Manager Recovery found merging granule
+				CODE_PROBE(true, "Blob Manager Recovery found merging granule");
 				UID mergeGranuleID = decodeBlobGranuleMergeKey(it.key);
 				KeyRange mergeRange;
 				std::vector<UID> parentGranuleIDs;
-				std::vector<KeyRange> parentGranuleRanges;
+				std::vector<Key> parentGranuleRanges;
 				std::vector<Version> parentGranuleStartVersions;
 				Version mergeVersion;
 				std::tie(mergeRange, mergeVersion, parentGranuleIDs, parentGranuleRanges, parentGranuleStartVersions) =
@@ -2593,24 +3352,51 @@ ACTOR Future<Void> resumeActiveMerges(Reference<BlobManagerData> bmData) {
 					           mergeVersion);
 				}
 
+				if (bmData->isForcePurging(mergeRange)) {
+					if (BM_DEBUG) {
+						fmt::print(
+						    "BM {0} cleaning up merge [{1} - {2}): {3} @ {4} in progress b/c it was force purged\n",
+						    bmData->epoch,
+						    mergeRange.begin.printable(),
+						    mergeRange.end.printable(),
+						    mergeGranuleID.toString().substr(0, 6),
+						    mergeVersion);
+					}
+					cleanupForcePurged.push_back(updateChangeFeed(&tr->getTransaction(),
+					                                              granuleIDToCFKey(mergeGranuleID),
+					                                              ChangeFeedStatus::CHANGE_FEED_DESTROY,
+					                                              mergeRange));
+					continue;
+				}
+
 				// want to mark in progress granule ranges as merging, BEFORE recovery is complete and workers can
 				// report updated status. Start with early (epoch, seqno) to guarantee lower than later status
 				BoundaryEvaluation eval(1, 0, BoundaryEvalType::MERGE, 1, 0);
 				ASSERT(!bmData->isMergeActive(mergeRange));
-				eval.inProgress = finishMergeGranules(bmData,
-				                                      mergeGranuleID,
-				                                      mergeRange,
-				                                      mergeVersion,
-				                                      parentGranuleIDs,
-				                                      parentGranuleRanges,
-				                                      parentGranuleStartVersions);
+				bmData->addActor.send(resumeMerge(finishMergeGranules(bmData,
+				                                                      mergeGranuleID,
+				                                                      mergeRange,
+				                                                      mergeVersion,
+				                                                      parentGranuleIDs,
+				                                                      parentGranuleRanges,
+				                                                      parentGranuleStartVersions),
+				                                  mergeRange));
 				bmData->boundaryEvaluations.insert(mergeRange, eval);
 				bmData->activeGranuleMerges.insert(mergeRange, mergeVersion);
+				bmData->setMergeCandidate(mergeRange, MergeCandidateMerging);
 			}
 
-			if (result.more) {
+			if (anyMore) {
 				currentRange = KeyRangeRef(keyAfter(result.back().key), currentRange.end);
-			} else {
+			}
+
+			if (!cleanupForcePurged.empty()) {
+				wait(waitForAll(cleanupForcePurged));
+				wait(tr->commit());
+				tr->reset();
+			}
+
+			if (!anyMore) {
 				return Void();
 			}
 		} catch (Error& e) {
@@ -2619,6 +3405,44 @@ ACTOR Future<Void> resumeActiveMerges(Reference<BlobManagerData> bmData) {
 	}
 }
 
+ACTOR Future<Void> loadBlobGranuleMergeBoundaries(Reference<BlobManagerData> bmData) {
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
+	state int rowLimit = BUGGIFY ? deterministicRandom()->randomInt(2, 10) : 10000;
+	state Key beginKey = blobGranuleMergeBoundaryKeys.begin;
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+
+			KeyRange nextRange(KeyRangeRef(beginKey, blobGranuleMergeBoundaryKeys.end));
+			// using the krm functions can produce incorrect behavior here as it does weird stuff with beginKey
+			state GetRangeLimits limits(rowLimit, GetRangeLimits::BYTE_LIMIT_UNLIMITED);
+			RangeResult results = wait(tr->getRange(nextRange, limits));
+
+			// Add the mappings to our in memory key range map
+			for (int i = 0; i < results.size(); i++) {
+				bmData->mergeBoundaries[results[i].key.removePrefix(blobGranuleMergeBoundaryKeys.begin)] =
+				    decodeBlobGranuleMergeBoundaryValue(results[i].value);
+			}
+
+			if (!results.more) {
+				break;
+			}
+			beginKey = keyAfter(results.back().key);
+		} catch (Error& e) {
+			if (BM_DEBUG) {
+				fmt::print("BM {0} got error reading granule merge boundaries during recovery: {1}\n",
+				           bmData->epoch,
+				           e.name());
+			}
+			wait(tr->onError(e));
+		}
+	}
+
+	return Void();
+}
+
 ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 	state double recoveryStartTime = now();
 	state Promise<Void> workerListReady;
@@ -2633,15 +3457,42 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 	// Once we acknowledge the existing blob workers, we can go ahead and recruit new ones
 	bmData->startRecruiting.trigger();
 
+	bmData->initBStore();
+	if (isFullRestoreMode())
+		wait(loadManifest(bmData->db, bmData->bstore));
+
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
+
+	// set up force purge keys if not done already
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			RangeResult existingForcePurgeKeys = wait(tr->getRange(blobGranuleForcePurgedKeys, 1));
+			if (!existingForcePurgeKeys.empty()) {
+				break;
+			}
+			wait(checkManagerLock(tr, bmData));
+			wait(krmSetRange(tr, blobGranuleForcePurgedKeys.begin, normalKeys, "0"_sr));
+			wait(tr->commit());
+			tr->reset();
+			break;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+
 	// skip the rest of the algorithm for the first blob manager
 	if (bmData->epoch == 1) {
 		bmData->doneRecovering.send(Void());
 		return Void();
 	}
 
-	state Future<Void> resumeMergesFuture = resumeActiveMerges(bmData);
+	state Future<Void> forcePurgedRanges = loadForcePurgedRanges(bmData);
+	state Future<Void> resumeMergesFuture = resumeActiveMerges(bmData, forcePurgedRanges);
 
-	TEST(true); // BM doing recovery
+	CODE_PROBE(true, "BM doing recovery");
 
 	wait(delay(0));
 
@@ -2663,7 +3514,6 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 
 	state KeyRangeMap<std::tuple<UID, int64_t, int64_t>> workerAssignments;
 	workerAssignments.insert(normalKeys, std::tuple(UID(), 0, 0));
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
 
 	// FIXME: use range stream instead
 	state int rowLimit = BUGGIFY ? deterministicRandom()->randomInt(2, 10) : 10000;
@@ -2722,7 +3572,7 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 				bmData->workerStats[workerId].numGranulesAssigned = reply.get().assignments.size();
 			}
 		} else {
-			TEST(true); // BM Recovery: BW didn't respond to assignments request
+			CODE_PROBE(true, "BM Recovery: BW didn't respond to assignments request");
 			// SOMEDAY: mark as failed and kill it
 			if (BM_DEBUG) {
 				fmt::print("  Worker {}: failed\n", workerId.toString().substr(0, 5));
@@ -2748,6 +3598,7 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			KeyRange nextRange(KeyRangeRef(beginKey, blobGranuleMappingKeys.end));
 			// using the krm functions can produce incorrect behavior here as it does weird stuff with beginKey
@@ -2798,6 +3649,7 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 		}
 	}
 
+	wait(forcePurgedRanges);
 	wait(resumeMergesFuture);
 
 	// Step 2. Send assign requests for all the granules and transfer assignments
@@ -2808,6 +3660,7 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			wait(checkManagerLock(tr, bmData));
 			wait(tr->commit());
 			break;
@@ -2826,7 +3679,7 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 	}
 
 	// revoke assignments that are old and incorrect
-	TEST(!outOfDateAssignments.empty()); // BM resolved conflicting assignments on recovery
+	CODE_PROBE(!outOfDateAssignments.empty(), "BM resolved conflicting assignments on recovery");
 	for (auto& it : outOfDateAssignments) {
 		if (BM_DEBUG) {
 			fmt::print("BM {0} revoking out of date assignment [{1} - {2}): {3}:\n",
@@ -2847,7 +3700,8 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 		fmt::print("BM {0} final ranges:\n", bmData->epoch);
 	}
 
-	int explicitAssignments = 0;
+	state int totalGranules = 0;
+	state int explicitAssignments = 0;
 	for (auto& range : workerAssignments.intersectingRanges(normalKeys)) {
 		int64_t epoch = std::get<1>(range.value());
 		int64_t seqno = std::get<2>(range.value());
@@ -2855,6 +3709,8 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 			continue;
 		}
 
+		totalGranules++;
+
 		UID workerId = std::get<0>(range.value());
 		bmData->workerAssignments.insert(range.range(), workerId);
 
@@ -2878,10 +3734,26 @@ ACTOR Future<Void> recoverBlobManager(Reference<BlobManagerData> bmData) {
 		}
 	}
 
+	// Load tenant data before letting blob granule operations continue.
+	tr->reset();
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			wait(loadTenantMap(tr, bmData));
+			break;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+
+	wait(loadBlobGranuleMergeBoundaries(bmData));
+
 	TraceEvent("BlobManagerRecovered", bmData->id)
 	    .detail("Epoch", bmData->epoch)
 	    .detail("Duration", now() - recoveryStartTime)
-	    .detail("Granules", bmData->workerAssignments.size()) // TODO this includes un-set ranges, so it is inaccurate
+	    .detail("Granules", totalGranules)
 	    .detail("Assigned", explicitAssignments)
 	    .detail("Revoked", outOfDateAssignments.size());
 
@@ -2896,11 +3768,11 @@ ACTOR Future<Void> chaosRangeMover(Reference<BlobManagerData> bmData) {
 	// KeyRange isn't hashable and this is only for simulation, so just use toString of range
 	state std::unordered_set<std::string> alreadyMoved;
 	ASSERT(g_network->isSimulated());
-	TEST(true); // BM chaos range mover enabled
+	CODE_PROBE(true, "BM chaos range mover enabled");
 	loop {
 		wait(delay(30.0));
 
-		if (g_simulator.speedUpSimulation) {
+		if (g_simulator->speedUpSimulation) {
 			if (BM_DEBUG) {
 				printf("Range mover stopping\n");
 			}
@@ -3000,7 +3872,7 @@ ACTOR Future<Void> initializeBlobWorker(Reference<BlobManagerData> self, Recruit
 		// if it failed in an expected way, add some delay before we try to recruit again
 		// on this worker
 		if (newBlobWorker.isError()) {
-			TEST(true); // BM got error recruiting BW
+			CODE_PROBE(true, "BM got error recruiting BW");
 			TraceEvent(SevWarn, "BMRecruitmentError", self->id)
 			    .error(newBlobWorker.getError())
 			    .detail("Epoch", self->epoch);
@@ -3104,7 +3976,7 @@ ACTOR Future<Void> blobWorkerRecruiter(
 			if (e.code() != error_code_timed_out) {
 				throw;
 			}
-			TEST(true); // Blob worker recruitment timed out
+			CODE_PROBE(true, "Blob worker recruitment timed out");
 		}
 	}
 }
@@ -3128,6 +4000,8 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Reference<BlobManagerData> bmData, U
 	state GranuleFiles files;
 	loop {
 		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID));
 			return files;
 		} catch (Error& e) {
@@ -3136,18 +4010,44 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Reference<BlobManagerData> bmData, U
 	}
 }
 
-ACTOR Future<Void> canDeleteFullGranule(Reference<BlobManagerData> self, UID granuleId) {
+ACTOR Future<bool> canDeleteFullGranuleSplit(Reference<BlobManagerData> self, UID granuleId) {
 	state Transaction tr(self->db);
 	state KeyRange splitRange = blobGranuleSplitKeyRangeFor(granuleId);
+	state KeyRange checkRange = splitRange;
+	state bool retry = false;
+
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Fully delete granule split check {1}\n", self->epoch, granuleId.toString());
+	}
 
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			state RangeResult splitState = wait(tr.getRange(splitRange, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT));
+			int lim = SERVER_KNOBS->BG_MAX_SPLIT_FANOUT;
+			if (BUGGIFY_WITH_PROB(0.1)) {
+				lim = deterministicRandom()->randomInt(1, std::max(2, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT));
+			}
+			state RangeResult splitState = wait(tr.getRange(checkRange, lim));
+			// if first try and empty, splitting state is fully cleaned up
+			if (!retry && checkRange == splitRange && splitState.empty() && !splitState.more) {
+				if (BM_PURGE_DEBUG) {
+					fmt::print("BM {0} Proceed with full deletion, no split state for {1}\n",
+					           self->epoch,
+					           granuleId.toString());
+				}
+				return true;
+			}
+			if (BM_PURGE_DEBUG) {
+				fmt::print("BM {0} Full delete check found {1} split states for {2}\n",
+				           self->epoch,
+				           splitState.size(),
+				           granuleId.toString());
+			}
 			state int i = 0;
-			state bool retry = false;
+
 			for (; i < splitState.size(); i++) {
 				UID parent, child;
 				BlobGranuleSplitState st;
@@ -3178,17 +4078,67 @@ ACTOR Future<Void> canDeleteFullGranule(Reference<BlobManagerData> self, UID gra
 			if (retry) {
 				tr.reset();
 				wait(delay(1.0));
+				retry = false;
+				checkRange = splitRange;
 			} else {
 				if (splitState.empty() || !splitState.more) {
 					break;
 				}
-				splitRange = KeyRangeRef(keyAfter(splitState.back().key), splitRange.end);
+				checkRange = KeyRangeRef(keyAfter(splitState.back().key), checkRange.end);
 			}
 		} catch (Error& e) {
 			wait(tr.onError(e));
 		}
 	}
-	return Void();
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Full delete check {1} done. Not deleting history key\n", self->epoch, granuleId.toString());
+	}
+	return false;
+}
+
+ACTOR Future<Void> canDeleteFullGranuleMerge(Reference<BlobManagerData> self, Optional<UID> mergeChildId) {
+	// if this granule is the parent of a merged granule, it needs to re-snapshot the merged granule before we can
+	// delete this one
+	if (!mergeChildId.present()) {
+		return Void();
+	}
+	CODE_PROBE(true, "checking canDeleteFullGranuleMerge");
+
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Fully delete granule merge check {1}\n", self->epoch, mergeChildId.get().toString());
+	}
+
+	state Transaction tr(self->db);
+	state KeyRange granuleFileRange = blobGranuleFileKeyRangeFor(mergeChildId.get());
+	// loop until granule has snapshotted
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			RangeResult files = wait(tr.getRange(granuleFileRange, 1));
+			if (!files.empty()) {
+				if (BM_PURGE_DEBUG) {
+					fmt::print("BM {0} Fully delete granule merge check {1} done\n",
+					           self->epoch,
+					           mergeChildId.get().toString());
+				}
+				return Void();
+			}
+			wait(delay(1.0));
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+ACTOR Future<bool> canDeleteFullGranule(Reference<BlobManagerData> self, UID granuleId, Optional<UID> mergeChildId) {
+	state Future<bool> split = canDeleteFullGranuleSplit(self, granuleId);
+	state Future<Void> merge = canDeleteFullGranuleMerge(self, mergeChildId);
+
+	wait(success(split) && merge);
+	bool canDeleteHistory = wait(split);
+	return canDeleteHistory;
 }
 
 static Future<Void> deleteFile(Reference<BlobConnectionProvider> bstoreProvider, std::string filePath) {
@@ -3222,14 +4172,30 @@ ACTOR Future<Void> fullyDeleteGranule(Reference<BlobManagerData> self,
                                       UID granuleId,
                                       Key historyKey,
                                       Version purgeVersion,
-                                      KeyRange granuleRange) {
-	if (BM_DEBUG) {
-		fmt::print("Fully deleting granule {0}: init\n", granuleId.toString());
+                                      KeyRange granuleRange,
+                                      Optional<UID> mergeChildID,
+                                      bool force) {
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Fully deleting granule [{1} - {2}): {3} @ {4}{5}\n",
+		           self->epoch,
+		           granuleRange.begin.printable(),
+		           granuleRange.end.printable(),
+		           granuleId.toString(),
+		           purgeVersion,
+		           force ? " (force)" : "");
 	}
 
 	// if granule is still splitting and files are needed for new sub-granules to re-snapshot, we can only partially
 	// delete the granule, since we need to keep the last snapshot and deltas for splitting
-	wait(canDeleteFullGranule(self, granuleId));
+	// Or, if the granule isn't finalized (still needs the history entry for the old change feed id, because all data
+	// from the old change feed hasn't yet been persisted in blob), we can delete the files but need to keep the granule
+	// history entry.
+	state bool canDeleteHistoryKey;
+	if (force) {
+		canDeleteHistoryKey = true;
+	} else {
+		wait(store(canDeleteHistoryKey, canDeleteFullGranule(self, granuleId, mergeChildID)));
+	}
 	state Reference<BlobConnectionProvider> bstore = wait(getBStoreForGranule(self, granuleRange));
 
 	// get files
@@ -3250,11 +4216,14 @@ ACTOR Future<Void> fullyDeleteGranule(Reference<BlobManagerData> self,
 		filesToDelete.emplace_back(fname);
 	}
 
-	if (BM_DEBUG) {
-		fmt::print("Fully deleting granule {0}: deleting {1} files\n", granuleId.toString(), filesToDelete.size());
-		for (auto filename : filesToDelete) {
-			fmt::print(" - {}\n", filename.c_str());
-		}
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Fully deleting granule {1}: deleting {2} files\n",
+		           self->epoch,
+		           granuleId.toString(),
+		           filesToDelete.size());
+		/*for (auto filename : filesToDelete) {
+		    fmt::print(" - {}\n", filename.c_str());
+		}*/
 	}
 
 	// delete the files before the corresponding metadata.
@@ -3264,19 +4233,33 @@ ACTOR Future<Void> fullyDeleteGranule(Reference<BlobManagerData> self,
 	wait(waitForAll(deletions));
 
 	// delete metadata in FDB (history entry and file keys)
-	if (BM_DEBUG) {
-		fmt::print("Fully deleting granule {0}: deleting history and file keys\n", granuleId.toString());
+	if (BM_PURGE_DEBUG) {
+		fmt::print(
+		    "BM {0} Fully deleting granule {1}: deleting history and file keys\n", self->epoch, granuleId.toString());
 	}
 
 	state Transaction tr(self->db);
-	tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
 	loop {
+		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 		try {
 			KeyRange fileRangeKey = blobGranuleFileKeyRangeFor(granuleId);
-			tr.clear(historyKey);
+			if (canDeleteHistoryKey) {
+				tr.clear(historyKey);
+			}
 			tr.clear(fileRangeKey);
+			if (force) {
+				// check manager lock to not delete metadata out from under a later recovering manager
+				wait(checkManagerLock(&tr, self));
+				wait(updateChangeFeed(
+				    &tr, granuleIDToCFKey(granuleId), ChangeFeedStatus::CHANGE_FEED_DESTROY, granuleRange));
+				tr.clear(blobGranuleLockKeyFor(granuleRange));
+				tr.clear(blobGranuleSplitKeyRangeFor(granuleId));
+				tr.clear(blobGranuleMergeKeyFor(granuleId));
+				// FIXME: also clear merge boundaries!
+			}
 			wait(tr.commit());
 			break;
 		} catch (Error& e) {
@@ -3284,11 +4267,14 @@ ACTOR Future<Void> fullyDeleteGranule(Reference<BlobManagerData> self,
 		}
 	}
 
-	if (BM_DEBUG) {
-		fmt::print("Fully deleting granule {0}: success\n", granuleId.toString());
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Fully deleting granule {1}: success {2}\n",
+		           self->epoch,
+		           granuleId.toString(),
+		           canDeleteHistoryKey ? "" : " ignoring history key!");
 	}
 
-	TraceEvent("GranuleFullPurge", self->id)
+	TraceEvent(SevDebug, "GranuleFullPurge", self->id)
 	    .detail("Epoch", self->epoch)
 	    .detail("GranuleID", granuleId)
 	    .detail("PurgeVersion", purgeVersion)
@@ -3297,6 +4283,8 @@ ACTOR Future<Void> fullyDeleteGranule(Reference<BlobManagerData> self,
 	++self->stats.granulesFullyPurged;
 	self->stats.filesPurged += filesToDelete.size();
 
+	CODE_PROBE(true, "full granule purged");
+
 	return Void();
 }
 
@@ -3312,8 +4300,8 @@ ACTOR Future<Void> partiallyDeleteGranule(Reference<BlobManagerData> self,
                                           UID granuleId,
                                           Version purgeVersion,
                                           KeyRange granuleRange) {
-	if (BM_DEBUG) {
-		fmt::print("Partially deleting granule {0}: init\n", granuleId.toString());
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Partially deleting granule {1}: init\n", self->epoch, granuleId.toString());
 	}
 
 	state Reference<BlobConnectionProvider> bstore = wait(getBStoreForGranule(self, granuleRange));
@@ -3362,11 +4350,14 @@ ACTOR Future<Void> partiallyDeleteGranule(Reference<BlobManagerData> self,
 		filesToDelete.emplace_back(fname);
 	}
 
-	if (BM_DEBUG) {
-		fmt::print("Partially deleting granule {0}: deleting {1} files\n", granuleId.toString(), filesToDelete.size());
-		for (auto filename : filesToDelete) {
-			fmt::print(" - {0}\n", filename);
-		}
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Partially deleting granule {1}: deleting {2} files\n",
+		           self->epoch,
+		           granuleId.toString(),
+		           filesToDelete.size());
+		/*for (auto filename : filesToDelete) {
+		    fmt::print(" - {0}\n", filename);
+		}*/
 	}
 
 	// TODO: the following comment relies on the assumption that BWs will not get requests to
@@ -3380,15 +4371,16 @@ ACTOR Future<Void> partiallyDeleteGranule(Reference<BlobManagerData> self,
 	wait(waitForAll(deletions));
 
 	// delete metadata in FDB (deleted file keys)
-	if (BM_DEBUG) {
-		fmt::print("Partially deleting granule {0}: deleting file keys\n", granuleId.toString());
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Partially deleting granule {1}: deleting file keys\n", self->epoch, granuleId.toString());
 	}
 
 	state Transaction tr(self->db);
-	tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
 	loop {
+		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 		try {
 			for (auto& key : deletedFileKeys) {
 				tr.clear(key);
@@ -3400,10 +4392,10 @@ ACTOR Future<Void> partiallyDeleteGranule(Reference<BlobManagerData> self,
 		}
 	}
 
-	if (BM_DEBUG) {
-		fmt::print("Partially deleting granule {0}: success\n", granuleId.toString());
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Partially deleting granule {1}: success\n", self->epoch, granuleId.toString());
 	}
-	TraceEvent("GranulePartialPurge", self->id)
+	TraceEvent(SevDebug, "GranulePartialPurge", self->id)
 	    .detail("Epoch", self->epoch)
 	    .detail("GranuleID", granuleId)
 	    .detail("PurgeVersion", purgeVersion)
@@ -3412,6 +4404,8 @@ ACTOR Future<Void> partiallyDeleteGranule(Reference<BlobManagerData> self,
 	++self->stats.granulesPartiallyPurged;
 	self->stats.filesPurged += filesToDelete.size();
 
+	CODE_PROBE(true, " partial granule purged");
+
 	return Void();
 }
 
@@ -3424,8 +4418,9 @@ ACTOR Future<Void> partiallyDeleteGranule(Reference<BlobManagerData> self,
  * processing this purge intent.
  */
 ACTOR Future<Void> purgeRange(Reference<BlobManagerData> self, KeyRangeRef range, Version purgeVersion, bool force) {
-	if (BM_DEBUG) {
-		fmt::print("purgeRange starting for range [{0} - {1}) @ purgeVersion={2}, force={3}\n",
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} purgeRange starting for range [{1} - {2}) @ purgeVersion={3}, force={4}\n",
+		           self->epoch,
 		           range.begin.printable(),
 		           range.end.printable(),
 		           purgeVersion,
@@ -3438,59 +4433,163 @@ ACTOR Future<Void> purgeRange(Reference<BlobManagerData> self, KeyRangeRef range
 	    .detail("PurgeVersion", purgeVersion)
 	    .detail("Force", force);
 
-	// queue of <range, startVersion, endVersion> for BFS traversal of history
-	state std::queue<std::tuple<KeyRange, Version, Version>> historyEntryQueue;
+	// queue of <range, startVersion, endVersion, mergeChildID> for BFS traversal of history
+	state std::queue<std::tuple<KeyRange, Version, Version, Optional<UID>>> historyEntryQueue;
 
-	// stacks of <granuleId, historyKey> and <granuleId> to track which granules to delete
-	state std::vector<std::tuple<UID, Key, KeyRange>> toFullyDelete;
+	// stacks of <granuleId, historyKey> and <granuleId> (and mergeChildID) to track which granules to delete
+	state std::vector<std::tuple<UID, Key, KeyRange, Optional<UID>>> toFullyDelete;
 	state std::vector<std::pair<UID, KeyRange>> toPartiallyDelete;
 
 	// track which granules we have already added to traversal
 	// note: (startKey, startVersion) uniquely identifies a granule
-	state std::unordered_set<std::pair<const uint8_t*, Version>, boost::hash<std::pair<const uint8_t*, Version>>>
-	    visited;
+	state std::unordered_set<std::pair<std::string, Version>, boost::hash<std::pair<std::string, Version>>> visited;
 
 	// find all active granules (that comprise the range) and add to the queue
-	state KeyRangeMap<UID>::Ranges activeRanges = self->workerAssignments.intersectingRanges(range);
 
 	state Transaction tr(self->db);
-	tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-	state KeyRangeMap<UID>::iterator activeRange;
-	for (activeRange = activeRanges.begin(); activeRange != activeRanges.end(); ++activeRange) {
-		if (BM_DEBUG) {
-			fmt::print("Checking if active range [{0} - {1}), owned by BW {2}, should be purged\n",
-			           activeRange.begin().printable(),
-			           activeRange.end().printable(),
-			           activeRange.value().toString());
+	if (force) {
+		// TODO could clean this up after force purge is done, but it's safer not to
+		self->forcePurgingRanges.insert(range, true);
+		// set force purged range, to prevent future operations on this range
+		loop {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			try {
+				// set force purged range, but don't clear mapping range yet, so that if a new BM recovers in the middle
+				// of purging, it still knows what granules to purge
+				wait(checkManagerLock(&tr, self));
+				// FIXME: need to handle this better if range is unaligned. Need to not truncate existing granules, and
+				// instead cover whole of intersecting granules at begin/end
+				wait(krmSetRangeCoalescing(&tr, blobGranuleForcePurgedKeys.begin, range, normalKeys, "1"_sr));
+				wait(tr.commit());
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+		tr.reset();
+		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+	}
+
+	// if range isn't in known blob ranges, do nothing after writing force purge range to database
+	bool anyKnownRanges = false;
+	auto knownRanges = self->knownBlobRanges.intersectingRanges(range);
+	for (auto& it : knownRanges) {
+		if (it.cvalue()) {
+			anyKnownRanges = true;
+			break;
+		}
+	}
+
+	if (!anyKnownRanges) {
+		CODE_PROBE(true, "skipping purge because not in known blob ranges");
+		TraceEvent("PurgeGranulesSkippingUnknownRange", self->id)
+		    .detail("Epoch", self->epoch)
+		    .detail("Range", range)
+		    .detail("PurgeVersion", purgeVersion)
+		    .detail("Force", force);
+		return Void();
+	}
+
+	// wait for all active splits and merges in the range to come to a stop, so no races with purging
+	std::vector<Future<Void>> activeBoundaryEvals;
+	auto boundaries = self->boundaryEvaluations.intersectingRanges(range);
+	for (auto& it : boundaries) {
+		auto& f = it.cvalue().inProgress;
+		if (f.isValid() && !f.isReady() && !f.isError()) {
+			activeBoundaryEvals.push_back(f);
+		}
+	}
+
+	if (!activeBoundaryEvals.empty()) {
+		wait(waitForAll(activeBoundaryEvals));
+	}
+
+	// some merges aren't counted in boundary evals, for merge/split race reasons
+	while (self->isMergeActive(range)) {
+		wait(delayJittered(1.0));
+	}
+
+	auto ranges = self->workerAssignments.intersectingRanges(range);
+	state std::vector<KeyRange> activeRanges;
+
+	// copy into state variable before waits
+	for (auto& it : ranges) {
+		activeRanges.push_back(it.range());
+	}
+
+	state std::set<Key> knownBoundariesPurged;
+
+	if (force) {
+		// revoke range from all active blob workers - AFTER we copy set of active ranges to purge
+		// if purge covers multiple blobbified ranges, revoke each separately
+		auto knownRanges = self->knownBlobRanges.intersectingRanges(range);
+		for (auto& it : knownRanges) {
+			if (it.cvalue()) {
+				RangeAssignment ra;
+				ra.isAssign = false;
+				ra.keyRange = range & it.range();
+				ra.revoke = RangeRevokeData(true); // dispose=true
+				if (ra.keyRange.begin > range.begin) {
+					knownBoundariesPurged.insert(ra.keyRange.begin);
+				}
+				if (ra.keyRange.end < range.end) {
+					knownBoundariesPurged.insert(ra.keyRange.end);
+				}
+				handleRangeAssign(self, ra);
+			}
+		}
+	}
+
+	state int rangeIdx;
+	for (rangeIdx = 0; rangeIdx < activeRanges.size(); rangeIdx++) {
+		state KeyRange activeRange = activeRanges[rangeIdx];
+		if (BM_PURGE_DEBUG) {
+			fmt::print("BM {0} Checking if active range [{1} - {2}) should be purged\n",
+			           self->epoch,
+			           activeRange.begin.printable(),
+			           activeRange.end.printable());
 		}
 
 		// assumption: purge boundaries must respect granule boundaries
-		if (activeRange.begin() < range.begin || activeRange.end() > range.end) {
+		if (activeRange.begin < range.begin || activeRange.end > range.end) {
+			TraceEvent(SevWarn, "GranulePurgeRangesUnaligned", self->id)
+			    .detail("Epoch", self->epoch)
+			    .detail("PurgeRange", range)
+			    .detail("GranuleRange", activeRange);
 			continue;
 		}
 
-		// TODO: if this is a force purge, then revoke the assignment from the corresponding BW first
-		// so that it doesn't try to interact with the granule (i.e. force it to give up gLock).
-		// we'll need some way to ack that the revoke was successful
-
 		loop {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			try {
-				if (BM_DEBUG) {
-					fmt::print("Fetching latest history entry for range [{0} - {1})\n",
-					           activeRange.begin().printable(),
-					           activeRange.end().printable());
+				if (BM_PURGE_DEBUG) {
+					fmt::print("BM {0} Fetching latest history entry for range [{1} - {2})\n",
+					           self->epoch,
+					           activeRange.begin.printable(),
+					           activeRange.end.printable());
 				}
-				Optional<GranuleHistory> history = wait(getLatestGranuleHistory(&tr, activeRange.range()));
+				// FIXME: doing this serially will likely be too slow for large purges
+				Optional<GranuleHistory> history = wait(getLatestGranuleHistory(&tr, activeRange));
 				// TODO: can we tell from the krm that this range is not valid, so that we don't need to do a
 				// get
 				if (history.present()) {
-					if (BM_DEBUG) {
-						printf("Adding range to history queue\n");
+					if (BM_PURGE_DEBUG) {
+						fmt::print("BM {0}   Adding range to history queue: [{1} - {2}) @ {3}\n",
+						           self->epoch,
+						           activeRange.begin.printable(),
+						           activeRange.end.printable(),
+						           history.get().version);
 					}
-					visited.insert({ activeRange.range().begin.begin(), history.get().version });
-					historyEntryQueue.push({ activeRange.range(), history.get().version, MAX_VERSION });
+					visited.insert({ activeRange.begin.toString(), history.get().version });
+					historyEntryQueue.push({ activeRange, history.get().version, MAX_VERSION, {} });
+				} else if (BM_PURGE_DEBUG) {
+					fmt::print("BM {0}   No history for range, ignoring\n", self->epoch);
 				}
 				break;
 			} catch (Error& e) {
@@ -3499,19 +4598,25 @@ ACTOR Future<Void> purgeRange(Reference<BlobManagerData> self, KeyRangeRef range
 		}
 	}
 
-	if (BM_DEBUG) {
-		printf("Beginning BFS traversal of history\n");
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0} Beginning BFS traversal of {1} history items for range [{2} - {3}) \n",
+		           self->epoch,
+		           historyEntryQueue.size(),
+		           range.begin.printable(),
+		           range.end.printable());
 	}
 	while (!historyEntryQueue.empty()) {
 		// process the node at the front of the queue and remove it
 		state KeyRange currRange;
 		state Version startVersion;
 		state Version endVersion;
-		std::tie(currRange, startVersion, endVersion) = historyEntryQueue.front();
+		state Optional<UID> mergeChildID;
+		std::tie(currRange, startVersion, endVersion, mergeChildID) = historyEntryQueue.front();
 		historyEntryQueue.pop();
 
-		if (BM_DEBUG) {
-			fmt::print("Processing history node [{0} - {1}) with versions [{2}, {3})\n",
+		if (BM_PURGE_DEBUG) {
+			fmt::print("BM {0} Processing history node [{1} - {2}) with versions [{3}, {4})\n",
+			           self->epoch,
 			           currRange.begin.printable(),
 			           currRange.end.printable(),
 			           startVersion,
@@ -3523,6 +4628,9 @@ ACTOR Future<Void> purgeRange(Reference<BlobManagerData> self, KeyRangeRef range
 		state Key historyKey = blobGranuleHistoryKeyFor(currRange, startVersion);
 		state bool foundHistory = false;
 		loop {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			try {
 				Optional<Value> persistedHistory = wait(tr.get(historyKey));
 				if (persistedHistory.present()) {
@@ -3536,11 +4644,15 @@ ACTOR Future<Void> purgeRange(Reference<BlobManagerData> self, KeyRangeRef range
 		}
 
 		if (!foundHistory) {
+			if (BM_PURGE_DEBUG) {
+				fmt::print("BM {0}  No history for this node, skipping\n", self->epoch);
+			}
 			continue;
 		}
 
-		if (BM_DEBUG) {
-			fmt::print("Found history entry for this node. It's granuleID is {0}\n",
+		if (BM_PURGE_DEBUG) {
+			fmt::print("BM {0}  Found history entry for this node. It's granuleID is {1}\n",
+			           self->epoch,
 			           currHistoryNode.granuleID.toString());
 		}
 
@@ -3551,39 +4663,56 @@ ACTOR Future<Void> purgeRange(Reference<BlobManagerData> self, KeyRangeRef range
 		//   and so this granule should be partially deleted
 		// - otherwise, this granule is active, so don't schedule it for deletion
 		if (force || endVersion <= purgeVersion) {
-			if (BM_DEBUG) {
-				fmt::print("Granule {0} will be FULLY deleted\n", currHistoryNode.granuleID.toString());
+			if (BM_PURGE_DEBUG) {
+				fmt::print(
+				    "BM {0}   Granule {1} will be FULLY deleted\n", self->epoch, currHistoryNode.granuleID.toString());
 			}
-			toFullyDelete.push_back({ currHistoryNode.granuleID, historyKey, currRange });
+			toFullyDelete.push_back({ currHistoryNode.granuleID, historyKey, currRange, mergeChildID });
 		} else if (startVersion < purgeVersion) {
-			if (BM_DEBUG) {
-				fmt::print("Granule {0} will be partially deleted\n", currHistoryNode.granuleID.toString());
+			if (BM_PURGE_DEBUG) {
+				fmt::print("BM {0}   Granule {1} will be partially deleted\n",
+				           self->epoch,
+				           currHistoryNode.granuleID.toString());
 			}
 			toPartiallyDelete.push_back({ currHistoryNode.granuleID, currRange });
 		}
 
 		// add all of the node's parents to the queue
-		for (auto& parent : currHistoryNode.parentGranules) {
+		if (BM_PURGE_DEBUG) {
+			fmt::print("BM {0}   Checking {1} parents\n", self->epoch, currHistoryNode.parentVersions.size());
+		}
+		Optional<UID> mergeChildID2 =
+		    currHistoryNode.parentVersions.size() > 1 ? currHistoryNode.granuleID : Optional<UID>();
+		for (int i = 0; i < currHistoryNode.parentVersions.size(); i++) {
+			// for (auto& parent : currHistoryNode.parentVersions.size()) {
 			// if we already added this node to queue, skip it; otherwise, mark it as visited
-			if (visited.count({ parent.first.begin.begin(), parent.second })) {
-				if (BM_DEBUG) {
-					fmt::print("Already added {0} to queue, so skipping it\n", currHistoryNode.granuleID.toString());
+			KeyRangeRef parentRange(currHistoryNode.parentBoundaries[i], currHistoryNode.parentBoundaries[i + 1]);
+			Version parentVersion = currHistoryNode.parentVersions[i];
+			std::string beginStr = parentRange.begin.toString();
+			if (!visited.insert({ beginStr, parentVersion }).second) {
+				if (BM_PURGE_DEBUG) {
+					fmt::print("BM {0}     Already added [{1} - {2}) @ {3} - {4} to queue, so skipping it\n",
+					           self->epoch,
+					           parentRange.begin.printable(),
+					           parentRange.end.printable(),
+					           parentVersion,
+					           startVersion);
 				}
 				continue;
 			}
-			visited.insert({ parent.first.begin.begin(), parent.second });
 
-			if (BM_DEBUG) {
-				fmt::print("Adding parent [{0} - {1}) with versions [{2} - {3}) to queue\n",
-				           parent.first.begin.printable(),
-				           parent.first.end.printable(),
-				           parent.second,
+			if (BM_PURGE_DEBUG) {
+				fmt::print("BM {0}     Adding parent [{1} - {2}) @ {3} - {4} to queue\n",
+				           self->epoch,
+				           parentRange.begin.printable(),
+				           parentRange.end.printable(),
+				           parentVersion,
 				           startVersion);
 			}
 
 			// the parent's end version is this node's startVersion,
 			// since this node must have started where it's parent finished
-			historyEntryQueue.push({ parent.first, parent.second, startVersion });
+			historyEntryQueue.push({ parentRange, parentVersion, startVersion, mergeChildID2 });
 		}
 	}
 
@@ -3602,46 +4731,86 @@ ACTOR Future<Void> purgeRange(Reference<BlobManagerData> self, KeyRangeRef range
 	// we won't run into any issues with trying to "re-delete" a blob file since deleting
 	// a file that doesn't exist is considered successful
 
+	TraceEvent("PurgeGranulesTraversalComplete", self->id)
+	    .detail("Epoch", self->epoch)
+	    .detail("Range", range)
+	    .detail("PurgeVersion", purgeVersion)
+	    .detail("Force", force)
+	    .detail("VisitedCount", visited.size())
+	    .detail("DeletingFullyCount", toFullyDelete.size())
+	    .detail("DeletingPartiallyCount", toPartiallyDelete.size());
+
 	state std::vector<Future<Void>> partialDeletions;
 	state int i;
-	if (BM_DEBUG) {
-		fmt::print("{0} granules to fully delete\n", toFullyDelete.size());
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0}: {1} granules to fully delete\n", self->epoch, toFullyDelete.size());
 	}
+	// Go backwards through set of granules to guarantee deleting oldest first. This avoids orphaning granules in the
+	// deletion process
+	// FIXME: could track explicit parent dependencies and parallelize so long as a parent and child aren't running in
+	// parallel, but that's non-trivial
 	for (i = toFullyDelete.size() - 1; i >= 0; --i) {
 		state UID granuleId;
 		Key historyKey;
 		KeyRange keyRange;
-		std::tie(granuleId, historyKey, keyRange) = toFullyDelete[i];
+		Optional<UID> mergeChildId;
+		std::tie(granuleId, historyKey, keyRange, mergeChildId) = toFullyDelete[i];
 		// FIXME: consider batching into a single txn (need to take care of txn size limit)
-		if (BM_DEBUG) {
-			fmt::print("About to fully delete granule {0}\n", granuleId.toString());
+		if (BM_PURGE_DEBUG) {
+			fmt::print("BM {0}: About to fully delete granule {1}\n", self->epoch, granuleId.toString());
 		}
-		wait(fullyDeleteGranule(self, granuleId, historyKey, purgeVersion, range));
+		wait(fullyDeleteGranule(self, granuleId, historyKey, purgeVersion, keyRange, mergeChildId, force));
 	}
 
-	if (BM_DEBUG) {
-		fmt::print("{0} granules to partially delete\n", toPartiallyDelete.size());
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0}: {1} granules to partially delete\n", self->epoch, toPartiallyDelete.size());
 	}
 
 	for (i = toPartiallyDelete.size() - 1; i >= 0; --i) {
 		UID granuleId;
-		KeyRange range;
-		std::tie(granuleId, range) = toPartiallyDelete[i];
-		if (BM_DEBUG) {
-			fmt::print("About to partially delete granule {0}\n", granuleId.toString());
+		KeyRange keyRange;
+		std::tie(granuleId, keyRange) = toPartiallyDelete[i];
+		if (BM_PURGE_DEBUG) {
+			fmt::print("BM {0}: About to partially delete granule {1}\n", self->epoch, granuleId.toString());
 		}
-		partialDeletions.emplace_back(partiallyDeleteGranule(self, granuleId, purgeVersion, range));
+		partialDeletions.emplace_back(partiallyDeleteGranule(self, granuleId, purgeVersion, keyRange));
 	}
 
 	wait(waitForAll(partialDeletions));
 
+	if (force) {
+		tr.reset();
+		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		loop {
+			try {
+				// clear mapping range, so that a new BM doesn't try to recover force purged granules, and clients can't
+				// read them
+				wait(checkManagerLock(&tr, self));
+				wait(krmSetRange(&tr, blobGranuleMappingKeys.begin, range, blobGranuleMappingValueFor(UID())));
+				// FIXME: there is probably a cleaner fix than setting extra keys in the database if someone does a
+				// purge that's not aligned to boundaries
+				for (auto& it : knownBoundariesPurged) {
+					// keep original bounds in granule mapping as to not confuse future managers on recovery
+					tr.set(it.withPrefix(blobGranuleMappingKeys.begin), blobGranuleMappingValueFor(UID()));
+				}
+				wait(tr.commit());
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
 	// Now that all the necessary granules and their files have been deleted, we can
 	// clear the purgeIntent key to signify that the work is done. However, there could have been
 	// another purgeIntent that got written for this table while we were processing this one.
 	// If that is the case, we should not clear the key. Otherwise, we can just clear the key.
 
-	if (BM_DEBUG) {
-		fmt::print("Successfully purged range [{0} - {1}) at purgeVersion={2}\n",
+	if (BM_PURGE_DEBUG) {
+		fmt::print("BM {0}: Successfully purged range [{1} - {2}) at purgeVersion={3}\n",
+		           self->epoch,
 		           range.begin.printable(),
 		           range.end.printable(),
 		           purgeVersion);
@@ -3653,6 +4822,8 @@ ACTOR Future<Void> purgeRange(Reference<BlobManagerData> self, KeyRangeRef range
 	    .detail("PurgeVersion", purgeVersion)
 	    .detail("Force", force);
 
+	CODE_PROBE(true, "range purge complete");
+
 	++self->stats.purgesProcessed;
 	return Void();
 }
@@ -3695,6 +4866,7 @@ ACTOR Future<Void> monitorPurgeKeys(Reference<BlobManagerData> self) {
 		loop {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			state std::vector<Future<Void>> purges;
 			state CoalescedKeyRangeMap<std::pair<Version, bool>> purgeMap;
@@ -3703,6 +4875,7 @@ ACTOR Future<Void> monitorPurgeKeys(Reference<BlobManagerData> self) {
 				// TODO: replace 10000 with a knob
 				state RangeResult purgeIntents = wait(tr->getRange(blobGranulePurgeKeys, BUGGIFY ? 1 : 10000));
 				if (purgeIntents.size()) {
+					CODE_PROBE(true, "BM found purges to process");
 					int rangeIdx = 0;
 					for (; rangeIdx < purgeIntents.size(); ++rangeIdx) {
 						Version purgeVersion;
@@ -3724,8 +4897,9 @@ ACTOR Future<Void> monitorPurgeKeys(Reference<BlobManagerData> self) {
 						}
 						purgeMap.insert(range, std::make_pair(purgeVersion, force));
 
-						if (BM_DEBUG) {
-							fmt::print("about to purge range [{0} - {1}) @ {2}, force={3}\n",
+						if (BM_PURGE_DEBUG) {
+							fmt::print("BM {0} about to purge range [{1} - {2}) @ {3}, force={4}\n",
+							           self->epoch,
 							           range.begin.printable(),
 							           range.end.printable(),
 							           purgeVersion,
@@ -3769,6 +4943,7 @@ ACTOR Future<Void> monitorPurgeKeys(Reference<BlobManagerData> self) {
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr->clear(KeyRangeRef(blobGranulePurgeKeys.begin, keyAfter(lastPurgeKey)));
 				wait(tr->commit());
 				break;
@@ -3777,9 +4952,11 @@ ACTOR Future<Void> monitorPurgeKeys(Reference<BlobManagerData> self) {
 			}
 		}
 
-		if (BM_DEBUG) {
-			printf("Done clearing current set of purge intents.\n");
+		if (BM_PURGE_DEBUG) {
+			fmt::print("BM {0} Done clearing current set of purge intents.\n", self->epoch);
 		}
+
+		CODE_PROBE(true, "BM finished processing purge intents");
 	}
 }
 
@@ -3789,7 +4966,7 @@ ACTOR Future<Void> doLockChecks(Reference<BlobManagerData> bmData) {
 		wait(check.getFuture());
 		wait(delay(0.5)); // don't do this too often if a lot of conflict
 
-		TEST(true); // BM doing lock checks after getting conflicts
+		CODE_PROBE(true, "BM doing lock checks after getting conflicts");
 
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bmData->db);
 
@@ -3797,6 +4974,7 @@ ACTOR Future<Void> doLockChecks(Reference<BlobManagerData> bmData) {
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				wait(checkManagerLock(tr, bmData));
 				wait(tr->commit());
 				break;
@@ -3869,6 +5047,28 @@ ACTOR Future<int64_t> bgccCheckGranule(Reference<BlobManagerData> bmData, KeyRan
 	return bytesRead;
 }
 
+// Check if there is any pending split. It's a precheck for manifest backup
+ACTOR Future<bool> hasPendingSplit(Reference<BlobManagerData> self) {
+	state Transaction tr(self->db);
+	loop {
+		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		try {
+			RangeResult result = wait(tr.getRange(blobGranuleSplitKeys, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+			for (auto& row : result) {
+				std::pair<BlobGranuleSplitState, Version> gss = decodeBlobGranuleSplitValue(row.value);
+				if (gss.first != BlobGranuleSplitState::Done) {
+					return true;
+				}
+			}
+			return false;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
 // FIXME: could eventually make this more thorough by storing some state in the DB or something
 // FIXME: simpler solution could be to shuffle ranges
 ACTOR Future<Void> bgConsistencyCheck(Reference<BlobManagerData> bmData) {
@@ -3880,15 +5080,25 @@ ACTOR Future<Void> bgConsistencyCheck(Reference<BlobManagerData> bmData) {
 	if (BM_DEBUG) {
 		fmt::print("BGCC starting\n");
 	}
+	if (isFullRestoreMode())
+		wait(printRestoreSummary(bmData->db, bmData->bstore));
 
 	loop {
-		if (g_network->isSimulated() && g_simulator.speedUpSimulation) {
+		if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
 			if (BM_DEBUG) {
 				printf("BGCC stopping\n");
 			}
 			return Void();
 		}
 
+		// Only dump blob manifest when there is no pending split to ensure data consistency
+		if (SERVER_KNOBS->BLOB_MANIFEST_BACKUP && !isFullRestoreMode()) {
+			bool pendingSplit = wait(hasPendingSplit(bmData));
+			if (!pendingSplit) {
+				wait(dumpManifest(bmData->db, bmData->bstore));
+			}
+		}
+
 		if (bmData->workersById.size() >= 1) {
 			int tries = 10;
 			state KeyRange range;
@@ -3938,6 +5148,15 @@ ACTOR Future<Void> bgConsistencyCheck(Reference<BlobManagerData> bmData) {
 // Simulation validation that multiple blob managers aren't started with the same epoch
 static std::map<int64_t, UID> managerEpochsSeen;
 
+ACTOR Future<Void> checkBlobManagerEpoch(Reference<AsyncVar<ServerDBInfo> const> dbInfo, int64_t epoch, UID dbgid) {
+	loop {
+		if (dbInfo->get().blobManager.present() && dbInfo->get().blobManager.get().epoch > epoch) {
+			throw worker_removed();
+		}
+		wait(dbInfo->onChange());
+	}
+}
+
 ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
                                Reference<AsyncVar<ServerDBInfo> const> dbInfo,
                                int64_t epoch) {
@@ -3953,10 +5172,11 @@ ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
 		managerEpochsSeen[epoch] = bmInterf.id();
 	}
 	state Reference<BlobManagerData> self =
-	    makeReference<BlobManagerData>(deterministicRandom()->randomUniqueID(),
+	    makeReference<BlobManagerData>(bmInterf.id(),
 	                                   dbInfo,
 	                                   openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True),
-	                                   bmInterf.locality.dcId());
+	                                   bmInterf.locality.dcId(),
+	                                   epoch);
 
 	state Future<Void> collection = actorCollection(self->addActor.getFuture());
 
@@ -3967,28 +5187,33 @@ ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
 
 	self->epoch = epoch;
 
-	// although we start the recruiter, we wait until existing workers are ack'd
-	auto recruitBlobWorker = IAsyncListener<RequestStream<RecruitBlobWorkerRequest>>::create(
-	    dbInfo, [](auto const& info) { return info.clusterInterface.recruitBlobWorker; });
-
-	self->addActor.send(blobWorkerRecruiter(self, recruitBlobWorker));
-
-	// we need to recover the old blob manager's state (e.g. granule assignments) before
-	// before the new blob manager does anything
-	wait(recoverBlobManager(self));
-
-	self->addActor.send(doLockChecks(self));
-	self->addActor.send(monitorClientRanges(self));
-	self->addActor.send(monitorPurgeKeys(self));
-	if (SERVER_KNOBS->BG_CONSISTENCY_CHECK_ENABLED) {
-		self->addActor.send(bgConsistencyCheck(self));
-	}
-
-	if (BUGGIFY) {
-		self->addActor.send(chaosRangeMover(self));
-	}
-
 	try {
+		// although we start the recruiter, we wait until existing workers are ack'd
+		auto recruitBlobWorker = IAsyncListener<RequestStream<RecruitBlobWorkerRequest>>::create(
+		    dbInfo, [](auto const& info) { return info.clusterInterface.recruitBlobWorker; });
+
+		self->addActor.send(blobWorkerRecruiter(self, recruitBlobWorker));
+		self->addActor.send(checkBlobManagerEpoch(dbInfo, epoch, bmInterf.id()));
+
+		// we need to recover the old blob manager's state (e.g. granule assignments) before
+		// before the new blob manager does anything
+		wait(recoverBlobManager(self) || collection);
+
+		self->addActor.send(doLockChecks(self));
+		self->addActor.send(monitorClientRanges(self));
+		self->addActor.send(monitorTenants(self));
+		self->addActor.send(monitorPurgeKeys(self));
+		if (SERVER_KNOBS->BG_CONSISTENCY_CHECK_ENABLED) {
+			self->addActor.send(bgConsistencyCheck(self));
+		}
+		if (SERVER_KNOBS->BG_ENABLE_MERGING) {
+			self->addActor.send(granuleMergeChecker(self));
+		}
+
+		if (BUGGIFY) {
+			self->addActor.send(chaosRangeMover(self));
+		}
+
 		loop choose {
 			when(wait(self->iAmReplaced.getFuture())) {
 				if (BM_DEBUG) {
@@ -4003,14 +5228,16 @@ ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
 				break;
 			}
 			when(state HaltBlobGranulesRequest req = waitNext(bmInterf.haltBlobGranules.getFuture())) {
-				wait(haltBlobGranules(self));
+				wait(haltBlobGranules(self) || collection);
 				req.reply.send(Void());
 				TraceEvent("BlobGranulesHalted", bmInterf.id()).detail("Epoch", epoch).detail("ReqID", req.requesterID);
 				break;
 			}
-			when(BlobManagerExclusionSafetyCheckRequest exclCheckReq =
-			         waitNext(bmInterf.blobManagerExclCheckReq.getFuture())) {
-				blobManagerExclusionSafetyCheck(self, exclCheckReq);
+			when(BlobManagerExclusionSafetyCheckRequest req = waitNext(bmInterf.blobManagerExclCheckReq.getFuture())) {
+				blobManagerExclusionSafetyCheck(self, req);
+			}
+			when(BlobManagerBlockedRequest req = waitNext(bmInterf.blobManagerBlockedReq.getFuture())) {
+				req.reply.send(BlobManagerBlockedReply(self->stats.blockedAssignments));
 			}
 			when(wait(collection)) {
 				TraceEvent(SevError, "BlobManagerActorCollectionError");
@@ -4021,6 +5248,9 @@ ACTOR Future<Void> blobManager(BlobManagerInterface bmInterf,
 	} catch (Error& err) {
 		TraceEvent("BlobManagerDied", bmInterf.id()).errorUnsuppressed(err);
 	}
+	// prevent a reference counting cycle
+	self->assignsInProgress = KeyRangeActorMap();
+	self->boundaryEvaluations.clear();
 	return Void();
 }
 
@@ -4044,47 +5274,44 @@ TEST_CASE("/blobmanager/updateranges") {
 	VectorRef<KeyRangeRef> added;
 	VectorRef<KeyRangeRef> removed;
 
-	StringRef active = LiteralStringRef("1");
-	StringRef inactive = StringRef();
-
 	RangeResult dbDataEmpty;
 	std::vector<std::pair<KeyRangeRef, bool>> kbrRanges;
 
-	StringRef keyA = StringRef(ar, LiteralStringRef("A"));
-	StringRef keyB = StringRef(ar, LiteralStringRef("B"));
-	StringRef keyC = StringRef(ar, LiteralStringRef("C"));
-	StringRef keyD = StringRef(ar, LiteralStringRef("D"));
+	StringRef keyA = StringRef(ar, "A"_sr);
+	StringRef keyB = StringRef(ar, "B"_sr);
+	StringRef keyC = StringRef(ar, "C"_sr);
+	StringRef keyD = StringRef(ar, "D"_sr);
 
 	// db data setup
 	RangeResult dbDataAB;
-	dbDataAB.emplace_back(ar, keyA, active);
-	dbDataAB.emplace_back(ar, keyB, inactive);
+	dbDataAB.emplace_back(ar, keyA, blobRangeActive);
+	dbDataAB.emplace_back(ar, keyB, blobRangeInactive);
 
 	RangeResult dbDataAC;
-	dbDataAC.emplace_back(ar, keyA, active);
-	dbDataAC.emplace_back(ar, keyC, inactive);
+	dbDataAC.emplace_back(ar, keyA, blobRangeActive);
+	dbDataAC.emplace_back(ar, keyC, blobRangeInactive);
 
 	RangeResult dbDataAD;
-	dbDataAD.emplace_back(ar, keyA, active);
-	dbDataAD.emplace_back(ar, keyD, inactive);
+	dbDataAD.emplace_back(ar, keyA, blobRangeActive);
+	dbDataAD.emplace_back(ar, keyD, blobRangeInactive);
 
 	RangeResult dbDataBC;
-	dbDataBC.emplace_back(ar, keyB, active);
-	dbDataBC.emplace_back(ar, keyC, inactive);
+	dbDataBC.emplace_back(ar, keyB, blobRangeActive);
+	dbDataBC.emplace_back(ar, keyC, blobRangeInactive);
 
 	RangeResult dbDataBD;
-	dbDataBD.emplace_back(ar, keyB, active);
-	dbDataBD.emplace_back(ar, keyD, inactive);
+	dbDataBD.emplace_back(ar, keyB, blobRangeActive);
+	dbDataBD.emplace_back(ar, keyD, blobRangeInactive);
 
 	RangeResult dbDataCD;
-	dbDataCD.emplace_back(ar, keyC, active);
-	dbDataCD.emplace_back(ar, keyD, inactive);
+	dbDataCD.emplace_back(ar, keyC, blobRangeActive);
+	dbDataCD.emplace_back(ar, keyD, blobRangeInactive);
 
 	RangeResult dbDataAB_CD;
-	dbDataAB_CD.emplace_back(ar, keyA, active);
-	dbDataAB_CD.emplace_back(ar, keyB, inactive);
-	dbDataAB_CD.emplace_back(ar, keyC, active);
-	dbDataAB_CD.emplace_back(ar, keyD, inactive);
+	dbDataAB_CD.emplace_back(ar, keyA, blobRangeActive);
+	dbDataAB_CD.emplace_back(ar, keyB, blobRangeInactive);
+	dbDataAB_CD.emplace_back(ar, keyC, blobRangeActive);
+	dbDataAB_CD.emplace_back(ar, keyD, blobRangeInactive);
 
 	// key ranges setup
 	KeyRangeRef rangeAB = KeyRangeRef(keyA, keyB);
diff --git a/fdbserver/BlobManifest.actor.cpp b/fdbserver/BlobManifest.actor.cpp
new file mode 100644
index 0000000000..5083f69b8f
--- /dev/null
+++ b/fdbserver/BlobManifest.actor.cpp
@@ -0,0 +1,374 @@
+/*
+ * BlobManifest.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/BackupContainer.h"
+#include "fdbserver/Knobs.h"
+#include "flow/FastRef.h"
+#include "flow/flow.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/BlobConnectionProvider.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/KeyRangeMap.h"
+#include "fdbclient/SystemData.h"
+#include "fdbclient/BackupContainerFileSystem.h"
+#include "fdbclient/BlobGranuleReader.actor.h"
+#include "fdbserver/BlobGranuleServerCommon.actor.h"
+
+#include "flow/actorcompiler.h" // has to be last include
+#include "fmt/core.h"
+
+//
+// This module offers routines to dump or load blob manifest file, which is used for full restore from granules
+//
+
+static std::string MANIFEST_FILENAME = "manifest"; // Default manifest file name on external blob storage
+
+#define ENABLE_DEBUG_PRINT true
+template <typename... T>
+inline void dprint(fmt::format_string<T...> fmt, T&&... args) {
+	if (ENABLE_DEBUG_PRINT)
+		fmt::print(fmt, std::forward<T>(args)...);
+}
+
+// This class dumps blob manifest to external blob storage.
+class BlobManifestDumper : public ReferenceCounted<BlobManifestDumper> {
+public:
+	BlobManifestDumper(Database& db, Reference<BlobConnectionProvider> blobConn) : db_(db), blobConn_(blobConn) {}
+	virtual ~BlobManifestDumper() {}
+
+	// Execute the dumper
+	ACTOR static Future<Void> execute(Reference<BlobManifestDumper> self) {
+		try {
+			state Standalone<BlobManifest> manifest;
+			Standalone<VectorRef<KeyValueRef>> rows = wait(getSystemKeys(self));
+			manifest.rows = rows;
+			Value data = encode(manifest);
+			wait(writeToFile(self, data));
+		} catch (Error& e) {
+			dprint("WARNING: unexpected blob manifest dumper error {}\n", e.what()); // skip error handling for now
+		}
+		return Void();
+	}
+
+private:
+	// Return system keys that to be backed up
+	ACTOR static Future<Standalone<VectorRef<KeyValueRef>>> getSystemKeys(Reference<BlobManifestDumper> self) {
+		state Standalone<VectorRef<KeyValueRef>> rows;
+		state Transaction tr(self->db_);
+		loop {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			try {
+				state std::vector<KeyRangeRef> ranges = {
+					blobGranuleMappingKeys, // Map granule to workers. Track the active granules
+					blobGranuleFileKeys, // Map a granule version to granule files. Track files for a granule
+					blobGranuleHistoryKeys, // Map granule to its parents and parent bundaries. for time-travel read
+					blobRangeKeys // Key ranges managed by blob
+				};
+				for (auto range : ranges) {
+					// todo use getRangeStream for better performance
+					RangeResult result = wait(tr.getRange(range, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+					for (auto& row : result) {
+						rows.push_back_deep(rows.arena(), KeyValueRef(row.key, row.value));
+					}
+				}
+				return rows;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	// Write data to blob manifest file
+	ACTOR static Future<Void> writeToFile(Reference<BlobManifestDumper> self, Value data) {
+		state Reference<BackupContainerFileSystem> writer;
+		state std::string fileName;
+
+		std::tie(writer, fileName) = self->blobConn_->createForWrite(MANIFEST_FILENAME);
+		state Reference<IBackupFile> file = wait(writer->writeFile(fileName));
+		wait(file->append(data.begin(), data.size()));
+		wait(file->finish());
+		dprint("Write blob manifest file with {} bytes\n", data.size());
+		return Void();
+	}
+
+	// Encode manifest as binary data
+	static Value encode(BlobManifest& manifest) {
+		BinaryWriter wr(IncludeVersion(ProtocolVersion::withBlobGranuleFile()));
+		wr << manifest;
+		return wr.toValue();
+	}
+
+	Database db_;
+	Reference<BlobConnectionProvider> blobConn_;
+};
+
+// Defines granule info that interests full restore
+struct BlobGranuleVersion {
+	// Two constructors required by VectorRef
+	BlobGranuleVersion() {}
+	BlobGranuleVersion(Arena& a, const BlobGranuleVersion& copyFrom)
+	  : granuleID(copyFrom.granuleID), keyRange(a, copyFrom.keyRange), version(copyFrom.version),
+	    sizeInBytes(copyFrom.sizeInBytes) {}
+
+	UID granuleID;
+	KeyRangeRef keyRange;
+	Version version;
+	int64_t sizeInBytes;
+};
+
+// Defines a vector for BlobGranuleVersion
+typedef Standalone<VectorRef<BlobGranuleVersion>> BlobGranuleVersionVector;
+
+// Defines filename, version, size for each granule file that interests full restore
+struct GranuleFileVersion {
+	Version version;
+	uint8_t fileType;
+	std::string filename;
+	int64_t sizeInBytes;
+};
+
+// This class is to load blob manifest into system key space, which is part of for bare metal restore
+class BlobManifestLoader : public ReferenceCounted<BlobManifestLoader> {
+public:
+	BlobManifestLoader(Database& db, Reference<BlobConnectionProvider> blobConn) : db_(db), blobConn_(blobConn) {}
+	virtual ~BlobManifestLoader() {}
+
+	// Execute the loader
+	ACTOR static Future<Void> execute(Reference<BlobManifestLoader> self) {
+		try {
+			Value data = wait(readFromFile(self));
+			Standalone<BlobManifest> manifest = decode(data);
+			wait(writeSystemKeys(self, manifest.rows));
+			BlobGranuleVersionVector _ = wait(listGranules(self));
+		} catch (Error& e) {
+			dprint("WARNING: unexpected manifest loader error {}\n", e.what()); // skip error handling so far
+		}
+		return Void();
+	}
+
+	// Print out a summary for blob granules
+	ACTOR static Future<Void> print(Reference<BlobManifestLoader> self) {
+		state BlobGranuleVersionVector granules = wait(listGranules(self));
+		for (auto granule : granules) {
+			wait(checkGranuleFiles(self, granule));
+		}
+		return Void();
+	}
+
+private:
+	// Read data from a manifest file
+	ACTOR static Future<Value> readFromFile(Reference<BlobManifestLoader> self) {
+		state Reference<BackupContainerFileSystem> readBstore = self->blobConn_->getForRead(MANIFEST_FILENAME);
+		state Reference<IAsyncFile> reader = wait(readBstore->readFile(MANIFEST_FILENAME));
+		state int64_t fileSize = wait(reader->size());
+		state Arena arena;
+		state uint8_t* data = new (arena) uint8_t[fileSize];
+		int readSize = wait(reader->read(data, fileSize, 0));
+		dprint("Blob manifest restoring {} bytes\n", readSize);
+		StringRef ref = StringRef(data, readSize);
+		return Value(ref, arena);
+	}
+
+	// Decode blob manifest from binary data
+	static Standalone<BlobManifest> decode(Value data) {
+		Standalone<BlobManifest> manifest;
+		BinaryReader binaryReader(data, IncludeVersion());
+		binaryReader >> manifest;
+		return manifest;
+	}
+
+	// Write system keys to database
+	ACTOR static Future<Void> writeSystemKeys(Reference<BlobManifestLoader> self, VectorRef<KeyValueRef> rows) {
+		state Transaction tr(self->db_);
+		loop {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			try {
+				for (auto& row : rows) {
+					tr.set(row.key, row.value);
+				}
+				wait(tr.commit());
+				dprint("Blob manifest loaded {} rows\n", rows.size());
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	// Iterate active granules and return their version/sizes
+	ACTOR static Future<BlobGranuleVersionVector> listGranules(Reference<BlobManifestLoader> self) {
+		state Transaction tr(self->db_);
+		loop {
+			state BlobGranuleVersionVector results;
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+
+			try {
+				std::vector<KeyRangeRef> granules;
+				state int i = 0;
+				auto limit = GetRangeLimits::BYTE_LIMIT_UNLIMITED;
+				state RangeResult blobRanges = wait(tr.getRange(blobGranuleMappingKeys, limit));
+				for (i = 0; i < blobRanges.size() - 1; i++) {
+					Key startKey = blobRanges[i].key.removePrefix(blobGranuleMappingKeys.begin);
+					Key endKey = blobRanges[i + 1].key.removePrefix(blobGranuleMappingKeys.begin);
+					state KeyRange granuleRange = KeyRangeRef(startKey, endKey);
+					try {
+						Standalone<BlobGranuleVersion> granule = wait(getGranule(&tr, granuleRange));
+						results.push_back_deep(results.arena(), granule);
+					} catch (Error& e) {
+						dprint("missing data for key range {} \n", granuleRange.toString());
+					}
+				}
+				return results;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	// Find the newest granule for a key range. The newest granule has the max version and relevant files
+	ACTOR static Future<Standalone<BlobGranuleVersion>> getGranule(Transaction* tr, KeyRangeRef range) {
+		state Standalone<BlobGranuleVersion> granuleVersion;
+		KeyRange historyKeyRange = blobGranuleHistoryKeyRangeFor(range);
+		// reverse lookup so that the first row is the newest version
+		state RangeResult results =
+		    wait(tr->getRange(historyKeyRange, GetRangeLimits::BYTE_LIMIT_UNLIMITED, Snapshot::False, Reverse::True));
+
+		for (KeyValueRef row : results) {
+			state KeyRange keyRange;
+			state Version version;
+			std::tie(keyRange, version) = decodeBlobGranuleHistoryKey(row.key);
+			Standalone<BlobGranuleHistoryValue> historyValue = decodeBlobGranuleHistoryValue(row.value);
+			state UID granuleID = historyValue.granuleID;
+
+			std::vector<GranuleFileVersion> files = wait(listGranuleFiles(tr, granuleID));
+			if (files.empty()) {
+				dprint("Granule {} doesn't have files for version {}\n", granuleID.toString(), version);
+				continue; // check previous version
+			}
+
+			granuleVersion.keyRange = KeyRangeRef(granuleVersion.arena(), keyRange);
+			granuleVersion.granuleID = granuleID;
+			granuleVersion.version = files.back().version;
+			granuleVersion.sizeInBytes = granuleSizeInBytes(files);
+
+			dprint("Granule {}: \n", granuleVersion.granuleID.toString());
+			dprint("  {} {} {}\n", keyRange.toString(), granuleVersion.version, granuleVersion.sizeInBytes);
+			for (auto& file : files) {
+				dprint("  File {}: {} bytes\n", file.filename, file.sizeInBytes);
+			}
+			return granuleVersion;
+		}
+		throw restore_missing_data(); // todo a better error code
+	}
+
+	// Return sum of last snapshot file size and delta files afterwards
+	static int64_t granuleSizeInBytes(std::vector<GranuleFileVersion> files) {
+		int64_t totalSize = 0;
+		for (auto it = files.rbegin(); it < files.rend(); ++it) {
+			totalSize += it->sizeInBytes;
+			if (it->fileType == BG_FILE_TYPE_SNAPSHOT)
+				break;
+		}
+		return totalSize;
+	}
+
+	// List all files for given granule
+	ACTOR static Future<std::vector<GranuleFileVersion>> listGranuleFiles(Transaction* tr, UID granuleID) {
+		state KeyRange fileKeyRange = blobGranuleFileKeyRangeFor(granuleID);
+		RangeResult results = wait(tr->getRange(fileKeyRange, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+
+		std::vector<GranuleFileVersion> files;
+		for (auto& row : results) {
+			UID gid;
+			Version version;
+			uint8_t fileType;
+			Standalone<StringRef> filename;
+			int64_t offset;
+			int64_t length;
+			int64_t fullFileLength;
+			Optional<BlobGranuleCipherKeysMeta> cipherKeysMeta;
+
+			std::tie(gid, version, fileType) = decodeBlobGranuleFileKey(row.key);
+			std::tie(filename, offset, length, fullFileLength, cipherKeysMeta) = decodeBlobGranuleFileValue(row.value);
+			GranuleFileVersion vs = { version, fileType, filename.toString(), length };
+			files.push_back(vs);
+		}
+		return files;
+	}
+
+	// Read data from granules and print out summary
+	ACTOR static Future<Void> checkGranuleFiles(Reference<BlobManifestLoader> self, BlobGranuleVersion granule) {
+		state KeyRangeRef range = granule.keyRange;
+		state Version readVersion = granule.version;
+		state Transaction tr(self->db_);
+		loop {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			try {
+				state Standalone<VectorRef<BlobGranuleChunkRef>> chunks =
+				    wait(tr.readBlobGranules(range, 0, readVersion));
+				state int count = 0;
+				for (const BlobGranuleChunkRef& chunk : chunks) {
+					RangeResult rows = wait(readBlobGranule(chunk, range, 0, readVersion, self->blobConn_));
+					count += rows.size();
+				}
+
+				dprint("Restorable blob granule {} @ {}\n", granule.granuleID.toString(), readVersion);
+				dprint("  Range: {}\n", range.toString());
+				dprint("  Keys : {}\n", count);
+				dprint("  Size : {} bytes\n", granule.sizeInBytes);
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	Database db_;
+	Reference<BlobConnectionProvider> blobConn_;
+};
+
+// API to dump a manifest copy to external storage
+ACTOR Future<Void> dumpManifest(Database db, Reference<BlobConnectionProvider> blobConn) {
+	Reference<BlobManifestDumper> dumper = makeReference<BlobManifestDumper>(db, blobConn);
+	wait(BlobManifestDumper::execute(dumper));
+	return Void();
+}
+
+// API to load manifest from external blob storage
+ACTOR Future<Void> loadManifest(Database db, Reference<BlobConnectionProvider> blobConn) {
+	Reference<BlobManifestLoader> loader = makeReference<BlobManifestLoader>(db, blobConn);
+	wait(BlobManifestLoader::execute(loader));
+	return Void();
+}
+
+// API to print summary for restorable granules
+ACTOR Future<Void> printRestoreSummary(Database db, Reference<BlobConnectionProvider> blobConn) {
+	Reference<BlobManifestLoader> loader = makeReference<BlobManifestLoader>(db, blobConn);
+	wait(BlobManifestLoader::print(loader));
+	return Void();
+}
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 45aa814a36..2776fe22fc 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/ClientBooleanParams.h"
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/BlobGranuleFiles.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/KeyRangeMap.h"
@@ -36,14 +37,14 @@
 #include "fdbclient/Notified.h"
 
 #include "fdbserver/BlobGranuleServerCommon.actor.h"
-#include "fdbserver/GetEncryptCipherKeys.actor.h"
+#include "fdbserver/EncryptionOpsUtils.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/MutationTracking.h"
 #include "fdbserver/ServerDBInfo.h"
 #include "fdbserver/WaitFailure.h"
 
 #include "flow/Arena.h"
-#include "flow/BlobCipher.h"
 #include "flow/CompressionUtils.h"
 #include "flow/EncryptUtils.h"
 #include "flow/Error.h"
@@ -83,7 +84,6 @@ struct GranuleStartState {
 	Optional<GranuleHistory> history;
 };
 
-// FIXME: add global byte limit for pending and buffered deltas
 struct GranuleMetadata : NonCopyable, ReferenceCounted<GranuleMetadata> {
 	KeyRange keyRange;
 
@@ -174,8 +174,6 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted<BlobWorkerData> {
 	UID id;
 	Database db;
 
-	BlobWorkerStats stats;
-
 	PromiseStream<Future<Void>> addActor;
 
 	LocalityData locality;
@@ -202,14 +200,32 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted<BlobWorkerData> {
 	NotifiedVersion grvVersion;
 	Promise<Void> fatalError;
 
-	FlowLock initialSnapshotLock;
+	Reference<FlowLock> initialSnapshotLock;
+	Reference<FlowLock> resnapshotLock;
+	Reference<FlowLock> deltaWritesLock;
+
+	BlobWorkerStats stats;
+
 	bool shuttingDown = false;
 
-	int changeFeedStreamReplyBufferSize = SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES / 2;
+	int changeFeedStreamReplyBufferSize = SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES / 4;
 
-	BlobWorkerData(UID id, Reference<AsyncVar<ServerDBInfo> const> dbInf, Database db)
-	  : id(id), db(db), stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL), tenantData(BGTenantMap(dbInf)), dbInfo(dbInf),
-	    initialSnapshotLock(SERVER_KNOBS->BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM) {}
+	bool isEncryptionEnabled = false;
+	bool buggifyFull = false;
+
+	int64_t memoryFullThreshold =
+	    (int64_t)(SERVER_KNOBS->BLOB_WORKER_REJECT_WHEN_FULL_THRESHOLD * SERVER_KNOBS->SERVER_MEM_LIMIT);
+	int64_t lastResidentMemory = 0;
+	double lastResidentMemoryCheckTime = -100.0;
+
+	BlobWorkerData(UID id, Reference<AsyncVar<ServerDBInfo> const> dbInfo, Database db)
+	  : id(id), db(db), tenantData(BGTenantMap(dbInfo)), dbInfo(dbInfo),
+	    initialSnapshotLock(new FlowLock(SERVER_KNOBS->BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM)),
+	    resnapshotLock(new FlowLock(SERVER_KNOBS->BLOB_WORKER_RESNAPSHOT_PARALLELISM)),
+	    deltaWritesLock(new FlowLock(SERVER_KNOBS->BLOB_WORKER_DELTA_FILE_WRITE_PARALLELISM)),
+	    stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, initialSnapshotLock, resnapshotLock, deltaWritesLock),
+	    isEncryptionEnabled(
+	        isEncryptionOpSupported(EncryptOperationType::BLOB_GRANULE_ENCRYPTION, db->clientInfo->get())) {}
 
 	bool managerEpochOk(int64_t epoch) {
 		if (epoch < currentManagerEpoch) {
@@ -231,14 +247,54 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted<BlobWorkerData> {
 			return true;
 		}
 	}
+
+	bool isFull() {
+		if (!SERVER_KNOBS->BLOB_WORKER_DO_REJECT_WHEN_FULL) {
+			return false;
+		}
+		if (g_network->isSimulated()) {
+			if (g_simulator->speedUpSimulation) {
+				return false;
+			}
+			return buggifyFull;
+		}
+
+		// TODO knob?
+		if (now() >= 1.0 + lastResidentMemoryCheckTime) {
+			// fdb as of 7.1 limits on resident memory instead of virtual memory
+			stats.lastResidentMemory = getResidentMemoryUsage();
+			lastResidentMemoryCheckTime = now();
+		}
+
+		// if we are already over threshold, no need to estimate extra memory
+		if (stats.lastResidentMemory >= memoryFullThreshold) {
+			return true;
+		}
+
+		// FIXME: since this isn't tested in simulation, could unit test this
+		// Try to model how much memory we *could* use given the already existing assignments and workload on this blob
+		// worker, before agreeing to take on a new assignment, given that several large sources of memory can grow and
+		// change post-assignment
+
+		// estimate slack in bytes buffered as max(0, assignments * (delta file size / 2) - bytesBuffered)
+		int64_t expectedExtraBytesBuffered = std::max(
+		    0, stats.numRangesAssigned * (SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES / 2) - stats.mutationBytesBuffered);
+		// estimate slack in potential pending delta file writes
+		int64_t maximumExtraDeltaWrite = SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES * deltaWritesLock->available();
+		// estimate slack in potential pending resnapshot
+		int64_t maximumExtraReSnapshot =
+		    (SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES + SERVER_KNOBS->BG_DELTA_BYTES_BEFORE_COMPACT) * 2 *
+		    resnapshotLock->available();
+
+		int64_t totalExtra = expectedExtraBytesBuffered + maximumExtraDeltaWrite + maximumExtraReSnapshot;
+		// assumes initial snapshot parallelism is small enough and uncommon enough to not add it to this computation
+		stats.estimatedMaxResidentMemory = stats.lastResidentMemory + totalExtra;
+
+		return stats.estimatedMaxResidentMemory >= memoryFullThreshold;
+	}
 };
 
 namespace {
-bool isBlobFileEncryptionSupported() {
-	bool supported = SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION && SERVER_KNOBS->BG_RANGE_SOURCE == "tenant";
-	ASSERT((supported && SERVER_KNOBS->ENABLE_ENCRYPTION) || !supported);
-	return supported;
-}
 
 Optional<CompressionFilter> getBlobFileCompressFilter() {
 	Optional<CompressionFilter> compFilter;
@@ -302,19 +358,20 @@ ACTOR Future<BlobGranuleCipherKeysCtx> getLatestGranuleCipherKeys(Reference<Blob
 	ASSERT(tenantData.isValid());
 
 	std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
-	domains.emplace(tenantData->entry.id, StringRef(*arena, tenantData->name));
+	domains.emplace(tenantData->entry.id, tenantData->name);
 	std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> domainKeyMap =
-	    wait(getLatestEncryptCipherKeys(bwData->dbInfo, domains));
+	    wait(getLatestEncryptCipherKeys(bwData->dbInfo, domains, BlobCipherMetrics::BLOB_GRANULE));
 
 	auto domainKeyItr = domainKeyMap.find(tenantData->entry.id);
 	ASSERT(domainKeyItr != domainKeyMap.end());
 	cipherKeysCtx.textCipherKey = BlobGranuleCipherKey::fromBlobCipherKey(domainKeyItr->second, *arena);
 
-	TextAndHeaderCipherKeys systemCipherKeys = wait(getLatestSystemEncryptCipherKeys(bwData->dbInfo));
+	TextAndHeaderCipherKeys systemCipherKeys =
+	    wait(getLatestSystemEncryptCipherKeys(bwData->dbInfo, BlobCipherMetrics::BLOB_GRANULE));
 	cipherKeysCtx.headerCipherKey = BlobGranuleCipherKey::fromBlobCipherKey(systemCipherKeys.cipherHeaderKey, *arena);
 
 	cipherKeysCtx.ivRef = makeString(AES_256_IV_LENGTH, *arena);
-	generateRandomData(mutateString(cipherKeysCtx.ivRef), AES_256_IV_LENGTH);
+	deterministicRandom()->randomBytes(mutateString(cipherKeysCtx.ivRef), AES_256_IV_LENGTH);
 
 	if (BG_ENCRYPT_COMPRESS_DEBUG) {
 		TraceEvent(SevDebug, "GetLatestGranuleCipherKey")
@@ -336,7 +393,7 @@ ACTOR Future<BlobGranuleCipherKey> lookupCipherKey(Reference<BlobWorkerData> bwD
 	std::unordered_set<BlobCipherDetails> cipherDetailsSet;
 	cipherDetailsSet.emplace(cipherDetails);
 	state std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> cipherKeyMap =
-	    wait(getEncryptCipherKeys(bwData->dbInfo, cipherDetailsSet));
+	    wait(getEncryptCipherKeys(bwData->dbInfo, cipherDetailsSet, BlobCipherMetrics::BLOB_GRANULE));
 
 	ASSERT(cipherKeyMap.size() == 1);
 
@@ -352,29 +409,27 @@ ACTOR Future<BlobGranuleCipherKey> lookupCipherKey(Reference<BlobWorkerData> bwD
 	return BlobGranuleCipherKey::fromBlobCipherKey(cipherKeyMapItr->second, *arena);
 }
 
-ACTOR Future<BlobGranuleCipherKeysCtx> getGranuleCipherKeys(Reference<BlobWorkerData> bwData,
-                                                            BlobGranuleCipherKeysMetaRef cipherKeysMetaRef,
-                                                            Arena* arena) {
+ACTOR Future<BlobGranuleCipherKeysCtx> getGranuleCipherKeysImpl(Reference<BlobWorkerData> bwData,
+                                                                BlobCipherDetails textCipherDetails,
+                                                                BlobCipherDetails headerCipherDetails,
+                                                                StringRef ivRef,
+                                                                Arena* arena) {
 	state BlobGranuleCipherKeysCtx cipherKeysCtx;
 
 	// Fetch 'textCipher' key
-	state BlobCipherDetails textCipherDetails(
-	    cipherKeysMetaRef.textDomainId, cipherKeysMetaRef.textBaseCipherId, cipherKeysMetaRef.textSalt);
 	BlobGranuleCipherKey textCipherKey = wait(lookupCipherKey(bwData, textCipherDetails, arena));
 	cipherKeysCtx.textCipherKey = textCipherKey;
 
 	// Fetch 'headerCipher' key
-	state BlobCipherDetails headerCipherDetails(
-	    cipherKeysMetaRef.headerDomainId, cipherKeysMetaRef.headerBaseCipherId, cipherKeysMetaRef.headerSalt);
 	BlobGranuleCipherKey headerCipherKey = wait(lookupCipherKey(bwData, headerCipherDetails, arena));
 	cipherKeysCtx.headerCipherKey = headerCipherKey;
 
 	// Populate 'Intialization Vector'
-	ASSERT_EQ(cipherKeysMetaRef.ivRef.size(), AES_256_IV_LENGTH);
-	cipherKeysCtx.ivRef = StringRef(*arena, cipherKeysMetaRef.ivRef);
+	ASSERT_EQ(ivRef.size(), AES_256_IV_LENGTH);
+	cipherKeysCtx.ivRef = StringRef(*arena, ivRef);
 
 	if (BG_ENCRYPT_COMPRESS_DEBUG) {
-		TraceEvent("GetGranuleCipherKey")
+		TraceEvent(SevDebug, "GetGranuleCipherKey")
 		    .detail("TextDomainId", cipherKeysCtx.textCipherKey.encryptDomainId)
 		    .detail("TextBaseCipherId", cipherKeysCtx.textCipherKey.baseCipherId)
 		    .detail("TextSalt", cipherKeysCtx.textCipherKey.salt)
@@ -387,6 +442,32 @@ ACTOR Future<BlobGranuleCipherKeysCtx> getGranuleCipherKeys(Reference<BlobWorker
 	return cipherKeysCtx;
 }
 
+Future<BlobGranuleCipherKeysCtx> getGranuleCipherKeysFromKeysMeta(Reference<BlobWorkerData> bwData,
+                                                                  BlobGranuleCipherKeysMeta cipherKeysMeta,
+                                                                  Arena* arena) {
+	BlobCipherDetails textCipherDetails(
+	    cipherKeysMeta.textDomainId, cipherKeysMeta.textBaseCipherId, cipherKeysMeta.textSalt);
+
+	BlobCipherDetails headerCipherDetails(
+	    cipherKeysMeta.headerDomainId, cipherKeysMeta.headerBaseCipherId, cipherKeysMeta.headerSalt);
+
+	StringRef ivRef = StringRef(*arena, cipherKeysMeta.ivStr);
+
+	return getGranuleCipherKeysImpl(bwData, textCipherDetails, headerCipherDetails, ivRef, arena);
+}
+
+Future<BlobGranuleCipherKeysCtx> getGranuleCipherKeysFromKeysMetaRef(Reference<BlobWorkerData> bwData,
+                                                                     BlobGranuleCipherKeysMetaRef cipherKeysMetaRef,
+                                                                     Arena* arena) {
+	BlobCipherDetails textCipherDetails(
+	    cipherKeysMetaRef.textDomainId, cipherKeysMetaRef.textBaseCipherId, cipherKeysMetaRef.textSalt);
+
+	BlobCipherDetails headerCipherDetails(
+	    cipherKeysMetaRef.headerDomainId, cipherKeysMetaRef.headerBaseCipherId, cipherKeysMetaRef.headerSalt);
+
+	return getGranuleCipherKeysImpl(bwData, textCipherDetails, headerCipherDetails, cipherKeysMetaRef.ivRef, arena);
+}
+
 ACTOR Future<Void> readAndCheckGranuleLock(Reference<ReadYourWritesTransaction> tr,
                                            KeyRange granuleRange,
                                            int64_t epoch,
@@ -394,7 +475,12 @@ ACTOR Future<Void> readAndCheckGranuleLock(Reference<ReadYourWritesTransaction>
 	state Key lockKey = blobGranuleLockKeyFor(granuleRange);
 	Optional<Value> lockValue = wait(tr->get(lockKey));
 
-	ASSERT(lockValue.present());
+	if (!lockValue.present()) {
+		// FIXME: could add some validation for simulation that a force purge was initiated
+		// for lock to be deleted out from under an active granule means a force purge must have happened.
+		throw granule_assignment_conflict();
+	}
+
 	std::tuple<int64_t, int64_t, UID> currentOwner = decodeBlobGranuleLockValue(lockValue.get());
 	checkGranuleLock(epoch, seqno, std::get<0>(currentOwner), std::get<1>(currentOwner));
 
@@ -414,6 +500,8 @@ ACTOR Future<GranuleFiles> loadHistoryFiles(Reference<BlobWorkerData> bwData, UI
 	state GranuleFiles files;
 	loop {
 		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID));
 			return files;
 		} catch (Error& e) {
@@ -539,7 +627,7 @@ ACTOR Future<Void> updateGranuleSplitState(Transaction* tr,
 
 			// tr->clear(singleKeyRange(oldGranuleLockKey));
 			tr->clear(currentRange);
-			TEST(true); // Granule split cleanup on last delta file persisted
+			CODE_PROBE(true, "Granule split cleanup on last delta file persisted");
 		} else {
 			tr->atomicOp(myStateKey, blobGranuleSplitValueFor(newState), MutationRef::SetVersionstampedValue);
 			if (newState == BlobGranuleSplitState::Assigned && currentState == BlobGranuleSplitState::Initialized &&
@@ -554,10 +642,10 @@ ACTOR Future<Void> updateGranuleSplitState(Transaction* tr,
 				wait(updateChangeFeed(
 				    tr, KeyRef(granuleIDToCFKey(parentGranuleID)), ChangeFeedStatus::CHANGE_FEED_STOP));
 			}
-			TEST(true); // Granule split stopping change feed
+			CODE_PROBE(true, "Granule split stopping change feed");
 		}
 	} else if (BW_DEBUG) {
-		TEST(true); // Out of order granule split state updates ignored
+		CODE_PROBE(true, "Out of order granule split state updates ignored");
 		fmt::print("Ignoring granule {0} split state from {1} {2} -> {3}\n",
 		           currentGranuleID.toString(),
 		           parentGranuleID.toString(),
@@ -597,13 +685,35 @@ ACTOR Future<BlobFileIndex> writeDeltaFile(Reference<BlobWorkerData> bwData,
                                            Version currentDeltaVersion,
                                            Future<BlobFileIndex> previousDeltaFileFuture,
                                            Future<Void> waitCommitted,
-                                           Optional<std::pair<KeyRange, UID>> oldGranuleComplete) {
+                                           Optional<std::pair<KeyRange, UID>> oldGranuleComplete,
+                                           Future<Void> startDeltaFileWrite) {
+	wait(startDeltaFileWrite);
+	state FlowLock::Releaser holdingLock(*bwData->deltaWritesLock);
+
 	wait(delay(0, TaskPriority::BlobWorkerUpdateStorage));
 
 	state std::string fileName = randomBGFilename(bwData->id, granuleID, currentDeltaVersion, ".delta");
 
-	state Value serialized = ObjectWriter::toValue(deltasToWrite, Unversioned());
+	state Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx;
+	state Optional<BlobGranuleCipherKeysMeta> cipherKeysMeta;
+	state Arena arena;
+
+	if (bwData->isEncryptionEnabled) {
+		BlobGranuleCipherKeysCtx ciphKeysCtx = wait(getLatestGranuleCipherKeys(bwData, keyRange, &arena));
+		cipherKeysCtx = std::move(ciphKeysCtx);
+		cipherKeysMeta = BlobGranuleCipherKeysCtx::toCipherKeysMeta(cipherKeysCtx.get());
+	}
+
+	state Optional<CompressionFilter> compressFilter = getBlobFileCompressFilter();
+	state Value serialized = serializeChunkedDeltaFile(StringRef(fileName),
+	                                                   deltasToWrite,
+	                                                   keyRange,
+	                                                   SERVER_KNOBS->BG_DELTA_FILE_TARGET_CHUNK_BYTES,
+	                                                   compressFilter,
+	                                                   cipherKeysCtx);
 	state size_t serializedSize = serialized.size();
+	bwData->stats.compressionBytesRaw += deltasToWrite.expectedSize();
+	bwData->stats.compressionBytesFinal += serializedSize;
 
 	// Free up deltasToWrite here to reduce memory
 	deltasToWrite = Standalone<GranuleDeltas>();
@@ -623,6 +733,10 @@ ACTOR Future<BlobFileIndex> writeDeltaFile(Reference<BlobWorkerData> bwData,
 	// free serialized since it is persisted in blob
 	serialized = Value();
 
+	// now that all buffered memory from file is gone, we can release memory flow lock
+	// we must unblock here to allow feed to continue to consume, so that waitCommitted returns
+	holdingLock.release();
+
 	state int numIterations = 0;
 	try {
 		// before updating FDB, wait for the delta file version to be committed and previous delta files to finish
@@ -634,13 +748,15 @@ ACTOR Future<BlobFileIndex> writeDeltaFile(Reference<BlobWorkerData> bwData,
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bwData->db);
 		loop {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			try {
 				wait(readAndCheckGranuleLock(tr, keyRange, epoch, seqno));
 				numIterations++;
 
 				Key dfKey = blobGranuleFileKeyFor(granuleID, currentDeltaVersion, 'D');
 				// TODO change once we support file multiplexing
-				Value dfValue = blobGranuleFileValueFor(fname, 0, serializedSize, serializedSize);
+				Value dfValue = blobGranuleFileValueFor(fname, 0, serializedSize, serializedSize, cipherKeysMeta);
 				tr->set(dfKey, dfValue);
 
 				if (oldGranuleComplete.present()) {
@@ -667,8 +783,16 @@ ACTOR Future<BlobFileIndex> writeDeltaFile(Reference<BlobWorkerData> bwData,
 				if (BUGGIFY_WITH_PROB(0.01)) {
 					wait(delay(deterministicRandom()->random01()));
 				}
+
+				if (BW_DEBUG) {
+					TraceEvent(SevDebug, "DeltaFileWritten")
+					    .detail("FileName", fname)
+					    .detail("Encrypted", cipherKeysCtx.present())
+					    .detail("Compressed", compressFilter.present());
+				}
+
 				// FIXME: change when we implement multiplexing
-				return BlobFileIndex(currentDeltaVersion, fname, 0, serializedSize, serializedSize);
+				return BlobFileIndex(currentDeltaVersion, fname, 0, serializedSize, serializedSize, cipherKeysMeta);
 			} catch (Error& e) {
 				wait(tr->onError(e));
 			}
@@ -678,19 +802,26 @@ ACTOR Future<BlobFileIndex> writeDeltaFile(Reference<BlobWorkerData> bwData,
 		// commit a transaction, we can and want to safely delete the file we wrote. Otherwise, we may have updated FDB
 		// with file and cannot safely delete it.
 		if (numIterations > 0) {
-			TEST(true); // Granule potentially leaving orphaned delta file
+			CODE_PROBE(true, "Granule potentially leaving orphaned delta file");
 			throw e;
 		}
 		if (BW_DEBUG) {
 			fmt::print("deleting delta file {0} after error {1}\n", fname, e.name());
 		}
-		TEST(true); // Granule cleaning up delta file after error
+		CODE_PROBE(true, "Granule cleaning up delta file after error");
 		++bwData->stats.s3DeleteReqs;
 		bwData->addActor.send(writeBStore->deleteFile(fname));
 		throw e;
 	}
 }
 
+ACTOR Future<Void> reevaluateInitialSplit(Reference<BlobWorkerData> bwData,
+                                          UID granuleID,
+                                          KeyRange keyRange,
+                                          int64_t epoch,
+                                          int64_t seqno,
+                                          Key proposedSplitKey);
+
 ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
                                           Reference<BlobConnectionProvider> bstore,
                                           KeyRange keyRange,
@@ -699,33 +830,59 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
                                           int64_t seqno,
                                           Version version,
                                           PromiseStream<RangeResult> rows,
-                                          bool createGranuleHistory) {
+                                          bool initialSnapshot) {
 	state std::string fileName = randomBGFilename(bwData->id, granuleID, version, ".snapshot");
 	state Standalone<GranuleSnapshot> snapshot;
+	state int64_t bytesRead = 0;
+	state bool injectTooBig = initialSnapshot && g_network->isSimulated() && BUGGIFY_WITH_PROB(0.1);
 
 	wait(delay(0, TaskPriority::BlobWorkerUpdateStorage));
 
 	loop {
 		try {
+			if (initialSnapshot && snapshot.size() > 1 &&
+			    (injectTooBig || bytesRead >= 3 * SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES)) {
+				// throw transaction too old either on injection for simulation, or if snapshot would be too large now
+				throw transaction_too_old();
+			}
 			RangeResult res = waitNext(rows.getFuture());
 			snapshot.arena().dependsOn(res.arena());
 			snapshot.append(snapshot.arena(), res.begin(), res.size());
+			bytesRead += res.expectedSize();
 			wait(yield(TaskPriority::BlobWorkerUpdateStorage));
 		} catch (Error& e) {
 			if (e.code() == error_code_end_of_stream) {
 				break;
 			}
-			throw e;
+			// if we got transaction_too_old naturally, have lower threshold for re-evaluating (2xlimit)
+			if (initialSnapshot && snapshot.size() > 1 && e.code() == error_code_transaction_too_old &&
+			    (injectTooBig || bytesRead >= 2 * SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES)) {
+				// idle this actor, while we tell the manager this is too big and to re-evaluate granules and revoke us
+				if (BW_DEBUG) {
+					fmt::print("Granule [{0} - {1}) re-evaluating snapshot after {2} bytes ({3} limit) {4}\n",
+					           keyRange.begin.printable(),
+					           keyRange.end.printable(),
+					           bytesRead,
+					           SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES,
+					           injectTooBig ? "(injected)" : "");
+				}
+				wait(reevaluateInitialSplit(
+				    bwData, granuleID, keyRange, epoch, seqno, snapshot[snapshot.size() / 2].key));
+				ASSERT(false);
+			} else {
+				throw e;
+			}
 		}
 	}
 
 	wait(delay(0, TaskPriority::BlobWorkerUpdateStorage));
 
 	if (BW_DEBUG) {
-		fmt::print("Granule [{0} - {1}) read {2} snapshot rows\n",
+		fmt::print("Granule [{0} - {1}) read {2} snapshot rows ({3} bytes)\n",
 		           keyRange.begin.printable(),
 		           keyRange.end.printable(),
-		           snapshot.size());
+		           snapshot.size(),
+		           bytesRead);
 	}
 
 	if (g_network->isSimulated()) {
@@ -746,16 +903,22 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
 	state Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx;
 	state Optional<BlobGranuleCipherKeysMeta> cipherKeysMeta;
 	state Arena arena;
-	if (isBlobFileEncryptionSupported()) {
+
+	if (bwData->isEncryptionEnabled) {
 		BlobGranuleCipherKeysCtx ciphKeysCtx = wait(getLatestGranuleCipherKeys(bwData, keyRange, &arena));
-		cipherKeysCtx = ciphKeysCtx;
+		cipherKeysCtx = std::move(ciphKeysCtx);
 		cipherKeysMeta = BlobGranuleCipherKeysCtx::toCipherKeysMeta(cipherKeysCtx.get());
 	}
 
-	Optional<CompressionFilter> compressFilter = getBlobFileCompressFilter();
-	state Value serialized =
-	    serializeChunkedSnapshot(snapshot, SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_CHUNKS, compressFilter, cipherKeysCtx);
+	state Optional<CompressionFilter> compressFilter = getBlobFileCompressFilter();
+	state Value serialized = serializeChunkedSnapshot(StringRef(fileName),
+	                                                  snapshot,
+	                                                  SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_CHUNK_BYTES,
+	                                                  compressFilter,
+	                                                  cipherKeysCtx);
 	state size_t serializedSize = serialized.size();
+	bwData->stats.compressionBytesRaw += snapshot.expectedSize();
+	bwData->stats.compressionBytesFinal += serializedSize;
 
 	// free snapshot to reduce memory
 	snapshot = Standalone<GranuleSnapshot>();
@@ -785,6 +948,8 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
 	try {
 		loop {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			try {
 				wait(readAndCheckGranuleLock(tr, keyRange, epoch, seqno));
 				numIterations++;
@@ -794,7 +959,7 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
 				    blobGranuleFileValueFor(fname, 0, serializedSize, serializedSize, cipherKeysMeta);
 				tr->set(snapshotFileKey, snapshotFileValue);
 				// create granule history at version if this is a new granule with the initial dump from FDB
-				if (createGranuleHistory) {
+				if (initialSnapshot) {
 					Key historyKey = blobGranuleHistoryKeyFor(keyRange, version);
 					Standalone<BlobGranuleHistoryValue> historyValue;
 					historyValue.granuleID = granuleID;
@@ -811,13 +976,13 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
 		// commit a transaction, we can and want to safely delete the file we wrote. Otherwise, we may have updated FDB
 		// with file and cannot safely delete it.
 		if (numIterations > 0) {
-			TEST(true); // Granule potentially leaving orphaned snapshot file
+			CODE_PROBE(true, "Granule potentially leaving orphaned snapshot file");
 			throw e;
 		}
 		if (BW_DEBUG) {
 			fmt::print("deleting snapshot file {0} after error {1}\n", fname, e.name());
 		}
-		TEST(true); // Granule deleting snapshot file after error
+		CODE_PROBE(true, "Granule deleting snapshot file after error");
 		++bwData->stats.s3DeleteReqs;
 		bwData->addActor.send(writeBStore->deleteFile(fname));
 		throw e;
@@ -835,6 +1000,13 @@ ACTOR Future<BlobFileIndex> writeSnapshot(Reference<BlobWorkerData> bwData,
 		wait(delay(deterministicRandom()->random01()));
 	}
 
+	if (BW_DEBUG) {
+		TraceEvent(SevDebug, "SnapshotFileWritten")
+		    .detail("FileName", fileName)
+		    .detail("Encrypted", cipherKeysCtx.present())
+		    .detail("Compressed", compressFilter.present());
+	}
+
 	// FIXME: change when we implement multiplexing
 	return BlobFileIndex(version, fname, 0, serializedSize, serializedSize, cipherKeysMeta);
 }
@@ -850,19 +1022,19 @@ ACTOR Future<BlobFileIndex> dumpInitialSnapshotFromFDB(Reference<BlobWorkerData>
 		           metadata->keyRange.begin.printable(),
 		           metadata->keyRange.end.printable());
 	}
-	wait(bwData->initialSnapshotLock.take());
-	state FlowLock::Releaser holdingDVL(bwData->initialSnapshotLock);
+	wait(bwData->initialSnapshotLock->take());
+	state FlowLock::Releaser holdingLock(*bwData->initialSnapshotLock);
 
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bwData->db);
-	state int64_t bytesRead = 0;
 	state int retries = 0;
 	state Version lastReadVersion = invalidVersion;
 	state Version readVersion = invalidVersion;
 
 	loop {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		// FIXME: proper tenant support in Blob Worker
 		tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		try {
 			Version rv = wait(tr->getReadVersion());
 			readVersion = rv;
@@ -886,19 +1058,18 @@ ACTOR Future<BlobFileIndex> dumpInitialSnapshotFromFDB(Reference<BlobWorkerData>
 			DEBUG_KEY_RANGE("BlobWorkerFDBSnapshot", readVersion, metadata->keyRange, bwData->id);
 
 			// initial snapshot is committed in fdb, we can pop the change feed up to this version
-			inFlightPops->push_back(bwData->db->popChangeFeedMutations(cfKey, readVersion));
+			inFlightPops->push_back(bwData->db->popChangeFeedMutations(cfKey, readVersion + 1));
 			return snapshotWriter.get();
 		} catch (Error& e) {
 			if (e.code() == error_code_operation_cancelled) {
 				throw e;
 			}
 			if (BW_DEBUG) {
-				fmt::print("Dumping snapshot {0} from FDB for [{1} - {2}) got error {3} after {4} bytes\n",
+				fmt::print("Dumping snapshot {0} from FDB for [{1} - {2}) got error {3}\n",
 				           retries + 1,
 				           metadata->keyRange.begin.printable(),
 				           metadata->keyRange.end.printable(),
-				           e.name(),
-				           bytesRead);
+				           e.name());
 			}
 			state Error err = e;
 			if (e.code() == error_code_server_overloaded) {
@@ -907,20 +1078,21 @@ ACTOR Future<BlobFileIndex> dumpInitialSnapshotFromFDB(Reference<BlobWorkerData>
 				wait(tr->onError(e));
 			}
 			retries++;
-			TEST(true); // Granule initial snapshot failed
+			CODE_PROBE(true, "Granule initial snapshot failed");
 			// FIXME: why can't we supress error event?
 			TraceEvent(retries < 10 ? SevDebug : SevWarn, "BlobGranuleInitialSnapshotRetry", bwData->id)
 			    .error(err)
 			    .detail("Granule", metadata->keyRange)
 			    .detail("Count", retries);
-			bytesRead = 0;
 			lastReadVersion = readVersion;
 			// Pop change feed up to readVersion, because that data will be before the next snapshot
 			// Do this to prevent a large amount of CF data from accumulating if we have consecutive failures to
 			// snapshot
 			// Also somewhat servers as a rate limiting function and checking that the database is available for this
 			// key range
-			wait(bwData->db->popChangeFeedMutations(cfKey, readVersion));
+			// FIXME: can't do this because this granule might not own the shard anymore and another worker might have
+			// successfully snapshotted already, but if it got an error before even getting to the lock check, it
+			// wouldn't realize wait(bwData->db->popChangeFeedMutations(cfKey, readVersion));
 		}
 	}
 }
@@ -933,6 +1105,8 @@ ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
                                             UID granuleID,
                                             std::vector<GranuleFiles> fileSet,
                                             Version version) {
+	wait(bwData->resnapshotLock->take());
+	state FlowLock::Releaser holdingLock(*bwData->resnapshotLock);
 	wait(delay(0, TaskPriority::BlobWorkerUpdateStorage));
 	if (BW_DEBUG) {
 		fmt::print("Compacting snapshot from blob for [{0} - {1}) @ {2}\n",
@@ -962,48 +1136,66 @@ ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
 		}
 		ASSERT(snapshotVersion < version);
 
+		state Optional<BlobGranuleCipherKeysCtx> snapCipherKeysCtx;
+		if (snapshotF.cipherKeysMeta.present()) {
+			ASSERT(bwData->isEncryptionEnabled);
+
+			BlobGranuleCipherKeysCtx keysCtx =
+			    wait(getGranuleCipherKeysFromKeysMeta(bwData, snapshotF.cipherKeysMeta.get(), &filenameArena));
+			snapCipherKeysCtx = std::move(keysCtx);
+		}
+
 		chunk.snapshotFile = BlobFilePointerRef(filenameArena,
 		                                        snapshotF.filename,
 		                                        snapshotF.offset,
 		                                        snapshotF.length,
 		                                        snapshotF.fullFileLength,
-		                                        snapshotF.cipherKeysMeta);
-
-		// TODO: optimization - batch 'encryption-key' lookup given the GranuleFile set is known
-		if (chunk.snapshotFile.get().cipherKeysMetaRef.present()) {
-			ASSERT(isBlobFileEncryptionSupported());
-			BlobGranuleCipherKeysCtx cipherKeysCtx =
-			    wait(getGranuleCipherKeys(bwData, chunk.snapshotFile.get().cipherKeysMetaRef.get(), &filenameArena));
-			chunk.cipherKeysCtx = cipherKeysCtx;
-		}
+		                                        snapCipherKeysCtx);
 
 		compactBytesRead += snapshotF.length;
-		int deltaIdx = files.deltaFiles.size() - 1;
+		state int deltaIdx = files.deltaFiles.size() - 1;
 		while (deltaIdx >= 0 && files.deltaFiles[deltaIdx].version > snapshotVersion) {
 			deltaIdx--;
 		}
 		deltaIdx++;
-		Version lastDeltaVersion = snapshotVersion;
+		state Version lastDeltaVersion = snapshotVersion;
+		state BlobFileIndex deltaF;
 		while (deltaIdx < files.deltaFiles.size() && lastDeltaVersion < version) {
-			BlobFileIndex deltaF = files.deltaFiles[deltaIdx];
-			chunk.deltaFiles.emplace_back_deep(
-			    filenameArena, deltaF.filename, deltaF.offset, deltaF.length, deltaF.fullFileLength);
+			state Optional<BlobGranuleCipherKeysCtx> deltaCipherKeysCtx;
+
+			deltaF = files.deltaFiles[deltaIdx];
+
+			if (deltaF.cipherKeysMeta.present()) {
+				ASSERT(isEncryptionOpSupported(EncryptOperationType::BLOB_GRANULE_ENCRYPTION,
+				                               bwData->dbInfo->get().client));
+
+				BlobGranuleCipherKeysCtx keysCtx =
+				    wait(getGranuleCipherKeysFromKeysMeta(bwData, deltaF.cipherKeysMeta.get(), &filenameArena));
+				deltaCipherKeysCtx = std::move(keysCtx);
+			}
+
+			chunk.deltaFiles.emplace_back_deep(filenameArena,
+			                                   deltaF.filename,
+			                                   deltaF.offset,
+			                                   deltaF.length,
+			                                   deltaF.fullFileLength,
+			                                   deltaCipherKeysCtx);
 			compactBytesRead += deltaF.length;
 			lastDeltaVersion = files.deltaFiles[deltaIdx].version;
 			deltaIdx++;
 		}
 		ASSERT(lastDeltaVersion >= version);
 		chunk.includedVersion = version;
-
-		if (BW_DEBUG) {
-			fmt::print("Re-snapshotting [{0} - {1}) @ {2} from blob\n",
-			           metadata->keyRange.begin.printable(),
-			           metadata->keyRange.end.printable(),
-			           version);
-		}
 		chunksToRead.push_back(readBlobGranule(chunk, metadata->keyRange, 0, version, bstore, &bwData->stats));
 	}
 
+	if (BW_DEBUG) {
+		fmt::print("Re-snapshotting [{0} - {1}) @ {2} from blob\n",
+		           metadata->keyRange.begin.printable(),
+		           metadata->keyRange.end.printable(),
+		           version);
+	}
+
 	try {
 		state PromiseStream<RangeResult> rowsStream;
 		state Future<BlobFileIndex> snapshotWriter = writeSnapshot(bwData,
@@ -1116,6 +1308,7 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
 				bwData->currentManagerStatusStream.get().send(GranuleStatusReply(metadata->keyRange,
 				                                                                 true,
 				                                                                 writeHot,
+				                                                                 false,
 				                                                                 statusEpoch,
 				                                                                 statusSeqno,
 				                                                                 granuleID,
@@ -1129,8 +1322,7 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
 				if (e.code() == error_code_operation_cancelled) {
 					throw e;
 				}
-				TEST(true); // Blob worker re-sending split evaluation to manager after not error/not hearing
-				            // back
+				CODE_PROBE(true, "Blob worker re-sending split evaluation to manager after not error/not hearing back");
 				// if we got broken promise while waiting, the old stream was killed, so we don't need to wait
 				// on change, just retry
 				if (e.code() == error_code_broken_promise) {
@@ -1180,6 +1372,100 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
 	return reSnapshotIdx;
 }
 
+ACTOR Future<BlobFileIndex> reSnapshotNoCheck(Reference<BlobWorkerData> bwData,
+                                              Reference<BlobConnectionProvider> bstore,
+                                              Reference<GranuleMetadata> metadata,
+                                              UID granuleID,
+                                              Future<BlobFileIndex> lastDeltaBeforeSnapshot) {
+	BlobFileIndex lastDeltaIdx = wait(lastDeltaBeforeSnapshot);
+	state Version reSnapshotVersion = lastDeltaIdx.version;
+	wait(delay(0, TaskPriority::BlobWorkerUpdateFDB));
+
+	CODE_PROBE(true, "re-snapshotting without BM check because still on old change feed!");
+
+	if (BW_DEBUG) {
+		fmt::print("Granule [{0} - {1}) re-snapshotting @ {2} WITHOUT checking with BM, because it is still on old "
+		           "change feed!\n",
+		           metadata->keyRange.begin.printable(),
+		           metadata->keyRange.end.printable(),
+		           reSnapshotVersion);
+	}
+
+	TraceEvent(SevDebug, "BlobGranuleReSnapshotOldFeed", bwData->id)
+	    .detail("Granule", metadata->keyRange)
+	    .detail("Version", reSnapshotVersion);
+
+	// wait for file updater to make sure that last delta file is in the metadata before
+	while (metadata->files.deltaFiles.empty() || metadata->files.deltaFiles.back().version < reSnapshotVersion) {
+		wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+	}
+
+	std::vector<GranuleFiles> toSnapshot;
+	toSnapshot.push_back(metadata->files);
+	BlobFileIndex reSnapshotIdx =
+	    wait(compactFromBlob(bwData, bstore, metadata, granuleID, toSnapshot, reSnapshotVersion));
+
+	return reSnapshotIdx;
+}
+
+// wait indefinitely to tell manager to re-evaluate this split, until the granule is revoked
+ACTOR Future<Void> reevaluateInitialSplit(Reference<BlobWorkerData> bwData,
+                                          UID granuleID,
+                                          KeyRange keyRange,
+                                          int64_t epoch,
+                                          int64_t seqno,
+                                          Key proposedSplitKey) {
+	// wait for first stream to be initialized
+	while (!bwData->statusStreamInitialized) {
+		wait(bwData->currentManagerStatusStream.onChange());
+	}
+	loop {
+		try {
+			// wait for manager stream to become ready, and send a message
+			loop {
+				choose {
+					when(wait(bwData->currentManagerStatusStream.get().onReady())) { break; }
+					when(wait(bwData->currentManagerStatusStream.onChange())) {}
+				}
+			}
+
+			GranuleStatusReply reply(keyRange,
+			                         true,
+			                         false,
+			                         true,
+			                         epoch,
+			                         seqno,
+			                         granuleID,
+			                         invalidVersion,
+			                         invalidVersion,
+			                         false,
+			                         epoch,
+			                         seqno);
+			reply.proposedSplitKey = proposedSplitKey;
+			bwData->currentManagerStatusStream.get().send(reply);
+			// if a new manager appears, also tell it about this granule being splittable, or retry after a certain
+			// amount of time of not hearing back
+			wait(success(timeout(bwData->currentManagerStatusStream.onChange(), 10.0)));
+			wait(delay(0));
+			CODE_PROBE(true, "Blob worker re-sending initialsplit too big");
+		} catch (Error& e) {
+			if (e.code() == error_code_operation_cancelled) {
+				throw e;
+			}
+
+			CODE_PROBE(true, "Blob worker re-sending merge candidate to manager after not error/not hearing back");
+
+			// if we got broken promise while waiting, the old stream was killed, so we don't need to wait
+			// on change, just retry
+			if (e.code() == error_code_broken_promise) {
+				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+			} else {
+				wait(bwData->currentManagerStatusStream.onChange());
+			}
+		}
+	}
+}
+
 ACTOR Future<Void> granuleCheckMergeCandidate(Reference<BlobWorkerData> bwData,
                                               Reference<GranuleMetadata> metadata,
                                               UID granuleID,
@@ -1189,30 +1475,29 @@ ACTOR Future<Void> granuleCheckMergeCandidate(Reference<BlobWorkerData> bwData,
 	}
 	// wait for the last snapshot to finish, so that the delay is from the last snapshot
 	wait(waitStart);
-	wait(delayJittered(SERVER_KNOBS->BG_MERGE_CANDIDATE_THRESHOLD_SECONDS));
+	double jitter = deterministicRandom()->random01() * 0.8 * SERVER_KNOBS->BG_MERGE_CANDIDATE_DELAY_SECONDS;
+	wait(delay(SERVER_KNOBS->BG_MERGE_CANDIDATE_THRESHOLD_SECONDS + jitter));
 	loop {
 		// this actor will be cancelled if a split check happened, or if the granule was moved away, so this
 		// being here means that granule is cold enough during that period. Now we just need to check if it is
 		// also small enough to be a merge candidate.
 		StorageMetrics currentMetrics = wait(bwData->db->getStorageMetrics(metadata->keyRange, CLIENT_KNOBS->TOO_MANY));
-		state int64_t granuleBytes = currentMetrics.bytes;
 
 		// FIXME: maybe separate knob and/or value for write rate?
 		if (currentMetrics.bytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES / 2 ||
 		    currentMetrics.bytesPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
 			wait(delayJittered(SERVER_KNOBS->BG_MERGE_CANDIDATE_THRESHOLD_SECONDS / 2.0));
-			TEST(true); // wait and check later to see if granule got smaller or colder
+			CODE_PROBE(true, "wait and check later to see if granule got smaller or colder");
 			continue;
 		}
 
-		TEST(true); // Blob Worker identified merge candidate granule
+		CODE_PROBE(true, "Blob Worker identified merge candidate granule");
 
 		// if we are a merge candidate, send a message to the BM. Once successful, this actor is complete
 		while (!bwData->statusStreamInitialized) {
 			wait(bwData->currentManagerStatusStream.onChange());
 		}
 
-		// FIXME: after a certain amount of retries/time, we may want to re-check anyway
 		state double sendTimeGiveUp = now() + SERVER_KNOBS->BG_MERGE_CANDIDATE_THRESHOLD_SECONDS / 2.0;
 		loop {
 			try {
@@ -1226,11 +1511,12 @@ ACTOR Future<Void> granuleCheckMergeCandidate(Reference<BlobWorkerData> bwData,
 				}
 
 				if (now() >= sendTimeGiveUp) {
-					TEST(true); // Blob worker could not send merge candidate in time, re-checking status
+					CODE_PROBE(true, "Blob worker could not send merge candidate in time, re-checking status");
 					break;
 				}
 
 				bwData->currentManagerStatusStream.get().send(GranuleStatusReply(metadata->keyRange,
+				                                                                 false,
 				                                                                 false,
 				                                                                 false,
 				                                                                 metadata->continueEpoch,
@@ -1242,18 +1528,16 @@ ACTOR Future<Void> granuleCheckMergeCandidate(Reference<BlobWorkerData> bwData,
 				                                                                 metadata->originalEpoch,
 				                                                                 metadata->originalSeqno));
 				// if a new manager appears, also tell it about this granule being mergeable
-				state int64_t lastSendEpoch = bwData->currentManagerEpoch;
-				while (lastSendEpoch == bwData->currentManagerEpoch) {
-					wait(bwData->currentManagerStatusStream.onChange());
-					wait(delay(0));
-				}
-				TEST(true); // Blob worker re-sending merge candidate to new manager
+				// or if a new stream from the existing manager, it may have missed the message due to a network issue
+				wait(bwData->currentManagerStatusStream.onChange());
+				wait(delay(0));
+				CODE_PROBE(true, "Blob worker re-sending merge candidate to new manager");
 			} catch (Error& e) {
 				if (e.code() == error_code_operation_cancelled) {
 					throw e;
 				}
 
-				TEST(true); // Blob worker re-sending merge candidate to manager after not error/not hearing back
+				CODE_PROBE(true, "Blob worker re-sending merge candidate to manager after not error/not hearing back");
 
 				// if we got broken promise while waiting, the old stream was killed, so we don't need to wait
 				// on change, just retry
@@ -1287,11 +1571,10 @@ void handleCompletedDeltaFile(Reference<BlobWorkerData> bwData,
 		}
 		// FIXME: for a write-hot shard, we could potentially batch these and only pop the largest one after
 		// several have completed
-		// FIXME: we actually want to pop at this version + 1 because pop is exclusive?
 		// FIXME: since this is async, and worker could die, new blob worker that opens granule should probably
 		// kick off an async pop at its previousDurableVersion after opening the granule to guarantee it is
 		// eventually popped?
-		Future<Void> popFuture = bwData->db->popChangeFeedMutations(cfKey, completedDeltaFile.version);
+		Future<Void> popFuture = bwData->db->popChangeFeedMutations(cfKey, completedDeltaFile.version + 1);
 		// Do pop asynchronously
 		inFlightPops.push_back(popFuture);
 	}
@@ -1320,6 +1603,7 @@ bool granuleCanRetry(const Error& e) {
 	case error_code_http_request_failed:
 	case error_code_connection_failed:
 	case error_code_lookup_failed: // dns
+	case error_code_platform_error: // injected faults
 		return true;
 	default:
 		return false;
@@ -1356,7 +1640,7 @@ Version doGranuleRollback(Reference<GranuleMetadata> metadata,
 		for (auto& f : inFlightFiles) {
 			if (f.snapshot) {
 				if (f.version > rollbackVersion) {
-					TEST(true); // Granule rollback cancelling snapshot file
+					CODE_PROBE(true, "Granule rollback cancelling snapshot file");
 					if (BW_DEBUG) {
 						fmt::print("[{0} - {1}) rollback cancelling snapshot file @ {2}\n",
 						           metadata->keyRange.begin.printable(),
@@ -1377,7 +1661,7 @@ Version doGranuleRollback(Reference<GranuleMetadata> metadata,
 						metadata->bytesInNewDeltaFiles -= f.bytes;
 					}
 					toPop++;
-					TEST(true); // Granule rollback cancelling delta file
+					CODE_PROBE(true, "Granule rollback cancelling delta file");
 					if (BW_DEBUG) {
 						fmt::print("[{0} - {1}) rollback cancelling delta file @ {2}\n",
 						           metadata->keyRange.begin.printable(),
@@ -1432,7 +1716,7 @@ Version doGranuleRollback(Reference<GranuleMetadata> metadata,
 
 	} else {
 		// No pending delta files to discard, just in-memory mutations
-		TEST(true); // Granule rollback discarding in memory mutations
+		CODE_PROBE(true, "Granule rollback discarding in memory mutations");
 
 		// FIXME: could binary search?
 		int mIdx = metadata->currentDeltas.size() - 1;
@@ -1557,6 +1841,44 @@ ACTOR Future<Void> waitVersionCommitted(Reference<BlobWorkerData> bwData,
 	return Void();
 }
 
+ACTOR Future<bool> checkFileNotFoundForcePurgeRace(Reference<BlobWorkerData> bwData, KeyRange range) {
+	state Transaction tr(bwData->db);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			ForcedPurgeState purgeState = wait(getForcePurgedState(&tr, range));
+			return purgeState != ForcedPurgeState::NonePurged;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+// Does a force flush after consuming all data post-split from the parent granule's old change feed.
+// This guarantees that all of the data from the old change feed gets persisted to blob storage, and the old feed can be
+// cleaned up. This is particularly necessary for sequential workloads, where only one child granule after the split has
+// new writes. Adds a delay so that, if the granule is not write-cold and would have written a delta file soon anyway,
+// this does not add any extra overhead.
+ACTOR Future<Void> forceFlushCleanup(Reference<BlobWorkerData> bwData, Reference<GranuleMetadata> metadata, Version v) {
+	double cleanupDelay = SERVER_KNOBS->BLOB_WORKER_FORCE_FLUSH_CLEANUP_DELAY;
+	if (cleanupDelay < 0) {
+		return Void();
+	}
+	wait(delay(cleanupDelay));
+	if (metadata->forceFlushVersion.get() < v && metadata->pendingDeltaVersion < v) {
+		metadata->forceFlushVersion.set(v);
+		++bwData->stats.forceFlushCleanups;
+		if (BW_DEBUG) {
+			fmt::print("Granule [{0} - {1}) forcing flush cleanup @ {2}\n",
+			           metadata->keyRange.begin.printable(),
+			           metadata->keyRange.end.printable(),
+			           v);
+		}
+	}
+	return Void();
+}
+
 // updater for a single granule
 // TODO: this is getting kind of large. Should try to split out this actor if it continues to grow?
 ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
@@ -1569,17 +1891,19 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 	state Future<Void> oldChangeFeedFuture;
 	state Future<Void> changeFeedFuture;
 	state Future<Void> checkMergeCandidate;
+	state Future<Void> forceFlushCleanupFuture;
 	state GranuleStartState startState;
 	state bool readOldChangeFeed;
-	state Optional<std::pair<KeyRange, UID>> oldChangeFeedDataComplete;
 	state Key cfKey;
 	state Optional<Key> oldCFKey;
 	state int pendingSnapshots = 0;
 	state Version lastForceFlushVersion = invalidVersion;
 	state std::deque<Version> forceFlushVersions;
+	state Future<Void> nextForceFlush = metadata->forceFlushVersion.whenAtLeast(1);
 
 	state std::deque<std::pair<Version, Version>> rollbacksInProgress;
 	state std::deque<std::pair<Version, Version>> rollbacksCompleted;
+	state Future<Void> startDeltaFileWrite = Future<Void>(Void());
 
 	state bool snapshotEligible; // just wrote a delta file or just took granule over from another worker
 	state bool justDidRollback = false;
@@ -1637,7 +1961,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 		}
 
 		if (!startState.doSnapshot) {
-			TEST(true); // Granule moved without split
+			CODE_PROBE(true, "Granule moved without split");
 			startVersion = startState.previousDurableVersion;
 			ASSERT(!metadata->files.snapshotFiles.empty());
 			metadata->pendingSnapshotVersion = metadata->files.snapshotFiles.back().version;
@@ -1674,6 +1998,10 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 			metadata->historyVersion = startState.history.present() ? startState.history.get().version : startVersion;
 		}
 
+		// No need to start Change Feed in full restore mode
+		if (isFullRestoreMode())
+			return Void();
+
 		checkMergeCandidate = granuleCheckMergeCandidate(bwData,
 		                                                 metadata,
 		                                                 startState.granuleID,
@@ -1685,9 +2013,9 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 		metadata->bufferedDeltaVersion = startVersion;
 		metadata->knownCommittedVersion = startVersion;
 
-		Reference<ChangeFeedData> cfData = makeReference<ChangeFeedData>();
+		Reference<ChangeFeedData> cfData = makeReference<ChangeFeedData>(bwData->db.getPtr());
 
-		if (startState.splitParentGranule.present() && startVersion < startState.changeFeedStartVersion) {
+		if (startState.splitParentGranule.present() && startVersion + 1 < startState.changeFeedStartVersion) {
 			// read from parent change feed up until our new change feed is started
 			// Required to have canReadPopped = false, otherwise another granule can take over the change feed,
 			// and pop it. That could cause this worker to think it has the full correct set of data if it then
@@ -1712,6 +2040,13 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 			                                                   metadata->keyRange,
 			                                                   bwData->changeFeedStreamReplyBufferSize,
 			                                                   false);
+			// in case previous worker died before popping the latest version, start another pop
+			if (startState.previousDurableVersion != invalidVersion) {
+				ASSERT(startState.previousDurableVersion + 1 >= startState.changeFeedStartVersion);
+				Future<Void> popFuture =
+				    bwData->db->popChangeFeedMutations(cfKey, startState.previousDurableVersion + 1);
+				inFlightPops.push_back(popFuture);
+			}
 		}
 
 		// Start actors BEFORE setting new change feed data to ensure the change feed data is properly
@@ -1798,7 +2133,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 						// popped up to V+1 is ok. Or in other words, if the last delta @ V, we only missed data
 						// at V+1 onward if popVersion >= V+2
 						if (metadata->bufferedDeltaVersion < metadata->activeCFData.get()->popVersion - 1) {
-							TEST(true); // Blob Worker detected popped
+							CODE_PROBE(true, "Blob Worker detected popped");
 							TraceEvent("BlobWorkerChangeFeedPopped", bwData->id)
 							    .detail("Granule", metadata->keyRange)
 							    .detail("GranuleID", startState.granuleID)
@@ -1809,7 +2144,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 						}
 					}
 					when(wait(inFlightFiles.empty() ? Never() : success(inFlightFiles.front().future))) {}
-					when(wait(metadata->forceFlushVersion.whenAtLeast(lastForceFlushVersion + 1))) {
+					when(wait(nextForceFlush)) {
 						if (forceFlushVersions.empty() ||
 						    forceFlushVersions.back() < metadata->forceFlushVersion.get()) {
 							forceFlushVersions.push_back(metadata->forceFlushVersion.get());
@@ -1817,6 +2152,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 						if (metadata->forceFlushVersion.get() > lastForceFlushVersion) {
 							lastForceFlushVersion = metadata->forceFlushVersion.get();
 						}
+						nextForceFlush = metadata->forceFlushVersion.whenAtLeast(lastForceFlushVersion + 1);
 					}
 				}
 			} catch (Error& e) {
@@ -1827,9 +2163,6 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 				ASSERT(readOldChangeFeed);
 
 				readOldChangeFeed = false;
-				// set this so next delta file write updates granule split metadata to done
-				ASSERT(startState.splitParentGranule.present());
-				oldChangeFeedDataComplete = startState.splitParentGranule.get();
 				if (BW_DEBUG) {
 					fmt::print("Granule [{0} - {1}) switching to new change feed {2} @ {3}, {4}\n",
 					           metadata->keyRange.begin.printable(),
@@ -1841,8 +2174,9 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 				ASSERT(metadata->bufferedDeltaVersion <= metadata->activeCFData.get()->getVersion());
 				// update this for change feed popped detection
 				metadata->bufferedDeltaVersion = metadata->activeCFData.get()->getVersion();
+				forceFlushCleanupFuture = forceFlushCleanup(bwData, metadata, metadata->bufferedDeltaVersion);
 
-				Reference<ChangeFeedData> cfData = makeReference<ChangeFeedData>();
+				Reference<ChangeFeedData> cfData = makeReference<ChangeFeedData>(bwData->db.getPtr());
 
 				changeFeedFuture = bwData->db->getChangeFeedStream(cfData,
 				                                                   cfKey,
@@ -1896,7 +2230,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 								if (metadata->pendingDeltaVersion <= rollbackVersion &&
 								    (metadata->currentDeltas.empty() ||
 								     metadata->currentDeltas.back().version <= rollbackVersion)) {
-									TEST(true); // Granule ignoring rollback
+									CODE_PROBE(true, "Granule ignoring rollback");
 
 									if (BW_DEBUG) {
 										fmt::print("Granule [{0} - {1}) on BW {2} skipping rollback {3} -> {4} "
@@ -1913,7 +2247,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 									// rollbackInProgress when we restart the stream.
 									rollbacksCompleted.push_back(std::pair(rollbackVersion, deltas.version));
 								} else {
-									TEST(true); // Granule processing rollback
+									CODE_PROBE(true, "Granule processing rollback");
 									if (BW_DEBUG) {
 										fmt::print("[{0} - {1}) on BW {2} ROLLBACK @ {3} -> {4}\n",
 										           metadata->keyRange.begin.printable(),
@@ -1927,6 +2261,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 										    .detail("RollbackVersion", rollbackVersion);
 									}
 
+									Version oldPendingSnapshot = metadata->pendingSnapshotVersion;
 									Version cfRollbackVersion = doGranuleRollback(metadata,
 									                                              deltas.version,
 									                                              rollbackVersion,
@@ -1934,9 +2269,29 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 									                                              rollbacksInProgress,
 									                                              rollbacksCompleted);
 
-									Reference<ChangeFeedData> cfData = makeReference<ChangeFeedData>();
+									if (oldPendingSnapshot > metadata->pendingSnapshotVersion) {
+										// If rollback cancelled in-flight snapshot, merge candidate checker also got
+										// cancelled. Restart it
+										CODE_PROBE(true,
+										           "Restarting merge candidate checker after rolling back snapshot");
+										checkMergeCandidate = granuleCheckMergeCandidate(
+										    bwData,
+										    metadata,
+										    startState.granuleID,
+										    inFlightFiles.empty() ? Future<Void>(Void())
+										                          : success(inFlightFiles.back().future));
+									}
+									// reset force flush state, requests should retry and add it back once feed is ready
+									forceFlushVersions.clear();
+									lastForceFlushVersion = 0;
+									metadata->forceFlushVersion = NotifiedVersion();
+									nextForceFlush = metadata->forceFlushVersion.whenAtLeast(1);
 
-									if (!readOldChangeFeed && cfRollbackVersion < startState.changeFeedStartVersion) {
+									Reference<ChangeFeedData> cfData =
+									    makeReference<ChangeFeedData>(bwData->db.getPtr());
+
+									if (!readOldChangeFeed &&
+									    cfRollbackVersion + 1 < startState.changeFeedStartVersion) {
 										// It isn't possible to roll back across the parent/child feed boundary,
 										// but as part of rolling back we may need to cancel in-flight delta
 										// files, and those delta files may include stuff from before the
@@ -1944,13 +2299,13 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 										// change feed
 										ASSERT(cfRollbackVersion >= startState.previousDurableVersion);
 										ASSERT(cfRollbackVersion >= metadata->durableDeltaVersion.get());
-										TEST(true); // rollback crossed change feed boundaries
+										CODE_PROBE(true, "rollback crossed change feed boundaries");
 										readOldChangeFeed = true;
-										oldChangeFeedDataComplete.reset();
+										forceFlushCleanupFuture = Never();
 									}
 
 									if (readOldChangeFeed) {
-										ASSERT(cfRollbackVersion < startState.changeFeedStartVersion);
+										ASSERT(cfRollbackVersion + 1 < startState.changeFeedStartVersion);
 										ASSERT(oldCFKey.present());
 										oldChangeFeedFuture =
 										    bwData->db->getChangeFeedStream(cfData,
@@ -1962,12 +2317,12 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 										                                    false);
 
 									} else {
-										if (cfRollbackVersion < startState.changeFeedStartVersion) {
+										if (cfRollbackVersion + 1 < startState.changeFeedStartVersion) {
 											fmt::print("Rollback past CF start??. rollback={0}, start={1}\n",
 											           cfRollbackVersion,
 											           startState.changeFeedStartVersion);
 										}
-										ASSERT(cfRollbackVersion >= startState.changeFeedStartVersion);
+										ASSERT(cfRollbackVersion + 1 >= startState.changeFeedStartVersion);
 
 										changeFeedFuture =
 										    bwData->db->getChangeFeedStream(cfData,
@@ -1990,7 +2345,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 							}
 						} else if (!rollbacksInProgress.empty() && rollbacksInProgress.front().first < deltas.version &&
 						           rollbacksInProgress.front().second > deltas.version) {
-							TEST(true); // Granule skipping mutations b/c prior rollback
+							CODE_PROBE(true, "Granule skipping mutations b/c prior rollback");
 							if (BW_DEBUG) {
 								fmt::print("Skipping mutations @ {} b/c prior rollback\n", deltas.version);
 							}
@@ -2033,7 +2388,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 			// The force flush contract is a version cannot be put in forceFlushVersion unless the change feed
 			// is already whenAtLeast that version
 			bool forceFlush = !forceFlushVersions.empty() && forceFlushVersions.back() > metadata->pendingDeltaVersion;
-			TEST(forceFlush); // Force flushing granule
+			CODE_PROBE(forceFlush, "Force flushing granule");
 			if (metadata->bufferedDeltaBytes >= SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES || forceFlush) {
 				TraceEvent(SevDebug, "BlobGranuleDeltaFile", bwData->id)
 				    .detail("Granule", metadata->keyRange)
@@ -2072,7 +2427,25 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 					// write/read a bunch of empty blob files
 					ASSERT(forceFlush);
 					ASSERT(!forceFlushVersions.empty());
-					TEST(true); // Force flushing empty delta file!
+					CODE_PROBE(true, "Force flushing empty delta file!");
+				}
+
+				// launch pipelined, but wait for previous operation to complete before persisting to FDB
+				Future<BlobFileIndex> previousFuture;
+				if (!inFlightFiles.empty()) {
+					previousFuture = inFlightFiles.back().future;
+				} else {
+					previousFuture = Future<BlobFileIndex>(BlobFileIndex());
+				}
+
+				// The last version included in the old change feed is startState.cfStartVersion - 1.
+				// So if the previous delta file did not include this version, and the new delta file does, the old
+				// change feed is considered complete.
+				Optional<std::pair<KeyRange, UID>> oldChangeFeedDataComplete;
+				if (startState.splitParentGranule.present() &&
+				    metadata->pendingDeltaVersion + 1 < startState.changeFeedStartVersion &&
+				    lastDeltaVersion + 1 >= startState.changeFeedStartVersion) {
+					oldChangeFeedDataComplete = startState.splitParentGranule.get();
 				}
 
 				if (BW_DEBUG) {
@@ -2084,13 +2457,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 					           oldChangeFeedDataComplete.present() ? ". Finalizing " : "");
 				}
 
-				// launch pipelined, but wait for previous operation to complete before persisting to FDB
-				Future<BlobFileIndex> previousFuture;
-				if (!inFlightFiles.empty()) {
-					previousFuture = inFlightFiles.back().future;
-				} else {
-					previousFuture = Future<BlobFileIndex>(BlobFileIndex());
-				}
+				startDeltaFileWrite = bwData->deltaWritesLock->take();
 				Future<BlobFileIndex> dfFuture =
 				    writeDeltaFile(bwData,
 				                   bstore,
@@ -2102,10 +2469,10 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 				                   lastDeltaVersion,
 				                   previousFuture,
 				                   waitVersionCommitted(bwData, metadata, lastDeltaVersion),
-				                   oldChangeFeedDataComplete);
+				                   oldChangeFeedDataComplete,
+				                   startDeltaFileWrite);
 				inFlightFiles.push_back(InFlightFile(dfFuture, lastDeltaVersion, metadata->bufferedDeltaBytes, false));
 
-				oldChangeFeedDataComplete.reset();
 				// add new pending delta file
 				ASSERT(metadata->pendingDeltaVersion < lastDeltaVersion);
 				metadata->pendingDeltaVersion = lastDeltaVersion;
@@ -2126,6 +2493,10 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 				// exhaust old change feed before compacting - otherwise we could end up with an endlessly
 				// growing list of previous change feeds in the worst case.
 				snapshotEligible = true;
+
+				// Wait on delta file starting here. If we have too many pending delta file writes, we need to not
+				// continue to consume from the change feed, as that will pile on even more delta files to write
+				wait(startDeltaFileWrite);
 			}
 
 			// FIXME: if we're still reading from old change feed, we should probably compact if we're
@@ -2133,8 +2504,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 			// yet
 
 			// If we have enough delta files, try to re-snapshot
-			if (snapshotEligible && metadata->bytesInNewDeltaFiles >= SERVER_KNOBS->BG_DELTA_BYTES_BEFORE_COMPACT &&
-			    metadata->pendingDeltaVersion >= startState.changeFeedStartVersion) {
+			if (snapshotEligible && metadata->bytesInNewDeltaFiles >= SERVER_KNOBS->BG_DELTA_BYTES_BEFORE_COMPACT) {
 				if (BW_DEBUG && !inFlightFiles.empty()) {
 					fmt::print("Granule [{0} - {1}) ready to re-snapshot at {2} after {3} > {4} bytes, "
 					           "waiting for "
@@ -2162,13 +2532,19 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 					previousFuture = Future<BlobFileIndex>(metadata->files.deltaFiles.back());
 				}
 				int64_t versionsSinceLastSnapshot = metadata->pendingDeltaVersion - metadata->pendingSnapshotVersion;
-				Future<BlobFileIndex> inFlightBlobSnapshot = checkSplitAndReSnapshot(bwData,
-				                                                                     bstore,
-				                                                                     metadata,
-				                                                                     startState.granuleID,
-				                                                                     metadata->bytesInNewDeltaFiles,
-				                                                                     previousFuture,
-				                                                                     versionsSinceLastSnapshot);
+				Future<BlobFileIndex> inFlightBlobSnapshot;
+				if (metadata->pendingDeltaVersion >= startState.changeFeedStartVersion) {
+					inFlightBlobSnapshot = checkSplitAndReSnapshot(bwData,
+					                                               bstore,
+					                                               metadata,
+					                                               startState.granuleID,
+					                                               metadata->bytesInNewDeltaFiles,
+					                                               previousFuture,
+					                                               versionsSinceLastSnapshot);
+				} else {
+					inFlightBlobSnapshot =
+					    reSnapshotNoCheck(bwData, bstore, metadata, startState.granuleID, previousFuture);
+				}
 				inFlightFiles.push_back(InFlightFile(inFlightBlobSnapshot, metadata->pendingDeltaVersion, 0, true));
 				pendingSnapshots++;
 
@@ -2200,7 +2576,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 						idx++;
 					}
 					while (waitIdx > 0) {
-						TEST(true); // Granule blocking on previous snapshot
+						CODE_PROBE(true, "Granule blocking on previous snapshot");
 						// TODO don't duplicate code
 						BlobFileIndex completedFile = wait(inFlightFiles.front().future);
 						if (inFlightFiles.front().snapshot) {
@@ -2241,7 +2617,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 				// queue too many files in parallel, and slow down change feed consuming to let file writing
 				// catch up
 
-				TEST(true); // Granule processing long tail of old change feed
+				CODE_PROBE(true, "Granule processing long tail of old change feed");
 				if (inFlightFiles.size() > 10 && inFlightFiles.front().version <= metadata->knownCommittedVersion) {
 					if (BW_DEBUG) {
 						fmt::print("[{0} - {1}) Waiting on delta file b/c old change feed\n",
@@ -2295,7 +2671,7 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 		++bwData->stats.granuleUpdateErrors;
 
 		if (granuleCanRetry(e)) {
-			TEST(true); // Granule close and re-open on error
+			CODE_PROBE(true, "Granule close and re-open on error");
 			TraceEvent("GranuleFileUpdaterRetriableError", bwData->id)
 			    .error(e)
 			    .detail("Granule", metadata->keyRange)
@@ -2317,17 +2693,31 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
 			throw e;
 		}
 
+		state Error e2 = e;
+		if (e.code() == error_code_file_not_found) {
+			// FIXME: better way to fix this?
+			bool isForcePurging = wait(checkFileNotFoundForcePurgeRace(bwData, metadata->keyRange));
+			if (isForcePurging) {
+				CODE_PROBE(true, "Granule got file not found from force purge");
+				TraceEvent("GranuleFileUpdaterFileNotFoundForcePurge", bwData->id)
+				    .error(e2)
+				    .detail("KeyRange", metadata->keyRange)
+				    .detail("GranuleID", startState.granuleID);
+				return Void();
+			}
+		}
+
 		TraceEvent(SevError, "GranuleFileUpdaterUnexpectedError", bwData->id)
-		    .error(e)
+		    .error(e2)
 		    .detail("Granule", metadata->keyRange)
 		    .detail("GranuleID", startState.granuleID);
 		ASSERT_WE_THINK(false);
 
 		// if not simulation, kill the BW
 		if (bwData->fatalError.canBeSet()) {
-			bwData->fatalError.sendError(e);
+			bwData->fatalError.sendError(e2);
 		}
-		throw e;
+		throw e2;
 	}
 }
 
@@ -2353,7 +2743,7 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 		GranuleStartState startState = wait(assignFuture);
 		state Optional<GranuleHistory> activeHistory = startState.history;
 
-		if (activeHistory.present() && activeHistory.get().value.parentGranules.size() > 0) {
+		if (activeHistory.present() && activeHistory.get().value.parentVersions.size() > 0) {
 			state int64_t loadId = nextHistoryLoadId++;
 			if (BW_HISTORY_DEBUG) {
 				fmt::print("HL {0} {1}) Loading history data for [{2} - {3})\n",
@@ -2369,7 +2759,7 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 			    std::priority_queue<OrderedHistoryKey, std::vector<OrderedHistoryKey>, std::greater<OrderedHistoryKey>>
 			        rootGranules;
 			state Transaction tr(bwData->db);
-			if (!activeHistory.get().value.parentGranules.empty()) {
+			if (!activeHistory.get().value.parentVersions.empty()) {
 				if (BW_HISTORY_DEBUG) {
 					fmt::print("HL {0} {1}) Starting history [{2} - {3}) @ {4}\n",
 					           bwData->id.shortString().substr(0, 5),
@@ -2398,7 +2788,7 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 				}
 
 				auto prev = bwData->granuleHistory.intersectingRanges(curHistory.range);
-				bool allLess = true;
+				bool allLess = true; // if the version is less than all existing granules
 				for (auto& it : prev) {
 					if (it.cvalue().isValid() && curHistory.version >= it.cvalue()->endVersion) {
 						allLess = false;
@@ -2435,20 +2825,23 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 				}
 
 				state int pIdx = 0;
-				state bool noParentsPresent = true;
+				state bool anyParentsMissing = curHistory.value.parentVersions.empty();
+				state std::vector<GranuleHistory> nexts;
+				nexts.reserve(curHistory.value.parentVersions.size());
 				// FIXME: parallelize this for all parents/all entries in queue?
 				loop {
-					if (pIdx >= curHistory.value.parentGranules.size()) {
+					if (pIdx >= curHistory.value.parentVersions.size()) {
 						break;
 					}
 					try {
-						Optional<Value> v =
-						    wait(tr.get(blobGranuleHistoryKeyFor(curHistory.value.parentGranules[pIdx].first,
-						                                         curHistory.value.parentGranules[pIdx].second)));
+						tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+						tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+						state KeyRangeRef parentRange(curHistory.value.parentBoundaries[pIdx],
+						                              curHistory.value.parentBoundaries[pIdx + 1]);
+						state Version parentVersion = curHistory.value.parentVersions[pIdx];
+						Optional<Value> v = wait(tr.get(blobGranuleHistoryKeyFor(parentRange, parentVersion)));
 						if (v.present()) {
-							next = GranuleHistory(curHistory.value.parentGranules[pIdx].first,
-							                      curHistory.value.parentGranules[pIdx].second,
-							                      decodeBlobGranuleHistoryValue(v.get()));
+							next = GranuleHistory(parentRange, parentVersion, decodeBlobGranuleHistoryValue(v.get()));
 							ASSERT(next.version != invalidVersion);
 
 							auto inserted = forwardHistory.insert({ next.value.granuleID, ForwardHistoryValue() });
@@ -2459,7 +2852,7 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 								// curHistory.version]
 								inserted.first->second.entry = makeReference<GranuleHistoryEntry>(
 								    next.range, next.value.granuleID, next.version, curHistory.version);
-								queue.push_back(next);
+								nexts.push_back(next);
 								if (BW_HISTORY_DEBUG) {
 									fmt::print("HL {0} {1}) [{2} - {3}) @ {4}: loaded parent [{5} - {6}) @ {7}\n",
 									           bwData->id.shortString().substr(0, 5),
@@ -2472,7 +2865,7 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 									           next.version);
 								}
 							} else {
-								TEST(true); // duplicate parent in granule history (split then merge)
+								CODE_PROBE(true, "duplicate parent in granule history (split then merge)");
 								if (BW_HISTORY_DEBUG) {
 									fmt::print("HL {0} {1}) [{2} - {3}) @ {4}: duplicate parent [{5} - "
 									           "{6}) @ {7}\n",
@@ -2487,7 +2880,8 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 								}
 								ASSERT(inserted.first->second.entry->endVersion == curHistory.version);
 							}
-							noParentsPresent = false;
+						} else {
+							anyParentsMissing = true;
 						}
 
 						pIdx++;
@@ -2496,16 +2890,21 @@ ACTOR Future<Void> blobGranuleLoadHistory(Reference<BlobWorkerData> bwData,
 					}
 				}
 
-				if (noParentsPresent) {
+				if (anyParentsMissing) {
 					if (BW_HISTORY_DEBUG) {
-						fmt::print("HL {0} {1}) [{2} - {3}) @ {4}: root b/c no parents\n",
+						fmt::print("HL {0} {1}) [{2} - {3}) @ {4}: root b/c parents missing\n",
 						           bwData->id.shortString().substr(0, 5),
 						           loadId,
 						           curHistory.range.begin.printable(),
 						           curHistory.range.end.printable(),
 						           curHistory.version);
 					}
+
 					rootGranules.push(OrderedHistoryKey(curHistory.version, curHistory.value.granuleID));
+				} else {
+					for (auto& it : nexts) {
+						queue.push_back(it);
+					}
 				}
 			}
 
@@ -2718,17 +3117,21 @@ std::vector<std::pair<KeyRange, Future<GranuleFiles>>> loadHistoryChunks(Referen
 		if (!it.cvalue().isValid()) {
 			throw blob_granule_transaction_too_old();
 		}
-		if (expectedEndVersion != it.cvalue()->endVersion) {
-			fmt::print("live granule history version {0} for [{1} - {2}) != history end version {3} for "
-			           "[{4} - {5})\n",
-			           expectedEndVersion,
-			           keyRange.begin.printable(),
-			           keyRange.end.printable(),
-			           it.cvalue()->endVersion,
-			           it.begin().printable(),
-			           it.end().printable());
+		if (expectedEndVersion > it.cvalue()->endVersion) {
+			if (BW_DEBUG) {
+				// history must have been pruned up to live granule, but BW still has previous history cached.
+				fmt::print("live granule history version {0} for [{1} - {2}) != history end version {3} for "
+				           "[{4} - {5}) on BW {6}\n",
+				           expectedEndVersion,
+				           keyRange.begin.printable(),
+				           keyRange.end.printable(),
+				           it.cvalue()->endVersion,
+				           it.begin().printable(),
+				           it.end().printable(),
+				           bwData->id.toString().substr(0, 5));
+			}
+			throw blob_granule_transaction_too_old();
 		}
-		ASSERT(expectedEndVersion == it.cvalue()->endVersion);
 		visited.insert(it.cvalue()->granuleID);
 		queue.push_back(it.cvalue());
 
@@ -2820,7 +3223,7 @@ std::vector<std::pair<KeyRange, Future<GranuleFiles>>> loadHistoryChunks(Referen
 
 	ASSERT(!resultChunks.empty());
 	if (resultChunks.size() >= 2) {
-		TEST(true); // Multiple history chunks for time travel query
+		CODE_PROBE(true, "Multiple history chunks for time travel query");
 		std::sort(resultChunks.begin(), resultChunks.end(), sort_result_chunks());
 		// Assert contiguous
 		for (int i = 0; i < resultChunks.size() - 1; i++) {
@@ -2857,14 +3260,15 @@ std::vector<std::pair<KeyRange, Future<GranuleFiles>>> loadHistoryChunks(Referen
 }
 
 // TODO might want to separate this out for valid values for range assignments vs read requests. Assignment
-// conflict isn't valid for read requests but is for assignments
+// conflict and blob_worker_full isn't valid for read requests but is for assignments
 bool canReplyWith(Error e) {
 	switch (e.code()) {
 	case error_code_blob_granule_transaction_too_old:
 	case error_code_transaction_too_old:
-	case error_code_future_version: // not thrown yet
+	case error_code_future_version:
 	case error_code_wrong_shard_server:
 	case error_code_process_behind: // not thrown yet
+	case error_code_blob_worker_full:
 		return true;
 	default:
 		return false;
@@ -2894,12 +3298,13 @@ ACTOR Future<Void> waitForVersion(Reference<GranuleMetadata> metadata, Version v
 	     metadata->durableDeltaVersion.get() == metadata->pendingDeltaVersion) &&
 	    (v <= metadata->durableSnapshotVersion.get() ||
 	     metadata->durableSnapshotVersion.get() == metadata->pendingSnapshotVersion)) {
-		TEST(true); // Granule read not waiting
+		CODE_PROBE(true, "Granule read not waiting");
 		return Void();
 	}
 
 	// wait for change feed version to catch up to ensure we have all data
 	if (metadata->activeCFData.get()->getVersion() < v) {
+		// FIXME: add future version timeout and throw here, same as SS
 		wait(metadata->activeCFData.get()->whenAtLeast(v));
 		ASSERT(metadata->activeCFData.get()->getVersion() >= v);
 	}
@@ -2911,7 +3316,7 @@ ACTOR Future<Void> waitForVersion(Reference<GranuleMetadata> metadata, Version v
 	// If there are mutations that are no longer buffered but have not been
 	// persisted to a delta file that are necessary for the query, wait for them
 	if (pendingDeltaV > metadata->durableDeltaVersion.get() && v > metadata->durableDeltaVersion.get()) {
-		TEST(true); // Granule read waiting for pending delta
+		CODE_PROBE(true, "Granule read waiting for pending delta");
 		wait(metadata->durableDeltaVersion.whenAtLeast(pendingDeltaV));
 		ASSERT(metadata->durableDeltaVersion.get() >= pendingDeltaV);
 	}
@@ -2919,7 +3324,7 @@ ACTOR Future<Void> waitForVersion(Reference<GranuleMetadata> metadata, Version v
 	// This isn't strictly needed, but if we're in the process of re-snapshotting, we'd likely rather
 	// return that snapshot file than the previous snapshot file and all its delta files.
 	if (pendingSnapshotV > metadata->durableSnapshotVersion.get() && v > metadata->durableSnapshotVersion.get()) {
-		TEST(true); // Granule read waiting for pending snapshot
+		CODE_PROBE(true, "Granule read waiting for pending snapshot");
 		wait(metadata->durableSnapshotVersion.whenAtLeast(pendingSnapshotV));
 		ASSERT(metadata->durableSnapshotVersion.get() >= pendingSnapshotV);
 	}
@@ -2929,7 +3334,7 @@ ACTOR Future<Void> waitForVersion(Reference<GranuleMetadata> metadata, Version v
 	// file instead of in memory mutations, so we wait for that delta file to complete
 
 	while (v > metadata->durableDeltaVersion.get() && metadata->pendingDeltaVersion > pendingDeltaV) {
-		TEST(true); // Granule mutations flushed while waiting for files to complete
+		CODE_PROBE(true, "Granule mutations flushed while waiting for files to complete");
 		Version waitVersion = std::min(v, metadata->pendingDeltaVersion);
 		pendingDeltaV = metadata->pendingDeltaVersion;
 		wait(metadata->durableDeltaVersion.whenAtLeast(waitVersion));
@@ -2960,7 +3365,7 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 			ASSERT(tenantEntry.get().id == req.tenantInfo.tenantId);
 			tenantPrefix = tenantEntry.get().prefix;
 		} else {
-			TEST(true); // Blob worker unknown tenant
+			CODE_PROBE(true, "Blob worker unknown tenant");
 			// FIXME - better way. Wait on retry here, or just have better model for tenant metadata?
 			// Just throw wrong_shard_server and make the client retry and assume we load it later
 			TraceEvent(SevDebug, "BlobWorkerRequestUnknownTenant", bwData->id)
@@ -3030,10 +3435,12 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 			}
 			state Reference<GranuleMetadata> metadata = m;
 			state Version granuleBeginVersion = req.beginVersion;
-
-			choose {
-				when(wait(metadata->readable.getFuture())) {}
-				when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); }
+			// skip waiting for CF ready for recovery mode
+			if (!isFullRestoreMode()) {
+				choose {
+					when(wait(metadata->readable.getFuture())) {}
+					when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); }
+				}
 			}
 
 			// in case both readable and cancelled are ready, check cancelled
@@ -3044,7 +3451,7 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 			state std::vector<std::pair<KeyRange, GranuleFiles>> rangeGranulePair;
 
 			if (req.readVersion < metadata->historyVersion) {
-				TEST(true); // Granule Time Travel Read
+				CODE_PROBE(true, "Granule Time Travel Read");
 				// this is a time travel query, find previous granule
 				if (metadata->historyLoaded.canBeSet()) {
 					choose {
@@ -3083,9 +3490,13 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 					throw blob_granule_transaction_too_old();
 				}
 
-				TEST(true); // Granule Active Read
+				CODE_PROBE(true, "Granule Active Read");
 				// this is an active granule query
 				loop {
+					// skip check since CF doesn't start for bare metal recovery mode
+					if (isFullRestoreMode()) {
+						break;
+					}
 					if (!metadata->activeCFData.get().isValid() || !metadata->cancelled.canBeSet()) {
 						throw wrong_shard_server();
 					}
@@ -3106,13 +3517,13 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 						// We can get change feed cancelled from whenAtLeast. This means the change feed may
 						// retry, or may be cancelled. Wait a bit and try again to see
 						if (e.code() == error_code_change_feed_popped) {
-							TEST(true); // Change feed popped while read waiting
+							CODE_PROBE(true, "Change feed popped while read waiting");
 							throw wrong_shard_server();
 						}
 						if (e.code() != error_code_change_feed_cancelled) {
 							throw e;
 						}
-						TEST(true); // Change feed switched while read waiting
+						CODE_PROBE(true, "Change feed switched while read waiting");
 						// wait 1ms and try again
 						wait(delay(0.001));
 					}
@@ -3123,6 +3534,18 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 						           req.readVersion);
 					}
 				}
+				// if feed was popped by another worker and BW only got empty versions, it wouldn't itself see that it
+				// got popped, but we can still reject the in theory this should never happen with other protections but
+				// it's a useful and inexpensive sanity check
+				if (!isFullRestoreMode()) {
+					Version emptyVersion = metadata->activeCFData.get()->popVersion - 1;
+					if (req.readVersion > metadata->durableDeltaVersion.get() &&
+					    emptyVersion > metadata->bufferedDeltaVersion) {
+						CODE_PROBE(true, "feed popped for read but granule updater didn't notice yet");
+						// FIXME: could try to cancel the actor here somehow, but it should find out eventually
+						throw wrong_shard_server();
+					}
+				}
 				rangeGranulePair.push_back(std::pair(metadata->keyRange, metadata->files));
 			}
 
@@ -3143,13 +3566,13 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 				Version granuleBeginVersion = req.beginVersion;
 				// Right now we force a collapse if the version range crosses granule boundaries, for simplicity
 				if (granuleBeginVersion > 0 && granuleBeginVersion <= item.second.snapshotFiles.front().version) {
-					TEST(true); // collapsed begin version request because of boundaries
+					CODE_PROBE(true, "collapsed begin version request because of boundaries");
 					didCollapse = true;
 					granuleBeginVersion = 0;
 				}
 				state BlobGranuleChunkRef chunk;
 				// TODO change with early reply
-				chunk.includedVersion = req.readVersion;
+
 				chunk.keyRange =
 				    KeyRangeRef(StringRef(rep.arena, item.first.begin), StringRef(rep.arena, item.first.end));
 				if (tenantPrefix.present()) {
@@ -3157,67 +3580,133 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 				}
 
 				int64_t deltaBytes = 0;
-				item.second.getFiles(
-				    granuleBeginVersion, req.readVersion, req.canCollapseBegin, chunk, rep.arena, deltaBytes);
+				item.second.getFiles(granuleBeginVersion,
+				                     req.readVersion,
+				                     req.canCollapseBegin,
+				                     chunk,
+				                     rep.arena,
+				                     deltaBytes,
+				                     req.summarize);
 				bwData->stats.readReqDeltaBytesReturned += deltaBytes;
 				if (granuleBeginVersion > 0 && chunk.snapshotFile.present()) {
-					TEST(true); // collapsed begin version request for efficiency
+					CODE_PROBE(true, "collapsed begin version request for efficiency");
 					didCollapse = true;
 				}
 
-				// TODO: optimization - batch 'encryption-key' lookup given the GranuleFile set is known
-				state Future<BlobGranuleCipherKeysCtx> cipherKeysCtx;
-				if (chunk.snapshotFile.present() && chunk.snapshotFile.get().cipherKeysMetaRef.present()) {
-					ASSERT(isBlobFileEncryptionSupported());
-					cipherKeysCtx =
-					    getGranuleCipherKeys(bwData, chunk.snapshotFile.get().cipherKeysMetaRef.get(), &rep.arena);
-				}
+				if (!req.summarize) {
+					chunk.includedVersion = req.readVersion;
+					// Invoke calls to populate 'EncryptionKeysCtx' for snapshot and/or deltaFiles asynchronously
+					state Optional<Future<BlobGranuleCipherKeysCtx>> snapCipherKeysCtx;
+					if (chunk.snapshotFile.present()) {
+						const bool encrypted = chunk.snapshotFile.get().cipherKeysMetaRef.present();
 
-				// new deltas (if version is larger than version of last delta file)
-				// FIXME: do trivial key bounds here if key range is not fully contained in request key
-				// range
-				if (req.readVersion > metadata->durableDeltaVersion.get() && !metadata->currentDeltas.empty()) {
-					if (metadata->durableDeltaVersion.get() != metadata->pendingDeltaVersion) {
-						fmt::print(
-						    "real-time read [{0} - {1}) @ {2} doesn't have mutations!! durable={3}, pending={4}\n",
-						    metadata->keyRange.begin.printable(),
-						    metadata->keyRange.end.printable(),
-						    req.readVersion,
-						    metadata->durableDeltaVersion.get(),
-						    metadata->pendingDeltaVersion);
-					}
-
-					// prune mutations based on begin version, if possible
-					ASSERT(metadata->durableDeltaVersion.get() == metadata->pendingDeltaVersion);
-					// FIXME: I think we can remove this dependsOn since we are doing push_back_deep
-					rep.arena.dependsOn(metadata->currentDeltas.arena());
-					MutationsAndVersionRef* mutationIt = metadata->currentDeltas.begin();
-					if (granuleBeginVersion > metadata->currentDeltas.back().version) {
-						TEST(true); // beginVersion pruning all in-memory mutations
-						mutationIt = metadata->currentDeltas.end();
-					} else if (granuleBeginVersion > metadata->currentDeltas.front().version) {
-						// binary search for beginVersion
-						TEST(true); // beginVersion pruning some in-memory mutations
-						mutationIt = std::lower_bound(metadata->currentDeltas.begin(),
-						                              metadata->currentDeltas.end(),
-						                              MutationsAndVersionRef(granuleBeginVersion, 0),
-						                              MutationsAndVersionRef::OrderByVersion());
-					}
-
-					// add mutations to response
-					while (mutationIt != metadata->currentDeltas.end()) {
-						if (mutationIt->version > req.readVersion) {
-							TEST(true); // readVersion pruning some in-memory mutations
-							break;
+						if (BW_DEBUG) {
+							TraceEvent("DoBlobGranuleFileRequestDelta_KeysCtxPrepare")
+							    .detail("FileName", chunk.snapshotFile.get().filename.toString())
+							    .detail("Encrypted", encrypted);
 						}
-						chunk.newDeltas.push_back_deep(rep.arena, *mutationIt);
-						mutationIt++;
-					}
-				}
 
-				if (chunk.snapshotFile.present() && chunk.snapshotFile.get().cipherKeysMetaRef.present()) {
-					BlobGranuleCipherKeysCtx ctx = wait(cipherKeysCtx);
-					chunk.cipherKeysCtx = std::move(ctx);
+						if (encrypted) {
+							ASSERT(bwData->isEncryptionEnabled);
+							ASSERT(!chunk.snapshotFile.get().cipherKeysCtx.present());
+
+							snapCipherKeysCtx = getGranuleCipherKeysFromKeysMetaRef(
+							    bwData, chunk.snapshotFile.get().cipherKeysMetaRef.get(), &rep.arena);
+						}
+					}
+					state std::unordered_map<int, Future<BlobGranuleCipherKeysCtx>> deltaCipherKeysCtxs;
+					for (int deltaIdx = 0; deltaIdx < chunk.deltaFiles.size(); deltaIdx++) {
+						const bool encrypted = chunk.deltaFiles[deltaIdx].cipherKeysMetaRef.present();
+
+						if (BW_DEBUG) {
+							TraceEvent("DoBlobGranuleFileRequestDelta_KeysCtxPrepare")
+							    .detail("FileName", chunk.deltaFiles[deltaIdx].filename.toString())
+							    .detail("Encrypted", encrypted);
+						}
+
+						if (encrypted) {
+							ASSERT(bwData->isEncryptionEnabled);
+							ASSERT(!chunk.deltaFiles[deltaIdx].cipherKeysCtx.present());
+
+							deltaCipherKeysCtxs.emplace(
+							    deltaIdx,
+							    getGranuleCipherKeysFromKeysMetaRef(
+							        bwData, chunk.deltaFiles[deltaIdx].cipherKeysMetaRef.get(), &rep.arena));
+						}
+					}
+
+					// new deltas (if version is larger than version of last delta file)
+					// FIXME: do trivial key bounds here if key range is not fully contained in request key
+					// range
+					if (req.readVersion > metadata->durableDeltaVersion.get() && !metadata->currentDeltas.empty()) {
+						if (metadata->durableDeltaVersion.get() != metadata->pendingDeltaVersion) {
+							fmt::print(
+							    "real-time read [{0} - {1}) @ {2} doesn't have mutations!! durable={3}, pending={4}\n",
+							    metadata->keyRange.begin.printable(),
+							    metadata->keyRange.end.printable(),
+							    req.readVersion,
+							    metadata->durableDeltaVersion.get(),
+							    metadata->pendingDeltaVersion);
+						}
+
+						// prune mutations based on begin version, if possible
+						ASSERT(metadata->durableDeltaVersion.get() == metadata->pendingDeltaVersion);
+						MutationsAndVersionRef* mutationIt = metadata->currentDeltas.begin();
+						if (granuleBeginVersion > metadata->currentDeltas.back().version) {
+							CODE_PROBE(true, "beginVersion pruning all in-memory mutations");
+							mutationIt = metadata->currentDeltas.end();
+						} else if (granuleBeginVersion > metadata->currentDeltas.front().version) {
+							// binary search for beginVersion
+							CODE_PROBE(true, "beginVersion pruning some in-memory mutations");
+							mutationIt = std::lower_bound(metadata->currentDeltas.begin(),
+							                              metadata->currentDeltas.end(),
+							                              MutationsAndVersionRef(granuleBeginVersion, 0),
+							                              MutationsAndVersionRef::OrderByVersion());
+						}
+
+						// add mutations to response
+						while (mutationIt != metadata->currentDeltas.end()) {
+							if (mutationIt->version > req.readVersion) {
+								CODE_PROBE(true, "readVersion pruning some in-memory mutations");
+								break;
+							}
+							chunk.newDeltas.push_back_deep(rep.arena, *mutationIt);
+							mutationIt++;
+						}
+					}
+
+					// Update EncryptionKeysCtx information for the chunk->snapshotFile
+					if (chunk.snapshotFile.present() && snapCipherKeysCtx.present()) {
+						ASSERT(chunk.snapshotFile.get().cipherKeysMetaRef.present());
+
+						BlobGranuleCipherKeysCtx keysCtx = wait(snapCipherKeysCtx.get());
+						chunk.snapshotFile.get().cipherKeysCtx = std::move(keysCtx);
+						// reclaim memory from non-serializable field
+						chunk.snapshotFile.get().cipherKeysMetaRef.reset();
+
+						if (BW_DEBUG) {
+							TraceEvent("DoBlobGranuleFileRequestSnap_KeysCtxDone")
+							    .detail("FileName", chunk.snapshotFile.get().filename.toString());
+						}
+					}
+
+					// Update EncryptionKeysCtx information for the chunk->deltaFiles
+					if (!deltaCipherKeysCtxs.empty()) {
+						ASSERT(!chunk.deltaFiles.empty());
+
+						state std::unordered_map<int, Future<BlobGranuleCipherKeysCtx>>::const_iterator itr;
+						for (itr = deltaCipherKeysCtxs.begin(); itr != deltaCipherKeysCtxs.end(); itr++) {
+							BlobGranuleCipherKeysCtx keysCtx = wait(itr->second);
+							chunk.deltaFiles[itr->first].cipherKeysCtx = std::move(keysCtx);
+							// reclaim memory from non-serializable field
+							chunk.deltaFiles[itr->first].cipherKeysMetaRef.reset();
+
+							if (BW_DEBUG) {
+								TraceEvent("DoBlobGranuleFileRequestDelta_KeysCtxDone")
+								    .detail("FileName", chunk.deltaFiles[itr->first].filename.toString());
+							}
+						}
+					}
 				}
 
 				rep.chunks.push_back(rep.arena, chunk);
@@ -3258,11 +3747,16 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 }
 
 ACTOR Future<Void> handleBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, BlobGranuleFileRequest req) {
+	++bwData->stats.readRequests;
+	++bwData->stats.activeReadRequests;
+	if (req.summarize) {
+		++bwData->stats.summaryReads;
+	}
 	choose {
 		when(wait(doBlobGranuleFileRequest(bwData, req))) {}
 		when(wait(delay(SERVER_KNOBS->BLOB_WORKER_REQUEST_TIMEOUT))) {
 			if (!req.reply.isSet()) {
-				TEST(true); // Blob Worker request timeout hit
+				CODE_PROBE(true, "Blob Worker request timeout hit");
 				if (BW_DEBUG) {
 					fmt::print("BW {0} request [{1} - {2}) @ {3} timed out, sending WSS\n",
 					           bwData->id.toString().substr(0, 5),
@@ -3314,28 +3808,44 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			state GranuleStartState info;
 			info.changeFeedStartVersion = invalidVersion;
 
 			state Future<Optional<Value>> fLockValue = tr.get(lockKey);
+			state Future<ForcedPurgeState> fForcedPurgeState = getForcePurgedState(&tr, req.keyRange);
 			Future<Optional<GranuleHistory>> fHistory = getLatestGranuleHistory(&tr, req.keyRange);
+
 			Optional<GranuleHistory> history = wait(fHistory);
 			info.history = history;
+
+			ForcedPurgeState purgeState = wait(fForcedPurgeState);
+			if (purgeState != ForcedPurgeState::NonePurged) {
+				CODE_PROBE(true, "Worker trying to open force purged granule");
+				if (BW_DEBUG) {
+					fmt::print("Granule [{0} - {1}) is force purged on BW {2}, abandoning\n",
+					           req.keyRange.begin.printable(),
+					           req.keyRange.end.printable(),
+					           bwData->id.toString().substr(0, 5));
+				}
+				throw granule_assignment_conflict();
+			}
+
 			Optional<Value> prevLockValue = wait(fLockValue);
 			state bool hasPrevOwner = prevLockValue.present();
 			state bool createChangeFeed = false;
+
 			if (hasPrevOwner) {
-				TEST(true); // Granule open found previous owner
+				CODE_PROBE(true, "Granule open found previous owner");
 				std::tuple<int64_t, int64_t, UID> prevOwner = decodeBlobGranuleLockValue(prevLockValue.get());
 
 				info.granuleID = std::get<2>(prevOwner);
-
 				state bool doLockCheck = true;
 				// if it's the first snapshot of a new granule, history won't be present
 				if (info.history.present()) {
 					if (info.granuleID != info.history.get().value.granuleID) {
-						TEST(true); // Blob Worker re-opening granule after merge+resplit
+						CODE_PROBE(true, "Blob Worker re-opening granule after merge+resplit");
 						// The only case this can happen is when a granule was merged into a larger granule,
 						// then split back out to the same one. Validate that this is a new granule that was
 						// split previously. Just check lock based on epoch, since seqno is intentionally
@@ -3394,9 +3904,28 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 					// if this granule is not derived from a split or merge, use new granule id
 					info.granuleID = newGranuleID;
 				}
-				createChangeFeed = true;
-				info.doSnapshot = true;
-				info.previousDurableVersion = invalidVersion;
+
+				// for recovery mode - don't create change feed, don't create snapshot
+				if (isFullRestoreMode()) {
+					createChangeFeed = false;
+					info.doSnapshot = false;
+					GranuleFiles granuleFiles = wait(loadPreviousFiles(&tr, info.granuleID));
+					info.existingFiles = granuleFiles;
+
+					if (info.existingFiles.get().snapshotFiles.empty()) {
+						ASSERT(info.existingFiles.get().deltaFiles.empty());
+						info.previousDurableVersion = invalidVersion;
+					} else if (info.existingFiles.get().deltaFiles.empty()) {
+						info.previousDurableVersion = info.existingFiles.get().snapshotFiles.back().version;
+					} else {
+						info.previousDurableVersion = info.existingFiles.get().deltaFiles.back().version;
+					}
+					info.changeFeedStartVersion = info.previousDurableVersion;
+				} else {
+					createChangeFeed = true;
+					info.doSnapshot = true;
+					info.previousDurableVersion = invalidVersion;
+				}
 			}
 
 			if (createChangeFeed) {
@@ -3411,12 +3940,13 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 			// If anything in previousGranules, need to do the handoff logic and set
 			// ret.previousChangeFeedId, and the previous durable version will come from the previous
 			// granules
-			if (info.history.present() && info.history.get().value.parentGranules.size() > 0) {
-				TEST(true); // Granule open found parent
-				if (info.history.get().value.parentGranules.size() == 1) { // split
-					state Key parentHistoryKey =
-					    blobGranuleHistoryKeyFor(info.history.get().value.parentGranules[0].first,
-					                             info.history.get().value.parentGranules[0].second);
+			if (info.history.present() && info.history.get().value.parentVersions.size() > 0 && !isFullRestoreMode()) {
+				CODE_PROBE(true, "Granule open found parent");
+				if (info.history.get().value.parentVersions.size() == 1) { // split
+					state KeyRangeRef parentRange(info.history.get().value.parentBoundaries[0],
+					                              info.history.get().value.parentBoundaries[1]);
+					state Version parentVersion = info.history.get().value.parentVersions[0];
+					state Key parentHistoryKey = blobGranuleHistoryKeyFor(parentRange, parentVersion);
 
 					Optional<Value> historyParentValue = wait(tr.get(parentHistoryKey));
 
@@ -3425,8 +3955,7 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 						    decodeBlobGranuleHistoryValue(historyParentValue.get());
 						UID parentGranuleID = val.granuleID;
 
-						info.splitParentGranule =
-						    std::pair(info.history.get().value.parentGranules[0].first, parentGranuleID);
+						info.splitParentGranule = std::pair(parentRange, parentGranuleID);
 
 						state std::pair<BlobGranuleSplitState, Version> granuleSplitState =
 						    std::pair(BlobGranuleSplitState::Initialized, invalidVersion);
@@ -3437,12 +3966,12 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 						}
 
 						if (granuleSplitState.first == BlobGranuleSplitState::Assigned) {
-							TEST(true); // Granule open found granule in assign state
+							CODE_PROBE(true, "Granule open found granule in assign state");
 							// was already assigned, use change feed start version
 							ASSERT(granuleSplitState.second > 0);
 							info.changeFeedStartVersion = granuleSplitState.second;
 						} else if (granuleSplitState.first == BlobGranuleSplitState::Initialized) {
-							TEST(true); // Granule open found granule in initialized state
+							CODE_PROBE(true, "Granule open found granule in initialized state");
 							wait(updateGranuleSplitState(&tr,
 							                             info.splitParentGranule.get().first,
 							                             info.splitParentGranule.get().second,
@@ -3451,7 +3980,7 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 							// change feed was created as part of this transaction, changeFeedStartVersion
 							// will be set later
 						} else {
-							TEST(true); // Granule open found granule in done state
+							CODE_PROBE(true, "Granule open found granule in done state");
 							// this sub-granule is done splitting, no need for split logic.
 							info.splitParentGranule.reset();
 						}
@@ -3470,7 +3999,7 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 						                                  : info.blobFilesToSnapshot[0].deltaFiles.back().version;
 					}
 				} else if (info.doSnapshot) {
-					TEST(true); // merge needs to snapshot at start
+					CODE_PROBE(true, "merge needs to snapshot at start");
 					state std::vector<Future<GranuleFiles>> parentGranulesToSnapshot;
 					ASSERT(info.previousDurableVersion == invalidVersion);
 					// need first snapshot to be at history version so this granule can serve the full range
@@ -3480,8 +4009,12 @@ ACTOR Future<GranuleStartState> openGranule(Reference<BlobWorkerData> bwData, As
 					// Can't roll back past re-snapshot version
 					info.changeFeedStartVersion = info.history.get().version;
 
-					for (auto& it : info.history.get().value.parentGranules) {
-						parentGranulesToSnapshot.push_back(loadParentGranuleForMergeSnapshot(&tr, it.first, it.second));
+					for (int i = 0; i < info.history.get().value.parentVersions.size(); i++) {
+						KeyRangeRef parentRange(info.history.get().value.parentBoundaries[i],
+						                        info.history.get().value.parentBoundaries[i + 1]);
+						Version parentVersion = info.history.get().value.parentVersions[i];
+						parentGranulesToSnapshot.push_back(
+						    loadParentGranuleForMergeSnapshot(&tr, parentRange, parentVersion));
 					}
 
 					state int pIdx;
@@ -3530,7 +4063,7 @@ ACTOR Future<Reference<BlobConnectionProvider>> loadBStoreForTenant(Reference<Bl
 			wait(delay(0));
 			return data->bstore;
 		} else {
-			TEST(true); // bstore for unknown tenant
+			CODE_PROBE(true, "bstore for unknown tenant");
 			// Assume not loaded yet, just wait a bit. Could do sophisticated mechanism but will redo tenant
 			// loading to be versioned anyway. 10 retries means it's likely not a transient race with
 			// loading tenants, and instead a persistent issue.
@@ -3768,6 +4301,17 @@ ACTOR Future<Void> handleRangeAssign(Reference<BlobWorkerData> bwData,
 		if (req.type == AssignRequestType::Continue) {
 			resumeBlobRange(bwData, req.keyRange, req.managerEpoch, req.managerSeqno);
 		} else {
+			if (!isSelfReassign && bwData->isFull()) {
+				if (BW_DEBUG) {
+					fmt::print("BW {0}: rejecting assignment [{1} - {2}) b/c full\n",
+					           bwData->id.toString().substr(0, 6),
+					           req.keyRange.begin.printable(),
+					           req.keyRange.end.printable());
+				}
+				++bwData->stats.fullRejections;
+				req.reply.sendError(blob_worker_full());
+				return Void();
+			}
 			std::vector<Future<Void>> toWait;
 			state bool shouldStart = changeBlobRange(bwData,
 			                                         req.keyRange,
@@ -3781,7 +4325,9 @@ ACTOR Future<Void> handleRangeAssign(Reference<BlobWorkerData> bwData,
 			wait(waitForAll(toWait));
 
 			if (shouldStart) {
-				bwData->stats.numRangesAssigned++;
+				if (!isSelfReassign) {
+					bwData->stats.numRangesAssigned++;
+				}
 				auto m = bwData->granuleMetadata.rangeContaining(req.keyRange.begin);
 				ASSERT(m.begin() == req.keyRange.begin && m.end() == req.keyRange.end);
 				if (m.value().activeMetadata.isValid()) {
@@ -3864,12 +4410,24 @@ ACTOR Future<Void> handleRangeRevoke(Reference<BlobWorkerData> bwData, RevokeBlo
 	}
 }
 
+void handleBlobVersionRequest(Reference<BlobWorkerData> bwData, MinBlobVersionRequest req) {
+	bwData->db->setDesiredChangeFeedVersion(
+	    std::max<Version>(0, req.grv - (SERVER_KNOBS->TARGET_BW_LAG_UPDATE * SERVER_KNOBS->VERSIONS_PER_SECOND)));
+	MinBlobVersionReply rep;
+	rep.version = bwData->db->getMinimumChangeFeedVersion();
+	bwData->stats.minimumCFVersion = rep.version;
+	bwData->stats.cfVersionLag = std::max((Version)0, req.grv - rep.version);
+	bwData->stats.notAtLatestChangeFeeds = bwData->db->notAtLatestChangeFeeds.size();
+	req.reply.send(rep);
+}
+
 ACTOR Future<Void> registerBlobWorker(Reference<BlobWorkerData> bwData, BlobWorkerInterface interf) {
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(bwData->db);
 	TraceEvent("BlobWorkerRegister", bwData->id);
 	loop {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		try {
 			Key blobWorkerListKey = blobWorkerListKeyFor(interf.id());
 			// FIXME: should be able to remove this conflict range
@@ -3906,10 +4464,11 @@ ACTOR Future<Void> monitorRemoval(Reference<BlobWorkerData> bwData) {
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
 				Optional<Value> val = wait(tr.get(blobWorkerListKey));
 				if (!val.present()) {
-					TEST(true); // Blob worker found out BM killed it from reading DB
+					CODE_PROBE(true, "Blob worker found out BM killed it from reading DB");
 					return Void();
 				}
 
@@ -3943,6 +4502,8 @@ ACTOR Future<Void> runGRVChecks(Reference<BlobWorkerData> bwData) {
 
 		tr.reset();
 		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			Version readVersion = wait(tr.getReadVersion());
 			ASSERT(readVersion >= bwData->grvVersion.get());
 			bwData->grvVersion.set(readVersion);
@@ -3963,29 +4524,23 @@ ACTOR Future<Void> monitorTenants(Reference<BlobWorkerData> bwData) {
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				state RangeResult tenantResults;
-				wait(store(tenantResults, tr->getRange(tenantMapKeys, CLIENT_KNOBS->TOO_MANY)));
-				ASSERT_WE_THINK(!tenantResults.more && tenantResults.size() < CLIENT_KNOBS->TOO_MANY);
-				if (tenantResults.more || tenantResults.size() >= CLIENT_KNOBS->TOO_MANY) {
-					TraceEvent(SevError, "BlobWorkerTooManyTenants", bwData->id)
-					    .detail("TenantCount", tenantResults.size());
-					wait(delay(600));
-					if (bwData->fatalError.canBeSet()) {
-						bwData->fatalError.sendError(internal_error());
-					}
-					throw internal_error();
-				}
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+				state KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenantResults;
+				wait(store(tenantResults,
+				           TenantMetadata::tenantMap().getRange(tr,
+				                                                Optional<TenantName>(),
+				                                                Optional<TenantName>(),
+				                                                CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)));
+				ASSERT(tenantResults.results.size() <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER && !tenantResults.more);
 
 				std::vector<std::pair<TenantName, TenantMapEntry>> tenants;
-				for (auto& it : tenantResults) {
+				for (auto& it : tenantResults.results) {
 					// FIXME: handle removing/moving tenants!
-					TenantNameRef tenantName = it.key.removePrefix(tenantMapPrefix);
-					TenantMapEntry entry = TenantMapEntry::decode(it.value);
-					tenants.push_back(std::pair(tenantName, entry));
+					tenants.push_back(std::pair(it.first, it.second));
 				}
 				bwData->tenantData.addTenants(tenants);
 
-				state Future<Void> watchChange = tr->watch(tenantLastIdKey);
+				state Future<Void> watchChange = tr->watch(TenantMetadata::lastTenantId().key);
 				wait(tr->commit());
 				wait(watchChange);
 				tr->reset();
@@ -4001,7 +4556,7 @@ void handleGetGranuleAssignmentsRequest(Reference<BlobWorkerData> self, const Ge
 	GetGranuleAssignmentsReply reply;
 	auto allRanges = self->granuleMetadata.intersectingRanges(normalKeys);
 	for (auto& it : allRanges) {
-		if (it.value().activeMetadata.isValid()) {
+		if (it.value().activeMetadata.isValid() && it.value().activeMetadata->cancelled.canBeSet()) {
 			// range is active, copy into reply's arena
 			StringRef start = StringRef(reply.arena, it.begin());
 			StringRef end = StringRef(reply.arena, it.end());
@@ -4060,94 +4615,107 @@ ACTOR Future<Void> handleFlushGranuleReq(Reference<BlobWorkerData> self, FlushGr
 				}
 			}
 
-			// force granule to flush at this version, and wait
-			if (req.flushVersion > metadata->pendingDeltaVersion) {
-				// first, wait for granule active
-
-				// wait for change feed version to catch up to ensure we have all data
-				if (metadata->activeCFData.get()->getVersion() < req.flushVersion) {
-					if (BW_DEBUG) {
-						fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: waiting for CF version "
-						           "(currently {4})\n",
-						           self->id.toString().substr(0, 5),
-						           req.granuleRange.begin.printable(),
-						           req.granuleRange.end.printable(),
-						           req.flushVersion,
-						           metadata->activeCFData.get()->getVersion());
+			loop {
+				// force granule to flush at this version, and wait
+				if (req.flushVersion > metadata->pendingDeltaVersion) {
+					// first, wait for granule active
+					if (!metadata->activeCFData.get().isValid()) {
+						req.reply.sendError(wrong_shard_server());
+						return Void();
 					}
 
-					loop {
-						choose {
-							when(wait(metadata->activeCFData.get().isValid()
-							              ? metadata->activeCFData.get()->whenAtLeast(req.flushVersion)
-							              : Never())) {
-								break;
-							}
-							when(wait(metadata->activeCFData.onChange())) {}
-							when(wait(granuleCancelled.getFuture())) {
-								if (BW_DEBUG) {
-									fmt::print("BW {0} flush granule [{1} - {2}) cancelled 2\n",
-									           self->id.toString().substr(0, 5),
-									           req.granuleRange.begin.printable(),
-									           req.granuleRange.end.printable());
+					// wait for change feed version to catch up to ensure we have all data
+					if (metadata->activeCFData.get()->getVersion() < req.flushVersion) {
+						if (BW_DEBUG) {
+							fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: waiting for CF version "
+							           "(currently {4})\n",
+							           self->id.toString().substr(0, 5),
+							           req.granuleRange.begin.printable(),
+							           req.granuleRange.end.printable(),
+							           req.flushVersion,
+							           metadata->activeCFData.get()->getVersion());
+						}
+
+						loop {
+							choose {
+								when(wait(metadata->activeCFData.get().isValid()
+								              ? metadata->activeCFData.get()->whenAtLeast(req.flushVersion)
+								              : Never())) {
+									break;
+								}
+								when(wait(metadata->activeCFData.onChange())) {}
+								when(wait(granuleCancelled.getFuture())) {
+									if (BW_DEBUG) {
+										fmt::print("BW {0} flush granule [{1} - {2}) cancelled 2\n",
+										           self->id.toString().substr(0, 5),
+										           req.granuleRange.begin.printable(),
+										           req.granuleRange.end.printable());
+									}
+									req.reply.sendError(wrong_shard_server());
+									return Void();
 								}
-								req.reply.sendError(wrong_shard_server());
-								return Void();
 							}
 						}
+
+						ASSERT(metadata->activeCFData.get()->getVersion() >= req.flushVersion);
+						if (BW_DEBUG) {
+							fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: got CF version\n",
+							           self->id.toString().substr(0, 5),
+							           req.granuleRange.begin.printable(),
+							           req.granuleRange.end.printable(),
+							           req.flushVersion);
+						}
 					}
 
-					ASSERT(metadata->activeCFData.get()->getVersion() >= req.flushVersion);
-					if (BW_DEBUG) {
-						fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: got CF version\n",
-						           self->id.toString().substr(0, 5),
-						           req.granuleRange.begin.printable(),
-						           req.granuleRange.end.printable(),
-						           req.flushVersion);
+					if (req.flushVersion > metadata->pendingDeltaVersion) {
+						if (BW_DEBUG) {
+							fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: setting force flush version\n",
+							           self->id.toString().substr(0, 5),
+							           req.granuleRange.begin.printable(),
+							           req.granuleRange.end.printable(),
+							           req.flushVersion);
+						}
+						// if after waiting for CF version, flushVersion still higher than pendingDeltaVersion,
+						// set forceFlushVersion
+						metadata->forceFlushVersion.set(req.flushVersion);
 					}
 				}
 
-				if (req.flushVersion > metadata->pendingDeltaVersion) {
-					if (BW_DEBUG) {
-						fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: setting force flush version\n",
-						           self->id.toString().substr(0, 5),
-						           req.granuleRange.begin.printable(),
-						           req.granuleRange.end.printable(),
-						           req.flushVersion);
-					}
-					// if after waiting for CF version, flushVersion still higher than pendingDeltaVersion,
-					// set forceFlushVersion
-					metadata->forceFlushVersion.set(req.flushVersion);
+				if (BW_DEBUG) {
+					fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: waiting durable\n",
+					           self->id.toString().substr(0, 5),
+					           req.granuleRange.begin.printable(),
+					           req.granuleRange.end.printable(),
+					           req.flushVersion);
 				}
-			}
+				choose {
+					when(wait(metadata->durableDeltaVersion.whenAtLeast(req.flushVersion))) {
+						if (BW_DEBUG) {
+							fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: got durable\n",
+							           self->id.toString().substr(0, 5),
+							           req.granuleRange.begin.printable(),
+							           req.granuleRange.end.printable(),
+							           req.flushVersion);
+						}
 
-			if (BW_DEBUG) {
-				fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: waiting durable\n",
-				           self->id.toString().substr(0, 5),
-				           req.granuleRange.begin.printable(),
-				           req.granuleRange.end.printable(),
-				           req.flushVersion);
-			}
-			choose {
-				when(wait(metadata->durableDeltaVersion.whenAtLeast(req.flushVersion))) {}
-				when(wait(granuleCancelled.getFuture())) {
-					if (BW_DEBUG) {
-						fmt::print("BW {0} flush granule [{1} - {2}) cancelled 3\n",
-						           self->id.toString().substr(0, 5),
-						           req.granuleRange.begin.printable(),
-						           req.granuleRange.end.printable());
+						req.reply.send(Void());
+						return Void();
+					}
+					when(wait(metadata->activeCFData.onChange())) {
+						// if a rollback happens, need to restart flush process
+					}
+					when(wait(granuleCancelled.getFuture())) {
+						if (BW_DEBUG) {
+							fmt::print("BW {0} flush granule [{1} - {2}) cancelled 3\n",
+							           self->id.toString().substr(0, 5),
+							           req.granuleRange.begin.printable(),
+							           req.granuleRange.end.printable());
+						}
+						req.reply.sendError(wrong_shard_server());
+						return Void();
 					}
-					req.reply.sendError(wrong_shard_server());
-					return Void();
 				}
 			}
-			if (BW_DEBUG) {
-				fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: got durable\n",
-				           self->id.toString().substr(0, 5),
-				           req.granuleRange.begin.printable(),
-				           req.granuleRange.end.printable(),
-				           req.flushVersion);
-			}
 		} catch (Error& e) {
 			if (BW_DEBUG) {
 				fmt::print("BW {0} flushing granule [{1} - {2}) @ {3}: got unexpected error {4}\n",
@@ -4167,10 +4735,72 @@ ACTOR Future<Void> handleFlushGranuleReq(Reference<BlobWorkerData> self, FlushGr
 			           req.granuleRange.end.printable(),
 			           req.flushVersion);
 		}
+
+		req.reply.send(Void());
+		return Void();
+	}
+}
+
+ACTOR Future<Void> simForceFileWriteContention(Reference<BlobWorkerData> bwData) {
+	// take the file write contention lock down to just 1 or 2 open writes
+	int numToLeave = deterministicRandom()->randomInt(1, 3);
+	state int numToTake = SERVER_KNOBS->BLOB_WORKER_DELTA_FILE_WRITE_PARALLELISM - numToLeave;
+	ASSERT(bwData->deltaWritesLock->available() >= numToTake);
+
+	if (numToTake <= 0) {
+		return Void();
+	}
+	if (BW_DEBUG) {
+		fmt::print("BW {0} forcing file contention down to {1}\n", bwData->id.toString().substr(0, 5), numToTake);
 	}
 
-	req.reply.send(Void());
-	return Void();
+	wait(bwData->deltaWritesLock->take(TaskPriority::DefaultYield, numToTake));
+	if (BW_DEBUG) {
+		fmt::print("BW {0} force acquired {1} file writes\n", bwData->id.toString().substr(0, 5), numToTake);
+	}
+	state FlowLock::Releaser holdingLock(*bwData->deltaWritesLock, numToTake);
+	state Future<Void> delayFor = delay(deterministicRandom()->randomInt(10, 60));
+	loop {
+		choose {
+			when(wait(delayFor)) {
+				if (BW_DEBUG) {
+					fmt::print("BW {0} releasing {1} file writes\n", bwData->id.toString().substr(0, 5), numToTake);
+				}
+				return Void();
+			}
+			// check for speed up sim
+			when(wait(delay(5.0))) {
+				if (g_simulator->speedUpSimulation) {
+					if (BW_DEBUG) {
+						fmt::print("BW {0} releasing {1} file writes b/c speed up simulation\n",
+						           bwData->id.toString().substr(0, 5),
+						           numToTake);
+					}
+					return Void();
+				}
+			}
+		}
+	}
+}
+
+ACTOR Future<Void> simForceFullMemory(Reference<BlobWorkerData> bwData) {
+	// instead of randomly rejecting each request or not, simulate periods in which BW is full
+	loop {
+		wait(delayJittered(deterministicRandom()->randomInt(5, 20)));
+		if (g_simulator->speedUpSimulation) {
+			bwData->buggifyFull = false;
+			if (BW_DEBUG) {
+				fmt::print("BW {0}: ForceFullMemory exiting\n", bwData->id.toString().substr(0, 6));
+			}
+			return Void();
+		}
+		bwData->buggifyFull = !bwData->buggifyFull;
+		if (BW_DEBUG) {
+			fmt::print("BW {0}: ForceFullMemory {1}\n",
+			           bwData->id.toString().substr(0, 6),
+			           bwData->buggifyFull ? "starting" : "stopping");
+		}
+	}
 }
 
 ACTOR Future<Void> blobWorker(BlobWorkerInterface bwInterf,
@@ -4188,7 +4818,7 @@ ACTOR Future<Void> blobWorker(BlobWorkerInterface bwInterf,
 	}
 
 	try {
-		if (SERVER_KNOBS->BG_RANGE_SOURCE != "tenant") {
+		if (SERVER_KNOBS->BG_METADATA_SOURCE != "tenant") {
 			if (BW_DEBUG) {
 				fmt::print("BW constructing backup container from {0}\n", SERVER_KNOBS->BG_URL);
 			}
@@ -4221,18 +4851,20 @@ ACTOR Future<Void> blobWorker(BlobWorkerInterface bwInterf,
 
 	self->addActor.send(waitFailureServer(bwInterf.waitFailure.getFuture()));
 	self->addActor.send(runGRVChecks(self));
-	if (SERVER_KNOBS->BG_RANGE_SOURCE == "tenant") {
-		self->addActor.send(monitorTenants(self));
-	}
+	self->addActor.send(monitorTenants(self));
 	state Future<Void> selfRemoved = monitorRemoval(self);
+	if (g_network->isSimulated() && BUGGIFY_WITH_PROB(0.25)) {
+		self->addActor.send(simForceFileWriteContention(self));
+	}
+	if (g_network->isSimulated() && SERVER_KNOBS->BLOB_WORKER_DO_REJECT_WHEN_FULL && BUGGIFY_WITH_PROB(0.25)) {
+		self->addActor.send(simForceFullMemory(self));
+	}
 
 	TraceEvent("BlobWorkerInit", self->id).log();
 
 	try {
 		loop choose {
 			when(BlobGranuleFileRequest req = waitNext(bwInterf.blobGranuleFileRequest.getFuture())) {
-				++self->stats.readRequests;
-				++self->stats.activeReadRequests;
 				self->addActor.send(handleBlobGranuleFileRequest(self, req));
 			}
 			when(state GranuleStatusStreamRequest req = waitNext(bwInterf.granuleStatusStreamRequest.getFuture())) {
@@ -4254,7 +4886,6 @@ ACTOR Future<Void> blobWorker(BlobWorkerInterface bwInterf,
 					if (self->statusStreamInitialized) {
 						copy = self->currentManagerStatusStream.get();
 					}
-					// TODO: pick a reasonable byte limit instead of just piggy-backing
 					req.reply.setByteLimit(SERVER_KNOBS->BLOBWORKERSTATUSSTREAM_LIMIT_BYTES);
 					self->statusStreamInitialized = true;
 
@@ -4304,7 +4935,8 @@ ACTOR Future<Void> blobWorker(BlobWorkerInterface bwInterf,
 				self->addActor.send(handleRangeAssign(self, granuleToReassign, true));
 			}
 			when(GetGranuleAssignmentsRequest req = waitNext(bwInterf.granuleAssignmentsRequest.getFuture())) {
-				if (self->managerEpochOk(req.managerEpoch)) {
+				// if request isn't from a manager and is just validation, let it check
+				if (req.managerEpoch == -1 || self->managerEpochOk(req.managerEpoch)) {
 					if (BW_DEBUG) {
 						fmt::print("Worker {0} got granule assignments request from BM {1}\n",
 						           self->id.toString(),
@@ -4329,6 +4961,9 @@ ACTOR Future<Void> blobWorker(BlobWorkerInterface bwInterf,
 					req.reply.sendError(blob_manager_replaced());
 				}
 			}
+			when(MinBlobVersionRequest req = waitNext(bwInterf.minBlobVersionRequest.getFuture())) {
+				handleBlobVersionRequest(self, req);
+			}
 			when(FlushGranuleRequest req = waitNext(bwInterf.flushGranuleRequest.getFuture())) {
 				if (self->managerEpochOk(req.managerEpoch)) {
 					if (BW_DEBUG) {
diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index d9deec50eb..2b1a26bf6d 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -6,9 +6,7 @@ else()
   list(APPEND FDBSERVER_SRCS coroimpl/CoroFlow.actor.cpp)
 endif()
 
-if (WITH_ROCKSDB_EXPERIMENTAL)
-  add_definitions(-DSSD_ROCKSDB_EXPERIMENTAL)
-
+if(WITH_ROCKSDB_EXPERIMENTAL)
   include(CompileRocksDB)
   # CompileRocksDB sets `lz4_LIBRARIES` to be the shared lib, we want to link
   # statically, so find the static library here.
@@ -23,13 +21,6 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/workloads)
 
 add_flow_target(EXECUTABLE NAME fdbserver SRCS ${FDBSERVER_SRCS})
 
-find_package(ZLIB)
-if(ZLIB_FOUND)
-  add_compile_definitions(ZLIB_LIB_SUPPORTED)
-else()
-  message(STATUS "ZLIB package not found")
-endif()
-
 target_include_directories(fdbserver PRIVATE
   ${CMAKE_SOURCE_DIR}/bindings/c
   ${CMAKE_BINARY_DIR}/bindings/c
@@ -51,6 +42,9 @@ else()
 endif()
 
 target_link_libraries(fdbserver PRIVATE toml11_target jemalloc rapidjson)
+if(WITH_ROCKSDB_EXPERIMENTAL)
+  target_compile_definitions(fdbserver PRIVATE SSD_ROCKSDB_EXPERIMENTAL)
+endif()
 # target_compile_definitions(fdbserver PRIVATE -DENABLE_SAMPLING)
 
 if(GPERFTOOLS_FOUND)
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 4a0ebb4a5c..1222c94f07 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -26,12 +26,14 @@
 #include <vector>
 
 #include "fdbclient/SystemData.h"
+#include "fdbclient/DatabaseContext.h"
 #include "fdbrpc/FailureMonitor.h"
-#include "fdbserver/EncryptKeyProxyInterface.h"
+#include "fdbclient/EncryptKeyProxyInterface.h"
 #include "fdbserver/Knobs.h"
 #include "flow/ActorCollection.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/TenantManagement.actor.h"
 #include "fdbserver/ApplyMetadataMutation.h"
 #include "fdbserver/BackupInterface.h"
 #include "fdbserver/BackupProgress.actor.h"
@@ -42,7 +44,6 @@
 #include "fdbserver/ClusterRecovery.actor.h"
 #include "fdbserver/DataDistributorInterface.h"
 #include "fdbserver/DBCoreState.h"
-#include "fdbserver/ConfigBroadcaster.h"
 #include "fdbserver/MoveKeys.actor.h"
 #include "fdbserver/LeaderElection.h"
 #include "fdbserver/LogSystem.h"
@@ -102,7 +103,7 @@ struct RatekeeperSingleton : Singleton<RatekeeperInterface> {
 		}
 	}
 	void halt(ClusterControllerData* cc, Optional<Standalone<StringRef>> pid) const {
-		if (interface.present()) {
+		if (interface.present() && cc->id_worker.count(pid)) {
 			cc->id_worker[pid].haltRatekeeper =
 			    brokenPromiseToNever(interface.get().haltRatekeeper.getReply(HaltRatekeeperRequest(cc->id)));
 		}
@@ -127,7 +128,7 @@ struct DataDistributorSingleton : Singleton<DataDistributorInterface> {
 		}
 	}
 	void halt(ClusterControllerData* cc, Optional<Standalone<StringRef>> pid) const {
-		if (interface.present()) {
+		if (interface.present() && cc->id_worker.count(pid)) {
 			cc->id_worker[pid].haltDistributor =
 			    brokenPromiseToNever(interface.get().haltDataDistributor.getReply(HaltDataDistributorRequest(cc->id)));
 		}
@@ -138,6 +139,31 @@ struct DataDistributorSingleton : Singleton<DataDistributorInterface> {
 	}
 };
 
+struct ConsistencyScanSingleton : Singleton<ConsistencyScanInterface> {
+
+	ConsistencyScanSingleton(const Optional<ConsistencyScanInterface>& interface) : Singleton(interface) {}
+
+	Role getRole() const { return Role::CONSISTENCYSCAN; }
+	ProcessClass::ClusterRole getClusterRole() const { return ProcessClass::ConsistencyScan; }
+
+	void setInterfaceToDbInfo(ClusterControllerData* cc) const {
+		if (interface.present()) {
+			TraceEvent("CCCK_SetInf", cc->id).detail("Id", interface.get().id());
+			cc->db.setConsistencyScan(interface.get());
+		}
+	}
+	void halt(ClusterControllerData* cc, Optional<Standalone<StringRef>> pid) const {
+		if (interface.present()) {
+			cc->id_worker[pid].haltConsistencyScan =
+			    brokenPromiseToNever(interface.get().haltConsistencyScan.getReply(HaltConsistencyScanRequest(cc->id)));
+		}
+	}
+	void recruit(ClusterControllerData* cc) const {
+		cc->lastRecruitTime = now();
+		cc->recruitConsistencyScan.set(true);
+	}
+};
+
 struct BlobManagerSingleton : Singleton<BlobManagerInterface> {
 
 	BlobManagerSingleton(const Optional<BlobManagerInterface>& interface) : Singleton(interface) {}
@@ -152,7 +178,7 @@ struct BlobManagerSingleton : Singleton<BlobManagerInterface> {
 		}
 	}
 	void halt(ClusterControllerData* cc, Optional<Standalone<StringRef>> pid) const {
-		if (interface.present()) {
+		if (interface.present() && cc->id_worker.count(pid)) {
 			cc->id_worker[pid].haltBlobManager =
 			    brokenPromiseToNever(interface.get().haltBlobManager.getReply(HaltBlobManagerRequest(cc->id)));
 		}
@@ -184,7 +210,7 @@ struct EncryptKeyProxySingleton : Singleton<EncryptKeyProxyInterface> {
 		}
 	}
 	void halt(ClusterControllerData* cc, Optional<Standalone<StringRef>> pid) const {
-		if (interface.present()) {
+		if (interface.present() && cc->id_worker.count(pid)) {
 			cc->id_worker[pid].haltEncryptKeyProxy =
 			    brokenPromiseToNever(interface.get().haltEncryptKeyProxy.getReply(HaltEncryptKeyProxyRequest(cc->id)));
 		}
@@ -195,13 +221,17 @@ struct EncryptKeyProxySingleton : Singleton<EncryptKeyProxyInterface> {
 	}
 };
 
-ACTOR Future<Void> handleLeaderReplacement(Reference<ClusterRecoveryData> self, Future<Void> leaderFail) {
-	loop choose {
-		when(wait(leaderFail)) {
-			TraceEvent("LeaderReplaced", self->controllerData->id).log();
-			// We are no longer the leader if this has changed.
-			self->controllerData->shouldCommitSuicide = true;
-			throw restart_cluster_controller();
+ACTOR Future<Optional<Value>> getPreviousCoordinators(ClusterControllerData* self) {
+	state ReadYourWritesTransaction tr(self->db.db);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			Optional<Value> previousCoordinators = wait(tr.get(previousCoordinatorsKey));
+			return previousCoordinators;
+		} catch (Error& e) {
+			wait(tr.onError(e));
 		}
 	}
 }
@@ -209,7 +239,6 @@ ACTOR Future<Void> handleLeaderReplacement(Reference<ClusterRecoveryData> self,
 ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
                                         ClusterControllerData::DBInfo* db,
                                         ServerCoordinators coordinators,
-                                        Future<Void> leaderFail,
                                         Future<Void> recoveredDiskFiles) {
 	state MasterInterface iMaster;
 	state Reference<ClusterRecoveryData> recoveryData;
@@ -245,11 +274,16 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 			dbInfo.ratekeeper = db->serverInfo->get().ratekeeper;
 			dbInfo.blobManager = db->serverInfo->get().blobManager;
 			dbInfo.encryptKeyProxy = db->serverInfo->get().encryptKeyProxy;
+			dbInfo.consistencyScan = db->serverInfo->get().consistencyScan;
 			dbInfo.latencyBandConfig = db->serverInfo->get().latencyBandConfig;
 			dbInfo.myLocality = db->serverInfo->get().myLocality;
 			dbInfo.client = ClientDBInfo();
-			dbInfo.client.tenantMode = db->config.tenantMode;
+			dbInfo.client.encryptKeyProxy = db->serverInfo->get().encryptKeyProxy;
+			dbInfo.client.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
+			dbInfo.client.tenantMode = TenantAPI::tenantModeForClusterType(db->clusterType, db->config.tenantMode);
 			dbInfo.client.clusterId = db->serverInfo->get().client.clusterId;
+			dbInfo.client.clusterType = db->clusterType;
+			dbInfo.client.metaclusterName = db->metaclusterName;
 
 			TraceEvent("CCWDB", cluster->id)
 			    .detail("NewMaster", dbInfo.master.id().toString())
@@ -269,7 +303,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 			                                                  db->serverInfo->get().masterLifetime,
 			                                                  coordinators,
 			                                                  db->serverInfo->get().clusterInterface,
-			                                                  LiteralStringRef(""),
+			                                                  ""_sr,
 			                                                  addActor,
 			                                                  db->forceRecovery);
 
@@ -302,7 +336,6 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 					TraceEvent(SevDebug, "BackupWorkerDoneRequest", cluster->id).log();
 				}
 				when(wait(collection)) { throw internal_error(); }
-				when(wait(handleLeaderReplacement(recoveryData, leaderFail))) { throw internal_error(); }
 			}
 			// failed master (better master exists) could happen while change-coordinators request processing is
 			// in-progress
@@ -316,7 +349,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 
 			wait(spinDelay);
 
-			TEST(true); // clusterWatchDatabase() master failed
+			CODE_PROBE(true, "clusterWatchDatabase() master failed");
 			TraceEvent(SevWarn, "DetectedFailedRecovery", cluster->id).detail("OldMaster", iMaster.id());
 		} catch (Error& e) {
 			state Error err = e;
@@ -328,13 +361,14 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 			wait(cleanupRecoveryActorCollection(recoveryData, true /* exThrown */));
 			ASSERT(addActor.isEmpty());
 
-			TEST(err.code() == error_code_tlog_failed); // Terminated due to tLog failure
-			TEST(err.code() == error_code_commit_proxy_failed); // Terminated due to commit proxy failure
-			TEST(err.code() == error_code_grv_proxy_failed); // Terminated due to GRV proxy failure
-			TEST(err.code() == error_code_resolver_failed); // Terminated due to resolver failure
-			TEST(err.code() == error_code_backup_worker_failed); // Terminated due to backup worker failure
-			TEST(err.code() == error_code_operation_failed); // Terminated due to failed operation
-			TEST(err.code() == error_code_restart_cluster_controller); // Terminated due to cluster-controller restart.
+			CODE_PROBE(err.code() == error_code_tlog_failed, "Terminated due to tLog failure");
+			CODE_PROBE(err.code() == error_code_commit_proxy_failed, "Terminated due to commit proxy failure");
+			CODE_PROBE(err.code() == error_code_grv_proxy_failed, "Terminated due to GRV proxy failure");
+			CODE_PROBE(err.code() == error_code_resolver_failed, "Terminated due to resolver failure");
+			CODE_PROBE(err.code() == error_code_backup_worker_failed, "Terminated due to backup worker failure");
+			CODE_PROBE(err.code() == error_code_operation_failed, "Terminated due to failed operation");
+			CODE_PROBE(err.code() == error_code_restart_cluster_controller,
+			           "Terminated due to cluster-controller restart.");
 
 			if (cluster->shouldCommitSuicide || err.code() == error_code_coordinators_changed) {
 				TraceEvent("ClusterControllerTerminate", cluster->id).errorUnsuppressed(err);
@@ -615,6 +649,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	// Try to find a new process for each singleton.
 	WorkerDetails newRKWorker = findNewProcessForSingleton(self, ProcessClass::Ratekeeper, id_used);
 	WorkerDetails newDDWorker = findNewProcessForSingleton(self, ProcessClass::DataDistributor, id_used);
+	WorkerDetails newCSWorker = findNewProcessForSingleton(self, ProcessClass::ConsistencyScan, id_used);
 
 	WorkerDetails newBMWorker;
 	if (self->db.blobGranulesEnabled.get()) {
@@ -629,6 +664,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	// Find best possible fitnesses for each singleton.
 	auto bestFitnessForRK = findBestFitnessForSingleton(self, newRKWorker, ProcessClass::Ratekeeper);
 	auto bestFitnessForDD = findBestFitnessForSingleton(self, newDDWorker, ProcessClass::DataDistributor);
+	auto bestFitnessForCS = findBestFitnessForSingleton(self, newCSWorker, ProcessClass::ConsistencyScan);
 
 	ProcessClass::Fitness bestFitnessForBM;
 	if (self->db.blobGranulesEnabled.get()) {
@@ -643,6 +679,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	auto& db = self->db.serverInfo->get();
 	auto rkSingleton = RatekeeperSingleton(db.ratekeeper);
 	auto ddSingleton = DataDistributorSingleton(db.distributor);
+	ConsistencyScanSingleton csSingleton(db.consistencyScan);
 	BlobManagerSingleton bmSingleton(db.blobManager);
 	EncryptKeyProxySingleton ekpSingleton(db.encryptKeyProxy);
 
@@ -654,6 +691,9 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	bool ddHealthy = isHealthySingleton<DataDistributorInterface>(
 	    self, newDDWorker, ddSingleton, bestFitnessForDD, self->recruitingDistributorID);
 
+	bool csHealthy = isHealthySingleton<ConsistencyScanInterface>(
+	    self, newCSWorker, csSingleton, bestFitnessForCS, self->recruitingConsistencyScanID);
+
 	bool bmHealthy = true;
 	if (self->db.blobGranulesEnabled.get()) {
 		bmHealthy = isHealthySingleton<BlobManagerInterface>(
@@ -667,7 +707,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	}
 	// if any of the singletons are unhealthy (rerecruited or not stable), then do not
 	// consider any further re-recruitments
-	if (!(rkHealthy && ddHealthy && bmHealthy && ekpHealthy)) {
+	if (!(rkHealthy && ddHealthy && bmHealthy && ekpHealthy && csHealthy)) {
 		return;
 	}
 
@@ -675,8 +715,10 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	// check if we can colocate the singletons in a more optimal way
 	Optional<Standalone<StringRef>> currRKProcessId = rkSingleton.interface.get().locality.processId();
 	Optional<Standalone<StringRef>> currDDProcessId = ddSingleton.interface.get().locality.processId();
+	Optional<Standalone<StringRef>> currCSProcessId = csSingleton.interface.get().locality.processId();
 	Optional<Standalone<StringRef>> newRKProcessId = newRKWorker.interf.locality.processId();
 	Optional<Standalone<StringRef>> newDDProcessId = newDDWorker.interf.locality.processId();
+	Optional<Standalone<StringRef>> newCSProcessId = newCSWorker.interf.locality.processId();
 
 	Optional<Standalone<StringRef>> currBMProcessId, newBMProcessId;
 	if (self->db.blobGranulesEnabled.get()) {
@@ -690,8 +732,8 @@ void checkBetterSingletons(ClusterControllerData* self) {
 		newEKPProcessId = newEKPWorker.interf.locality.processId();
 	}
 
-	std::vector<Optional<Standalone<StringRef>>> currPids = { currRKProcessId, currDDProcessId };
-	std::vector<Optional<Standalone<StringRef>>> newPids = { newRKProcessId, newDDProcessId };
+	std::vector<Optional<Standalone<StringRef>>> currPids = { currRKProcessId, currDDProcessId, currCSProcessId };
+	std::vector<Optional<Standalone<StringRef>>> newPids = { newRKProcessId, newDDProcessId, newCSProcessId };
 	if (self->db.blobGranulesEnabled.get()) {
 		currPids.emplace_back(currBMProcessId);
 		newPids.emplace_back(newBMProcessId);
@@ -721,7 +763,8 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	if (newColocMap[newRKProcessId] <= currColocMap[currRKProcessId] &&
 	    newColocMap[newDDProcessId] <= currColocMap[currDDProcessId] &&
 	    newColocMap[newBMProcessId] <= currColocMap[currBMProcessId] &&
-	    newColocMap[newEKPProcessId] <= currColocMap[currEKPProcessId]) {
+	    newColocMap[newEKPProcessId] <= currColocMap[currEKPProcessId] &&
+	    newColocMap[newCSProcessId] <= currColocMap[currCSProcessId]) {
 		// rerecruit the singleton for which we have found a better process, if any
 		if (newColocMap[newRKProcessId] < currColocMap[currRKProcessId]) {
 			rkSingleton.recruit(self);
@@ -731,6 +774,8 @@ void checkBetterSingletons(ClusterControllerData* self) {
 			bmSingleton.recruit(self);
 		} else if (SERVER_KNOBS->ENABLE_ENCRYPTION && newColocMap[newEKPProcessId] < currColocMap[currEKPProcessId]) {
 			ekpSingleton.recruit(self);
+		} else if (newColocMap[newCSProcessId] < currColocMap[currCSProcessId]) {
+			csSingleton.recruit(self);
 		}
 	}
 }
@@ -797,6 +842,7 @@ void checkOutstandingRequests(ClusterControllerData* self) {
 
 ACTOR Future<Void> rebootAndCheck(ClusterControllerData* cluster, Optional<Standalone<StringRef>> processID) {
 	{
+		ASSERT(processID.present());
 		auto watcher = cluster->id_worker.find(processID);
 		ASSERT(watcher != cluster->id_worker.end());
 
@@ -1010,7 +1056,11 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co
 	// Construct the client information
 	if (db->clientInfo->get().commitProxies != req.commitProxies ||
 	    db->clientInfo->get().grvProxies != req.grvProxies ||
-	    db->clientInfo->get().tenantMode != db->config.tenantMode || db->clientInfo->get().clusterId != req.clusterId) {
+	    db->clientInfo->get().tenantMode != db->config.tenantMode || db->clientInfo->get().clusterId != req.clusterId ||
+	    db->clientInfo->get().isEncryptionEnabled != SERVER_KNOBS->ENABLE_ENCRYPTION ||
+	    db->clientInfo->get().clusterType != db->clusterType ||
+	    db->clientInfo->get().metaclusterName != db->metaclusterName ||
+	    db->clientInfo->get().encryptKeyProxy != db->serverInfo->get().encryptKeyProxy) {
 		TraceEvent("PublishNewClientInfo", self->id)
 		    .detail("Master", dbInfo.master.id())
 		    .detail("GrvProxies", db->clientInfo->get().grvProxies)
@@ -1020,15 +1070,24 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co
 		    .detail("TenantMode", db->clientInfo->get().tenantMode.toString())
 		    .detail("ReqTenantMode", db->config.tenantMode.toString())
 		    .detail("ClusterId", db->clientInfo->get().clusterId)
-		    .detail("ReqClusterId", req.clusterId);
+		    .detail("ReqClusterId", req.clusterId)
+		    .detail("EncryptionEnabled", SERVER_KNOBS->ENABLE_ENCRYPTION)
+		    .detail("ClusterType", db->clientInfo->get().clusterType)
+		    .detail("ReqClusterType", db->clusterType)
+		    .detail("MetaclusterName", db->clientInfo->get().metaclusterName)
+		    .detail("ReqMetaclusterName", db->metaclusterName);
 		isChanged = true;
 		// TODO why construct a new one and not just copy the old one and change proxies + id?
 		ClientDBInfo clientInfo;
+		clientInfo.encryptKeyProxy = db->serverInfo->get().encryptKeyProxy;
 		clientInfo.id = deterministicRandom()->randomUniqueID();
+		clientInfo.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
 		clientInfo.commitProxies = req.commitProxies;
 		clientInfo.grvProxies = req.grvProxies;
-		clientInfo.tenantMode = db->config.tenantMode;
+		clientInfo.tenantMode = TenantAPI::tenantModeForClusterType(db->clusterType, db->config.tenantMode);
 		clientInfo.clusterId = req.clusterId;
+		clientInfo.clusterType = db->clusterType;
+		clientInfo.metaclusterName = db->metaclusterName;
 		db->clientInfo->set(clientInfo);
 		dbInfo.client = db->clientInfo->get();
 	}
@@ -1202,13 +1261,12 @@ ACTOR Future<Void> registerWorker(RegisterWorkerRequest req,
 		    w.locality.processId() == self->db.serverInfo->get().master.locality.processId()) {
 			self->masterProcessId = w.locality.processId();
 		}
-		if (configBroadcaster != nullptr && isCoordinator) {
-			self->addActor.send(configBroadcaster->registerNode(
-			    w,
-			    req.lastSeenKnobVersion,
-			    req.knobConfigClassSet,
-			    self->id_worker[w.locality.processId()].watcher,
-			    self->id_worker[w.locality.processId()].details.interf.configBroadcastInterface));
+		if (configBroadcaster != nullptr && req.lastSeenKnobVersion.present() && req.knobConfigClassSet.present()) {
+			self->addActor.send(configBroadcaster->registerNode(req.configBroadcastInterface,
+			                                                    req.lastSeenKnobVersion.get(),
+			                                                    req.knobConfigClassSet.get(),
+			                                                    self->id_worker[w.locality.processId()].watcher,
+			                                                    isCoordinator));
 		}
 		self->updateDBInfoEndpoints.insert(w.updateServerDBInfo.getEndpoint());
 		self->updateDBInfo.trigger();
@@ -1229,22 +1287,26 @@ ACTOR Future<Void> registerWorker(RegisterWorkerRequest req,
 		if (info->second.details.interf.id() != w.id()) {
 			self->removedDBInfoEndpoints.insert(info->second.details.interf.updateServerDBInfo.getEndpoint());
 			info->second.details.interf = w;
+			// Cancel the existing watcher actor; possible race condition could be, the older registered watcher
+			// detects failures and removes the worker from id_worker even before the new watcher starts monitoring the
+			// new interface
+			info->second.watcher.cancel();
 			info->second.watcher = workerAvailabilityWatch(w, newProcessClass, self);
 		}
 		if (req.requestDbInfo) {
 			self->updateDBInfoEndpoints.insert(w.updateServerDBInfo.getEndpoint());
 			self->updateDBInfo.trigger();
 		}
-		if (configBroadcaster != nullptr && isCoordinator) {
-			self->addActor.send(configBroadcaster->registerNode(w,
-			                                                    req.lastSeenKnobVersion,
-			                                                    req.knobConfigClassSet,
+		if (configBroadcaster != nullptr && req.lastSeenKnobVersion.present() && req.knobConfigClassSet.present()) {
+			self->addActor.send(configBroadcaster->registerNode(req.configBroadcastInterface,
+			                                                    req.lastSeenKnobVersion.get(),
+			                                                    req.knobConfigClassSet.get(),
 			                                                    info->second.watcher,
-			                                                    info->second.details.interf.configBroadcastInterface));
+			                                                    isCoordinator));
 		}
 		checkOutstandingRequests(self);
 	} else {
-		TEST(true); // Received an old worker registration request.
+		CODE_PROBE(true, "Received an old worker registration request.");
 	}
 
 	// For each singleton
@@ -1278,6 +1340,13 @@ ACTOR Future<Void> registerWorker(RegisterWorkerRequest req,
 		    self, w, currSingleton, registeringSingleton, self->recruitingEncryptKeyProxyID);
 	}
 
+	if (req.consistencyScanInterf.present()) {
+		auto currSingleton = ConsistencyScanSingleton(self->db.serverInfo->get().consistencyScan);
+		auto registeringSingleton = ConsistencyScanSingleton(req.consistencyScanInterf);
+		haltRegisteringOrCurrentSingleton<ConsistencyScanInterface>(
+		    self, w, currSingleton, registeringSingleton, self->recruitingConsistencyScanID);
+	}
+
 	// Notify the worker to register again with new process class/exclusive property
 	if (!req.reply.isSet() && newPriorityInfo != req.priorityInfo) {
 		req.reply.send(RegisterWorkerReply(newProcessClass, newPriorityInfo));
@@ -1286,7 +1355,7 @@ ACTOR Future<Void> registerWorker(RegisterWorkerRequest req,
 	return Void();
 }
 
-#define TIME_KEEPER_VERSION LiteralStringRef("1")
+#define TIME_KEEPER_VERSION "1"_sr
 
 ACTOR Future<Void> timeKeeperSetVersion(ClusterControllerData* self) {
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->cx);
@@ -2034,8 +2103,9 @@ ACTOR Future<Void> monitorDataDistributor(ClusterControllerData* self) {
 			choose {
 				when(wait(waitFailureClient(self->db.serverInfo->get().distributor.get().waitFailure,
 				                            SERVER_KNOBS->DD_FAILURE_TIME))) {
-					TraceEvent("CCDataDistributorDied", self->id)
-					    .detail("DDID", self->db.serverInfo->get().distributor.get().id());
+					const auto& distributor = self->db.serverInfo->get().distributor;
+					TraceEvent("CCDataDistributorDied", self->id).detail("DDID", distributor.get().id());
+					DataDistributorSingleton(distributor).halt(self, distributor.get().locality.processId());
 					self->db.clearInterf(ProcessClass::DataDistributorClass);
 				}
 				when(wait(self->recruitDistributor.onChange())) {}
@@ -2125,8 +2195,9 @@ ACTOR Future<Void> monitorRatekeeper(ClusterControllerData* self) {
 			choose {
 				when(wait(waitFailureClient(self->db.serverInfo->get().ratekeeper.get().waitFailure,
 				                            SERVER_KNOBS->RATEKEEPER_FAILURE_TIME))) {
-					TraceEvent("CCRatekeeperDied", self->id)
-					    .detail("RKID", self->db.serverInfo->get().ratekeeper.get().id());
+					const auto& ratekeeper = self->db.serverInfo->get().ratekeeper;
+					TraceEvent("CCRatekeeperDied", self->id).detail("RKID", ratekeeper.get().id());
+					RatekeeperSingleton(ratekeeper).halt(self, ratekeeper.get().locality.processId());
 					self->db.clearInterf(ProcessClass::RatekeeperClass);
 				}
 				when(wait(self->recruitRatekeeper.onChange())) {}
@@ -2137,6 +2208,101 @@ ACTOR Future<Void> monitorRatekeeper(ClusterControllerData* self) {
 	}
 }
 
+ACTOR Future<Void> startConsistencyScan(ClusterControllerData* self) {
+	wait(delay(0.0)); // If master fails at the same time, give it a chance to clear master PID.
+	TraceEvent("CCStartConsistencyScan", self->id).log();
+	loop {
+		try {
+			state bool no_consistencyScan = !self->db.serverInfo->get().consistencyScan.present();
+			while (!self->masterProcessId.present() ||
+			       self->masterProcessId != self->db.serverInfo->get().master.locality.processId() ||
+			       self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+				wait(self->db.serverInfo->onChange() || delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY));
+			}
+			if (no_consistencyScan && self->db.serverInfo->get().consistencyScan.present()) {
+				// Existing consistencyScan registers while waiting, so skip.
+				return Void();
+			}
+
+			std::map<Optional<Standalone<StringRef>>, int> id_used = self->getUsedIds();
+			WorkerFitnessInfo csWorker = self->getWorkerForRoleInDatacenter(self->clusterControllerDcId,
+			                                                                ProcessClass::ConsistencyScan,
+			                                                                ProcessClass::NeverAssign,
+			                                                                self->db.config,
+			                                                                id_used);
+
+			InitializeConsistencyScanRequest req(deterministicRandom()->randomUniqueID());
+			state WorkerDetails worker = csWorker.worker;
+			if (self->onMasterIsBetter(worker, ProcessClass::ConsistencyScan)) {
+				worker = self->id_worker[self->masterProcessId.get()].details;
+			}
+
+			self->recruitingConsistencyScanID = req.reqId;
+			TraceEvent("CCRecruitConsistencyScan", self->id)
+			    .detail("Addr", worker.interf.address())
+			    .detail("CSID", req.reqId);
+
+			ErrorOr<ConsistencyScanInterface> interf = wait(worker.interf.consistencyScan.getReplyUnlessFailedFor(
+			    req, SERVER_KNOBS->WAIT_FOR_CONSISTENCYSCAN_JOIN_DELAY, 0));
+			if (interf.present()) {
+				self->recruitConsistencyScan.set(false);
+				self->recruitingConsistencyScanID = interf.get().id();
+				const auto& consistencyScan = self->db.serverInfo->get().consistencyScan;
+				TraceEvent("CCConsistencyScanRecruited", self->id)
+				    .detail("Addr", worker.interf.address())
+				    .detail("CKID", interf.get().id());
+				if (consistencyScan.present() && consistencyScan.get().id() != interf.get().id() &&
+				    self->id_worker.count(consistencyScan.get().locality.processId())) {
+					TraceEvent("CCHaltConsistencyScanAfterRecruit", self->id)
+					    .detail("CKID", consistencyScan.get().id())
+					    .detail("DcID", printable(self->clusterControllerDcId));
+					ConsistencyScanSingleton(consistencyScan).halt(self, consistencyScan.get().locality.processId());
+				}
+				if (!consistencyScan.present() || consistencyScan.get().id() != interf.get().id()) {
+					self->db.setConsistencyScan(interf.get());
+				}
+				checkOutstandingRequests(self);
+				return Void();
+			} else {
+				TraceEvent("CCConsistencyScanRecruitEmpty", self->id).log();
+			}
+		} catch (Error& e) {
+			TraceEvent("CCConsistencyScanRecruitError", self->id).error(e);
+			if (e.code() != error_code_no_more_servers) {
+				throw;
+			}
+		}
+		wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY));
+	}
+}
+
+ACTOR Future<Void> monitorConsistencyScan(ClusterControllerData* self) {
+	while (self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+		TraceEvent("CCMonitorConsistencyScanWaitingForRecovery", self->id).log();
+		wait(self->db.serverInfo->onChange());
+	}
+
+	TraceEvent("CCMonitorConsistencyScan", self->id).log();
+	loop {
+		if (self->db.serverInfo->get().consistencyScan.present() && !self->recruitConsistencyScan.get()) {
+			state Future<Void> wfClient =
+			    waitFailureClient(self->db.serverInfo->get().consistencyScan.get().waitFailure,
+			                      SERVER_KNOBS->CONSISTENCYSCAN_FAILURE_TIME);
+			choose {
+				when(wait(wfClient)) {
+					TraceEvent("CCMonitorConsistencyScanDied", self->id)
+					    .detail("CKID", self->db.serverInfo->get().consistencyScan.get().id());
+					self->db.clearInterf(ProcessClass::ConsistencyScanClass);
+				}
+				when(wait(self->recruitConsistencyScan.onChange())) {}
+			}
+		} else {
+			TraceEvent("CCMonitorConsistencyScanStarting", self->id).log();
+			wait(startConsistencyScan(self));
+		}
+	}
+}
+
 ACTOR Future<Void> startEncryptKeyProxy(ClusterControllerData* self, double waitTime) {
 	// If master fails at the same time, give it a chance to clear master PID.
 	// Also wait to avoid too many consecutive recruits in a small time window.
@@ -2221,6 +2387,8 @@ ACTOR Future<Void> monitorEncryptKeyProxy(ClusterControllerData* self) {
 				when(wait(waitFailureClient(self->db.serverInfo->get().encryptKeyProxy.get().waitFailure,
 				                            SERVER_KNOBS->ENCRYPT_KEY_PROXY_FAILURE_TIME))) {
 					TraceEvent("CCEKP_Died", self->id);
+					const auto& encryptKeyProxy = self->db.serverInfo->get().encryptKeyProxy;
+					EncryptKeyProxySingleton(encryptKeyProxy).halt(self, encryptKeyProxy.get().locality.processId());
 					self->db.clearInterf(ProcessClass::EncryptKeyProxyClass);
 				}
 				when(wait(self->recruitEncryptKeyProxy.onChange())) {}
@@ -2293,7 +2461,8 @@ ACTOR Future<Void> startBlobManager(ClusterControllerData* self, double waitTime
 			self->recruitingBlobManagerID = req.reqId;
 			TraceEvent("CCRecruitBlobManager", self->id)
 			    .detail("Addr", worker.interf.address())
-			    .detail("BMID", req.reqId);
+			    .detail("BMID", req.reqId)
+			    .detail("Epoch", nextEpoch);
 
 			ErrorOr<BlobManagerInterface> interf = wait(worker.interf.blobManager.getReplyUnlessFailedFor(
 			    req, SERVER_KNOBS->WAIT_FOR_BLOB_MANAGER_JOIN_DELAY, 0));
@@ -2339,7 +2508,7 @@ ACTOR Future<Void> watchBlobGranulesConfigKey(ClusterControllerData* self) {
 
 			Optional<Value> blobConfig = wait(tr->get(blobGranuleConfigKey));
 			if (blobConfig.present()) {
-				self->db.blobGranulesEnabled.set(blobConfig.get() == LiteralStringRef("1"));
+				self->db.blobGranulesEnabled.set(blobConfig.get() == "1"_sr);
 			}
 
 			state Future<Void> watch = tr->watch(blobGranuleConfigKey);
@@ -2364,8 +2533,9 @@ ACTOR Future<Void> monitorBlobManager(ClusterControllerData* self) {
 			loop {
 				choose {
 					when(wait(wfClient)) {
-						TraceEvent("CCBlobManagerDied", self->id)
-						    .detail("BMID", self->db.serverInfo->get().blobManager.get().id());
+						const auto& blobManager = self->db.serverInfo->get().blobManager;
+						TraceEvent("CCBlobManagerDied", self->id).detail("BMID", blobManager.get().id());
+						BlobManagerSingleton(blobManager).halt(self, blobManager.get().locality.processId());
 						self->db.clearInterf(ProcessClass::BlobManagerClass);
 						break;
 					}
@@ -2519,17 +2689,20 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
                                          ConfigDBType configDBType,
                                          Future<Void> recoveredDiskFiles) {
 	state ClusterControllerData self(interf, locality, coordinators);
-	state ConfigBroadcaster configBroadcaster(coordinators, configDBType);
 	state Future<Void> coordinationPingDelay = delay(SERVER_KNOBS->WORKER_COORDINATION_PING_DELAY);
 	state uint64_t step = 0;
 	state Future<ErrorOr<Void>> error = errorOr(actorCollection(self.addActor.getFuture()));
+	state ConfigBroadcaster configBroadcaster;
+	if (configDBType != ConfigDBType::DISABLED) {
+		configBroadcaster = ConfigBroadcaster(coordinators, configDBType, getPreviousCoordinators(&self));
+	}
 
 	// EncryptKeyProxy is necessary for TLog recovery, recruit it as the first process
 	if (SERVER_KNOBS->ENABLE_ENCRYPTION) {
 		self.addActor.send(monitorEncryptKeyProxy(&self));
 	}
-	self.addActor.send(clusterWatchDatabase(
-	    &self, &self.db, coordinators, leaderFail, recoveredDiskFiles)); // Start the master database
+	self.addActor.send(
+	    clusterWatchDatabase(&self, &self.db, coordinators, recoveredDiskFiles)); // Start the master database
 	self.addActor.send(self.updateWorkerList.init(self.db.db));
 	self.addActor.send(statusServer(interf.clientInterface.databaseStatus.getFuture(),
 	                                &self,
@@ -2547,6 +2720,7 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 	self.addActor.send(monitorRatekeeper(&self));
 	self.addActor.send(monitorBlobManager(&self));
 	self.addActor.send(watchBlobGranulesConfigKey(&self));
+	self.addActor.send(monitorConsistencyScan(&self));
 	self.addActor.send(dbInfoUpdater(&self));
 	self.addActor.send(traceCounters("ClusterControllerMetrics",
 	                                 self.id,
@@ -2639,6 +2813,12 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 		when(GetServerDBInfoRequest req = waitNext(interf.getServerDBInfo.getFuture())) {
 			self.addActor.send(clusterGetServerInfo(&self.db, req.knownServerInfoID, req.reply));
 		}
+		when(wait(leaderFail)) {
+			// We are no longer the leader if this has changed.
+			endRole(Role::CLUSTER_CONTROLLER, interf.id(), "Leader Replaced", true);
+			CODE_PROBE(true, "Leader replaced");
+			return Void();
+		}
 		when(ReplyPromise<Void> ping = waitNext(interf.clientInterface.ping.getFuture())) { ping.send(Void()); }
 	}
 }
@@ -2729,7 +2909,7 @@ ACTOR Future<Void> clusterController(Reference<IClusterConnectionRecord> connRec
 	state bool hasConnected = false;
 	loop {
 		try {
-			ServerCoordinators coordinators(connRecord);
+			ServerCoordinators coordinators(connRecord, configDBType);
 			wait(clusterController(
 			    coordinators, currentCC, hasConnected, asyncPriorityInfo, locality, configDBType, recoveredDiskFiles));
 			hasConnected = true;
@@ -2814,6 +2994,20 @@ TEST_CASE("/fdbserver/clustercontroller/updateWorkerHealth") {
 		ASSERT_EQ(health.degradedPeers[badPeer3].lastRefreshTime, previousRefreshTime);
 	}
 
+	// Create a `UpdateWorkerHealthRequest` with disconnected peers, which should update the bad peer's lastRefreshTime.
+	{
+		wait(delay(0.001));
+		UpdateWorkerHealthRequest req;
+		req.address = workerAddress;
+		req.disconnectedPeers.push_back(badPeer3);
+		data.updateWorkerHealth(req);
+		ASSERT(data.workerHealth.find(workerAddress) != data.workerHealth.end());
+		auto& health = data.workerHealth[workerAddress];
+		ASSERT_EQ(health.degradedPeers.size(), 3);
+		ASSERT(health.degradedPeers.find(badPeer3) != health.degradedPeers.end());
+		ASSERT_LT(health.degradedPeers[badPeer3].startTime, health.degradedPeers[badPeer3].lastRefreshTime);
+	}
+
 	return Void();
 }
 
diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp
index 5cc4ce3743..946a5913b7 100644
--- a/fdbserver/ClusterRecovery.actor.cpp
+++ b/fdbserver/ClusterRecovery.actor.cpp
@@ -18,10 +18,12 @@
  * limitations under the License.
  */
 
+#include "fdbclient/Metacluster.h"
 #include "fdbrpc/sim_validation.h"
 #include "fdbserver/ApplyMetadataMutation.h"
 #include "fdbserver/BackupProgress.actor.h"
 #include "fdbserver/ClusterRecovery.actor.h"
+#include "fdbserver/EncryptionOpsUtils.h"
 #include "fdbserver/MasterInterface.h"
 #include "fdbserver/WaitFailure.h"
 
@@ -58,7 +60,7 @@ ACTOR Future<Void> recoveryTerminateOnConflict(UID dbgid,
 		when(wait(onConflict)) {
 			if (!fullyRecovered.isSet()) {
 				TraceEvent("RecoveryTerminated", dbgid).detail("Reason", "Conflict");
-				TEST(true); // Coordinated state conflict, recovery terminating
+				CODE_PROBE(true, "Coordinated state conflict, recovery terminating");
 				throw worker_removed();
 			}
 			return Void();
@@ -110,7 +112,7 @@ ACTOR Future<Void> recruitNewMaster(ClusterControllerData* cluster,
 
 			return Void();
 		} else {
-			TEST(true); // clusterWatchDatabase() !newMaster.present()
+			CODE_PROBE(true, "clusterWatchDatabase() !newMaster.present()");
 			wait(delay(SERVER_KNOBS->MASTER_SPIN_DELAY));
 		}
 	}
@@ -118,7 +120,7 @@ ACTOR Future<Void> recruitNewMaster(ClusterControllerData* cluster,
 
 ACTOR Future<Void> clusterRecruitFromConfiguration(ClusterControllerData* self, Reference<RecruitWorkersInfo> req) {
 	// At the moment this doesn't really need to be an actor (it always completes immediately)
-	TEST(true); // ClusterController RecruitTLogsRequest
+	CODE_PROBE(true, "ClusterController RecruitTLogsRequest");
 	loop {
 		try {
 			req->rep = self->findWorkersForConfiguration(req->req);
@@ -150,7 +152,7 @@ ACTOR Future<RecruitRemoteFromConfigurationReply> clusterRecruitRemoteFromConfig
     ClusterControllerData* self,
     Reference<RecruitRemoteWorkersInfo> req) {
 	// At the moment this doesn't really need to be an actor (it always completes immediately)
-	TEST(true); // ClusterController RecruitTLogsRequest Remote
+	CODE_PROBE(true, "ClusterController RecruitTLogsRequest Remote");
 	loop {
 		try {
 			auto rep = self->findRemoteWorkersForConfiguration(req->req);
@@ -355,7 +357,7 @@ ACTOR Future<Void> newSeedServers(Reference<ClusterRecoveryData> self,
 			    !newServer.isError(error_code_request_maybe_delivered))
 				throw newServer.getError();
 
-			TEST(true); // initial storage recuitment loop failed to get new server
+			CODE_PROBE(true, "initial storage recuitment loop failed to get new server");
 			wait(delay(SERVER_KNOBS->STORAGE_RECRUITMENT_DELAY));
 		} else {
 			if (!dcId_tags.count(recruits.storageServers[idx].locality.dcId())) {
@@ -521,6 +523,19 @@ ACTOR Future<Void> changeCoordinators(Reference<ClusterRecoveryData> self) {
 		TraceEvent("ChangeCoordinators", self->dbgid).log();
 		++self->changeCoordinatorsRequests;
 		state ChangeCoordinatorsRequest changeCoordinatorsRequest = req;
+		if (self->masterInterface.id() != changeCoordinatorsRequest.masterId) {
+			// Make sure the request is coming from a proxy from the same
+			// generation. If not, throw coordinators_changed - this is OK
+			// because the client will still receive commit_unknown_result, and
+			// will retry the request. This check is necessary because
+			// otherwise in rare circumstances where a recovery occurs between
+			// the change coordinators request from the client and the cstate
+			// actually being moved, the client may think the change
+			// coordinators command failed when it is still in progress. So we
+			// preempt the issue here and force failure if the generations
+			// don't match.
+			throw coordinators_changed();
+		}
 
 		// Kill cluster controller to facilitate coordinator registration update
 		if (self->controllerData->shouldCommitSuicide) {
@@ -736,7 +751,7 @@ ACTOR Future<Void> updateLogsValue(Reference<ClusterRecoveryData> self, Database
 			}
 
 			if (!found) {
-				TEST(true); // old master attempted to change logsKey
+				CODE_PROBE(true, "old master attempted to change logsKey");
 				return Void();
 			}
 
@@ -815,7 +830,7 @@ ACTOR Future<Void> updateRegistration(Reference<ClusterRecoveryData> self, Refer
 			                            std::vector<UID>()));
 		} else {
 			// The cluster should enter the accepting commits phase soon, and then we will register again
-			TEST(true); // cstate is updated but we aren't accepting commits yet
+			CODE_PROBE(true, "cstate is updated but we aren't accepting commits yet");
 		}
 	}
 }
@@ -1044,14 +1059,15 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 	if (self->txnStateStore)
 		self->txnStateStore->close();
 	self->txnStateLogAdapter = openDiskQueueAdapter(oldLogSystem, myLocality, txsPoppedVersion);
-	self->txnStateStore = keyValueStoreLogSystem(self->txnStateLogAdapter,
-	                                             self->dbInfo,
-	                                             self->dbgid,
-	                                             self->memoryLimit,
-	                                             false,
-	                                             false,
-	                                             true,
-	                                             SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION);
+	self->txnStateStore = keyValueStoreLogSystem(
+	    self->txnStateLogAdapter,
+	    self->dbInfo,
+	    self->dbgid,
+	    self->memoryLimit,
+	    false,
+	    false,
+	    true,
+	    isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION, self->dbInfo->get().client));
 
 	// Version 0 occurs at the version epoch. The version epoch is the number
 	// of microseconds since the Unix epoch. It can be set through fdbcli.
@@ -1096,7 +1112,7 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 		self->recoveryTransactionVersion += deterministicRandom()->randomInt64(0, 10000000);
 	}
 
-	TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_RECOVERED_EVENT_NAME).c_str(),
+	TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_RECOVERING_EVENT_NAME).c_str(),
 	           self->dbgid)
 	    .detail("LastEpochEnd", self->lastEpochEnd)
 	    .detail("RecoveryTransactionVersion", self->recoveryTransactionVersion);
@@ -1144,6 +1160,17 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 		self->allTags.push_back(decodeServerTagValue(kv.value));
 	}
 
+	Optional<Value> metaclusterRegistrationVal =
+	    wait(self->txnStateStore->readValue(MetaclusterMetadata::metaclusterRegistration().key));
+	Optional<MetaclusterRegistrationEntry> metaclusterRegistration =
+	    MetaclusterRegistrationEntry::decode(metaclusterRegistrationVal);
+	if (metaclusterRegistration.present()) {
+		self->controllerData->db.metaclusterName = metaclusterRegistration.get().metaclusterName;
+		self->controllerData->db.clusterType = metaclusterRegistration.get().clusterType;
+	} else {
+		self->controllerData->db.clusterType = ClusterType::STANDALONE;
+	}
+
 	uniquify(self->allTags);
 
 	// auto kvs = self->txnStateStore->readRange( systemKeys );
@@ -1274,7 +1301,7 @@ void updateConfigForForcedRecovery(Reference<ClusterRecoveryData> self,
 	Standalone<CommitTransactionRef> regionCommit;
 	regionCommit.mutations.push_back_deep(
 	    regionCommit.arena(),
-	    MutationRef(MutationRef::SetValue, configKeysPrefix.toString() + "usable_regions", LiteralStringRef("1")));
+	    MutationRef(MutationRef::SetValue, configKeysPrefix.toString() + "usable_regions", "1"_sr));
 	self->configuration.applyMutation(regionCommit.mutations.back());
 	if (regionsChanged) {
 		std::sort(
@@ -1357,7 +1384,7 @@ ACTOR Future<Void> recoverFrom(Reference<ClusterRecoveryData> self,
 			}
 			when(Standalone<CommitTransactionRef> _req = wait(provisional)) {
 				state Standalone<CommitTransactionRef> req = _req; // mutable
-				TEST(true); // Emergency transaction processing during recovery
+				CODE_PROBE(true, "Emergency transaction processing during recovery");
 				TraceEvent("EmergencyTransaction", self->dbgid).log();
 				for (auto m = req.mutations.begin(); m != req.mutations.end(); ++m)
 					TraceEvent("EmergencyTransactionMutation", self->dbgid)
@@ -1411,7 +1438,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 
 	wait(self->cstate.read());
 
-	if (self->cstate.prevDBState.lowestCompatibleProtocolVersion > currentProtocolVersion) {
+	if (self->cstate.prevDBState.lowestCompatibleProtocolVersion > currentProtocolVersion()) {
 		TraceEvent(SevWarnAlways, "IncompatibleProtocolVersion", self->dbgid).log();
 		throw internal_error();
 	}
@@ -1454,8 +1481,8 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 			           (self->cstate.myDBState.oldTLogData.size() - CLIENT_KNOBS->RECOVERY_DELAY_START_GENERATION)));
 		}
 		if (g_network->isSimulated() && self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->MAX_GENERATIONS_SIM) {
-			g_simulator.connectionFailuresDisableDuration = 1e6;
-			g_simulator.speedUpSimulation = true;
+			g_simulator->connectionFailuresDisableDuration = 1e6;
+			g_simulator->speedUpSimulation = true;
 			TraceEvent(SevWarnAlways, "DisableConnectionFailures_TooManyGenerations").log();
 		}
 	}
@@ -1472,10 +1499,10 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 	DBCoreState newState = self->cstate.myDBState;
 	newState.recoveryCount++;
 	if (self->cstate.prevDBState.newestProtocolVersion.isInvalid() ||
-	    self->cstate.prevDBState.newestProtocolVersion < currentProtocolVersion) {
+	    self->cstate.prevDBState.newestProtocolVersion < currentProtocolVersion()) {
 		ASSERT(self->cstate.myDBState.lowestCompatibleProtocolVersion.isInvalid() ||
 		       !self->cstate.myDBState.newestProtocolVersion.isInvalid());
-		newState.newestProtocolVersion = currentProtocolVersion;
+		newState.newestProtocolVersion = currentProtocolVersion();
 		newState.lowestCompatibleProtocolVersion = minCompatibleProtocolVersion;
 	}
 	wait(self->cstate.write(newState) || recoverAndEndEpoch);
@@ -1559,7 +1586,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 		    .detail("SnapRecoveryFlag", snapRecoveryFlag.present() ? snapRecoveryFlag.get().toString() : "N/A")
 		    .detail("LastEpochEnd", self->lastEpochEnd);
 		if (snapRecoveryFlag.present()) {
-			TEST(true); // Recovering from snapshot, writing to snapShotEndVersionKey
+			CODE_PROBE(true, "Recovering from snapshot, writing to snapShotEndVersionKey");
 			BinaryWriter bw(Unversioned());
 			tr.set(recoveryCommitRequest.arena, snapshotEndVersionKey, (bw << self->lastEpochEnd).toValue());
 			// Pause the backups that got restored in this snapshot to avoid data corruption
@@ -1610,6 +1637,11 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 	tr.set(
 	    recoveryCommitRequest.arena, primaryLocalityKey, BinaryWriter::toValue(self->primaryLocality, Unversioned()));
 	tr.set(recoveryCommitRequest.arena, backupVersionKey, backupVersionValue);
+	Optional<Value> txnStateStoreCoords = self->txnStateStore->readValue(coordinatorsKey).get();
+	if (txnStateStoreCoords.present() &&
+	    txnStateStoreCoords.get() != self->coordinators.ccr->getConnectionString().toString()) {
+		tr.set(recoveryCommitRequest.arena, previousCoordinatorsKey, txnStateStoreCoords.get());
+	}
 	tr.set(recoveryCommitRequest.arena, coordinatorsKey, self->coordinators.ccr->getConnectionString().toString());
 	tr.set(recoveryCommitRequest.arena, logsKey, self->logSystem->getLogsValue());
 	tr.set(recoveryCommitRequest.arena,
@@ -1636,7 +1668,8 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 	                       self->dbgid,
 	                       recoveryCommitRequest.arena,
 	                       tr.mutations.slice(mmApplied, tr.mutations.size()),
-	                       self->txnStateStore);
+	                       self->txnStateStore,
+	                       self->dbInfo);
 	mmApplied = tr.mutations.size();
 
 	tr.read_snapshot = self->recoveryTransactionVersion; // lastEpochEnd would make more sense, but isn't in the initial
@@ -1659,7 +1692,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 	// unless we want to change TLogs
 	wait((success(recoveryCommit) && sendInitialCommitToResolvers(self)));
 	if (recoveryCommit.isReady() && recoveryCommit.get().isError()) {
-		TEST(true); // Cluster recovery failed because of the initial commit failed
+		CODE_PROBE(true, "Cluster recovery failed because of the initial commit failed");
 		throw cluster_recovery_failed();
 	}
 
@@ -1716,6 +1749,12 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 	    .detail("RecoveryDuration", recoveryDuration)
 	    .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey);
 
+	TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_AVAILABLE_EVENT_NAME).c_str(),
+	           self->dbgid)
+	    .detail("NumOfOldGensOfLogs", self->cstate.myDBState.oldTLogData.size())
+	    .detail("AvailableAtVersion", self->recoveryTransactionVersion)
+	    .trackLatest(self->clusterRecoveryAvailableEventHolder->trackingKey);
+
 	self->addActor.send(changeCoordinators(self));
 	Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
 	self->addActor.send(configurationMonitor(self, cx));
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index 4ec7dee14e..4c1a037a6f 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -22,6 +22,8 @@
 #include <tuple>
 
 #include "fdbclient/Atomic.h"
+#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/FDBTypes.h"
@@ -29,14 +31,16 @@
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
+#include "fdbclient/Tenant.h"
 #include "fdbclient/TransactionLineage.h"
+#include "fdbrpc/TenantInfo.h"
 #include "fdbrpc/sim_validation.h"
 #include "fdbserver/ApplyMetadataMutation.h"
 #include "fdbserver/ConflictSet.h"
 #include "fdbserver/DataDistributorInterface.h"
-#include "fdbserver/EncryptedMutationMessage.h"
+#include "fdbserver/EncryptionOpsUtils.h"
 #include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/LogSystem.h"
@@ -47,10 +51,11 @@
 #include "fdbserver/RatekeeperInterface.h"
 #include "fdbserver/RecoveryState.h"
 #include "fdbserver/RestoreUtil.h"
+#include "fdbserver/ServerDBInfo.actor.h"
 #include "fdbserver/WaitFailure.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "flow/ActorCollection.h"
-#include "flow/BlobCipher.h"
+#include "flow/EncryptUtils.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/Knobs.h"
@@ -58,6 +63,7 @@
 #include "fdbclient/Tracing.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/network.h"
 
 ACTOR Future<Void> broadcastTxnRequest(TxnStateRequest req, int sendAmount, bool sendReply) {
 	state ReplyPromise<Void> reply = req.reply;
@@ -283,16 +289,20 @@ bool verifyTenantPrefix(ProxyCommitData* const commitData, const CommitTransacti
 				if (!m.param1.startsWith(tenantPrefix)) {
 					TraceEvent(SevWarnAlways, "TenantPrefixMismatch")
 					    .suppressFor(60)
-					    .detail("Prefix", tenantPrefix.toHexString())
-					    .detail("Key", m.param1.toHexString());
+					    .detail("Tenant", req.tenantInfo.name)
+					    .detail("TenantID", req.tenantInfo.tenantId)
+					    .detail("Prefix", tenantPrefix)
+					    .detail("Key", m.param1);
 					return false;
 				}
 
 				if (m.type == MutationRef::ClearRange && !m.param2.startsWith(tenantPrefix)) {
 					TraceEvent(SevWarnAlways, "TenantClearRangePrefixMismatch")
 					    .suppressFor(60)
-					    .detail("Prefix", tenantPrefix.toHexString())
-					    .detail("Key", m.param2.toHexString());
+					    .detail("Tenant", req.tenantInfo.name)
+					    .detail("TenantID", req.tenantInfo.tenantId)
+					    .detail("Prefix", tenantPrefix)
+					    .detail("Key", m.param2);
 					return false;
 				} else if (m.type == MutationRef::SetVersionstampedKey) {
 					ASSERT(m.param1.size() >= 4);
@@ -301,8 +311,10 @@ bool verifyTenantPrefix(ProxyCommitData* const commitData, const CommitTransacti
 					if (*offset < tenantPrefix.size()) {
 						TraceEvent(SevWarnAlways, "TenantVersionstampInvalidOffset")
 						    .suppressFor(60)
-						    .detail("Prefix", tenantPrefix.toHexString())
-						    .detail("Key", m.param1.toHexString())
+						    .detail("Tenant", req.tenantInfo.name)
+						    .detail("TenantID", req.tenantInfo.tenantId)
+						    .detail("Prefix", tenantPrefix)
+						    .detail("Key", m.param1)
 						    .detail("Offset", *offset);
 						return false;
 					}
@@ -315,9 +327,11 @@ bool verifyTenantPrefix(ProxyCommitData* const commitData, const CommitTransacti
 			    (!rc.begin.startsWith(tenantPrefix) || !rc.end.startsWith(tenantPrefix))) {
 				TraceEvent(SevWarnAlways, "TenantReadConflictPrefixMismatch")
 				    .suppressFor(60)
-				    .detail("Prefix", tenantPrefix.toHexString())
-				    .detail("BeginKey", rc.begin.toHexString())
-				    .detail("EndKey", rc.end.toHexString());
+				    .detail("Tenant", req.tenantInfo.name)
+				    .detail("TenantID", req.tenantInfo.tenantId)
+				    .detail("Prefix", tenantPrefix)
+				    .detail("BeginKey", rc.begin)
+				    .detail("EndKey", rc.end);
 				return false;
 			}
 		}
@@ -327,9 +341,11 @@ bool verifyTenantPrefix(ProxyCommitData* const commitData, const CommitTransacti
 			    (!wc.begin.startsWith(tenantPrefix) || !wc.end.startsWith(tenantPrefix))) {
 				TraceEvent(SevWarnAlways, "TenantWriteConflictPrefixMismatch")
 				    .suppressFor(60)
-				    .detail("Prefix", tenantPrefix.toHexString())
-				    .detail("BeginKey", wc.begin.toHexString())
-				    .detail("EndKey", wc.end.toHexString());
+				    .detail("Tenant", req.tenantInfo.name)
+				    .detail("TenantID", req.tenantInfo.tenantId)
+				    .detail("Prefix", tenantPrefix)
+				    .detail("BeginKey", wc.begin)
+				    .detail("EndKey", wc.end);
 				return false;
 			}
 		}
@@ -370,7 +386,7 @@ ACTOR Future<Void> commitBatcher(ProxyCommitData* commitData,
 					// Drop requests if memory is under severe pressure
 					if (commitData->commitBatchesMemBytesCount + bytes > memBytesLimit) {
 						++commitData->stats.txnCommitErrors;
-						req.reply.sendError(proxy_memory_limit_exceeded());
+						req.reply.sendError(commit_proxy_memory_limit_exceeded());
 						TraceEvent(SevWarnAlways, "ProxyCommitBatchMemoryThresholdExceeded")
 						    .suppressFor(60)
 						    .detail("MemBytesCount", commitData->commitBatchesMemBytesCount)
@@ -432,7 +448,7 @@ void createWhitelistBinPathVec(const std::string& binPath, std::vector<Standalon
 	TraceEvent(SevDebug, "BinPathConverter").detail("Input", binPath);
 	StringRef input(binPath);
 	while (input != StringRef()) {
-		StringRef token = input.eat(LiteralStringRef(","));
+		StringRef token = input.eat(","_sr);
 		if (token != StringRef()) {
 			const uint8_t* ptr = token.begin();
 			while (ptr != token.end() && *ptr == ' ') {
@@ -608,7 +624,7 @@ struct CommitBatchContext {
 	bool isMyFirstBatch;
 	bool firstStateMutations;
 
-	Optional<Value> oldCoordinators;
+	Optional<Value> previousCoordinators;
 
 	StoreCommit_t storeCommits;
 
@@ -755,8 +771,8 @@ bool canReject(const std::vector<CommitTransactionRequest>& trs) {
 	for (const auto& tr : trs) {
 		if (tr.transaction.mutations.empty())
 			continue;
-		if (!tr.tenantInfo.name.present() && (tr.transaction.mutations[0].param1.startsWith(LiteralStringRef("\xff")) ||
-		                                      tr.transaction.read_conflict_ranges.empty())) {
+		if (!tr.tenantInfo.name.present() &&
+		    (tr.transaction.mutations[0].param1.startsWith("\xff"_sr) || tr.transaction.read_conflict_ranges.empty())) {
 			return false;
 		}
 	}
@@ -789,7 +805,7 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 	}
 
 	// Pre-resolution the commits
-	TEST(pProxyCommitData->latestLocalCommitBatchResolving.get() < localBatchNumber - 1); // Wait for local batch
+	CODE_PROBE(pProxyCommitData->latestLocalCommitBatchResolving.get() < localBatchNumber - 1, "Wait for local batch");
 	wait(pProxyCommitData->latestLocalCommitBatchResolving.whenAtLeast(localBatchNumber - 1));
 	pProxyCommitData->stats.computeLatency.addMeasurement(now() - timeStart);
 	double queuingDelay = g_network->now() - timeStart;
@@ -798,7 +814,7 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 	     (g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01))) &&
 	    SERVER_KNOBS->PROXY_REJECT_BATCH_QUEUED_TOO_LONG && canReject(trs)) {
 		// Disabled for the recovery transaction. otherwise, recovery can't finish and keeps doing more recoveries.
-		TEST(true); // Reject transactions in the batch
+		CODE_PROBE(true, "Reject transactions in the batch");
 		TraceEvent(SevWarnAlways, "ProxyReject", pProxyCommitData->dbgid)
 		    .suppressFor(0.1)
 		    .detail("QDelay", queuingDelay)
@@ -864,11 +880,86 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 	return Void();
 }
 
+namespace {
+// Routine allows caller to find TenantName for a given TenantId. It returns empty optional when either TenantId is
+// invalid or tenant is unknown
+Optional<TenantName> getTenantName(ProxyCommitData* commitData, int64_t tenantId) {
+	if (tenantId != TenantInfo::INVALID_TENANT) {
+		auto itr = commitData->tenantIdIndex.find(tenantId);
+		if (itr != commitData->tenantIdIndex.end()) {
+			return Optional<TenantName>(itr->second);
+		}
+	}
+
+	return Optional<TenantName>();
+}
+
+std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromMutationRef(ProxyCommitData* commitData,
+                                                                                           MutationRef m) {
+	std::pair<EncryptCipherDomainName, EncryptCipherDomainId> details(EncryptCipherDomainName(),
+	                                                                  INVALID_ENCRYPT_DOMAIN_ID);
+
+	// Possible scenarios:
+	// 1. Encryption domain (Tenant details) weren't explicitly provided, extract Tenant details using
+	// TenantPrefix (first 8 bytes of FDBKey)
+	// 2. Encryption domain isn't available, leverage 'default encryption domain'
+
+	if (isSystemKey(m.param1)) {
+		// Encryption domain == FDB SystemKeyspace encryption domain
+		details.first = EncryptCipherDomainName(FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
+		details.second = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
+	} else if (commitData->tenantMap.empty()) {
+		// Cluster serves no-tenants; use 'default encryption domain'
+	} else if (isSingleKeyMutation((MutationRef::Type)m.type)) {
+		ASSERT_NE((MutationRef::Type)m.type, MutationRef::Type::ClearRange);
+
+		if (m.param1.size() >= TENANT_PREFIX_SIZE) {
+			// Parse mutation key to determine mutation encryption domain
+			StringRef prefix = m.param1.substr(0, TENANT_PREFIX_SIZE);
+			int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False);
+			if (tenantId != TenantInfo::INVALID_TENANT) {
+				Optional<TenantName> tenantName = getTenantName(commitData, tenantId);
+				if (tenantName.present()) {
+					details.first = tenantName.get();
+					details.second = tenantId;
+				}
+			} else {
+				// Leverage 'default encryption domain'
+			}
+		}
+	} else {
+		// ClearRange is the 'only' MultiKey transaction allowed
+		ASSERT_EQ((MutationRef::Type)m.type, MutationRef::Type::ClearRange);
+
+		// FIXME: Handle Clear-range transaction, actions needed:
+		// 1. Transaction range can spawn multiple encryption domains (tenants)
+		// 2. Transaction can be a multi-key transaction spawning multiple tenants
+		// For now fallback to 'default encryption domain'
+
+		CODE_PROBE(true, "ClearRange mutation encryption");
+	}
+
+	// Unknown tenant, fallback to fdb default encryption domain
+	if (details.second == INVALID_ENCRYPT_DOMAIN_ID) {
+		ASSERT_EQ(details.first.size(), 0);
+		details.first = EncryptCipherDomainName(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
+		details.second = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
+
+		CODE_PROBE(true, "Default domain mutation encryption");
+	}
+
+	ASSERT_GT(details.first.size(), 0);
+
+	return details;
+}
+
+} // namespace
+
 ACTOR Future<Void> getResolution(CommitBatchContext* self) {
 	state double resolutionStart = now();
 	// Sending these requests is the fuzzy border between phase 1 and phase 2; it could conceivably overlap with
 	// resolution processing but is still using CPU
-	ProxyCommitData* pProxyCommitData = self->pProxyCommitData;
+	state ProxyCommitData* pProxyCommitData = self->pProxyCommitData;
 	std::vector<CommitTransactionRequest>& trs = self->trs;
 	state Span span("MP:getResolution"_loc, self->span.context);
 
@@ -906,22 +997,36 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
 
 	// Fetch cipher keys if needed.
 	state Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getCipherKeys;
-	if (SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION) {
+	if (pProxyCommitData->isEncryptionEnabled) {
 		static std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> defaultDomains = {
-			{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME },
-			{ ENCRYPT_HEADER_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
+			{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
+			{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME },
+			{ FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
 		};
 		std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> encryptDomains = defaultDomains;
 		for (int t = 0; t < trs.size(); t++) {
-			int64_t tenantId = trs[t].tenantInfo.tenantId;
-			Optional<TenantName> tenantName = trs[t].tenantInfo.name;
-			// TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data
+			TenantInfo const& tenantInfo = trs[t].tenantInfo;
+			int64_t tenantId = tenantInfo.tenantId;
+			Optional<TenantNameRef> const& tenantName = tenantInfo.name;
 			if (tenantId != TenantInfo::INVALID_TENANT) {
 				ASSERT(tenantName.present());
-				encryptDomains[tenantId] = tenantName.get();
+				encryptDomains[tenantId] = Standalone(tenantName.get(), tenantInfo.arena);
+			} else {
+				// Optimization: avoid enumerating mutations if cluster only serves default encryption domains
+				if (pProxyCommitData->tenantMap.size() > 0) {
+					for (auto m : trs[t].transaction.mutations) {
+						std::pair<EncryptCipherDomainName, int64_t> details =
+						    getEncryptDetailsFromMutationRef(pProxyCommitData, m);
+						encryptDomains[details.second] = details.first;
+					}
+				} else {
+					// Ensure default encryption domain-ids are present.
+					ASSERT_EQ(encryptDomains.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID), 1);
+					ASSERT_EQ(encryptDomains.count(FDB_DEFAULT_ENCRYPT_DOMAIN_ID), 1);
+				}
 			}
 		}
-		getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains);
+		getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains, BlobCipherMetrics::TLOG);
 	}
 
 	self->releaseFuture = releaseResolvingAfter(pProxyCommitData, self->releaseDelay, self->localBatchNumber);
@@ -948,8 +1053,7 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
 		g_traceBatch.addEvent(
 		    "CommitDebug", self->debugID.get().first(), "CommitProxyServer.commitBatch.AfterResolution");
 	}
-
-	if (SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION) {
+	if (pProxyCommitData->isEncryptionEnabled) {
 		std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys = wait(getCipherKeys);
 		self->cipherKeys = cipherKeys;
 	}
@@ -1093,7 +1197,7 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 				                       pProxyCommitData->logSystem,
 				                       trs[t].transaction.mutations,
 				                       SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS ? nullptr : &self->toCommit,
-				                       SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION ? &self->cipherKeys : nullptr,
+				                       pProxyCommitData->isEncryptionEnabled ? &self->cipherKeys : nullptr,
 				                       self->forceRecovery,
 				                       self->commitVersion,
 				                       self->commitVersion + 1,
@@ -1136,28 +1240,64 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 	ASSERT(self->commitVersion);
 
 	if (!self->isMyFirstBatch &&
-	    pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get() != self->oldCoordinators.get()) {
+	    pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get() != self->previousCoordinators.get()) {
 		wait(brokenPromiseToNever(pProxyCommitData->db->get().clusterInterface.changeCoordinators.getReply(
-		    ChangeCoordinatorsRequest(pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get()))));
+		    ChangeCoordinatorsRequest(pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get(),
+		                              self->pProxyCommitData->master.id()))));
 		ASSERT(false); // ChangeCoordinatorsRequest should always throw
 	}
 
 	return Void();
 }
 
-void writeMutation(CommitBatchContext* self, int64_t tenantId, const MutationRef& mutation) {
-	static_assert(TenantInfo::INVALID_TENANT == ENCRYPT_INVALID_DOMAIN_ID);
-	if (!SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION || tenantId == TenantInfo::INVALID_TENANT) {
-		// TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data
-		bool isRawAccess = tenantId == TenantInfo::INVALID_TENANT && !isSystemKey(mutation.param1) &&
-		                   !(mutation.type == MutationRef::ClearRange && isSystemKey(mutation.param2)) &&
-		                   self->pProxyCommitData->db->get().client.tenantMode == TenantMode::REQUIRED;
-		TEST(isRawAccess); // Raw access to tenant key space
-		self->toCommit.writeTypedMessage(mutation);
+ACTOR Future<MutationRef> writeMutation(CommitBatchContext* self,
+                                        int64_t tenantId,
+                                        const MutationRef* mutation,
+                                        Optional<MutationRef>* encryptedMutationOpt,
+                                        Arena* arena) {
+	static_assert(TenantInfo::INVALID_TENANT == INVALID_ENCRYPT_DOMAIN_ID);
+
+	if (self->pProxyCommitData->isEncryptionEnabled) {
+		state EncryptCipherDomainId domainId = tenantId;
+		state MutationRef encryptedMutation;
+
+		if (encryptedMutationOpt->present()) {
+			CODE_PROBE(true, "using already encrypted mutation");
+			encryptedMutation = encryptedMutationOpt->get();
+			ASSERT(encryptedMutation.isEncrypted());
+			// During simulation check whether the encrypted mutation matches the decrpyted mutation
+			if (g_network && g_network->isSimulated()) {
+				Reference<AsyncVar<ServerDBInfo> const> dbInfo = self->pProxyCommitData->db;
+				state const BlobCipherEncryptHeader* header = encryptedMutation.encryptionHeader();
+				TextAndHeaderCipherKeys cipherKeys =
+				    wait(getEncryptCipherKeys(dbInfo, *header, BlobCipherMetrics::TLOG));
+				MutationRef decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG);
+				ASSERT(decryptedMutation.param1 == mutation->param1 && decryptedMutation.param2 == mutation->param2 &&
+				       decryptedMutation.type == mutation->type);
+			}
+		} else {
+			if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
+				std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
+				    getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
+				domainId = p.second;
+
+				if (self->cipherKeys.find(domainId) == self->cipherKeys.end()) {
+					Reference<BlobCipherKey> cipherKey = wait(getLatestEncryptCipherKey(
+					    self->pProxyCommitData->db, domainId, p.first, BlobCipherMetrics::TLOG));
+					self->cipherKeys[domainId] = cipherKey;
+				}
+
+				CODE_PROBE(true, "Raw access mutation encryption");
+			}
+
+			ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID);
+			encryptedMutation = mutation->encrypt(self->cipherKeys, domainId, *arena, BlobCipherMetrics::TLOG);
+		}
+		self->toCommit.writeTypedMessage(encryptedMutation);
+		return encryptedMutation;
 	} else {
-		Arena arena;
-		self->toCommit.writeTypedMessage(
-		    EncryptedMutationMessage::encrypt(arena, self->cipherKeys, tenantId /*domainId*/, mutation));
+		self->toCommit.writeTypedMessage(*mutation);
+		return *mutation;
 	}
 }
 
@@ -1177,6 +1317,9 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
 		state Optional<ClientTrCommitCostEstimation>* trCost = &trs[self->transactionNum].commitCostEstimation;
 		state int mutationNum = 0;
 		state VectorRef<MutationRef>* pMutations = &trs[self->transactionNum].transaction.mutations;
+		state VectorRef<Optional<MutationRef>>* encryptedMutations =
+		    &trs[self->transactionNum].transaction.encryptedMutations;
+		ASSERT(encryptedMutations->size() == 0 || encryptedMutations->size() == pMutations->size());
 		state int64_t tenantId = trs[self->transactionNum].tenantInfo.tenantId;
 
 		self->toCommit.addTransactionInfo(trs[self->transactionNum].spanContext);
@@ -1191,13 +1334,17 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
 				}
 			}
 
-			auto& m = (*pMutations)[mutationNum];
+			state MutationRef m = (*pMutations)[mutationNum];
+			state Optional<MutationRef> encryptedMutation =
+			    encryptedMutations->size() > 0 ? (*encryptedMutations)[mutationNum] : Optional<MutationRef>();
+			state Arena arena;
+			state MutationRef writtenMutation;
 			self->mutationCount++;
 			self->mutationBytes += m.expectedSize();
 			self->yieldBytes += m.expectedSize();
+			ASSERT(!m.isEncrypted());
 			// Determine the set of tags (responsible storage servers) for the mutation, splitting it
 			// if necessary.  Serialize (splits of) the mutation into the message buffer and add the tags.
-
 			if (isSingleKeyMutation((MutationRef::Type)m.type)) {
 				auto& tags = pProxyCommitData->tagsForKey(m.param1);
 
@@ -1235,7 +1382,11 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
 				if (pProxyCommitData->cacheInfo[m.param1]) {
 					self->toCommit.addTag(cacheTag);
 				}
-				writeMutation(self, tenantId, m);
+				if (encryptedMutation.present()) {
+					ASSERT(encryptedMutation.get().isEncrypted());
+				}
+				MutationRef tempMutation = wait(writeMutation(self, tenantId, &m, &encryptedMutation, &arena));
+				writtenMutation = tempMutation;
 			} else if (m.type == MutationRef::ClearRange) {
 				KeyRangeRef clearRange(KeyRangeRef(m.param1, m.param2));
 				auto ranges = pProxyCommitData->keyInfo.intersectingRanges(clearRange);
@@ -1259,7 +1410,7 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
 						trCost->get().clearIdxCosts.pop_front();
 					}
 				} else {
-					TEST(true); // A clear range extends past a shard boundary
+					CODE_PROBE(true, "A clear range extends past a shard boundary");
 					std::set<Tag> allSources;
 					for (auto r : ranges) {
 						r.value().populateTags();
@@ -1288,21 +1439,41 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
 				if (pProxyCommitData->needsCacheTag(clearRange)) {
 					self->toCommit.addTag(cacheTag);
 				}
-				writeMutation(self, tenantId, m);
+				MutationRef tempMutation = wait(writeMutation(self, tenantId, &m, &encryptedMutation, &arena));
+				writtenMutation = tempMutation;
 			} else {
 				UNREACHABLE();
 			}
 
-			// Check on backing up key, if backup ranges are defined and a normal key
-			if (!(pProxyCommitData->vecBackupKeys.size() > 1 &&
-			      (normalKeys.contains(m.param1) || m.param1 == metadataVersionKey))) {
+			if (pProxyCommitData->vecBackupKeys.size() <= 1) {
+				continue;
+			}
+
+			// Check whether the mutation intersects any legal backup ranges
+			// If so, it will be clamped to the intersecting range(s) later
+			bool hasCandidateBackupKeys = false;
+			if (normalKeys.contains(m.param1) || m.param1 == metadataVersionKey) {
+				hasCandidateBackupKeys = true;
+			} else if (m.type != MutationRef::Type::ClearRange) {
+				hasCandidateBackupKeys = systemBackupMutationMask().rangeContaining(m.param1).value();
+			} else {
+				for (auto& r : systemBackupMutationMask().intersectingRanges(KeyRangeRef(m.param1, m.param2))) {
+					if (r->value()) {
+						hasCandidateBackupKeys = true;
+						break;
+					}
+				}
+			}
+			if (!hasCandidateBackupKeys) {
 				continue;
 			}
 
 			if (m.type != MutationRef::Type::ClearRange) {
 				// Add the mutation to the relevant backup tag
 				for (auto backupName : pProxyCommitData->vecBackupKeys[m.param1]) {
-					self->logRangeMutations[backupName].push_back_deep(self->logRangeMutationsArena, m);
+					// If encryption is enabled make sure the mutation we are writing is also encrypted
+					ASSERT(!self->pProxyCommitData->isEncryptionEnabled || writtenMutation.isEncrypted());
+					self->logRangeMutations[backupName].push_back_deep(self->logRangeMutationsArena, writtenMutation);
 				}
 			} else {
 				KeyRangeRef mutationRange(m.param1, m.param2);
@@ -1320,6 +1491,21 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
 					MutationRef backupMutation(
 					    MutationRef::Type::ClearRange, intersectionRange.begin, intersectionRange.end);
 
+					// TODO (Nim): Currently clear ranges are encrypted using the default encryption key, this must be
+					// changed to account for clear ranges which span tenant boundaries
+					if (self->pProxyCommitData->isEncryptionEnabled) {
+						if (backupMutation.param1 == m.param1 && backupMutation.param2 == m.param2 &&
+						    encryptedMutation.present()) {
+							backupMutation = encryptedMutation.get();
+						} else {
+							std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
+							    getEncryptDetailsFromMutationRef(self->pProxyCommitData, backupMutation);
+							EncryptCipherDomainId domainId = p.second;
+							backupMutation =
+							    backupMutation.encrypt(self->cipherKeys, domainId, arena, BlobCipherMetrics::BACKUP);
+						}
+					}
+
 					// Add the mutation to the relevant backup tag
 					for (auto backupName : backupRange.value()) {
 						self->logRangeMutations[backupName].push_back_deep(self->logRangeMutationsArena,
@@ -1347,7 +1533,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 	state Span span("MP:postResolution"_loc, self->span.context);
 
 	bool queuedCommits = pProxyCommitData->latestLocalCommitBatchLogging.get() < localBatchNumber - 1;
-	TEST(queuedCommits); // Queuing post-resolution commit processing
+	CODE_PROBE(queuedCommits, "Queuing post-resolution commit processing");
 	wait(pProxyCommitData->latestLocalCommitBatchLogging.whenAtLeast(localBatchNumber - 1));
 	state double postResolutionQueuing = now();
 	pProxyCommitData->stats.postResolutionDist->sampleSeconds(postResolutionQueuing - postResolutionStart);
@@ -1363,7 +1549,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 	}
 
 	self->isMyFirstBatch = !pProxyCommitData->version.get();
-	self->oldCoordinators = pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get();
+	self->previousCoordinators = pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get();
 
 	assertResolutionStateMutationsSizeConsistent(self->resolution);
 
@@ -1413,9 +1599,9 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 	pProxyCommitData->stats.mutations += self->mutationCount;
 	pProxyCommitData->stats.mutationBytes += self->mutationBytes;
 
-	// Storage servers mustn't make durable versions which are not fully committed (because then they are impossible to
-	// roll back) We prevent this by limiting the number of versions which are semi-committed but not fully committed to
-	// be less than the MVCC window
+	// Storage servers mustn't make durable versions which are not fully committed (because then they are impossible
+	// to roll back) We prevent this by limiting the number of versions which are semi-committed but not fully
+	// committed to be less than the MVCC window
 	if (pProxyCommitData->committedVersion.get() <
 	    self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) {
 		self->computeDuration += g_network->timer() - self->computeStart;
@@ -1424,7 +1610,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 		       self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) {
 			// This should be *extremely* rare in the real world, but knob buggification should make it happen in
 			// simulation
-			TEST(true); // Semi-committed pipeline limited by MVCC window
+			CODE_PROBE(true, "Semi-committed pipeline limited by MVCC window");
 			//TraceEvent("ProxyWaitingForCommitted", pProxyCommitData->dbgid).detail("CommittedVersion", pProxyCommitData->committedVersion.get()).detail("NeedToCommit", commitVersion);
 			waitVersionSpan = Span("MP:overMaxReadTransactionLifeVersions"_loc, span.context);
 			choose {
@@ -1592,7 +1778,7 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
 	state ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
 	state Span span("MP:reply"_loc, self->span.context);
 
-	const Optional<UID>& debugID = self->debugID;
+	state const Optional<UID>& debugID = self->debugID;
 
 	if (self->prevVersion && self->commitVersion - self->prevVersion < SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT / 2) {
 		//TraceEvent("CPAdvanceMinVersion", self->pProxyCommitData->dbgid).detail("PrvVersion", self->prevVersion).detail("CommitVersion", self->commitVersion).detail("Master", self->pProxyCommitData->master.id().toString()).detail("TxSize", self->trs.size());
@@ -1614,10 +1800,11 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
 	// After logging finishes, we report the commit version to master so that every other proxy can get the most
 	// up-to-date live committed version. We also maintain the invariant that master's committed version >=
 	// self->committedVersion by reporting commit version first before updating self->committedVersion. Otherwise, a
-	// client may get a commit version that the master is not aware of, and next GRV request may get a version less than
-	// self->committedVersion.
+	// client may get a commit version that the master is not aware of, and next GRV request may get a version less
+	// than self->committedVersion.
 
-	TEST(pProxyCommitData->committedVersion.get() > self->commitVersion); // later version was reported committed first
+	CODE_PROBE(pProxyCommitData->committedVersion.get() > self->commitVersion,
+	           "later version was reported committed first");
 
 	if (self->commitVersion >= pProxyCommitData->committedVersion.get()) {
 		state Optional<std::set<Tag>> writtenTags;
@@ -1633,6 +1820,12 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
 		                                     writtenTags),
 		    TaskPriority::ProxyMasterVersionReply));
 	}
+
+	if (debugID.present()) {
+		g_traceBatch.addEvent(
+		    "CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.AfterReportRawCommittedVersion");
+	}
+
 	if (self->commitVersion > pProxyCommitData->committedVersion.get()) {
 		pProxyCommitData->locked = self->lockedAfter;
 		pProxyCommitData->metadataVersion = self->metadataVersionAfter;
@@ -1667,11 +1860,11 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
 				for (int resolverInd : self->transactionResolverMap[t]) {
 					auto const& cKRs =
 					    self->resolution[resolverInd]
-					        .conflictingKeyRangeMap[self->nextTr[resolverInd]]; // nextTr[resolverInd] -> index of this
-					                                                            // trs[t] on the resolver
+					        .conflictingKeyRangeMap[self->nextTr[resolverInd]]; // nextTr[resolverInd] -> index of
+					                                                            // this trs[t] on the resolver
 					for (auto const& rCRIndex : cKRs)
-						// read_conflict_range can change when sent to resolvers, mapping the index from resolver-side
-						// to original index in commitTransactionRef
+						// read_conflict_range can change when sent to resolvers, mapping the index from
+						// resolver-side to original index in commitTransactionRef
 						conflictingKRIndices.push_back(conflictingKRIndices.arena(),
 						                               self->txReadConflictRangeIndexMap[t][resolverInd][rCRIndex]);
 				}
@@ -1838,7 +2031,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
 	while (tenantEntry.isError()) {
 		bool finalQuery = commitData->version.get() >= minTenantVersion;
 		ErrorOr<Optional<TenantMapEntry>> _tenantEntry =
-		    getTenantEntry(commitData, req.tenant, Optional<int64_t>(), finalQuery);
+		    getTenantEntry(commitData, req.tenant.name, Optional<int64_t>(), finalQuery);
 		tenantEntry = _tenantEntry;
 
 		if (tenantEntry.isError()) {
@@ -1929,7 +2122,7 @@ ACTOR static Future<Void> readRequestServer(CommitProxyInterface proxy,
 		    commitData->stats.keyServerLocationIn.getValue() - commitData->stats.keyServerLocationOut.getValue() >
 		        SERVER_KNOBS->KEY_LOCATION_MAX_QUEUE_SIZE) {
 			++commitData->stats.keyServerLocationErrors;
-			req.reply.sendError(proxy_memory_limit_exceeded());
+			req.reply.sendError(commit_proxy_memory_limit_exceeded());
 			TraceEvent(SevWarnAlways, "ProxyLocationRequestThresholdExceeded").suppressFor(60);
 		} else {
 			++commitData->stats.keyServerLocationIn;
@@ -2121,9 +2314,7 @@ ACTOR Future<Void> proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* co
 			throw snap_not_fully_recovered_unsupported();
 		}
 
-		auto result =
-		    commitData->txnStateStore->readValue(LiteralStringRef("log_anti_quorum").withPrefix(configKeysPrefix))
-		        .get();
+		auto result = commitData->txnStateStore->readValue("log_anti_quorum"_sr.withPrefix(configKeysPrefix)).get();
 		int logAntiQuorum = 0;
 		if (result.present()) {
 			logAntiQuorum = atoi(result.get().toString().c_str());
@@ -2155,7 +2346,8 @@ ACTOR Future<Void> proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* co
 				TraceEvent("SnapCommitProxy_DDSnapResponseError")
 				    .errorUnsuppressed(e)
 				    .detail("SnapPayload", snapReq.snapPayload)
-				    .detail("SnapUID", snapReq.snapUID);
+				    .detail("SnapUID", snapReq.snapUID)
+				    .detail("Retry", snapReqRetry);
 				// Retry if we have network issues
 				if (e.code() != error_code_request_maybe_delivered ||
 				    ++snapReqRetry > SERVER_KNOBS->SNAP_NETWORK_FAILURE_RETRY_LIMIT)
@@ -2451,7 +2643,8 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
 	// Wait until we can load the "real" logsystem, since we don't support switching them currently
 	while (!(masterLifetime.isEqual(commitData.db->get().masterLifetime) &&
 	         commitData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION &&
-	         (!SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION || commitData.db->get().encryptKeyProxy.present()))) {
+	         (!isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION, db->get().client) ||
+	          commitData.db->get().encryptKeyProxy.present()))) {
 		//TraceEvent("ProxyInit2", proxy.id()).detail("LSEpoch", db->get().logSystemConfig.epoch).detail("Need", epoch);
 		wait(commitData.db->onChange());
 	}
@@ -2462,10 +2655,8 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
 	commitData.localTLogCount = commitData.db->get().logSystemConfig.numLogs();
 	ASSERT(commitData.resolvers.size() != 0);
 	for (int i = 0; i < commitData.resolvers.size(); ++i) {
-		commitData.stats.resolverDist.push_back(
-		    Histogram::getHistogram(LiteralStringRef("CommitProxy"),
-		                            "ToResolver_" + commitData.resolvers[i].id().toString(),
-		                            Histogram::Unit::microseconds));
+		commitData.stats.resolverDist.push_back(Histogram::getHistogram(
+		    "CommitProxy"_sr, "ToResolver_" + commitData.resolvers[i].id().toString(), Histogram::Unit::microseconds));
 	}
 
 	// Initialize keyResolvers map
@@ -2477,8 +2668,15 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
 	commitData.logSystem = ILogSystem::fromServerDBInfo(proxy.id(), commitData.db->get(), false, addActor);
 	commitData.logAdapter =
 	    new LogSystemDiskQueueAdapter(commitData.logSystem, Reference<AsyncVar<PeekTxsInfo>>(), 1, false);
-	commitData.txnStateStore = keyValueStoreLogSystem(
-	    commitData.logAdapter, commitData.db, proxy.id(), 2e9, true, true, true, SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION);
+	commitData.txnStateStore =
+	    keyValueStoreLogSystem(commitData.logAdapter,
+	                           commitData.db,
+	                           proxy.id(),
+	                           2e9,
+	                           true,
+	                           true,
+	                           true,
+	                           isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION, db->get().client));
 	createWhitelistBinPathVec(whitelistBinPaths, commitData.whitelistedBinPathVec);
 
 	commitData.updateLatencyBandConfig(commitData.db->get().latencyBandConfig);
@@ -2603,7 +2801,7 @@ ACTOR Future<Void> commitProxyServer(CommitProxyInterface proxy,
 		    e.code() != error_code_failed_to_progress) {
 			throw;
 		}
-		TEST(e.code() == error_code_failed_to_progress); // Commit proxy failed to progress
+		CODE_PROBE(e.code() == error_code_failed_to_progress, "Commit proxy failed to progress");
 	}
 	return Void();
 }
diff --git a/fdbserver/ConfigBroadcaster.actor.cpp b/fdbserver/ConfigBroadcaster.actor.cpp
index c402ab7b27..eafce9c9cb 100644
--- a/fdbserver/ConfigBroadcaster.actor.cpp
+++ b/fdbserver/ConfigBroadcaster.actor.cpp
@@ -20,6 +20,7 @@
 
 #include <algorithm>
 
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbserver/ConfigBroadcaster.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/IConfigConsumer.h"
@@ -77,13 +78,25 @@ class ConfigBroadcasterImpl {
 	std::deque<VersionedConfigMutation> mutationHistory;
 	std::deque<VersionedConfigCommitAnnotation> annotationHistory;
 	Version lastCompactedVersion;
+	Version largestLiveVersion;
 	Version mostRecentVersion;
+	CoordinatorsHash coordinatorsHash;
 	std::unique_ptr<IConfigConsumer> consumer;
 	Future<Void> consumerFuture;
 	ActorCollection actors{ false };
+	std::unordered_map<NetworkAddress, Future<Void>> registrationActors;
 	std::map<UID, BroadcastClientDetails> clients;
 	std::map<UID, Future<Void>> clientFailures;
 
+	// State related to changing coordinators
+
+	// Used to read a snapshot from the previous coordinators after a change
+	// coordinators command.
+	Version maxLastSeenVersion = ::invalidVersion;
+	Future<Optional<Value>> previousCoordinatorsFuture;
+	std::unique_ptr<IConfigConsumer> previousCoordinatorsConsumer;
+	Future<Void> previousCoordinatorsSnapshotFuture;
+
 	UID id;
 	CounterCollection cc;
 	Counter compactRequest;
@@ -95,6 +108,7 @@ class ConfigBroadcasterImpl {
 	int coordinators = 0;
 	std::unordered_set<NetworkAddress> activeConfigNodes;
 	std::unordered_set<NetworkAddress> registrationResponses;
+	std::unordered_set<NetworkAddress> registrationResponsesUnregistered;
 	bool disallowUnregistered = false;
 	Promise<Void> newConfigNodesAllowed;
 
@@ -111,6 +125,11 @@ class ConfigBroadcasterImpl {
 			}
 		}
 		request.version = snapshotVersion;
+		// TODO: Don't need a delay if there are no atomic changes.
+		// Delay restarting the cluster controller to allow messages to be sent to workers.
+		request.restartDelay = client.broadcastInterface.address() == g_network->getLocalAddress()
+		                           ? SERVER_KNOBS->BROADCASTER_SELF_UPDATE_DELAY
+		                           : 0.0;
 		TraceEvent(SevDebug, "ConfigBroadcasterSnapshotRequest", id)
 		    .detail("Size", request.snapshot.size())
 		    .detail("Version", request.version);
@@ -150,13 +169,18 @@ class ConfigBroadcasterImpl {
 
 		client.lastSeenVersion = mostRecentVersion;
 		req.mostRecentVersion = mostRecentVersion;
+		// TODO: Don't need a delay if there are no atomic changes.
+		// Delay restarting the cluster controller to allow messages to be sent to workers.
+		req.restartDelay = client.broadcastInterface.address() == g_network->getLocalAddress()
+		                       ? SERVER_KNOBS->BROADCASTER_SELF_UPDATE_DELAY
+		                       : 0.0;
 		++successfulChangeRequest;
 		return success(client.broadcastInterface.changes.getReply(req));
 	}
 
 	ConfigBroadcasterImpl()
-	  : lastCompactedVersion(0), mostRecentVersion(0), id(deterministicRandom()->randomUniqueID()),
-	    cc("ConfigBroadcaster"), compactRequest("CompactRequest", cc),
+	  : lastCompactedVersion(0), largestLiveVersion(0), mostRecentVersion(0),
+	    id(deterministicRandom()->randomUniqueID()), cc("ConfigBroadcaster"), compactRequest("CompactRequest", cc),
 	    successfulChangeRequest("SuccessfulChangeRequest", cc), failedChangeRequest("FailedChangeRequest", cc),
 	    snapshotRequest("SnapshotRequest", cc) {
 		logger = traceCounters(
@@ -183,45 +207,44 @@ class ConfigBroadcasterImpl {
 		}
 	}
 
-	template <class Snapshot>
-	Future<Void> setSnapshot(Snapshot&& snapshot, Version snapshotVersion) {
-		this->snapshot = std::forward<Snapshot>(snapshot);
-		this->lastCompactedVersion = snapshotVersion;
+	ACTOR static Future<Void> pushSnapshotAndChanges(ConfigBroadcasterImpl* self, Version snapshotVersion) {
 		std::vector<Future<Void>> futures;
-		for (const auto& [id, client] : clients) {
-			futures.push_back(brokenPromiseToNever(pushSnapshot(snapshotVersion, client)));
+		for (const auto& [id, client] : self->clients) {
+			futures.push_back(brokenPromiseToNever(self->pushSnapshot(snapshotVersion, client)));
 		}
-		return waitForAll(futures);
-	}
-
-	ACTOR template <class Snapshot>
-	static Future<Void> pushSnapshotAndChanges(ConfigBroadcasterImpl* self,
-	                                           Snapshot snapshot,
-	                                           Version snapshotVersion,
-	                                           Standalone<VectorRef<VersionedConfigMutationRef>> changes,
-	                                           Version changesVersion,
-	                                           Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations) {
-		// Make sure all snapshot messages were received before sending changes.
-		wait(self->setSnapshot(snapshot, snapshotVersion));
-		self->addChanges(changes, changesVersion, annotations);
+		wait(waitForAll(futures));
 		return Void();
 	}
 
 	ACTOR static Future<Void> waitForFailure(ConfigBroadcasterImpl* self,
 	                                         Future<Void> watcher,
 	                                         UID clientUID,
-	                                         NetworkAddress clientAddress) {
+	                                         NetworkAddress clientAddress,
+	                                         bool isCoordinator) {
 		wait(watcher);
 		TraceEvent(SevDebug, "ConfigBroadcastClientDied", self->id)
 		    .detail("ClientID", clientUID)
-		    .detail("Address", clientAddress);
+		    .detail("Address", clientAddress)
+		    .detail("IsUnregistered",
+		            self->registrationResponsesUnregistered.find(clientAddress) !=
+		                self->registrationResponsesUnregistered.end())
+		    .detail("IsActive", self->activeConfigNodes.find(clientAddress) != self->activeConfigNodes.end());
 		self->clients.erase(clientUID);
 		self->clientFailures.erase(clientUID);
-		self->activeConfigNodes.erase(clientAddress);
-		self->registrationResponses.erase(clientAddress);
-		// See comment where this promise is reset below.
-		if (self->newConfigNodesAllowed.isSet()) {
-			self->newConfigNodesAllowed.reset();
+		if (isCoordinator) {
+			self->registrationResponses.erase(clientAddress);
+			if (self->activeConfigNodes.find(clientAddress) != self->activeConfigNodes.end()) {
+				self->activeConfigNodes.erase(clientAddress);
+				if (self->registrationResponsesUnregistered.find(clientAddress) !=
+				    self->registrationResponsesUnregistered.end()) {
+					self->registrationResponsesUnregistered.erase(clientAddress);
+					self->disallowUnregistered = false;
+					// See comment where this promise is reset below.
+					if (self->newConfigNodesAllowed.isSet()) {
+						self->newConfigNodesAllowed.reset();
+					}
+				}
+			}
 		}
 		return Void();
 	}
@@ -231,84 +254,167 @@ class ConfigBroadcasterImpl {
 	// ensure strict serializability, some nodes may be temporarily restricted
 	// from participation until the other nodes in the system are brought up to
 	// date.
-	ACTOR static Future<Void> registerNodeInternal(ConfigBroadcasterImpl* self,
-	                                               WorkerInterface w,
-	                                               Version lastSeenVersion) {
+	ACTOR static Future<Void> registerNodeInternal(ConfigBroadcaster* broadcaster,
+	                                               ConfigBroadcasterImpl* self,
+	                                               ConfigBroadcastInterface configBroadcastInterface) {
 		if (self->configDBType == ConfigDBType::SIMPLE) {
-			wait(success(retryBrokenPromise(w.configBroadcastInterface.ready, ConfigBroadcastReadyRequest{})));
+			self->consumerFuture = self->consumer->consume(*broadcaster);
+			wait(success(brokenPromiseToNever(
+			    configBroadcastInterface.ready.getReply(ConfigBroadcastReadyRequest{ 0, {}, -1, -1 }))));
 			return Void();
 		}
 
-		state NetworkAddress address = w.address();
-
+		state NetworkAddress address = configBroadcastInterface.address();
 		// Ask the registering ConfigNode whether it has registered in the past.
-		ConfigBroadcastRegisteredReply reply =
-		    wait(w.configBroadcastInterface.registered.getReply(ConfigBroadcastRegisteredRequest{}));
+		state ConfigBroadcastRegisteredReply reply = wait(
+		    brokenPromiseToNever(configBroadcastInterface.registered.getReply(ConfigBroadcastRegisteredRequest{})));
+		self->maxLastSeenVersion = std::max(self->maxLastSeenVersion, reply.lastSeenVersion);
 		state bool registered = reply.registered;
+		TraceEvent("ConfigBroadcasterRegisterNodeReceivedRegistrationReply", self->id)
+		    .detail("Address", address)
+		    .detail("Registered", registered)
+		    .detail("DisallowUnregistered", self->disallowUnregistered)
+		    .detail("LastSeenVersion", reply.lastSeenVersion);
 
 		if (self->activeConfigNodes.find(address) != self->activeConfigNodes.end()) {
 			self->activeConfigNodes.erase(address);
-			// Since a node can die and re-register before the broadcaster
-			// receives notice that the node has died, we need to check for
-			// re-registration of a node here. There are two places that can
-			// reset the promise to allow new nodes, make sure the promise is
-			// actually set before resetting it. This prevents a node from
-			// dying, registering, waiting on the promise, then the broadcaster
-			// receives the notification the node has died and resets the
-			// promise again.
-			if (self->newConfigNodesAllowed.isSet()) {
-				self->newConfigNodesAllowed.reset();
+			if (self->registrationResponsesUnregistered.find(address) !=
+			    self->registrationResponsesUnregistered.end()) {
+				self->registrationResponsesUnregistered.erase(address);
+				// If an unregistered node died which was active, reset the
+				// disallow unregistered flag so if it re-registers it can be
+				// set as active again.
+				self->disallowUnregistered = false;
+				// Since a node can die and re-register before the broadcaster
+				// receives notice that the node has died, we need to check for
+				// re-registration of a node here. There are two places that can
+				// reset the promise to allow new nodes, so make sure the promise
+				// is actually set before resetting it. This prevents a node from
+				// dying, registering, waiting on the promise, then the broadcaster
+				// receives the notification the node has died and resets the
+				// promise again.
+				if (self->newConfigNodesAllowed.isSet()) {
+					self->newConfigNodesAllowed.reset();
+				}
 			}
 		}
-		self->registrationResponses.insert(address);
+		int responsesRemaining = self->coordinators - (int)self->registrationResponses.size();
+		int nodesTillQuorum = self->coordinators / 2 + 1 - (int)self->activeConfigNodes.size();
 
 		if (registered) {
-			if (!self->disallowUnregistered) {
-				self->activeConfigNodes.clear();
-			}
 			self->activeConfigNodes.insert(address);
 			self->disallowUnregistered = true;
 		} else if ((self->activeConfigNodes.size() < self->coordinators / 2 + 1 && !self->disallowUnregistered) ||
-		           self->coordinators - self->registrationResponses.size() <=
-		               self->coordinators / 2 + 1 - self->activeConfigNodes.size()) {
+		           (self->registrationResponsesUnregistered.size() < self->coordinators / 2 &&
+		            responsesRemaining <= nodesTillQuorum)) {
 			// Received a registration request from an unregistered node. There
 			// are two cases where we want to allow unregistered nodes to
 			// register:
 			// 	 * the cluster is just starting and no nodes are registered
-			// 	 * a minority of nodes are registered and a majority are
-			// 	   unregistered. This situation should only occur in rare
-			// 	   circumstances where the cluster controller dies with only a
-			// 	   minority of config nodes having received a
-			// 	   ConfigBroadcastReadyRequest
+			// 	 * there are registered and unregistered nodes, but the
+			// 	   registered nodes may not represent a majority due to previous
+			// 	   data loss. In this case, unregistered nodes must be allowed
+			// 	   to register so they can be rolled forward and form a quorum.
+			// 	   But only a minority of unregistered nodes should be allowed
+			// 	   to register so they cannot override the registered nodes as
+			// 	   a source of truth
 			self->activeConfigNodes.insert(address);
-			if (self->activeConfigNodes.size() >= self->coordinators / 2 + 1 &&
+			self->registrationResponsesUnregistered.insert(address);
+			if ((self->activeConfigNodes.size() >= self->coordinators / 2 + 1 ||
+			     self->registrationResponsesUnregistered.size() >= self->coordinators / 2 + 1) &&
 			    self->newConfigNodesAllowed.canBeSet()) {
 				self->newConfigNodesAllowed.send(Void());
 			}
 		} else {
 			self->disallowUnregistered = true;
 		}
+		self->registrationResponses.insert(address);
 
-		if (!registered) {
+		// Read previous coordinators and fetch snapshot from them if they
+		// exist. This path should only be hit once after the coordinators are
+		// changed.
+		wait(yield());
+		Optional<Value> previousCoordinators = wait(self->previousCoordinatorsFuture);
+		TraceEvent("ConfigBroadcasterRegisterNodeReadPreviousCoordinators", self->id)
+		    .detail("PreviousCoordinators", previousCoordinators)
+		    .detail("HasStartedConsumer", self->previousCoordinatorsSnapshotFuture.isValid());
+
+		if (previousCoordinators.present()) {
+			if (!self->previousCoordinatorsSnapshotFuture.isValid()) {
+				// Create a consumer to read a snapshot from the previous
+				// coordinators. The snapshot will be forwarded to the new
+				// coordinators to bring them up to date.
+				size_t previousCoordinatorsHash = std::hash<std::string>()(previousCoordinators.get().toString());
+				if (previousCoordinatorsHash != self->coordinatorsHash) {
+					ServerCoordinators previousCoordinatorsData(Reference<IClusterConnectionRecord>(
+					    new ClusterConnectionMemoryRecord(previousCoordinators.get().toString())));
+					TraceEvent("ConfigBroadcasterRegisterNodeStartingConsumer", self->id).log();
+					self->previousCoordinatorsConsumer = IConfigConsumer::createPaxos(
+					    previousCoordinatorsData, 0.5, SERVER_KNOBS->COMPACTION_INTERVAL, true);
+					self->previousCoordinatorsSnapshotFuture =
+					    self->previousCoordinatorsConsumer->readSnapshot(*broadcaster);
+				} else {
+					// If the cluster controller restarts without a coordinator
+					// change having taken place, there is no need to read a
+					// previous snapshot.
+					self->previousCoordinatorsSnapshotFuture = Void();
+				}
+			}
+			wait(self->previousCoordinatorsSnapshotFuture);
+		}
+
+		state bool sendSnapshot =
+		    self->previousCoordinatorsConsumer && reply.lastSeenVersion <= self->mostRecentVersion;
+		// Unregistered nodes need to wait for either:
+		//   1. A quorum of registered nodes to register and send their
+		//      snapshots, so the unregistered nodes can be rolled forward, or
+		//   2. A quorum of unregistered nodes to contact the broadcaster (this
+		//      means there is no previous data in the configuration database)
+		// The above conditions do not apply when changing coordinators, as a
+		// snapshot of the current state of the configuration database needs to
+		// be sent to all new coordinators.
+		TraceEvent("ConfigBroadcasterRegisterNodeDetermineEligibility", self->id)
+		    .detail("Registered", registered)
+		    .detail("SendSnapshot", sendSnapshot);
+		if (!registered && !sendSnapshot) {
 			wait(self->newConfigNodesAllowed.getFuture());
 		}
 
-		wait(success(w.configBroadcastInterface.ready.getReply(ConfigBroadcastReadyRequest{})));
+		sendSnapshot = sendSnapshot || (!registered && self->snapshot.size() > 0);
+		TraceEvent("ConfigBroadcasterRegisterNodeSendingReadyRequest", self->id)
+		    .detail("ConfigNodeAddress", address)
+		    .detail("SendSnapshot", sendSnapshot)
+		    .detail("SnapshotVersion", self->mostRecentVersion)
+		    .detail("SnapshotSize", self->snapshot.size())
+		    .detail("LargestLiveVersion", self->largestLiveVersion);
+		if (sendSnapshot) {
+			Version liveVersion = std::max(self->largestLiveVersion, self->mostRecentVersion);
+			wait(success(brokenPromiseToNever(configBroadcastInterface.ready.getReply(ConfigBroadcastReadyRequest{
+			    self->coordinatorsHash, self->snapshot, self->mostRecentVersion, liveVersion }))));
+		} else {
+			wait(success(brokenPromiseToNever(configBroadcastInterface.ready.getReply(
+			    ConfigBroadcastReadyRequest{ self->coordinatorsHash, {}, -1, -1 }))));
+		}
+
+		// Start the consumer last, so at least some nodes will be registered.
+		if (!self->consumerFuture.isValid()) {
+			if (sendSnapshot) {
+				self->consumer->allowSpecialCaseRollforward();
+			}
+			self->consumerFuture = self->consumer->consume(*broadcaster);
+		}
 		return Void();
 	}
 
 	ACTOR static Future<Void> registerNode(ConfigBroadcaster* self,
 	                                       ConfigBroadcasterImpl* impl,
-	                                       WorkerInterface w,
+	                                       ConfigBroadcastInterface broadcastInterface,
 	                                       Version lastSeenVersion,
 	                                       ConfigClassSet configClassSet,
 	                                       Future<Void> watcher,
-	                                       ConfigBroadcastInterface broadcastInterface) {
+	                                       bool isCoordinator) {
 		state BroadcastClientDetails client(
 		    watcher, std::move(configClassSet), lastSeenVersion, std::move(broadcastInterface));
-		if (!impl->consumerFuture.isValid()) {
-			impl->consumerFuture = impl->consumer->consume(*self);
-		}
 
 		if (impl->clients.count(broadcastInterface.id())) {
 			// Client already registered
@@ -317,26 +423,32 @@ class ConfigBroadcasterImpl {
 
 		TraceEvent(SevDebug, "ConfigBroadcasterRegisteringWorker", impl->id)
 		    .detail("ClientID", broadcastInterface.id())
-		    .detail("MostRecentVersion", impl->mostRecentVersion);
+		    .detail("MostRecentVersion", impl->mostRecentVersion)
+		    .detail("IsCoordinator", isCoordinator);
 
-		impl->actors.add(registerNodeInternal(impl, w, lastSeenVersion));
+		if (isCoordinator) {
+			// A client re-registering will cause the old actor to be
+			// cancelled.
+			impl->registrationActors[broadcastInterface.address()] =
+			    registerNodeInternal(self, impl, broadcastInterface);
+		}
 
 		// Push full snapshot to worker if it isn't up to date.
 		wait(impl->pushSnapshot(impl->mostRecentVersion, client));
 		impl->clients[broadcastInterface.id()] = client;
 		impl->clientFailures[broadcastInterface.id()] =
-		    waitForFailure(impl, watcher, broadcastInterface.id(), w.address());
+		    waitForFailure(impl, watcher, broadcastInterface.id(), broadcastInterface.address(), isCoordinator);
 		return Void();
 	}
 
 public:
 	Future<Void> registerNode(ConfigBroadcaster& self,
-	                          WorkerInterface const& w,
+	                          ConfigBroadcastInterface const& broadcastInterface,
 	                          Version lastSeenVersion,
 	                          ConfigClassSet configClassSet,
 	                          Future<Void> watcher,
-	                          ConfigBroadcastInterface const& broadcastInterface) {
-		return registerNode(&self, this, w, lastSeenVersion, configClassSet, watcher, broadcastInterface);
+	                          bool isCoordinator) {
+		return registerNode(&self, this, broadcastInterface, lastSeenVersion, configClassSet, watcher, isCoordinator);
 	}
 
 	// Updates the broadcasters knowledge of which replicas are fully up to
@@ -360,8 +472,9 @@ public:
 	                  Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations,
 	                  std::vector<ConfigFollowerInterface> const& readReplicas) {
 		if (mostRecentVersion >= 0) {
-			TraceEvent(SevDebug, "ConfigBroadcasterApplyingChanges", id)
+			TraceEvent(SevInfo, "ConfigBroadcasterApplyingChanges", id)
 			    .detail("ChangesSize", changes.size())
+			    .detail("AnnotationsSize", annotations.size())
 			    .detail("CurrentMostRecentVersion", this->mostRecentVersion)
 			    .detail("NewMostRecentVersion", mostRecentVersion)
 			    .detail("ActiveReplicas", readReplicas.size());
@@ -377,17 +490,37 @@ public:
 	                             Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
 	                             Version changesVersion,
 	                             Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations,
-	                             std::vector<ConfigFollowerInterface> const& readReplicas) {
-		TraceEvent(SevDebug, "ConfigBroadcasterApplyingSnapshotAndChanges", id)
+	                             std::vector<ConfigFollowerInterface> const& readReplicas,
+	                             Version largestLiveVersion,
+	                             bool fromPreviousCoordinators) {
+		TraceEvent(SevInfo, "ConfigBroadcasterApplyingSnapshotAndChanges", id)
 		    .detail("CurrentMostRecentVersion", this->mostRecentVersion)
 		    .detail("SnapshotSize", snapshot.size())
 		    .detail("SnapshotVersion", snapshotVersion)
 		    .detail("ChangesSize", changes.size())
 		    .detail("ChangesVersion", changesVersion)
-		    .detail("ActiveReplicas", readReplicas.size());
-		actors.add(pushSnapshotAndChanges(this, snapshot, snapshotVersion, changes, changesVersion, annotations));
+		    .detail("AnnotationsSize", annotations.size())
+		    .detail("ActiveReplicas", readReplicas.size())
+		    .detail("LargestLiveVersion", largestLiveVersion)
+		    .detail("FromPreviousCoordinators", fromPreviousCoordinators);
+		// Avoid updating state if the snapshot contains no mutations, or if it
+		// contains old mutations. This can happen when the set of coordinators
+		// is changed, and a new coordinator comes online that has not yet had
+		// the current configuration database pushed to it, or when a new
+		// coordinator contains state from an old configuration database
+		// generation.
+		if ((snapshot.size() != 0 || changes.size() != 0) &&
+		    (snapshotVersion > this->mostRecentVersion || changesVersion > this->mostRecentVersion)) {
+			this->snapshot = std::forward<Snapshot>(snapshot);
+			this->lastCompactedVersion = snapshotVersion;
+			this->largestLiveVersion = std::max(this->largestLiveVersion, largestLiveVersion);
+			addChanges(changes, changesVersion, annotations);
+			actors.add(pushSnapshotAndChanges(this, snapshotVersion));
+		}
 
-		updateKnownReplicas(readReplicas);
+		if (!fromPreviousCoordinators) {
+			updateKnownReplicas(readReplicas);
+		}
 	}
 
 	ConfigBroadcasterImpl(ConfigFollowerInterface const& cfi) : ConfigBroadcasterImpl() {
@@ -397,18 +530,28 @@ public:
 		TraceEvent(SevDebug, "ConfigBroadcasterStartingConsumer", id).detail("Consumer", consumer->getID());
 	}
 
-	ConfigBroadcasterImpl(ServerCoordinators const& coordinators, ConfigDBType configDBType) : ConfigBroadcasterImpl() {
+	ConfigBroadcasterImpl(ServerCoordinators const& coordinators,
+	                      ConfigDBType configDBType,
+	                      Future<Optional<Value>> previousCoordinatorsFuture)
+	  : ConfigBroadcasterImpl() {
 		this->configDBType = configDBType;
 		this->coordinators = coordinators.configServers.size();
 		if (configDBType != ConfigDBType::DISABLED) {
 			if (configDBType == ConfigDBType::SIMPLE) {
 				consumer = IConfigConsumer::createSimple(coordinators, 0.5, SERVER_KNOBS->COMPACTION_INTERVAL);
 			} else {
+				this->previousCoordinatorsFuture = previousCoordinatorsFuture;
 				consumer = IConfigConsumer::createPaxos(coordinators, 0.5, SERVER_KNOBS->COMPACTION_INTERVAL);
 			}
-			TraceEvent(SevDebug, "ConfigBroadcasterStartingConsumer", id)
+
+			coordinatorsHash = std::hash<std::string>()(coordinators.ccr->getConnectionString().toString());
+
+			TraceEvent(SevInfo, "ConfigBroadcasterStartingConsumer", id)
 			    .detail("Consumer", consumer->getID())
-			    .detail("UsingSimpleConsumer", configDBType == ConfigDBType::SIMPLE);
+			    .detail("UsingSimpleConsumer", configDBType == ConfigDBType::SIMPLE)
+			    .detail("CoordinatorsCount", this->coordinators)
+			    .detail("CoordinatorsHash", coordinatorsHash)
+			    .detail("CompactionInterval", SERVER_KNOBS->COMPACTION_INTERVAL);
 		}
 	}
 
@@ -419,9 +562,12 @@ public:
 			JsonBuilderObject mutationObject;
 			mutationObject["version"] = versionedMutation.version;
 			const auto& mutation = versionedMutation.mutation;
+			mutationObject["type"] = mutation.isSet() ? "set" : "clear";
 			mutationObject["config_class"] = mutation.getConfigClass().orDefault("<global>"_sr);
 			mutationObject["knob_name"] = mutation.getKnobName();
-			mutationObject["knob_value"] = mutation.getValue().toString();
+			if (mutation.isSet()) {
+				mutationObject["knob_value"] = mutation.getValue().toString();
+			}
 			mutationsArray.push_back(std::move(mutationObject));
 		}
 		result["mutations"] = std::move(mutationsArray);
@@ -477,11 +623,15 @@ public:
 	static void runPendingRequestStoreTest(bool includeGlobalMutation, int expectedMatches);
 };
 
+ConfigBroadcaster::ConfigBroadcaster() {}
+
 ConfigBroadcaster::ConfigBroadcaster(ConfigFollowerInterface const& cfi)
   : impl(PImpl<ConfigBroadcasterImpl>::create(cfi)) {}
 
-ConfigBroadcaster::ConfigBroadcaster(ServerCoordinators const& coordinators, ConfigDBType configDBType)
-  : impl(PImpl<ConfigBroadcasterImpl>::create(coordinators, configDBType)) {}
+ConfigBroadcaster::ConfigBroadcaster(ServerCoordinators const& coordinators,
+                                     ConfigDBType configDBType,
+                                     Future<Optional<Value>> previousCoordinatorsFuture)
+  : impl(PImpl<ConfigBroadcasterImpl>::create(coordinators, configDBType, previousCoordinatorsFuture)) {}
 
 ConfigBroadcaster::ConfigBroadcaster(ConfigBroadcaster&&) = default;
 
@@ -489,12 +639,12 @@ ConfigBroadcaster& ConfigBroadcaster::operator=(ConfigBroadcaster&&) = default;
 
 ConfigBroadcaster::~ConfigBroadcaster() = default;
 
-Future<Void> ConfigBroadcaster::registerNode(WorkerInterface const& w,
+Future<Void> ConfigBroadcaster::registerNode(ConfigBroadcastInterface const& broadcastInterface,
                                              Version lastSeenVersion,
                                              ConfigClassSet const& configClassSet,
                                              Future<Void> watcher,
-                                             ConfigBroadcastInterface const& broadcastInterface) {
-	return impl->registerNode(*this, w, lastSeenVersion, configClassSet, watcher, broadcastInterface);
+                                             bool isCoordinator) {
+	return impl->registerNode(*this, broadcastInterface, lastSeenVersion, configClassSet, watcher, isCoordinator);
 }
 
 void ConfigBroadcaster::applyChanges(Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
@@ -510,8 +660,17 @@ void ConfigBroadcaster::applySnapshotAndChanges(
     Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
     Version changesVersion,
     Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations,
-    std::vector<ConfigFollowerInterface> const& readReplicas) {
-	impl->applySnapshotAndChanges(snapshot, snapshotVersion, changes, changesVersion, annotations, readReplicas);
+    std::vector<ConfigFollowerInterface> const& readReplicas,
+    Version largestLiveVersion,
+    bool fromPreviousCoordinators) {
+	impl->applySnapshotAndChanges(snapshot,
+	                              snapshotVersion,
+	                              changes,
+	                              changesVersion,
+	                              annotations,
+	                              readReplicas,
+	                              largestLiveVersion,
+	                              fromPreviousCoordinators);
 }
 
 void ConfigBroadcaster::applySnapshotAndChanges(
@@ -520,9 +679,17 @@ void ConfigBroadcaster::applySnapshotAndChanges(
     Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
     Version changesVersion,
     Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations,
-    std::vector<ConfigFollowerInterface> const& readReplicas) {
-	impl->applySnapshotAndChanges(
-	    std::move(snapshot), snapshotVersion, changes, changesVersion, annotations, readReplicas);
+    std::vector<ConfigFollowerInterface> const& readReplicas,
+    Version largestLiveVersion,
+    bool fromPreviousCoordinators) {
+	impl->applySnapshotAndChanges(std::move(snapshot),
+	                              snapshotVersion,
+	                              changes,
+	                              changesVersion,
+	                              annotations,
+	                              readReplicas,
+	                              largestLiveVersion,
+	                              fromPreviousCoordinators);
 }
 
 Future<Void> ConfigBroadcaster::getError() const {
@@ -544,3 +711,31 @@ JsonBuilderObject ConfigBroadcaster::getStatus() const {
 void ConfigBroadcaster::compact(Version compactionVersion) {
 	impl->compact(compactionVersion);
 }
+
+ACTOR static Future<Void> lockConfigNodesImpl(ServerCoordinators coordinators) {
+	if (coordinators.configServers.empty()) {
+		return Void();
+	}
+
+	CoordinatorsHash coordinatorsHash = std::hash<std::string>()(coordinators.ccr->getConnectionString().toString());
+
+	std::vector<Future<Void>> lockRequests;
+	lockRequests.reserve(coordinators.configServers.size());
+	for (int i = 0; i < coordinators.configServers.size(); i++) {
+		if (coordinators.configServers[i].hostname.present()) {
+			lockRequests.push_back(retryGetReplyFromHostname(ConfigFollowerLockRequest{ coordinatorsHash },
+			                                                 coordinators.configServers[i].hostname.get(),
+			                                                 WLTOKEN_CONFIGFOLLOWER_LOCK));
+		} else {
+			lockRequests.push_back(
+			    retryBrokenPromise(coordinators.configServers[i].lock, ConfigFollowerLockRequest{ coordinatorsHash }));
+		}
+	}
+	int quorum_size = lockRequests.size() / 2 + 1;
+	wait(quorum(lockRequests, quorum_size));
+	return Void();
+}
+
+Future<Void> ConfigBroadcaster::lockConfigNodes(ServerCoordinators coordinators) {
+	return lockConfigNodesImpl(coordinators);
+}
diff --git a/fdbserver/ConfigDatabaseUnitTests.actor.cpp b/fdbserver/ConfigDatabaseUnitTests.actor.cpp
index 40ff575b61..3472fe28af 100644
--- a/fdbserver/ConfigDatabaseUnitTests.actor.cpp
+++ b/fdbserver/ConfigDatabaseUnitTests.actor.cpp
@@ -114,7 +114,7 @@ public:
 	                         Standalone<VectorRef<VersionedConfigMutationRef>> mutations,
 	                         Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations) {
 		return cfi.rollforward.getReply(
-		    ConfigFollowerRollforwardRequest{ rollback, lastKnownCommitted, target, mutations, annotations });
+		    ConfigFollowerRollforwardRequest{ rollback, lastKnownCommitted, target, mutations, annotations, false });
 	}
 
 	void restartNode() {
@@ -232,7 +232,7 @@ class LocalConfigEnvironment {
 	Future<Void> addMutation(Optional<KeyRef> configClass, KeyRef knobName, Optional<KnobValueRef> value) {
 		Standalone<VectorRef<VersionedConfigMutationRef>> versionedMutations;
 		appendVersionedMutation(versionedMutations, ++lastWrittenVersion, configClass, knobName, value);
-		return readFrom.getMutableLocalConfiguration().addChanges(versionedMutations, lastWrittenVersion);
+		return readFrom.getMutableLocalConfiguration().addChanges(versionedMutations, lastWrittenVersion, 0);
 	}
 
 public:
@@ -269,8 +269,8 @@ class BroadcasterToLocalConfigEnvironment {
 		wait(self->readFrom.setup());
 		self->cbi = makeReference<AsyncVar<ConfigBroadcastInterface>>();
 		self->readFrom.connectToBroadcaster(self->cbi);
-		self->broadcastServer = self->broadcaster.registerNode(
-		    WorkerInterface(), 0, configClassSet, self->workerFailure.getFuture(), self->cbi->get());
+		self->broadcastServer =
+		    self->broadcaster.registerNode(self->cbi->get(), 0, configClassSet, self->workerFailure.getFuture(), true);
 		return Void();
 	}
 
@@ -305,11 +305,8 @@ public:
 		broadcastServer.cancel();
 		cbi->set(ConfigBroadcastInterface{});
 		readFrom.connectToBroadcaster(cbi);
-		broadcastServer = broadcaster.registerNode(WorkerInterface(),
-		                                           readFrom.lastSeenVersion(),
-		                                           readFrom.configClassSet(),
-		                                           workerFailure.getFuture(),
-		                                           cbi->get());
+		broadcastServer = broadcaster.registerNode(
+		    cbi->get(), readFrom.lastSeenVersion(), readFrom.configClassSet(), workerFailure.getFuture(), true);
 	}
 
 	Future<Void> restartLocalConfig(std::string const& newConfigPath) {
@@ -441,8 +438,8 @@ class TransactionToLocalConfigEnvironment {
 		wait(self->readFrom.setup());
 		self->cbi = makeReference<AsyncVar<ConfigBroadcastInterface>>();
 		self->readFrom.connectToBroadcaster(self->cbi);
-		self->broadcastServer = self->broadcaster.registerNode(
-		    WorkerInterface(), 0, configClassSet, self->workerFailure.getFuture(), self->cbi->get());
+		self->broadcastServer =
+		    self->broadcaster.registerNode(self->cbi->get(), 0, configClassSet, self->workerFailure.getFuture(), true);
 		return Void();
 	}
 
@@ -461,11 +458,8 @@ public:
 		broadcastServer.cancel();
 		cbi->set(ConfigBroadcastInterface{});
 		readFrom.connectToBroadcaster(cbi);
-		broadcastServer = broadcaster.registerNode(WorkerInterface(),
-		                                           readFrom.lastSeenVersion(),
-		                                           readFrom.configClassSet(),
-		                                           workerFailure.getFuture(),
-		                                           cbi->get());
+		broadcastServer = broadcaster.registerNode(
+		    cbi->get(), readFrom.lastSeenVersion(), readFrom.configClassSet(), workerFailure.getFuture(), true);
 	}
 
 	Future<Void> restartLocalConfig(std::string const& newConfigPath) {
diff --git a/fdbserver/ConfigFollowerInterface.cpp b/fdbserver/ConfigFollowerInterface.cpp
index d95f772d85..54b964d2b4 100644
--- a/fdbserver/ConfigFollowerInterface.cpp
+++ b/fdbserver/ConfigFollowerInterface.cpp
@@ -29,6 +29,7 @@ void ConfigFollowerInterface::setupWellKnownEndpoints() {
 	compact.makeWellKnownEndpoint(WLTOKEN_CONFIGFOLLOWER_COMPACT, TaskPriority::Coordination);
 	rollforward.makeWellKnownEndpoint(WLTOKEN_CONFIGFOLLOWER_ROLLFORWARD, TaskPriority::Coordination);
 	getCommittedVersion.makeWellKnownEndpoint(WLTOKEN_CONFIGFOLLOWER_GETCOMMITTEDVERSION, TaskPriority::Coordination);
+	lock.makeWellKnownEndpoint(WLTOKEN_CONFIGFOLLOWER_LOCK, TaskPriority::Coordination);
 }
 
 ConfigFollowerInterface::ConfigFollowerInterface() : _id(deterministicRandom()->randomUniqueID()) {}
@@ -39,7 +40,8 @@ ConfigFollowerInterface::ConfigFollowerInterface(NetworkAddress const& remote)
     getChanges(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_GETCHANGES)),
     compact(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_COMPACT)),
     rollforward(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_ROLLFORWARD)),
-    getCommittedVersion(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_GETCOMMITTEDVERSION)) {}
+    getCommittedVersion(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_GETCOMMITTEDVERSION)),
+    lock(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_LOCK)) {}
 
 ConfigFollowerInterface::ConfigFollowerInterface(Hostname const& remote)
   : _id(deterministicRandom()->randomUniqueID()), hostname(remote) {}
diff --git a/fdbserver/ConfigNode.actor.cpp b/fdbserver/ConfigNode.actor.cpp
index c76ee882cf..fce3ee5010 100644
--- a/fdbserver/ConfigNode.actor.cpp
+++ b/fdbserver/ConfigNode.actor.cpp
@@ -32,9 +32,11 @@
 
 namespace {
 
+const KeyRef coordinatorsHashKey = "id"_sr;
 const KeyRef lastCompactedVersionKey = "lastCompactedVersion"_sr;
 const KeyRef currentGenerationKey = "currentGeneration"_sr;
 const KeyRef registeredKey = "registered"_sr;
+const KeyRef lockedKey = "locked"_sr;
 const KeyRangeRef kvKeys = KeyRangeRef("kv/"_sr, "kv0"_sr);
 const KeyRangeRef mutationKeys = KeyRangeRef("mutation/"_sr, "mutation0"_sr);
 const KeyRangeRef annotationKeys = KeyRangeRef("annotation/"_sr, "annotation0"_sr);
@@ -122,6 +124,7 @@ class ConfigNodeImpl {
 	Counter failedChangeRequests;
 	Counter snapshotRequests;
 	Counter getCommittedVersionRequests;
+	Counter lockRequests;
 
 	// Transaction counters
 	Counter successfulCommits;
@@ -132,6 +135,27 @@ class ConfigNodeImpl {
 	Counter getGenerationRequests;
 	Future<Void> logger;
 
+	ACTOR static Future<CoordinatorsHash> getCoordinatorsHash(ConfigNodeImpl* self) {
+		state CoordinatorsHash coordinatorsHash = 0;
+		Optional<Value> value = wait(self->kvStore->readValue(coordinatorsHashKey));
+		if (value.present()) {
+			coordinatorsHash = BinaryReader::fromStringRef<CoordinatorsHash>(value.get(), IncludeVersion());
+		} else {
+			self->kvStore->set(
+			    KeyValueRef(coordinatorsHashKey, BinaryWriter::toValue(coordinatorsHash, IncludeVersion())));
+			wait(self->kvStore->commit());
+		}
+		return coordinatorsHash;
+	}
+
+	ACTOR static Future<Optional<CoordinatorsHash>> getLocked(ConfigNodeImpl* self) {
+		Optional<Value> value = wait(self->kvStore->readValue(lockedKey));
+		if (!value.present()) {
+			return Optional<CoordinatorsHash>();
+		}
+		return BinaryReader::fromStringRef<Optional<CoordinatorsHash>>(value.get(), IncludeVersion());
+	}
+
 	ACTOR static Future<ConfigGeneration> getGeneration(ConfigNodeImpl* self) {
 		state ConfigGeneration generation;
 		Optional<Value> value = wait(self->kvStore->readValue(currentGenerationKey));
@@ -206,7 +230,7 @@ class ConfigNodeImpl {
 			// Handle a very rare case where a ConfigNode loses data between
 			// responding with a committed version and responding to the
 			// subsequent get changes request.
-			TEST(true); // ConfigNode data loss occurred on a minority of coordinators
+			CODE_PROBE(true, "ConfigNode data loss occurred on a minority of coordinators");
 			req.reply.sendError(process_behind()); // Reuse the process_behind error
 			return Void();
 		}
@@ -214,8 +238,9 @@ class ConfigNodeImpl {
 		    wait(getMutations(self, req.lastSeenVersion + 1, committedVersion));
 		state Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> versionedAnnotations =
 		    wait(getAnnotations(self, req.lastSeenVersion + 1, committedVersion));
-		TraceEvent(SevDebug, "ConfigNodeSendingChanges", self->id)
+		TraceEvent(SevInfo, "ConfigNodeSendingChanges", self->id)
 		    .detail("ReqLastSeenVersion", req.lastSeenVersion)
+		    .detail("ReqMostRecentVersion", req.mostRecentVersion)
 		    .detail("CommittedVersion", committedVersion)
 		    .detail("NumMutations", versionedMutations.size())
 		    .detail("NumCommits", versionedAnnotations.size());
@@ -227,10 +252,16 @@ class ConfigNodeImpl {
 	// New transactions increment the database's current live version. This effectively serves as a lock, providing
 	// serializability
 	ACTOR static Future<Void> getNewGeneration(ConfigNodeImpl* self, ConfigTransactionGetGenerationRequest req) {
+		state CoordinatorsHash coordinatorsHash = wait(getCoordinatorsHash(self));
+		if (req.coordinatorsHash != coordinatorsHash) {
+			req.reply.sendError(coordinators_changed());
+			return Void();
+		}
 		state ConfigGeneration generation = wait(getGeneration(self));
 		++generation.liveVersion;
 		if (req.lastSeenLiveVersion.present()) {
-			TEST(req.lastSeenLiveVersion.get() >= generation.liveVersion); // Node is lagging behind some other node
+			CODE_PROBE(req.lastSeenLiveVersion.get() >= generation.liveVersion,
+			           "Node is lagging behind some other node");
 			generation.liveVersion = std::max(generation.liveVersion, req.lastSeenLiveVersion.get() + 1);
 		}
 		self->kvStore->set(KeyValueRef(currentGenerationKey, BinaryWriter::toValue(generation, IncludeVersion())));
@@ -240,6 +271,17 @@ class ConfigNodeImpl {
 	}
 
 	ACTOR static Future<Void> get(ConfigNodeImpl* self, ConfigTransactionGetRequest req) {
+		state Optional<CoordinatorsHash> locked = wait(getLocked(self));
+		if (locked.present()) {
+			CODE_PROBE(true, "attempting to read from a locked ConfigNode");
+			req.reply.sendError(coordinators_changed());
+			return Void();
+		}
+		state CoordinatorsHash coordinatorsHash = wait(getCoordinatorsHash(self));
+		if (req.coordinatorsHash != coordinatorsHash) {
+			req.reply.sendError(coordinators_changed());
+			return Void();
+		}
 		ConfigGeneration currentGeneration = wait(getGeneration(self));
 		if (req.generation != currentGeneration) {
 			// TODO: Also send information about highest seen version
@@ -272,6 +314,17 @@ class ConfigNodeImpl {
 	// TODO: Currently it is possible that extra configuration classes may be returned, we
 	// may want to fix this to clean up the contract
 	ACTOR static Future<Void> getConfigClasses(ConfigNodeImpl* self, ConfigTransactionGetConfigClassesRequest req) {
+		state Optional<CoordinatorsHash> locked = wait(getLocked(self));
+		if (locked.present()) {
+			CODE_PROBE(true, "attempting to read config classes from locked ConfigNode");
+			req.reply.sendError(coordinators_changed());
+			return Void();
+		}
+		state CoordinatorsHash coordinatorsHash = wait(getCoordinatorsHash(self));
+		if (req.coordinatorsHash != coordinatorsHash) {
+			req.reply.sendError(coordinators_changed());
+			return Void();
+		}
 		ConfigGeneration currentGeneration = wait(getGeneration(self));
 		if (req.generation != currentGeneration) {
 			req.reply.sendError(transaction_too_old());
@@ -305,6 +358,18 @@ class ConfigNodeImpl {
 
 	// Retrieve all knobs explicitly defined for the specified configuration class
 	ACTOR static Future<Void> getKnobs(ConfigNodeImpl* self, ConfigTransactionGetKnobsRequest req) {
+		state Optional<CoordinatorsHash> locked = wait(getLocked(self));
+		if (locked.present()) {
+			CODE_PROBE(true, "attempting to read knobs from locked ConfigNode");
+			req.reply.sendError(coordinators_changed());
+			return Void();
+		}
+		state CoordinatorsHash coordinatorsHash = wait(getCoordinatorsHash(self));
+		if (req.coordinatorsHash != coordinatorsHash) {
+			req.reply.sendError(coordinators_changed());
+			return Void();
+		}
+
 		ConfigGeneration currentGeneration = wait(getGeneration(self));
 		if (req.generation != currentGeneration) {
 			req.reply.sendError(transaction_too_old());
@@ -343,7 +408,8 @@ class ConfigNodeImpl {
 	ACTOR static Future<Void> commitMutations(ConfigNodeImpl* self,
 	                                          Standalone<VectorRef<VersionedConfigMutationRef>> mutations,
 	                                          Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations,
-	                                          Version commitVersion) {
+	                                          Version commitVersion,
+	                                          Version liveVersion = ::invalidVersion) {
 		Version latestVersion = 0;
 		int index = 0;
 		for (const auto& mutation : mutations) {
@@ -375,6 +441,9 @@ class ConfigNodeImpl {
 			                               BinaryWriter::toValue(annotation.annotation, IncludeVersion())));
 		}
 		ConfigGeneration newGeneration = { commitVersion, commitVersion };
+		if (liveVersion != ::invalidVersion) {
+			newGeneration.liveVersion = liveVersion;
+		}
 		self->kvStore->set(KeyValueRef(currentGenerationKey, BinaryWriter::toValue(newGeneration, IncludeVersion())));
 		wait(self->kvStore->commit());
 		++self->successfulCommits;
@@ -382,6 +451,18 @@ class ConfigNodeImpl {
 	}
 
 	ACTOR static Future<Void> commit(ConfigNodeImpl* self, ConfigTransactionCommitRequest req) {
+		state Optional<CoordinatorsHash> locked = wait(getLocked(self));
+		if (locked.present()) {
+			CODE_PROBE(true, "attempting to write to locked ConfigNode");
+			req.reply.sendError(coordinators_changed());
+			return Void();
+		}
+		state CoordinatorsHash coordinatorsHash = wait(getCoordinatorsHash(self));
+		if (req.coordinatorsHash != coordinatorsHash) {
+			req.reply.sendError(coordinators_changed());
+			return Void();
+		}
+
 		ConfigGeneration currentGeneration = wait(getGeneration(self));
 		if (req.generation.committedVersion != currentGeneration.committedVersion) {
 			++self->failedCommits;
@@ -398,6 +479,7 @@ class ConfigNodeImpl {
 		}
 		Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations;
 		annotations.emplace_back_deep(annotations.arena(), req.generation.liveVersion, req.annotation);
+
 		wait(commitMutations(self, mutations, annotations, req.generation.liveVersion));
 		req.reply.send(Void());
 		return Void();
@@ -440,10 +522,11 @@ class ConfigNodeImpl {
 		wait(store(reply.snapshotVersion, getLastCompactedVersion(self)));
 		wait(store(reply.changes, getMutations(self, reply.snapshotVersion + 1, req.mostRecentVersion)));
 		wait(store(reply.annotations, getAnnotations(self, reply.snapshotVersion + 1, req.mostRecentVersion)));
-		TraceEvent(SevDebug, "ConfigNodeGettingSnapshot", self->id)
+		TraceEvent(SevInfo, "ConfigNodeGettingSnapshot", self->id)
 		    .detail("SnapshotVersion", reply.snapshotVersion)
 		    .detail("SnapshotSize", reply.snapshot.size())
-		    .detail("ChangesSize", reply.changes.size());
+		    .detail("ChangesSize", reply.changes.size())
+		    .detail("AnnotationsSize", reply.annotations.size());
 		req.reply.send(reply);
 		return Void();
 	}
@@ -453,7 +536,7 @@ class ConfigNodeImpl {
 	// However, commit annotations for compacted mutations are lost
 	ACTOR static Future<Void> compact(ConfigNodeImpl* self, ConfigFollowerCompactRequest req) {
 		state Version lastCompactedVersion = wait(getLastCompactedVersion(self));
-		TraceEvent(SevDebug, "ConfigNodeCompacting", self->id)
+		TraceEvent(SevInfo, "ConfigNodeCompacting", self->id)
 		    .detail("Version", req.version)
 		    .detail("LastCompacted", lastCompactedVersion);
 		if (req.version <= lastCompactedVersion) {
@@ -505,11 +588,14 @@ class ConfigNodeImpl {
 			req.reply.sendError(transaction_too_old());
 			return Void();
 		}
-		TraceEvent("ConfigNodeRollforward")
+		TraceEvent("ConfigNodeRollforward", self->id)
 		    .detail("RollbackTo", req.rollback)
 		    .detail("Target", req.target)
 		    .detail("LastKnownCommitted", req.lastKnownCommitted)
-		    .detail("Committed", currentGeneration.committedVersion);
+		    .detail("Committed", currentGeneration.committedVersion)
+		    .detail("CurrentGeneration", currentGeneration.toString())
+		    .detail("LastCompactedVersion", lastCompactedVersion)
+		    .detail("SpecialZeroQuorum", req.specialZeroQuorum);
 		// Rollback to prior known committed version to erase any commits not
 		// made on a quorum.
 		if (req.rollback.present() && req.rollback.get() < currentGeneration.committedVersion) {
@@ -538,8 +624,15 @@ class ConfigNodeImpl {
 		}
 		// Now rollforward by applying all mutations between last known
 		// committed version and rollforward version.
-		ASSERT_GT(req.mutations[0].version, currentGeneration.committedVersion);
-		wait(commitMutations(self, req.mutations, req.annotations, req.target));
+		if (req.mutations.size() > 0) {
+			ASSERT_GT(req.mutations.size(), 0);
+			ASSERT_GT(req.mutations[0].version, currentGeneration.committedVersion);
+			wait(commitMutations(self,
+			                     req.mutations,
+			                     req.annotations,
+			                     req.target,
+			                     req.specialZeroQuorum ? currentGeneration.liveVersion : ::invalidVersion));
+		}
 
 		req.reply.send(Void());
 		return Void();
@@ -547,39 +640,20 @@ class ConfigNodeImpl {
 
 	ACTOR static Future<Void> getCommittedVersion(ConfigNodeImpl* self, ConfigFollowerGetCommittedVersionRequest req) {
 		state Version lastCompacted = wait(getLastCompactedVersion(self));
-		ConfigGeneration generation = wait(getGeneration(self));
-		req.reply.send(ConfigFollowerGetCommittedVersionReply{ lastCompacted, generation.committedVersion });
+		state ConfigGeneration generation = wait(getGeneration(self));
+		bool isRegistered = wait(registered(self));
+		req.reply.send(ConfigFollowerGetCommittedVersionReply{
+		    isRegistered, lastCompacted, generation.liveVersion, generation.committedVersion });
 		return Void();
 	}
 
-	ACTOR static Future<Void> serve(ConfigNodeImpl* self, ConfigFollowerInterface const* cfi) {
-		loop {
-			choose {
-				when(ConfigFollowerGetSnapshotAndChangesRequest req =
-				         waitNext(cfi->getSnapshotAndChanges.getFuture())) {
-					++self->snapshotRequests;
-					wait(getSnapshotAndChanges(self, req));
-				}
-				when(ConfigFollowerGetChangesRequest req = waitNext(cfi->getChanges.getFuture())) {
-					wait(getChanges(self, req));
-				}
-				when(ConfigFollowerCompactRequest req = waitNext(cfi->compact.getFuture())) {
-					++self->compactRequests;
-					wait(compact(self, req));
-				}
-				when(ConfigFollowerRollforwardRequest req = waitNext(cfi->rollforward.getFuture())) {
-					++self->rollforwardRequests;
-					wait(rollforward(self, req));
-				}
-				when(ConfigFollowerGetCommittedVersionRequest req = waitNext(cfi->getCommittedVersion.getFuture())) {
-					++self->getCommittedVersionRequests;
-					wait(getCommittedVersion(self, req));
-				}
-				when(wait(self->kvStore->getError())) { ASSERT(false); }
-			}
-		}
-	}
-
+	// Requires ConfigNodes to register with the ConfigBroadcaster before being
+	// allowed to respond to most requests. The ConfigBroadcaster will first
+	// ask the ConfigNode whether it is registered (kickstarted by the worker
+	// registering with the cluster controller). Then, the ConfigBroadcaster
+	// will send the ConfigNode a ready message, containing a snapshot if the
+	// ConfigNode is a new coordinator and needs updated state, or empty
+	// otherwise.
 	ACTOR static Future<Void> serve(ConfigNodeImpl* self, ConfigBroadcastInterface const* cbi, bool infinite) {
 		loop {
 			// Normally, the ConfigBroadcaster will first send a
@@ -592,10 +666,61 @@ class ConfigNodeImpl {
 			// ConfigNode.
 			choose {
 				when(state ConfigBroadcastRegisteredRequest req = waitNext(cbi->registered.getFuture())) {
-					bool isRegistered = wait(registered(self));
-					req.reply.send(ConfigBroadcastRegisteredReply{ isRegistered });
+					state bool isRegistered = wait(registered(self));
+					ConfigGeneration generation = wait(getGeneration(self));
+					TraceEvent("ConfigNodeSendingRegisteredReply", self->id)
+					    .detail("Generation", generation.toString());
+					req.reply.send(ConfigBroadcastRegisteredReply{ isRegistered, generation.committedVersion });
 				}
-				when(ConfigBroadcastReadyRequest readyReq = waitNext(cbi->ready.getFuture())) {
+				when(state ConfigBroadcastReadyRequest readyReq = waitNext(cbi->ready.getFuture())) {
+					state Optional<CoordinatorsHash> locked = wait(getLocked(self));
+
+					// New ConfigNodes with no previous state should always
+					// apply snapshots from the ConfigBroadcaster. Otherwise,
+					// the ConfigNode must be part of a new generation to
+					// accept a snapshot. An existing ConfigNode that restarts
+					// shouldn't apply a snapshot and overwrite its state if
+					// the set of coordinators hasn't changed.
+					if ((!infinite && !locked.present()) ||
+					    (locked.present() && locked.get() != readyReq.coordinatorsHash)) {
+						// Apply snapshot if necessary.
+						if (readyReq.snapshot.size() > 0) {
+							for (const auto& [configKey, knobValue] : readyReq.snapshot) {
+								TraceEvent("ConfigNodeSettingFromSnapshot", self->id)
+								    .detail("ConfigClass", configKey.configClass)
+								    .detail("KnobName", configKey.knobName)
+								    .detail("Value", knobValue.toString())
+								    .detail("Version", readyReq.snapshotVersion);
+								self->kvStore->set(KeyValueRef(
+								    BinaryWriter::toValue(configKey, IncludeVersion()).withPrefix(kvKeys.begin),
+								    ObjectWriter::toValue(knobValue, IncludeVersion())));
+							}
+							ConfigGeneration newGeneration = { readyReq.snapshotVersion, readyReq.liveVersion };
+							self->kvStore->set(KeyValueRef(currentGenerationKey,
+							                               BinaryWriter::toValue(newGeneration, IncludeVersion())));
+							// Clear out any mutations to the keys. If these
+							// aren't cleared, they will overwrite the
+							// snapshotted values when the knobs are read.
+							self->kvStore->clear(KeyRangeRef(versionedMutationKey(0, 0),
+							                                 versionedMutationKey(readyReq.snapshotVersion + 1, 0)));
+							self->kvStore->clear(KeyRangeRef(versionedAnnotationKey(0),
+							                                 versionedAnnotationKey(readyReq.snapshotVersion + 1)));
+
+							self->kvStore->set(
+							    KeyValueRef(lastCompactedVersionKey,
+							                BinaryWriter::toValue(readyReq.snapshotVersion, IncludeVersion())));
+						}
+						// Make sure freshly up to date ConfigNode isn't
+						// locked! This is possible if it was a coordinator in
+						// a previous generation.
+						self->kvStore->set(KeyValueRef(
+						    lockedKey, BinaryWriter::toValue(Optional<CoordinatorsHash>(), IncludeVersion())));
+					}
+					self->kvStore->set(KeyValueRef(coordinatorsHashKey,
+					                               BinaryWriter::toValue(readyReq.coordinatorsHash, IncludeVersion())));
+					wait(self->kvStore->commit());
+
+					TraceEvent("ConfigNodeReady", self->id).detail("WasLocked", locked.present());
 					readyReq.reply.send(ConfigBroadcastReadyReply{});
 					if (!infinite) {
 						return Void();
@@ -605,17 +730,70 @@ class ConfigNodeImpl {
 		}
 	}
 
+	ACTOR static Future<Void> serveRegistered(ConfigNodeImpl* self, ConfigFollowerInterface const* cfi) {
+		loop {
+			ConfigFollowerCompactRequest req = waitNext(cfi->compact.getFuture());
+			++self->compactRequests;
+			wait(compact(self, req));
+		}
+	}
+
+	// Many of the ConfigNode interfaces need to be served before the
+	// ConfigNode is officially registered with the ConfigBroadcaster. This is
+	// necessary due to edge cases around coordinator changes. For example, a
+	// ConfigNode that loses its coordinator status but then restarts before
+	// serving its snapshot to the new coordinators needs to be able to
+	// continue serving its snapshot interface when it restarts, even though it
+	// is no longer a coordinator.
+	ACTOR static Future<Void> serveUnregistered(ConfigNodeImpl* self, ConfigFollowerInterface const* cfi) {
+		loop {
+			choose {
+				when(ConfigFollowerGetSnapshotAndChangesRequest req =
+				         waitNext(cfi->getSnapshotAndChanges.getFuture())) {
+					++self->snapshotRequests;
+					wait(getSnapshotAndChanges(self, req));
+				}
+				when(ConfigFollowerGetChangesRequest req = waitNext(cfi->getChanges.getFuture())) {
+					wait(getChanges(self, req));
+				}
+				when(ConfigFollowerRollforwardRequest req = waitNext(cfi->rollforward.getFuture())) {
+					++self->rollforwardRequests;
+					wait(rollforward(self, req));
+				}
+				when(ConfigFollowerGetCommittedVersionRequest req = waitNext(cfi->getCommittedVersion.getFuture())) {
+					++self->getCommittedVersionRequests;
+					wait(getCommittedVersion(self, req));
+				}
+				when(state ConfigFollowerLockRequest req = waitNext(cfi->lock.getFuture())) {
+					++self->lockRequests;
+					CoordinatorsHash coordinatorsHash = wait(getCoordinatorsHash(self));
+					if (coordinatorsHash == 0 || coordinatorsHash == req.coordinatorsHash) {
+						TraceEvent("ConfigNodeLocking", self->id).log();
+						self->kvStore->set(KeyValueRef(registeredKey, BinaryWriter::toValue(false, IncludeVersion())));
+						self->kvStore->set(KeyValueRef(
+						    lockedKey,
+						    BinaryWriter::toValue(Optional<CoordinatorsHash>(req.coordinatorsHash), IncludeVersion())));
+						wait(self->kvStore->commit());
+					}
+					req.reply.send(Void());
+				}
+				when(wait(self->kvStore->getError())) { ASSERT(false); }
+			}
+		}
+	}
+
 	ACTOR static Future<Void> serve(ConfigNodeImpl* self,
 	                                ConfigBroadcastInterface const* cbi,
 	                                ConfigTransactionInterface const* cti,
 	                                ConfigFollowerInterface const* cfi) {
+		state Future<Void> serveUnregisteredFuture = serveUnregistered(self, cfi);
 		wait(serve(self, cbi, false));
 
 		self->kvStore->set(KeyValueRef(registeredKey, BinaryWriter::toValue(true, IncludeVersion())));
 		wait(self->kvStore->commit());
 
 		// Shouldn't return (coordinationServer will throw an error if it does).
-		wait(serve(self, cbi, true) || serve(self, cti) || serve(self, cfi));
+		wait(serve(self, cbi, true) || serve(self, cti) || serveRegistered(self, cfi) || serveUnregisteredFuture);
 		return Void();
 	}
 
@@ -630,11 +808,12 @@ public:
 	    compactRequests("CompactRequests", cc), rollbackRequests("RollbackRequests", cc),
 	    rollforwardRequests("RollforwardRequests", cc), successfulChangeRequests("SuccessfulChangeRequests", cc),
 	    failedChangeRequests("FailedChangeRequests", cc), snapshotRequests("SnapshotRequests", cc),
-	    getCommittedVersionRequests("GetCommittedVersionRequests", cc), successfulCommits("SuccessfulCommits", cc),
-	    failedCommits("FailedCommits", cc), setMutations("SetMutations", cc), clearMutations("ClearMutations", cc),
+	    getCommittedVersionRequests("GetCommittedVersionRequests", cc), lockRequests("LockRequests", cc),
+	    successfulCommits("SuccessfulCommits", cc), failedCommits("FailedCommits", cc),
+	    setMutations("SetMutations", cc), clearMutations("ClearMutations", cc),
 	    getValueRequests("GetValueRequests", cc), getGenerationRequests("GetGenerationRequests", cc) {
 		logger = traceCounters("ConfigNodeMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "ConfigNode");
-		TraceEvent(SevDebug, "StartingConfigNode", id).detail("KVStoreAlreadyExists", kvStore.exists());
+		TraceEvent(SevInfo, "StartingConfigNode", id).detail("KVStoreAlreadyExists", kvStore.exists());
 	}
 
 	Future<Void> serve(ConfigBroadcastInterface const& cbi,
@@ -643,9 +822,13 @@ public:
 		return serve(this, &cbi, &cti, &cfi);
 	}
 
+	Future<Void> serve(ConfigBroadcastInterface const& cbi) { return serve(this, &cbi, true); }
+
 	Future<Void> serve(ConfigTransactionInterface const& cti) { return serve(this, &cti); }
 
-	Future<Void> serve(ConfigFollowerInterface const& cfi) { return serve(this, &cfi); }
+	Future<Void> serve(ConfigFollowerInterface const& cfi) {
+		return serveUnregistered(this, &cfi) && serveRegistered(this, &cfi);
+	}
 
 	void close() { kvStore.close(); }
 
@@ -662,6 +845,10 @@ Future<Void> ConfigNode::serve(ConfigBroadcastInterface const& cbi,
 	return impl->serve(cbi, cti, cfi);
 }
 
+Future<Void> ConfigNode::serve(ConfigBroadcastInterface const& cbi) {
+	return impl->serve(cbi);
+}
+
 Future<Void> ConfigNode::serve(ConfigTransactionInterface const& cti) {
 	return impl->serve(cti);
 }
diff --git a/fdbserver/ConsistencyScan.actor.cpp b/fdbserver/ConsistencyScan.actor.cpp
new file mode 100644
index 0000000000..1a6046c659
--- /dev/null
+++ b/fdbserver/ConsistencyScan.actor.cpp
@@ -0,0 +1,1132 @@
+/*
+ * ConsistencyScan.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbrpc/TenantInfo.h"
+#include "fdbserver/WorkerInterface.actor.h"
+#include "flow/IRandom.h"
+#include "flow/IndexedSet.h"
+#include "fdbrpc/FailureMonitor.h"
+#include "fdbrpc/Smoother.h"
+#include "fdbrpc/simulator.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/TagThrottle.actor.h"
+#include "fdbserver/Knobs.h"
+#include "fdbserver/StorageMetrics.h"
+#include "fdbserver/DataDistribution.actor.h"
+#include "fdbserver/RatekeeperInterface.h"
+#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/WaitFailure.h"
+#include "fdbserver/TesterInterface.actor.h"
+#include "flow/DeterministicRandom.h"
+#include "flow/Trace.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+// Core of the data consistency checking (checkDataConsistency) and many of the supporting functions are shared between
+// the ConsistencyScan role and the ConsistencyCheck workload. They are currently part of this file. ConsistencyScan
+// role's main goal is to simply validate data across all shards, while ConsistencyCheck workload does more than that.
+// Potentially a re-factor candidate!
+
+struct ConsistencyScanData {
+	UID id;
+	Database db;
+
+	DatabaseConfiguration configuration;
+	PromiseStream<Future<Void>> addActor;
+
+	// TODO: Consider holding a ConsistencyScanInfo object to use as its state, as many of the members are the same.
+	int64_t restart = 1;
+	int64_t maxRate = 0;
+	int64_t targetInterval = 0;
+	int64_t bytesReadInPrevRound = 0;
+	int finishedRounds = 0;
+	KeyRef progressKey;
+	AsyncVar<bool> consistencyScanEnabled = false;
+
+	ConsistencyScanData(UID id, Database db) : id(id), db(db) {}
+};
+
+// Gets a version at which to read from the storage servers
+ACTOR Future<Version> getVersion(Database cx) {
+	loop {
+		state Transaction tr(cx);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		try {
+			Version version = wait(tr.getReadVersion());
+			return version;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+void testFailure(std::string message, bool performQuiescentChecks, bool isError) {
+	TraceEvent failEvent(isError ? SevError : SevWarn, "TestFailure");
+	if (performQuiescentChecks)
+		failEvent.detail("Workload", "QuiescentCheck");
+	else
+		failEvent.detail("Workload", "ConsistencyCheck");
+
+	failEvent.detail("Reason", "Consistency check: " + message);
+}
+
+// Get a list of storage servers(persisting keys within range "kr") from the master and compares them with the
+// TLogs. If this is a quiescent check, then each commit proxy needs to respond, otherwise only one needs to
+// respond. Returns false if there is a failure (in this case, keyServersPromise will never be set)
+ACTOR Future<bool> getKeyServers(
+    Database cx,
+    Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise,
+    KeyRangeRef kr,
+    bool performQuiescentChecks) {
+	state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers;
+
+	// Try getting key server locations from the master proxies
+	state std::vector<Future<ErrorOr<GetKeyServerLocationsReply>>> keyServerLocationFutures;
+	state Key begin = kr.begin;
+	state Key end = kr.end;
+	state int limitKeyServers = BUGGIFY ? 1 : 100;
+	state Span span(SpanContext(deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUInt64()),
+	                "WL:ConsistencyCheck"_loc);
+
+	while (begin < end) {
+		state Reference<CommitProxyInfo> commitProxyInfo =
+		    wait(cx->getCommitProxiesFuture(UseProvisionalProxies::False));
+		keyServerLocationFutures.clear();
+		for (int i = 0; i < commitProxyInfo->size(); i++)
+			keyServerLocationFutures.push_back(
+			    commitProxyInfo->get(i, &CommitProxyInterface::getKeyServersLocations)
+			        .getReplyUnlessFailedFor(
+			            GetKeyServerLocationsRequest(
+			                span.context, TenantInfo(), begin, end, limitKeyServers, false, latestVersion, Arena()),
+			            2,
+			            0));
+
+		state bool keyServersInsertedForThisIteration = false;
+		choose {
+			when(wait(waitForAll(keyServerLocationFutures))) {
+				// Read the key server location results
+				for (int i = 0; i < keyServerLocationFutures.size(); i++) {
+					ErrorOr<GetKeyServerLocationsReply> shards = keyServerLocationFutures[i].get();
+
+					// If performing quiescent check, then all master proxies should be reachable.  Otherwise, only
+					// one needs to be reachable
+					if (performQuiescentChecks && !shards.present()) {
+						TraceEvent("ConsistencyCheck_CommitProxyUnavailable")
+						    .detail("CommitProxyID", commitProxyInfo->getId(i));
+						testFailure("Commit proxy unavailable", performQuiescentChecks, true);
+						return false;
+					}
+
+					// Get the list of shards if one was returned.  If not doing a quiescent check, we can break if
+					// it is. If we are doing a quiescent check, then we only need to do this for the first shard.
+					if (shards.present() && !keyServersInsertedForThisIteration) {
+						keyServers.insert(keyServers.end(), shards.get().results.begin(), shards.get().results.end());
+						keyServersInsertedForThisIteration = true;
+						begin = shards.get().results.back().first.end;
+
+						if (!performQuiescentChecks)
+							break;
+					}
+				} // End of For
+			}
+			when(wait(cx->onProxiesChanged())) {}
+		} // End of choose
+
+		if (!keyServersInsertedForThisIteration) // Retry the entire workflow
+			wait(delay(1.0));
+
+	} // End of while
+
+	keyServersPromise.send(keyServers);
+	return true;
+}
+
+// Retrieves the locations of all shards in the database
+// Returns false if there is a failure (in this case, keyLocationPromise will never be set)
+ACTOR Future<bool> getKeyLocations(Database cx,
+                                   std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> shards,
+                                   Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise,
+                                   bool performQuiescentChecks) {
+	state Standalone<VectorRef<KeyValueRef>> keyLocations;
+	state Key beginKey = allKeys.begin.withPrefix(keyServersPrefix);
+	state Key endKey = allKeys.end.withPrefix(keyServersPrefix);
+	state int i = 0;
+	state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
+
+	// If the responses are too big, we may use multiple requests to get the key locations.  Each request begins
+	// where the last left off
+	for (; i < shards.size(); i++) {
+		while (beginKey < std::min<KeyRef>(shards[i].first.end, endKey)) {
+			try {
+				Version version = wait(getVersion(cx));
+
+				GetKeyValuesRequest req;
+				req.begin = firstGreaterOrEqual(beginKey);
+				req.end = firstGreaterOrEqual(std::min<KeyRef>(shards[i].first.end, endKey));
+				req.limit = SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT;
+				req.limitBytes = SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES;
+				req.version = version;
+				req.tags = TagSet();
+
+				// Try getting the shard locations from the key servers
+				state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
+				for (const auto& kv : shards[i].second) {
+					resetReply(req);
+					keyValueFutures.push_back(kv.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
+				}
+
+				wait(waitForAll(keyValueFutures));
+
+				int firstValidStorageServer = -1;
+
+				// Read the shard location results
+				for (int j = 0; j < keyValueFutures.size(); j++) {
+					ErrorOr<GetKeyValuesReply> reply = keyValueFutures[j].get();
+
+					if (!reply.present() || reply.get().error.present()) {
+						// If no storage servers replied, then throw all_alternatives_failed to force a retry
+						if (firstValidStorageServer < 0 && j == keyValueFutures.size() - 1)
+							throw all_alternatives_failed();
+					}
+
+					// If this is the first storage server, store the locations to send back to the caller
+					else if (firstValidStorageServer < 0) {
+						firstValidStorageServer = j;
+
+						// Otherwise, compare the data to the results from the first storage server.  If they are
+						// different, then the check fails
+					} else if (reply.get().data != keyValueFutures[firstValidStorageServer].get().get().data ||
+					           reply.get().more != keyValueFutures[firstValidStorageServer].get().get().more) {
+						TraceEvent("ConsistencyCheck_InconsistentKeyServers")
+						    .detail("StorageServer1", shards[i].second[firstValidStorageServer].id())
+						    .detail("StorageServer2", shards[i].second[j].id());
+						testFailure("Key servers inconsistent", performQuiescentChecks, true);
+						return false;
+					}
+				}
+
+				auto keyValueResponse = keyValueFutures[firstValidStorageServer].get().get();
+				RangeResult currentLocations = krmDecodeRanges(
+				    keyServersPrefix,
+				    KeyRangeRef(beginKey.removePrefix(keyServersPrefix),
+				                std::min<KeyRef>(shards[i].first.end, endKey).removePrefix(keyServersPrefix)),
+				    RangeResultRef(keyValueResponse.data, keyValueResponse.more));
+
+				if (keyValueResponse.data.size() && beginKey == keyValueResponse.data[0].key) {
+					keyLocations.push_back_deep(keyLocations.arena(), currentLocations[0]);
+				}
+
+				if (currentLocations.size() > 2) {
+					keyLocations.append_deep(keyLocations.arena(), &currentLocations[1], currentLocations.size() - 2);
+				}
+
+				// Next iteration should pick up where we left off
+				ASSERT(currentLocations.size() > 1);
+				if (!keyValueResponse.more) {
+					beginKey = shards[i].first.end;
+				} else {
+					beginKey = keyValueResponse.data.end()[-1].key;
+				}
+
+				// If this is the last iteration, then push the allKeys.end KV pair
+				if (beginKey >= endKey)
+					keyLocations.push_back_deep(keyLocations.arena(), currentLocations.end()[-1]);
+			} catch (Error& e) {
+				state Error err = e;
+				wait(onErrorTr.onError(err));
+				TraceEvent("ConsistencyCheck_RetryGetKeyLocations").error(err);
+			}
+		}
+	}
+
+	keyLocationPromise.send(keyLocations);
+	return true;
+}
+
+// Retrieves a vector of the storage servers' estimates for the size of a particular shard
+// If a storage server can't be reached, its estimate will be -1
+// If there is an error, then the returned vector will have 0 size
+ACTOR Future<std::vector<int64_t>> getStorageSizeEstimate(std::vector<StorageServerInterface> storageServers,
+                                                          KeyRangeRef shard) {
+	state std::vector<int64_t> estimatedBytes;
+
+	state WaitMetricsRequest req;
+	req.keys = shard;
+	req.max.bytes = -1;
+	req.min.bytes = 0;
+
+	state std::vector<Future<ErrorOr<StorageMetrics>>> metricFutures;
+
+	try {
+		// Check the size of the shard on each storage server
+		for (int i = 0; i < storageServers.size(); i++) {
+			resetReply(req);
+			metricFutures.push_back(storageServers[i].waitMetrics.getReplyUnlessFailedFor(req, 2, 0));
+		}
+
+		// Wait for the storage servers to respond
+		wait(waitForAll(metricFutures));
+
+		int firstValidStorageServer = -1;
+
+		// Retrieve the size from the storage server responses
+		for (int i = 0; i < storageServers.size(); i++) {
+			ErrorOr<StorageMetrics> reply = metricFutures[i].get();
+
+			// If the storage server doesn't reply, then return -1
+			if (!reply.present()) {
+				TraceEvent("ConsistencyCheck_FailedToFetchMetrics")
+				    .error(reply.getError())
+				    .detail("Begin", printable(shard.begin))
+				    .detail("End", printable(shard.end))
+				    .detail("StorageServer", storageServers[i].id())
+				    .detail("IsTSS", storageServers[i].isTss() ? "True" : "False");
+				estimatedBytes.push_back(-1);
+			}
+
+			// Add the result to the list of estimates
+			else if (reply.present()) {
+				int64_t numBytes = reply.get().bytes;
+				estimatedBytes.push_back(numBytes);
+				if (firstValidStorageServer < 0)
+					firstValidStorageServer = i;
+				else if (estimatedBytes[firstValidStorageServer] != numBytes) {
+					TraceEvent("ConsistencyCheck_InconsistentStorageMetrics")
+					    .detail("ByteEstimate1", estimatedBytes[firstValidStorageServer])
+					    .detail("ByteEstimate2", numBytes)
+					    .detail("Begin", shard.begin)
+					    .detail("End", shard.end)
+					    .detail("StorageServer1", storageServers[firstValidStorageServer].id())
+					    .detail("StorageServer2", storageServers[i].id())
+					    .detail("IsTSS",
+					            storageServers[i].isTss() || storageServers[firstValidStorageServer].isTss() ? "True"
+					                                                                                         : "False");
+				}
+			}
+		}
+	} catch (Error& e) {
+		TraceEvent("ConsistencyCheck_ErrorFetchingMetrics")
+		    .error(e)
+		    .detail("Begin", printable(shard.begin))
+		    .detail("End", printable(shard.end));
+		estimatedBytes.clear();
+	}
+
+	return estimatedBytes;
+}
+
+ACTOR Future<int64_t> getDatabaseSize(Database cx) {
+	state Transaction tr(cx);
+	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+	loop {
+		try {
+			StorageMetrics metrics =
+			    wait(tr.getDatabase()->getStorageMetrics(KeyRangeRef(allKeys.begin, keyServersPrefix), 100000));
+			return metrics.bytes;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+// Checks that the data in each shard is the same on each storage server that it resides on.  Also performs some
+// sanity checks on the sizes of shards and storage servers. Returns false if there is a failure
+// TODO: Future optimization: Use streaming reads
+ACTOR Future<bool> checkDataConsistency(Database cx,
+                                        VectorRef<KeyValueRef> keyLocations,
+                                        DatabaseConfiguration configuration,
+                                        std::map<UID, StorageServerInterface> tssMapping,
+                                        bool performQuiescentChecks,
+                                        bool performTSSCheck,
+                                        bool firstClient,
+                                        bool failureIsError,
+                                        int clientId,
+                                        int clientCount,
+                                        bool distributed,
+                                        bool shuffleShards,
+                                        int shardSampleFactor,
+                                        int64_t sharedRandomNumber,
+                                        int64_t repetitions,
+                                        int64_t* bytesReadInPrevRound,
+                                        int restart,
+                                        int64_t maxRate,
+                                        int64_t targetInterval,
+                                        KeyRef progressKey) {
+	// Stores the total number of bytes on each storage server
+	// In a distributed test, this will be an estimated size
+	state std::map<UID, int64_t> storageServerSizes;
+
+	// Iterate through each shard, checking its values on all of its storage servers
+	// If shardSampleFactor > 1, then not all shards are processed
+	// Also, in a distributed data consistency check, each client processes a subset of the shards
+	// Note: this may cause some shards to be processed more than once or not at all in a non-quiescent database
+	state int effectiveClientCount = distributed ? clientCount : 1;
+	state int i = clientId * (shardSampleFactor + 1);
+	state int increment = (distributed && !firstClient) ? effectiveClientCount * shardSampleFactor : 1;
+	state int64_t rateLimitForThisRound =
+	    *bytesReadInPrevRound == 0
+	        ? maxRate
+	        : std::min(maxRate, static_cast<int64_t>(ceil(*bytesReadInPrevRound / (float)targetInterval)));
+	ASSERT(rateLimitForThisRound >= 0 && rateLimitForThisRound <= maxRate);
+	TraceEvent("ConsistencyCheck_RateLimitForThisRound").detail("RateLimit", rateLimitForThisRound);
+	state Reference<IRateControl> rateLimiter = Reference<IRateControl>(new SpeedLimit(rateLimitForThisRound, 1));
+	state double rateLimiterStartTime = now();
+	state int64_t bytesReadInthisRound = 0;
+	state bool resume = !(restart || shuffleShards);
+
+	state double dbSize = 100e12;
+	if (g_network->isSimulated()) {
+		// This call will get all shard ranges in the database, which is too expensive on real clusters.
+		int64_t _dbSize = wait(getDatabaseSize(cx));
+		dbSize = _dbSize;
+	}
+
+	state std::vector<KeyRangeRef> ranges;
+
+	for (int k = 0; k < keyLocations.size() - 1; k++) {
+		// TODO: check if this is sufficient
+		if (resume && keyLocations[k].key < progressKey) {
+			TraceEvent("ConsistencyCheck_SkippingRange")
+			    .detail("KeyBegin", keyLocations[k].key.toString())
+			    .detail("KeyEnd", keyLocations[k + 1].key.toString())
+			    .detail("PrevKey", progressKey.toString());
+			continue;
+		}
+		KeyRangeRef range(keyLocations[k].key, keyLocations[k + 1].key);
+		ranges.push_back(range);
+	}
+
+	state std::vector<int> shardOrder;
+	shardOrder.reserve(ranges.size());
+	for (int k = 0; k < ranges.size(); k++)
+		shardOrder.push_back(k);
+	if (shuffleShards) {
+		uint32_t seed = sharedRandomNumber + repetitions;
+		DeterministicRandom sharedRandom(seed == 0 ? 1 : seed);
+		sharedRandom.randomShuffle(shardOrder);
+	}
+
+	for (; i < ranges.size(); i++) {
+		state int shard = shardOrder[i];
+
+		state KeyRangeRef range = ranges[shard];
+		state std::vector<UID> sourceStorageServers;
+		state std::vector<UID> destStorageServers;
+		state Transaction tr(cx);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		state int bytesReadInRange = 0;
+
+		RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+		ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
+		decodeKeyServersValue(UIDtoTagMap, keyLocations[shard].value, sourceStorageServers, destStorageServers, false);
+
+		// If the destStorageServers is non-empty, then this shard is being relocated
+		state bool isRelocating = destStorageServers.size() > 0;
+
+		// In a quiescent database, check that the team size is the same as the desired team size
+		if (firstClient && performQuiescentChecks &&
+		    sourceStorageServers.size() != configuration.usableRegions * configuration.storageTeamSize) {
+			TraceEvent("ConsistencyCheck_InvalidTeamSize")
+			    .detail("ShardBegin", printable(range.begin))
+			    .detail("ShardEnd", printable(range.end))
+			    .detail("SourceTeamSize", sourceStorageServers.size())
+			    .detail("DestServerSize", destStorageServers.size())
+			    .detail("ConfigStorageTeamSize", configuration.storageTeamSize)
+			    .detail("UsableRegions", configuration.usableRegions);
+			// Record the server reponsible for the problematic shards
+			int k = 0;
+			for (auto& id : sourceStorageServers) {
+				TraceEvent("IncorrectSizeTeamInfo").detail("ServerUID", id).detail("TeamIndex", k++);
+			}
+			testFailure("Invalid team size", performQuiescentChecks, failureIsError);
+			return false;
+		}
+
+		state std::vector<UID> storageServers = (isRelocating) ? destStorageServers : sourceStorageServers;
+		state std::vector<StorageServerInterface> storageServerInterfaces;
+
+		loop {
+			try {
+				std::vector<Future<Optional<Value>>> serverListEntries;
+				serverListEntries.reserve(storageServers.size());
+				for (int s = 0; s < storageServers.size(); s++)
+					serverListEntries.push_back(tr.get(serverListKeyFor(storageServers[s])));
+				state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+				for (int s = 0; s < serverListValues.size(); s++) {
+					if (serverListValues[s].present())
+						storageServerInterfaces.push_back(decodeServerListValue(serverListValues[s].get()));
+					else if (performQuiescentChecks)
+						testFailure(
+						    "/FF/serverList changing in a quiescent database", performQuiescentChecks, failureIsError);
+				}
+
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		// add TSS to end of list, if configured and if not relocating
+		if (!isRelocating && performTSSCheck) {
+			int initialSize = storageServers.size();
+			for (int i = 0; i < initialSize; i++) {
+				auto tssPair = tssMapping.find(storageServers[i]);
+				if (tssPair != tssMapping.end()) {
+					CODE_PROBE(true, "TSS checked in consistency check");
+					storageServers.push_back(tssPair->second.id());
+					storageServerInterfaces.push_back(tssPair->second);
+				}
+			}
+		}
+
+		state std::vector<int64_t> estimatedBytes = wait(getStorageSizeEstimate(storageServerInterfaces, range));
+
+		// Gets permitted size range of shard
+		int64_t maxShardSize = getMaxShardSize(dbSize);
+		state ShardSizeBounds shardBounds = getShardSizeBounds(range, maxShardSize);
+
+		if (firstClient) {
+			// If there was an error retrieving shard estimated size
+			if (performQuiescentChecks && estimatedBytes.size() == 0)
+				testFailure("Error fetching storage metrics", performQuiescentChecks, failureIsError);
+
+			// If running a distributed test, storage server size is an accumulation of shard estimates
+			else if (distributed && firstClient)
+				for (int j = 0; j < storageServers.size(); j++)
+					storageServerSizes[storageServers[j]] += std::max(estimatedBytes[j], (int64_t)0);
+		}
+
+		// The first client may need to skip the rest of the loop contents if it is just processing this shard to
+		// get a size estimate
+		if (!firstClient || shard % (effectiveClientCount * shardSampleFactor) == 0) {
+			state int shardKeys = 0;
+			state int shardBytes = 0;
+			state int sampledBytes = 0;
+			state int splitBytes = 0;
+			state int firstKeySampledBytes = 0;
+			state int sampledKeys = 0;
+			state int sampledKeysWithProb = 0;
+			state double shardVariance = 0;
+			state bool canSplit = false;
+			state Key lastSampleKey;
+			state Key lastStartSampleKey;
+			state int64_t totalReadAmount = 0;
+
+			state KeySelector begin = firstGreaterOrEqual(range.begin);
+			state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
+
+			// Read a limited number of entries at a time, repeating until all keys in the shard have been read
+			loop {
+				try {
+					lastSampleKey = lastStartSampleKey;
+
+					// Get the min version of the storage servers
+					Version version = wait(getVersion(cx));
+
+					state GetKeyValuesRequest req;
+					req.begin = begin;
+					req.end = firstGreaterOrEqual(range.end);
+					req.limit = 1e4;
+					req.limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
+					req.version = version;
+					req.tags = TagSet();
+
+					// Try getting the entries in the specified range
+					state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
+					state int j = 0;
+					TraceEvent("ConsistencyCheck_StoringGetFutures").detail("SSISize", storageServerInterfaces.size());
+					for (j = 0; j < storageServerInterfaces.size(); j++) {
+						resetReply(req);
+						keyValueFutures.push_back(
+						    storageServerInterfaces[j].getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
+					}
+
+					wait(waitForAll(keyValueFutures));
+
+					// Read the resulting entries
+					state int firstValidServer = -1;
+					totalReadAmount = 0;
+					for (j = 0; j < storageServerInterfaces.size(); j++) {
+						ErrorOr<GetKeyValuesReply> rangeResult = keyValueFutures[j].get();
+
+						// Compare the results with other storage servers
+						if (rangeResult.present() && !rangeResult.get().error.present()) {
+							state GetKeyValuesReply current = rangeResult.get();
+							TraceEvent("ConsistencyCheck_GetKeyValuesStream")
+							    .detail("DataSize", current.data.size())
+							    .detail(format("StorageServer%d", j).c_str(), storageServers[j].toString());
+							totalReadAmount += current.data.expectedSize();
+							// If we haven't encountered a valid storage server yet, then mark this as the baseline
+							// to compare against
+							if (firstValidServer == -1) {
+								TraceEvent("ConsistencyCheck_FirstValidServer").detail("Iter", j);
+								firstValidServer = j;
+								// Compare this shard against the first
+							} else {
+								GetKeyValuesReply reference = keyValueFutures[firstValidServer].get().get();
+
+								if (current.data != reference.data || current.more != reference.more) {
+									// Be especially verbose if in simulation
+									if (g_network->isSimulated()) {
+										int invalidIndex = -1;
+										printf("\n%sSERVER %d (%s); shard = %s - %s:\n",
+										       "",
+										       j,
+										       storageServerInterfaces[j].address().toString().c_str(),
+										       printable(req.begin.getKey()).c_str(),
+										       printable(req.end.getKey()).c_str());
+										for (int k = 0; k < current.data.size(); k++) {
+											printf("%d. %s => %s\n",
+											       k,
+											       printable(current.data[k].key).c_str(),
+											       printable(current.data[k].value).c_str());
+											if (invalidIndex < 0 && (k >= reference.data.size() ||
+											                         current.data[k].key != reference.data[k].key ||
+											                         current.data[k].value != reference.data[k].value))
+												invalidIndex = k;
+										}
+
+										printf("\n%sSERVER %d (%s); shard = %s - %s:\n",
+										       "",
+										       firstValidServer,
+										       storageServerInterfaces[firstValidServer].address().toString().c_str(),
+										       printable(req.begin.getKey()).c_str(),
+										       printable(req.end.getKey()).c_str());
+										for (int k = 0; k < reference.data.size(); k++) {
+											printf("%d. %s => %s\n",
+											       k,
+											       printable(reference.data[k].key).c_str(),
+											       printable(reference.data[k].value).c_str());
+											if (invalidIndex < 0 && (k >= current.data.size() ||
+											                         reference.data[k].key != current.data[k].key ||
+											                         reference.data[k].value != current.data[k].value))
+												invalidIndex = k;
+										}
+
+										printf("\nMISMATCH AT %d\n\n", invalidIndex);
+									}
+
+									// Data for trace event
+									// The number of keys unique to the current shard
+									int currentUniques = 0;
+									// The number of keys unique to the reference shard
+									int referenceUniques = 0;
+									// The number of keys in both shards with conflicting values
+									int valueMismatches = 0;
+									// The number of keys in both shards with matching values
+									int matchingKVPairs = 0;
+									// Last unique key on the current shard
+									KeyRef currentUniqueKey;
+									// Last unique key on the reference shard
+									KeyRef referenceUniqueKey;
+									// Last value mismatch
+									KeyRef valueMismatchKey;
+
+									// Loop indeces
+									int currentI = 0;
+									int referenceI = 0;
+									while (currentI < current.data.size() || referenceI < reference.data.size()) {
+										if (currentI >= current.data.size()) {
+											referenceUniqueKey = reference.data[referenceI].key;
+											referenceUniques++;
+											referenceI++;
+										} else if (referenceI >= reference.data.size()) {
+											currentUniqueKey = current.data[currentI].key;
+											currentUniques++;
+											currentI++;
+										} else {
+											KeyValueRef currentKV = current.data[currentI];
+											KeyValueRef referenceKV = reference.data[referenceI];
+
+											if (currentKV.key == referenceKV.key) {
+												if (currentKV.value == referenceKV.value)
+													matchingKVPairs++;
+												else {
+													valueMismatchKey = currentKV.key;
+													valueMismatches++;
+												}
+
+												currentI++;
+												referenceI++;
+											} else if (currentKV.key < referenceKV.key) {
+												currentUniqueKey = currentKV.key;
+												currentUniques++;
+												currentI++;
+											} else {
+												referenceUniqueKey = referenceKV.key;
+												referenceUniques++;
+												referenceI++;
+											}
+										}
+									}
+
+									TraceEvent("ConsistencyCheck_DataInconsistent")
+									    .detail(format("StorageServer%d", j).c_str(), storageServers[j].toString())
+									    .detail(format("StorageServer%d", firstValidServer).c_str(),
+									            storageServers[firstValidServer].toString())
+									    .detail("ShardBegin", req.begin.getKey())
+									    .detail("ShardEnd", req.end.getKey())
+									    .detail("VersionNumber", req.version)
+									    .detail(format("Server%dUniques", j).c_str(), currentUniques)
+									    .detail(format("Server%dUniqueKey", j).c_str(), currentUniqueKey)
+									    .detail(format("Server%dUniques", firstValidServer).c_str(), referenceUniques)
+									    .detail(format("Server%dUniqueKey", firstValidServer).c_str(),
+									            referenceUniqueKey)
+									    .detail("ValueMismatches", valueMismatches)
+									    .detail("ValueMismatchKey", valueMismatchKey)
+									    .detail("MatchingKVPairs", matchingKVPairs)
+									    .detail("IsTSS",
+									            storageServerInterfaces[j].isTss() ||
+									                    storageServerInterfaces[firstValidServer].isTss()
+									                ? "True"
+									                : "False");
+
+									if ((g_network->isSimulated() &&
+									     g_simulator->tssMode != ISimulator::TSSMode::EnabledDropMutations) ||
+									    (!storageServerInterfaces[j].isTss() &&
+									     !storageServerInterfaces[firstValidServer].isTss())) {
+										testFailure("Data inconsistent", performQuiescentChecks, true);
+										return false;
+									}
+								}
+							}
+						}
+
+						// If the data is not available and we aren't relocating this shard
+						else if (!isRelocating) {
+							Error e = rangeResult.isError() ? rangeResult.getError() : rangeResult.get().error.get();
+
+							TraceEvent("ConsistencyCheck_StorageServerUnavailable")
+							    .errorUnsuppressed(e)
+							    .suppressFor(1.0)
+							    .detail("StorageServer", storageServers[j])
+							    .detail("ShardBegin", printable(range.begin))
+							    .detail("ShardEnd", printable(range.end))
+							    .detail("Address", storageServerInterfaces[j].address())
+							    .detail("UID", storageServerInterfaces[j].id())
+							    .detail("GetKeyValuesToken",
+							            storageServerInterfaces[j].getKeyValues.getEndpoint().token)
+							    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False");
+
+							// All shards should be available in quiscence
+							if (performQuiescentChecks && !storageServerInterfaces[j].isTss()) {
+								testFailure("Storage server unavailable", performQuiescentChecks, failureIsError);
+								return false;
+							}
+						}
+					}
+
+					if (firstValidServer >= 0) {
+						state VectorRef<KeyValueRef> data = keyValueFutures[firstValidServer].get().get().data;
+
+						// Persist the last key of the range we just verified as the progressKey
+						if (data.size()) {
+							state Reference<ReadYourWritesTransaction> csInfoTr =
+							    makeReference<ReadYourWritesTransaction>(cx);
+							progressKey = data[data.size() - 1].key;
+							loop {
+								try {
+									csInfoTr->reset();
+									csInfoTr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+
+									state Optional<Value> val = wait(ConsistencyScanInfo::getInfo(csInfoTr));
+									wait(csInfoTr->commit());
+									if (val.present()) {
+										ConsistencyScanInfo consistencyScanInfo =
+										    ObjectReader::fromStringRef<ConsistencyScanInfo>(val.get(),
+										                                                     IncludeVersion());
+										consistencyScanInfo.progress_key = progressKey;
+										csInfoTr->reset();
+										csInfoTr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+										wait(ConsistencyScanInfo::setInfo(csInfoTr, consistencyScanInfo));
+										wait(csInfoTr->commit());
+									}
+									break;
+								} catch (Error& e) {
+									wait(csInfoTr->onError(e));
+								}
+							}
+						}
+
+						// Calculate the size of the shard, the variance of the shard size estimate, and the correct
+						// shard size estimate
+						for (int k = 0; k < data.size(); k++) {
+							ByteSampleInfo sampleInfo = isKeyValueInSample(data[k]);
+							shardBytes += sampleInfo.size;
+							double itemProbability = ((double)sampleInfo.size) / sampleInfo.sampledSize;
+							if (itemProbability < 1)
+								shardVariance +=
+								    itemProbability * (1 - itemProbability) * pow((double)sampleInfo.sampledSize, 2);
+
+							if (sampleInfo.inSample) {
+								sampledBytes += sampleInfo.sampledSize;
+								if (!canSplit && sampledBytes >= shardBounds.min.bytes &&
+								    data[k].key.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT &&
+								    sampledBytes <=
+								        shardBounds.max.bytes * CLIENT_KNOBS->STORAGE_METRICS_UNFAIR_SPLIT_LIMIT / 2) {
+									canSplit = true;
+									splitBytes = sampledBytes;
+								}
+
+								/*TraceEvent("ConsistencyCheck_ByteSample").detail("ShardBegin", printable(range.begin)).detail("ShardEnd", printable(range.end))
+								  .detail("SampledBytes", sampleInfo.sampledSize).detail("Key",
+								  printable(data[k].key)).detail("KeySize", data[k].key.size()).detail("ValueSize",
+								  data[k].value.size());*/
+
+								// In data distribution, the splitting process ignores the first key in a shard.
+								// Thus, we shouldn't consider it when validating the upper bound of estimated shard
+								// sizes
+								if (k == 0)
+									firstKeySampledBytes += sampleInfo.sampledSize;
+
+								sampledKeys++;
+								if (itemProbability < 1) {
+									sampledKeysWithProb++;
+								}
+							}
+						}
+
+						// Accumulate number of keys in this shard
+						shardKeys += data.size();
+					}
+					// after requesting each shard, enforce rate limit based on how much data will likely be read
+					if (rateLimitForThisRound > 0) {
+						TraceEvent("ConsistencyCheck_RateLimit")
+						    .detail("RateLimitForThisRound", rateLimitForThisRound)
+						    .detail("TotalAmountRead", totalReadAmount);
+						wait(rateLimiter->getAllowance(totalReadAmount));
+						TraceEvent("ConsistencyCheck_AmountRead1").detail("TotalAmountRead", totalReadAmount);
+						// Set ratelimit to max allowed if current round has been going on for a while
+						if (now() - rateLimiterStartTime > 1.1 * targetInterval && rateLimitForThisRound != maxRate) {
+							rateLimitForThisRound = maxRate;
+							rateLimiter = Reference<IRateControl>(new SpeedLimit(rateLimitForThisRound, 1));
+							rateLimiterStartTime = now();
+							TraceEvent(SevInfo, "ConsistencyCheck_RateLimitSetMaxForThisRound")
+							    .detail("RateLimit", rateLimitForThisRound);
+						}
+					}
+					bytesReadInRange += totalReadAmount;
+					bytesReadInthisRound += totalReadAmount;
+					TraceEvent("ConsistencyCheck_BytesRead")
+					    .detail("BytesReadInRange", bytesReadInRange)
+					    .detail("BytesReadInthisRound", bytesReadInthisRound);
+
+					// Advance to the next set of entries
+					if (firstValidServer >= 0 && keyValueFutures[firstValidServer].get().get().more) {
+						VectorRef<KeyValueRef> result = keyValueFutures[firstValidServer].get().get().data;
+						ASSERT(result.size() > 0);
+						begin = firstGreaterThan(result[result.size() - 1].key);
+						ASSERT(begin.getKey() != allKeys.end);
+						lastStartSampleKey = lastSampleKey;
+					} else
+						break;
+				} catch (Error& e) {
+					state Error err = e;
+					wait(onErrorTr.onError(err));
+					TraceEvent("ConsistencyCheck_RetryDataConsistency").error(err);
+				}
+			}
+
+			canSplit = canSplit && sampledBytes - splitBytes >= shardBounds.min.bytes && sampledBytes > splitBytes;
+
+			// Update the size of all storage servers containing this shard
+			// This is only done in a non-distributed consistency check; the distributed check uses shard size
+			// estimates
+			if (!distributed)
+				for (int j = 0; j < storageServers.size(); j++)
+					storageServerSizes[storageServers[j]] += shardBytes;
+
+			// If the storage servers' sampled estimate of shard size is different from ours
+			if (performQuiescentChecks) {
+				for (int j = 0; j < estimatedBytes.size(); j++) {
+					if (estimatedBytes[j] >= 0 && estimatedBytes[j] != sampledBytes) {
+						TraceEvent("ConsistencyCheck_IncorrectEstimate")
+						    .detail("EstimatedBytes", estimatedBytes[j])
+						    .detail("CorrectSampledBytes", sampledBytes)
+						    .detail("StorageServer", storageServers[j])
+						    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False");
+
+						if (!storageServerInterfaces[j].isTss()) {
+							testFailure("Storage servers had incorrect sampled estimate",
+							            performQuiescentChecks,
+							            failureIsError);
+						}
+
+						break;
+					} else if (estimatedBytes[j] < 0 && ((g_network->isSimulated() &&
+					                                      g_simulator->tssMode <= ISimulator::TSSMode::EnabledNormal) ||
+					                                     !storageServerInterfaces[j].isTss())) {
+						// Ignore a non-responding TSS outside of simulation, or if tss fault injection is enabled
+						break;
+					}
+				}
+			}
+
+			// Compute the difference between the shard size estimate and its actual size.  If it is sufficiently
+			// large, then fail
+			double stdDev = sqrt(shardVariance);
+
+			double failErrorNumStdDev = 7;
+			int estimateError = abs(shardBytes - sampledBytes);
+
+			// Only perform the check if there are sufficient keys to get a distribution that should resemble a
+			// normal distribution
+			if (sampledKeysWithProb > 30 && estimateError > failErrorNumStdDev * stdDev) {
+				double numStdDev = estimateError / sqrt(shardVariance);
+				TraceEvent("ConsistencyCheck_InaccurateShardEstimate")
+				    .detail("Min", shardBounds.min.bytes)
+				    .detail("Max", shardBounds.max.bytes)
+				    .detail("Estimate", sampledBytes)
+				    .detail("Actual", shardBytes)
+				    .detail("NumStdDev", numStdDev)
+				    .detail("Variance", shardVariance)
+				    .detail("StdDev", stdDev)
+				    .detail("ShardBegin", printable(range.begin))
+				    .detail("ShardEnd", printable(range.end))
+				    .detail("NumKeys", shardKeys)
+				    .detail("NumSampledKeys", sampledKeys)
+				    .detail("NumSampledKeysWithProb", sampledKeysWithProb);
+
+				testFailure(format("Shard size is more than %f std dev from estimate", failErrorNumStdDev),
+				            performQuiescentChecks,
+				            failureIsError);
+			}
+
+			// In a quiescent database, check that the (estimated) size of the shard is within permitted bounds
+			// Min and max shard sizes have a 3 * shardBounds.permittedError.bytes cushion for error since shard
+			// sizes are not precise Shard splits ignore the first key in a shard, so its size shouldn't be
+			// considered when checking the upper bound 0xff shards are not checked
+			if (canSplit && sampledKeys > 5 && performQuiescentChecks && !range.begin.startsWith(keyServersPrefix) &&
+			    (sampledBytes < shardBounds.min.bytes - 3 * shardBounds.permittedError.bytes ||
+			     sampledBytes - firstKeySampledBytes > shardBounds.max.bytes + 3 * shardBounds.permittedError.bytes)) {
+				TraceEvent("ConsistencyCheck_InvalidShardSize")
+				    .detail("Min", shardBounds.min.bytes)
+				    .detail("Max", shardBounds.max.bytes)
+				    .detail("Size", shardBytes)
+				    .detail("EstimatedSize", sampledBytes)
+				    .detail("ShardBegin", printable(range.begin))
+				    .detail("ShardEnd", printable(range.end))
+				    .detail("ShardCount", ranges.size())
+				    .detail("SampledKeys", sampledKeys);
+				testFailure(format("Shard size in quiescent database is too %s",
+				                   (sampledBytes < shardBounds.min.bytes) ? "small" : "large"),
+				            performQuiescentChecks,
+				            failureIsError);
+				return false;
+			}
+		}
+
+		if (bytesReadInRange > 0) {
+			TraceEvent("ConsistencyCheck_ReadRange")
+			    .suppressFor(1.0)
+			    .detail("Range", range)
+			    .detail("BytesRead", bytesReadInRange);
+		}
+	}
+
+	*bytesReadInPrevRound = bytesReadInthisRound;
+	return true;
+}
+
+ACTOR Future<Void> runDataValidationCheck(ConsistencyScanData* self) {
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->db);
+	state ConsistencyScanInfo csInfo = ConsistencyScanInfo();
+	csInfo.consistency_scan_enabled = true;
+	csInfo.restart = self->restart;
+	csInfo.max_rate = self->maxRate;
+	csInfo.target_interval = self->targetInterval;
+	csInfo.last_round_start = StorageMetadataType::currentTime();
+	try {
+		// Get a list of key servers; verify that the TLogs and master all agree about who the key servers are
+		state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
+		state std::map<UID, StorageServerInterface> tssMapping;
+		bool keyServerResult = wait(getKeyServers(self->db, keyServerPromise, keyServersKeys, false));
+		if (keyServerResult) {
+			state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
+			    keyServerPromise.getFuture().get();
+
+			// Get the locations of all the shards in the database
+			state Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
+			bool keyLocationResult = wait(getKeyLocations(self->db, keyServers, keyLocationPromise, false));
+			if (keyLocationResult) {
+				state Standalone<VectorRef<KeyValueRef>> keyLocations = keyLocationPromise.getFuture().get();
+
+				// Check that each shard has the same data on all storage servers that it resides on
+				wait(::success(checkDataConsistency(self->db,
+				                                    keyLocations,
+				                                    self->configuration,
+				                                    tssMapping,
+				                                    false /* quiescentCheck */,
+				                                    false /* tssCheck */,
+				                                    true /* firstClient */,
+				                                    false /* failureIsError */,
+				                                    0 /* clientId */,
+				                                    1 /* clientCount */,
+				                                    false /* distributed */,
+				                                    false /* shuffleShards */,
+				                                    1 /* shardSampleFactor */,
+				                                    deterministicRandom()->randomInt64(0, 10000000),
+				                                    self->finishedRounds /* repetitions */,
+				                                    &(self->bytesReadInPrevRound),
+				                                    self->restart,
+				                                    self->maxRate,
+				                                    self->targetInterval,
+				                                    self->progressKey)));
+			}
+		}
+	} catch (Error& e) {
+		if (e.code() == error_code_transaction_too_old || e.code() == error_code_future_version ||
+		    e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
+		    e.code() == error_code_process_behind || e.code() == error_code_actor_cancelled)
+			TraceEvent("ConsistencyScan_Retry").error(e); // FIXME: consistency check does not retry in this case
+		else
+			throw;
+	}
+
+	TraceEvent("ConsistencyScan_FinishedCheck");
+
+	//  Update the ConsistencyScanInfo object and persist to the database
+	csInfo.last_round_finish = StorageMetadataType::currentTime();
+	csInfo.finished_rounds = self->finishedRounds + 1;
+	auto duration = csInfo.last_round_finish - csInfo.last_round_start;
+	csInfo.smoothed_round_duration.setTotal((double)duration);
+	csInfo.progress_key = self->progressKey;
+	csInfo.bytes_read_prev_round = self->bytesReadInPrevRound;
+	loop {
+		try {
+			tr->reset();
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			wait(ConsistencyScanInfo::setInfo(tr, csInfo));
+			wait(tr->commit());
+			break;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+
+	return Void();
+}
+
+ACTOR Future<Void> watchConsistencyScanInfoKey(ConsistencyScanData* self) {
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->db);
+
+	loop {
+		try {
+			tr->reset();
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+
+			state Optional<Value> val = wait(ConsistencyScanInfo::getInfo(tr));
+			if (val.present()) {
+				ConsistencyScanInfo consistencyScanInfo =
+				    ObjectReader::fromStringRef<ConsistencyScanInfo>(val.get(), IncludeVersion());
+				self->restart = consistencyScanInfo.restart;
+				self->maxRate = consistencyScanInfo.max_rate;
+				self->targetInterval = consistencyScanInfo.target_interval;
+				self->progressKey = consistencyScanInfo.progress_key;
+				self->bytesReadInPrevRound = consistencyScanInfo.bytes_read_prev_round;
+				self->finishedRounds = consistencyScanInfo.finished_rounds;
+				self->consistencyScanEnabled.set(consistencyScanInfo.consistency_scan_enabled);
+				//TraceEvent("ConsistencyScan_WatchGotVal", self->id)
+				//  .detail("Enabled", consistencyScanInfo.consistency_scan_enabled)
+				//  .detail("MaxRateRead", consistencyScanInfo.max_rate)
+				//  .detail("MaxRateSelf", self->maxRate);
+			}
+			state Future<Void> watch = tr->watch(consistencyScanInfoKey);
+			wait(tr->commit());
+			wait(watch);
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+}
+
+ACTOR Future<Void> consistencyScan(ConsistencyScanInterface csInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	state ConsistencyScanData self(csInterf.id(),
+	                               openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True));
+	state Promise<Void> err;
+	state Future<Void> collection = actorCollection(self.addActor.getFuture());
+	state ConsistencyScanInfo csInfo = ConsistencyScanInfo();
+
+	TraceEvent("ConsistencyScan_Starting", csInterf.id()).log();
+
+	// Randomly enable consistencyScan in simulation
+	if (g_network->isSimulated()) {
+		if (deterministicRandom()->random01() < 0.5) {
+			csInfo.consistency_scan_enabled = false;
+		} else {
+			csInfo.consistency_scan_enabled = true;
+			csInfo.restart = false;
+			csInfo.max_rate = 50e6;
+			csInfo.target_interval = 24 * 7 * 60 * 60;
+		}
+		TraceEvent("SimulatedConsistencyScanConfigRandom")
+		    .detail("ConsistencyScanEnabled", csInfo.consistency_scan_enabled)
+		    .detail("MaxRate", csInfo.max_rate)
+		    .detail("Interval", csInfo.target_interval);
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self.db);
+		loop {
+			try {
+				tr->reset();
+				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				wait(ConsistencyScanInfo::setInfo(tr, csInfo));
+				wait(tr->commit());
+				break;
+			} catch (Error& e) {
+				wait(tr->onError(e));
+			}
+		}
+	}
+
+	self.addActor.send(waitFailureServer(csInterf.waitFailure.getFuture()));
+	self.addActor.send(traceRole(Role::CONSISTENCYSCAN, csInterf.id()));
+	self.addActor.send(watchConsistencyScanInfoKey(&self));
+
+	loop {
+		if (self.consistencyScanEnabled.get()) {
+			try {
+				loop choose {
+					when(wait(runDataValidationCheck(&self))) {
+						TraceEvent("ConsistencyScan_Done", csInterf.id()).log();
+						return Void();
+					}
+					when(HaltConsistencyScanRequest req = waitNext(csInterf.haltConsistencyScan.getFuture())) {
+						req.reply.send(Void());
+						TraceEvent("ConsistencyScan_Halted", csInterf.id()).detail("ReqID", req.requesterID);
+						break;
+					}
+					when(wait(err.getFuture())) {}
+					when(wait(collection)) {
+						ASSERT(false);
+						throw internal_error();
+					}
+				}
+			} catch (Error& err) {
+				if (err.code() == error_code_actor_cancelled) {
+					TraceEvent("ConsistencyScan_ActorCanceled", csInterf.id()).errorUnsuppressed(err);
+					return Void();
+				}
+				TraceEvent("ConsistencyScan_Died", csInterf.id()).errorUnsuppressed(err);
+			}
+		} else {
+			TraceEvent("ConsistencyScan_WaitingForConfigChange", self.id).log();
+			wait(self.consistencyScanEnabled.onChange());
+		}
+	}
+}
\ No newline at end of file
diff --git a/fdbserver/CoordinatedState.actor.cpp b/fdbserver/CoordinatedState.actor.cpp
index 72e75bf3be..b8852fe80a 100644
--- a/fdbserver/CoordinatedState.actor.cpp
+++ b/fdbserver/CoordinatedState.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
+#include "fdbserver/ConfigBroadcaster.h"
 #include "fdbserver/CoordinatedState.h"
 #include "fdbserver/CoordinationInterface.h"
 #include "fdbserver/Knobs.h"
@@ -79,13 +80,13 @@ struct CoordinatedStateImpl {
 
 	CoordinatedStateImpl(ServerCoordinators const& c)
 	  : coordinators(c), stage(0), conflictGen(0), doomed(false), ac(false), initial(false) {}
-	uint64_t getConflict() { return conflictGen; }
+	uint64_t getConflict() const { return conflictGen; }
 
-	bool isDoomed(GenerationRegReadReply const& rep) {
-		return rep.gen > gen // setExclusive is doomed, because there was a write at least started at a higher
-		                     // generation, which means a read completed at that higher generation
-		                     // || rep.rgen > gen // setExclusive isn't absolutely doomed, but it may/probably will fail
-		    ;
+	bool isDoomed(GenerationRegReadReply const& rep) const {
+		return rep.gen > gen;
+		// setExclusive is doomed, because there was a write at least started at a higher
+		// generation, which means a read completed at that higher generation
+		// || rep.rgen > gen // setExclusive isn't absolutely doomed, but it may/probably will fail
 	}
 
 	ACTOR static Future<Value> read(CoordinatedStateImpl* self) {
@@ -216,7 +217,7 @@ struct CoordinatedStateImpl {
 };
 
 CoordinatedState::CoordinatedState(ServerCoordinators const& coord)
-  : impl(std::make_unique<CoordinatedStateImpl>(coord)) {}
+  : impl(PImpl<CoordinatedStateImpl>::create(coord)) {}
 CoordinatedState::~CoordinatedState() = default;
 Future<Value> CoordinatedState::read() {
 	return CoordinatedStateImpl::read(impl.get());
@@ -227,7 +228,7 @@ Future<Void> CoordinatedState::onConflict() {
 Future<Void> CoordinatedState::setExclusive(Value v) {
 	return CoordinatedStateImpl::setExclusive(impl.get(), v);
 }
-uint64_t CoordinatedState::getConflict() {
+uint64_t CoordinatedState::getConflict() const {
 	return impl->getConflict();
 }
 
@@ -273,7 +274,7 @@ struct MovableCoordinatedStateImpl {
 		// SOMEDAY: If moveState.mode == MovingFrom, read (without locking) old state and assert that it corresponds
 		// with our state and is ReallyTo(coordinators)
 		if (moveState.mode == MovableValue::MaybeTo) {
-			TEST(true); // Maybe moveto state
+			CODE_PROBE(true, "Maybe moveto state");
 			ASSERT(moveState.other.present());
 			wait(self->moveTo(
 			    self, &self->cs, ClusterConnectionString(moveState.other.get().toString()), moveState.value));
@@ -322,7 +323,7 @@ struct MovableCoordinatedStateImpl {
 
 		Value oldQuorumState = wait(cs.read());
 		if (oldQuorumState != self->lastCSValue.get()) {
-			TEST(true); // Quorum change aborted by concurrent write to old coordination state
+			CODE_PROBE(true, "Quorum change aborted by concurrent write to old coordination state");
 			TraceEvent("QuorumChangeAbortedByConcurrency").log();
 			throw coordinated_state_conflict();
 		}
@@ -346,7 +347,8 @@ struct MovableCoordinatedStateImpl {
 		// SOMEDAY: If we are worried about someone magically getting the new cluster ID and interfering, do a second
 		// cs.setExclusive( encode( ReallyTo, ... ) )
 		TraceEvent("ChangingQuorum").detail("ConnectionString", nc.toString());
-		wait(changeLeaderCoordinators(self->coordinators, StringRef(nc.toString())));
+		wait(ConfigBroadcaster::lockConfigNodes(self->coordinators) &&
+		     changeLeaderCoordinators(self->coordinators, StringRef(nc.toString())));
 		TraceEvent("ChangedQuorum").detail("ConnectionString", nc.toString());
 		throw coordinators_changed();
 	}
@@ -354,7 +356,7 @@ struct MovableCoordinatedStateImpl {
 
 MovableCoordinatedState& MovableCoordinatedState::operator=(MovableCoordinatedState&&) = default;
 MovableCoordinatedState::MovableCoordinatedState(class ServerCoordinators const& coord)
-  : impl(std::make_unique<MovableCoordinatedStateImpl>(coord)) {}
+  : impl(PImpl<MovableCoordinatedStateImpl>::create(coord)) {}
 MovableCoordinatedState::~MovableCoordinatedState() = default;
 Future<Value> MovableCoordinatedState::read() {
 	return MovableCoordinatedStateImpl::read(impl.get());
@@ -368,3 +370,15 @@ Future<Void> MovableCoordinatedState::setExclusive(Value v) {
 Future<Void> MovableCoordinatedState::move(ClusterConnectionString const& nc) {
 	return MovableCoordinatedStateImpl::move(impl.get(), nc);
 }
+
+Optional<Value> updateCCSInMovableValue(ValueRef movableVal, KeyRef oldClusterKey, KeyRef newClusterKey) {
+	Optional<Value> result;
+	MovableValue moveVal = BinaryReader::fromStringRef<MovableValue>(
+	    movableVal, IncludeVersion(ProtocolVersion::withMovableCoordinatedStateV2()));
+	if (moveVal.other.present() && moveVal.other.get().startsWith(oldClusterKey)) {
+		TraceEvent(SevDebug, "UpdateCCSInMovableValue").detail("OldConnectionString", moveVal.other.get());
+		moveVal.other = moveVal.other.get().removePrefix(oldClusterKey).withPrefix(newClusterKey);
+		result = BinaryWriter::toValue(moveVal, IncludeVersion(ProtocolVersion::withMovableCoordinatedStateV2()));
+	}
+	return result;
+}
diff --git a/fdbserver/Coordination.actor.cpp b/fdbserver/Coordination.actor.cpp
index e25278915f..087ee3af24 100644
--- a/fdbserver/Coordination.actor.cpp
+++ b/fdbserver/Coordination.actor.cpp
@@ -97,17 +97,22 @@ LeaderElectionRegInterface::LeaderElectionRegInterface(INetwork* local) : Client
 	forward.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_FORWARD, TaskPriority::Coordination);
 }
 
-ServerCoordinators::ServerCoordinators(Reference<IClusterConnectionRecord> ccr) : ClientCoordinators(ccr) {
+ServerCoordinators::ServerCoordinators(Reference<IClusterConnectionRecord> ccr, ConfigDBType configDBType)
+  : ClientCoordinators(ccr) {
 	ClusterConnectionString cs = ccr->getConnectionString();
 	for (auto h : cs.hostnames) {
 		leaderElectionServers.emplace_back(h);
 		stateServers.emplace_back(h);
-		configServers.emplace_back(h);
+		if (configDBType != ConfigDBType::DISABLED) {
+			configServers.emplace_back(h);
+		}
 	}
 	for (auto s : cs.coords) {
 		leaderElectionServers.emplace_back(s);
 		stateServers.emplace_back(s);
-		configServers.emplace_back(s);
+		if (configDBType != ConfigDBType::DISABLED) {
+			configServers.emplace_back(s);
+		}
 	}
 }
 
@@ -183,8 +188,8 @@ TEST_CASE("/fdbserver/Coordination/localGenerationReg/simple") {
 	}
 
 	{
-		UniqueGeneration g = wait(
-		    reg.write.getReply(GenerationRegWriteRequest(KeyValueRef(the_key, LiteralStringRef("Value1")), firstGen)));
+		UniqueGeneration g =
+		    wait(reg.write.getReply(GenerationRegWriteRequest(KeyValueRef(the_key, "Value1"_sr), firstGen)));
 		//   (gen1==gen is considered a "successful" write)
 		ASSERT(g == firstGen);
 	}
@@ -193,7 +198,7 @@ TEST_CASE("/fdbserver/Coordination/localGenerationReg/simple") {
 		GenerationRegReadReply r = wait(reg.read.getReply(GenerationRegReadRequest(the_key, UniqueGeneration())));
 		// read(key,gen2) returns (value,gen,rgen).
 		//     There was some earlier or concurrent write(key,value,gen).
-		ASSERT(r.value == LiteralStringRef("Value1"));
+		ASSERT(r.value == "Value1"_sr);
 		ASSERT(r.gen == firstGen);
 		//     There was some earlier or concurrent read(key,rgen).
 		ASSERT(r.rgen == firstGen);
@@ -393,6 +398,7 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 				notify[i].send(newInfo);
 			notify.clear();
 			ClientDBInfo outInfo;
+			outInfo.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
 			outInfo.id = deterministicRandom()->randomUniqueID();
 			outInfo.forward = req.conn.toString();
 			clientData.clientInfo->set(CachedSerialization<ClientDBInfo>(outInfo));
@@ -483,16 +489,16 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 // Generation register values are stored without prefixing in the coordinated state, but always begin with an
 // alphanumeric character (they are always derived from a ClusterConnectionString key). Forwarding values are stored in
 // this range:
-const KeyRangeRef fwdKeys(LiteralStringRef("\xff"
-                                           "fwd"),
-                          LiteralStringRef("\xff"
-                                           "fwe"));
+const KeyRangeRef fwdKeys("\xff"
+                          "fwd"_sr,
+                          "\xff"
+                          "fwe"_sr);
 
 // The time when forwarding was last set is stored in this range:
-const KeyRangeRef fwdTimeKeys(LiteralStringRef("\xff"
-                                               "fwdTime"),
-                              LiteralStringRef("\xff"
-                                               "fwdTimf"));
+const KeyRangeRef fwdTimeKeys("\xff"
+                              "fwdTime"_sr,
+                              "\xff"
+                              "fwdTimf"_sr);
 struct LeaderRegisterCollection {
 	// SOMEDAY: Factor this into a generic tool?  Extend ActorCollection to support removal actions?  What?
 	ActorCollection actors;
@@ -632,6 +638,7 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 			Optional<LeaderInfo> forward = regs.getForward(req.clusterKey);
 			if (forward.present()) {
 				ClientDBInfo info;
+				info.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
 				info.id = deterministicRandom()->randomUniqueID();
 				info.forward = forward.get().serializedInfo;
 				req.reply.send(CachedSerialization<ClientDBInfo>(info));
@@ -723,9 +730,9 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 		}
 		when(ForwardRequest req = waitNext(interf.forward.getFuture())) {
 			Optional<LeaderInfo> forward = regs.getForward(req.key);
-			if (forward.present())
+			if (forward.present()) {
 				req.reply.send(Void());
-			else {
+			} else {
 				StringRef clusterName = ccr->getConnectionString().clusterKeyName();
 				if (!SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT && getClusterDescriptor(req.key).compare(clusterName)) {
 					TraceEvent(SevWarn, "CCRMismatch")
@@ -759,12 +766,14 @@ ACTOR Future<Void> coordinationServer(std::string dataFolder,
 	state Future<Void> configDatabaseServer = Never();
 	TraceEvent("CoordinationServer", myID)
 	    .detail("MyInterfaceAddr", myInterface.read.getEndpoint().getPrimaryAddress())
-	    .detail("Folder", dataFolder);
+	    .detail("Folder", dataFolder)
+	    .detail("ConfigNodeValid", configNode.isValid());
 
 	if (configNode.isValid()) {
 		configTransactionInterface.setupWellKnownEndpoints();
 		configFollowerInterface.setupWellKnownEndpoints();
-		configDatabaseServer = configNode->serve(cbi, configTransactionInterface, configFollowerInterface);
+		configDatabaseServer =
+		    brokenPromiseToNever(configNode->serve(cbi, configTransactionInterface, configFollowerInterface));
 	}
 
 	try {
@@ -776,3 +785,78 @@ ACTOR Future<Void> coordinationServer(std::string dataFolder,
 		throw;
 	}
 }
+
+ACTOR Future<Void> changeClusterDescription(std::string datafolder, KeyRef newClusterKey, KeyRef oldClusterKey) {
+	state UID myID = deterministicRandom()->randomUniqueID();
+	state OnDemandStore store(datafolder, myID, "coordination-");
+	RangeResult res = wait(store->readRange(allKeys));
+	// Context, in coordinators' kv-store
+	// cluster description and the random id are always appear together as the clusterKey
+	// The old cluster key, (call it oldCKey) below can appear in the following scenarios:
+	// 1. oldCKey is a key in the store: the value is a binary format of _GenerationRegVal_ which contains a different
+	// clusterKey(either movedFrom or moveTo)
+	// 2. oldCKey appears in a key for forwarding message:
+	// 		2.1: the prefix is _fwdKeys.begin_: the value is the new connection string
+	//		2.2: the prefix is _fwdTimeKeys.begin_: the value is the time
+	// 3. oldCKey does not appear in any keys but in a value:
+	// 		3.1: it's in the value of a forwarding message(see 2.1)
+	//		3.2: it's inside the value of _GenerationRegVal_ (see 1), which is a cluster connection string.
+	//		it seems that even we do not change it the cluster should still be good, but to be safe we still update it.
+	for (auto& [key, value] : res) {
+		if (key.startsWith(fwdKeys.begin)) {
+			if (key.removePrefix(fwdKeys.begin) == oldClusterKey) {
+				store->clear(singleKeyRange(key));
+				store->set(KeyValueRef(newClusterKey.withPrefix(fwdKeys.begin), value));
+			} else if (value.startsWith(oldClusterKey)) {
+				store->set(KeyValueRef(key, value.removePrefix(oldClusterKey).withPrefix(newClusterKey)));
+			}
+		} else if (key.startsWith(fwdTimeKeys.begin) && key.removePrefix(fwdTimeKeys.begin) == oldClusterKey) {
+			store->clear(singleKeyRange(key));
+			store->set(KeyValueRef(newClusterKey.withPrefix(fwdTimeKeys.begin), value));
+		} else if (key == oldClusterKey) {
+			store->clear(singleKeyRange(key));
+			store->set(KeyValueRef(newClusterKey, value));
+		} else {
+			// parse the value part
+			GenerationRegVal regVal = BinaryReader::fromStringRef<GenerationRegVal>(value, IncludeVersion());
+			if (regVal.val.present()) {
+				Optional<Value> newVal = updateCCSInMovableValue(regVal.val.get(), oldClusterKey, newClusterKey);
+				if (newVal.present()) {
+					regVal.val = newVal.get();
+					store->set(KeyValueRef(
+					    key, BinaryWriter::toValue(regVal, IncludeVersion(ProtocolVersion::withGenerationRegVal()))));
+				}
+			}
+		}
+	}
+	wait(store->commit());
+	return Void();
+}
+
+Future<Void> coordChangeClusterKey(std::string dataFolder, KeyRef newClusterKey, KeyRef oldClusterKey) {
+	TraceEvent(SevInfo, "CoordChangeClusterKey")
+	    .detail("DataFolder", dataFolder)
+	    .detail("NewClusterKey", newClusterKey)
+	    .detail("OldClusterKey", oldClusterKey);
+	std::string absDataFolder = abspath(dataFolder);
+	std::vector<std::string> returnList = platform::listDirectories(absDataFolder);
+	std::vector<Future<Void>> futures;
+	for (const auto& dirEntry : returnList) {
+		if (dirEntry == "." || dirEntry == "..") {
+			continue;
+		}
+		std::string processDir = dataFolder + "/" + dirEntry;
+		TraceEvent(SevInfo, "UpdatingCoordDataForProcess").detail("ProcessDataDir", processDir);
+		std::vector<std::string> returnFiles = platform::listFiles(processDir, "");
+		bool isCoord = false;
+		for (const auto& fileEntry : returnFiles) {
+			if (fileEntry.rfind("coordination-", 0) == 0) {
+				isCoord = true;
+			}
+		}
+		if (!isCoord)
+			continue;
+		futures.push_back(changeClusterDescription(processDir, newClusterKey, oldClusterKey));
+	}
+	return waitForAll(futures);
+}
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DDRelocationQueue.actor.cpp
similarity index 73%
rename from fdbserver/DataDistributionQueue.actor.cpp
rename to fdbserver/DDRelocationQueue.actor.cpp
index 6553132bcb..ad38e64c3c 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DDRelocationQueue.actor.cpp
@@ -29,11 +29,13 @@
 #include "fdbrpc/sim_validation.h"
 #include "fdbclient/SystemData.h"
 #include "fdbserver/DataDistribution.actor.h"
+#include "fdbserver/DDSharedContext.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbserver/MoveKeys.actor.h"
 #include "fdbserver/Knobs.h"
 #include "fdbrpc/simulator.h"
 #include "fdbserver/DDTxnProcessor.h"
+#include "flow/DebugTrace.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 #define WORK_FULL_UTILIZATION 10000 // This is not a knob; it is a fixed point scaling factor!
@@ -41,12 +43,6 @@
 typedef Reference<IDataDistributionTeam> ITeamRef;
 typedef std::pair<ITeamRef, ITeamRef> SrcDestTeamPair;
 
-// FIXME: Always use DataMovementReason to invoke these functions.
-inline bool isDiskRebalancePriority(int priority) {
-	return priority == SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM ||
-	       priority == SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM;
-}
-
 inline bool isDataMovementForDiskBalancing(DataMovementReason reason) {
 	return reason == DataMovementReason::REBALANCE_UNDERUTILIZED_TEAM ||
 	       reason == DataMovementReason::REBALANCE_OVERUTILIZED_TEAM;
@@ -57,16 +53,12 @@ inline bool isDataMovementForReadBalancing(DataMovementReason reason) {
 	       reason == DataMovementReason::REBALANCE_READ_UNDERUTIL_TEAM;
 }
 
-inline bool isMountainChopperPriority(int priority) {
-	return priority == SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM ||
-	       priority == SERVER_KNOBS->PRIORITY_REBALANCE_READ_OVERUTIL_TEAM;
-}
-
 inline bool isDataMovementForMountainChopper(DataMovementReason reason) {
 	return reason == DataMovementReason::REBALANCE_OVERUTILIZED_TEAM ||
 	       reason == DataMovementReason::REBALANCE_READ_OVERUTIL_TEAM;
 }
 
+// FIXME: Always use DataMovementReason to invoke these functions.
 inline bool isValleyFillerPriority(int priority) {
 	return priority == SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM ||
 	       priority == SERVER_KNOBS->PRIORITY_REBALANCE_READ_UNDERUTIL_TEAM;
@@ -77,62 +69,55 @@ inline bool isDataMovementForValleyFiller(DataMovementReason reason) {
 	       reason == DataMovementReason::REBALANCE_READ_UNDERUTIL_TEAM;
 }
 
-int dataMovementPriority(DataMovementReason reason) {
-	int priority;
-	switch (reason) {
-	case DataMovementReason::RECOVER_MOVE:
-		priority = SERVER_KNOBS->PRIORITY_RECOVER_MOVE;
-		break;
-	case DataMovementReason::REBALANCE_UNDERUTILIZED_TEAM:
-		priority = SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM;
-		break;
-	case DataMovementReason::REBALANCE_OVERUTILIZED_TEAM:
-		priority = SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM;
-		break;
-	case DataMovementReason::REBALANCE_READ_OVERUTIL_TEAM:
-		priority = SERVER_KNOBS->PRIORITY_REBALANCE_READ_OVERUTIL_TEAM;
-		break;
-	case DataMovementReason::REBALANCE_READ_UNDERUTIL_TEAM:
-		priority = SERVER_KNOBS->PRIORITY_REBALANCE_READ_UNDERUTIL_TEAM;
-		break;
-	case DataMovementReason::PERPETUAL_STORAGE_WIGGLE:
-		priority = SERVER_KNOBS->PRIORITY_PERPETUAL_STORAGE_WIGGLE;
-		break;
-	case DataMovementReason::TEAM_HEALTHY:
-		priority = SERVER_KNOBS->PRIORITY_TEAM_HEALTHY;
-		break;
-	case DataMovementReason::TEAM_CONTAINS_UNDESIRED_SERVER:
-		priority = SERVER_KNOBS->PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER;
-		break;
-	case DataMovementReason::TEAM_REDUNDANT:
-		priority = SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT;
-		break;
-	case DataMovementReason::MERGE_SHARD:
-		priority = SERVER_KNOBS->PRIORITY_MERGE_SHARD;
-		break;
-	case DataMovementReason::POPULATE_REGION:
-		priority = SERVER_KNOBS->PRIORITY_POPULATE_REGION;
-		break;
-	case DataMovementReason::TEAM_UNHEALTHY:
-		priority = SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY;
-		break;
-	case DataMovementReason::TEAM_2_LEFT:
-		priority = SERVER_KNOBS->PRIORITY_TEAM_2_LEFT;
-		break;
-	case DataMovementReason::TEAM_1_LEFT:
-		priority = SERVER_KNOBS->PRIORITY_TEAM_1_LEFT;
-		break;
-	case DataMovementReason::TEAM_FAILED:
-		priority = SERVER_KNOBS->PRIORITY_TEAM_FAILED;
-		break;
-	case DataMovementReason::TEAM_0_LEFT:
-		priority = SERVER_KNOBS->PRIORITY_TEAM_0_LEFT;
-		break;
-	case DataMovementReason::SPLIT_SHARD:
-		priority = SERVER_KNOBS->PRIORITY_SPLIT_SHARD;
-		break;
+typedef std::map<DataMovementReason, int> DmReasonPriorityMapping;
+typedef std::map<int, DataMovementReason> PriorityDmReasonMapping;
+std::pair<const DmReasonPriorityMapping*, const PriorityDmReasonMapping*> buildPriorityMappings() {
+	static DmReasonPriorityMapping reasonPriority{
+		{ DataMovementReason::INVALID, -1 },
+		{ DataMovementReason::RECOVER_MOVE, SERVER_KNOBS->PRIORITY_RECOVER_MOVE },
+		{ DataMovementReason::REBALANCE_UNDERUTILIZED_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM },
+		{ DataMovementReason::REBALANCE_OVERUTILIZED_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM },
+		{ DataMovementReason::REBALANCE_READ_OVERUTIL_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_READ_OVERUTIL_TEAM },
+		{ DataMovementReason::REBALANCE_READ_UNDERUTIL_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_READ_UNDERUTIL_TEAM },
+		{ DataMovementReason::PERPETUAL_STORAGE_WIGGLE, SERVER_KNOBS->PRIORITY_PERPETUAL_STORAGE_WIGGLE },
+		{ DataMovementReason::TEAM_HEALTHY, SERVER_KNOBS->PRIORITY_TEAM_HEALTHY },
+		{ DataMovementReason::TEAM_CONTAINS_UNDESIRED_SERVER, SERVER_KNOBS->PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER },
+		{ DataMovementReason::TEAM_REDUNDANT, SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT },
+		{ DataMovementReason::MERGE_SHARD, SERVER_KNOBS->PRIORITY_MERGE_SHARD },
+		{ DataMovementReason::POPULATE_REGION, SERVER_KNOBS->PRIORITY_POPULATE_REGION },
+		{ DataMovementReason::TEAM_UNHEALTHY, SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY },
+		{ DataMovementReason::TEAM_2_LEFT, SERVER_KNOBS->PRIORITY_TEAM_2_LEFT },
+		{ DataMovementReason::TEAM_1_LEFT, SERVER_KNOBS->PRIORITY_TEAM_1_LEFT },
+		{ DataMovementReason::TEAM_FAILED, SERVER_KNOBS->PRIORITY_TEAM_FAILED },
+		{ DataMovementReason::TEAM_0_LEFT, SERVER_KNOBS->PRIORITY_TEAM_0_LEFT },
+		{ DataMovementReason::SPLIT_SHARD, SERVER_KNOBS->PRIORITY_SPLIT_SHARD },
+		{ DataMovementReason::ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD,
+		  SERVER_KNOBS->PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD }
+	};
+
+	static PriorityDmReasonMapping priorityReason;
+	if (priorityReason.empty()) { // only build once
+		for (const auto& [r, p] : reasonPriority) {
+			priorityReason[p] = r;
+		}
+		// Don't allow 2 priorities value being the same.
+		if (priorityReason.size() != reasonPriority.size()) {
+			TraceEvent(SevError, "DuplicateDataMovementPriority").log();
+			ASSERT(false);
+		}
 	}
-	return priority;
+
+	return std::make_pair(&reasonPriority, &priorityReason);
+}
+
+int dataMovementPriority(DataMovementReason reason) {
+	auto [reasonPriority, _] = buildPriorityMappings();
+	return reasonPriority->at(reason);
+}
+
+DataMovementReason priorityToDataMovementReason(int priority) {
+	auto [_, priorityReason] = buildPriorityMappings();
+	return priorityReason->at(priority);
 }
 
 struct RelocateData {
@@ -143,7 +128,7 @@ struct RelocateData {
 	RelocateReason reason;
 
 	double startTime;
-	UID randomId;
+	UID randomId; // inherit from RelocateShard.traceId
 	UID dataMoveId;
 	int workFactor;
 	std::vector<UID> src;
@@ -155,17 +140,18 @@ struct RelocateData {
 	std::shared_ptr<DataMove> dataMove;
 
 	RelocateData()
-	  : priority(-1), boundaryPriority(-1), healthPriority(-1), reason(RelocateReason::INVALID), startTime(-1),
+	  : priority(-1), boundaryPriority(-1), healthPriority(-1), reason(RelocateReason::OTHER), startTime(-1),
 	    dataMoveId(anonymousShardId), workFactor(0), wantsNewServers(false), cancellable(false),
 	    interval("QueuedRelocation") {}
 	explicit RelocateData(RelocateShard const& rs)
 	  : keys(rs.keys), priority(rs.priority), boundaryPriority(isBoundaryPriority(rs.priority) ? rs.priority : -1),
 	    healthPriority(isHealthPriority(rs.priority) ? rs.priority : -1), reason(rs.reason), startTime(now()),
-	    randomId(deterministicRandom()->randomUniqueID()), dataMoveId(rs.dataMoveId), workFactor(0),
-	    wantsNewServers(isMountainChopperPriority(rs.priority) || isValleyFillerPriority(rs.priority) ||
-	                    rs.priority == SERVER_KNOBS->PRIORITY_SPLIT_SHARD ||
-	                    rs.priority == SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT),
-	    cancellable(true), interval("QueuedRelocation"), dataMove(rs.dataMove) {
+	    randomId(rs.traceId.isValid() ? rs.traceId : deterministicRandom()->randomUniqueID()),
+	    dataMoveId(rs.dataMoveId), workFactor(0), wantsNewServers(isDataMovementForMountainChopper(rs.moveReason) ||
+	                                                              isDataMovementForValleyFiller(rs.moveReason) ||
+	                                                              rs.moveReason == DataMovementReason::SPLIT_SHARD ||
+	                                                              rs.moveReason == DataMovementReason::TEAM_REDUNDANT),
+	    cancellable(true), interval("QueuedRelocation", randomId), dataMove(rs.dataMove) {
 		if (dataMove != nullptr) {
 			this->src.insert(this->src.end(), dataMove->meta.src.begin(), dataMove->meta.src.end());
 		}
@@ -526,14 +512,14 @@ void complete(RelocateData const& relocation, std::map<UID, Busyness>& busymap,
 }
 
 // Cancells in-flight data moves intersecting with range.
-ACTOR Future<Void> cancelDataMove(struct DDQueueData* self, KeyRange range, const DDEnabledState* ddEnabledState);
+ACTOR Future<Void> cancelDataMove(struct DDQueue* self, KeyRange range, const DDEnabledState* ddEnabledState);
 
-ACTOR Future<Void> dataDistributionRelocator(struct DDQueueData* self,
+ACTOR Future<Void> dataDistributionRelocator(struct DDQueue* self,
                                              RelocateData rd,
                                              Future<Void> prevCleanup,
                                              const DDEnabledState* ddEnabledState);
 
-struct DDQueueData {
+struct DDQueue : public IDDRelocationQueue {
 	struct DDDataMove {
 		DDDataMove() = default;
 		explicit DDDataMove(UID id) : id(id) {}
@@ -544,14 +530,109 @@ struct DDQueueData {
 		Future<Void> cancel;
 	};
 
+	struct ServerCounter {
+		enum CountType : uint8_t { ProposedSource = 0, QueuedSource, LaunchedSource, LaunchedDest, __COUNT };
+
+	private:
+		typedef std::array<int, (int)__COUNT> Item; // one for each CountType
+		typedef std::array<Item, RelocateReason::typeCount()> ReasonItem; // one for each RelocateReason
+
+		std::unordered_map<UID, ReasonItem> counter;
+
+		std::string toString(const Item& item) const {
+			return format("%d %d %d %d", item[0], item[1], item[2], item[3]);
+		}
+
+		void traceReasonItem(TraceEvent* event, const ReasonItem& item) const {
+			for (int i = 0; i < item.size(); ++i) {
+				if (std::accumulate(item[i].cbegin(), item[i].cend(), 0) > 0) {
+					// "PQSD" corresponding to CounterType
+					event->detail(RelocateReason(i).toString() + "PQSD", toString(item[i]));
+				}
+			}
+		}
+
+		bool countNonZero(const ReasonItem& item, CountType type) const {
+			return std::any_of(item.cbegin(), item.cend(), [type](const Item& item) { return item[(int)type] > 0; });
+		}
+
+		void increase(const UID& id, RelocateReason reason, CountType type) {
+			int idx = (int)(reason);
+			// if (idx < 0 || idx >= RelocateReason::typeCount()) {
+			// 	TraceEvent(SevWarnAlways, "ServerCounterDebug").detail("Reason", reason.toString());
+			// }
+			ASSERT(idx >= 0 && idx < RelocateReason::typeCount());
+			counter[id][idx][(int)type] += 1;
+		}
+
+		void summarizeLaunchedServers(decltype(counter.cbegin()) begin,
+		                              decltype(counter.cend()) end,
+		                              TraceEvent* event) const {
+			if (begin == end)
+				return;
+
+			std::string execSrc, execDest;
+			for (; begin != end; ++begin) {
+				if (countNonZero(begin->second, LaunchedSource)) {
+					execSrc += begin->first.shortString() + ",";
+				}
+				if (countNonZero(begin->second, LaunchedDest)) {
+					execDest += begin->first.shortString() + ",";
+				}
+			}
+			event->detail("RemainedLaunchedSources", execSrc).detail("RemainedLaunchedDestinations", execDest);
+		}
+
+	public:
+		void clear() { counter.clear(); }
+
+		int get(const UID& id, RelocateReason reason, CountType type) const {
+			return counter.at(id)[(int)reason][(int)type];
+		}
+
+		void increaseForTeam(const std::vector<UID>& ids, RelocateReason reason, CountType type) {
+			for (auto& id : ids) {
+				increase(id, reason, type);
+			}
+		}
+
+		void traceAll(const UID& debugId = UID()) const {
+			auto it = counter.cbegin();
+			int count = 0;
+			for (; count < SERVER_KNOBS->DD_QUEUE_COUNTER_MAX_LOG && it != counter.cend(); ++count, ++it) {
+				TraceEvent event("DDQueueServerCounter", debugId);
+				event.detail("ServerId", it->first);
+				traceReasonItem(&event, it->second);
+			}
+
+			if (it != counter.cend()) {
+				TraceEvent e(SevWarn, "DDQueueServerCounterTooMany", debugId);
+				e.detail("Servers", size());
+				if (SERVER_KNOBS->DD_QUEUE_COUNTER_SUMMARIZE) {
+					summarizeLaunchedServers(it, counter.cend(), &e);
+					return;
+				}
+			}
+		}
+
+		size_t size() const { return counter.size(); }
+
+		// for random test
+		static CountType randomCountType() {
+			int i = deterministicRandom()->randomInt(0, (int)__COUNT);
+			return (CountType)i;
+		}
+	};
+
 	ActorCollectionNoErrors noErrorActors; // has to be the last one to be destroyed because other Actors may use it.
 	UID distributorId;
 	MoveKeysLock lock;
 	Database cx;
-	std::shared_ptr<IDDTxnProcessor> txnProcessor;
+	Reference<IDDTxnProcessor> txnProcessor;
 
 	std::vector<TeamCollectionInterface> teamCollections;
 	Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure;
+	Reference<PhysicalShardCollection> physicalShardCollection;
 	PromiseStream<Promise<int64_t>> getAverageShardBytes;
 
 	FlowLock startMoveKeysParallelismLock;
@@ -577,6 +658,7 @@ struct DDQueueData {
 	// The last time one server was selected as source team for read rebalance reason. We want to throttle read
 	// rebalance on time bases because the read workload sample update has delay after the previous moving
 	std::map<UID, double> lastAsSource;
+	ServerCounter serverCounter;
 
 	KeyRangeMap<RelocateData> inFlight;
 	// Track all actors that relocates specified keys to a good place; Key: keyRange; Value: actor
@@ -644,20 +726,22 @@ struct DDQueueData {
 		}
 	}
 
-	DDQueueData(UID mid,
-	            MoveKeysLock lock,
-	            Database cx,
-	            std::vector<TeamCollectionInterface> teamCollections,
-	            Reference<ShardsAffectedByTeamFailure> sABTF,
-	            PromiseStream<Promise<int64_t>> getAverageShardBytes,
-	            int teamSize,
-	            int singleRegionTeamSize,
-	            PromiseStream<RelocateShard> output,
-	            FutureStream<RelocateShard> input,
-	            PromiseStream<GetMetricsRequest> getShardMetrics,
-	            PromiseStream<GetTopKMetricsRequest> getTopKMetrics)
-	  : distributorId(mid), lock(lock), cx(cx), txnProcessor(new DDTxnProcessor(cx)), teamCollections(teamCollections),
-	    shardsAffectedByTeamFailure(sABTF), getAverageShardBytes(getAverageShardBytes),
+	DDQueue(UID mid,
+	        MoveKeysLock lock,
+	        Reference<IDDTxnProcessor> db,
+	        std::vector<TeamCollectionInterface> teamCollections,
+	        Reference<ShardsAffectedByTeamFailure> sABTF,
+	        Reference<PhysicalShardCollection> physicalShardCollection,
+	        PromiseStream<Promise<int64_t>> getAverageShardBytes,
+	        int teamSize,
+	        int singleRegionTeamSize,
+	        PromiseStream<RelocateShard> output,
+	        FutureStream<RelocateShard> input,
+	        PromiseStream<GetMetricsRequest> getShardMetrics,
+	        PromiseStream<GetTopKMetricsRequest> getTopKMetrics)
+	  : IDDRelocationQueue(), distributorId(mid), lock(lock), cx(db->context()), txnProcessor(db),
+	    teamCollections(teamCollections), shardsAffectedByTeamFailure(sABTF),
+	    physicalShardCollection(physicalShardCollection), getAverageShardBytes(getAverageShardBytes),
 	    startMoveKeysParallelismLock(SERVER_KNOBS->DD_MOVE_KEYS_PARALLELISM),
 	    finishMoveKeysParallelismLock(SERVER_KNOBS->DD_MOVE_KEYS_PARALLELISM),
 	    cleanUpDataMoveParallelismLock(SERVER_KNOBS->DD_MOVE_KEYS_PARALLELISM),
@@ -667,6 +751,7 @@ struct DDQueueData {
 	    suppressIntervals(0), rawProcessingUnhealthy(new AsyncVar<bool>(false)),
 	    rawProcessingWiggle(new AsyncVar<bool>(false)), unhealthyRelocations(0),
 	    movedKeyServersEventHolder(makeReference<EventCacheHolder>("MovedKeyServers")) {}
+	DDQueue() = default;
 
 	void validate() {
 		if (EXPENSIVE_VALIDATION) {
@@ -812,8 +897,7 @@ struct DDQueueData {
 		}
 	}
 
-	ACTOR static Future<Void> getSourceServersForRange(DDQueueData* self,
-	                                                   Database cx,
+	ACTOR static Future<Void> getSourceServersForRange(DDQueue* self,
 	                                                   RelocateData input,
 	                                                   PromiseStream<RelocateData> output,
 	                                                   Reference<FlowLock> fetchLock) {
@@ -915,11 +999,14 @@ struct DDQueueData {
 
 			if (rrs.src.size() == 0 && (rrs.keys == rd.keys || fetchingSourcesQueue.erase(rrs) > 0)) {
 				rrs.keys = affectedQueuedItems[r];
+				rrs.interval = TraceInterval("QueuedRelocation", rrs.randomId); // inherit the old randomId
+
+				DebugRelocationTraceEvent(rrs.interval.begin(), distributorId)
+				    .detail("KeyBegin", rrs.keys.begin)
+				    .detail("KeyEnd", rrs.keys.end)
+				    .detail("Priority", rrs.priority)
+				    .detail("WantsNewServers", rrs.wantsNewServers);
 
-				rrs.interval = TraceInterval("QueuedRelocation");
-				/*TraceEvent(rrs.interval.begin(), distributorId);
-				  .detail("KeyBegin", rrs.keys.begin).detail("KeyEnd", rrs.keys.end)
-				    .detail("Priority", rrs.priority).detail("WantsNewServers", rrs.wantsNewServers);*/
 				queuedRelocations++;
 				TraceEvent(SevVerbose, "QueuedRelocationsChanged")
 				    .detail("DataMoveID", rrs.dataMoveId)
@@ -929,7 +1016,7 @@ struct DDQueueData {
 
 				fetchingSourcesQueue.insert(rrs);
 				getSourceActors.insert(
-				    rrs.keys, getSourceServersForRange(this, cx, rrs, fetchSourceServersComplete, fetchSourceLock));
+				    rrs.keys, getSourceServersForRange(this, rrs, fetchSourceServersComplete, fetchSourceLock));
 			} else {
 				RelocateData newData(rrs);
 				newData.keys = affectedQueuedItems[r];
@@ -941,11 +1028,15 @@ struct DDQueueData {
 
 					if (serverQueue.erase(rrs) > 0) {
 						if (!foundActiveRelocation) {
-							newData.interval = TraceInterval("QueuedRelocation");
-							/*TraceEvent(newData.interval.begin(), distributorId);
-							  .detail("KeyBegin", newData.keys.begin).detail("KeyEnd", newData.keys.end)
-							    .detail("Priority", newData.priority).detail("WantsNewServers",
-							  newData.wantsNewServers);*/
+							newData.interval =
+							    TraceInterval("QueuedRelocation", rrs.randomId); // inherit the old randomId
+
+							DebugRelocationTraceEvent(newData.interval.begin(), distributorId)
+							    .detail("KeyBegin", newData.keys.begin)
+							    .detail("KeyEnd", newData.keys.end)
+							    .detail("Priority", newData.priority)
+							    .detail("WantsNewServers", newData.wantsNewServers);
+
 							queuedRelocations++;
 							TraceEvent(SevVerbose, "QueuedRelocationsChanged")
 							    .detail("DataMoveID", newData.dataMoveId)
@@ -966,11 +1057,11 @@ struct DDQueueData {
 			}
 		}
 
-		/*TraceEvent("ReceivedRelocateShard", distributorId)
-		  .detail("KeyBegin", rd.keys.begin)
-		  .detail("KeyEnd", rd.keys.end)
+		DebugRelocationTraceEvent("ReceivedRelocateShard", distributorId)
+		    .detail("KeyBegin", rd.keys.begin)
+		    .detail("KeyEnd", rd.keys.end)
 		    .detail("Priority", rd.priority)
-		    .detail("AffectedRanges", affectedQueuedItems.size()); */
+		    .detail("AffectedRanges", affectedQueuedItems.size());
 	}
 
 	void completeSourceFetch(const RelocateData& results) {
@@ -984,6 +1075,7 @@ struct DDQueueData {
 			queue[results.src[i]].insert(results);
 		}
 		updateLastAsSource(results.src);
+		serverCounter.increaseForTeam(results.src, results.reason, ServerCounter::CountType::QueuedSource);
 	}
 
 	void logRelocation(const RelocateData& rd, const char* title) {
@@ -1052,10 +1144,12 @@ struct DDQueueData {
 				if (fetchKeysComplete.count(it->value()) && inFlightActors.liveActorAt(it->range().begin) &&
 				    !rd.keys.contains(it->range()) && it->value().priority >= rd.priority &&
 				    rd.healthPriority < SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY) {
-					/*TraceEvent("OverlappingInFlight", distributorId)
+
+					DebugRelocationTraceEvent("OverlappingInFlight", distributorId)
 					    .detail("KeyBegin", it->value().keys.begin)
 					    .detail("KeyEnd", it->value().keys.end)
-					    .detail("Priority", it->value().priority);*/
+					    .detail("Priority", it->value().priority);
+
 					overlappingInFlight = true;
 					break;
 				}
@@ -1090,8 +1184,8 @@ struct DDQueueData {
 			// because they do not have too much inflight data movement.
 
 			// logRelocation( rd, "LaunchingRelocation" );
+			DebugRelocationTraceEvent(rd.interval.end(), distributorId).detail("Result", "Success");
 
-			//TraceEvent(rd.interval.end(), distributorId).detail("Result","Success");
 			if (!rd.isRestore()) {
 				queuedRelocations--;
 				TraceEvent(SevVerbose, "QueuedRelocationsChanged")
@@ -1107,7 +1201,7 @@ struct DDQueueData {
 			}
 
 			Future<Void> fCleanup =
-			    CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA ? cancelDataMove(this, rd.keys, ddEnabledState) : Void();
+			    SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA ? cancelDataMove(this, rd.keys, ddEnabledState) : Void();
 
 			// If there is a job in flight that wants data relocation which we are about to cancel/modify,
 			//     make sure that we keep the relocation intent for the job that we launch
@@ -1129,14 +1223,18 @@ struct DDQueueData {
 				rrs.keys = ranges[r];
 				if (rd.keys == ranges[r] && rd.isRestore()) {
 					ASSERT(rd.dataMove != nullptr);
-					ASSERT(CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+					ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
 					rrs.dataMoveId = rd.dataMove->meta.id;
 				} else {
 					ASSERT_WE_THINK(!rd.isRestore()); // Restored data move should not overlap.
 					// TODO(psm): The shard id is determined by DD.
 					rrs.dataMove.reset();
-					if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
-						rrs.dataMoveId = deterministicRandom()->randomUniqueID();
+					if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+						if (SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+							rrs.dataMoveId = UID();
+						} else {
+							rrs.dataMoveId = deterministicRandom()->randomUniqueID();
+						}
 					} else {
 						rrs.dataMoveId = anonymousShardId;
 					}
@@ -1195,6 +1293,7 @@ struct DDQueueData {
 
 	// Schedules cancellation of a data move.
 	void enqueueCancelledDataMove(UID dataMoveId, KeyRange range, const DDEnabledState* ddEnabledState) {
+		ASSERT(!txnProcessor->isMocked()); // the mock implementation currently doesn't support data move
 		std::vector<Future<Void>> cleanup;
 		auto f = this->dataMoves.intersectingRanges(range);
 		for (auto it = f.begin(); it != f.end(); ++it) {
@@ -1208,7 +1307,7 @@ struct DDQueueData {
 			}
 		}
 
-		DDQueueData::DDDataMove dataMove(dataMoveId);
+		DDQueue::DDDataMove dataMove(dataMoveId);
 		dataMove.cancel = cleanUpDataMove(
 		    this->cx, dataMoveId, this->lock, &this->cleanUpDataMoveParallelismLock, range, ddEnabledState);
 		this->dataMoves.insert(range, dataMove);
@@ -1216,9 +1315,37 @@ struct DDQueueData {
 		    .detail("DataMoveID", dataMoveId)
 		    .detail("Range", range);
 	}
+
+	Future<Void> periodicalRefreshCounter() {
+		auto f = [this]() {
+			serverCounter.traceAll(distributorId);
+			serverCounter.clear();
+		};
+		return recurring(f, SERVER_KNOBS->DD_QUEUE_COUNTER_REFRESH_INTERVAL);
+	}
+
+	int getUnhealthyRelocationCount() override { return unhealthyRelocations; }
+
+	Future<SrcDestTeamPair> getSrcDestTeams(const int& teamCollectionIndex,
+	                                        const GetTeamRequest& srcReq,
+	                                        const GetTeamRequest& destReq,
+	                                        const int& priority,
+	                                        TraceEvent* traceEvent);
+
+	Future<bool> rebalanceReadLoad(DataMovementReason moveReason,
+	                               Reference<IDataDistributionTeam> sourceTeam,
+	                               Reference<IDataDistributionTeam> destTeam,
+	                               bool primary,
+	                               TraceEvent* traceEvent);
+
+	Future<bool> rebalanceTeams(DataMovementReason moveReason,
+	                            Reference<IDataDistributionTeam const> sourceTeam,
+	                            Reference<IDataDistributionTeam const> destTeam,
+	                            bool primary,
+	                            TraceEvent* traceEvent);
 };
 
-ACTOR Future<Void> cancelDataMove(struct DDQueueData* self, KeyRange range, const DDEnabledState* ddEnabledState) {
+ACTOR Future<Void> cancelDataMove(struct DDQueue* self, KeyRange range, const DDEnabledState* ddEnabledState) {
 	std::vector<Future<Void>> cleanup;
 	auto f = self->dataMoves.intersectingRanges(range);
 	for (auto it = f.begin(); it != f.end(); ++it) {
@@ -1239,7 +1366,7 @@ ACTOR Future<Void> cancelDataMove(struct DDQueueData* self, KeyRange range, cons
 	wait(waitForAll(cleanup));
 	auto ranges = self->dataMoves.getAffectedRangesAfterInsertion(range);
 	if (!ranges.empty()) {
-		self->dataMoves.insert(KeyRangeRef(ranges.front().begin, ranges.back().end), DDQueueData::DDDataMove());
+		self->dataMoves.insert(KeyRangeRef(ranges.front().begin, ranges.back().end), DDQueue::DDDataMove());
 	}
 	return Void();
 }
@@ -1258,12 +1385,12 @@ static std::string destServersString(std::vector<std::pair<Reference<IDataDistri
 
 // This actor relocates the specified keys to a good place.
 // The inFlightActor key range map stores the actor for each RelocateData
-ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
+ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
                                              RelocateData rd,
                                              Future<Void> prevCleanup,
                                              const DDEnabledState* ddEnabledState) {
 	state Promise<Void> errorOut(self->error);
-	state TraceInterval relocateShardInterval("RelocateShard");
+	state TraceInterval relocateShardInterval("RelocateShard", rd.randomId);
 	state PromiseStream<RelocateData> dataTransferComplete(self->dataTransferComplete);
 	state PromiseStream<RelocateData> relocationComplete(self->relocationComplete);
 	state bool signalledTransferComplete = false;
@@ -1279,6 +1406,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 	state std::vector<std::pair<Reference<IDataDistributionTeam>, bool>> bestTeams;
 	state double startTime = now();
 	state std::vector<UID> destIds;
+	state uint64_t debugID = deterministicRandom()->randomUInt64();
 
 	try {
 		if (now() - self->lastInterval < 1.0) {
@@ -1290,7 +1418,6 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 		    .detail("KeyBegin", rd.keys.begin)
 		    .detail("KeyEnd", rd.keys.end)
 		    .detail("Priority", rd.priority)
-		    .detail("RelocationID", relocateShardInterval.pairID)
 		    .detail("SuppressedEventCount", self->suppressIntervals);
 
 		if (relocateShardInterval.severity != SevDebug) {
@@ -1298,7 +1425,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 			self->suppressIntervals = 0;
 		}
 
-		if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+		if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
 			auto inFlightRange = self->inFlight.rangeContaining(rd.keys.begin);
 			ASSERT(inFlightRange.range() == rd.keys);
 			ASSERT(inFlightRange.value().randomId == rd.randomId);
@@ -1318,12 +1445,20 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 					    .detail("Range", kr);
 				}
 			}
-			self->dataMoves.insert(rd.keys, DDQueueData::DDDataMove(rd.dataMoveId));
+			if (rd.isRestore() || !SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+				if (SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+					ASSERT(rd.dataMoveId.isValid());
+				}
+				self->dataMoves.insert(rd.keys, DDQueue::DDDataMove(rd.dataMoveId));
+			}
 		}
 
 		state StorageMetrics metrics =
 		    wait(brokenPromiseToNever(self->getShardMetrics.getReply(GetMetricsRequest(rd.keys))));
 
+		state uint64_t physicalShardIDCandidate = UID().first();
+		state bool forceToUseNewPhysicalShard = false;
+
 		ASSERT(rd.src.size());
 		loop {
 			destOverloadedCount = 0;
@@ -1340,7 +1475,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 				bestTeams.clear();
 				// Get team from teamCollections in different DCs and find the best one
 				while (tciIndex < self->teamCollections.size()) {
-					if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA && rd.isRestore()) {
+					if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && rd.isRestore()) {
 						auto req = GetTeamRequest(tciIndex == 0 ? rd.dataMove->primaryDest : rd.dataMove->remoteDest);
 						Future<std::pair<Optional<Reference<IDataDistributionTeam>>, bool>> fbestTeam =
 						    brokenPromiseToNever(self->teamCollections[tciIndex].getTeam.getReply(req));
@@ -1384,6 +1519,20 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 						req.src = rd.src;
 						req.completeSources = rd.completeSources;
 
+						if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD &&
+						    tciIndex == 1) {
+							ASSERT(physicalShardIDCandidate != UID().first() &&
+							       physicalShardIDCandidate != anonymousShardId.first());
+							Optional<ShardsAffectedByTeamFailure::Team> remoteTeamWithPhysicalShard =
+							    self->physicalShardCollection->tryGetAvailableRemoteTeamWith(
+							        physicalShardIDCandidate, metrics, debugID);
+							if (remoteTeamWithPhysicalShard.present()) {
+								// Exists a remoteTeam in the mapping that has the physicalShardIDCandidate
+								// use the remoteTeam with the physicalShard as the bestTeam
+								req = GetTeamRequest(remoteTeamWithPhysicalShard.get().servers);
+							}
+						}
+
 						// bestTeam.second = false if the bestTeam in the teamCollection (in the DC) does not have any
 						// server that hosts the relocateData. This is possible, for example, in a fearless
 						// configuration when the remote DC is just brought up.
@@ -1415,10 +1564,62 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 							anyWithSource = true;
 						}
 
-						bestTeams.emplace_back(bestTeam.first.get(), bestTeam.second);
+						if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+							// critical to the correctness of team selection by PhysicalShardCollection
+							// tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a primary
+							// team Thus, tryGetAvailableRemoteTeamWith() may select an almost full remote team In this
+							// case, we must re-select a remote team We set foundTeams = false to avoid finishing team
+							// selection Then, forceToUseNewPhysicalShard is set, which enforce to use getTeam to select
+							// a remote team
+							if (tciIndex == 1 && !forceToUseNewPhysicalShard) {
+								bool minAvailableSpaceRatio = bestTeam.first.get()->getMinAvailableSpaceRatio(true);
+								if (minAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) {
+									foundTeams = false;
+									break;
+								}
+							}
+						}
+
+						if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+							bestTeams.emplace_back(bestTeam.first.get(), true);
+							// Always set bestTeams[i].second = true to disable optimization in data move between DCs
+							// for the correctness of PhysicalShardCollection
+							// Currently, enabling the optimization will break the invariant of PhysicalShardCollection
+							// Invariant: once a physical shard is created with a specific set of SSes, this SS set will
+							// never get changed.
+						} else {
+							bestTeams.emplace_back(bestTeam.first.get(), bestTeam.second);
+						}
+
+						// get physicalShardIDCandidate
+						if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD &&
+						    tciIndex == 0) {
+							ASSERT(foundTeams);
+							ShardsAffectedByTeamFailure::Team primaryTeam =
+							    ShardsAffectedByTeamFailure::Team(bestTeams[0].first->getServerIDs(), true);
+							physicalShardIDCandidate =
+							    self->physicalShardCollection->determinePhysicalShardIDGivenPrimaryTeam(
+							        primaryTeam, metrics, forceToUseNewPhysicalShard, debugID);
+							ASSERT(physicalShardIDCandidate != UID().first() &&
+							       physicalShardIDCandidate != anonymousShardId.first());
+						}
 					}
 					tciIndex++;
 				}
+
+				// critical to the correctness of team selection by PhysicalShardCollection
+				// tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a primary team
+				// Thus, tryGetAvailableRemoteTeamWith() may select an unhealthy remote team
+				// In this case, we must re-select a remote team
+				// We set foundTeams = false to avoid finishing team selection
+				// Then, forceToUseNewPhysicalShard is set, which enforce to use getTeam to select a remote team
+				if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD &&
+				    bestTeams.size() > 1 && !forceToUseNewPhysicalShard) {
+					if (!bestTeams[1].first->isHealthy()) {
+						foundTeams = false;
+					}
+				}
+
 				// once we've found healthy candidate teams, make sure they're not overloaded with outstanding moves
 				// already
 				anyDestOverloaded = !canLaunchDest(bestTeams, rd.priority, self->destBusymap);
@@ -1429,7 +1630,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 				}
 
 				if (anyDestOverloaded) {
-					TEST(true); // Destination overloaded throttled move
+					CODE_PROBE(true, "Destination overloaded throttled move");
 					destOverloadedCount++;
 					TraceEvent(destOverloadedCount > 50 ? SevInfo : SevDebug, "DestSSBusy", distributorId)
 					    .suppressFor(1.0)
@@ -1439,9 +1640,14 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 					    .detail("AnyDestOverloaded", anyDestOverloaded)
 					    .detail("NumOfTeamCollections", self->teamCollections.size())
 					    .detail("Servers", destServersString(bestTeams));
+					if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+						if (rd.isRestore() && destOverloadedCount > 50) {
+							throw data_move_dest_team_not_found();
+						}
+					}
 					wait(delay(SERVER_KNOBS->DEST_OVERLOADED_DELAY, TaskPriority::DataDistributionLaunch));
 				} else {
-					TEST(true); // did not find a healthy destination team on the first attempt
+					CODE_PROBE(true, "did not find a healthy destination team on the first attempt");
 					stuckCount++;
 					TraceEvent(stuckCount > 50 ? SevWarnAlways : SevWarn, "BestTeamStuck", distributorId)
 					    .suppressFor(1.0)
@@ -1455,10 +1661,42 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 					}
 					wait(delay(SERVER_KNOBS->BEST_TEAM_STUCK_DELAY, TaskPriority::DataDistributionLaunch));
 				}
+				// When forceToUseNewPhysicalShard = false, we get paired primary team and remote team
+				// However, this may be failed
+				// Any retry triggers to use new physicalShard which enters the normal routine
+				if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+					forceToUseNewPhysicalShard = true;
+				}
 
 				// TODO different trace event + knob for overloaded? Could wait on an async var for done moves
 			}
 
+			if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+				if (!rd.isRestore()) {
+					// when !rd.isRestore(), dataMoveId is just decided as physicalShardIDCandidate
+					// thus, update the physicalShardIDCandidate to related data structures
+					ASSERT(physicalShardIDCandidate != UID().first());
+					rd.dataMoveId = newShardId(physicalShardIDCandidate, AssignEmptyRange::False);
+					auto inFlightRange = self->inFlight.rangeContaining(rd.keys.begin);
+					inFlightRange.value().dataMoveId = rd.dataMoveId;
+					auto f = self->dataMoves.intersectingRanges(rd.keys);
+					for (auto it = f.begin(); it != f.end(); ++it) {
+						KeyRangeRef kr(it->range().begin, it->range().end);
+						const UID mId = it->value().id;
+						if (mId.isValid() && mId != rd.dataMoveId) {
+							TraceEvent("DDRelocatorConflictingDataMoveAfterGetTeam", distributorId)
+							    .detail("CurrentDataMoveID", rd.dataMoveId)
+							    .detail("DataMoveID", mId)
+							    .detail("Range", kr);
+						}
+					}
+					self->dataMoves.insert(rd.keys, DDQueue::DDDataMove(rd.dataMoveId));
+				}
+				ASSERT(rd.dataMoveId.first() != UID().first());
+				auto dataMoveRange = self->dataMoves.rangeContaining(rd.keys.begin);
+				ASSERT(dataMoveRange.value().id == rd.dataMoveId);
+			}
+
 			// set cancellable to false on inFlight's entry for this key range
 			auto inFlightRange = self->inFlight.rangeContaining(rd.keys.begin);
 			ASSERT(inFlightRange.range() == rd.keys);
@@ -1544,22 +1782,26 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 				    .detail("ExtraIds", describe(extraIds));
 			}
 
+			self->serverCounter.increaseForTeam(rd.src, rd.reason, DDQueue::ServerCounter::LaunchedSource);
+			self->serverCounter.increaseForTeam(destIds, rd.reason, DDQueue::ServerCounter::LaunchedDest);
+			self->serverCounter.increaseForTeam(extraIds, rd.reason, DDQueue::ServerCounter::LaunchedDest);
+
 			state Error error = success();
 			state Promise<Void> dataMovementComplete;
 			// Move keys from source to destination by changing the serverKeyList and keyServerList system keys
-			state Future<Void> doMoveKeys = moveKeys(self->cx,
-			                                         rd.dataMoveId,
-			                                         rd.keys,
-			                                         destIds,
-			                                         healthyIds,
-			                                         self->lock,
-			                                         dataMovementComplete,
-			                                         &self->startMoveKeysParallelismLock,
-			                                         &self->finishMoveKeysParallelismLock,
-			                                         self->teamCollections.size() > 1,
-			                                         relocateShardInterval.pairID,
-			                                         ddEnabledState,
-			                                         CancelConflictingDataMoves::False);
+			state Future<Void> doMoveKeys =
+			    self->txnProcessor->moveKeys(MoveKeysParams{ rd.dataMoveId,
+			                                                 rd.keys,
+			                                                 destIds,
+			                                                 healthyIds,
+			                                                 self->lock,
+			                                                 dataMovementComplete,
+			                                                 &self->startMoveKeysParallelismLock,
+			                                                 &self->finishMoveKeysParallelismLock,
+			                                                 self->teamCollections.size() > 1,
+			                                                 relocateShardInterval.pairID,
+			                                                 ddEnabledState,
+			                                                 CancelConflictingDataMoves::False });
 			state Future<Void> pollHealth =
 			    signalledTransferComplete ? Never()
 			                              : delay(SERVER_KNOBS->HEALTH_POLL_TIME, TaskPriority::DataDistributionLaunch);
@@ -1572,26 +1814,26 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 								healthyIds.insert(healthyIds.end(), extraIds.begin(), extraIds.end());
 								extraIds.clear();
 								ASSERT(totalIds == destIds.size()); // Sanity check the destIDs before we move keys
-								doMoveKeys = moveKeys(self->cx,
-								                      rd.dataMoveId,
-								                      rd.keys,
-								                      destIds,
-								                      healthyIds,
-								                      self->lock,
-								                      Promise<Void>(),
-								                      &self->startMoveKeysParallelismLock,
-								                      &self->finishMoveKeysParallelismLock,
-								                      self->teamCollections.size() > 1,
-								                      relocateShardInterval.pairID,
-								                      ddEnabledState,
-								                      CancelConflictingDataMoves::False);
+								doMoveKeys =
+								    self->txnProcessor->moveKeys(MoveKeysParams{ rd.dataMoveId,
+								                                                 rd.keys,
+								                                                 destIds,
+								                                                 healthyIds,
+								                                                 self->lock,
+								                                                 Promise<Void>(),
+								                                                 &self->startMoveKeysParallelismLock,
+								                                                 &self->finishMoveKeysParallelismLock,
+								                                                 self->teamCollections.size() > 1,
+								                                                 relocateShardInterval.pairID,
+								                                                 ddEnabledState,
+								                                                 CancelConflictingDataMoves::False });
 							} else {
 								self->fetchKeysComplete.insert(rd);
-								if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+								if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
 									auto ranges = self->dataMoves.getAffectedRangesAfterInsertion(rd.keys);
 									if (ranges.size() == 1 && static_cast<KeyRange>(ranges[0]) == rd.keys &&
 									    ranges[0].value.id == rd.dataMoveId && !ranges[0].value.cancel.isValid()) {
-										self->dataMoves.insert(rd.keys, DDQueueData::DDDataMove());
+										self->dataMoves.insert(rd.keys, DDQueue::DDDataMove());
 										TraceEvent(SevVerbose, "DequeueDataMoveOnSuccess", self->distributorId)
 										    .detail("DataMoveID", rd.dataMoveId)
 										    .detail("DataMoveRange", rd.keys);
@@ -1639,6 +1881,10 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
 				auto readLoad = metrics.bytesReadPerKSecond;
+				// Note: It’s equal to trigger([healthyDestinations, readLoad], which is a value capture of
+				// healthyDestinations. Have to create a reference to healthyDestinations because in ACTOR the state
+				// variable is actually a member variable, I can’t write trigger([healthyDestinations, readLoad]
+				// directly.
 				auto& destinationRef = healthyDestinations;
 				self->noErrorActors.add(
 				    trigger([destinationRef, readLoad]() mutable { destinationRef.addReadInFlightToTeam(-readLoad); },
@@ -1669,12 +1915,26 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self,
 					self->bytesWritten += metrics.bytes;
 					self->shardsAffectedByTeamFailure->finishMove(rd.keys);
 					relocationComplete.send(rd);
+
+					if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+						// update physical shard collection
+						std::vector<ShardsAffectedByTeamFailure::Team> selectedTeams;
+						for (int i = 0; i < bestTeams.size(); i++) {
+							auto serverIds = bestTeams[i].first->getServerIDs();
+							selectedTeams.push_back(ShardsAffectedByTeamFailure::Team(serverIds, i == 0));
+						}
+						// The update of PhysicalShardToTeams, PhysicalShardInstances, keyRangePhysicalShardIDMap should
+						// be atomic
+						self->physicalShardCollection->updatePhysicalShardCollection(
+						    rd.keys, rd.isRestore(), selectedTeams, rd.dataMoveId.first(), metrics, debugID);
+					}
+
 					return Void();
 				} else {
 					throw error;
 				}
 			} else {
-				TEST(true); // move to removed server
+				CODE_PROBE(true, "move to removed server");
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
 				auto readLoad = metrics.bytesReadPerKSecond;
 				auto& destinationRef = healthyDestinations;
@@ -1734,13 +1994,13 @@ inline double getWorstCpu(const HealthMetrics& metrics, const std::vector<UID>&
 
 // Move the shard with the top K highest read density of sourceTeam's to destTeam if sourceTeam has much more read load
 // than destTeam
-ACTOR Future<bool> rebalanceReadLoad(DDQueueData* self,
-                                     int priority,
+ACTOR Future<bool> rebalanceReadLoad(DDQueue* self,
+                                     DataMovementReason moveReason,
                                      Reference<IDataDistributionTeam> sourceTeam,
                                      Reference<IDataDistributionTeam> destTeam,
                                      bool primary,
                                      TraceEvent* traceEvent) {
-	if (g_network->isSimulated() && g_simulator.speedUpSimulation) {
+	if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
 		traceEvent->detail("CancelingDueToSimulationSpeedup", true);
 		return false;
 	}
@@ -1774,14 +2034,10 @@ ACTOR Future<bool> rebalanceReadLoad(DDQueueData* self,
 		return false;
 	}
 	// randomly choose topK shards
-	int topK = std::min(int(0.1 * shards.size()), SERVER_KNOBS->READ_REBALANCE_SHARD_TOPK);
-	state Future<HealthMetrics> healthMetrics = self->cx->getHealthMetrics(true);
+	int topK = std::max(1, std::min(int(0.1 * shards.size()), SERVER_KNOBS->READ_REBALANCE_SHARD_TOPK));
+	state Future<HealthMetrics> healthMetrics = self->txnProcessor->getHealthMetrics(true);
 	state GetTopKMetricsRequest req(
 	    shards, topK, (srcLoad - destLoad) * SERVER_KNOBS->READ_REBALANCE_MAX_SHARD_FRAC, srcLoad / shards.size());
-	req.comparator = [](const StorageMetrics& a, const StorageMetrics& b) {
-		return a.bytesReadPerKSecond / std::max(a.bytes * 1.0, 1.0) >
-		       b.bytesReadPerKSecond / std::max(b.bytes * 1.0, 1.0);
-	};
 	state GetTopKMetricsReply reply = wait(brokenPromiseToNever(self->getTopKMetrics.getReply(req)));
 	wait(ready(healthMetrics));
 	auto cpu = getWorstCpu(healthMetrics.get(), sourceTeam->getServerIDs());
@@ -1790,32 +2046,32 @@ ACTOR Future<bool> rebalanceReadLoad(DDQueueData* self,
 		return false;
 	}
 
-	auto& metricsList = reply.metrics;
+	auto& metricsList = reply.shardMetrics;
 	// NOTE: randomize is important here since we don't want to always push the same shard into the queue
 	deterministicRandom()->randomShuffle(metricsList);
 	traceEvent->detail("MinReadLoad", reply.minReadLoad).detail("MaxReadLoad", reply.maxReadLoad);
 
-	int chosenIdx = -1;
-	for (int i = 0; i < metricsList.size(); ++i) {
-		if (metricsList[i].keys.present()) {
-			chosenIdx = i;
-			break;
-		}
-	}
-	if (chosenIdx == -1) {
+	if (metricsList.empty()) {
 		traceEvent->detail("SkipReason", "NoEligibleShards");
 		return false;
 	}
 
-	auto& metrics = metricsList[chosenIdx];
+	auto& [shard, metrics] = metricsList[0];
 	traceEvent->detail("ShardReadBandwidth", metrics.bytesReadPerKSecond);
 	//  Verify the shard is still in ShardsAffectedByTeamFailure
 	shards = self->shardsAffectedByTeamFailure->getShardsFor(
 	    ShardsAffectedByTeamFailure::Team(sourceTeam->getServerIDs(), primary));
 	for (int i = 0; i < shards.size(); i++) {
-		if (metrics.keys == shards[i]) {
-			self->output.send(RelocateShard(metrics.keys.get(), priority, RelocateReason::REBALANCE_READ));
-			self->updateLastAsSource(sourceTeam->getServerIDs());
+		if (shard == shards[i]) {
+			UID traceId = deterministicRandom()->randomUniqueID();
+			self->output.send(RelocateShard(shard, moveReason, RelocateReason::REBALANCE_READ, traceId));
+			traceEvent->detail("TraceId", traceId);
+
+			auto serverIds = sourceTeam->getServerIDs();
+			self->updateLastAsSource(serverIds);
+
+			self->serverCounter.increaseForTeam(
+			    serverIds, RelocateReason::REBALANCE_READ, DDQueue::ServerCounter::ProposedSource);
 			return true;
 		}
 	}
@@ -1824,13 +2080,13 @@ ACTOR Future<bool> rebalanceReadLoad(DDQueueData* self,
 }
 
 // Move a random shard from sourceTeam if sourceTeam has much more data than provided destTeam
-ACTOR static Future<bool> rebalanceTeams(DDQueueData* self,
-                                         int priority,
+ACTOR static Future<bool> rebalanceTeams(DDQueue* self,
+                                         DataMovementReason moveReason,
                                          Reference<IDataDistributionTeam const> sourceTeam,
                                          Reference<IDataDistributionTeam const> destTeam,
                                          bool primary,
                                          TraceEvent* traceEvent) {
-	if (g_network->isSimulated() && g_simulator.speedUpSimulation) {
+	if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
 		traceEvent->detail("CancelingDueToSimulationSpeedup", true);
 		return false;
 	}
@@ -1886,7 +2142,12 @@ ACTOR static Future<bool> rebalanceTeams(DDQueueData* self,
 	    ShardsAffectedByTeamFailure::Team(sourceTeam->getServerIDs(), primary));
 	for (int i = 0; i < shards.size(); i++) {
 		if (moveShard == shards[i]) {
-			self->output.send(RelocateShard(moveShard, priority, RelocateReason::REBALANCE_DISK));
+			UID traceId = deterministicRandom()->randomUniqueID();
+			self->output.send(RelocateShard(moveShard, moveReason, RelocateReason::REBALANCE_DISK, traceId));
+			traceEvent->detail("TraceId", traceId);
+
+			self->serverCounter.increaseForTeam(
+			    sourceTeam->getServerIDs(), RelocateReason::REBALANCE_DISK, DDQueue::ServerCounter::ProposedSource);
 			return true;
 		}
 	}
@@ -1895,7 +2156,7 @@ ACTOR static Future<bool> rebalanceTeams(DDQueueData* self,
 	return false;
 }
 
-ACTOR Future<SrcDestTeamPair> getSrcDestTeams(DDQueueData* self,
+ACTOR Future<SrcDestTeamPair> getSrcDestTeams(DDQueue* self,
                                               int teamCollectionIndex,
                                               GetTeamRequest srcReq,
                                               GetTeamRequest destReq,
@@ -1922,16 +2183,60 @@ ACTOR Future<SrcDestTeamPair> getSrcDestTeams(DDQueueData* self,
 	return {};
 }
 
-ACTOR Future<Void> BgDDLoadRebalance(DDQueueData* self, int teamCollectionIndex, DataMovementReason reason) {
-	state int resetCount = SERVER_KNOBS->DD_REBALANCE_RESET_AMOUNT;
-	state Transaction tr(self->cx);
+Future<SrcDestTeamPair> DDQueue::getSrcDestTeams(const int& teamCollectionIndex,
+                                                 const GetTeamRequest& srcReq,
+                                                 const GetTeamRequest& destReq,
+                                                 const int& priority,
+                                                 TraceEvent* traceEvent) {
+	return ::getSrcDestTeams(this, teamCollectionIndex, srcReq, destReq, priority, traceEvent);
+}
+Future<bool> DDQueue::rebalanceReadLoad(DataMovementReason moveReason,
+                                        Reference<IDataDistributionTeam> sourceTeam,
+                                        Reference<IDataDistributionTeam> destTeam,
+                                        bool primary,
+                                        TraceEvent* traceEvent) {
+	return ::rebalanceReadLoad(this, moveReason, sourceTeam, destTeam, primary, traceEvent);
+}
+
+Future<bool> DDQueue::rebalanceTeams(DataMovementReason moveReason,
+                                     Reference<const IDataDistributionTeam> sourceTeam,
+                                     Reference<const IDataDistributionTeam> destTeam,
+                                     bool primary,
+                                     TraceEvent* traceEvent) {
+	return ::rebalanceTeams(this, moveReason, sourceTeam, destTeam, primary, traceEvent);
+}
+
+ACTOR Future<bool> getSkipRebalanceValue(Reference<IDDTxnProcessor> txnProcessor, bool readRebalance) {
+	Optional<Value> val = wait(txnProcessor->readRebalanceDDIgnoreKey());
+
+	if (!val.present())
+		return false;
+
+	bool skipCurrentLoop = false;
+	// NOTE: check special value "" and "on" might written in old version < 7.2
+	if (val.get().size() > 0 && val.get() != "on"_sr) {
+		int ddIgnore = BinaryReader::fromStringRef<uint8_t>(val.get(), Unversioned());
+		if (readRebalance) {
+			skipCurrentLoop = (ddIgnore & DDIgnore::REBALANCE_READ) > 0;
+		} else {
+			skipCurrentLoop = (ddIgnore & DDIgnore::REBALANCE_DISK) > 0;
+		}
+	} else {
+		skipCurrentLoop = true;
+	}
+
+	return skipCurrentLoop;
+}
+
+ACTOR Future<Void> BgDDLoadRebalance(DDQueue* self, int teamCollectionIndex, DataMovementReason reason) {
+	state int resetCount = 0;
 	state double lastRead = 0;
 	state bool skipCurrentLoop = false;
-	state Future<Void> delayF = Never();
 	state const bool readRebalance = isDataMovementForReadBalancing(reason);
-	state const char* eventName =
-	    isDataMovementForMountainChopper(reason) ? "BgDDMountainChopper_New" : "BgDDValleyFiller_New";
+	state const char* eventName = isDataMovementForMountainChopper(reason) ? "BgDDMountainChopper" : "BgDDValleyFiller";
 	state int ddPriority = dataMovementPriority(reason);
+	state double rebalancePollingInterval = 0;
+
 	loop {
 		state bool moved = false;
 		state Reference<IDataDistributionTeam> sourceTeam;
@@ -1940,74 +2245,44 @@ ACTOR Future<Void> BgDDLoadRebalance(DDQueueData* self, int teamCollectionIndex,
 		state GetTeamRequest destReq;
 		state TraceEvent traceEvent(eventName, self->distributorId);
 		traceEvent.suppressFor(5.0)
-		    .detail("PollingInterval", SERVER_KNOBS->BG_REBALANCE_POLLING_INTERVAL)
+		    .detail("PollingInterval", rebalancePollingInterval)
 		    .detail("Rebalance", readRebalance ? "Read" : "Disk");
 
+		// NOTE: the DD throttling relies on DDQueue, so here just trigger the balancer periodically
+		wait(delay(rebalancePollingInterval, TaskPriority::DataDistributionLaunch));
 		try {
-			// NOTE: the DD throttling relies on DDQueue
-			delayF = delay(SERVER_KNOBS->BG_REBALANCE_POLLING_INTERVAL, TaskPriority::DataDistributionLaunch);
 			if ((now() - lastRead) > SERVER_KNOBS->BG_REBALANCE_SWITCH_CHECK_INTERVAL) {
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				Optional<Value> val = wait(tr.get(rebalanceDDIgnoreKey));
+				wait(store(skipCurrentLoop, getSkipRebalanceValue(self->txnProcessor, readRebalance)));
 				lastRead = now();
-				if (!val.present()) {
-					skipCurrentLoop = false;
-				} else {
-					// NOTE: check special value "" and "on" might written in old version < 7.2
-					if (val.get().size() > 0 && val.get() != "on"_sr) {
-						int ddIgnore = BinaryReader::fromStringRef<uint8_t>(val.get(), Unversioned());
-						if (readRebalance) {
-							skipCurrentLoop = (ddIgnore & DDIgnore::REBALANCE_READ) > 0;
-						} else {
-							skipCurrentLoop = (ddIgnore & DDIgnore::REBALANCE_DISK) > 0;
-						}
-					} else {
-						skipCurrentLoop = true;
-					}
-				}
 			}
-
 			traceEvent.detail("Enabled", !skipCurrentLoop);
 
-			wait(delayF);
 			if (skipCurrentLoop) {
-				tr.reset();
+				rebalancePollingInterval =
+				    std::max(rebalancePollingInterval, SERVER_KNOBS->BG_REBALANCE_SWITCH_CHECK_INTERVAL);
 				continue;
+			} else {
+				rebalancePollingInterval = SERVER_KNOBS->BG_REBALANCE_POLLING_INTERVAL;
 			}
 
 			traceEvent.detail("QueuedRelocations", self->priority_relocations[ddPriority]);
 
 			if (self->priority_relocations[ddPriority] < SERVER_KNOBS->DD_REBALANCE_PARALLELISM) {
-				if (isDataMovementForMountainChopper(reason)) {
-					srcReq = GetTeamRequest(WantNewServers::True,
-					                        WantTrueBest::True,
-					                        PreferLowerDiskUtil::False,
-					                        TeamMustHaveShards::True,
-					                        ForReadBalance(readRebalance),
-					                        PreferLowerReadUtil::False);
-					destReq = GetTeamRequest(WantNewServers::True,
-					                         WantTrueBest::False,
-					                         PreferLowerDiskUtil::True,
-					                         TeamMustHaveShards::False,
-					                         ForReadBalance(readRebalance),
-					                         PreferLowerReadUtil::True);
-				} else {
-					srcReq = GetTeamRequest(WantNewServers::True,
-					                        WantTrueBest::False,
-					                        PreferLowerDiskUtil::False,
-					                        TeamMustHaveShards::True,
-					                        ForReadBalance(readRebalance),
-					                        PreferLowerReadUtil::False);
-					destReq = GetTeamRequest(WantNewServers::True,
-					                         WantTrueBest::True,
-					                         PreferLowerDiskUtil::True,
-					                         TeamMustHaveShards::False,
-					                         ForReadBalance(readRebalance),
-					                         PreferLowerReadUtil::True);
-				}
+				bool mcMove = isDataMovementForMountainChopper(reason);
+				srcReq = GetTeamRequest(WantNewServers::True,
+				                        WantTrueBest(mcMove),
+				                        PreferLowerDiskUtil::False,
+				                        TeamMustHaveShards::True,
+				                        ForReadBalance(readRebalance),
+				                        PreferLowerReadUtil::False);
+				destReq = GetTeamRequest(WantNewServers::True,
+				                         WantTrueBest(!mcMove),
+				                         PreferLowerDiskUtil::True,
+				                         TeamMustHaveShards::False,
+				                         ForReadBalance(readRebalance),
+				                         PreferLowerReadUtil::True);
 				state Future<SrcDestTeamPair> getTeamFuture =
-				    getSrcDestTeams(self, teamCollectionIndex, srcReq, destReq, ddPriority, &traceEvent);
+				    self->getSrcDestTeams(teamCollectionIndex, srcReq, destReq, ddPriority, &traceEvent);
 				wait(ready(getTeamFuture));
 				sourceTeam = getTeamFuture.get().first;
 				destTeam = getTeamFuture.get().second;
@@ -2015,9 +2290,9 @@ ACTOR Future<Void> BgDDLoadRebalance(DDQueueData* self, int teamCollectionIndex,
 				// clang-format off
 				if (sourceTeam.isValid() && destTeam.isValid()) {
 					if (readRebalance) {
-						wait(store(moved,rebalanceReadLoad(self, ddPriority, sourceTeam, destTeam, teamCollectionIndex == 0, &traceEvent)));
+						wait(store(moved,self->rebalanceReadLoad( reason, sourceTeam, destTeam, teamCollectionIndex == 0, &traceEvent)));
 					} else {
-						wait(store(moved,rebalanceTeams(self, ddPriority, sourceTeam, destTeam, teamCollectionIndex == 0, &traceEvent)));
+						wait(store(moved,self->rebalanceTeams( reason, sourceTeam, destTeam, teamCollectionIndex == 0, &traceEvent)));
 					}
 				}
 				// clang-format on
@@ -2025,11 +2300,10 @@ ACTOR Future<Void> BgDDLoadRebalance(DDQueueData* self, int teamCollectionIndex,
 			}
 
 			traceEvent.detail("ResetCount", resetCount);
-			tr.reset();
 		} catch (Error& e) {
 			// Log actor_cancelled because it's not legal to suppress an event that's initialized
 			traceEvent.errorUnsuppressed(e);
-			wait(tr.onError(e));
+			throw;
 		}
 
 		traceEvent.detail("Moved", moved);
@@ -2037,204 +2311,7 @@ ACTOR Future<Void> BgDDLoadRebalance(DDQueueData* self, int teamCollectionIndex,
 	}
 }
 
-ACTOR Future<Void> BgDDMountainChopper(DDQueueData* self, int teamCollectionIndex) {
-	state double rebalancePollingInterval = SERVER_KNOBS->BG_REBALANCE_POLLING_INTERVAL;
-	state Transaction tr(self->cx);
-	state double lastRead = 0;
-	state bool skipCurrentLoop = false;
-	loop {
-		state std::pair<Optional<Reference<IDataDistributionTeam>>, bool> randomTeam;
-		state bool moved = false;
-		state TraceEvent traceEvent("BgDDMountainChopper_Old", self->distributorId);
-		traceEvent.suppressFor(5.0).detail("PollingInterval", rebalancePollingInterval).detail("Rebalance", "Disk");
-
-		try {
-			state Future<Void> delayF = delay(rebalancePollingInterval, TaskPriority::DataDistributionLaunch);
-			if ((now() - lastRead) > SERVER_KNOBS->BG_REBALANCE_SWITCH_CHECK_INTERVAL) {
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				Optional<Value> val = wait(tr.get(rebalanceDDIgnoreKey));
-				lastRead = now();
-				if (!val.present()) {
-					// reset loop interval
-					if (skipCurrentLoop) {
-						rebalancePollingInterval = SERVER_KNOBS->BG_REBALANCE_POLLING_INTERVAL;
-					}
-					skipCurrentLoop = false;
-				} else {
-					// NOTE: check special value "" and "on" might written in old version < 7.2
-					if (val.get().size() > 0 && val.get() != "on"_sr) {
-						int ddIgnore = BinaryReader::fromStringRef<uint8_t>(val.get(), Unversioned());
-						skipCurrentLoop = (ddIgnore & DDIgnore::REBALANCE_DISK) > 0;
-					} else {
-						skipCurrentLoop = true;
-					}
-				}
-			}
-
-			traceEvent.detail("Enabled", !skipCurrentLoop);
-
-			wait(delayF);
-			if (skipCurrentLoop) {
-				// set loop interval to avoid busy wait here.
-				rebalancePollingInterval =
-				    std::max(rebalancePollingInterval, SERVER_KNOBS->BG_REBALANCE_SWITCH_CHECK_INTERVAL);
-				tr.reset();
-				continue;
-			}
-
-			traceEvent.detail("QueuedRelocations",
-			                  self->priority_relocations[SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM]);
-			if (self->priority_relocations[SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM] <
-			    SERVER_KNOBS->DD_REBALANCE_PARALLELISM) {
-				std::pair<Optional<Reference<IDataDistributionTeam>>, bool> _randomTeam =
-				    wait(brokenPromiseToNever(self->teamCollections[teamCollectionIndex].getTeam.getReply(
-				        GetTeamRequest(WantNewServers::True,
-				                       WantTrueBest::False,
-				                       PreferLowerDiskUtil::True,
-				                       TeamMustHaveShards::False))));
-				randomTeam = _randomTeam;
-				traceEvent.detail("DestTeam",
-				                  printable(randomTeam.first.map<std::string>(
-				                      [](const Reference<IDataDistributionTeam>& team) { return team->getDesc(); })));
-
-				if (randomTeam.first.present()) {
-					std::pair<Optional<Reference<IDataDistributionTeam>>, bool> loadedTeam =
-					    wait(brokenPromiseToNever(self->teamCollections[teamCollectionIndex].getTeam.getReply(
-					        GetTeamRequest(WantNewServers::True,
-					                       WantTrueBest::True,
-					                       PreferLowerDiskUtil::False,
-					                       TeamMustHaveShards::True))));
-
-					traceEvent.detail(
-					    "SourceTeam",
-					    printable(loadedTeam.first.map<std::string>(
-					        [](const Reference<IDataDistributionTeam>& team) { return team->getDesc(); })));
-
-					if (loadedTeam.first.present()) {
-						bool _moved = wait(rebalanceTeams(self,
-						                                  SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM,
-						                                  loadedTeam.first.get(),
-						                                  randomTeam.first.get(),
-						                                  teamCollectionIndex == 0,
-						                                  &traceEvent));
-						moved = _moved;
-					}
-				}
-			}
-
-			tr.reset();
-		} catch (Error& e) {
-			// Log actor_cancelled because it's not legal to suppress an event that's initialized
-			traceEvent.errorUnsuppressed(e);
-			wait(tr.onError(e));
-		}
-
-		traceEvent.detail("Moved", moved);
-		traceEvent.log();
-	}
-}
-
-ACTOR Future<Void> BgDDValleyFiller(DDQueueData* self, int teamCollectionIndex) {
-	state double rebalancePollingInterval = SERVER_KNOBS->BG_REBALANCE_POLLING_INTERVAL;
-	state Transaction tr(self->cx);
-	state double lastRead = 0;
-	state bool skipCurrentLoop = false;
-
-	loop {
-		state std::pair<Optional<Reference<IDataDistributionTeam>>, bool> randomTeam;
-		state bool moved = false;
-		state TraceEvent traceEvent("BgDDValleyFiller_Old", self->distributorId);
-		traceEvent.suppressFor(5.0).detail("PollingInterval", rebalancePollingInterval).detail("Rebalance", "Disk");
-
-		try {
-			state Future<Void> delayF = delay(rebalancePollingInterval, TaskPriority::DataDistributionLaunch);
-			if ((now() - lastRead) > SERVER_KNOBS->BG_REBALANCE_SWITCH_CHECK_INTERVAL) {
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				Optional<Value> val = wait(tr.get(rebalanceDDIgnoreKey));
-				lastRead = now();
-				if (!val.present()) {
-					// reset loop interval
-					if (skipCurrentLoop) {
-						rebalancePollingInterval = SERVER_KNOBS->BG_REBALANCE_POLLING_INTERVAL;
-					}
-					skipCurrentLoop = false;
-				} else {
-					// NOTE: check special value "" and "on" might written in old version < 7.2
-					if (val.get().size() > 0 && val.get() != "on"_sr) {
-						int ddIgnore = BinaryReader::fromStringRef<uint8_t>(val.get(), Unversioned());
-						skipCurrentLoop = (ddIgnore & DDIgnore::REBALANCE_DISK) > 0;
-					} else {
-						skipCurrentLoop = true;
-					}
-				}
-			}
-
-			traceEvent.detail("Enabled", !skipCurrentLoop);
-
-			wait(delayF);
-			if (skipCurrentLoop) {
-				// set loop interval to avoid busy wait here.
-				rebalancePollingInterval =
-				    std::max(rebalancePollingInterval, SERVER_KNOBS->BG_REBALANCE_SWITCH_CHECK_INTERVAL);
-				tr.reset();
-				continue;
-			}
-
-			traceEvent.detail("QueuedRelocations",
-			                  self->priority_relocations[SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM]);
-			if (self->priority_relocations[SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM] <
-			    SERVER_KNOBS->DD_REBALANCE_PARALLELISM) {
-				std::pair<Optional<Reference<IDataDistributionTeam>>, bool> _randomTeam =
-				    wait(brokenPromiseToNever(self->teamCollections[teamCollectionIndex].getTeam.getReply(
-				        GetTeamRequest(WantNewServers::True,
-				                       WantTrueBest::False,
-				                       PreferLowerDiskUtil::False,
-				                       TeamMustHaveShards::True))));
-				randomTeam = _randomTeam;
-				traceEvent.detail("SourceTeam",
-				                  printable(randomTeam.first.map<std::string>(
-				                      [](const Reference<IDataDistributionTeam>& team) { return team->getDesc(); })));
-
-				if (randomTeam.first.present()) {
-					std::pair<Optional<Reference<IDataDistributionTeam>>, bool> unloadedTeam =
-					    wait(brokenPromiseToNever(self->teamCollections[teamCollectionIndex].getTeam.getReply(
-					        GetTeamRequest(WantNewServers::True,
-					                       WantTrueBest::True,
-					                       PreferLowerDiskUtil::True,
-					                       TeamMustHaveShards::False))));
-
-					traceEvent.detail(
-					    "DestTeam",
-					    printable(unloadedTeam.first.map<std::string>(
-					        [](const Reference<IDataDistributionTeam>& team) { return team->getDesc(); })));
-
-					if (unloadedTeam.first.present()) {
-						bool _moved = wait(rebalanceTeams(self,
-						                                  SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM,
-						                                  randomTeam.first.get(),
-						                                  unloadedTeam.first.get(),
-						                                  teamCollectionIndex == 0,
-						                                  &traceEvent));
-						moved = _moved;
-					}
-				}
-			}
-
-			tr.reset();
-		} catch (Error& e) {
-			// Log actor_cancelled because it's not legal to suppress an event that's initialized
-			traceEvent.errorUnsuppressed(e);
-			wait(tr.onError(e));
-		}
-
-		traceEvent.detail("Moved", moved);
-		traceEvent.log();
-	}
-}
-
-ACTOR Future<Void> dataDistributionQueue(Database cx,
+ACTOR Future<Void> dataDistributionQueue(Reference<IDDTxnProcessor> db,
                                          PromiseStream<RelocateShard> output,
                                          FutureStream<RelocateShard> input,
                                          PromiseStream<GetMetricsRequest> getShardMetrics,
@@ -2243,6 +2320,7 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
                                          Reference<AsyncVar<bool>> processingWiggle,
                                          std::vector<TeamCollectionInterface> teamCollections,
                                          Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
+                                         Reference<PhysicalShardCollection> physicalShardCollection,
                                          MoveKeysLock lock,
                                          PromiseStream<Promise<int64_t>> getAverageShardBytes,
                                          FutureStream<Promise<int>> getUnhealthyRelocationCount,
@@ -2250,41 +2328,40 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
                                          int teamSize,
                                          int singleRegionTeamSize,
                                          const DDEnabledState* ddEnabledState) {
-	state DDQueueData self(distributorId,
-	                       lock,
-	                       cx,
-	                       teamCollections,
-	                       shardsAffectedByTeamFailure,
-	                       getAverageShardBytes,
-	                       teamSize,
-	                       singleRegionTeamSize,
-	                       output,
-	                       input,
-	                       getShardMetrics,
-	                       getTopKMetrics);
+	state DDQueue self(distributorId,
+	                   lock,
+	                   db,
+	                   teamCollections,
+	                   shardsAffectedByTeamFailure,
+	                   physicalShardCollection,
+	                   getAverageShardBytes,
+	                   teamSize,
+	                   singleRegionTeamSize,
+	                   output,
+	                   input,
+	                   getShardMetrics,
+	                   getTopKMetrics);
 	state std::set<UID> serversToLaunchFrom;
 	state KeyRange keysToLaunchFrom;
 	state RelocateData launchData;
 	state Future<Void> recordMetrics = delay(SERVER_KNOBS->DD_QUEUE_LOGGING_INTERVAL);
 
-	state std::vector<Future<Void>> balancingFutures;
+	state std::vector<Future<Void>> ddQueueFutures;
 
 	state PromiseStream<KeyRange> rangesComplete;
 	state Future<Void> launchQueuedWorkTimeout = Never();
 
 	for (int i = 0; i < teamCollections.size(); i++) {
-		// FIXME: Use BgDDLoadBalance for disk rebalance too after DD simulation test proof.
-		// balancingFutures.push_back(BgDDLoadRebalance(&self, i, SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM));
-		// balancingFutures.push_back(BgDDLoadRebalance(&self, i, SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM));
+		ddQueueFutures.push_back(BgDDLoadRebalance(&self, i, DataMovementReason::REBALANCE_OVERUTILIZED_TEAM));
+		ddQueueFutures.push_back(BgDDLoadRebalance(&self, i, DataMovementReason::REBALANCE_UNDERUTILIZED_TEAM));
 		if (SERVER_KNOBS->READ_SAMPLING_ENABLED) {
-			balancingFutures.push_back(BgDDLoadRebalance(&self, i, DataMovementReason::REBALANCE_READ_OVERUTIL_TEAM));
-			balancingFutures.push_back(BgDDLoadRebalance(&self, i, DataMovementReason::REBALANCE_READ_UNDERUTIL_TEAM));
+			ddQueueFutures.push_back(BgDDLoadRebalance(&self, i, DataMovementReason::REBALANCE_READ_OVERUTIL_TEAM));
+			ddQueueFutures.push_back(BgDDLoadRebalance(&self, i, DataMovementReason::REBALANCE_READ_UNDERUTIL_TEAM));
 		}
-		balancingFutures.push_back(BgDDMountainChopper(&self, i));
-		balancingFutures.push_back(BgDDValleyFiller(&self, i));
 	}
-	balancingFutures.push_back(delayedAsyncVar(self.rawProcessingUnhealthy, processingUnhealthy, 0));
-	balancingFutures.push_back(delayedAsyncVar(self.rawProcessingWiggle, processingWiggle, 0));
+	ddQueueFutures.push_back(delayedAsyncVar(self.rawProcessingUnhealthy, processingUnhealthy, 0));
+	ddQueueFutures.push_back(delayedAsyncVar(self.rawProcessingWiggle, processingWiggle, 0));
+	ddQueueFutures.push_back(self.periodicalRefreshCounter());
 
 	try {
 		loop {
@@ -2397,8 +2474,10 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 					                                // key we use here must match the key used in the holder.
 				}
 				when(wait(self.error.getFuture())) {} // Propagate errors from dataDistributionRelocator
-				when(wait(waitForAll(balancingFutures))) {}
-				when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) { r.send(self.unhealthyRelocations); }
+				when(wait(waitForAll(ddQueueFutures))) {}
+				when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) {
+					r.send(self.getUnhealthyRelocationCount());
+				}
 			}
 		}
 	} catch (Error& e) {
@@ -2410,3 +2489,29 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 		throw e;
 	}
 }
+
+ACTOR Future<Void> dataDistributionQueue(Reference<DDSharedContext> context, Database cx);
+
+TEST_CASE("/DataDistribution/DDQueue/ServerCounterTrace") {
+	state double duration = 2.5 * SERVER_KNOBS->DD_QUEUE_COUNTER_REFRESH_INTERVAL;
+	state DDQueue self;
+	state Future<Void> counterFuture = self.periodicalRefreshCounter();
+	state Future<Void> finishFuture = delay(duration);
+	std::cout << "Start trace counter unit test for " << duration << "s ...\n";
+	loop choose {
+		when(wait(counterFuture)) {}
+		when(wait(finishFuture)) { break; }
+		when(wait(delayJittered(2.0))) {
+			std::vector<UID> team(3);
+			for (int i = 0; i < team.size(); ++i) {
+				team[i] = UID(deterministicRandom()->randomInt(1, 400), 0);
+			}
+			auto reason = RelocateReason(deterministicRandom()->randomInt(0, RelocateReason::typeCount()));
+			auto countType = DDQueue::ServerCounter::randomCountType();
+			self.serverCounter.increaseForTeam(team, reason, countType);
+			ASSERT(self.serverCounter.get(team[0], reason, countType));
+		}
+	}
+	std::cout << "Finished.";
+	return Void();
+}
diff --git a/fdbserver/DDShardTracker.actor.cpp b/fdbserver/DDShardTracker.actor.cpp
new file mode 100644
index 0000000000..9150bd1daa
--- /dev/null
+++ b/fdbserver/DDShardTracker.actor.cpp
@@ -0,0 +1,2103 @@
+/*
+ * DataDistributionTracker.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/StorageServerInterface.h"
+#include "fdbrpc/FailureMonitor.h"
+#include "fdbclient/SystemData.h"
+#include "fdbserver/DataDistribution.actor.h"
+#include "fdbserver/DDSharedContext.h"
+#include "fdbserver/TenantCache.h"
+#include "fdbserver/Knobs.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbclient/DatabaseContext.h"
+#include "flow/ActorCollection.h"
+#include "flow/Arena.h"
+#include "flow/CodeProbe.h"
+#include "flow/FastRef.h"
+#include "flow/Trace.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+// The used bandwidth of a shard. The higher the value is, the busier the shard is.
+enum BandwidthStatus { BandwidthStatusLow, BandwidthStatusNormal, BandwidthStatusHigh };
+
+enum ReadBandwidthStatus { ReadBandwidthStatusNormal, ReadBandwidthStatusHigh };
+
+BandwidthStatus getBandwidthStatus(StorageMetrics const& metrics) {
+	if (metrics.bytesPerKSecond > SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC)
+		return BandwidthStatusHigh;
+	else if (metrics.bytesPerKSecond < SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC)
+		return BandwidthStatusLow;
+
+	return BandwidthStatusNormal;
+}
+
+ReadBandwidthStatus getReadBandwidthStatus(StorageMetrics const& metrics) {
+	if (metrics.bytesReadPerKSecond <= SERVER_KNOBS->SHARD_READ_HOT_BANDWIDTH_MIN_PER_KSECONDS ||
+	    metrics.bytesReadPerKSecond <= SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO * metrics.bytes *
+	                                       SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS) {
+		return ReadBandwidthStatusNormal;
+	} else {
+		return ReadBandwidthStatusHigh;
+	}
+}
+
+ACTOR Future<Void> updateMaxShardSize(Reference<AsyncVar<int64_t>> dbSizeEstimate,
+                                      Reference<AsyncVar<Optional<int64_t>>> maxShardSize) {
+	state int64_t lastDbSize = 0;
+	state int64_t granularity = g_network->isSimulated() ? SERVER_KNOBS->DD_SHARD_SIZE_GRANULARITY_SIM
+	                                                     : SERVER_KNOBS->DD_SHARD_SIZE_GRANULARITY;
+	loop {
+		auto sizeDelta = std::abs(dbSizeEstimate->get() - lastDbSize);
+		if (sizeDelta > granularity || !maxShardSize->get().present()) {
+			auto v = getMaxShardSize(dbSizeEstimate->get());
+			maxShardSize->set(v);
+			lastDbSize = dbSizeEstimate->get();
+		}
+		wait(dbSizeEstimate->onChange());
+	}
+}
+
+struct DataDistributionTracker : public IDDShardTracker {
+	Reference<IDDTxnProcessor> db;
+	UID distributorId;
+
+	// At now, the lifetime of shards is guaranteed longer than DataDistributionTracker.
+	KeyRangeMap<ShardTrackedData>* shards = nullptr;
+	ActorCollection sizeChanges;
+
+	int64_t systemSizeEstimate = 0;
+	Reference<AsyncVar<int64_t>> dbSizeEstimate;
+	Reference<AsyncVar<Optional<int64_t>>> maxShardSize;
+	Future<Void> maxShardSizeUpdater;
+
+	// CapacityTracker
+	PromiseStream<RelocateShard> output;
+	Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure;
+
+	// PhysicalShard Tracker
+	Reference<PhysicalShardCollection> physicalShardCollection;
+
+	Promise<Void> readyToStart;
+	Reference<AsyncVar<bool>> anyZeroHealthyTeams;
+
+	// Read hot detection
+	PromiseStream<KeyRange> readHotShard;
+
+	// The reference to trackerCancelled must be extracted by actors,
+	// because by the time (trackerCancelled == true) this memory cannot
+	// be accessed
+	bool* trackerCancelled = nullptr;
+
+	// This class extracts the trackerCancelled reference from a DataDistributionTracker object
+	// Because some actors spawned by the dataDistributionTracker outlive the DataDistributionTracker
+	// object, we must guard against memory errors by using a GetTracker functor to access
+	// the DataDistributionTracker object.
+	class SafeAccessor {
+		bool const& trackerCancelled;
+		DataDistributionTracker& tracker;
+
+	public:
+		SafeAccessor(DataDistributionTracker* tracker)
+		  : trackerCancelled(*tracker->trackerCancelled), tracker(*tracker) {
+			ASSERT(!trackerCancelled);
+		}
+
+		DataDistributionTracker* operator()() {
+			if (trackerCancelled) {
+				CODE_PROBE(true, "Trying to access DataDistributionTracker after tracker has been cancelled");
+				throw dd_tracker_cancelled();
+			}
+			return &tracker;
+		}
+	};
+
+	Optional<Reference<TenantCache>> ddTenantCache;
+
+	DataDistributionTracker() = default;
+
+	DataDistributionTracker(Reference<IDDTxnProcessor> db,
+	                        UID distributorId,
+	                        Promise<Void> const& readyToStart,
+	                        PromiseStream<RelocateShard> const& output,
+	                        Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
+	                        Reference<PhysicalShardCollection> physicalShardCollection,
+	                        Reference<AsyncVar<bool>> anyZeroHealthyTeams,
+	                        KeyRangeMap<ShardTrackedData>* shards,
+	                        bool* trackerCancelled,
+	                        Optional<Reference<TenantCache>> ddTenantCache)
+	  : IDDShardTracker(), db(db), distributorId(distributorId), shards(shards), sizeChanges(false),
+	    systemSizeEstimate(0), dbSizeEstimate(new AsyncVar<int64_t>()), maxShardSize(new AsyncVar<Optional<int64_t>>()),
+	    output(output), shardsAffectedByTeamFailure(shardsAffectedByTeamFailure),
+	    physicalShardCollection(physicalShardCollection), readyToStart(readyToStart),
+	    anyZeroHealthyTeams(anyZeroHealthyTeams), trackerCancelled(trackerCancelled), ddTenantCache(ddTenantCache) {}
+
+	~DataDistributionTracker() override {
+		if (trackerCancelled) {
+			*trackerCancelled = true;
+		}
+		// Cancel all actors so they aren't waiting on sizeChanged broken promise
+		sizeChanges.clear(false);
+	}
+
+	double getAverageShardBytes() override { return maxShardSize->get().get() / 2.0; }
+};
+
+void restartShardTrackers(DataDistributionTracker* self,
+                          KeyRangeRef keys,
+                          Optional<ShardMetrics> startingMetrics = Optional<ShardMetrics>(),
+                          bool whenDDInit = false);
+
+// Gets the permitted size and IO bounds for a shard. A shard that starts at allKeys.begin
+//  (i.e. '') will have a permitted size of 0, since the database can contain no data.
+ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize) {
+	ShardSizeBounds bounds;
+
+	if (shard.begin >= keyServersKeys.begin) {
+		bounds.max.bytes = SERVER_KNOBS->KEY_SERVER_SHARD_BYTES;
+	} else {
+		bounds.max.bytes = maxShardSize;
+	}
+
+	bounds.max.bytesPerKSecond = bounds.max.infinity;
+	bounds.max.iosPerKSecond = bounds.max.infinity;
+	bounds.max.bytesReadPerKSecond = bounds.max.infinity;
+
+	// The first shard can have arbitrarily small size
+	if (shard.begin == allKeys.begin) {
+		bounds.min.bytes = 0;
+	} else {
+		bounds.min.bytes = maxShardSize / SERVER_KNOBS->SHARD_BYTES_RATIO;
+	}
+
+	bounds.min.bytesPerKSecond = 0;
+	bounds.min.iosPerKSecond = 0;
+	bounds.min.bytesReadPerKSecond = 0;
+
+	// The permitted error is 1/3 of the general-case minimum bytes (even in the special case where this is the last
+	// shard)
+	bounds.permittedError.bytes = bounds.max.bytes / SERVER_KNOBS->SHARD_BYTES_RATIO / 3;
+	bounds.permittedError.bytesPerKSecond = bounds.permittedError.infinity;
+	bounds.permittedError.iosPerKSecond = bounds.permittedError.infinity;
+	bounds.permittedError.bytesReadPerKSecond = bounds.permittedError.infinity;
+
+	return bounds;
+}
+
+int64_t getMaxShardSize(double dbSizeEstimate) {
+	return std::min((SERVER_KNOBS->MIN_SHARD_BYTES + (int64_t)std::sqrt(std::max<double>(dbSizeEstimate, 0)) *
+	                                                     SERVER_KNOBS->SHARD_BYTES_PER_SQRT_BYTES) *
+	                    SERVER_KNOBS->SHARD_BYTES_RATIO,
+	                (int64_t)SERVER_KNOBS->MAX_SHARD_BYTES);
+}
+
+ShardSizeBounds calculateShardSizeBounds(const KeyRange& keys,
+                                         const Reference<AsyncVar<Optional<ShardMetrics>>>& shardMetrics,
+                                         const BandwidthStatus& bandwidthStatus,
+                                         PromiseStream<KeyRange> readHotShard) {
+	ShardSizeBounds bounds;
+	if (shardMetrics->get().present()) {
+		auto bytes = shardMetrics->get().get().metrics.bytes;
+		auto readBandwidthStatus = getReadBandwidthStatus(shardMetrics->get().get().metrics);
+
+		bounds.max.bytes = std::max(int64_t(bytes * 1.1), (int64_t)SERVER_KNOBS->MIN_SHARD_BYTES);
+		bounds.min.bytes = std::min(int64_t(bytes * 0.9),
+		                            std::max(int64_t(bytes - (SERVER_KNOBS->MIN_SHARD_BYTES * 0.1)), (int64_t)0));
+		bounds.permittedError.bytes = bytes * 0.1;
+		if (bandwidthStatus == BandwidthStatusNormal) { // Not high or low
+			bounds.max.bytesPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
+			bounds.min.bytesPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
+			bounds.permittedError.bytesPerKSecond = bounds.min.bytesPerKSecond / 4;
+		} else if (bandwidthStatus == BandwidthStatusHigh) { // > 10MB/sec for 100MB shard, proportionally lower
+			                                                 // for smaller shard, > 200KB/sec no matter what
+			bounds.max.bytesPerKSecond = bounds.max.infinity;
+			bounds.min.bytesPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
+			bounds.permittedError.bytesPerKSecond = bounds.min.bytesPerKSecond / 4;
+		} else if (bandwidthStatus == BandwidthStatusLow) { // < 10KB/sec
+			bounds.max.bytesPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
+			bounds.min.bytesPerKSecond = 0;
+			bounds.permittedError.bytesPerKSecond = bounds.max.bytesPerKSecond / 4;
+		} else {
+			ASSERT(false);
+		}
+		// handle read bandwidth status
+		if (readBandwidthStatus == ReadBandwidthStatusNormal) {
+			bounds.max.bytesReadPerKSecond =
+			    std::max((int64_t)(SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO * bytes *
+			                       SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS *
+			                       (1.0 + SERVER_KNOBS->SHARD_MAX_BYTES_READ_PER_KSEC_JITTER)),
+			             SERVER_KNOBS->SHARD_READ_HOT_BANDWIDTH_MIN_PER_KSECONDS);
+			bounds.min.bytesReadPerKSecond = 0;
+			bounds.permittedError.bytesReadPerKSecond = bounds.min.bytesReadPerKSecond / 4;
+		} else if (readBandwidthStatus == ReadBandwidthStatusHigh) {
+			bounds.max.bytesReadPerKSecond = bounds.max.infinity;
+			bounds.min.bytesReadPerKSecond = SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO * bytes *
+			                                 SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS *
+			                                 (1.0 - SERVER_KNOBS->SHARD_MAX_BYTES_READ_PER_KSEC_JITTER);
+			bounds.permittedError.bytesReadPerKSecond = bounds.min.bytesReadPerKSecond / 4;
+			// TraceEvent("RHDTriggerReadHotLoggingForShard")
+			//     .detail("ShardBegin", keys.begin.printable().c_str())
+			//     .detail("ShardEnd", keys.end.printable().c_str());
+			readHotShard.send(keys);
+		} else {
+			ASSERT(false);
+		}
+	} else {
+		bounds.max.bytes = -1;
+		bounds.min.bytes = -1;
+		bounds.permittedError.bytes = -1;
+		bounds.max.bytesPerKSecond = bounds.max.infinity;
+		bounds.min.bytesPerKSecond = 0;
+		bounds.permittedError.bytesPerKSecond = bounds.permittedError.infinity;
+		bounds.max.bytesReadPerKSecond = bounds.max.infinity;
+		bounds.min.bytesReadPerKSecond = 0;
+		bounds.permittedError.bytesReadPerKSecond = bounds.permittedError.infinity;
+	}
+
+	bounds.max.iosPerKSecond = bounds.max.infinity;
+	bounds.min.iosPerKSecond = 0;
+	bounds.permittedError.iosPerKSecond = bounds.permittedError.infinity;
+	return bounds;
+}
+
+ACTOR Future<Void> trackShardMetrics(DataDistributionTracker::SafeAccessor self,
+                                     KeyRange keys,
+                                     Reference<AsyncVar<Optional<ShardMetrics>>> shardMetrics,
+                                     bool whenDDInit) {
+	state BandwidthStatus bandwidthStatus =
+	    shardMetrics->get().present() ? getBandwidthStatus(shardMetrics->get().get().metrics) : BandwidthStatusNormal;
+	state double lastLowBandwidthStartTime =
+	    shardMetrics->get().present() ? shardMetrics->get().get().lastLowBandwidthStartTime : now();
+	state int shardCount = shardMetrics->get().present() ? shardMetrics->get().get().shardCount : 1;
+	state ReadBandwidthStatus readBandwidthStatus = shardMetrics->get().present()
+	                                                    ? getReadBandwidthStatus(shardMetrics->get().get().metrics)
+	                                                    : ReadBandwidthStatusNormal;
+	state bool initWithNewMetrics = whenDDInit;
+	wait(delay(0, TaskPriority::DataDistribution));
+
+	/*TraceEvent("TrackShardMetricsStarting")
+	    .detail("TrackerID", trackerID)
+	    .detail("Keys", keys)
+	    .detail("TrackedBytesInitiallyPresent", shardMetrics->get().present())
+	    .detail("StartingMetrics", shardMetrics->get().present() ? shardMetrics->get().get().metrics.bytes : 0)
+	    .detail("StartingMerges", shardMetrics->get().present() ? shardMetrics->get().get().merges : 0);*/
+
+	try {
+		loop {
+			state ShardSizeBounds bounds;
+			bounds = calculateShardSizeBounds(keys, shardMetrics, bandwidthStatus, self()->readHotShard);
+
+			loop {
+				// metrics.second is the number of key-ranges (i.e., shards) in the 'keys' key-range
+				std::pair<Optional<StorageMetrics>, int> metrics =
+				    wait(self()->db->waitStorageMetrics(keys,
+				                                        bounds.min,
+				                                        bounds.max,
+				                                        bounds.permittedError,
+				                                        CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
+				                                        shardCount));
+				if (metrics.first.present()) {
+					BandwidthStatus newBandwidthStatus = getBandwidthStatus(metrics.first.get());
+					if (newBandwidthStatus == BandwidthStatusLow && bandwidthStatus != BandwidthStatusLow) {
+						lastLowBandwidthStartTime = now();
+					}
+					bandwidthStatus = newBandwidthStatus;
+
+					/*TraceEvent("ShardSizeUpdate")
+					    .detail("Keys", keys)
+					    .detail("UpdatedSize", metrics.metrics.bytes)
+					    .detail("Bandwidth", metrics.metrics.bytesPerKSecond)
+					    .detail("BandwidthStatus", getBandwidthStatus(metrics))
+					    .detail("BytesLower", bounds.min.bytes)
+					    .detail("BytesUpper", bounds.max.bytes)
+					    .detail("BandwidthLower", bounds.min.bytesPerKSecond)
+					    .detail("BandwidthUpper", bounds.max.bytesPerKSecond)
+					    .detail("ShardSizePresent", shardSize->get().present())
+					    .detail("OldShardSize", shardSize->get().present() ? shardSize->get().get().metrics.bytes : 0)
+					    .detail("TrackerID", trackerID);*/
+
+					if (shardMetrics->get().present()) {
+						self()->dbSizeEstimate->set(self()->dbSizeEstimate->get() + metrics.first.get().bytes -
+						                            shardMetrics->get().get().metrics.bytes);
+						if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+							// update physicalShard metrics and return whether the keys needs to move out of
+							// physicalShard
+							const MoveKeyRangeOutPhysicalShard needToMove =
+							    self()->physicalShardCollection->trackPhysicalShard(
+							        keys, metrics.first.get(), shardMetrics->get().get().metrics, initWithNewMetrics);
+							if (needToMove) {
+								// Do we need to update shardsAffectedByTeamFailure here?
+								self()->output.send(
+								    RelocateShard(keys,
+								                  DataMovementReason::ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD,
+								                  RelocateReason::OTHER));
+							}
+							if (initWithNewMetrics) {
+								initWithNewMetrics = false;
+							}
+						}
+						if (keys.begin >= systemKeys.begin) {
+							self()->systemSizeEstimate +=
+							    metrics.first.get().bytes - shardMetrics->get().get().metrics.bytes;
+						}
+					}
+
+					shardMetrics->set(ShardMetrics(metrics.first.get(), lastLowBandwidthStartTime, shardCount));
+					break;
+				} else {
+					shardCount = metrics.second;
+					if (shardMetrics->get().present()) {
+						auto newShardMetrics = shardMetrics->get().get();
+						newShardMetrics.shardCount = shardCount;
+						shardMetrics->set(newShardMetrics);
+					}
+				}
+			}
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled && e.code() != error_code_dd_tracker_cancelled) {
+			// The above loop use Database cx, but those error should only be thrown in a code using transaction.
+			ASSERT(transactionRetryableErrors.count(e.code()) == 0);
+			self()->output.sendError(e); // Propagate failure to dataDistributionTracker
+		}
+		throw e;
+	}
+}
+
+ACTOR Future<Void> readHotDetector(DataDistributionTracker* self) {
+	try {
+		loop {
+			state KeyRange keys = waitNext(self->readHotShard.getFuture());
+			Standalone<VectorRef<ReadHotRangeWithMetrics>> readHotRanges = wait(self->db->getReadHotRanges(keys));
+
+			for (const auto& keyRange : readHotRanges) {
+				TraceEvent("ReadHotRangeLog")
+				    .detail("ReadDensity", keyRange.density)
+				    .detail("ReadBandwidth", keyRange.readBandwidth)
+				    .detail("ReadDensityThreshold", SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO)
+				    .detail("KeyRangeBegin", keyRange.keys.begin)
+				    .detail("KeyRangeEnd", keyRange.keys.end);
+			}
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled) {
+			// Those error should only be thrown in a code using transaction.
+			ASSERT(transactionRetryableErrors.count(e.code()) == 0);
+			self->output.sendError(e); // Propagate failure to dataDistributionTracker
+		}
+		throw e;
+	}
+}
+
+/*
+ACTOR Future<Void> extrapolateShardBytes( Reference<AsyncVar<Optional<int64_t>>> inBytes,
+Reference<AsyncVar<Optional<int64_t>>> outBytes ) { state std::deque< std::pair<double,int64_t> > past; loop { wait(
+inBytes->onChange() ); if( inBytes->get().present() ) { past.emplace_back(now(),inBytes->get().get()); if
+(past.size() < 2) outBytes->set( inBytes->get() ); else { while (past.size() > 1 && past.end()[-1].first -
+past.begin()[1].first > 1.0) past.pop_front(); double rate = std::max(0.0,
+double(past.end()[-1].second-past.begin()[0].second)/(past.end()[-1].first - past.begin()[0].first)); outBytes->set(
+inBytes->get().get() + rate * 10.0 );
+            }
+        }
+    }
+}*/
+
+ACTOR Future<int64_t> getFirstSize(Reference<AsyncVar<Optional<ShardMetrics>>> stats) {
+	loop {
+		if (stats->get().present())
+			return stats->get().get().metrics.bytes;
+		wait(stats->onChange());
+	}
+}
+
+ACTOR Future<Void> changeSizes(DataDistributionTracker* self, KeyRange keys, int64_t oldShardsEndingSize) {
+	state std::vector<Future<int64_t>> sizes;
+	state std::vector<Future<int64_t>> systemSizes;
+	for (auto it : self->shards->intersectingRanges(keys)) {
+		Future<int64_t> thisSize = getFirstSize(it->value().stats);
+		sizes.push_back(thisSize);
+		if (it->range().begin >= systemKeys.begin) {
+			systemSizes.push_back(thisSize);
+		}
+	}
+
+	wait(waitForAll(sizes));
+	wait(yield(TaskPriority::DataDistribution));
+
+	int64_t newShardsStartingSize = 0;
+	for (const auto& size : sizes) {
+		newShardsStartingSize += size.get();
+	}
+
+	int64_t newSystemShardsStartingSize = 0;
+	for (const auto& systemSize : systemSizes) {
+		newSystemShardsStartingSize += systemSize.get();
+	}
+
+	int64_t totalSizeEstimate = self->dbSizeEstimate->get();
+	/*TraceEvent("TrackerChangeSizes")
+	    .detail("TotalSizeEstimate", totalSizeEstimate)
+	    .detail("EndSizeOfOldShards", oldShardsEndingSize)
+	    .detail("StartingSizeOfNewShards", newShardsStartingSize);*/
+	self->dbSizeEstimate->set(totalSizeEstimate + newShardsStartingSize - oldShardsEndingSize);
+	self->systemSizeEstimate += newSystemShardsStartingSize;
+	if (keys.begin >= systemKeys.begin) {
+		self->systemSizeEstimate -= oldShardsEndingSize;
+	}
+	return Void();
+}
+
+struct HasBeenTrueFor : ReferenceCounted<HasBeenTrueFor> {
+	explicit HasBeenTrueFor(const Optional<ShardMetrics>& value) {
+		if (value.present()) {
+			lowBandwidthStartTime = value.get().lastLowBandwidthStartTime;
+			trigger =
+			    delayJittered(std::max(0.0, SERVER_KNOBS->DD_MERGE_COALESCE_DELAY + lowBandwidthStartTime - now()),
+			                  TaskPriority::DataDistributionLow) ||
+			    cleared.getFuture();
+		}
+	}
+
+	Future<Void> set(double lastLowBandwidthStartTime) {
+		if (!trigger.isValid() || lowBandwidthStartTime != lastLowBandwidthStartTime) {
+			cleared = Promise<Void>();
+			trigger =
+			    delayJittered(SERVER_KNOBS->DD_MERGE_COALESCE_DELAY + std::max(lastLowBandwidthStartTime - now(), 0.0),
+			                  TaskPriority::DataDistributionLow) ||
+			    cleared.getFuture();
+
+			lowBandwidthStartTime = lastLowBandwidthStartTime;
+		}
+		return trigger;
+	}
+	void clear() {
+		if (!trigger.isValid()) {
+			return;
+		}
+		trigger = Future<Void>();
+		cleared.send(Void());
+		lowBandwidthStartTime = 0;
+	}
+
+	// True if this->value is true and has been true for this->seconds
+	bool hasBeenTrueForLongEnough() const { return trigger.isValid() && trigger.isReady(); }
+
+private:
+	double lowBandwidthStartTime = 0;
+	Future<Void> trigger;
+	Promise<Void> cleared;
+};
+
+std::string describeSplit(KeyRange keys, Standalone<VectorRef<KeyRef>>& splitKeys) {
+	std::string s;
+	s += "[" + keys.begin.toString() + ", " + keys.end.toString() + ") -> ";
+
+	for (auto& sk : splitKeys) {
+		s += sk.printable() + " ";
+	}
+
+	return s;
+}
+void traceSplit(KeyRange keys, Standalone<VectorRef<KeyRef>>& splitKeys) {
+	auto s = describeSplit(keys, splitKeys);
+	TraceEvent(SevInfo, "ExecutingShardSplit").detail("AtKeys", s);
+}
+
+void executeShardSplit(DataDistributionTracker* self,
+                       KeyRange keys,
+                       Standalone<VectorRef<KeyRef>> splitKeys,
+                       Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
+                       bool relocate,
+                       RelocateReason reason) {
+
+	int numShards = splitKeys.size() - 1;
+	ASSERT(numShards > 1);
+
+	int skipRange = deterministicRandom()->randomInt(0, numShards);
+
+	auto s = describeSplit(keys, splitKeys);
+	TraceEvent(SevInfo, "ExecutingShardSplit").suppressFor(0.5).detail("Splitting", s).detail("NumShards", numShards);
+
+	// The queue can't deal with RelocateShard requests which split an existing shard into three pieces, so
+	// we have to send the unskipped ranges in this order (nibbling in from the edges of the old range)
+	for (int i = 0; i < skipRange; i++)
+		restartShardTrackers(self, KeyRangeRef(splitKeys[i], splitKeys[i + 1]));
+	restartShardTrackers(self, KeyRangeRef(splitKeys[skipRange], splitKeys[skipRange + 1]));
+	for (int i = numShards - 1; i > skipRange; i--)
+		restartShardTrackers(self, KeyRangeRef(splitKeys[i], splitKeys[i + 1]));
+
+	for (int i = 0; i < skipRange; i++) {
+		KeyRangeRef r(splitKeys[i], splitKeys[i + 1]);
+		self->shardsAffectedByTeamFailure->defineShard(r);
+		if (relocate) {
+			self->output.send(RelocateShard(r, DataMovementReason::SPLIT_SHARD, reason));
+		}
+	}
+	for (int i = numShards - 1; i > skipRange; i--) {
+		KeyRangeRef r(splitKeys[i], splitKeys[i + 1]);
+		self->shardsAffectedByTeamFailure->defineShard(r);
+		if (relocate) {
+			self->output.send(RelocateShard(r, DataMovementReason::SPLIT_SHARD, reason));
+		}
+	}
+
+	self->sizeChanges.add(changeSizes(self, keys, shardSize->get().get().metrics.bytes));
+}
+
+struct RangeToSplit {
+	RangeMap<Standalone<StringRef>, ShardTrackedData, KeyRangeRef>::iterator shard;
+	Standalone<VectorRef<KeyRef>> faultLines;
+
+	RangeToSplit(RangeMap<Standalone<StringRef>, ShardTrackedData, KeyRangeRef>::iterator shard,
+	             Standalone<VectorRef<KeyRef>> faultLines)
+	  : shard(shard), faultLines(faultLines) {}
+};
+
+Standalone<VectorRef<KeyRef>> findShardFaultLines(KeyRef shardBegin,
+                                                  KeyRef shardEnd,
+                                                  KeyRef tenantBegin,
+                                                  KeyRef tenantEnd) {
+	Standalone<VectorRef<KeyRef>> faultLines;
+
+	ASSERT((shardBegin < tenantBegin && shardEnd > tenantBegin) || (shardBegin < tenantEnd && shardEnd > tenantEnd));
+
+	faultLines.push_back_deep(faultLines.arena(), shardBegin);
+	if (shardBegin < tenantBegin && shardEnd > tenantBegin) {
+		faultLines.push_back_deep(faultLines.arena(), tenantBegin);
+	}
+	if (shardBegin < tenantEnd && shardEnd > tenantEnd) {
+		faultLines.push_back_deep(faultLines.arena(), tenantEnd);
+	}
+	faultLines.push_back_deep(faultLines.arena(), shardEnd);
+
+	return faultLines;
+}
+
+std::vector<RangeToSplit> findTenantShardBoundaries(KeyRangeMap<ShardTrackedData>* shards, KeyRange tenantKeys) {
+
+	std::vector<RangeToSplit> result;
+	auto shardContainingTenantStart = shards->rangeContaining(tenantKeys.begin);
+	auto shardContainingTenantEnd = shards->rangeContainingKeyBefore(tenantKeys.end);
+
+	// same shard
+	if (shardContainingTenantStart == shardContainingTenantEnd) {
+		// If shard boundaries are not aligned with tenantKeys
+		if (shardContainingTenantStart.begin() != tenantKeys.begin ||
+		    shardContainingTenantStart.end() != tenantKeys.end) {
+
+			CODE_PROBE(true, "Splitting a shard that contains complete tenant key range");
+
+			auto startShardSize = shardContainingTenantStart->value().stats;
+
+			if (startShardSize->get().present()) {
+				auto faultLines = findShardFaultLines(shardContainingTenantStart->begin(),
+				                                      shardContainingTenantStart->end(),
+				                                      tenantKeys.begin,
+				                                      tenantKeys.end);
+				result.emplace_back(shardContainingTenantStart, faultLines);
+			} else {
+				CODE_PROBE(true,
+				           "Shard that contains complete tenant key range not split since shard stats are unavailable");
+			}
+		}
+	} else {
+		auto startShardSize = shardContainingTenantStart->value().stats;
+		auto endShardSize = shardContainingTenantEnd->value().stats;
+
+		CODE_PROBE(true, "Splitting multiple shards that a tenant key range straddles");
+
+		if (startShardSize->get().present() && endShardSize->get().present()) {
+			if (shardContainingTenantStart->begin() != tenantKeys.begin) {
+				auto faultLines = findShardFaultLines(shardContainingTenantStart->begin(),
+				                                      shardContainingTenantStart->end(),
+				                                      tenantKeys.begin,
+				                                      tenantKeys.end);
+				result.emplace_back(shardContainingTenantStart, faultLines);
+			}
+
+			if (shardContainingTenantEnd->end() != tenantKeys.end) {
+				auto faultLines = findShardFaultLines(shardContainingTenantEnd->begin(),
+				                                      shardContainingTenantEnd->end(),
+				                                      tenantKeys.begin,
+				                                      tenantKeys.end);
+				result.emplace_back(shardContainingTenantEnd, faultLines);
+			}
+		} else {
+			CODE_PROBE(true, "Shards that contain tenant key range not split since shard stats are unavailable");
+		}
+	}
+
+	return result;
+}
+
+bool faultLinesMatch(std::vector<RangeToSplit>& ranges, std::vector<std::vector<KeyRef>>& expectedFaultLines) {
+	if (ranges.size() != expectedFaultLines.size()) {
+		return false;
+	}
+
+	for (auto& range : ranges) {
+		KeyRangeRef keys = KeyRangeRef(range.shard->begin(), range.shard->end());
+		traceSplit(keys, range.faultLines);
+	}
+
+	for (int r = 0; r < ranges.size(); r++) {
+		if (ranges[r].faultLines.size() != expectedFaultLines[r].size()) {
+			return false;
+		}
+		for (int fl = 0; fl < ranges[r].faultLines.size(); fl++) {
+			if (ranges[r].faultLines[fl] != expectedFaultLines[r][fl]) {
+				return false;
+			}
+		}
+	}
+
+	return true;
+}
+
+TEST_CASE("/DataDistribution/Tenant/SingleShardSplit") {
+	wait(Future<Void>(Void()));
+	ShardTrackedData data;
+	ShardMetrics sm(StorageMetrics(), now(), 1);
+	data.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+
+	KeyRangeMap<ShardTrackedData> shards;
+
+	KeyRef begin = "a"_sr, end = "f"_sr;
+	KeyRangeRef k(begin, end);
+	shards.insert(k, data);
+
+	KeyRangeRef tenantKeys("b"_sr, "c"_sr);
+
+	data.stats->set(sm);
+
+	std::vector<RangeToSplit> result = findTenantShardBoundaries(&shards, tenantKeys);
+
+	std::vector<std::vector<KeyRef>> expectedFaultLines = { { "a"_sr, "b"_sr, "c"_sr, "f"_sr } };
+	ASSERT(faultLinesMatch(result, expectedFaultLines));
+
+	return Void();
+}
+
+TEST_CASE("/DataDistribution/Tenant/SingleShardTenantAligned") {
+	wait(Future<Void>(Void()));
+	ShardTrackedData data;
+	ShardMetrics sm(StorageMetrics(), now(), 1);
+	data.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+
+	KeyRangeMap<ShardTrackedData> shards;
+
+	KeyRef begin = "a"_sr, end = "f"_sr;
+	KeyRangeRef k(begin, end);
+	shards.insert(k, data);
+
+	KeyRangeRef tenantKeys("a"_sr, "f"_sr);
+
+	data.stats->set(sm);
+
+	std::vector<RangeToSplit> result = findTenantShardBoundaries(&shards, tenantKeys);
+
+	std::vector<std::vector<KeyRef>> expectedFaultLines = {};
+	ASSERT(faultLinesMatch(result, expectedFaultLines));
+
+	return Void();
+}
+
+TEST_CASE("/DataDistribution/Tenant/SingleShardTenantAlignedAtStart") {
+	wait(Future<Void>(Void()));
+	ShardTrackedData data;
+	ShardMetrics sm(StorageMetrics(), now(), 1);
+	data.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+
+	KeyRangeMap<ShardTrackedData> shards;
+
+	KeyRef begin = "a"_sr, end = "f"_sr;
+	KeyRangeRef k(begin, end);
+	shards.insert(k, data);
+
+	KeyRangeRef tenantKeys("a"_sr, "d"_sr);
+
+	data.stats->set(sm);
+
+	std::vector<RangeToSplit> result = findTenantShardBoundaries(&shards, tenantKeys);
+
+	std::vector<std::vector<KeyRef>> expectedFaultLines = { { "a"_sr, "d"_sr, "f"_sr } };
+	ASSERT(faultLinesMatch(result, expectedFaultLines));
+
+	return Void();
+}
+
+TEST_CASE("/DataDistribution/Tenant/SingleShardTenantAlignedAtEnd") {
+	wait(Future<Void>(Void()));
+	ShardTrackedData data;
+	ShardMetrics sm(StorageMetrics(), now(), 1);
+	data.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+
+	KeyRangeMap<ShardTrackedData> shards;
+
+	KeyRef begin = "a"_sr, end = "f"_sr;
+	KeyRangeRef k(begin, end);
+	shards.insert(k, data);
+
+	KeyRangeRef tenantKeys("b"_sr, "f"_sr);
+
+	data.stats->set(sm);
+
+	std::vector<RangeToSplit> result = findTenantShardBoundaries(&shards, tenantKeys);
+
+	std::vector<std::vector<KeyRef>> expectedFaultLines = { { "a"_sr, "b"_sr, "f"_sr } };
+	ASSERT(faultLinesMatch(result, expectedFaultLines));
+
+	return Void();
+}
+
+TEST_CASE("/DataDistribution/Tenant/DoubleShardSplit") {
+	wait(Future<Void>(Void()));
+	ShardTrackedData data1, data2;
+	ShardMetrics sm(StorageMetrics(), now(), 1);
+	data1.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+	data2.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+
+	KeyRangeMap<ShardTrackedData> shards;
+
+	KeyRef begin1 = "a"_sr, end1 = "c"_sr;
+	KeyRef begin2 = "d"_sr, end2 = "f"_sr;
+	KeyRangeRef k1(begin1, end1);
+	KeyRangeRef k2(begin2, end2);
+
+	shards.insert(k1, data1);
+	shards.insert(k2, data2);
+
+	KeyRangeRef tenantKeys("b"_sr, "e"_sr);
+
+	data1.stats->set(sm);
+	data2.stats->set(sm);
+
+	std::vector<RangeToSplit> result = findTenantShardBoundaries(&shards, tenantKeys);
+
+	for (auto& range : result) {
+		KeyRangeRef keys = KeyRangeRef(range.shard->begin(), range.shard->end());
+		traceSplit(keys, range.faultLines);
+	}
+
+	std::vector<std::vector<KeyRef>> expectedFaultLines = { { "a"_sr, "b"_sr, "c"_sr }, { "d"_sr, "e"_sr, "f"_sr } };
+	ASSERT(faultLinesMatch(result, expectedFaultLines));
+
+	return Void();
+}
+
+TEST_CASE("/DataDistribution/Tenant/DoubleShardTenantAlignedAtStart") {
+	wait(Future<Void>(Void()));
+	ShardTrackedData data1, data2;
+	ShardMetrics sm(StorageMetrics(), now(), 1);
+	data1.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+	data2.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+
+	KeyRangeMap<ShardTrackedData> shards;
+
+	KeyRef begin1 = "a"_sr, end1 = "c"_sr;
+	KeyRef begin2 = "d"_sr, end2 = "f"_sr;
+	KeyRangeRef k1(begin1, end1);
+	KeyRangeRef k2(begin2, end2);
+
+	shards.insert(k1, data1);
+	shards.insert(k2, data2);
+
+	KeyRangeRef tenantKeys("a"_sr, "e"_sr);
+
+	data1.stats->set(sm);
+	data2.stats->set(sm);
+
+	std::vector<RangeToSplit> result = findTenantShardBoundaries(&shards, tenantKeys);
+
+	std::vector<std::vector<KeyRef>> expectedFaultLines = { { "d"_sr, "e"_sr, "f"_sr } };
+	ASSERT(faultLinesMatch(result, expectedFaultLines));
+
+	return Void();
+}
+
+TEST_CASE("/DataDistribution/Tenant/DoubleShardTenantAlignedAtEnd") {
+	wait(Future<Void>(Void()));
+	ShardTrackedData data1, data2;
+	ShardMetrics sm(StorageMetrics(), now(), 1);
+	data1.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+	data2.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+
+	KeyRangeMap<ShardTrackedData> shards;
+
+	KeyRef begin1 = "a"_sr, end1 = "c"_sr;
+	KeyRef begin2 = "d"_sr, end2 = "f"_sr;
+	KeyRangeRef k1(begin1, end1);
+	KeyRangeRef k2(begin2, end2);
+
+	shards.insert(k1, data1);
+	shards.insert(k2, data2);
+
+	KeyRangeRef tenantKeys("b"_sr, "f"_sr);
+
+	data1.stats->set(sm);
+	data2.stats->set(sm);
+
+	std::vector<RangeToSplit> result = findTenantShardBoundaries(&shards, tenantKeys);
+
+	std::vector<std::vector<KeyRef>> expectedFaultLines = { { "a"_sr, "b"_sr, "c"_sr } };
+	ASSERT(faultLinesMatch(result, expectedFaultLines));
+
+	return Void();
+}
+
+ACTOR Future<Void> tenantShardSplitter(DataDistributionTracker* self, KeyRange tenantKeys) {
+	wait(Future<Void>(Void()));
+	std::vector<RangeToSplit> rangesToSplit = findTenantShardBoundaries(self->shards, tenantKeys);
+
+	for (auto& range : rangesToSplit) {
+		KeyRangeRef keys = KeyRangeRef(range.shard->begin(), range.shard->end());
+		traceSplit(keys, range.faultLines);
+		executeShardSplit(self, keys, range.faultLines, range.shard->value().stats, true, RelocateReason::TENANT_SPLIT);
+	}
+
+	return Void();
+}
+
+ACTOR Future<Void> tenantCreationHandling(DataDistributionTracker* self, TenantCacheTenantCreated req) {
+	TraceEvent(SevInfo, "TenantCacheTenantCreated").detail("Begin", req.keys.begin).detail("End", req.keys.end);
+
+	wait(tenantShardSplitter(self, req.keys));
+	req.reply.send(true);
+	return Void();
+}
+
+ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
+                                 KeyRange keys,
+                                 Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
+                                 ShardSizeBounds shardBounds,
+                                 RelocateReason reason) {
+	state StorageMetrics metrics = shardSize->get().get().metrics;
+	state BandwidthStatus bandwidthStatus = getBandwidthStatus(metrics);
+
+	// Split
+	CODE_PROBE(true, "shard to be split");
+
+	StorageMetrics splitMetrics;
+	splitMetrics.bytes = shardBounds.max.bytes / 2;
+	splitMetrics.bytesPerKSecond =
+	    keys.begin >= keyServersKeys.begin ? splitMetrics.infinity : SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
+	splitMetrics.iosPerKSecond = splitMetrics.infinity;
+	splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidth
+
+	state Standalone<VectorRef<KeyRef>> splitKeys =
+	    wait(self->db->splitStorageMetrics(keys, splitMetrics, metrics, SERVER_KNOBS->MIN_SHARD_BYTES));
+	// fprintf(stderr, "split keys:\n");
+	// for( int i = 0; i < splitKeys.size(); i++ ) {
+	//	fprintf(stderr, "   %s\n", printable(splitKeys[i]).c_str());
+	//}
+	int numShards = splitKeys.size() - 1;
+
+	TraceEvent("RelocateShardStartSplit", self->distributorId)
+	    .suppressFor(1.0)
+	    .detail("Begin", keys.begin)
+	    .detail("End", keys.end)
+	    .detail("MaxBytes", shardBounds.max.bytes)
+	    .detail("MetricsBytes", metrics.bytes)
+	    .detail("Bandwidth",
+	            bandwidthStatus == BandwidthStatusHigh     ? "High"
+	            : bandwidthStatus == BandwidthStatusNormal ? "Normal"
+	                                                       : "Low")
+	    .detail("BytesPerKSec", metrics.bytesPerKSecond)
+	    .detail("NumShards", numShards);
+
+	if (numShards > 1) {
+		executeShardSplit(self, keys, splitKeys, shardSize, true, reason);
+	} else {
+		wait(delay(1.0, TaskPriority::DataDistribution)); // In case the reason the split point was off was due to a
+		                                                  // discrepancy between storage servers
+	}
+	return Void();
+}
+
+ACTOR Future<Void> brokenPromiseToReady(Future<Void> f) {
+	try {
+		wait(f);
+	} catch (Error& e) {
+		if (e.code() != error_code_broken_promise) {
+			throw;
+		}
+	}
+	return Void();
+}
+
+static bool shardMergeFeasible(DataDistributionTracker* self, KeyRange const& keys, KeyRangeRef adjRange) {
+	bool honorTenantKeyspaceBoundaries = self->ddTenantCache.present();
+
+	if (!honorTenantKeyspaceBoundaries) {
+		return true;
+	}
+
+	Optional<Reference<TCTenantInfo>> tenantOwningRange = {};
+	Optional<Reference<TCTenantInfo>> tenantOwningAdjRange = {};
+
+	tenantOwningRange = self->ddTenantCache.get()->tenantOwning(keys.begin);
+	tenantOwningAdjRange = self->ddTenantCache.get()->tenantOwning(adjRange.begin);
+
+	if ((tenantOwningRange.present() != tenantOwningAdjRange.present()) ||
+	    (tenantOwningRange.present() && (tenantOwningRange != tenantOwningAdjRange))) {
+		return false;
+	}
+
+	return true;
+}
+
+static bool shardForwardMergeFeasible(DataDistributionTracker* self, KeyRange const& keys, KeyRangeRef nextRange) {
+	if (keys.end == allKeys.end) {
+		return false;
+	}
+
+	return shardMergeFeasible(self, keys, nextRange);
+}
+
+static bool shardBackwardMergeFeasible(DataDistributionTracker* self, KeyRange const& keys, KeyRangeRef prevRange) {
+	if (keys.begin == allKeys.begin) {
+		return false;
+	}
+
+	return shardMergeFeasible(self, keys, prevRange);
+}
+
+Future<Void> shardMerger(DataDistributionTracker* self,
+                         KeyRange const& keys,
+                         Reference<AsyncVar<Optional<ShardMetrics>>> shardSize) {
+	int64_t maxShardSize = self->maxShardSize->get().get();
+
+	auto prevIter = self->shards->rangeContaining(keys.begin);
+	auto nextIter = self->shards->rangeContaining(keys.begin);
+
+	CODE_PROBE(true, "shard to be merged");
+	ASSERT(keys.begin > allKeys.begin);
+
+	// This will merge shards both before and after "this" shard in keyspace.
+	int shardsMerged = 1;
+	bool forwardComplete = false;
+	KeyRangeRef merged;
+
+	StorageMetrics endingStats = shardSize->get().get().metrics;
+	int shardCount = shardSize->get().get().shardCount;
+	double lastLowBandwidthStartTime = shardSize->get().get().lastLowBandwidthStartTime;
+	if (FLOW_KNOBS->DELAY_JITTER_OFFSET * SERVER_KNOBS->DD_MERGE_COALESCE_DELAY >
+	        SERVER_KNOBS->DD_LOW_BANDWIDTH_DELAY &&
+	    now() - lastLowBandwidthStartTime < SERVER_KNOBS->DD_LOW_BANDWIDTH_DELAY) {
+		TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "ShardMergeTooSoon", self->distributorId)
+		    .detail("Keys", keys)
+		    .detail("LastLowBandwidthStartTime", lastLowBandwidthStartTime);
+	}
+
+	int64_t systemBytes = keys.begin >= systemKeys.begin ? shardSize->get().get().metrics.bytes : 0;
+
+	loop {
+		Optional<ShardMetrics> newMetrics;
+		if (!forwardComplete) {
+			if (nextIter->range().end == allKeys.end) {
+				forwardComplete = true;
+				continue;
+			}
+
+			++nextIter;
+			if (!shardForwardMergeFeasible(self, keys, nextIter->range())) {
+				--nextIter;
+				forwardComplete = true;
+				continue;
+			}
+
+			newMetrics = nextIter->value().stats->get();
+
+			// If going forward, give up when the next shard's stats are not yet present, or if the
+			// the shard is already over the merge bounds.
+			if (!newMetrics.present() || shardCount + newMetrics.get().shardCount >= CLIENT_KNOBS->SHARD_COUNT_LIMIT ||
+			    (endingStats.bytes + newMetrics.get().metrics.bytes > maxShardSize)) {
+				--nextIter;
+				forwardComplete = true;
+				continue;
+			}
+		} else {
+			--prevIter;
+			newMetrics = prevIter->value().stats->get();
+
+			if (!shardBackwardMergeFeasible(self, keys, prevIter->range())) {
+				++prevIter;
+				break;
+			}
+
+			// If going backward, stop when the stats are not present or if the shard is already over the merge
+			//  bounds. If this check triggers right away (if we have not merged anything) then return a trigger
+			//  on the previous shard changing "size".
+			if (!newMetrics.present() || shardCount + newMetrics.get().shardCount >= CLIENT_KNOBS->SHARD_COUNT_LIMIT ||
+			    (endingStats.bytes + newMetrics.get().metrics.bytes > maxShardSize)) {
+				if (shardsMerged == 1) {
+					CODE_PROBE(true, "shardMerger cannot merge anything");
+					return brokenPromiseToReady(prevIter->value().stats->onChange());
+				}
+
+				++prevIter;
+				break;
+			}
+		}
+
+		merged = KeyRangeRef(prevIter->range().begin, nextIter->range().end);
+		endingStats += newMetrics.get().metrics;
+		shardCount += newMetrics.get().shardCount;
+		lastLowBandwidthStartTime = newMetrics.get().lastLowBandwidthStartTime;
+		if ((forwardComplete ? prevIter->range().begin : nextIter->range().begin) >= systemKeys.begin) {
+			systemBytes += newMetrics.get().metrics.bytes;
+		}
+		shardsMerged++;
+
+		auto shardBounds = getShardSizeBounds(merged, maxShardSize);
+		// If we just recently get the current shard's metrics (i.e., less than DD_LOW_BANDWIDTH_DELAY ago), it
+		// means the shard's metric may not be stable yet. So we cannot continue merging in this direction.
+		if (endingStats.bytes >= shardBounds.min.bytes || getBandwidthStatus(endingStats) != BandwidthStatusLow ||
+		    now() - lastLowBandwidthStartTime < SERVER_KNOBS->DD_LOW_BANDWIDTH_DELAY ||
+		    shardsMerged >= SERVER_KNOBS->DD_MERGE_LIMIT) {
+			// The merged range is larger than the min bounds so we cannot continue merging in this direction.
+			//  This means that:
+			//  1. If we were going forwards (the starting direction), we roll back the last speculative merge.
+			//      In this direction we do not want to go above this boundary since we will merge at least one in
+			//      the other direction, even when that goes over the bounds.
+			//  2. If we were going backwards we always want to merge one more shard on (to make sure we go over
+			//      the shard min bounds) so we "break" without resetting the merged range.
+			if (forwardComplete)
+				break;
+
+			// If going forward, remove most recently added range
+			endingStats -= newMetrics.get().metrics;
+			shardCount -= newMetrics.get().shardCount;
+			if (nextIter->range().begin >= systemKeys.begin) {
+				systemBytes -= newMetrics.get().metrics.bytes;
+			}
+			shardsMerged--;
+			--nextIter;
+			merged = KeyRangeRef(prevIter->range().begin, nextIter->range().end);
+			forwardComplete = true;
+		}
+	}
+
+	if (shardsMerged == 1) {
+		return brokenPromiseToReady(nextIter->value().stats->onChange());
+	}
+
+	// restarting shard tracker will derefenced values in the shard map, so make a copy
+	KeyRange mergeRange = merged;
+
+	// OldKeys: Shards in the key range are merged as one shard defined by NewKeys;
+	// NewKeys: New key range after shards are merged;
+	// EndingSize: The new merged shard size in bytes;
+	// BatchedMerges: The number of shards merged. Each shard is defined in self->shards;
+	// LastLowBandwidthStartTime: When does a shard's bandwidth status becomes BandwidthStatusLow. If a shard's
+	// status
+	//   becomes BandwidthStatusLow less than DD_LOW_BANDWIDTH_DELAY ago, the merging logic will stop at the shard;
+	// ShardCount: The number of non-splittable shards that are merged. Each shard is defined in self->shards may
+	// have
+	//   more than 1 shards.
+	TraceEvent("RelocateShardMergeMetrics", self->distributorId)
+	    .detail("OldKeys", keys)
+	    .detail("NewKeys", mergeRange)
+	    .detail("EndingSize", endingStats.bytes)
+	    .detail("BatchedMerges", shardsMerged)
+	    .detail("LastLowBandwidthStartTime", lastLowBandwidthStartTime)
+	    .detail("ShardCount", shardCount);
+
+	if (mergeRange.begin < systemKeys.begin) {
+		self->systemSizeEstimate -= systemBytes;
+	}
+	restartShardTrackers(self, mergeRange, ShardMetrics(endingStats, lastLowBandwidthStartTime, shardCount));
+	self->shardsAffectedByTeamFailure->defineShard(mergeRange);
+	self->output.send(RelocateShard(mergeRange, DataMovementReason::MERGE_SHARD, RelocateReason::MERGE_SHARD));
+
+	// We are about to be cancelled by the call to restartShardTrackers
+	return Void();
+}
+
+ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
+                                  KeyRange keys,
+                                  Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
+                                  Reference<HasBeenTrueFor> wantsToMerge) {
+	Future<Void> onChange = shardSize->onChange() || yieldedFuture(self->maxShardSize->onChange());
+
+	// There are the bounds inside of which we are happy with the shard size.
+	// getShardSizeBounds() will allways have shardBounds.min.bytes == 0 for shards that start at allKeys.begin,
+	//  so will will never attempt to merge that shard with the one previous.
+	ShardSizeBounds shardBounds = getShardSizeBounds(keys, self->maxShardSize->get().get());
+	StorageMetrics const& stats = shardSize->get().get().metrics;
+	auto bandwidthStatus = getBandwidthStatus(stats);
+
+	bool sizeSplit = stats.bytes > shardBounds.max.bytes,
+	     writeSplit = bandwidthStatus == BandwidthStatusHigh && keys.begin < keyServersKeys.begin;
+	bool shouldSplit = sizeSplit || writeSplit;
+
+	auto prevIter = self->shards->rangeContaining(keys.begin);
+	if (keys.begin > allKeys.begin)
+		--prevIter;
+
+	auto nextIter = self->shards->rangeContaining(keys.begin);
+	if (keys.end < allKeys.end)
+		++nextIter;
+
+	bool shouldMerge = stats.bytes < shardBounds.min.bytes && bandwidthStatus == BandwidthStatusLow &&
+	                   (shardForwardMergeFeasible(self, keys, nextIter.range()) ||
+	                    shardBackwardMergeFeasible(self, keys, prevIter.range()));
+
+	// Every invocation must set this or clear it
+	if (shouldMerge && !self->anyZeroHealthyTeams->get()) {
+		auto whenLongEnough = wantsToMerge->set(shardSize->get().get().lastLowBandwidthStartTime);
+		if (!wantsToMerge->hasBeenTrueForLongEnough()) {
+			onChange = onChange || whenLongEnough;
+		}
+	} else {
+		wantsToMerge->clear();
+		if (shouldMerge) {
+			onChange = onChange || self->anyZeroHealthyTeams->onChange();
+		}
+	}
+
+	// TraceEvent("EdgeCaseTraceShardEvaluator", self->distributorId)
+	//     .detail("BeginKey", keys.begin.printable())
+	//     .detail("EndKey", keys.end.printable())
+	//     .detail("ShouldSplit", shouldSplit)
+	//     .detail("ShouldMerge", shouldMerge)
+	//     .detail("HasBeenTrueLongEnough", wantsToMerge->hasBeenTrueForLongEnough())
+	//     .detail("CurrentMetrics", stats.toString())
+	//     .detail("ShardBoundsMaxBytes", shardBounds.max.bytes)
+	//     .detail("ShardBoundsMinBytes", shardBounds.min.bytes)
+	//     .detail("WriteBandwitdhStatus", bandwidthStatus)
+	//     .detail("SplitBecauseHighWriteBandWidth", writeSplit ? "Yes" : "No");
+
+	if (!self->anyZeroHealthyTeams->get() && wantsToMerge->hasBeenTrueForLongEnough()) {
+		onChange = onChange || shardMerger(self, keys, shardSize);
+	}
+	if (shouldSplit) {
+		RelocateReason reason = writeSplit ? RelocateReason::WRITE_SPLIT : RelocateReason::SIZE_SPLIT;
+		onChange = onChange || shardSplitter(self, keys, shardSize, shardBounds, reason);
+	}
+
+	wait(onChange);
+	return Void();
+}
+
+ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
+                                KeyRange keys,
+                                Reference<AsyncVar<Optional<ShardMetrics>>> shardSize) {
+	wait(yieldedFuture(self()->readyToStart.getFuture()));
+
+	if (!shardSize->get().present())
+		wait(shardSize->onChange());
+
+	if (!self()->maxShardSize->get().present())
+		wait(yieldedFuture(self()->maxShardSize->onChange()));
+
+	// Since maxShardSize will become present for all shards at once, avoid slow tasks with a short delay
+	wait(delay(0, TaskPriority::DataDistribution));
+
+	// Survives multiple calls to shardEvaluator and keeps merges from happening too quickly.
+	state Reference<HasBeenTrueFor> wantsToMerge(new HasBeenTrueFor(shardSize->get()));
+
+	/*TraceEvent("ShardTracker", self()->distributorId)
+	    .detail("Begin", keys.begin)
+	    .detail("End", keys.end)
+	    .detail("TrackerID", trackerID)
+	    .detail("MaxBytes", self()->maxShardSize->get().get())
+	    .detail("ShardSize", shardSize->get().get().bytes)
+	    .detail("BytesPerKSec", shardSize->get().get().bytesPerKSecond);*/
+
+	try {
+		loop {
+			// Use the current known size to check for (and start) splits and merges.
+			wait(shardEvaluator(self(), keys, shardSize, wantsToMerge));
+
+			// We could have a lot of actors being released from the previous wait at the same time. Immediately
+			// calling delay(0) mitigates the resulting SlowTask
+			wait(delay(0, TaskPriority::DataDistribution));
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled && e.code() != error_code_dd_tracker_cancelled) {
+			self()->output.sendError(e); // Propagate failure to dataDistributionTracker
+		}
+		throw e;
+	}
+}
+
+void restartShardTrackers(DataDistributionTracker* self,
+                          KeyRangeRef keys,
+                          Optional<ShardMetrics> startingMetrics,
+                          bool whenDDInit) {
+	auto ranges = self->shards->getAffectedRangesAfterInsertion(keys, ShardTrackedData());
+	for (int i = 0; i < ranges.size(); i++) {
+		if (!ranges[i].value.trackShard.isValid() && ranges[i].begin != keys.begin) {
+			// When starting, key space will be full of "dummy" default contructed entries.
+			// This should happen when called from trackInitialShards()
+			ASSERT(!self->readyToStart.isSet());
+			continue;
+		}
+
+		auto shardMetrics = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+
+		// For the case where the new tracker will take over at the boundaries of current shard(s)
+		//  we can use the old size if it is available. This will be the case when merging shards.
+		if (startingMetrics.present()) {
+			ASSERT(ranges.size() == 1);
+			/*TraceEvent("ShardTrackerSizePreset", self->distributorId)
+			    .detail("Keys", keys)
+			    .detail("Size", startingMetrics.get().metrics.bytes)
+			    .detail("Merges", startingMetrics.get().merges);*/
+			CODE_PROBE(true, "shardTracker started with trackedBytes already set");
+			shardMetrics->set(startingMetrics);
+		}
+
+		ShardTrackedData data;
+		data.stats = shardMetrics;
+		data.trackShard = shardTracker(DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics);
+		data.trackBytes =
+		    trackShardMetrics(DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics, whenDDInit);
+		self->shards->insert(ranges[i], data);
+	}
+}
+
+ACTOR Future<Void> trackInitialShards(DataDistributionTracker* self, Reference<InitialDataDistribution> initData) {
+	TraceEvent("TrackInitialShards", self->distributorId).detail("InitialShardCount", initData->shards.size());
+
+	// This line reduces the priority of shard initialization to prevent interference with failure monitoring.
+	// SOMEDAY: Figure out what this priority should actually be
+	wait(delay(0.0, TaskPriority::DataDistribution));
+
+	state int s;
+	for (s = 0; s < initData->shards.size() - 1; s++) {
+		restartShardTrackers(
+		    self, KeyRangeRef(initData->shards[s].key, initData->shards[s + 1].key), Optional<ShardMetrics>(), true);
+		wait(yield(TaskPriority::DataDistribution));
+	}
+
+	Future<Void> initialSize = changeSizes(self, KeyRangeRef(allKeys.begin, allKeys.end), 0);
+	self->readyToStart.send(Void());
+	wait(initialSize);
+	self->maxShardSizeUpdater = updateMaxShardSize(self->dbSizeEstimate, self->maxShardSize);
+
+	return Void();
+}
+
+ACTOR Future<Void> fetchTopKShardMetrics_impl(DataDistributionTracker* self, GetTopKMetricsRequest req) {
+	state Future<Void> onChange;
+	state std::vector<GetTopKMetricsReply::KeyRangeStorageMetrics> returnMetrics;
+	// random pick a portion of shard
+	if (req.keys.size() > SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT) {
+		deterministicRandom()->randomShuffle(req.keys, SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT);
+	}
+	try {
+		loop {
+			onChange = Future<Void>();
+			returnMetrics.clear();
+			state int64_t minReadLoad = -1;
+			state int64_t maxReadLoad = -1;
+			state int i;
+			for (i = 0; i < SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT && i < req.keys.size(); ++i) {
+				auto range = req.keys[i];
+				StorageMetrics metrics;
+				for (auto t : self->shards->intersectingRanges(range)) {
+					auto& stats = t.value().stats;
+					if (!stats->get().present()) {
+						onChange = stats->onChange();
+						break;
+					}
+					metrics += t.value().stats->get().get().metrics;
+				}
+
+				// skip if current stats is invalid
+				if (onChange.isValid()) {
+					break;
+				}
+
+				if (metrics.bytesReadPerKSecond > 0) {
+					if (minReadLoad == -1) {
+						minReadLoad = metrics.bytesReadPerKSecond;
+					} else {
+						minReadLoad = std::min(metrics.bytesReadPerKSecond, minReadLoad);
+					}
+					maxReadLoad = std::max(metrics.bytesReadPerKSecond, maxReadLoad);
+					if (req.minBytesReadPerKSecond <= metrics.bytesReadPerKSecond &&
+					    metrics.bytesReadPerKSecond <= req.maxBytesReadPerKSecond) {
+						returnMetrics.emplace_back(range, metrics);
+					}
+				}
+
+				wait(yield());
+			}
+			// FIXME(xwang): Do we need to track slow task here?
+			if (!onChange.isValid()) {
+				if (req.getTopK() >= returnMetrics.size())
+					req.reply.send(GetTopKMetricsReply(returnMetrics, minReadLoad, maxReadLoad));
+				else {
+					std::nth_element(returnMetrics.begin(),
+					                 returnMetrics.begin() + req.getTopK() - 1,
+					                 returnMetrics.end(),
+					                 GetTopKMetricsRequest::compare);
+					req.reply.send(
+					    GetTopKMetricsReply(std::vector<GetTopKMetricsReply::KeyRangeStorageMetrics>(
+					                            returnMetrics.begin(), returnMetrics.begin() + req.getTopK()),
+					                        minReadLoad,
+					                        maxReadLoad));
+				}
+				return Void();
+			}
+			wait(onChange);
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled && !req.reply.isSet())
+			req.reply.sendError(e);
+		throw;
+	}
+}
+
+ACTOR Future<Void> fetchTopKShardMetrics(DataDistributionTracker* self, GetTopKMetricsRequest req) {
+	choose {
+		// simulate time_out
+		when(wait(g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01) ? Never()
+		                                                              : fetchTopKShardMetrics_impl(self, req))) {}
+		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT))) {
+			CODE_PROBE(true, "TopK DD_SHARD_METRICS_TIMEOUT");
+			req.reply.send(GetTopKMetricsReply());
+		}
+	}
+	return Void();
+}
+
+ACTOR Future<Void> fetchShardMetrics_impl(DataDistributionTracker* self, GetMetricsRequest req) {
+	try {
+		loop {
+			Future<Void> onChange;
+			StorageMetrics returnMetrics;
+			for (auto t : self->shards->intersectingRanges(req.keys)) {
+				auto& stats = t.value().stats;
+				if (!stats->get().present()) {
+					onChange = stats->onChange();
+					break;
+				}
+				returnMetrics += t.value().stats->get().get().metrics;
+			}
+
+			if (!onChange.isValid()) {
+				req.reply.send(returnMetrics);
+				return Void();
+			}
+
+			wait(onChange);
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled && !req.reply.isSet())
+			req.reply.sendError(e);
+		throw;
+	}
+}
+
+ACTOR Future<Void> fetchShardMetrics(DataDistributionTracker* self, GetMetricsRequest req) {
+	choose {
+		when(wait(fetchShardMetrics_impl(self, req))) {}
+		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT, TaskPriority::DataDistribution))) {
+			CODE_PROBE(true, "DD_SHARD_METRICS_TIMEOUT");
+			StorageMetrics largeMetrics;
+			largeMetrics.bytes = getMaxShardSize(self->dbSizeEstimate->get());
+			req.reply.send(largeMetrics);
+		}
+	}
+	return Void();
+}
+
+ACTOR Future<Void> fetchShardMetricsList_impl(DataDistributionTracker* self, GetMetricsListRequest req) {
+	try {
+		loop {
+			// used to control shard limit
+			int shardNum = 0;
+			// list of metrics, regenerate on loop when full range unsuccessful
+			Standalone<VectorRef<DDMetricsRef>> result;
+			Future<Void> onChange;
+			auto beginIter = self->shards->containedRanges(req.keys).begin();
+			auto endIter = self->shards->intersectingRanges(req.keys).end();
+			for (auto t = beginIter; t != endIter; ++t) {
+				auto& stats = t.value().stats;
+				if (!stats->get().present()) {
+					onChange = stats->onChange();
+					break;
+				}
+				result.push_back_deep(result.arena(),
+				                      DDMetricsRef(stats->get().get().metrics.bytes, KeyRef(t.begin().toString())));
+				++shardNum;
+				if (shardNum >= req.shardLimit) {
+					break;
+				}
+			}
+
+			if (!onChange.isValid()) {
+				req.reply.send(result);
+				return Void();
+			}
+
+			wait(onChange);
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled && !req.reply.isSet())
+			req.reply.sendError(e);
+		throw;
+	}
+}
+
+ACTOR Future<Void> fetchShardMetricsList(DataDistributionTracker* self, GetMetricsListRequest req) {
+	choose {
+		when(wait(fetchShardMetricsList_impl(self, req))) {}
+		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT))) { req.reply.sendError(timed_out()); }
+	}
+	return Void();
+}
+
+ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> initData,
+                                           Reference<IDDTxnProcessor> db,
+                                           PromiseStream<RelocateShard> output,
+                                           Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
+                                           Reference<PhysicalShardCollection> physicalShardCollection,
+                                           PromiseStream<GetMetricsRequest> getShardMetrics,
+                                           FutureStream<GetTopKMetricsRequest> getTopKMetrics,
+                                           PromiseStream<GetMetricsListRequest> getShardMetricsList,
+                                           FutureStream<Promise<int64_t>> getAverageShardBytes,
+                                           Promise<Void> readyToStart,
+                                           Reference<AsyncVar<bool>> zeroHealthyTeams,
+                                           UID distributorId,
+                                           KeyRangeMap<ShardTrackedData>* shards,
+                                           bool* trackerCancelled,
+                                           Optional<Reference<TenantCache>> ddTenantCache) {
+	state DataDistributionTracker self(db,
+	                                   distributorId,
+	                                   readyToStart,
+	                                   output,
+	                                   shardsAffectedByTeamFailure,
+	                                   physicalShardCollection,
+	                                   zeroHealthyTeams,
+	                                   shards,
+	                                   trackerCancelled,
+	                                   ddTenantCache);
+	state Future<Void> loggingTrigger = Void();
+	state Future<Void> readHotDetect = readHotDetector(&self);
+	state Reference<EventCacheHolder> ddTrackerStatsEventHolder = makeReference<EventCacheHolder>("DDTrackerStats");
+	try {
+		wait(trackInitialShards(&self, initData));
+		initData.clear(); // we can release initData after initialization
+
+		state PromiseStream<TenantCacheTenantCreated> tenantCreationSignal;
+		if (self.ddTenantCache.present()) {
+			tenantCreationSignal = self.ddTenantCache.get()->tenantCreationSignal;
+		}
+
+		loop choose {
+			when(Promise<int64_t> req = waitNext(getAverageShardBytes)) { req.send(self.getAverageShardBytes()); }
+			when(wait(loggingTrigger)) {
+				TraceEvent("DDTrackerStats", self.distributorId)
+				    .detail("Shards", self.shards->size())
+				    .detail("TotalSizeBytes", self.dbSizeEstimate->get())
+				    .detail("SystemSizeBytes", self.systemSizeEstimate)
+				    .trackLatest(ddTrackerStatsEventHolder->trackingKey);
+
+				loggingTrigger = delay(SERVER_KNOBS->DATA_DISTRIBUTION_LOGGING_INTERVAL, TaskPriority::FlushTrace);
+			}
+			when(GetMetricsRequest req = waitNext(getShardMetrics.getFuture())) {
+				self.sizeChanges.add(fetchShardMetrics(&self, req));
+			}
+			when(GetTopKMetricsRequest req = waitNext(getTopKMetrics)) {
+				self.sizeChanges.add(fetchTopKShardMetrics(&self, req));
+			}
+			when(GetMetricsListRequest req = waitNext(getShardMetricsList.getFuture())) {
+				self.sizeChanges.add(fetchShardMetricsList(&self, req));
+			}
+			when(wait(self.sizeChanges.getResult())) {}
+
+			when(TenantCacheTenantCreated newTenant = waitNext(tenantCreationSignal.getFuture())) {
+				self.sizeChanges.add(tenantCreationHandling(&self, newTenant));
+			}
+
+			when(KeyRange req = waitNext(self.shardsAffectedByTeamFailure->restartShardTracker.getFuture())) {
+				restartShardTrackers(&self, req);
+			}
+		}
+	} catch (Error& e) {
+		TraceEvent(SevError, "DataDistributionTrackerError", self.distributorId).error(e);
+		throw e;
+	}
+}
+
+// Not used yet
+ACTOR Future<Void> dataDistributionTracker(Reference<DDSharedContext> context,
+                                           Reference<InitialDataDistribution> initData,
+                                           Database cx,
+                                           KeyRangeMap<ShardTrackedData>* shards);
+
+// Methods for PhysicalShardCollection
+FDB_DEFINE_BOOLEAN_PARAM(InAnonymousPhysicalShard);
+FDB_DEFINE_BOOLEAN_PARAM(PhysicalShardHasMoreThanKeyRange);
+FDB_DEFINE_BOOLEAN_PARAM(InOverSizePhysicalShard);
+FDB_DEFINE_BOOLEAN_PARAM(PhysicalShardAvailable);
+FDB_DEFINE_BOOLEAN_PARAM(MoveKeyRangeOutPhysicalShard);
+
+// Decide whether a physical shard is available at the moment when the func is calling
+PhysicalShardAvailable PhysicalShardCollection::checkPhysicalShardAvailable(uint64_t physicalShardID,
+                                                                            StorageMetrics const& moveInMetrics) {
+	ASSERT(physicalShardID != UID().first() && physicalShardID != anonymousShardId.first());
+	ASSERT(physicalShardInstances.count(physicalShardID) > 0);
+	if (physicalShardInstances[physicalShardID].metrics.bytes + moveInMetrics.bytes >
+	    SERVER_KNOBS->MAX_PHYSICAL_SHARD_BYTES) {
+		return PhysicalShardAvailable::False;
+	}
+	return PhysicalShardAvailable::True;
+}
+
+std::string PhysicalShardCollection::convertIDsToString(std::set<uint64_t> ids) {
+	std::string r = "";
+	for (auto id : ids) {
+		r = r + std::to_string(id) + " ";
+	}
+	return r;
+}
+
+void PhysicalShardCollection::updateTeamPhysicalShardIDsMap(uint64_t inputPhysicalShardID,
+                                                            std::vector<ShardsAffectedByTeamFailure::Team> inputTeams,
+                                                            uint64_t debugID) {
+	ASSERT(inputTeams.size() <= 2);
+	ASSERT(inputPhysicalShardID != anonymousShardId.first() && inputPhysicalShardID != UID().first());
+	for (auto inputTeam : inputTeams) {
+		if (teamPhysicalShardIDs.count(inputTeam) == 0) {
+			std::set<uint64_t> physicalShardIDSet;
+			physicalShardIDSet.insert(inputPhysicalShardID);
+			teamPhysicalShardIDs.insert(std::make_pair(inputTeam, physicalShardIDSet));
+		} else {
+			teamPhysicalShardIDs[inputTeam].insert(inputPhysicalShardID);
+		}
+	}
+	return;
+}
+
+void PhysicalShardCollection::insertPhysicalShardToCollection(uint64_t physicalShardID,
+                                                              StorageMetrics const& metrics,
+                                                              std::vector<ShardsAffectedByTeamFailure::Team> teams,
+                                                              uint64_t debugID,
+                                                              PhysicalShardCreationTime whenCreated) {
+	ASSERT(physicalShardID != anonymousShardId.first() && physicalShardID != UID().first());
+	ASSERT(physicalShardInstances.count(physicalShardID) == 0);
+	physicalShardInstances.insert(
+	    std::make_pair(physicalShardID, PhysicalShard(physicalShardID, metrics, teams, whenCreated)));
+	return;
+}
+
+void PhysicalShardCollection::updatekeyRangePhysicalShardIDMap(KeyRange keyRange,
+                                                               uint64_t physicalShardID,
+                                                               uint64_t debugID) {
+	ASSERT(physicalShardID != UID().first());
+	keyRangePhysicalShardIDMap.insert(keyRange, physicalShardID);
+	return;
+}
+
+// At beginning of the transition from the initial state without physical shard notion
+// to the physical shard aware state, the physicalShard set only contains one element which is anonymousShardId[0]
+// After a period in the transition, the physicalShard set of the team contains some meaningful physicalShardIDs
+Optional<uint64_t> PhysicalShardCollection::trySelectAvailablePhysicalShardFor(ShardsAffectedByTeamFailure::Team team,
+                                                                               StorageMetrics const& moveInMetrics,
+                                                                               uint64_t debugID) {
+	ASSERT(team.servers.size() > 0);
+	// Case: The team is not tracked in the mapping (teamPhysicalShardIDs)
+	if (teamPhysicalShardIDs.count(team) == 0) {
+		return Optional<uint64_t>();
+	}
+	ASSERT(teamPhysicalShardIDs[team].size() >= 1);
+	// Case: The team is tracked in the mapping and the system already has physical shard notion
+	// 		and the number of physicalShard is large
+	std::vector<uint64_t> availablePhysicalShardIDs;
+	for (auto physicalShardID : teamPhysicalShardIDs[team]) {
+		if (physicalShardID == anonymousShardId.first() || physicalShardID == UID().first()) {
+			ASSERT(false);
+		}
+		ASSERT(physicalShardInstances.count(physicalShardID));
+		/*TraceEvent("TryGetPhysicalShardIDCandidates")
+		    .detail("PhysicalShardID", physicalShardID)
+		    .detail("Bytes", physicalShardInstances[physicalShardID].metrics.bytes)
+		    .detail("BelongTeam", team.toString())
+		    .detail("DebugID", debugID);*/
+		if (!checkPhysicalShardAvailable(physicalShardID, moveInMetrics)) {
+			continue;
+		}
+		availablePhysicalShardIDs.push_back(physicalShardID);
+	}
+	if (availablePhysicalShardIDs.size() == 0) {
+		/*TraceEvent("TryGetPhysicalShardIDResultFailed")
+		    .detail("Reason", "no valid physicalShard")
+		    .detail("MoveInBytes", moveInMetrics.bytes)
+		    .detail("MaxPhysicalShardBytes", SERVER_KNOBS->MAX_PHYSICAL_SHARD_BYTES)
+		    .detail("DebugID", debugID);*/
+		return Optional<uint64_t>();
+	}
+	return deterministicRandom()->randomChoice(availablePhysicalShardIDs);
+}
+
+uint64_t PhysicalShardCollection::generateNewPhysicalShardID(uint64_t debugID) {
+	uint64_t physicalShardID = UID().first();
+	int stuckCount = 0;
+	while (physicalShardID == UID().first() || physicalShardID == anonymousShardId.first()) {
+		physicalShardID = deterministicRandom()->randomUInt64();
+		stuckCount = stuckCount + 1;
+		if (stuckCount > 50) {
+			ASSERT(false);
+		}
+	}
+	ASSERT(physicalShardID != UID().first() && physicalShardID != anonymousShardId.first());
+	//TraceEvent("GenerateNewPhysicalShardID").detail("PhysicalShardID", physicalShardID).detail("DebugID", debugID);
+	return physicalShardID;
+}
+
+void PhysicalShardCollection::reduceMetricsForMoveOut(uint64_t physicalShardID, StorageMetrics const& moveOutMetrics) {
+	ASSERT(physicalShardInstances.count(physicalShardID) != 0);
+	ASSERT(physicalShardID != UID().first() && physicalShardID != anonymousShardId.first());
+	physicalShardInstances[physicalShardID].metrics = physicalShardInstances[physicalShardID].metrics - moveOutMetrics;
+	return;
+}
+
+void PhysicalShardCollection::increaseMetricsForMoveIn(uint64_t physicalShardID, StorageMetrics const& moveInMetrics) {
+	ASSERT(physicalShardInstances.count(physicalShardID) != 0);
+	ASSERT(physicalShardID != UID().first() && physicalShardID != anonymousShardId.first());
+	physicalShardInstances[physicalShardID].metrics = physicalShardInstances[physicalShardID].metrics + moveInMetrics;
+	return;
+}
+
+void PhysicalShardCollection::updatePhysicalShardMetricsByKeyRange(KeyRange keyRange,
+                                                                   StorageMetrics const& newMetrics,
+                                                                   StorageMetrics const& oldMetrics,
+                                                                   bool initWithNewMetrics) {
+	auto ranges = keyRangePhysicalShardIDMap.intersectingRanges(keyRange);
+	std::set<uint64_t> physicalShardIDSet;
+	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+		physicalShardIDSet.insert(it->value());
+	}
+	StorageMetrics delta;
+	if (initWithNewMetrics) {
+		delta = newMetrics;
+	} else {
+		delta = newMetrics - oldMetrics;
+	}
+	for (auto physicalShardID : physicalShardIDSet) {
+		ASSERT(physicalShardID != UID().first());
+		if (physicalShardID == anonymousShardId.first()) {
+			continue; // we ignore anonymousShard when updating physicalShard metrics
+		}
+		increaseMetricsForMoveIn(physicalShardID, (delta * (1.0 / physicalShardIDSet.size())));
+	}
+	return;
+}
+
+InAnonymousPhysicalShard PhysicalShardCollection::isInAnonymousPhysicalShard(KeyRange keyRange) {
+	InAnonymousPhysicalShard res = InAnonymousPhysicalShard::True;
+	auto ranges = keyRangePhysicalShardIDMap.intersectingRanges(keyRange);
+	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+		uint64_t physicalShardID = it->value();
+		if (physicalShardID != anonymousShardId.first()) {
+			// res = false if exists a part of keyRange belongs to a non-anonymous physicalShard
+			// exist a case where some keyRange of anonymousShard is decided to move
+			// to a non-anonymous physicalShard but not completes
+			res = InAnonymousPhysicalShard::False;
+		}
+	}
+	return res;
+}
+
+// TODO: require optimize
+// It is slow to go through the keyRangePhysicalShardIDRanges for each time
+// Do we need a D/S to store the keyRange for each physicalShard?
+PhysicalShardHasMoreThanKeyRange PhysicalShardCollection::whetherPhysicalShardHasMoreThanKeyRange(
+    uint64_t physicalShardID,
+    KeyRange keyRange) {
+	KeyRangeMap<uint64_t>::Ranges keyRangePhysicalShardIDRanges = keyRangePhysicalShardIDMap.ranges();
+	KeyRangeMap<uint64_t>::iterator it = keyRangePhysicalShardIDRanges.begin();
+	for (; it != keyRangePhysicalShardIDRanges.end(); ++it) {
+		if (it->value() != physicalShardID) {
+			continue;
+		}
+		auto keyRangePiece = KeyRangeRef(it->range().begin, it->range().end);
+		if (!keyRange.intersects(keyRangePiece)) {
+			return PhysicalShardHasMoreThanKeyRange::True;
+		}
+		// if keyRange and keyRangePiece have intersection
+		if (!keyRange.contains(keyRangePiece)) {
+			return PhysicalShardHasMoreThanKeyRange::True;
+		}
+	}
+	return PhysicalShardHasMoreThanKeyRange::False;
+}
+
+InOverSizePhysicalShard PhysicalShardCollection::isInOverSizePhysicalShard(KeyRange keyRange) {
+	auto ranges = keyRangePhysicalShardIDMap.intersectingRanges(keyRange);
+	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+		uint64_t physicalShardID = it->value();
+		ASSERT(physicalShardID != UID().first());
+		if (physicalShardID == anonymousShardId.first()) {
+			continue;
+		}
+		if (checkPhysicalShardAvailable(physicalShardID, StorageMetrics())) {
+			continue;
+		}
+		if (!whetherPhysicalShardHasMoreThanKeyRange(physicalShardID, keyRange)) {
+			continue;
+		}
+		return InOverSizePhysicalShard::True;
+	}
+	return InOverSizePhysicalShard::False;
+}
+
+uint64_t PhysicalShardCollection::determinePhysicalShardIDGivenPrimaryTeam(
+    ShardsAffectedByTeamFailure::Team primaryTeam,
+    StorageMetrics const& metrics,
+    bool forceToUseNewPhysicalShard,
+    uint64_t debugID) {
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
+	ASSERT(primaryTeam.primary == true);
+	if (forceToUseNewPhysicalShard) {
+		return generateNewPhysicalShardID(debugID);
+	}
+	Optional<uint64_t> physicalShardIDFetch = trySelectAvailablePhysicalShardFor(primaryTeam, metrics, debugID);
+	if (!physicalShardIDFetch.present()) {
+		return generateNewPhysicalShardID(debugID);
+	}
+	return physicalShardIDFetch.get();
+}
+
+// May return a problematic remote team
+Optional<ShardsAffectedByTeamFailure::Team> PhysicalShardCollection::tryGetAvailableRemoteTeamWith(
+    uint64_t inputPhysicalShardID,
+    StorageMetrics const& moveInMetrics,
+    uint64_t debugID) {
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
+	ASSERT(inputPhysicalShardID != anonymousShardId.first() && inputPhysicalShardID != UID().first());
+	if (physicalShardInstances.count(inputPhysicalShardID) == 0) {
+		return Optional<ShardsAffectedByTeamFailure::Team>();
+	}
+	if (!checkPhysicalShardAvailable(inputPhysicalShardID, moveInMetrics)) {
+		return Optional<ShardsAffectedByTeamFailure::Team>();
+	}
+	for (auto team : physicalShardInstances[inputPhysicalShardID].teams) {
+		if (team.primary == false) {
+			/*TraceEvent("TryGetRemoteTeamWith")
+			    .detail("PhysicalShardID", inputPhysicalShardID)
+			    .detail("Team", team.toString())
+			    .detail("TeamSize", team.servers.size())
+			    .detail("PhysicalShardsOfTeam", convertIDsToString(teamPhysicalShardIDs[team]))
+			    .detail("DebugID", debugID);*/
+			return team;
+		}
+	}
+	UNREACHABLE();
+}
+
+// The update of PhysicalShardToTeams, Collection, keyRangePhysicalShardIDMap should be atomic
+void PhysicalShardCollection::initPhysicalShardCollection(KeyRange keys,
+                                                          std::vector<ShardsAffectedByTeamFailure::Team> selectedTeams,
+                                                          uint64_t physicalShardID,
+                                                          uint64_t debugID) {
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
+	ASSERT(physicalShardID != UID().first());
+	if (physicalShardID != anonymousShardId.first()) {
+		updateTeamPhysicalShardIDsMap(physicalShardID, selectedTeams, debugID);
+		if (physicalShardInstances.count(physicalShardID) == 0) {
+			insertPhysicalShardToCollection(
+			    physicalShardID, StorageMetrics(), selectedTeams, debugID, PhysicalShardCreationTime::DDInit);
+		} else {
+			// This assertion will be broken if we enable the optimization of data move traffic between DCs
+			ASSERT(physicalShardInstances[physicalShardID].teams == selectedTeams);
+		}
+	} else {
+		// If any physicalShard restored when DD init is the anonymousShard,
+		// Then DD enters Transition state where DD graduatelly moves Shard (or KeyRange)
+		// out of the anonymousShard
+		setTransitionCheck();
+	}
+	updatekeyRangePhysicalShardIDMap(keys, physicalShardID, debugID);
+	return;
+}
+
+// The update of PhysicalShardToTeams, Collection, keyRangePhysicalShardIDMap should be atomic
+void PhysicalShardCollection::updatePhysicalShardCollection(
+    KeyRange keys,
+    bool isRestore,
+    std::vector<ShardsAffectedByTeamFailure::Team> selectedTeams,
+    uint64_t physicalShardID,
+    const StorageMetrics& metrics,
+    uint64_t debugID) {
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
+	ASSERT(physicalShardID != UID().first());
+	/*TraceEvent e("UpdatePhysicalShard");
+	e.detail("DebugID", debugID);
+	e.detail("KeyRange", keys);
+	e.detail("IsRestore", isRestore);*/
+	// When updates metrics in physicalShard collection, we assume:
+	// It is impossible to move a keyRange from anonymousShard to a valid physicalShard
+	// Thus, we ignore anonymousShard when updating metrics
+	if (physicalShardID != anonymousShardId.first()) {
+		updateTeamPhysicalShardIDsMap(physicalShardID, selectedTeams, debugID);
+		// Update physicalShardInstances
+		// Add the metrics to in-physicalShard
+		// e.detail("PhysicalShardIDIn", physicalShardID);
+		if (physicalShardInstances.count(physicalShardID) == 0) {
+			// e.detail("Op", "Insert");
+			insertPhysicalShardToCollection(
+			    physicalShardID, metrics, selectedTeams, debugID, PhysicalShardCreationTime::DDRelocator);
+		} else {
+			// e.detail("Op", "Update");
+			//  This assertion is true since we disable the optimization of data move traffic between DCs
+			ASSERT(physicalShardInstances[physicalShardID].teams == selectedTeams);
+			increaseMetricsForMoveIn(physicalShardID, metrics);
+		}
+	}
+	// Minus the metrics from the existing (multiple) out-physicalShard(s)
+	auto ranges = keyRangePhysicalShardIDMap.intersectingRanges(keys);
+	std::set<uint64_t> physicalShardIDSet;
+	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+		physicalShardIDSet.insert(it->value());
+	}
+	/*std::string physicalShardIDOut = "";
+	for (auto id : physicalShardIDSet) {
+	    physicalShardIDOut = physicalShardIDOut + std::to_string(id) + " ";
+	}*/
+	// e.detail("PhysicalShardIDOut", physicalShardIDOut);
+	for (auto physicalShardID : physicalShardIDSet) { // imprecise: evenly move out bytes
+		if (physicalShardID == anonymousShardId.first()) {
+			continue; // we ignore anonymousShard when updating physicalShard metrics
+		}
+		StorageMetrics toReduceMetrics = metrics * (1.0 / physicalShardIDSet.size());
+		reduceMetricsForMoveOut(physicalShardID, toReduceMetrics);
+	}
+	// keyRangePhysicalShardIDMap must be update after updating the metrics of physicalShardInstances
+	updatekeyRangePhysicalShardIDMap(keys, physicalShardID, debugID);
+	return;
+}
+
+// return false if no need to move keyRange out of current physical shard
+MoveKeyRangeOutPhysicalShard PhysicalShardCollection::trackPhysicalShard(KeyRange keyRange,
+                                                                         StorageMetrics const& newMetrics,
+                                                                         StorageMetrics const& oldMetrics,
+                                                                         bool initWithNewMetrics) {
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
+	updatePhysicalShardMetricsByKeyRange(keyRange, newMetrics, oldMetrics, initWithNewMetrics);
+	if (requireTransitionCheck() &&
+	    now() - lastTransitionStartTime > SERVER_KNOBS->ANONYMOUS_PHYSICAL_SHARD_TRANSITION_TIME) {
+		if (isInAnonymousPhysicalShard(keyRange)) {
+			// Currently, whenever a shard updates metrics, it checks whether is in AnonymousPhysicalShard
+			// If yes, and if the shard has been created for long time, then triggers a data move on the shard.
+			resetLastTransitionStartTime();
+			TraceEvent("PhysicalShardTiggerTransitionMove")
+			    .detail("KeyRange", keyRange)
+			    .detail("TransitionCoolDownTime", SERVER_KNOBS->ANONYMOUS_PHYSICAL_SHARD_TRANSITION_TIME);
+			return MoveKeyRangeOutPhysicalShard::True;
+		}
+	}
+	if (isInOverSizePhysicalShard(keyRange)) {
+		return MoveKeyRangeOutPhysicalShard::True;
+	}
+	return MoveKeyRangeOutPhysicalShard::False;
+}
+
+// The update of PhysicalShardToTeams, PhysicalShardInstances, KeyRangePhysicalShardIDMap should be atomic
+void PhysicalShardCollection::cleanUpPhysicalShardCollection() {
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
+	std::set<uint64_t> physicalShardsInUse;
+	std::map<uint64_t, StorageMetrics> metricsReplies;
+	KeyRangeMap<uint64_t>::Ranges keyRangePhysicalShardIDRanges = keyRangePhysicalShardIDMap.ranges();
+	KeyRangeMap<uint64_t>::iterator it = keyRangePhysicalShardIDRanges.begin();
+	// Assume that once a physical shard is disappear in keyRangePhysicalShardIDMap,
+	// the physical shard (with the deleted id) should be deprecated.
+	// This function aims at clean up those deprecated physical shards in PhysicalShardCollection
+	// This function collects the physicalShard usage info from KeyRangePhysicalShardIDMap,
+	// then based on the info to update PhysicalShardToTeams and PhysicalShardInstances
+
+	// keyRangePhysicalShardIDMap indicates which physicalShard actually has data
+	// Step 1: Clear unused physicalShard in physicalShardInstances based on keyRangePhysicalShardIDMap
+	for (; it != keyRangePhysicalShardIDRanges.end(); ++it) {
+		uint64_t physicalShardID = it->value();
+		if (physicalShardID == anonymousShardId.first()) {
+			continue;
+		}
+		physicalShardsInUse.insert(physicalShardID);
+	}
+	for (auto it = physicalShardInstances.begin(); it != physicalShardInstances.end();) {
+		uint64_t physicalShardID = it->first;
+		ASSERT(physicalShardInstances.count(physicalShardID) > 0);
+		if (physicalShardsInUse.count(physicalShardID) == 0) {
+			/*TraceEvent("PhysicalShardisEmpty")
+			    .detail("PhysicalShard", physicalShardID)
+			    .detail("RemainBytes", physicalShardInstances[physicalShardID].metrics.bytes);*/
+			// "RemainBytes" indicates the deviation of current physical shard metric update
+			it = physicalShardInstances.erase(it);
+		} else {
+			it++;
+		}
+	}
+	// Step 2: Clean up teamPhysicalShardIDs
+	std::set<ShardsAffectedByTeamFailure::Team> toRemoveTeams;
+	for (auto [team, _] : teamPhysicalShardIDs) {
+		for (auto it = teamPhysicalShardIDs[team].begin(); it != teamPhysicalShardIDs[team].end();) {
+			uint64_t physicalShardID = *it;
+			if (physicalShardInstances.count(physicalShardID) == 0) {
+				// physicalShardID has been removed from physicalShardInstances (see step 1)
+				// So, remove the physicalShard from teamPhysicalShardID[team]
+				it = teamPhysicalShardIDs[team].erase(it);
+			} else {
+				it++;
+			}
+		}
+		if (teamPhysicalShardIDs[team].size() == 0) {
+			// If a team has no physicalShard, remove the team from teamPhysicalShardID
+			toRemoveTeams.insert(team);
+		}
+	}
+	for (auto team : toRemoveTeams) {
+		teamPhysicalShardIDs.erase(team);
+	}
+}
+
+void PhysicalShardCollection::logPhysicalShardCollection() {
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
+	// Step 1: Logging non-empty physicalShard
+	for (auto [physicalShardID, physicalShard] : physicalShardInstances) {
+		ASSERT(physicalShardID == physicalShard.id);
+		TraceEvent e("PhysicalShardStatus");
+		e.detail("PhysicalShardID", physicalShardID);
+		e.detail("TotalBytes", physicalShard.metrics.bytes);
+	}
+	// Step 2: Logging TeamPhysicalShardStatus
+	for (auto [team, physicalShardIDs] : teamPhysicalShardIDs) {
+		TraceEvent e("TeamPhysicalShardStatus");
+		e.detail("Team", team.toString());
+		// std::string metricsStr = "";
+		int64_t counter = 0;
+		int64_t totalBytes = 0;
+		int64_t maxPhysicalShardBytes = -1;
+		int64_t minPhysicalShardBytes = StorageMetrics::infinity;
+		uint64_t maxPhysicalShardID = 0;
+		uint64_t minPhysicalShardID = 0;
+		for (auto physicalShardID : physicalShardIDs) {
+			ASSERT(physicalShardInstances.count(physicalShardID) > 0);
+			uint64_t id = physicalShardInstances[physicalShardID].id;
+			int64_t bytes = physicalShardInstances[physicalShardID].metrics.bytes;
+			if (bytes > maxPhysicalShardBytes) {
+				maxPhysicalShardBytes = bytes;
+				maxPhysicalShardID = id;
+			}
+			if (bytes < minPhysicalShardBytes) {
+				minPhysicalShardBytes = bytes;
+				minPhysicalShardID = id;
+			}
+			totalBytes = totalBytes + bytes;
+			/* metricsStr = metricsStr + std::to_string(id) + ":" + std::to_string(bytes);
+			if (counter < physicalShardIDs.size() - 1) {
+			    metricsStr = metricsStr + ",";
+			} */
+			counter = counter + 1;
+		}
+		// e.detail("Metrics", metricsStr);
+		e.detail("TotalBytes", totalBytes);
+		e.detail("NumPhysicalShards", counter);
+		e.detail("MaxPhysicalShard", std::to_string(maxPhysicalShardID) + ":" + std::to_string(maxPhysicalShardBytes));
+		e.detail("MinPhysicalShard", std::to_string(minPhysicalShardID) + ":" + std::to_string(minPhysicalShardBytes));
+	}
+	// Step 3: Logging StorageServerPhysicalShardStatus
+	std::map<UID, std::map<uint64_t, int64_t>> storageServerPhysicalShardStatus;
+	for (auto [team, _] : teamPhysicalShardIDs) {
+		for (auto ssid : team.servers) {
+			for (auto it = teamPhysicalShardIDs[team].begin(); it != teamPhysicalShardIDs[team].end();) {
+				uint64_t physicalShardID = *it;
+				if (storageServerPhysicalShardStatus.count(ssid) != 0) {
+					if (storageServerPhysicalShardStatus[ssid].count(physicalShardID) == 0) {
+						ASSERT(physicalShardInstances.count(physicalShardID) > 0);
+						storageServerPhysicalShardStatus[ssid].insert(
+						    std::make_pair(physicalShardID, physicalShardInstances[physicalShardID].metrics.bytes));
+					}
+				} else {
+					ASSERT(physicalShardInstances.count(physicalShardID) > 0);
+					std::map<uint64_t, int64_t> tmp;
+					tmp.insert(std::make_pair(physicalShardID, physicalShardInstances[physicalShardID].metrics.bytes));
+					storageServerPhysicalShardStatus.insert(std::make_pair(ssid, tmp));
+				}
+				it++;
+			}
+		}
+	}
+	for (auto [serverID, physicalShardMetrics] : storageServerPhysicalShardStatus) {
+		TraceEvent e("ServerPhysicalShardStatus");
+		e.detail("Server", serverID);
+		e.detail("NumPhysicalShards", physicalShardMetrics.size());
+		int64_t totalBytes = 0;
+		int64_t maxPhysicalShardBytes = -1;
+		int64_t minPhysicalShardBytes = StorageMetrics::infinity;
+		uint64_t maxPhysicalShardID = 0;
+		uint64_t minPhysicalShardID = 0;
+		// std::string metricsStr = "";
+		// int64_t counter = 0;
+		for (auto [physicalShardID, bytes] : physicalShardMetrics) {
+			totalBytes = totalBytes + bytes;
+			if (bytes > maxPhysicalShardBytes) {
+				maxPhysicalShardBytes = bytes;
+				maxPhysicalShardID = physicalShardID;
+			}
+			if (bytes < minPhysicalShardBytes) {
+				minPhysicalShardBytes = bytes;
+				minPhysicalShardID = physicalShardID;
+			}
+			/* metricsStr = metricsStr + std::to_string(physicalShardID) + ":" + std::to_string(bytes);
+			if (counter < physicalShardMetrics.size() - 1) {
+			        metricsStr = metricsStr + ",";
+			}
+			counter = counter + 1; */
+		}
+		e.detail("TotalBytes", totalBytes);
+		e.detail("MaxPhysicalShard", std::to_string(maxPhysicalShardID) + ":" + std::to_string(maxPhysicalShardBytes));
+		e.detail("MinPhysicalShard", std::to_string(minPhysicalShardID) + ":" + std::to_string(minPhysicalShardBytes));
+	}
+}
+
+// FIXME: complete this test with non-empty range
+TEST_CASE("/DataDistributor/Tracker/FetchTopK") {
+	state DataDistributionTracker self;
+	state std::vector<KeyRange> ranges;
+	// for (int i = 1; i <= 10; i += 2) {
+	//     ranges.emplace_back(KeyRangeRef(doubleToTestKey(i), doubleToTestKey(i + 2)));
+	//     std::cout << "add range: " << ranges.back().begin.toString() << "\n";
+	// }
+	state GetTopKMetricsRequest req(ranges, 3, 1000, 100000);
+
+	// double targetDensities[10] = { 2, 1, 3, 5, 4, 10, 6, 8, 7, 0 };
+
+	wait(fetchTopKShardMetrics(&self, req));
+	auto& reply = req.reply.getFuture().get();
+	ASSERT(reply.shardMetrics.empty());
+	ASSERT(reply.maxReadLoad == -1);
+	ASSERT(reply.minReadLoad == -1);
+
+	return Void();
+}
\ No newline at end of file
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index b121056c35..1de3fb0080 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -942,19 +942,20 @@ public:
 											                                : SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY);
 										}
 									} else {
-										TEST(true); // A removed server is still associated with a team in
-										            // ShardsAffectedByTeamFailure
+										CODE_PROBE(true,
+										           "A removed server is still associated with a team in "
+										           "ShardsAffectedByTeamFailure");
 									}
 								}
 							}
 
-							RelocateShard rs;
-							rs.keys = shards[i];
-							rs.priority = maxPriority;
+							RelocateShard rs(
+							    shards[i], maxPriority, RelocateReason::OTHER, deterministicRandom()->randomUniqueID());
 
 							self->output.send(rs);
 							TraceEvent("SendRelocateToDDQueue", self->distributorId)
 							    .suppressFor(1.0)
+							    .detail("TraceId", rs.traceId)
 							    .detail("ServerPrimary", self->primary)
 							    .detail("ServerTeam", team->getDesc())
 							    .detail("KeyBegin", rs.keys.begin)
@@ -1253,7 +1254,7 @@ public:
 
 						server->updateLastKnown(newInterface.first, newInterface.second);
 						if (localityChanged && !isTss) {
-							TEST(true); // Server locality changed
+							CODE_PROBE(true, "Server locality changed");
 
 							// The locality change of a server will affect machine teams related to the server if
 							// the server's machine locality is changed
@@ -1320,7 +1321,7 @@ public:
 								}
 							}
 							if (addedNewBadTeam && self->badTeamRemover.isReady()) {
-								TEST(true); // Server locality change created bad teams
+								CODE_PROBE(true, "Server locality change created bad teams");
 								self->doBuildTeams = true;
 								self->badTeamRemover = removeBadTeams(self);
 								self->addActor.send(self->badTeamRemover);
@@ -1724,7 +1725,7 @@ public:
 					// in the serverTeams vector in the machine team.
 					--teamIndex;
 					self->addTeam(team->getServers(), IsInitialTeam::True, IsRedundantTeam::True);
-					TEST(true); // Removed machine team
+					CODE_PROBE(true, "Removed machine team");
 				}
 
 				self->doBuildTeams = true;
@@ -1808,7 +1809,7 @@ public:
 				bool foundTeam = self->removeTeam(st);
 				ASSERT(foundTeam);
 				self->addTeam(st->getServers(), IsInitialTeam::True, IsRedundantTeam::True);
-				TEST(true); // Marked team as a bad team
+				CODE_PROBE(true, "Marked team as a bad team");
 
 				self->doBuildTeams = true;
 
@@ -2052,7 +2053,7 @@ public:
 			if (self->wigglingId.present()) {
 				state UID id = self->wigglingId.get();
 				if (self->pauseWiggle->get()) {
-					TEST(true); // paused because cluster is unhealthy
+					CODE_PROBE(true, "paused because cluster is unhealthy");
 					moveFinishFuture = Never();
 					self->includeStorageServersForWiggle();
 					self->storageWiggler->setWiggleState(StorageWiggler::PAUSE);
@@ -2068,7 +2069,7 @@ public:
 				} else {
 					choose {
 						when(wait(self->waitUntilHealthy())) {
-							TEST(true); // start wiggling
+							CODE_PROBE(true, "start wiggling");
 							wait(self->storageWiggler->startWiggle());
 							auto fv = self->excludeStorageServersForWiggle(id);
 							moveFinishFuture = fv;
@@ -2431,10 +2432,10 @@ public:
 		// SS and/or TSS recruitment failed at this point, update tssState
 		if (recruitTss && tssState->tssRecruitFailed()) {
 			tssState->markComplete();
-			TEST(true); // TSS recruitment failed for some reason
+			CODE_PROBE(true, "TSS recruitment failed for some reason");
 		}
 		if (!recruitTss && tssState->ssRecruitFailed()) {
-			TEST(true); // SS with pair TSS recruitment failed for some reason
+			CODE_PROBE(true, "SS with pair TSS recruitment failed for some reason");
 		}
 
 		self->recruitingStream.set(self->recruitingStream.get() - 1);
@@ -2467,16 +2468,7 @@ public:
 
 		loop {
 			try {
-				// Divide TSS evenly in each DC if there are multiple
-				// TODO would it be better to put all of them in primary DC?
-				targetTSSInDC = self->configuration.desiredTSSCount;
-				if (self->configuration.usableRegions > 1) {
-					targetTSSInDC /= self->configuration.usableRegions;
-					if (self->primary) {
-						// put extras in primary DC if it's uneven
-						targetTSSInDC += (self->configuration.desiredTSSCount % self->configuration.usableRegions);
-					}
-				}
+				targetTSSInDC = self->getTargetTSSInDC();
 				int newTssToRecruit = targetTSSInDC - self->tss_info_by_pair.size() - inProgressTSSCount;
 				// FIXME: Should log this if the recruit count stays the same but the other numbers update?
 				if (newTssToRecruit != tssToRecruit) {
@@ -2575,7 +2567,7 @@ public:
 							    .detail("Addr", candidateSSAddr.toString())
 							    .detail("Locality", candidateWorker.worker.locality.toString());
 
-							TEST(true); // Starting TSS recruitment
+							CODE_PROBE(true, "Starting TSS recruitment");
 							self->isTssRecruiting = true;
 							tssState = makeReference<TSSPairState>(candidateWorker.worker.locality);
 
@@ -2585,7 +2577,7 @@ public:
 							checkTss = self->initialFailureReactionDelay;
 						} else {
 							if (tssState->active && tssState->inDataZone(candidateWorker.worker.locality)) {
-								TEST(true); // TSS recruits pair in same dc/datahall
+								CODE_PROBE(true, "TSS recruits pair in same dc/datahall");
 								self->isTssRecruiting = false;
 								TraceEvent("TSS_Recruit", self->distributorId)
 								    .detail("Stage", "PairSS")
@@ -2596,8 +2588,9 @@ public:
 								// successfully started recruitment of pair, reset tss recruitment state
 								tssState = makeReference<TSSPairState>();
 							} else {
-								TEST(tssState->active); // TSS recruitment skipped potential pair because it's in a
-								                        // different dc/datahall
+								CODE_PROBE(
+								    tssState->active,
+								    "TSS recruitment skipped potential pair because it's in a different dc/datahall");
 								self->addActor.send(initializeStorage(
 								    self, candidateWorker, ddEnabledState, false, makeReference<TSSPairState>()));
 							}
@@ -2617,8 +2610,9 @@ public:
 						int tssToKill = std::min((int)self->tss_info_by_pair.size(),
 						                         std::max(-tssToRecruit, self->zeroHealthyTeams->get() ? 1 : 0));
 						if (cancelTss) {
-							TEST(tssToRecruit < 0); // tss recruitment cancelled due to too many TSS
-							TEST(self->zeroHealthyTeams->get()); // tss recruitment cancelled due zero healthy teams
+							CODE_PROBE(tssToRecruit < 0, "tss recruitment cancelled due to too many TSS");
+							CODE_PROBE(self->zeroHealthyTeams->get(),
+							           "tss recruitment cancelled due zero healthy teams");
 
 							TraceEvent(SevWarn, "TSS_RecruitCancelled", self->distributorId)
 							    .detail("Reason", tssToRecruit <= 0 ? "TooMany" : "ZeroHealthyTeams");
@@ -2637,8 +2631,8 @@ public:
 								if (self->shouldHandleServer(tssi) && self->server_and_tss_info.count(tssId)) {
 									Promise<Void> killPromise = itr->second->killTss;
 									if (killPromise.canBeSet()) {
-										TEST(tssToRecruit < 0); // Killing TSS due to too many TSS
-										TEST(self->zeroHealthyTeams->get()); // Killing TSS due zero healthy teams
+										CODE_PROBE(tssToRecruit < 0, "Killing TSS due to too many TSS");
+										CODE_PROBE(self->zeroHealthyTeams->get(), "Killing TSS due zero healthy teams");
 										TraceEvent(SevWarn, "TSS_DDKill", self->distributorId)
 										    .detail("TSSID", tssId)
 										    .detail("Reason",
@@ -2672,7 +2666,7 @@ public:
 				if (e.code() != error_code_timed_out) {
 					throw;
 				}
-				TEST(true); // Storage recruitment timed out
+				CODE_PROBE(true, "Storage recruitment timed out");
 			}
 		}
 	}
@@ -2690,33 +2684,21 @@ public:
 		    .detail("Primary", self->primary)
 		    .detail("DcId", dcId)
 		    .detail("Replicas", self->configuration.storageTeamSize);
-		state Transaction tr(self->cx);
-		loop {
-			try {
-				Optional<Value> val = wait(tr.get(datacenterReplicasKeyFor(dcId)));
-				state int oldReplicas = val.present() ? decodeDatacenterReplicasValue(val.get()) : 0;
-				if (oldReplicas == self->configuration.storageTeamSize) {
-					TraceEvent("DDUpdatedAlready", self->distributorId)
-					    .detail("Primary", self->primary)
-					    .detail("DcId", dcId)
-					    .detail("Replicas", self->configuration.storageTeamSize);
-					return Void();
-				}
-				if (oldReplicas < self->configuration.storageTeamSize) {
-					tr.set(rebootWhenDurableKey, StringRef());
-				}
-				tr.set(datacenterReplicasKeyFor(dcId), datacenterReplicasValue(self->configuration.storageTeamSize));
-				wait(tr.commit());
-				TraceEvent("DDUpdatedReplicas", self->distributorId)
-				    .detail("Primary", self->primary)
-				    .detail("DcId", dcId)
-				    .detail("Replicas", self->configuration.storageTeamSize)
-				    .detail("OldReplicas", oldReplicas);
-				return Void();
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
+
+		int oldReplicas = wait(self->db->tryUpdateReplicasKeyForDc(dcId, self->configuration.storageTeamSize));
+		if (oldReplicas == self->configuration.storageTeamSize) {
+			TraceEvent("DDUpdatedAlready", self->distributorId)
+			    .detail("Primary", self->primary)
+			    .detail("DcId", dcId)
+			    .detail("Replicas", self->configuration.storageTeamSize);
+		} else {
+			TraceEvent("DDUpdatedReplicas", self->distributorId)
+			    .detail("Primary", self->primary)
+			    .detail("DcId", dcId)
+			    .detail("Replicas", self->configuration.storageTeamSize)
+			    .detail("OldReplicas", oldReplicas);
 		}
+		return Void();
 	}
 
 	ACTOR static Future<Void> serverGetTeamRequests(DDTeamCollection* self, TeamCollectionInterface tci) {
@@ -2743,140 +2725,98 @@ public:
 		}
 	}
 
-	ACTOR static Future<UID> getClusterId(DDTeamCollection* self) {
-		state ReadYourWritesTransaction tr(self->cx);
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Optional<Value> clusterId = wait(tr.get(clusterIdKey));
-				ASSERT(clusterId.present());
-				return BinaryReader::fromStringRef<UID>(clusterId.get(), Unversioned());
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-
 	ACTOR static Future<Void> waitServerListChange(DDTeamCollection* self,
 	                                               FutureStream<Void> serverRemoved,
 	                                               const DDEnabledState* ddEnabledState) {
 		state Future<Void> checkSignal = delay(SERVER_KNOBS->SERVER_LIST_DELAY, TaskPriority::DataDistributionLaunch);
-		state Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> serverListAndProcessClasses =
-		    Never();
+		state Future<ServerWorkerInfos> serverListAndProcessClasses = Never();
 		state bool isFetchingResults = false;
-		state Transaction tr(self->cx);
 		loop {
-			try {
-				choose {
-					when(wait(checkSignal)) {
-						checkSignal = Never();
-						isFetchingResults = true;
-						serverListAndProcessClasses = NativeAPI::getServerListAndProcessClasses(&tr);
-					}
-					when(std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
-					         wait(serverListAndProcessClasses)) {
-						serverListAndProcessClasses = Never();
-						isFetchingResults = false;
+			choose {
+				when(wait(checkSignal)) {
+					checkSignal = Never();
+					isFetchingResults = true;
+					serverListAndProcessClasses = self->db->getServerListAndProcessClasses();
+				}
+				when(ServerWorkerInfos infos = wait(serverListAndProcessClasses)) {
+					auto& servers = infos.servers;
+					serverListAndProcessClasses = Never();
+					isFetchingResults = false;
 
-						for (int i = 0; i < results.size(); i++) {
-							UID serverId = results[i].first.id();
-							StorageServerInterface const& ssi = results[i].first;
-							ProcessClass const& processClass = results[i].second;
-							if (!self->shouldHandleServer(ssi)) {
-								continue;
-							} else if (self->server_and_tss_info.count(serverId)) {
-								auto& serverInfo = self->server_and_tss_info[serverId];
-								if (ssi.getValue.getEndpoint() !=
-								        serverInfo->getLastKnownInterface().getValue.getEndpoint() ||
-								    processClass != serverInfo->getLastKnownClass().classType()) {
-									Promise<std::pair<StorageServerInterface, ProcessClass>> currentInterfaceChanged =
-									    serverInfo->interfaceChanged;
-									serverInfo->interfaceChanged =
-									    Promise<std::pair<StorageServerInterface, ProcessClass>>();
-									serverInfo->onInterfaceChanged =
-									    Future<std::pair<StorageServerInterface, ProcessClass>>(
-									        serverInfo->interfaceChanged.getFuture());
-									currentInterfaceChanged.send(std::make_pair(ssi, processClass));
-								}
-							} else if (!self->recruitingIds.count(ssi.id())) {
-								self->addServer(ssi,
-								                processClass,
-								                self->serverTrackerErrorOut,
-								                tr.getReadVersion().get(),
-								                *ddEnabledState);
+					for (int i = 0; i < servers.size(); i++) {
+						UID serverId = servers[i].first.id();
+						StorageServerInterface const& ssi = servers[i].first;
+						ProcessClass const& processClass = servers[i].second;
+						if (!self->shouldHandleServer(ssi)) {
+							continue;
+						} else if (self->server_and_tss_info.count(serverId)) {
+							auto& serverInfo = self->server_and_tss_info[serverId];
+							if (ssi.getValue.getEndpoint() !=
+							        serverInfo->getLastKnownInterface().getValue.getEndpoint() ||
+							    processClass != serverInfo->getLastKnownClass().classType()) {
+								Promise<std::pair<StorageServerInterface, ProcessClass>> currentInterfaceChanged =
+								    serverInfo->interfaceChanged;
+								serverInfo->interfaceChanged =
+								    Promise<std::pair<StorageServerInterface, ProcessClass>>();
+								serverInfo->onInterfaceChanged =
+								    Future<std::pair<StorageServerInterface, ProcessClass>>(
+								        serverInfo->interfaceChanged.getFuture());
+								currentInterfaceChanged.send(std::make_pair(ssi, processClass));
 							}
+						} else if (!self->recruitingIds.count(ssi.id())) {
+							self->addServer(ssi,
+							                processClass,
+							                self->serverTrackerErrorOut,
+							                infos.readVersion.get(),
+							                *ddEnabledState);
 						}
-
-						tr = Transaction(self->cx);
-						checkSignal = delay(SERVER_KNOBS->SERVER_LIST_DELAY, TaskPriority::DataDistributionLaunch);
 					}
-					when(waitNext(serverRemoved)) {
-						if (isFetchingResults) {
-							tr = Transaction(self->cx);
-							serverListAndProcessClasses = NativeAPI::getServerListAndProcessClasses(&tr);
-						}
+
+					checkSignal = delay(SERVER_KNOBS->SERVER_LIST_DELAY, TaskPriority::DataDistributionLaunch);
+				}
+				when(waitNext(serverRemoved)) {
+					if (isFetchingResults) {
+						serverListAndProcessClasses = self->db->getServerListAndProcessClasses();
 					}
 				}
-			} catch (Error& e) {
-				wait(tr.onError(e));
-				serverListAndProcessClasses = Never();
-				isFetchingResults = false;
-				checkSignal = Void();
 			}
 		}
 	}
 
-	ACTOR static Future<UID> getNextWigglingServerID(DDTeamCollection* teamCollection) {
-		state Optional<Value> localityKey;
-		state Optional<Value> localityValue;
-
-		// NOTE: because normal \xff/conf change through `changeConfig` now will cause DD throw `movekeys_conflict()`
-		// then recruit a new DD, we only need to read current configuration once
-		if (teamCollection->configuration.perpetualStorageWiggleLocality != "0") {
-			// parsing format is like "datahall:0"
-			std::string& localityKeyValue = teamCollection->configuration.perpetualStorageWiggleLocality;
-			ASSERT(isValidPerpetualStorageWiggleLocality(localityKeyValue));
-			// get key and value from perpetual_storage_wiggle_locality.
-			int split = localityKeyValue.find(':');
-			localityKey = Optional<Value>(ValueRef((uint8_t*)localityKeyValue.c_str(), split));
-			localityValue = Optional<Value>(
-			    ValueRef((uint8_t*)localityKeyValue.c_str() + split + 1, localityKeyValue.size() - split - 1));
-		}
-
+	ACTOR static Future<UID> getNextWigglingServerID(Reference<StorageWiggler> wiggler,
+	                                                 Optional<Value> localityKey = Optional<Value>(),
+	                                                 Optional<Value> localityValue = Optional<Value>(),
+	                                                 DDTeamCollection* teamCollection = nullptr) {
+		ASSERT(wiggler->teamCollection == teamCollection);
 		loop {
-			// wait until the wiggle queue is not empty
-			if (teamCollection->storageWiggler->empty()) {
-				wait(teamCollection->storageWiggler->nonEmpty.onChange());
+			// when the DC need more
+			state Optional<UID> id =
+			    wiggler->getNextServerId(teamCollection == nullptr || teamCollection->reachTSSPairTarget());
+			if (!id.present()) {
+				wait(wiggler->onCheck());
+				continue;
 			}
 
 			// if perpetual_storage_wiggle_locality has value and not 0(disabled).
 			if (localityKey.present()) {
 				// Whether the selected server matches the locality
-				auto id = teamCollection->storageWiggler->getNextServerId();
-				if (!id.present())
-					continue;
 				auto server = teamCollection->server_info.at(id.get());
 
 				// TraceEvent("PerpetualLocality").detail("Server", server->getLastKnownInterface().locality.get(localityKey)).detail("Desire", localityValue);
 				if (server->getLastKnownInterface().locality.get(localityKey.get()) == localityValue) {
 					return id.get();
-				} else {
-					if (teamCollection->storageWiggler->empty()) {
-						// None of the entries in wiggle queue matches the given locality.
-						TraceEvent("PerpetualStorageWiggleEmptyQueue", teamCollection->distributorId)
-						    .detail("WriteValue", "No process matched the given perpetualStorageWiggleLocality")
-						    .detail("PerpetualStorageWiggleLocality",
-						            teamCollection->configuration.perpetualStorageWiggleLocality);
-					}
-					continue;
 				}
-			} else {
-				auto id = teamCollection->storageWiggler->getNextServerId();
-				if (!id.present())
-					continue;
-				return id.get();
+
+				if (wiggler->empty()) {
+					// None of the entries in wiggle queue matches the given locality.
+					TraceEvent("PerpetualStorageWiggleEmptyQueue", teamCollection->distributorId)
+					    .detail("WriteValue", "No process matched the given perpetualStorageWiggleLocality")
+					    .detail("PerpetualStorageWiggleLocality",
+					            teamCollection->configuration.perpetualStorageWiggleLocality);
+				}
+				continue;
 			}
+			return id.get();
 		}
 	}
 
@@ -2950,7 +2890,7 @@ public:
 
 		try {
 			wait(self->init(initData, *ddEnabledState));
-			initData = Reference<InitialDataDistribution>();
+			initData.clear(); // release reference count
 			self->addActor.send(self->serverGetTeamRequests(tci));
 
 			TraceEvent("DDTeamCollectionBegin", self->distributorId).detail("Primary", self->primary);
@@ -2984,22 +2924,25 @@ public:
 			self->addActor.send(self->storageRecruiter(recruitStorage, *ddEnabledState));
 			self->addActor.send(self->monitorStorageServerRecruitment());
 			self->addActor.send(self->waitServerListChange(serverRemoved.getFuture(), *ddEnabledState));
-			self->addActor.send(self->trackExcludedServers());
 			self->addActor.send(self->monitorHealthyTeams());
-			self->addActor.send(self->waitHealthyZoneChange());
-			self->addActor.send(self->monitorPerpetualStorageWiggle());
+
+			if (!self->db->isMocked()) {
+				self->addActor.send(self->trackExcludedServers());
+				self->addActor.send(self->waitHealthyZoneChange());
+				self->addActor.send(self->monitorPerpetualStorageWiggle());
+			}
 			// SOMEDAY: Monitor FF/serverList for (new) servers that aren't in allServers and add or remove them
 
 			loop choose {
 				when(UID removedServer = waitNext(self->removedServers.getFuture())) {
-					TEST(true); // Storage server removed from database
+					CODE_PROBE(true, "Storage server removed from database");
 					self->removeServer(removedServer);
 					serverRemoved.send(Void());
 
 					self->restartRecruiting.trigger();
 				}
 				when(UID removedTSS = waitNext(self->removedTSS.getFuture())) {
-					TEST(true); // TSS removed from database
+					CODE_PROBE(true, "TSS removed from database");
 					self->removeTSS(removedTSS);
 					serverRemoved.send(Void());
 
@@ -3061,220 +3004,230 @@ public:
 		state int traceEventsPrinted = 0;
 		state std::vector<const UID*> serverIDs;
 		state double lastPrintTime = 0;
-		state ReadYourWritesTransaction tr(self->cx);
 		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				state Future<Void> watchFuture = tr.watch(triggerDDTeamInfoPrintKey);
-				wait(tr.commit());
-				wait(self->printDetailedTeamsInfo.onTrigger() || watchFuture);
-				tr.reset();
-				if (now() - lastPrintTime < SERVER_KNOBS->DD_TEAMS_INFO_PRINT_INTERVAL) {
-					continue;
-				}
-				lastPrintTime = now();
 
-				traceEventsPrinted = 0;
+			wait(self->printDetailedTeamsInfo.onTrigger() || self->db->waitDDTeamInfoPrintSignal());
 
-				double snapshotStart = now();
+			if (now() - lastPrintTime < SERVER_KNOBS->DD_TEAMS_INFO_PRINT_INTERVAL) {
+				continue;
+			}
+			lastPrintTime = now();
 
-				configuration = self->configuration;
-				server_info = self->server_info;
-				teams = self->teams;
-				// Perform deep copy so we have a consistent snapshot, even if yields are performed
-				for (const auto& [machineId, info] : self->machine_info) {
-					machine_info.emplace(machineId, info->clone());
-				}
-				machineTeams = self->machineTeams;
-				// internedLocalityRecordKeyNameStrings = self->machineLocalityMap._keymap->_lookuparray;
-				// machineLocalityMapEntryArraySize = self->machineLocalityMap.size();
-				// machineLocalityMapRecordArray = self->machineLocalityMap.getRecordArray();
-				std::vector<const UID*> _uids = self->machineLocalityMap.getObjects();
-				serverIDs = _uids;
+			traceEventsPrinted = 0;
 
-				auto const& keys = self->server_status.getKeys();
-				for (auto const& key : keys) {
-					// Add to or update the local server_status map
-					server_status[key] = self->server_status.get(key);
-				}
+			double snapshotStart = now();
 
-				TraceEvent("DDPrintSnapshotTeasmInfo", self->getDistributorId())
-				    .detail("SnapshotSpeed", now() - snapshotStart)
-				    .detail("Primary", self->isPrimary());
+			configuration = self->configuration;
+			server_info = self->server_info;
+			teams = self->teams;
+			// Perform deep copy so we have a consistent snapshot, even if yields are performed
+			for (const auto& [machineId, info] : self->machine_info) {
+				machine_info.emplace(machineId, info->clone());
+			}
+			machineTeams = self->machineTeams;
+			// internedLocalityRecordKeyNameStrings = self->machineLocalityMap._keymap->_lookuparray;
+			// machineLocalityMapEntryArraySize = self->machineLocalityMap.size();
+			// machineLocalityMapRecordArray = self->machineLocalityMap.getRecordArray();
+			std::vector<const UID*> _uids = self->machineLocalityMap.getObjects();
+			serverIDs = _uids;
 
-				// Print to TraceEvents
-				TraceEvent("DDConfig", self->getDistributorId())
-				    .detail("StorageTeamSize", configuration.storageTeamSize)
-				    .detail("DesiredTeamsPerServer", SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER)
-				    .detail("MaxTeamsPerServer", SERVER_KNOBS->MAX_TEAMS_PER_SERVER)
-				    .detail("Primary", self->isPrimary());
+			auto const& keys = self->server_status.getKeys();
+			for (auto const& key : keys) {
+				// Add to or update the local server_status map
+				server_status[key] = self->server_status.get(key);
+			}
 
+			TraceEvent("DDPrintSnapshotTeamsInfo", self->getDistributorId())
+			    .detail("SnapshotSpeed", now() - snapshotStart)
+			    .detail("Primary", self->isPrimary());
+
+			// Print to TraceEvents
+			TraceEvent("DDConfig", self->getDistributorId())
+			    .detail("StorageTeamSize", configuration.storageTeamSize)
+			    .detail("DesiredTeamsPerServer", SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER)
+			    .detail("MaxTeamsPerServer", SERVER_KNOBS->MAX_TEAMS_PER_SERVER)
+			    .detail("Primary", self->isPrimary());
+
+			TraceEvent("ServerInfo", self->getDistributorId())
+			    .detail("Size", server_info.size())
+			    .detail("Primary", self->isPrimary());
+
+			state int i;
+			state std::map<UID, Reference<TCServerInfo>>::iterator server = server_info.begin();
+			for (i = 0; i < server_info.size(); i++) {
 				TraceEvent("ServerInfo", self->getDistributorId())
-				    .detail("Size", server_info.size())
+				    .detail("ServerInfoIndex", i)
+				    .detail("ServerID", server->first.toString())
+				    .detail("ServerTeamOwned", server->second->getTeams().size())
+				    .detail("MachineID", server->second->machine->machineID.contents().toString())
 				    .detail("Primary", self->isPrimary());
-				state int i;
-				state std::map<UID, Reference<TCServerInfo>>::iterator server = server_info.begin();
-				for (i = 0; i < server_info.size(); i++) {
-					TraceEvent("ServerInfo", self->getDistributorId())
-					    .detail("ServerInfoIndex", i)
-					    .detail("ServerID", server->first.toString())
-					    .detail("ServerTeamOwned", server->second->getTeams().size())
-					    .detail("MachineID", server->second->machine->machineID.contents().toString())
-					    .detail("Primary", self->isPrimary());
-					server++;
-					if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-						wait(yield());
-					}
+				server++;
+				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
+					wait(yield());
+				}
+			}
+
+			server = server_info.begin();
+			for (i = 0; i < server_info.size(); i++) {
+				const UID& uid = server->first;
+
+				TraceEvent e("ServerStatus", self->getDistributorId());
+				e.detail("ServerUID", uid)
+				    .detail("MachineIsValid", server_info[uid]->machine.isValid())
+				    .detail("MachineTeamSize",
+				            server_info[uid]->machine.isValid() ? server_info[uid]->machine->machineTeams.size() : -1)
+				    .detail("Primary", self->isPrimary());
+
+				// ServerStatus might not be known if server was very recently added and
+				// storageServerFailureTracker() has not yet updated self->server_status If the UID is not found, do
+				// not assume the server is healthy or unhealthy
+				auto it = server_status.find(uid);
+				if (it != server_status.end()) {
+					e.detail("Healthy", !it->second.isUnhealthy());
 				}
 
-				server = server_info.begin();
-				for (i = 0; i < server_info.size(); i++) {
-					const UID& uid = server->first;
-
-					TraceEvent e("ServerStatus", self->getDistributorId());
-					e.detail("ServerUID", uid)
-					    .detail("MachineIsValid", server_info[uid]->machine.isValid())
-					    .detail("MachineTeamSize",
-					            server_info[uid]->machine.isValid() ? server_info[uid]->machine->machineTeams.size()
-					                                                : -1)
-					    .detail("Primary", self->isPrimary());
-
-					// ServerStatus might not be known if server was very recently added and
-					// storageServerFailureTracker() has not yet updated self->server_status If the UID is not found, do
-					// not assume the server is healthy or unhealthy
-					auto it = server_status.find(uid);
-					if (it != server_status.end()) {
-						e.detail("Healthy", !it->second.isUnhealthy());
-					}
-
-					server++;
-					if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-						wait(yield());
-					}
+				server++;
+				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
+					wait(yield());
 				}
+			}
+
+			TraceEvent("ServerTeamInfo", self->getDistributorId())
+			    .detail("Size", teams.size())
+			    .detail("Primary", self->isPrimary());
+
+			for (i = 0; i < teams.size(); i++) {
+				const auto& team = teams[i];
 
 				TraceEvent("ServerTeamInfo", self->getDistributorId())
-				    .detail("Size", teams.size())
-				    .detail("Primary", self->isPrimary());
-				for (i = 0; i < teams.size(); i++) {
-					const auto& team = teams[i];
-
-					TraceEvent("ServerTeamInfo", self->getDistributorId())
-					    .detail("TeamIndex", i)
-					    .detail("Healthy", team->isHealthy())
-					    .detail("TeamSize", team->size())
-					    .detail("MemberIDs", team->getServerIDsStr())
-					    .detail("Primary", self->isPrimary())
-					    .detail("TeamID", team->getTeamID())
-					    .detail(
-					        "Shards",
-					        self->shardsAffectedByTeamFailure
-					            ->getShardsFor(ShardsAffectedByTeamFailure::Team(team->getServerIDs(), self->primary))
-					            .size());
-					if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-						wait(yield());
-					}
+				    .detail("TeamIndex", i)
+				    .detail("Healthy", team->isHealthy())
+				    .detail("TeamSize", team->size())
+				    .detail("MemberIDs", team->getServerIDsStr())
+				    .detail("Primary", self->isPrimary())
+				    .detail("TeamID", team->getTeamID())
+				    .detail("Shards",
+				            self->shardsAffectedByTeamFailure
+				                ->getShardsFor(ShardsAffectedByTeamFailure::Team(team->getServerIDs(), self->primary))
+				                .size());
+				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
+					wait(yield());
 				}
-
-				TraceEvent("MachineInfo", self->getDistributorId())
-				    .detail("Size", machine_info.size())
-				    .detail("Primary", self->isPrimary());
-				state std::map<Standalone<StringRef>, Reference<TCMachineInfo>>::iterator machine =
-				    machine_info.begin();
-				state bool isMachineHealthy = false;
-				for (i = 0; i < machine_info.size(); i++) {
-					Reference<TCMachineInfo> _machine = machine->second;
-					if (!_machine.isValid() || machine_info.find(_machine->machineID) == machine_info.end() ||
-					    _machine->serversOnMachine.empty()) {
-						isMachineHealthy = false;
-					}
-
-					// Healthy machine has at least one healthy server
-					for (auto& server : _machine->serversOnMachine) {
-						// ServerStatus might not be known if server was very recently added and
-						// storageServerFailureTracker() has not yet updated self->server_status If the UID is not
-						// found, do not assume the server is healthy
-						auto it = server_status.find(server->getId());
-						if (it != server_status.end() && !it->second.isUnhealthy()) {
-							isMachineHealthy = true;
-						}
-					}
-
-					isMachineHealthy = false;
-					TraceEvent("MachineInfo", self->getDistributorId())
-					    .detail("MachineInfoIndex", i)
-					    .detail("Healthy", isMachineHealthy)
-					    .detail("MachineID", machine->first.contents().toString())
-					    .detail("MachineTeamOwned", machine->second->machineTeams.size())
-					    .detail("ServerNumOnMachine", machine->second->serversOnMachine.size())
-					    .detail("ServersID", machine->second->getServersIDStr())
-					    .detail("Primary", self->isPrimary());
-					machine++;
-					if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-						wait(yield());
-					}
-				}
-
-				TraceEvent("MachineTeamInfo", self->getDistributorId())
-				    .detail("Size", machineTeams.size())
-				    .detail("Primary", self->isPrimary());
-				for (i = 0; i < machineTeams.size(); i++) {
-					const auto& team = machineTeams[i];
-					TraceEvent("MachineTeamInfo", self->getDistributorId())
-					    .detail("TeamIndex", i)
-					    .detail("MachineIDs", team->getMachineIDsStr())
-					    .detail("ServerTeams", team->getServerTeams().size())
-					    .detail("Primary", self->isPrimary());
-					if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-						wait(yield());
-					}
-				}
-
-				// TODO: re-enable the following logging or remove them.
-				// TraceEvent("LocalityRecordKeyName", self->getDistributorId())
-				//     .detail("Size", internedLocalityRecordKeyNameStrings.size())
-				//     .detail("Primary", self->isPrimary());
-				// for (i = 0; i < internedLocalityRecordKeyNameStrings.size(); i++) {
-				// 	TraceEvent("LocalityRecordKeyIndexName", self->getDistributorId())
-				// 	    .detail("KeyIndex", i)
-				// 	    .detail("KeyName", internedLocalityRecordKeyNameStrings[i])
-				// 	    .detail("Primary", self->isPrimary());
-				// 	if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-				// 		wait(yield());
-				// 	}
-				// }
-
-				// TraceEvent("MachineLocalityMap", self->getDistributorId())
-				//     .detail("Size", machineLocalityMapEntryArraySize)
-				//     .detail("Primary", self->isPrimary());
-				// for (i = 0; i < serverIDs.size(); i++) {
-				// 	const auto& serverID = serverIDs[i];
-				// 	Reference<LocalityRecord> record = machineLocalityMapRecordArray[i];
-				// 	if (record.isValid()) {
-				// 		TraceEvent("MachineLocalityMap", self->getDistributorId())
-				// 		    .detail("LocalityIndex", i)
-				// 		    .detail("UID", serverID->toString())
-				// 		    .detail("LocalityRecord", record->toString())
-				// 		    .detail("Primary", self->isPrimary());
-				// 	} else {
-				// 		TraceEvent("MachineLocalityMap", self->getDistributorId())
-				// 		    .detail("LocalityIndex", i)
-				// 		    .detail("UID", serverID->toString())
-				// 		    .detail("LocalityRecord", "[NotFound]")
-				// 		    .detail("Primary", self->isPrimary());
-				// 	}
-				// 	if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-				// 		wait(yield());
-				// 	}
-				// }
-			} catch (Error& e) {
-				wait(tr.onError(e));
 			}
+
+			TraceEvent("MachineInfo", self->getDistributorId())
+			    .detail("Size", machine_info.size())
+			    .detail("Primary", self->isPrimary());
+
+			state std::map<Standalone<StringRef>, Reference<TCMachineInfo>>::iterator machine = machine_info.begin();
+			state bool isMachineHealthy = false;
+			for (i = 0; i < machine_info.size(); i++) {
+				Reference<TCMachineInfo> _machine = machine->second;
+				if (!_machine.isValid() || machine_info.find(_machine->machineID) == machine_info.end() ||
+				    _machine->serversOnMachine.empty()) {
+					isMachineHealthy = false;
+				}
+
+				// Healthy machine has at least one healthy server
+				for (auto& server : _machine->serversOnMachine) {
+					// ServerStatus might not be known if server was very recently added and
+					// storageServerFailureTracker() has not yet updated self->server_status If the UID is not
+					// found, do not assume the server is healthy
+					auto it = server_status.find(server->getId());
+					if (it != server_status.end() && !it->second.isUnhealthy()) {
+						isMachineHealthy = true;
+					}
+				}
+
+				isMachineHealthy = false;
+				TraceEvent("MachineInfo", self->getDistributorId())
+				    .detail("MachineInfoIndex", i)
+				    .detail("Healthy", isMachineHealthy)
+				    .detail("MachineID", machine->first.contents().toString())
+				    .detail("MachineTeamOwned", machine->second->machineTeams.size())
+				    .detail("ServerNumOnMachine", machine->second->serversOnMachine.size())
+				    .detail("ServersID", machine->second->getServersIDStr())
+				    .detail("Primary", self->isPrimary());
+				machine++;
+				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
+					wait(yield());
+				}
+			}
+
+			TraceEvent("MachineTeamInfo", self->getDistributorId())
+			    .detail("Size", machineTeams.size())
+			    .detail("Primary", self->isPrimary());
+
+			for (i = 0; i < machineTeams.size(); i++) {
+				const auto& team = machineTeams[i];
+				TraceEvent("MachineTeamInfo", self->getDistributorId())
+				    .detail("TeamIndex", i)
+				    .detail("MachineIDs", team->getMachineIDsStr())
+				    .detail("ServerTeams", team->getServerTeams().size())
+				    .detail("Primary", self->isPrimary());
+				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
+					wait(yield());
+				}
+			}
+
+			// TODO: re-enable the following logging or remove them.
+			// TraceEvent("LocalityRecordKeyName", self->getDistributorId())
+			//     .detail("Size", internedLocalityRecordKeyNameStrings.size())
+			//     .detail("Primary", self->isPrimary());
+			// for (i = 0; i < internedLocalityRecordKeyNameStrings.size(); i++) {
+			// 	TraceEvent("LocalityRecordKeyIndexName", self->getDistributorId())
+			// 	    .detail("KeyIndex", i)
+			// 	    .detail("KeyName", internedLocalityRecordKeyNameStrings[i])
+			// 	    .detail("Primary", self->isPrimary());
+			// 	if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
+			// 		wait(yield());
+			// 	}
+			// }
+
+			// TraceEvent("MachineLocalityMap", self->getDistributorId())
+			//     .detail("Size", machineLocalityMapEntryArraySize)
+			//     .detail("Primary", self->isPrimary());
+			// for (i = 0; i < serverIDs.size(); i++) {
+			// 	const auto& serverID = serverIDs[i];
+			// 	Reference<LocalityRecord> record = machineLocalityMapRecordArray[i];
+			// 	if (record.isValid()) {
+			// 		TraceEvent("MachineLocalityMap", self->getDistributorId())
+			// 		    .detail("LocalityIndex", i)
+			// 		    .detail("UID", serverID->toString())
+			// 		    .detail("LocalityRecord", record->toString())
+			// 		    .detail("Primary", self->isPrimary());
+			// 	} else {
+			// 		TraceEvent("MachineLocalityMap", self->getDistributorId())
+			// 		    .detail("LocalityIndex", i)
+			// 		    .detail("UID", serverID->toString())
+			// 		    .detail("LocalityRecord", "[NotFound]")
+			// 		    .detail("Primary", self->isPrimary());
+			// 	}
+			// 	if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
+			// 		wait(yield());
+			// 	}
+			// }
 		}
 	}
 
 }; // class DDTeamCollectionImpl
 
+int32_t DDTeamCollection::getTargetTSSInDC() const {
+	int32_t targetTSSInDC = configuration.desiredTSSCount;
+	if (configuration.usableRegions > 1) {
+		targetTSSInDC /= configuration.usableRegions;
+		if (primary) {
+			// put extras in primary DC if it's uneven
+			targetTSSInDC += (configuration.desiredTSSCount % configuration.usableRegions);
+		}
+	}
+	return targetTSSInDC;
+}
+
+bool DDTeamCollection::reachTSSPairTarget() const {
+	return tss_info_by_pair.size() >= getTargetTSSInDC();
+}
+
 Reference<TCMachineTeamInfo> DDTeamCollection::findMachineTeam(
     std::vector<Standalone<StringRef>> const& machineIDs) const {
 	if (machineIDs.empty()) {
@@ -3467,6 +3420,7 @@ Future<Void> DDTeamCollection::serverTeamRemover() {
 }
 
 Future<Void> DDTeamCollection::trackExcludedServers() {
+	ASSERT(!db->isMocked());
 	return DDTeamCollectionImpl::trackExcludedServers(this);
 }
 
@@ -3489,6 +3443,7 @@ Future<Void> DDTeamCollection::perpetualStorageWiggler(AsyncVar<bool>& stopSigna
 }
 
 Future<Void> DDTeamCollection::monitorPerpetualStorageWiggle() {
+	ASSERT(!db->isMocked());
 	return DDTeamCollectionImpl::monitorPerpetualStorageWiggle(this);
 }
 
@@ -3498,6 +3453,7 @@ Future<Void> DDTeamCollection::waitServerListChange(FutureStream<Void> serverRem
 }
 
 Future<Void> DDTeamCollection::waitHealthyZoneChange() {
+	ASSERT(!db->isMocked());
 	return DDTeamCollectionImpl::waitHealthyZoneChange(this);
 }
 
@@ -3531,11 +3487,27 @@ Future<Void> DDTeamCollection::monitorHealthyTeams() {
 }
 
 Future<UID> DDTeamCollection::getClusterId() {
-	return DDTeamCollectionImpl::getClusterId(this);
+	return db->getClusterId();
 }
 
 Future<UID> DDTeamCollection::getNextWigglingServerID() {
-	return DDTeamCollectionImpl::getNextWigglingServerID(this);
+	Optional<Value> localityKey;
+	Optional<Value> localityValue;
+
+	// NOTE: because normal \xff/conf change through `changeConfig` now will cause DD throw `movekeys_conflict()`
+	// then recruit a new DD, we only need to read current configuration once
+	if (configuration.perpetualStorageWiggleLocality != "0") {
+		// parsing format is like "datahall:0"
+		std::string& localityKeyValue = configuration.perpetualStorageWiggleLocality;
+		ASSERT(isValidPerpetualStorageWiggleLocality(localityKeyValue));
+		// get key and value from perpetual_storage_wiggle_locality.
+		int split = localityKeyValue.find(':');
+		localityKey = Optional<Value>(ValueRef((uint8_t*)localityKeyValue.c_str(), split));
+		localityValue = Optional<Value>(
+		    ValueRef((uint8_t*)localityKeyValue.c_str() + split + 1, localityKeyValue.size() - split - 1));
+	}
+
+	return DDTeamCollectionImpl::getNextWigglingServerID(storageWiggler, localityKey, localityValue, this);
 }
 
 Future<Void> DDTeamCollection::readStorageWiggleMap() {
@@ -3571,7 +3543,7 @@ bool DDTeamCollection::satisfiesPolicy(const std::vector<Reference<TCServerInfo>
 	return result && resultEntries.size() == 0;
 }
 
-DDTeamCollection::DDTeamCollection(Database const& cx,
+DDTeamCollection::DDTeamCollection(Reference<IDDTxnProcessor>& db,
                                    UID distributorId,
                                    MoveKeysLock const& lock,
                                    PromiseStream<RelocateShard> const& output,
@@ -3587,7 +3559,7 @@ DDTeamCollection::DDTeamCollection(Database const& cx,
                                    PromiseStream<GetMetricsRequest> getShardMetrics,
                                    Promise<UID> removeFailedServer,
                                    PromiseStream<Promise<int>> getUnhealthyRelocationCount)
-  : doBuildTeams(true), lastBuildTeamsFailed(false), teamBuilder(Void()), lock(lock), output(output),
+  : db(db), doBuildTeams(true), lastBuildTeamsFailed(false), teamBuilder(Void()), lock(lock), output(output),
     unhealthyServers(0), storageWiggler(makeReference<StorageWiggler>(this)), processingWiggle(processingWiggle),
     shardsAffectedByTeamFailure(shardsAffectedByTeamFailure),
     initialFailureReactionDelay(
@@ -3605,8 +3577,13 @@ DDTeamCollection::DDTeamCollection(Database const& cx,
     teamCollectionInfoEventHolder(makeReference<EventCacheHolder>("TeamCollectionInfo")),
     storageServerRecruitmentEventHolder(
         makeReference<EventCacheHolder>("StorageServerRecruitment_" + distributorId.toString())),
-    primary(primary), distributorId(distributorId), cx(cx), configuration(configuration),
+    primary(primary), distributorId(distributorId), configuration(configuration),
     storageServerSet(new LocalityMap<UID>()) {
+
+	if (!db->isMocked()) {
+		cx = this->db->context();
+	}
+
 	if (!primary || configuration.usableRegions == 1) {
 		TraceEvent("DDTrackerStarting", distributorId)
 		    .detail("State", "Inactive")
@@ -4808,7 +4785,7 @@ Reference<TCMachineInfo> DDTeamCollection::checkAndCreateMachine(Reference<TCSer
 	Reference<TCMachineInfo> machineInfo;
 	if (machine_info.find(machine_id) == machine_info.end()) {
 		// uid is the first storage server process on the machine
-		TEST(true); // First storage server in process on the machine
+		CODE_PROBE(true, "First storage server in process on the machine");
 		// For each machine, store the first server's localityEntry into machineInfo for later use.
 		LocalityEntry localityEntry = machineLocalityMap.add(locality, &server->getId());
 		machineInfo = makeReference<TCMachineInfo>(server, localityEntry);
@@ -5131,18 +5108,19 @@ Future<Void> DDTeamCollection::printSnapshotTeamsInfo(Reference<DDTeamCollection
 }
 
 class DDTeamCollectionUnitTest {
+public:
 	static std::unique_ptr<DDTeamCollection> testTeamCollection(int teamSize,
 	                                                            Reference<IReplicationPolicy> policy,
 	                                                            int processCount) {
 		Database database = DatabaseContext::create(
 		    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::False);
-
+		auto txnProcessor = Reference<IDDTxnProcessor>(new DDTxnProcessor(database));
 		DatabaseConfiguration conf;
 		conf.storageTeamSize = teamSize;
 		conf.storagePolicy = policy;
 
 		auto collection =
-		    std::unique_ptr<DDTeamCollection>(new DDTeamCollection(database,
+		    std::unique_ptr<DDTeamCollection>(new DDTeamCollection(txnProcessor,
 		                                                           UID(0, 0),
 		                                                           MoveKeysLock(),
 		                                                           PromiseStream<RelocateShard>(),
@@ -5163,9 +5141,9 @@ class DDTeamCollectionUnitTest {
 			UID uid(id, 0);
 			StorageServerInterface interface;
 			interface.uniqueID = uid;
-			interface.locality.set(LiteralStringRef("machineid"), Standalone<StringRef>(std::to_string(id)));
-			interface.locality.set(LiteralStringRef("zoneid"), Standalone<StringRef>(std::to_string(id % 5)));
-			interface.locality.set(LiteralStringRef("data_hall"), Standalone<StringRef>(std::to_string(id % 3)));
+			interface.locality.set("machineid"_sr, Standalone<StringRef>(std::to_string(id)));
+			interface.locality.set("zoneid"_sr, Standalone<StringRef>(std::to_string(id % 5)));
+			interface.locality.set("data_hall"_sr, Standalone<StringRef>(std::to_string(id % 3)));
 			collection->server_info[uid] = makeReference<TCServerInfo>(
 			    interface, collection.get(), ProcessClass(), true, collection->storageServerSet);
 			collection->server_status.set(uid, ServerStatus(false, false, false, interface.locality));
@@ -5180,13 +5158,13 @@ class DDTeamCollectionUnitTest {
 	                                                                   int processCount) {
 		Database database = DatabaseContext::create(
 		    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::False);
-
+		auto txnProcessor = Reference<IDDTxnProcessor>(new DDTxnProcessor(database));
 		DatabaseConfiguration conf;
 		conf.storageTeamSize = teamSize;
 		conf.storagePolicy = policy;
 
 		auto collection =
-		    std::unique_ptr<DDTeamCollection>(new DDTeamCollection(database,
+		    std::unique_ptr<DDTeamCollection>(new DDTeamCollection(txnProcessor,
 		                                                           UID(0, 0),
 		                                                           MoveKeysLock(),
 		                                                           PromiseStream<RelocateShard>(),
@@ -5218,11 +5196,11 @@ class DDTeamCollectionUnitTest {
 			       zone_id,
 			       machine_id,
 			       interface.address().toString().c_str());
-			interface.locality.set(LiteralStringRef("processid"), Standalone<StringRef>(std::to_string(process_id)));
-			interface.locality.set(LiteralStringRef("machineid"), Standalone<StringRef>(std::to_string(machine_id)));
-			interface.locality.set(LiteralStringRef("zoneid"), Standalone<StringRef>(std::to_string(zone_id)));
-			interface.locality.set(LiteralStringRef("data_hall"), Standalone<StringRef>(std::to_string(data_hall_id)));
-			interface.locality.set(LiteralStringRef("dcid"), Standalone<StringRef>(std::to_string(dc_id)));
+			interface.locality.set("processid"_sr, Standalone<StringRef>(std::to_string(process_id)));
+			interface.locality.set("machineid"_sr, Standalone<StringRef>(std::to_string(machine_id)));
+			interface.locality.set("zoneid"_sr, Standalone<StringRef>(std::to_string(zone_id)));
+			interface.locality.set("data_hall"_sr, Standalone<StringRef>(std::to_string(data_hall_id)));
+			interface.locality.set("dcid"_sr, Standalone<StringRef>(std::to_string(dc_id)));
 			collection->server_info[uid] = makeReference<TCServerInfo>(
 			    interface, collection.get(), ProcessClass(), true, collection->storageServerSet);
 
@@ -5235,7 +5213,6 @@ class DDTeamCollectionUnitTest {
 		return collection;
 	}
 
-public:
 	ACTOR static Future<Void> AddTeamsBestOf_UseMachineID() {
 		wait(Future<Void>(Void()));
 
@@ -5869,3 +5846,78 @@ TEST_CASE("/DataDistribution/GetTeam/DeprioritizeWigglePausedTeam") {
 	wait(DDTeamCollectionUnitTest::GetTeam_DeprioritizeWigglePausedTeam());
 	return Void();
 }
+
+TEST_CASE("/DataDistribution/StorageWiggler/NextIdWithMinAge") {
+	state StorageWiggler wiggler(nullptr);
+	state double startTime = now();
+	wiggler.addServer(UID(1, 0),
+	                  StorageMetadataType(startTime - SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC + 5.0,
+	                                      KeyValueStoreType::SSD_BTREE_V2));
+	wiggler.addServer(UID(2, 0),
+	                  StorageMetadataType(
+	                      startTime + SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC, KeyValueStoreType::MEMORY, true));
+	wiggler.addServer(UID(3, 0), StorageMetadataType(startTime - 5.0, KeyValueStoreType::SSD_ROCKSDB_V1, true));
+	wiggler.addServer(UID(4, 0),
+	                  StorageMetadataType(startTime - SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC - 1.0,
+	                                      KeyValueStoreType::SSD_BTREE_V2));
+	std::vector<Optional<UID>> correctResult{ UID(3, 0), UID(2, 0), UID(4, 0), Optional<UID>() };
+	for (int i = 0; i < 4; ++i) {
+		auto id = wiggler.getNextServerId();
+		ASSERT(id == correctResult[i]);
+	}
+
+	{
+		std::cout << "Finish Initial Check. Start test getNextWigglingServerID() loop...\n";
+		// test the getNextWigglingServerID() loop
+		UID id = wait(DDTeamCollectionImpl::getNextWigglingServerID(Reference<StorageWiggler>::addRef(&wiggler)));
+		ASSERT(id == UID(1, 0));
+	}
+
+	std::cout << "Test after addServer() ...\n";
+	state Future<UID> nextFuture =
+	    DDTeamCollectionImpl::getNextWigglingServerID(Reference<StorageWiggler>::addRef(&wiggler));
+	ASSERT(!nextFuture.isReady());
+	startTime = now();
+	StorageMetadataType metadata(startTime + SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC + 100.0,
+	                             KeyValueStoreType::SSD_BTREE_V2);
+	wiggler.addServer(UID(5, 0), metadata);
+	ASSERT(!nextFuture.isReady());
+
+	std::cout << "Test after updateServer() ...\n";
+	StorageWiggler* ptr = &wiggler;
+	wait(trigger(
+	    [ptr]() {
+		    ptr->updateMetadata(UID(5, 0),
+		                        StorageMetadataType(now() - SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC,
+		                                            KeyValueStoreType::SSD_BTREE_V2));
+	    },
+	    delay(5.0)));
+	wait(success(nextFuture));
+	ASSERT(now() - startTime < SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC + 100.0);
+	ASSERT(nextFuture.get() == UID(5, 0));
+	return Void();
+}
+
+TEST_CASE("/DataDistribution/StorageWiggler/NextIdWithTSS") {
+	state std::unique_ptr<DDTeamCollection> collection =
+	    DDTeamCollectionUnitTest::testMachineTeamCollection(1, Reference<IReplicationPolicy>(new PolicyOne()), 5);
+	state StorageWiggler wiggler(collection.get());
+
+	std::cout << "Test when need TSS ... \n";
+	collection->configuration.usableRegions = 1;
+	collection->configuration.desiredTSSCount = 1;
+	state double startTime = now();
+	wiggler.addServer(UID(1, 0),
+	                  StorageMetadataType(startTime + SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC + 150.0,
+	                                      KeyValueStoreType::SSD_BTREE_V2));
+	wiggler.addServer(UID(2, 0),
+	                  StorageMetadataType(startTime + SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC + 150.0,
+	                                      KeyValueStoreType::SSD_BTREE_V2));
+	ASSERT(!wiggler.getNextServerId(true).present());
+	ASSERT(wiggler.getNextServerId(collection->reachTSSPairTarget()) == UID(1, 0));
+	UID id = wait(DDTeamCollectionImpl::getNextWigglingServerID(
+	    Reference<StorageWiggler>::addRef(&wiggler), Optional<Value>(), Optional<Value>(), collection.get()));
+	ASSERT(now() - startTime < SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC + 150.0);
+	ASSERT(id == UID(2, 0));
+	return Void();
+}
\ No newline at end of file
diff --git a/fdbserver/DDTxnProcessor.actor.cpp b/fdbserver/DDTxnProcessor.actor.cpp
index 2d3279163c..57c32537df 100644
--- a/fdbserver/DDTxnProcessor.actor.cpp
+++ b/fdbserver/DDTxnProcessor.actor.cpp
@@ -22,11 +22,26 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbserver/DataDistribution.actor.h"
+#include "fdbclient/DatabaseContext.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 class DDTxnProcessorImpl {
 	friend class DDTxnProcessor;
 
+	ACTOR static Future<ServerWorkerInfos> getServerListAndProcessClasses(Database cx) {
+		state Transaction tr(cx);
+		state ServerWorkerInfos res;
+		loop {
+			try {
+				wait(store(res.servers, NativeAPI::getServerListAndProcessClasses(&tr)));
+				res.readVersion = tr.getReadVersion().get();
+				return res;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
 	// return {sourceServers, completeSources}
 	ACTOR static Future<IDDTxnProcessor::SourceServers> getSourceServersForRange(Database cx, KeyRangeRef keys) {
 		state std::set<UID> servers;
@@ -126,6 +141,43 @@ class DDTxnProcessorImpl {
 		return Void();
 	}
 
+	ACTOR static Future<int> tryUpdateReplicasKeyForDc(Database cx, Optional<Key> dcId, int storageTeamSize) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				Optional<Value> val = wait(tr.get(datacenterReplicasKeyFor(dcId)));
+				state int oldReplicas = val.present() ? decodeDatacenterReplicasValue(val.get()) : 0;
+				if (oldReplicas == storageTeamSize) {
+					return oldReplicas;
+				}
+				if (oldReplicas < storageTeamSize) {
+					tr.set(rebootWhenDurableKey, StringRef());
+				}
+				tr.set(datacenterReplicasKeyFor(dcId), datacenterReplicasValue(storageTeamSize));
+				wait(tr.commit());
+
+				return oldReplicas;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR static Future<UID> getClusterId(Database cx) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				Optional<Value> clusterId = wait(tr.get(clusterIdKey));
+				ASSERT(clusterId.present());
+				return BinaryReader::fromStringRef<UID>(clusterId.get(), Unversioned());
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
 	// Read keyservers, return unique set of teams
 	ACTOR static Future<Reference<InitialDataDistribution>> getInitialDataDistribution(
 	    Database cx,
@@ -250,7 +302,7 @@ class DDTxnProcessorImpl {
 		// If keyServers is too large to read in a single transaction, then we will have to break this process up into
 		// multiple transactions. In that case, each iteration should begin where the previous left off
 		while (beginKey < allKeys.end) {
-			TEST(beginKey > allKeys.begin); // Multi-transactional getInitialDataDistribution
+			CODE_PROBE(beginKey > allKeys.begin, "Multi-transactional getInitialDataDistribution");
 			loop {
 				succeeded = false;
 				try {
@@ -349,7 +401,7 @@ class DDTxnProcessorImpl {
 		// a dummy shard at the end with no keys or servers makes life easier for trackInitialShards()
 		result->shards.push_back(DDShardInfo(allKeys.end));
 
-		if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA && numDataMoves > 0) {
+		if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && numDataMoves > 0) {
 			for (int shard = 0; shard < result->shards.size() - 1; ++shard) {
 				const DDShardInfo& iShard = result->shards[shard];
 				KeyRangeRef keys = KeyRangeRef(iShard.key, result->shards[shard + 1].key);
@@ -395,18 +447,101 @@ class DDTxnProcessorImpl {
 			}
 		}
 	}
+
+	ACTOR static Future<bool> isDataDistributionEnabled(Database cx, const DDEnabledState* ddEnabledState) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				Optional<Value> mode = wait(tr.get(dataDistributionModeKey));
+				if (!mode.present() && ddEnabledState->isDDEnabled())
+					return true;
+				if (mode.present()) {
+					BinaryReader rd(mode.get(), Unversioned());
+					int m;
+					rd >> m;
+					if (m && ddEnabledState->isDDEnabled()) {
+						TraceEvent(SevDebug, "IsDDEnabledSucceeded")
+						    .detail("Mode", m)
+						    .detail("IsDDEnabled", ddEnabledState->isDDEnabled());
+						return true;
+					}
+				}
+				// SOMEDAY: Write a wrapper in MoveKeys.actor.h
+				Optional<Value> readVal = wait(tr.get(moveKeysLockOwnerKey));
+				UID currentOwner =
+				    readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
+				if (ddEnabledState->isDDEnabled() && (currentOwner != dataDistributionModeLock)) {
+					TraceEvent(SevDebug, "IsDDEnabledSucceeded")
+					    .detail("CurrentOwner", currentOwner)
+					    .detail("DDModeLock", dataDistributionModeLock)
+					    .detail("IsDDEnabled", ddEnabledState->isDDEnabled());
+					return true;
+				}
+				TraceEvent(SevDebug, "IsDDEnabledFailed")
+				    .detail("CurrentOwner", currentOwner)
+				    .detail("DDModeLock", dataDistributionModeLock)
+				    .detail("IsDDEnabled", ddEnabledState->isDDEnabled());
+				return false;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR static Future<Void> pollMoveKeysLock(Database cx, MoveKeysLock lock, const DDEnabledState* ddEnabledState) {
+		loop {
+			wait(delay(SERVER_KNOBS->MOVEKEYS_LOCK_POLLING_DELAY));
+			state Transaction tr(cx);
+			loop {
+				try {
+					wait(checkMoveKeysLockReadOnly(&tr, lock, ddEnabledState));
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+		}
+	}
+
+	ACTOR static Future<Optional<Value>> readRebalanceDDIgnoreKey(Database cx) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				Optional<Value> res = wait(tr.get(rebalanceDDIgnoreKey));
+				return res;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR static Future<Void> waitDDTeamInfoPrintSignal(Database cx) {
+		state ReadYourWritesTransaction tr(cx);
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				state Future<Void> watchFuture = tr.watch(triggerDDTeamInfoPrintKey);
+				wait(tr.commit());
+				wait(watchFuture);
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
 };
 
 Future<IDDTxnProcessor::SourceServers> DDTxnProcessor::getSourceServersForRange(const KeyRangeRef range) {
 	return DDTxnProcessorImpl::getSourceServersForRange(cx, range);
 }
 
-Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> DDTxnProcessor::getServerListAndProcessClasses() {
-	Transaction tr(cx);
-	return NativeAPI::getServerListAndProcessClasses(&tr);
+Future<ServerWorkerInfos> DDTxnProcessor::getServerListAndProcessClasses() {
+	return DDTxnProcessorImpl::getServerListAndProcessClasses(cx);
 }
 
-Future<MoveKeysLock> DDTxnProcessor::takeMoveKeysLock(UID ddId) const {
+Future<MoveKeysLock> DDTxnProcessor::takeMoveKeysLock(const UID& ddId) const {
 	return ::takeMoveKeysLock(cx, ddId);
 }
 
@@ -430,4 +565,219 @@ Future<Reference<InitialDataDistribution>> DDTxnProcessor::getInitialDataDistrib
 
 Future<Void> DDTxnProcessor::waitForDataDistributionEnabled(const DDEnabledState* ddEnabledState) const {
 	return DDTxnProcessorImpl::waitForDataDistributionEnabled(cx, ddEnabledState);
-}
\ No newline at end of file
+}
+
+Future<bool> DDTxnProcessor::isDataDistributionEnabled(const DDEnabledState* ddEnabledState) const {
+	return DDTxnProcessorImpl::isDataDistributionEnabled(cx, ddEnabledState);
+}
+
+Future<Void> DDTxnProcessor::pollMoveKeysLock(const MoveKeysLock& lock, const DDEnabledState* ddEnabledState) const {
+	return DDTxnProcessorImpl::pollMoveKeysLock(cx, lock, ddEnabledState);
+}
+
+Future<std::pair<Optional<StorageMetrics>, int>> DDTxnProcessor::waitStorageMetrics(
+    const KeyRange& keys,
+    const StorageMetrics& min,
+    const StorageMetrics& max,
+    const StorageMetrics& permittedError,
+    int shardLimit,
+    int expectedShardCount) const {
+	return cx->waitStorageMetrics(keys, min, max, permittedError, shardLimit, expectedShardCount);
+}
+
+Future<Standalone<VectorRef<KeyRef>>> DDTxnProcessor::splitStorageMetrics(const KeyRange& keys,
+                                                                          const StorageMetrics& limit,
+                                                                          const StorageMetrics& estimated,
+                                                                          const Optional<int>& minSplitBytes) const {
+	return cx->splitStorageMetrics(keys, limit, estimated, minSplitBytes);
+}
+
+Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> DDTxnProcessor::getReadHotRanges(const KeyRange& keys) const {
+	return cx->getReadHotRanges(keys);
+}
+
+Future<HealthMetrics> DDTxnProcessor::getHealthMetrics(bool detailed) const {
+	return cx->getHealthMetrics(detailed);
+}
+
+Future<Optional<Value>> DDTxnProcessor::readRebalanceDDIgnoreKey() const {
+	return DDTxnProcessorImpl::readRebalanceDDIgnoreKey(cx);
+}
+
+Future<int> DDTxnProcessor::tryUpdateReplicasKeyForDc(const Optional<Key>& dcId, const int& storageTeamSize) const {
+	return DDTxnProcessorImpl::tryUpdateReplicasKeyForDc(cx, dcId, storageTeamSize);
+}
+
+Future<UID> DDTxnProcessor::getClusterId() const {
+	return DDTxnProcessorImpl::getClusterId(cx);
+}
+
+Future<Void> DDTxnProcessor::waitDDTeamInfoPrintSignal() const {
+	return DDTxnProcessorImpl::waitDDTeamInfoPrintSignal(cx);
+}
+
+Future<ServerWorkerInfos> DDMockTxnProcessor::getServerListAndProcessClasses() {
+	ServerWorkerInfos res;
+	for (auto& [_, mss] : mgs->allServers) {
+		res.servers.emplace_back(mss.ssi, ProcessClass(ProcessClass::StorageClass, ProcessClass::DBSource));
+	}
+	// FIXME(xwang): possible generate version from time?
+	res.readVersion = 0;
+	return res;
+}
+
+std::pair<std::set<std::vector<UID>>, std::set<std::vector<UID>>> getAllTeamsInRegion(
+    const std::vector<DDShardInfo>& shards) {
+	std::set<std::vector<UID>> primary, remote;
+	for (auto& info : shards) {
+		if (!info.primarySrc.empty())
+			primary.emplace(info.primarySrc);
+		if (!info.primaryDest.empty())
+			primary.emplace(info.primaryDest);
+		if (!info.remoteSrc.empty())
+			remote.emplace(info.remoteSrc);
+		if (!info.remoteDest.empty())
+			remote.emplace(info.remoteDest);
+	}
+	return { primary, remote };
+}
+
+inline void transformTeamsToServerIds(std::vector<ShardsAffectedByTeamFailure::Team>& teams,
+                                      std::vector<UID>& primaryIds,
+                                      std::vector<UID>& remoteIds) {
+	std::set<UID> primary, remote;
+	for (auto& team : teams) {
+		team.primary ? primary.insert(team.servers.begin(), team.servers.end())
+		             : remote.insert(team.servers.begin(), team.servers.end());
+	}
+	primaryIds = std::vector<UID>(primary.begin(), primary.end());
+	remoteIds = std::vector<UID>(remote.begin(), remote.end());
+}
+
+// reconstruct DDShardInfos from shardMapping
+std::vector<DDShardInfo> DDMockTxnProcessor::getDDShardInfos() const {
+	std::vector<DDShardInfo> res;
+	res.reserve(mgs->shardMapping->getNumberOfShards());
+	auto allRange = mgs->shardMapping->getAllRanges();
+	ASSERT(allRange.end().begin() == allKeys.end);
+	for (auto it = allRange.begin(); it != allRange.end(); ++it) {
+		// FIXME: now just use anonymousShardId
+		KeyRangeRef curRange = it->range();
+		DDShardInfo info(curRange.begin);
+
+		auto teams = mgs->shardMapping->getTeamsFor(curRange);
+		if (!teams.first.empty() && !teams.second.empty()) {
+			CODE_PROBE(true, "Mock InitialDataDistribution In-Flight shard");
+			info.hasDest = true;
+			info.destId = anonymousShardId;
+			info.srcId = anonymousShardId;
+			transformTeamsToServerIds(teams.second, info.primarySrc, info.remoteSrc);
+			transformTeamsToServerIds(teams.first, info.primaryDest, info.remoteDest);
+		} else if (!teams.first.empty()) {
+			CODE_PROBE(true, "Mock InitialDataDistribution Static shard");
+			info.srcId = anonymousShardId;
+			transformTeamsToServerIds(teams.first, info.primarySrc, info.remoteSrc);
+		} else {
+			ASSERT(false);
+		}
+
+		res.push_back(std::move(info));
+	}
+	res.emplace_back(allKeys.end);
+
+	return res;
+}
+
+Future<Reference<InitialDataDistribution>> DDMockTxnProcessor::getInitialDataDistribution(
+    const UID& distributorId,
+    const MoveKeysLock& moveKeysLock,
+    const std::vector<Optional<Key>>& remoteDcIds,
+    const DDEnabledState* ddEnabledState) {
+
+	// FIXME: now we just ignore ddEnabledState and moveKeysLock, will fix it in the future
+	Reference<InitialDataDistribution> res = makeReference<InitialDataDistribution>();
+	res->mode = 1;
+	res->allServers = getServerListAndProcessClasses().get().servers;
+	res->shards = getDDShardInfos();
+	std::tie(res->primaryTeams, res->remoteTeams) = getAllTeamsInRegion(res->shards);
+	return res;
+}
+
+Future<Void> DDMockTxnProcessor::removeKeysFromFailedServer(const UID& serverID,
+                                                            const std::vector<UID>& teamForDroppedRange,
+                                                            const MoveKeysLock& lock,
+                                                            const DDEnabledState* ddEnabledState) const {
+
+	// This function only takes effect when user exclude failed IP:PORT in the fdbcli. In the first version , the mock
+	// class won’t support this.
+	UNREACHABLE();
+}
+
+Future<Void> DDMockTxnProcessor::removeStorageServer(const UID& serverID,
+                                                     const Optional<UID>& tssPairID,
+                                                     const MoveKeysLock& lock,
+                                                     const DDEnabledState* ddEnabledState) const {
+	ASSERT(mgs->allShardRemovedFromServer(serverID));
+	mgs->allServers.erase(serverID);
+	return Void();
+}
+
+void DDMockTxnProcessor::setupMockGlobalState(Reference<InitialDataDistribution> initData) {
+	for (auto& [ssi, pInfo] : initData->allServers) {
+		mgs->addStorageServer(ssi);
+	}
+	mgs->shardMapping->setCheckMode(ShardsAffectedByTeamFailure::CheckMode::ForceNoCheck);
+
+	for (int i = 0; i < initData->shards.size() - 1; ++i) {
+		// insert to keyServers
+		auto& shardInfo = initData->shards[i];
+		ASSERT(shardInfo.remoteSrc.empty() && shardInfo.remoteDest.empty());
+
+		uint64_t shardBytes =
+		    deterministicRandom()->randomInt(SERVER_KNOBS->MIN_SHARD_BYTES, SERVER_KNOBS->MAX_SHARD_BYTES);
+		KeyRangeRef keys(shardInfo.key, initData->shards[i + 1].key);
+		mgs->shardMapping->assignRangeToTeams(keys, { { shardInfo.primarySrc, true } });
+		if (shardInfo.hasDest) {
+			mgs->shardMapping->moveShard(keys, { { shardInfo.primaryDest, true } });
+		}
+		// insert to serverKeys
+		for (auto& id : shardInfo.primarySrc) {
+			mgs->allServers[id].serverKeys.insert(keys, { MockShardStatus::COMPLETED, shardBytes });
+		}
+		for (auto& id : shardInfo.primaryDest) {
+			mgs->allServers[id].serverKeys.insert(keys, { MockShardStatus::INFLIGHT, shardBytes });
+		}
+	}
+
+	mgs->shardMapping->setCheckMode(ShardsAffectedByTeamFailure::CheckMode::Normal);
+}
+
+// FIXME: finish moveKeys implementation
+Future<Void> DDMockTxnProcessor::moveKeys(const MoveKeysParams& params) {
+	UNREACHABLE();
+}
+
+// FIXME: finish implementation
+Future<HealthMetrics> DDMockTxnProcessor::getHealthMetrics(bool detailed) const {
+	return Future<HealthMetrics>();
+}
+
+// FIXME: finish implementation
+Future<Standalone<VectorRef<KeyRef>>> DDMockTxnProcessor::splitStorageMetrics(
+    const KeyRange& keys,
+    const StorageMetrics& limit,
+    const StorageMetrics& estimated,
+    const Optional<int>& minSplitBytes) const {
+	return Future<Standalone<VectorRef<KeyRef>>>();
+}
+
+// FIXME: finish implementation
+Future<std::pair<Optional<StorageMetrics>, int>> DDMockTxnProcessor::waitStorageMetrics(
+    const KeyRange& keys,
+    const StorageMetrics& min,
+    const StorageMetrics& max,
+    const StorageMetrics& permittedError,
+    int shardLimit,
+    int expectedShardCount) const {
+	return Future<std::pair<Optional<StorageMetrics>, int>>();
+}
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index e4682f44bd..191ba67c5f 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -41,6 +41,7 @@
 #include "fdbserver/TenantCache.h"
 #include "fdbserver/TLogInterface.h"
 #include "fdbserver/WaitFailure.h"
+#include "fdbserver/workloads/workloads.actor.h"
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/BooleanParam.h"
@@ -48,7 +49,7 @@
 #include "flow/serialize.h"
 #include "flow/Trace.h"
 #include "flow/UnitTest.h"
-
+#include "fdbserver/DDSharedContext.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 void DataMove::validateShard(const DDShardInfo& shard, KeyRangeRef range, int priority) {
@@ -102,13 +103,16 @@ void DataMove::validateShard(const DDShardInfo& shard, KeyRangeRef range, int pr
 	}
 }
 
+Future<Void> StorageWiggler::onCheck() const {
+	return delay(MIN_ON_CHECK_DELAY_SEC);
+}
+
 // add server to wiggling queue
 void StorageWiggler::addServer(const UID& serverId, const StorageMetadataType& metadata) {
 	// std::cout << "size: " << pq_handles.size() << " add " << serverId.toString() << " DC: "
 	//           << teamCollection->isPrimary() << std::endl;
 	ASSERT(!pq_handles.count(serverId));
 	pq_handles[serverId] = wiggle_pq.emplace(metadata, serverId);
-	nonEmpty.set(true);
 }
 
 void StorageWiggler::removeServer(const UID& serverId) {
@@ -119,7 +123,6 @@ void StorageWiggler::removeServer(const UID& serverId) {
 		pq_handles.erase(serverId);
 		wiggle_pq.erase(handle);
 	}
-	nonEmpty.set(!wiggle_pq.empty());
 }
 
 void StorageWiggler::updateMetadata(const UID& serverId, const StorageMetadataType& metadata) {
@@ -132,9 +135,16 @@ void StorageWiggler::updateMetadata(const UID& serverId, const StorageMetadataTy
 	wiggle_pq.update(handle, std::make_pair(metadata, serverId));
 }
 
-Optional<UID> StorageWiggler::getNextServerId() {
+bool StorageWiggler::necessary(const UID& serverId, const StorageMetadataType& metadata) const {
+	return metadata.wrongConfigured || (now() - metadata.createdTime > SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC);
+}
+
+Optional<UID> StorageWiggler::getNextServerId(bool necessaryOnly) {
 	if (!wiggle_pq.empty()) {
 		auto [metadata, id] = wiggle_pq.top();
+		if (necessaryOnly && !necessary(id, metadata)) {
+			return {};
+		}
 		wiggle_pq.pop();
 		pq_handles.erase(id);
 		return Optional<UID>(id);
@@ -192,46 +202,6 @@ ACTOR Future<Void> remoteRecovered(Reference<AsyncVar<ServerDBInfo> const> db) {
 	return Void();
 }
 
-ACTOR Future<bool> isDataDistributionEnabled(Database cx, const DDEnabledState* ddEnabledState) {
-	state Transaction tr(cx);
-	loop {
-		try {
-			Optional<Value> mode = wait(tr.get(dataDistributionModeKey));
-			if (!mode.present() && ddEnabledState->isDDEnabled())
-				return true;
-			if (mode.present()) {
-				BinaryReader rd(mode.get(), Unversioned());
-				int m;
-				rd >> m;
-				if (m && ddEnabledState->isDDEnabled()) {
-					TraceEvent(SevDebug, "IsDDEnabledSucceeded")
-					    .detail("Mode", m)
-					    .detail("IsDDEnabled", ddEnabledState->isDDEnabled());
-					return true;
-				}
-			}
-			// SOMEDAY: Write a wrapper in MoveKeys.actor.h
-			Optional<Value> readVal = wait(tr.get(moveKeysLockOwnerKey));
-			UID currentOwner =
-			    readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
-			if (ddEnabledState->isDDEnabled() && (currentOwner != dataDistributionModeLock)) {
-				TraceEvent(SevDebug, "IsDDEnabledSucceeded")
-				    .detail("CurrentOwner", currentOwner)
-				    .detail("DDModeLock", dataDistributionModeLock)
-				    .detail("IsDDEnabled", ddEnabledState->isDDEnabled());
-				return true;
-			}
-			TraceEvent(SevDebug, "IsDDEnabledFailed")
-			    .detail("CurrentOwner", currentOwner)
-			    .detail("DDModeLock", dataDistributionModeLock)
-			    .detail("IsDDEnabled", ddEnabledState->isDDEnabled());
-			return false;
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-}
-
 // Ensures that the serverKeys key space is properly coalesced
 // This method is only used for testing and is not implemented in a manner that is safe for large databases
 ACTOR Future<Void> debugCheckCoalescing(Database cx) {
@@ -274,28 +244,15 @@ static std::set<int> const& normalDDQueueErrors() {
 	return s;
 }
 
-ACTOR Future<Void> pollMoveKeysLock(Database cx, MoveKeysLock lock, const DDEnabledState* ddEnabledState) {
-	loop {
-		wait(delay(SERVER_KNOBS->MOVEKEYS_LOCK_POLLING_DELAY));
-		state Transaction tr(cx);
-		loop {
-			try {
-				wait(checkMoveKeysLockReadOnly(&tr, lock, ddEnabledState));
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-}
-
 struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
+public:
 	Reference<AsyncVar<ServerDBInfo> const> dbInfo;
+	Reference<DDSharedContext> context;
 	UID ddId;
 	PromiseStream<Future<Void>> addActor;
 
 	// State initialized when bootstrap
-	std::shared_ptr<IDDTxnProcessor> txnProcessor;
+	Reference<IDDTxnProcessor> txnProcessor;
 	MoveKeysLock lock;
 	DatabaseConfiguration configuration;
 	std::vector<Optional<Key>> primaryDcId;
@@ -311,10 +268,16 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 	// fully-functional.
 	DDTeamCollection* teamCollection;
 	Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure;
-	PromiseStream<RelocateShard> relocationProducer, relocationConsumer; // comsumer is a yield stream from producer
+	// consumer is a yield stream from producer. The RelocateShard is pushed into relocationProducer and popped from
+	// relocationConsumer (by DDQueue)
+	PromiseStream<RelocateShard> relocationProducer, relocationConsumer;
+	Reference<PhysicalShardCollection> physicalShardCollection;
 
-	DataDistributor(Reference<AsyncVar<ServerDBInfo> const> const& db, UID id)
-	  : dbInfo(db), ddId(id), txnProcessor(nullptr), initialDDEventHolder(makeReference<EventCacheHolder>("InitialDD")),
+	StorageQuotaInfo storageQuotaInfo;
+
+	DataDistributor(Reference<AsyncVar<ServerDBInfo> const> const& db, UID id, Reference<DDSharedContext> context)
+	  : dbInfo(db), context(context), ddId(id), txnProcessor(nullptr),
+	    initialDDEventHolder(makeReference<EventCacheHolder>("InitialDD")),
 	    movingDataEventHolder(makeReference<EventCacheHolder>("MovingData")),
 	    totalDataInFlightEventHolder(makeReference<EventCacheHolder>("TotalDataInFlight")),
 	    totalDataInFlightRemoteEventHolder(makeReference<EventCacheHolder>("TotalDataInFlightRemote")),
@@ -330,13 +293,13 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 		return txnProcessor->updateReplicaKeys(primaryDcId, remoteDcIds, configuration);
 	}
 
-	Future<Void> loadInitialDataDistribution(const DDEnabledState* ddEnabledState) {
+	Future<Void> loadInitialDataDistribution() {
 		return store(initData,
 		             txnProcessor->getInitialDataDistribution(
 		                 ddId,
 		                 lock,
 		                 configuration.usableRegions > 1 ? remoteDcIds : std::vector<Optional<Key>>(),
-		                 ddEnabledState));
+		                 context->ddEnabledState.get()));
 	}
 
 	void initDcInfo() {
@@ -351,14 +314,14 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 		}
 	}
 
-	Future<Void> waitDataDistributorEnabled(const DDEnabledState* ddEnabledState) const {
-		return txnProcessor->waitForDataDistributionEnabled(ddEnabledState);
+	Future<Void> waitDataDistributorEnabled() const {
+		return txnProcessor->waitForDataDistributionEnabled(context->ddEnabledState.get());
 	}
 
 	// Initialize the required internal states of DataDistributor. It's necessary before DataDistributor start working.
 	// Doesn't include initialization of optional components, like TenantCache, DDQueue, Tracker, TeamCollection. The
 	// components should call its own ::init methods.
-	ACTOR static Future<Void> init(Reference<DataDistributor> self, const DDEnabledState* ddEnabledState) {
+	ACTOR static Future<Void> init(Reference<DataDistributor> self) {
 		loop {
 			TraceEvent("DDInitTakingMoveKeysLock", self->ddId).log();
 			wait(self->takeMoveKeysLock());
@@ -371,7 +334,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 			wait(self->updateReplicaKeys());
 			TraceEvent("DDInitUpdatedReplicaKeys", self->ddId).log();
 
-			wait(self->loadInitialDataDistribution(ddEnabledState));
+			wait(self->loadInitialDataDistribution());
 
 			if (self->initData->shards.size() > 1) {
 				TraceEvent("DDInitGotInitialDD", self->ddId)
@@ -389,7 +352,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				    .trackLatest(self->initialDDEventHolder->trackingKey);
 			}
 
-			if (self->initData->mode && ddEnabledState->isDDEnabled()) {
+			if (self->initData->mode && self->context->isDDEnabled()) {
 				// mode may be set true by system operator using fdbcli and isDDEnabled() set to true
 				break;
 			}
@@ -430,16 +393,26 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 			    .detail("HighestPriority", self->configuration.usableRegions > 1 ? 0 : -1)
 			    .trackLatest(self->totalDataInFlightRemoteEventHolder->trackingKey);
 
-			wait(self->waitDataDistributorEnabled(ddEnabledState));
+			wait(self->waitDataDistributorEnabled());
 			TraceEvent("DataDistributionEnabled").log();
 		}
 		return Void();
 	}
 
-	// Resume inflight relocations from the previous DD
-	// TODO: add a test to verify the inflight relocation correctness and measure the memory usage with 4 million shards
-	ACTOR static Future<Void> resumeRelocations(Reference<DataDistributor> self) {
-		ASSERT(self->shardsAffectedByTeamFailure); // has to be allocated
+	ACTOR static Future<Void> resumeFromShards(Reference<DataDistributor> self, bool traceShard) {
+		// All physicalShard init must be completed before issuing data move
+		if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+			for (int i = 0; i < self->initData->shards.size() - 1; i++) {
+				const DDShardInfo& iShard = self->initData->shards[i];
+				KeyRangeRef keys = KeyRangeRef(iShard.key, self->initData->shards[i + 1].key);
+				std::vector<ShardsAffectedByTeamFailure::Team> teams;
+				teams.push_back(ShardsAffectedByTeamFailure::Team(iShard.primarySrc, true));
+				if (self->configuration.usableRegions > 1) {
+					teams.push_back(ShardsAffectedByTeamFailure::Team(iShard.remoteSrc, false));
+				}
+				self->physicalShardCollection->initPhysicalShardCollection(keys, teams, iShard.srcId.first(), 0);
+			}
+		}
 
 		state int shard = 0;
 		for (; shard < self->initData->shards.size() - 1; shard++) {
@@ -452,8 +425,8 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 			if (self->configuration.usableRegions > 1) {
 				teams.push_back(ShardsAffectedByTeamFailure::Team(iShard.remoteSrc, false));
 			}
-			if (g_network->isSimulated()) {
-				TraceEvent("DDInitShard")
+			if (traceShard) {
+				TraceEvent(SevDebug, "DDInitShard")
 				    .detail("Keys", keys)
 				    .detail("PrimarySrc", describe(iShard.primarySrc))
 				    .detail("RemoteSrc", describe(iShard.remoteSrc))
@@ -472,20 +445,27 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				if (!unhealthy && self->configuration.usableRegions > 1) {
 					unhealthy = iShard.remoteSrc.size() != self->configuration.storageTeamSize;
 				}
-				self->relocationProducer.send(RelocateShard(keys,
-				                                            unhealthy ? SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY
-				                                                      : SERVER_KNOBS->PRIORITY_RECOVER_MOVE,
-				                                            RelocateReason::OTHER));
+				self->relocationProducer.send(
+				    RelocateShard(keys,
+				                  unhealthy ? DataMovementReason::TEAM_UNHEALTHY : DataMovementReason::RECOVER_MOVE,
+				                  RelocateReason::OTHER));
 			}
 
 			wait(yield(TaskPriority::DataDistribution));
 		}
+		return Void();
+	}
 
+	// TODO: unit test needed
+	ACTOR static Future<Void> resumeFromDataMoves(Reference<DataDistributor> self, Future<Void> readyToStart) {
 		state KeyRangeMap<std::shared_ptr<DataMove>>::iterator it = self->initData->dataMoveMap.ranges().begin();
+
+		wait(readyToStart);
+
 		for (; it != self->initData->dataMoveMap.ranges().end(); ++it) {
 			const DataMoveMetaData& meta = it.value()->meta;
-			if (it.value()->isCancelled() || (it.value()->valid && !CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA)) {
-				RelocateShard rs(meta.range, SERVER_KNOBS->PRIORITY_RECOVER_MOVE, RelocateReason::OTHER);
+			if (it.value()->isCancelled() || (it.value()->valid && !SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA)) {
+				RelocateShard rs(meta.range, DataMovementReason::RECOVER_MOVE, RelocateReason::OTHER);
 				rs.dataMoveId = meta.id;
 				rs.cancelled = true;
 				self->relocationProducer.send(rs);
@@ -494,7 +474,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				TraceEvent(SevDebug, "DDInitFoundDataMove", self->ddId).detail("DataMove", meta.toString());
 				ASSERT(meta.range == it.range());
 				// TODO: Persist priority in DataMoveMetaData.
-				RelocateShard rs(meta.range, SERVER_KNOBS->PRIORITY_RECOVER_MOVE, RelocateReason::OTHER);
+				RelocateShard rs(meta.range, DataMovementReason::RECOVER_MOVE, RelocateReason::OTHER);
 				rs.dataMoveId = meta.id;
 				rs.dataMove = it.value();
 				std::vector<ShardsAffectedByTeamFailure::Team> teams;
@@ -517,15 +497,71 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 		}
 		return Void();
 	}
+
+	// Resume inflight relocations from the previous DD
+	// TODO: The initialDataDistribution is unused once resumeRelocations and
+	// DataDistributionTracker::trackInitialShards are done. In the future, we can release the object to save memory
+	// usage if it turns out to be a problem.
+	Future<Void> resumeRelocations() {
+		ASSERT(shardsAffectedByTeamFailure); // has to be allocated
+		Future<Void> shardsReady = resumeFromShards(Reference<DataDistributor>::addRef(this), g_network->isSimulated());
+		return resumeFromDataMoves(Reference<DataDistributor>::addRef(this), shardsReady);
+	}
+
+	Future<Void> pollMoveKeysLock() { return txnProcessor->pollMoveKeysLock(lock, context->ddEnabledState.get()); }
+
+	Future<bool> isDataDistributionEnabled() const {
+		return txnProcessor->isDataDistributionEnabled(context->ddEnabledState.get());
+	}
+
+	Future<Void> removeKeysFromFailedServer(const UID& serverID, const std::vector<UID>& teamForDroppedRange) const {
+		return txnProcessor->removeKeysFromFailedServer(
+		    serverID, teamForDroppedRange, lock, context->ddEnabledState.get());
+	}
+
+	Future<Void> removeStorageServer(const UID& serverID, const Optional<UID>& tssPairID = Optional<UID>()) const {
+		return txnProcessor->removeStorageServer(serverID, tssPairID, lock, context->ddEnabledState.get());
+	}
 };
 
+ACTOR Future<Void> storageQuotaTracker(Database cx, StorageQuotaInfo* storageQuotaInfo) {
+	loop {
+		state Transaction tr(cx);
+		loop {
+			try {
+				state RangeResult currentQuotas = wait(tr.getRange(storageQuotaKeys, CLIENT_KNOBS->TOO_MANY));
+				TraceEvent("StorageQuota_ReadCurrentQuotas").detail("Size", currentQuotas.size());
+				for (auto const kv : currentQuotas) {
+					Key const key = kv.key.removePrefix(storageQuotaPrefix);
+					uint64_t const quota = BinaryReader::fromStringRef<uint64_t>(kv.value, Unversioned());
+					storageQuotaInfo->quotaMap[key] = quota;
+				}
+				wait(delay(5.0));
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+}
+
+// Periodically check and log the physicalShard status; clean up empty physicalShard;
+ACTOR Future<Void> monitorPhysicalShardStatus(Reference<PhysicalShardCollection> self) {
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
+	loop {
+		self->cleanUpPhysicalShardCollection();
+		self->logPhysicalShardCollection();
+		wait(delay(SERVER_KNOBS->PHYSICAL_SHARD_METRICS_DELAY));
+	}
+}
+
 // Runs the data distribution algorithm for FDB, including the DD Queue, DD tracker, and DD team collection
 ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
-                                    PromiseStream<GetMetricsListRequest> getShardMetricsList,
-                                    const DDEnabledState* ddEnabledState) {
+                                    PromiseStream<GetMetricsListRequest> getShardMetricsList) {
 	state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DataDistributionLaunch, LockAware::True);
 	cx->locationCacheSize = SERVER_KNOBS->DD_LOCATION_CACHE_SIZE;
-	self->txnProcessor = std::shared_ptr<IDDTxnProcessor>(new DDTxnProcessor(cx));
+	self->txnProcessor = Reference<IDDTxnProcessor>(new DDTxnProcessor(cx));
 
 	// cx->setOption( FDBDatabaseOptions::LOCATION_CACHE_SIZE, StringRef((uint8_t*)
 	// &SERVER_KNOBS->DD_LOCATION_CACHE_SIZE, 8) ); ASSERT( cx->locationCacheSize ==
@@ -546,13 +582,7 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 		state KeyRangeMap<ShardTrackedData> shards;
 		state Promise<UID> removeFailedServer;
 		try {
-			wait(DataDistributor::init(self, ddEnabledState));
-
-			state Reference<TenantCache> ddTenantCache;
-			if (ddIsTenantAware) {
-				ddTenantCache = makeReference<TenantCache>(cx, self->ddId);
-				wait(ddTenantCache->build(cx));
-			}
+			wait(DataDistributor::init(self));
 
 			// When/If this assertion fails, Evan owes Ben a pat on the back for his foresight
 			ASSERT(self->configuration.storageTeamSize > 0);
@@ -564,18 +594,26 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 			state Reference<AsyncVar<bool>> processingUnhealthy(new AsyncVar<bool>(false));
 			state Reference<AsyncVar<bool>> processingWiggle(new AsyncVar<bool>(false));
 			state Promise<Void> readyToStart;
+
+			state Optional<Reference<TenantCache>> ddTenantCache;
+			if (ddIsTenantAware) {
+				ddTenantCache = makeReference<TenantCache>(cx, self->ddId);
+				wait(ddTenantCache.get()->build());
+			}
+
 			self->shardsAffectedByTeamFailure = makeReference<ShardsAffectedByTeamFailure>();
-			wait(DataDistributor::resumeRelocations(self));
+			self->physicalShardCollection = makeReference<PhysicalShardCollection>();
+			wait(self->resumeRelocations());
 
-			std::vector<TeamCollectionInterface> tcis;
+			std::vector<TeamCollectionInterface> tcis; // primary and remote region interface
+			Reference<AsyncVar<bool>> anyZeroHealthyTeams; // true if primary or remote has zero healthy team
+			std::vector<Reference<AsyncVar<bool>>> zeroHealthyTeams; // primary and remote
 
-			Reference<AsyncVar<bool>> anyZeroHealthyTeams;
-			std::vector<Reference<AsyncVar<bool>>> zeroHealthyTeams;
 			tcis.push_back(TeamCollectionInterface());
 			zeroHealthyTeams.push_back(makeReference<AsyncVar<bool>>(true));
 			int storageTeamSize = self->configuration.storageTeamSize;
 
-			std::vector<Future<Void>> actors;
+			std::vector<Future<Void>> actors; // the container of ACTORs
 			if (self->configuration.usableRegions > 1) {
 				tcis.push_back(TeamCollectionInterface());
 				storageTeamSize = 2 * self->configuration.storageTeamSize;
@@ -586,16 +624,13 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 			} else {
 				anyZeroHealthyTeams = zeroHealthyTeams[0];
 			}
-			if (ddIsTenantAware) {
-				actors.push_back(reportErrorsExcept(
-				    ddTenantCache->monitorTenantMap(), "DDTenantCacheMonitor", self->ddId, &normalDDQueueErrors()));
-			}
 
-			actors.push_back(pollMoveKeysLock(cx, self->lock, ddEnabledState));
+			actors.push_back(self->pollMoveKeysLock());
 			actors.push_back(reportErrorsExcept(dataDistributionTracker(self->initData,
-			                                                            cx,
+			                                                            self->txnProcessor,
 			                                                            self->relocationProducer,
 			                                                            self->shardsAffectedByTeamFailure,
+			                                                            self->physicalShardCollection,
 			                                                            getShardMetrics,
 			                                                            getTopKShardMetrics.getFuture(),
 			                                                            getShardMetricsList,
@@ -604,11 +639,12 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 			                                                            anyZeroHealthyTeams,
 			                                                            self->ddId,
 			                                                            &shards,
-			                                                            &trackerCancelled),
+			                                                            &trackerCancelled,
+			                                                            ddTenantCache),
 			                                    "DDTracker",
 			                                    self->ddId,
 			                                    &normalDDQueueErrors()));
-			actors.push_back(reportErrorsExcept(dataDistributionQueue(cx,
+			actors.push_back(reportErrorsExcept(dataDistributionQueue(self->txnProcessor,
 			                                                          self->relocationProducer,
 			                                                          self->relocationConsumer.getFuture(),
 			                                                          getShardMetrics,
@@ -617,20 +653,33 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 			                                                          processingWiggle,
 			                                                          tcis,
 			                                                          self->shardsAffectedByTeamFailure,
+			                                                          self->physicalShardCollection,
 			                                                          self->lock,
 			                                                          getAverageShardBytes,
 			                                                          getUnhealthyRelocationCount.getFuture(),
 			                                                          self->ddId,
 			                                                          storageTeamSize,
 			                                                          self->configuration.storageTeamSize,
-			                                                          ddEnabledState),
+			                                                          self->context->ddEnabledState.get()),
 			                                    "DDQueue",
 			                                    self->ddId,
 			                                    &normalDDQueueErrors()));
 
+			actors.push_back(reportErrorsExcept(storageQuotaTracker(cx, &self->storageQuotaInfo),
+			                                    "StorageQuotaTracker",
+			                                    self->ddId,
+			                                    &normalDDQueueErrors()));
+
+			if (ddIsTenantAware) {
+				actors.push_back(reportErrorsExcept(ddTenantCache.get()->monitorTenantMap(),
+				                                    "DDTenantCacheMonitor",
+				                                    self->ddId,
+				                                    &normalDDQueueErrors()));
+			}
+
 			std::vector<DDTeamCollection*> teamCollectionsPtrs;
 			primaryTeamCollection = makeReference<DDTeamCollection>(
-			    cx,
+			    self->txnProcessor,
 			    self->ddId,
 			    self->lock,
 			    self->relocationProducer,
@@ -651,7 +700,7 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 			    self->dbInfo, [](auto const& info) { return info.clusterInterface.recruitStorage; });
 			if (self->configuration.usableRegions > 1) {
 				remoteTeamCollection =
-				    makeReference<DDTeamCollection>(cx,
+				    makeReference<DDTeamCollection>(self->txnProcessor,
 				                                    self->ddId,
 				                                    self->lock,
 				                                    self->relocationProducer,
@@ -669,24 +718,32 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 				                                    getUnhealthyRelocationCount);
 				teamCollectionsPtrs.push_back(remoteTeamCollection.getPtr());
 				remoteTeamCollection->teamCollections = teamCollectionsPtrs;
-				actors.push_back(reportErrorsExcept(
-				    DDTeamCollection::run(
-				        remoteTeamCollection, self->initData, tcis[1], recruitStorage, *ddEnabledState),
-				    "DDTeamCollectionSecondary",
-				    self->ddId,
-				    &normalDDQueueErrors()));
+				actors.push_back(reportErrorsExcept(DDTeamCollection::run(remoteTeamCollection,
+				                                                          self->initData,
+				                                                          tcis[1],
+				                                                          recruitStorage,
+				                                                          *self->context->ddEnabledState.get()),
+				                                    "DDTeamCollectionSecondary",
+				                                    self->ddId,
+				                                    &normalDDQueueErrors()));
 				actors.push_back(DDTeamCollection::printSnapshotTeamsInfo(remoteTeamCollection));
 			}
 			primaryTeamCollection->teamCollections = teamCollectionsPtrs;
 			self->teamCollection = primaryTeamCollection.getPtr();
-			actors.push_back(reportErrorsExcept(
-			    DDTeamCollection::run(primaryTeamCollection, self->initData, tcis[0], recruitStorage, *ddEnabledState),
-			    "DDTeamCollectionPrimary",
-			    self->ddId,
-			    &normalDDQueueErrors()));
+			actors.push_back(reportErrorsExcept(DDTeamCollection::run(primaryTeamCollection,
+			                                                          self->initData,
+			                                                          tcis[0],
+			                                                          recruitStorage,
+			                                                          *self->context->ddEnabledState.get()),
+			                                    "DDTeamCollectionPrimary",
+			                                    self->ddId,
+			                                    &normalDDQueueErrors()));
 
 			actors.push_back(DDTeamCollection::printSnapshotTeamsInfo(primaryTeamCollection));
 			actors.push_back(yieldPromiseStream(self->relocationProducer.getFuture(), self->relocationConsumer));
+			if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+				actors.push_back(monitorPhysicalShardStatus(self->physicalShardCollection));
+			}
 
 			wait(waitForAll(actors));
 			return Void();
@@ -723,17 +780,14 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 			TraceEvent("DataDistributorTeamCollectionsDestroyed").error(err);
 			if (removeFailedServer.getFuture().isReady() && !removeFailedServer.getFuture().isError()) {
 				TraceEvent("RemoveFailedServer", removeFailedServer.getFuture().get()).error(err);
-				wait(removeKeysFromFailedServer(
-				    cx, removeFailedServer.getFuture().get(), teamForDroppedRange, self->lock, ddEnabledState));
-				Optional<UID> tssPairID;
-				wait(removeStorageServer(
-				    cx, removeFailedServer.getFuture().get(), tssPairID, self->lock, ddEnabledState));
+				wait(self->removeKeysFromFailedServer(removeFailedServer.getFuture().get(), teamForDroppedRange));
+				wait(self->removeStorageServer(removeFailedServer.getFuture().get()));
 			} else {
 				if (err.code() != error_code_movekeys_conflict) {
 					throw err;
 				}
 
-				bool ddEnabled = wait(isDataDistributionEnabled(cx, ddEnabledState));
+				bool ddEnabled = wait(self->isDataDistributionEnabled());
 				TraceEvent("DataDistributionMoveKeysConflict").error(err).detail("DataDistributionEnabled", ddEnabled);
 				if (ddEnabled) {
 					throw err;
@@ -778,7 +832,8 @@ ACTOR Future<ErrorOr<Void>> trySendSnapReq(RequestStream<WorkerSnapRequest> stre
 		if (reply.isError()) {
 			TraceEvent("SnapDataDistributor_ReqError")
 			    .errorUnsuppressed(reply.getError())
-			    .detail("Peer", stream.getEndpoint().getPrimaryAddress());
+			    .detail("Peer", stream.getEndpoint().getPrimaryAddress())
+			    .detail("Retry", snapReqRetry);
 			if (reply.getError().code() != error_code_request_maybe_delivered ||
 			    ++snapReqRetry > SERVER_KNOBS->SNAP_NETWORK_FAILURE_RETRY_LIMIT)
 				return ErrorOr<Void>(reply.getError());
@@ -794,24 +849,6 @@ ACTOR Future<ErrorOr<Void>> trySendSnapReq(RequestStream<WorkerSnapRequest> stre
 	return ErrorOr<Void>(Void());
 }
 
-ACTOR static Future<Void> waitForMost(std::vector<Future<ErrorOr<Void>>> futures,
-                                      int faultTolerance,
-                                      Error e,
-                                      double waitMultiplierForSlowFutures = 1.0) {
-	state std::vector<Future<bool>> successFutures;
-	state double startTime = now();
-	successFutures.reserve(futures.size());
-	for (const auto& future : futures) {
-		successFutures.push_back(fmap([](auto const& result) { return result.present(); }, future));
-	}
-	bool success = wait(quorumEqualsTrue(successFutures, successFutures.size() - faultTolerance));
-	if (!success) {
-		throw e;
-	}
-	wait(delay((now() - startTime) * waitMultiplierForSlowFutures) || waitForAll(successFutures));
-	return Void();
-}
-
 ACTOR Future<std::map<NetworkAddress, std::pair<WorkerInterface, std::string>>> getStatefulWorkers(
     Database cx,
     Reference<AsyncVar<ServerDBInfo> const> dbInfo,
@@ -845,8 +882,7 @@ ACTOR Future<std::map<NetworkAddress, std::pair<WorkerInterface, std::string>>>
 				workersMap[worker.interf.address()] = worker.interf;
 			}
 
-			Optional<Value> regionsValue =
-			    wait(tr.get(LiteralStringRef("usable_regions").withPrefix(configKeysPrefix)));
+			Optional<Value> regionsValue = wait(tr.get("usable_regions"_sr.withPrefix(configKeysPrefix)));
 			int usableRegions = 1;
 			if (regionsValue.present()) {
 				usableRegions = atoi(regionsValue.get().toString().c_str());
@@ -881,7 +917,7 @@ ACTOR Future<std::map<NetworkAddress, std::pair<WorkerInterface, std::string>>>
 			                                  configuration.storageTeamSize - 1) -
 			                         storageFailures;
 			if (*storageFaultTolerance < 0) {
-				TEST(true); // Too many failed storage servers to complete snapshot
+				CODE_PROBE(true, "Too many failed storage servers to complete snapshot");
 				throw snap_storage_failed();
 			}
 			// tlogs
@@ -902,6 +938,7 @@ ACTOR Future<std::map<NetworkAddress, std::pair<WorkerInterface, std::string>>>
 			// get coordinators
 			Optional<Value> coordinators = wait(tr.get(coordinatorsKey));
 			if (!coordinators.present()) {
+				CODE_PROBE(true, "Failed to read the coordinatorsKey");
 				throw operation_failed();
 			}
 			ClusterConnectionString ccs(coordinators.get().toString());
@@ -992,7 +1029,8 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 
 		TraceEvent("SnapDataDistributor_GotStatefulWorkers")
 		    .detail("SnapPayload", snapReq.snapPayload)
-		    .detail("SnapUID", snapReq.snapUID);
+		    .detail("SnapUID", snapReq.snapUID)
+		    .detail("StorageFaultTolerance", storageFaultTolerance);
 
 		// we need to snapshot storage nodes before snapshot any tlogs
 		std::vector<Future<ErrorOr<Void>>> storageSnapReqs;
@@ -1004,7 +1042,6 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 		}
 		wait(waitForMost(storageSnapReqs, storageFaultTolerance, snap_storage_failed()));
 		TraceEvent("SnapDataDistributor_AfterSnapStorage")
-		    .detail("FaultTolerance", storageFaultTolerance)
 		    .detail("SnapPayload", snapReq.snapPayload)
 		    .detail("SnapUID", snapReq.snapUID);
 
@@ -1290,12 +1327,12 @@ ACTOR Future<Void> ddGetMetrics(GetDataDistributorMetricsRequest req,
 }
 
 ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncVar<ServerDBInfo> const> db) {
-	state Reference<DataDistributor> self(new DataDistributor(db, di.id()));
+	state Reference<DDSharedContext> context(new DDSharedContext(di.id()));
+	state Reference<DataDistributor> self(new DataDistributor(db, di.id(), context));
 	state Future<Void> collection = actorCollection(self->addActor.getFuture());
 	state PromiseStream<GetMetricsListRequest> getShardMetricsList;
 	state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True);
 	state ActorCollection actors(false);
-	state DDEnabledState ddEnabledState;
 	state std::map<UID, DistributorSnapRequest> ddSnapReqMap;
 	state std::map<UID, ErrorOr<Void>> ddSnapReqResultMap;
 	self->addActor.send(actors.getResult());
@@ -1305,11 +1342,8 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 		TraceEvent("DataDistributorRunning", di.id());
 		self->addActor.send(waitFailureServer(di.waitFailure.getFuture()));
 		self->addActor.send(cacheServerWatcher(&cx));
-		state Future<Void> distributor =
-		    reportErrorsExcept(dataDistribution(self, getShardMetricsList, &ddEnabledState),
-		                       "DataDistribution",
-		                       di.id(),
-		                       &normalDataDistributorErrors());
+		state Future<Void> distributor = reportErrorsExcept(
+		    dataDistribution(self, getShardMetricsList), "DataDistribution", di.id(), &normalDataDistributorErrors());
 
 		loop choose {
 			when(wait(distributor || collection)) {
@@ -1327,20 +1361,21 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 			when(DistributorSnapRequest snapReq = waitNext(di.distributorSnapReq.getFuture())) {
 				auto& snapUID = snapReq.snapUID;
 				if (ddSnapReqResultMap.count(snapUID)) {
-					TEST(true); // Data distributor received a duplicate finished snap request
+					CODE_PROBE(true, "Data distributor received a duplicate finished snapshot request");
 					auto result = ddSnapReqResultMap[snapUID];
 					result.isError() ? snapReq.reply.sendError(result.getError()) : snapReq.reply.send(result.get());
 					TraceEvent("RetryFinishedDistributorSnapRequest")
 					    .detail("SnapUID", snapUID)
 					    .detail("Result", result.isError() ? result.getError().code() : 0);
 				} else if (ddSnapReqMap.count(snapReq.snapUID)) {
-					TEST(true); // Data distributor received a duplicate ongoing snap request
+					CODE_PROBE(true, "Data distributor received a duplicate ongoing snapshot request");
 					TraceEvent("RetryOngoingDistributorSnapRequest").detail("SnapUID", snapUID);
 					ASSERT(snapReq.snapPayload == ddSnapReqMap[snapUID].snapPayload);
 					ddSnapReqMap[snapUID] = snapReq;
 				} else {
 					ddSnapReqMap[snapUID] = snapReq;
-					actors.add(ddSnapCreate(snapReq, db, &ddEnabledState, &ddSnapReqMap, &ddSnapReqResultMap));
+					actors.add(ddSnapCreate(
+					    snapReq, db, self->context->ddEnabledState.get(), &ddSnapReqMap, &ddSnapReqResultMap));
 					auto* ddSnapReqResultMapPtr = &ddSnapReqResultMap;
 					actors.add(fmap(
 					    [ddSnapReqResultMapPtr, snapUID](Void _) {
@@ -1371,65 +1406,33 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 
 namespace data_distribution_test {
 
-static Future<ErrorOr<Void>> goodTestFuture(double duration) {
-	return tag(delay(duration), ErrorOr<Void>(Void()));
+inline DDShardInfo doubleToNoLocationShardInfo(double d, bool hasDest) {
+	DDShardInfo res(doubleToTestKey(d), anonymousShardId, anonymousShardId);
+	res.primarySrc.emplace_back((uint64_t)d, 0);
+	if (hasDest) {
+		res.primaryDest.emplace_back((uint64_t)d + 1, 0);
+		res.hasDest = true;
+	}
+	return res;
 }
 
-static Future<ErrorOr<Void>> badTestFuture(double duration, Error e) {
-	return tag(delay(duration), ErrorOr<Void>(e));
+inline int getRandomShardCount() {
+#if defined(USE_SANITIZER)
+	return deterministicRandom()->randomInt(1000, 24000); // 24000 * MAX_SHARD_SIZE = 12TB
+#else
+	return deterministicRandom()->randomInt(1000, CLIENT_KNOBS->TOO_MANY); // 2000000000; OOM
+#endif
 }
 
 } // namespace data_distribution_test
 
-TEST_CASE("/DataDistribution/WaitForMost") {
-	state std::vector<Future<ErrorOr<Void>>> futures;
-	{
-		futures = { data_distribution_test::goodTestFuture(1),
-			        data_distribution_test::goodTestFuture(2),
-			        data_distribution_test::goodTestFuture(3) };
-		wait(waitForMost(futures, 1, operation_failed(), 0.0)); // Don't wait for slowest future
-		ASSERT(!futures[2].isReady());
-	}
-	{
-		futures = { data_distribution_test::goodTestFuture(1),
-			        data_distribution_test::goodTestFuture(2),
-			        data_distribution_test::goodTestFuture(3) };
-		wait(waitForMost(futures, 0, operation_failed(), 0.0)); // Wait for all futures
-		ASSERT(futures[2].isReady());
-	}
-	{
-		futures = { data_distribution_test::goodTestFuture(1),
-			        data_distribution_test::goodTestFuture(2),
-			        data_distribution_test::goodTestFuture(3) };
-		wait(waitForMost(futures, 1, operation_failed(), 1.0)); // Wait for slowest future
-		ASSERT(futures[2].isReady());
-	}
-	{
-		futures = { data_distribution_test::goodTestFuture(1),
-			        data_distribution_test::goodTestFuture(2),
-			        data_distribution_test::badTestFuture(1, success()) };
-		wait(waitForMost(futures, 1, operation_failed(), 1.0)); // Error ignored
-	}
-	{
-		futures = { data_distribution_test::goodTestFuture(1),
-			        data_distribution_test::goodTestFuture(2),
-			        data_distribution_test::badTestFuture(1, success()) };
-		try {
-			wait(waitForMost(futures, 0, operation_failed(), 1.0));
-			ASSERT(false);
-		} catch (Error& e) {
-			ASSERT_EQ(e.code(), error_code_operation_failed);
-		}
-	}
-	return Void();
-}
-
-TEST_CASE("/DataDistributor/StorageWiggler/Order") {
+TEST_CASE("/DataDistribution/StorageWiggler/Order") {
 	StorageWiggler wiggler(nullptr);
-	wiggler.addServer(UID(1, 0), StorageMetadataType(1, KeyValueStoreType::SSD_BTREE_V2));
-	wiggler.addServer(UID(2, 0), StorageMetadataType(2, KeyValueStoreType::MEMORY, true));
-	wiggler.addServer(UID(3, 0), StorageMetadataType(3, KeyValueStoreType::SSD_ROCKSDB_V1, true));
-	wiggler.addServer(UID(4, 0), StorageMetadataType(4, KeyValueStoreType::SSD_BTREE_V2));
+	double startTime = now() - SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC - 0.4;
+	wiggler.addServer(UID(1, 0), StorageMetadataType(startTime, KeyValueStoreType::SSD_BTREE_V2));
+	wiggler.addServer(UID(2, 0), StorageMetadataType(startTime + 0.1, KeyValueStoreType::MEMORY, true));
+	wiggler.addServer(UID(3, 0), StorageMetadataType(startTime + 0.2, KeyValueStoreType::SSD_ROCKSDB_V1, true));
+	wiggler.addServer(UID(4, 0), StorageMetadataType(startTime + 0.3, KeyValueStoreType::SSD_BTREE_V2));
 
 	std::vector<UID> correctOrder{ UID(2, 0), UID(3, 0), UID(1, 0), UID(4, 0) };
 	for (int i = 0; i < correctOrder.size(); ++i) {
@@ -1439,4 +1442,49 @@ TEST_CASE("/DataDistributor/StorageWiggler/Order") {
 	}
 	ASSERT(!wiggler.getNextServerId().present());
 	return Void();
+}
+
+TEST_CASE("/DataDistribution/Initialization/ResumeFromShard") {
+	state Reference<DDSharedContext> context(new DDSharedContext(UID()));
+	state Reference<AsyncVar<ServerDBInfo> const> dbInfo;
+	state Reference<DataDistributor> self(new DataDistributor(dbInfo, UID(), context));
+
+	self->shardsAffectedByTeamFailure = makeReference<ShardsAffectedByTeamFailure>();
+	if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
+		self->physicalShardCollection = makeReference<PhysicalShardCollection>();
+	}
+	self->initData = makeReference<InitialDataDistribution>();
+	self->configuration.usableRegions = 1;
+	self->configuration.storageTeamSize = 1;
+
+	// add DDShardInfo
+	self->shardsAffectedByTeamFailure->setCheckMode(
+	    ShardsAffectedByTeamFailure::CheckMode::ForceNoCheck); // skip check when build
+	int shardNum = data_distribution_test::getRandomShardCount();
+	std::cout << "generating " << shardNum << " shards...\n";
+	for (int i = 1; i <= SERVER_KNOBS->DD_MOVE_KEYS_PARALLELISM; ++i) {
+		self->initData->shards.emplace_back(data_distribution_test::doubleToNoLocationShardInfo(i, true));
+	}
+	for (int i = SERVER_KNOBS->DD_MOVE_KEYS_PARALLELISM + 1; i <= shardNum; ++i) {
+		self->initData->shards.emplace_back(data_distribution_test::doubleToNoLocationShardInfo(i, false));
+	}
+	self->initData->shards.emplace_back(DDShardInfo(allKeys.end));
+	std::cout << "Start resuming...\n";
+	wait(DataDistributor::resumeFromShards(self, false));
+	std::cout << "Start validation...\n";
+	auto relocateFuture = self->relocationProducer.getFuture();
+	for (int i = 0; i < SERVER_KNOBS->DD_MOVE_KEYS_PARALLELISM; ++i) {
+		ASSERT(relocateFuture.isReady());
+		auto rs = relocateFuture.pop();
+		ASSERT(rs.isRestore() == false);
+		ASSERT(rs.cancelled == false);
+		ASSERT(rs.dataMoveId == anonymousShardId);
+		ASSERT(rs.priority == SERVER_KNOBS->PRIORITY_RECOVER_MOVE);
+		// std::cout << rs.keys.begin.toString() << " " << self->initData->shards[i].key.toString() << " \n";
+		ASSERT(rs.keys.begin.compare(self->initData->shards[i].key) == 0);
+		ASSERT(rs.keys.end == self->initData->shards[i + 1].key);
+	}
+	self->shardsAffectedByTeamFailure->setCheckMode(ShardsAffectedByTeamFailure::CheckMode::ForceCheck);
+	self->shardsAffectedByTeamFailure->check();
+	return Void();
 }
\ No newline at end of file
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
deleted file mode 100644
index 3976c76674..0000000000
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ /dev/null
@@ -1,1223 +0,0 @@
-/*
- * DataDistributionTracker.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbrpc/FailureMonitor.h"
-#include "fdbclient/SystemData.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbclient/DatabaseContext.h"
-#include "flow/ActorCollection.h"
-#include "flow/FastRef.h"
-#include "flow/Trace.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// The used bandwidth of a shard. The higher the value is, the busier the shard is.
-enum BandwidthStatus { BandwidthStatusLow, BandwidthStatusNormal, BandwidthStatusHigh };
-
-enum ReadBandwidthStatus { ReadBandwidthStatusNormal, ReadBandwidthStatusHigh };
-
-BandwidthStatus getBandwidthStatus(StorageMetrics const& metrics) {
-	if (metrics.bytesPerKSecond > SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC)
-		return BandwidthStatusHigh;
-	else if (metrics.bytesPerKSecond < SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC)
-		return BandwidthStatusLow;
-
-	return BandwidthStatusNormal;
-}
-
-ReadBandwidthStatus getReadBandwidthStatus(StorageMetrics const& metrics) {
-	if (metrics.bytesReadPerKSecond <= SERVER_KNOBS->SHARD_READ_HOT_BANDWIDTH_MIN_PER_KSECONDS ||
-	    metrics.bytesReadPerKSecond <= SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO * metrics.bytes *
-	                                       SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS) {
-		return ReadBandwidthStatusNormal;
-	} else {
-		return ReadBandwidthStatusHigh;
-	}
-}
-
-ACTOR Future<Void> updateMaxShardSize(Reference<AsyncVar<int64_t>> dbSizeEstimate,
-                                      Reference<AsyncVar<Optional<int64_t>>> maxShardSize) {
-	state int64_t lastDbSize = 0;
-	state int64_t granularity = g_network->isSimulated() ? SERVER_KNOBS->DD_SHARD_SIZE_GRANULARITY_SIM
-	                                                     : SERVER_KNOBS->DD_SHARD_SIZE_GRANULARITY;
-	loop {
-		auto sizeDelta = std::abs(dbSizeEstimate->get() - lastDbSize);
-		if (sizeDelta > granularity || !maxShardSize->get().present()) {
-			auto v = getMaxShardSize(dbSizeEstimate->get());
-			maxShardSize->set(v);
-			lastDbSize = dbSizeEstimate->get();
-		}
-		wait(dbSizeEstimate->onChange());
-	}
-}
-
-struct DataDistributionTracker {
-	Database cx;
-	UID distributorId;
-	KeyRangeMap<ShardTrackedData>& shards;
-	ActorCollection sizeChanges;
-
-	int64_t systemSizeEstimate;
-	Reference<AsyncVar<int64_t>> dbSizeEstimate;
-	Reference<AsyncVar<Optional<int64_t>>> maxShardSize;
-	Future<Void> maxShardSizeUpdater;
-
-	// CapacityTracker
-	PromiseStream<RelocateShard> output;
-	Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure;
-
-	Promise<Void> readyToStart;
-	Reference<AsyncVar<bool>> anyZeroHealthyTeams;
-
-	// Read hot detection
-	PromiseStream<KeyRange> readHotShard;
-
-	// The reference to trackerCancelled must be extracted by actors,
-	// because by the time (trackerCancelled == true) this memory cannot
-	// be accessed
-	bool& trackerCancelled;
-
-	// This class extracts the trackerCancelled reference from a DataDistributionTracker object
-	// Because some actors spawned by the dataDistributionTracker outlive the DataDistributionTracker
-	// object, we must guard against memory errors by using a GetTracker functor to access
-	// the DataDistributionTracker object.
-	class SafeAccessor {
-		bool const& trackerCancelled;
-		DataDistributionTracker& tracker;
-
-	public:
-		SafeAccessor(DataDistributionTracker* tracker)
-		  : trackerCancelled(tracker->trackerCancelled), tracker(*tracker) {
-			ASSERT(!trackerCancelled);
-		}
-
-		DataDistributionTracker* operator()() {
-			if (trackerCancelled) {
-				TEST(true); // Trying to access DataDistributionTracker after tracker has been cancelled
-				throw dd_tracker_cancelled();
-			}
-			return &tracker;
-		}
-	};
-
-	DataDistributionTracker(Database cx,
-	                        UID distributorId,
-	                        Promise<Void> const& readyToStart,
-	                        PromiseStream<RelocateShard> const& output,
-	                        Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
-	                        Reference<AsyncVar<bool>> anyZeroHealthyTeams,
-	                        KeyRangeMap<ShardTrackedData>& shards,
-	                        bool& trackerCancelled)
-	  : cx(cx), distributorId(distributorId), shards(shards), sizeChanges(false), systemSizeEstimate(0),
-	    dbSizeEstimate(new AsyncVar<int64_t>()), maxShardSize(new AsyncVar<Optional<int64_t>>()), output(output),
-	    shardsAffectedByTeamFailure(shardsAffectedByTeamFailure), readyToStart(readyToStart),
-	    anyZeroHealthyTeams(anyZeroHealthyTeams), trackerCancelled(trackerCancelled) {}
-
-	~DataDistributionTracker() {
-		trackerCancelled = true;
-		// Cancel all actors so they aren't waiting on sizeChanged broken promise
-		sizeChanges.clear(false);
-	}
-};
-
-void restartShardTrackers(DataDistributionTracker* self,
-                          KeyRangeRef keys,
-                          Optional<ShardMetrics> startingMetrics = Optional<ShardMetrics>());
-
-// Gets the permitted size and IO bounds for a shard. A shard that starts at allKeys.begin
-//  (i.e. '') will have a permitted size of 0, since the database can contain no data.
-ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize) {
-	ShardSizeBounds bounds;
-
-	if (shard.begin >= keyServersKeys.begin) {
-		bounds.max.bytes = SERVER_KNOBS->KEY_SERVER_SHARD_BYTES;
-	} else {
-		bounds.max.bytes = maxShardSize;
-	}
-
-	bounds.max.bytesPerKSecond = bounds.max.infinity;
-	bounds.max.iosPerKSecond = bounds.max.infinity;
-	bounds.max.bytesReadPerKSecond = bounds.max.infinity;
-
-	// The first shard can have arbitrarily small size
-	if (shard.begin == allKeys.begin) {
-		bounds.min.bytes = 0;
-	} else {
-		bounds.min.bytes = maxShardSize / SERVER_KNOBS->SHARD_BYTES_RATIO;
-	}
-
-	bounds.min.bytesPerKSecond = 0;
-	bounds.min.iosPerKSecond = 0;
-	bounds.min.bytesReadPerKSecond = 0;
-
-	// The permitted error is 1/3 of the general-case minimum bytes (even in the special case where this is the last
-	// shard)
-	bounds.permittedError.bytes = bounds.max.bytes / SERVER_KNOBS->SHARD_BYTES_RATIO / 3;
-	bounds.permittedError.bytesPerKSecond = bounds.permittedError.infinity;
-	bounds.permittedError.iosPerKSecond = bounds.permittedError.infinity;
-	bounds.permittedError.bytesReadPerKSecond = bounds.permittedError.infinity;
-
-	return bounds;
-}
-
-int64_t getMaxShardSize(double dbSizeEstimate) {
-	return std::min((SERVER_KNOBS->MIN_SHARD_BYTES + (int64_t)std::sqrt(std::max<double>(dbSizeEstimate, 0)) *
-	                                                     SERVER_KNOBS->SHARD_BYTES_PER_SQRT_BYTES) *
-	                    SERVER_KNOBS->SHARD_BYTES_RATIO,
-	                (int64_t)SERVER_KNOBS->MAX_SHARD_BYTES);
-}
-
-ACTOR Future<Void> trackShardMetrics(DataDistributionTracker::SafeAccessor self,
-                                     KeyRange keys,
-                                     Reference<AsyncVar<Optional<ShardMetrics>>> shardMetrics) {
-	state BandwidthStatus bandwidthStatus =
-	    shardMetrics->get().present() ? getBandwidthStatus(shardMetrics->get().get().metrics) : BandwidthStatusNormal;
-	state double lastLowBandwidthStartTime =
-	    shardMetrics->get().present() ? shardMetrics->get().get().lastLowBandwidthStartTime : now();
-	state int shardCount = shardMetrics->get().present() ? shardMetrics->get().get().shardCount : 1;
-	state ReadBandwidthStatus readBandwidthStatus = shardMetrics->get().present()
-	                                                    ? getReadBandwidthStatus(shardMetrics->get().get().metrics)
-	                                                    : ReadBandwidthStatusNormal;
-
-	wait(delay(0, TaskPriority::DataDistribution));
-
-	/*TraceEvent("TrackShardMetricsStarting")
-	    .detail("TrackerID", trackerID)
-	    .detail("Keys", keys)
-	    .detail("TrackedBytesInitiallyPresent", shardMetrics->get().present())
-	    .detail("StartingMetrics", shardMetrics->get().present() ? shardMetrics->get().get().metrics.bytes : 0)
-	    .detail("StartingMerges", shardMetrics->get().present() ? shardMetrics->get().get().merges : 0);*/
-
-	try {
-		loop {
-			state ShardSizeBounds bounds;
-			if (shardMetrics->get().present()) {
-				auto bytes = shardMetrics->get().get().metrics.bytes;
-				auto readBandwidthStatus = getReadBandwidthStatus(shardMetrics->get().get().metrics);
-
-				bounds.max.bytes = std::max(int64_t(bytes * 1.1), (int64_t)SERVER_KNOBS->MIN_SHARD_BYTES);
-				bounds.min.bytes = std::min(
-				    int64_t(bytes * 0.9), std::max(int64_t(bytes - (SERVER_KNOBS->MIN_SHARD_BYTES * 0.1)), (int64_t)0));
-				bounds.permittedError.bytes = bytes * 0.1;
-				if (bandwidthStatus == BandwidthStatusNormal) { // Not high or low
-					bounds.max.bytesPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
-					bounds.min.bytesPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
-					bounds.permittedError.bytesPerKSecond = bounds.min.bytesPerKSecond / 4;
-				} else if (bandwidthStatus == BandwidthStatusHigh) { // > 10MB/sec for 100MB shard, proportionally lower
-					                                                 // for smaller shard, > 200KB/sec no matter what
-					bounds.max.bytesPerKSecond = bounds.max.infinity;
-					bounds.min.bytesPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
-					bounds.permittedError.bytesPerKSecond = bounds.min.bytesPerKSecond / 4;
-				} else if (bandwidthStatus == BandwidthStatusLow) { // < 10KB/sec
-					bounds.max.bytesPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
-					bounds.min.bytesPerKSecond = 0;
-					bounds.permittedError.bytesPerKSecond = bounds.max.bytesPerKSecond / 4;
-				} else {
-					ASSERT(false);
-				}
-				// handle read bandkwith status
-				if (readBandwidthStatus == ReadBandwidthStatusNormal) {
-					bounds.max.bytesReadPerKSecond =
-					    std::max((int64_t)(SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO * bytes *
-					                       SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS *
-					                       (1.0 + SERVER_KNOBS->SHARD_MAX_BYTES_READ_PER_KSEC_JITTER)),
-					             SERVER_KNOBS->SHARD_READ_HOT_BANDWIDTH_MIN_PER_KSECONDS);
-					bounds.min.bytesReadPerKSecond = 0;
-					bounds.permittedError.bytesReadPerKSecond = bounds.min.bytesReadPerKSecond / 4;
-				} else if (readBandwidthStatus == ReadBandwidthStatusHigh) {
-					bounds.max.bytesReadPerKSecond = bounds.max.infinity;
-					bounds.min.bytesReadPerKSecond = SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO * bytes *
-					                                 SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS *
-					                                 (1.0 - SERVER_KNOBS->SHARD_MAX_BYTES_READ_PER_KSEC_JITTER);
-					bounds.permittedError.bytesReadPerKSecond = bounds.min.bytesReadPerKSecond / 4;
-					// TraceEvent("RHDTriggerReadHotLoggingForShard")
-					//     .detail("ShardBegin", keys.begin.printable().c_str())
-					//     .detail("ShardEnd", keys.end.printable().c_str());
-					self()->readHotShard.send(keys);
-				} else {
-					ASSERT(false);
-				}
-			} else {
-				bounds.max.bytes = -1;
-				bounds.min.bytes = -1;
-				bounds.permittedError.bytes = -1;
-				bounds.max.bytesPerKSecond = bounds.max.infinity;
-				bounds.min.bytesPerKSecond = 0;
-				bounds.permittedError.bytesPerKSecond = bounds.permittedError.infinity;
-				bounds.max.bytesReadPerKSecond = bounds.max.infinity;
-				bounds.min.bytesReadPerKSecond = 0;
-				bounds.permittedError.bytesReadPerKSecond = bounds.permittedError.infinity;
-			}
-
-			bounds.max.iosPerKSecond = bounds.max.infinity;
-			bounds.min.iosPerKSecond = 0;
-			bounds.permittedError.iosPerKSecond = bounds.permittedError.infinity;
-
-			loop {
-				Transaction tr(self()->cx);
-				// metrics.second is the number of key-ranges (i.e., shards) in the 'keys' key-range
-				std::pair<Optional<StorageMetrics>, int> metrics =
-				    wait(self()->cx->waitStorageMetrics(keys,
-				                                        bounds.min,
-				                                        bounds.max,
-				                                        bounds.permittedError,
-				                                        CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
-				                                        shardCount));
-				if (metrics.first.present()) {
-					BandwidthStatus newBandwidthStatus = getBandwidthStatus(metrics.first.get());
-					if (newBandwidthStatus == BandwidthStatusLow && bandwidthStatus != BandwidthStatusLow) {
-						lastLowBandwidthStartTime = now();
-					}
-					bandwidthStatus = newBandwidthStatus;
-
-					/*TraceEvent("ShardSizeUpdate")
-					    .detail("Keys", keys)
-					    .detail("UpdatedSize", metrics.metrics.bytes)
-					    .detail("Bandwidth", metrics.metrics.bytesPerKSecond)
-					    .detail("BandwidthStatus", getBandwidthStatus(metrics))
-					    .detail("BytesLower", bounds.min.bytes)
-					    .detail("BytesUpper", bounds.max.bytes)
-					    .detail("BandwidthLower", bounds.min.bytesPerKSecond)
-					    .detail("BandwidthUpper", bounds.max.bytesPerKSecond)
-					    .detail("ShardSizePresent", shardSize->get().present())
-					    .detail("OldShardSize", shardSize->get().present() ? shardSize->get().get().metrics.bytes : 0)
-					    .detail("TrackerID", trackerID);*/
-
-					if (shardMetrics->get().present()) {
-						self()->dbSizeEstimate->set(self()->dbSizeEstimate->get() + metrics.first.get().bytes -
-						                            shardMetrics->get().get().metrics.bytes);
-						if (keys.begin >= systemKeys.begin) {
-							self()->systemSizeEstimate +=
-							    metrics.first.get().bytes - shardMetrics->get().get().metrics.bytes;
-						}
-					}
-
-					shardMetrics->set(ShardMetrics(metrics.first.get(), lastLowBandwidthStartTime, shardCount));
-					break;
-				} else {
-					shardCount = metrics.second;
-					if (shardMetrics->get().present()) {
-						auto newShardMetrics = shardMetrics->get().get();
-						newShardMetrics.shardCount = shardCount;
-						shardMetrics->set(newShardMetrics);
-					}
-				}
-			}
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled && e.code() != error_code_dd_tracker_cancelled) {
-			self()->output.sendError(e); // Propagate failure to dataDistributionTracker
-		}
-		throw e;
-	}
-}
-
-ACTOR Future<Void> readHotDetector(DataDistributionTracker* self) {
-	try {
-		loop {
-			state KeyRange keys = waitNext(self->readHotShard.getFuture());
-			state Transaction tr(self->cx);
-			loop {
-				try {
-					Standalone<VectorRef<ReadHotRangeWithMetrics>> readHotRanges =
-					    wait(self->cx->getReadHotRanges(keys));
-					for (const auto& keyRange : readHotRanges) {
-						TraceEvent("ReadHotRangeLog")
-						    .detail("ReadDensity", keyRange.density)
-						    .detail("ReadBandwidth", keyRange.readBandwidth)
-						    .detail("ReadDensityThreshold", SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO)
-						    .detail("KeyRangeBegin", keyRange.keys.begin)
-						    .detail("KeyRangeEnd", keyRange.keys.end);
-					}
-					break;
-				} catch (Error& e) {
-					wait(tr.onError(e));
-				}
-			}
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled)
-			self->output.sendError(e); // Propagate failure to dataDistributionTracker
-		throw e;
-	}
-}
-
-/*
-ACTOR Future<Void> extrapolateShardBytes( Reference<AsyncVar<Optional<int64_t>>> inBytes,
-Reference<AsyncVar<Optional<int64_t>>> outBytes ) { state std::deque< std::pair<double,int64_t> > past; loop { wait(
-inBytes->onChange() ); if( inBytes->get().present() ) { past.emplace_back(now(),inBytes->get().get()); if
-(past.size() < 2) outBytes->set( inBytes->get() ); else { while (past.size() > 1 && past.end()[-1].first -
-past.begin()[1].first > 1.0) past.pop_front(); double rate = std::max(0.0,
-double(past.end()[-1].second-past.begin()[0].second)/(past.end()[-1].first - past.begin()[0].first)); outBytes->set(
-inBytes->get().get() + rate * 10.0 );
-            }
-        }
-    }
-}*/
-
-ACTOR Future<Standalone<VectorRef<KeyRef>>> getSplitKeys(DataDistributionTracker* self,
-                                                         KeyRange splitRange,
-                                                         StorageMetrics splitMetrics,
-                                                         StorageMetrics estimated) {
-	loop {
-		state Transaction tr(self->cx);
-		try {
-			Standalone<VectorRef<KeyRef>> keys =
-			    wait(self->cx->splitStorageMetrics(splitRange, splitMetrics, estimated, SERVER_KNOBS->MIN_SHARD_BYTES));
-			return keys;
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-}
-
-ACTOR Future<int64_t> getFirstSize(Reference<AsyncVar<Optional<ShardMetrics>>> stats) {
-	loop {
-		if (stats->get().present())
-			return stats->get().get().metrics.bytes;
-		wait(stats->onChange());
-	}
-}
-
-ACTOR Future<Void> changeSizes(DataDistributionTracker* self, KeyRange keys, int64_t oldShardsEndingSize) {
-	state std::vector<Future<int64_t>> sizes;
-	state std::vector<Future<int64_t>> systemSizes;
-	for (auto it : self->shards.intersectingRanges(keys)) {
-		Future<int64_t> thisSize = getFirstSize(it->value().stats);
-		sizes.push_back(thisSize);
-		if (it->range().begin >= systemKeys.begin) {
-			systemSizes.push_back(thisSize);
-		}
-	}
-
-	wait(waitForAll(sizes));
-	wait(yield(TaskPriority::DataDistribution));
-
-	int64_t newShardsStartingSize = 0;
-	for (const auto& size : sizes) {
-		newShardsStartingSize += size.get();
-	}
-
-	int64_t newSystemShardsStartingSize = 0;
-	for (const auto& systemSize : systemSizes) {
-		newSystemShardsStartingSize += systemSize.get();
-	}
-
-	int64_t totalSizeEstimate = self->dbSizeEstimate->get();
-	/*TraceEvent("TrackerChangeSizes")
-	    .detail("TotalSizeEstimate", totalSizeEstimate)
-	    .detail("EndSizeOfOldShards", oldShardsEndingSize)
-	    .detail("StartingSizeOfNewShards", newShardsStartingSize);*/
-	self->dbSizeEstimate->set(totalSizeEstimate + newShardsStartingSize - oldShardsEndingSize);
-	self->systemSizeEstimate += newSystemShardsStartingSize;
-	if (keys.begin >= systemKeys.begin) {
-		self->systemSizeEstimate -= oldShardsEndingSize;
-	}
-	return Void();
-}
-
-struct HasBeenTrueFor : ReferenceCounted<HasBeenTrueFor> {
-	explicit HasBeenTrueFor(const Optional<ShardMetrics>& value) {
-		if (value.present()) {
-			lowBandwidthStartTime = value.get().lastLowBandwidthStartTime;
-			trigger =
-			    delayJittered(std::max(0.0, SERVER_KNOBS->DD_MERGE_COALESCE_DELAY + lowBandwidthStartTime - now()),
-			                  TaskPriority::DataDistributionLow) ||
-			    cleared.getFuture();
-		}
-	}
-
-	Future<Void> set(double lastLowBandwidthStartTime) {
-		if (!trigger.isValid() || lowBandwidthStartTime != lastLowBandwidthStartTime) {
-			cleared = Promise<Void>();
-			trigger =
-			    delayJittered(SERVER_KNOBS->DD_MERGE_COALESCE_DELAY + std::max(lastLowBandwidthStartTime - now(), 0.0),
-			                  TaskPriority::DataDistributionLow) ||
-			    cleared.getFuture();
-
-			lowBandwidthStartTime = lastLowBandwidthStartTime;
-		}
-		return trigger;
-	}
-	void clear() {
-		if (!trigger.isValid()) {
-			return;
-		}
-		trigger = Future<Void>();
-		cleared.send(Void());
-		lowBandwidthStartTime = 0;
-	}
-
-	// True if this->value is true and has been true for this->seconds
-	bool hasBeenTrueForLongEnough() const { return trigger.isValid() && trigger.isReady(); }
-
-private:
-	double lowBandwidthStartTime = 0;
-	Future<Void> trigger;
-	Promise<Void> cleared;
-};
-
-ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
-                                 KeyRange keys,
-                                 Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
-                                 ShardSizeBounds shardBounds) {
-	state StorageMetrics metrics = shardSize->get().get().metrics;
-	state BandwidthStatus bandwidthStatus = getBandwidthStatus(metrics);
-
-	// Split
-	TEST(true); // shard to be split
-
-	StorageMetrics splitMetrics;
-	splitMetrics.bytes = shardBounds.max.bytes / 2;
-	splitMetrics.bytesPerKSecond =
-	    keys.begin >= keyServersKeys.begin ? splitMetrics.infinity : SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
-	splitMetrics.iosPerKSecond = splitMetrics.infinity;
-	splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidth
-
-	state Standalone<VectorRef<KeyRef>> splitKeys = wait(getSplitKeys(self, keys, splitMetrics, metrics));
-	// fprintf(stderr, "split keys:\n");
-	// for( int i = 0; i < splitKeys.size(); i++ ) {
-	//	fprintf(stderr, "   %s\n", printable(splitKeys[i]).c_str());
-	//}
-	int numShards = splitKeys.size() - 1;
-
-	TraceEvent("RelocateShardStartSplit", self->distributorId)
-	    .suppressFor(1.0)
-	    .detail("Begin", keys.begin)
-	    .detail("End", keys.end)
-	    .detail("MaxBytes", shardBounds.max.bytes)
-	    .detail("MetricsBytes", metrics.bytes)
-	    .detail("Bandwidth",
-	            bandwidthStatus == BandwidthStatusHigh     ? "High"
-	            : bandwidthStatus == BandwidthStatusNormal ? "Normal"
-	                                                       : "Low")
-	    .detail("BytesPerKSec", metrics.bytesPerKSecond)
-	    .detail("NumShards", numShards);
-
-	if (numShards > 1) {
-		int skipRange = deterministicRandom()->randomInt(0, numShards);
-		// The queue can't deal with RelocateShard requests which split an existing shard into three pieces, so
-		// we have to send the unskipped ranges in this order (nibbling in from the edges of the old range)
-		for (int i = 0; i < skipRange; i++)
-			restartShardTrackers(self, KeyRangeRef(splitKeys[i], splitKeys[i + 1]));
-		restartShardTrackers(self, KeyRangeRef(splitKeys[skipRange], splitKeys[skipRange + 1]));
-		for (int i = numShards - 1; i > skipRange; i--)
-			restartShardTrackers(self, KeyRangeRef(splitKeys[i], splitKeys[i + 1]));
-
-		for (int i = 0; i < skipRange; i++) {
-			KeyRangeRef r(splitKeys[i], splitKeys[i + 1]);
-			self->shardsAffectedByTeamFailure->defineShard(r);
-			self->output.send(RelocateShard(r, SERVER_KNOBS->PRIORITY_SPLIT_SHARD, RelocateReason::OTHER));
-		}
-		for (int i = numShards - 1; i > skipRange; i--) {
-			KeyRangeRef r(splitKeys[i], splitKeys[i + 1]);
-			self->shardsAffectedByTeamFailure->defineShard(r);
-			self->output.send(RelocateShard(r, SERVER_KNOBS->PRIORITY_SPLIT_SHARD, RelocateReason::OTHER));
-		}
-
-		self->sizeChanges.add(changeSizes(self, keys, shardSize->get().get().metrics.bytes));
-	} else {
-		wait(delay(1.0, TaskPriority::DataDistribution)); // In case the reason the split point was off was due to a
-		                                                  // discrepancy between storage servers
-	}
-	return Void();
-}
-
-ACTOR Future<Void> brokenPromiseToReady(Future<Void> f) {
-	try {
-		wait(f);
-	} catch (Error& e) {
-		if (e.code() != error_code_broken_promise) {
-			throw;
-		}
-	}
-	return Void();
-}
-
-Future<Void> shardMerger(DataDistributionTracker* self,
-                         KeyRange const& keys,
-                         Reference<AsyncVar<Optional<ShardMetrics>>> shardSize) {
-	int64_t maxShardSize = self->maxShardSize->get().get();
-
-	auto prevIter = self->shards.rangeContaining(keys.begin);
-	auto nextIter = self->shards.rangeContaining(keys.begin);
-
-	TEST(true); // shard to be merged
-	ASSERT(keys.begin > allKeys.begin);
-
-	// This will merge shards both before and after "this" shard in keyspace.
-	int shardsMerged = 1;
-	bool forwardComplete = false;
-	KeyRangeRef merged;
-	StorageMetrics endingStats = shardSize->get().get().metrics;
-	int shardCount = shardSize->get().get().shardCount;
-	double lastLowBandwidthStartTime = shardSize->get().get().lastLowBandwidthStartTime;
-	if (FLOW_KNOBS->DELAY_JITTER_OFFSET * SERVER_KNOBS->DD_MERGE_COALESCE_DELAY >
-	        SERVER_KNOBS->DD_LOW_BANDWIDTH_DELAY &&
-	    now() - lastLowBandwidthStartTime < SERVER_KNOBS->DD_LOW_BANDWIDTH_DELAY) {
-		TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "ShardMergeTooSoon", self->distributorId)
-		    .detail("Keys", keys)
-		    .detail("LastLowBandwidthStartTime", lastLowBandwidthStartTime);
-	}
-
-	int64_t systemBytes = keys.begin >= systemKeys.begin ? shardSize->get().get().metrics.bytes : 0;
-
-	loop {
-		Optional<ShardMetrics> newMetrics;
-		if (!forwardComplete) {
-			if (nextIter->range().end == allKeys.end) {
-				forwardComplete = true;
-				continue;
-			}
-			++nextIter;
-			newMetrics = nextIter->value().stats->get();
-
-			// If going forward, give up when the next shard's stats are not yet present.
-			if (!newMetrics.present() || shardCount + newMetrics.get().shardCount >= CLIENT_KNOBS->SHARD_COUNT_LIMIT) {
-				--nextIter;
-				forwardComplete = true;
-				continue;
-			}
-		} else {
-			--prevIter;
-			newMetrics = prevIter->value().stats->get();
-
-			// If going backward, stop when the stats are not present or if the shard is already over the merge
-			//  bounds. If this check triggers right away (if we have not merged anything) then return a trigger
-			//  on the previous shard changing "size".
-			if (!newMetrics.present() || shardCount + newMetrics.get().shardCount >= CLIENT_KNOBS->SHARD_COUNT_LIMIT) {
-				if (shardsMerged == 1) {
-					TEST(true); // shardMerger cannot merge anything
-					return brokenPromiseToReady(prevIter->value().stats->onChange());
-				}
-
-				++prevIter;
-				break;
-			}
-		}
-
-		merged = KeyRangeRef(prevIter->range().begin, nextIter->range().end);
-		endingStats += newMetrics.get().metrics;
-		shardCount += newMetrics.get().shardCount;
-		lastLowBandwidthStartTime = newMetrics.get().lastLowBandwidthStartTime;
-		if ((forwardComplete ? prevIter->range().begin : nextIter->range().begin) >= systemKeys.begin) {
-			systemBytes += newMetrics.get().metrics.bytes;
-		}
-		shardsMerged++;
-
-		auto shardBounds = getShardSizeBounds(merged, maxShardSize);
-		// If we just recently get the current shard's metrics (i.e., less than DD_LOW_BANDWIDTH_DELAY ago), it means
-		// the shard's metric may not be stable yet. So we cannot continue merging in this direction.
-		if (endingStats.bytes >= shardBounds.min.bytes || getBandwidthStatus(endingStats) != BandwidthStatusLow ||
-		    now() - lastLowBandwidthStartTime < SERVER_KNOBS->DD_LOW_BANDWIDTH_DELAY ||
-		    shardsMerged >= SERVER_KNOBS->DD_MERGE_LIMIT) {
-			// The merged range is larger than the min bounds so we cannot continue merging in this direction.
-			//  This means that:
-			//  1. If we were going forwards (the starting direction), we roll back the last speculative merge.
-			//      In this direction we do not want to go above this boundary since we will merge at least one in
-			//      the other direction, even when that goes over the bounds.
-			//  2. If we were going backwards we always want to merge one more shard on (to make sure we go over
-			//      the shard min bounds) so we "break" without resetting the merged range.
-			if (forwardComplete)
-				break;
-
-			// If going forward, remove most recently added range
-			endingStats -= newMetrics.get().metrics;
-			shardCount -= newMetrics.get().shardCount;
-			if (nextIter->range().begin >= systemKeys.begin) {
-				systemBytes -= newMetrics.get().metrics.bytes;
-			}
-			shardsMerged--;
-			--nextIter;
-			merged = KeyRangeRef(prevIter->range().begin, nextIter->range().end);
-			forwardComplete = true;
-		}
-	}
-
-	// restarting shard tracker will derefenced values in the shard map, so make a copy
-	KeyRange mergeRange = merged;
-
-	// OldKeys: Shards in the key range are merged as one shard defined by NewKeys;
-	// NewKeys: New key range after shards are merged;
-	// EndingSize: The new merged shard size in bytes;
-	// BatchedMerges: The number of shards merged. Each shard is defined in self->shards;
-	// LastLowBandwidthStartTime: When does a shard's bandwidth status becomes BandwidthStatusLow. If a shard's status
-	//   becomes BandwidthStatusLow less than DD_LOW_BANDWIDTH_DELAY ago, the merging logic will stop at the shard;
-	// ShardCount: The number of non-splittable shards that are merged. Each shard is defined in self->shards may have
-	//   more than 1 shards.
-	TraceEvent("RelocateShardMergeMetrics", self->distributorId)
-	    .detail("OldKeys", keys)
-	    .detail("NewKeys", mergeRange)
-	    .detail("EndingSize", endingStats.bytes)
-	    .detail("BatchedMerges", shardsMerged)
-	    .detail("LastLowBandwidthStartTime", lastLowBandwidthStartTime)
-	    .detail("ShardCount", shardCount);
-
-	if (mergeRange.begin < systemKeys.begin) {
-		self->systemSizeEstimate -= systemBytes;
-	}
-	restartShardTrackers(self, mergeRange, ShardMetrics(endingStats, lastLowBandwidthStartTime, shardCount));
-	self->shardsAffectedByTeamFailure->defineShard(mergeRange);
-	self->output.send(RelocateShard(mergeRange, SERVER_KNOBS->PRIORITY_MERGE_SHARD, RelocateReason::OTHER));
-
-	// We are about to be cancelled by the call to restartShardTrackers
-	return Void();
-}
-
-ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
-                                  KeyRange keys,
-                                  Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
-                                  Reference<HasBeenTrueFor> wantsToMerge) {
-	Future<Void> onChange = shardSize->onChange() || yieldedFuture(self->maxShardSize->onChange());
-
-	// There are the bounds inside of which we are happy with the shard size.
-	// getShardSizeBounds() will allways have shardBounds.min.bytes == 0 for shards that start at allKeys.begin,
-	//  so will will never attempt to merge that shard with the one previous.
-	ShardSizeBounds shardBounds = getShardSizeBounds(keys, self->maxShardSize->get().get());
-	StorageMetrics const& stats = shardSize->get().get().metrics;
-	auto bandwidthStatus = getBandwidthStatus(stats);
-
-	bool shouldSplit = stats.bytes > shardBounds.max.bytes ||
-	                   (bandwidthStatus == BandwidthStatusHigh && keys.begin < keyServersKeys.begin);
-	bool shouldMerge = stats.bytes < shardBounds.min.bytes && bandwidthStatus == BandwidthStatusLow;
-
-	// Every invocation must set this or clear it
-	if (shouldMerge && !self->anyZeroHealthyTeams->get()) {
-		auto whenLongEnough = wantsToMerge->set(shardSize->get().get().lastLowBandwidthStartTime);
-		if (!wantsToMerge->hasBeenTrueForLongEnough()) {
-			onChange = onChange || whenLongEnough;
-		}
-	} else {
-		wantsToMerge->clear();
-		if (shouldMerge) {
-			onChange = onChange || self->anyZeroHealthyTeams->onChange();
-		}
-	}
-
-	// TraceEvent("EdgeCaseTraceShardEvaluator", self->distributorId)
-	//     .detail("BeginKey", keys.begin.printable())
-	//     .detail("EndKey", keys.end.printable())
-	//     .detail("ShouldSplit", shouldSplit)
-	//     .detail("ShouldMerge", shouldMerge)
-	//     .detail("HasBeenTrueLongEnough", wantsToMerge->hasBeenTrueForLongEnough())
-	//     .detail("CurrentMetrics", stats.toString())
-	//     .detail("ShardBoundsMaxBytes", shardBounds.max.bytes)
-	//     .detail("ShardBoundsMinBytes", shardBounds.min.bytes)
-	//     .detail("WriteBandwitdhStatus", bandwidthStatus)
-	//     .detail("SplitBecauseHighWriteBandWidth",
-	//             (bandwidthStatus == BandwidthStatusHigh && keys.begin < keyServersKeys.begin) ? "Yes" : "No");
-
-	if (!self->anyZeroHealthyTeams->get() && wantsToMerge->hasBeenTrueForLongEnough()) {
-		onChange = onChange || shardMerger(self, keys, shardSize);
-	}
-	if (shouldSplit) {
-		onChange = onChange || shardSplitter(self, keys, shardSize, shardBounds);
-	}
-
-	wait(onChange);
-	return Void();
-}
-
-ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
-                                KeyRange keys,
-                                Reference<AsyncVar<Optional<ShardMetrics>>> shardSize) {
-	wait(yieldedFuture(self()->readyToStart.getFuture()));
-
-	if (!shardSize->get().present())
-		wait(shardSize->onChange());
-
-	if (!self()->maxShardSize->get().present())
-		wait(yieldedFuture(self()->maxShardSize->onChange()));
-
-	// Since maxShardSize will become present for all shards at once, avoid slow tasks with a short delay
-	wait(delay(0, TaskPriority::DataDistribution));
-
-	// Survives multiple calls to shardEvaluator and keeps merges from happening too quickly.
-	state Reference<HasBeenTrueFor> wantsToMerge(new HasBeenTrueFor(shardSize->get()));
-
-	/*TraceEvent("ShardTracker", self()->distributorId)
-	    .detail("Begin", keys.begin)
-	    .detail("End", keys.end)
-	    .detail("TrackerID", trackerID)
-	    .detail("MaxBytes", self()->maxShardSize->get().get())
-	    .detail("ShardSize", shardSize->get().get().bytes)
-	    .detail("BytesPerKSec", shardSize->get().get().bytesPerKSecond);*/
-
-	try {
-		loop {
-			// Use the current known size to check for (and start) splits and merges.
-			wait(shardEvaluator(self(), keys, shardSize, wantsToMerge));
-
-			// We could have a lot of actors being released from the previous wait at the same time. Immediately calling
-			// delay(0) mitigates the resulting SlowTask
-			wait(delay(0, TaskPriority::DataDistribution));
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled && e.code() != error_code_dd_tracker_cancelled) {
-			self()->output.sendError(e); // Propagate failure to dataDistributionTracker
-		}
-		throw e;
-	}
-}
-
-void restartShardTrackers(DataDistributionTracker* self, KeyRangeRef keys, Optional<ShardMetrics> startingMetrics) {
-	auto ranges = self->shards.getAffectedRangesAfterInsertion(keys, ShardTrackedData());
-	for (int i = 0; i < ranges.size(); i++) {
-		if (!ranges[i].value.trackShard.isValid() && ranges[i].begin != keys.begin) {
-			// When starting, key space will be full of "dummy" default contructed entries.
-			// This should happen when called from trackInitialShards()
-			ASSERT(!self->readyToStart.isSet());
-			continue;
-		}
-
-		auto shardMetrics = makeReference<AsyncVar<Optional<ShardMetrics>>>();
-
-		// For the case where the new tracker will take over at the boundaries of current shard(s)
-		//  we can use the old size if it is available. This will be the case when merging shards.
-		if (startingMetrics.present()) {
-			ASSERT(ranges.size() == 1);
-			/*TraceEvent("ShardTrackerSizePreset", self->distributorId)
-			    .detail("Keys", keys)
-			    .detail("Size", startingMetrics.get().metrics.bytes)
-			    .detail("Merges", startingMetrics.get().merges);*/
-			TEST(true); // shardTracker started with trackedBytes already set
-			shardMetrics->set(startingMetrics);
-		}
-
-		ShardTrackedData data;
-		data.stats = shardMetrics;
-		data.trackShard = shardTracker(DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics);
-		data.trackBytes = trackShardMetrics(DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics);
-		self->shards.insert(ranges[i], data);
-	}
-}
-
-ACTOR Future<Void> trackInitialShards(DataDistributionTracker* self, Reference<InitialDataDistribution> initData) {
-	TraceEvent("TrackInitialShards", self->distributorId).detail("InitialShardCount", initData->shards.size());
-
-	// This line reduces the priority of shard initialization to prevent interference with failure monitoring.
-	// SOMEDAY: Figure out what this priority should actually be
-	wait(delay(0.0, TaskPriority::DataDistribution));
-
-	state int s;
-	for (s = 0; s < initData->shards.size() - 1; s++) {
-		restartShardTrackers(self, KeyRangeRef(initData->shards[s].key, initData->shards[s + 1].key));
-		wait(yield(TaskPriority::DataDistribution));
-	}
-
-	Future<Void> initialSize = changeSizes(self, KeyRangeRef(allKeys.begin, allKeys.end), 0);
-	self->readyToStart.send(Void());
-	wait(initialSize);
-	self->maxShardSizeUpdater = updateMaxShardSize(self->dbSizeEstimate, self->maxShardSize);
-
-	return Void();
-}
-
-ACTOR Future<Void> fetchTopKShardMetrics_impl(DataDistributionTracker* self, GetTopKMetricsRequest req) {
-	ASSERT(req.comparator);
-	state Future<Void> onChange;
-	state std::vector<StorageMetrics> returnMetrics;
-	// random pick a portion of shard
-	if (req.keys.size() > SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT) {
-		deterministicRandom()->randomShuffle(req.keys, SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT);
-	}
-	try {
-		loop {
-			onChange = Future<Void>();
-			returnMetrics.clear();
-			state int64_t minReadLoad = std::numeric_limits<int64_t>::max();
-			state int64_t maxReadLoad = std::numeric_limits<int64_t>::min();
-			state int i;
-			for (i = 0; i < SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT && i < req.keys.size(); ++i) {
-				auto range = req.keys[i];
-				StorageMetrics metrics;
-				for (auto t : self->shards.intersectingRanges(range)) {
-					auto& stats = t.value().stats;
-					if (!stats->get().present()) {
-						onChange = stats->onChange();
-						break;
-					}
-					metrics += t.value().stats->get().get().metrics;
-				}
-
-				// skip if current stats is invalid
-				if (onChange.isValid()) {
-					break;
-				}
-
-				if (metrics.bytesReadPerKSecond > 0) {
-					minReadLoad = std::min(metrics.bytesReadPerKSecond, minReadLoad);
-					maxReadLoad = std::max(metrics.bytesReadPerKSecond, maxReadLoad);
-					if (req.minBytesReadPerKSecond <= metrics.bytesReadPerKSecond &&
-					    metrics.bytesReadPerKSecond <= req.maxBytesReadPerKSecond) {
-						metrics.keys = range;
-						returnMetrics.push_back(metrics);
-					}
-				}
-
-				wait(yield());
-			}
-			// FIXME(xwang): Do we need to track slow task here?
-			if (!onChange.isValid()) {
-				if (req.topK >= returnMetrics.size())
-					req.reply.send(GetTopKMetricsReply(returnMetrics, minReadLoad, maxReadLoad));
-				else {
-					std::nth_element(returnMetrics.begin(),
-					                 returnMetrics.begin() + req.topK - 1,
-					                 returnMetrics.end(),
-					                 req.comparator);
-					req.reply.send(GetTopKMetricsReply(
-					    std::vector<StorageMetrics>(returnMetrics.begin(), returnMetrics.begin() + req.topK),
-					    minReadLoad,
-					    maxReadLoad));
-				}
-				return Void();
-			}
-			wait(onChange);
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled && !req.reply.isSet())
-			req.reply.sendError(e);
-		throw;
-	}
-}
-
-ACTOR Future<Void> fetchTopKShardMetrics(DataDistributionTracker* self, GetTopKMetricsRequest req) {
-	choose {
-		when(wait(fetchTopKShardMetrics_impl(self, req))) {}
-		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT))) {
-			TEST(true); // TopK DD_SHARD_METRICS_TIMEOUT
-			req.reply.send(GetTopKMetricsReply());
-		}
-	}
-	return Void();
-}
-
-ACTOR Future<Void> fetchShardMetrics_impl(DataDistributionTracker* self, GetMetricsRequest req) {
-	try {
-		loop {
-			Future<Void> onChange;
-			StorageMetrics returnMetrics;
-			for (auto t : self->shards.intersectingRanges(req.keys)) {
-				auto& stats = t.value().stats;
-				if (!stats->get().present()) {
-					onChange = stats->onChange();
-					break;
-				}
-				returnMetrics += t.value().stats->get().get().metrics;
-			}
-
-			if (!onChange.isValid()) {
-				req.reply.send(returnMetrics);
-				return Void();
-			}
-
-			wait(onChange);
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled && !req.reply.isSet())
-			req.reply.sendError(e);
-		throw;
-	}
-}
-
-ACTOR Future<Void> fetchShardMetrics(DataDistributionTracker* self, GetMetricsRequest req) {
-	choose {
-		when(wait(fetchShardMetrics_impl(self, req))) {}
-		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT, TaskPriority::DataDistribution))) {
-			TEST(true); // DD_SHARD_METRICS_TIMEOUT
-			StorageMetrics largeMetrics;
-			largeMetrics.bytes = getMaxShardSize(self->dbSizeEstimate->get());
-			req.reply.send(largeMetrics);
-		}
-	}
-	return Void();
-}
-
-ACTOR Future<Void> fetchShardMetricsList_impl(DataDistributionTracker* self, GetMetricsListRequest req) {
-	try {
-		loop {
-			// used to control shard limit
-			int shardNum = 0;
-			// list of metrics, regenerate on loop when full range unsuccessful
-			Standalone<VectorRef<DDMetricsRef>> result;
-			Future<Void> onChange;
-			auto beginIter = self->shards.containedRanges(req.keys).begin();
-			auto endIter = self->shards.intersectingRanges(req.keys).end();
-			for (auto t = beginIter; t != endIter; ++t) {
-				auto& stats = t.value().stats;
-				if (!stats->get().present()) {
-					onChange = stats->onChange();
-					break;
-				}
-				result.push_back_deep(result.arena(),
-				                      DDMetricsRef(stats->get().get().metrics.bytes, KeyRef(t.begin().toString())));
-				++shardNum;
-				if (shardNum >= req.shardLimit) {
-					break;
-				}
-			}
-
-			if (!onChange.isValid()) {
-				req.reply.send(result);
-				return Void();
-			}
-
-			wait(onChange);
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled && !req.reply.isSet())
-			req.reply.sendError(e);
-		throw;
-	}
-}
-
-ACTOR Future<Void> fetchShardMetricsList(DataDistributionTracker* self, GetMetricsListRequest req) {
-	choose {
-		when(wait(fetchShardMetricsList_impl(self, req))) {}
-		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT))) { req.reply.sendError(timed_out()); }
-	}
-	return Void();
-}
-
-ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> initData,
-                                           Database cx,
-                                           PromiseStream<RelocateShard> output,
-                                           Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
-                                           PromiseStream<GetMetricsRequest> getShardMetrics,
-                                           FutureStream<GetTopKMetricsRequest> getTopKMetrics,
-                                           PromiseStream<GetMetricsListRequest> getShardMetricsList,
-                                           FutureStream<Promise<int64_t>> getAverageShardBytes,
-                                           Promise<Void> readyToStart,
-                                           Reference<AsyncVar<bool>> anyZeroHealthyTeams,
-                                           UID distributorId,
-                                           KeyRangeMap<ShardTrackedData>* shards,
-                                           bool* trackerCancelled) {
-	state DataDistributionTracker self(cx,
-	                                   distributorId,
-	                                   readyToStart,
-	                                   output,
-	                                   shardsAffectedByTeamFailure,
-	                                   anyZeroHealthyTeams,
-	                                   *shards,
-	                                   *trackerCancelled);
-	state Future<Void> loggingTrigger = Void();
-	state Future<Void> readHotDetect = readHotDetector(&self);
-	state Reference<EventCacheHolder> ddTrackerStatsEventHolder = makeReference<EventCacheHolder>("DDTrackerStats");
-	try {
-		wait(trackInitialShards(&self, initData));
-		initData = Reference<InitialDataDistribution>();
-
-		loop choose {
-			when(Promise<int64_t> req = waitNext(getAverageShardBytes)) {
-				req.send(self.maxShardSize->get().get() / 2);
-			}
-			when(wait(loggingTrigger)) {
-				TraceEvent("DDTrackerStats", self.distributorId)
-				    .detail("Shards", self.shards.size())
-				    .detail("TotalSizeBytes", self.dbSizeEstimate->get())
-				    .detail("SystemSizeBytes", self.systemSizeEstimate)
-				    .trackLatest(ddTrackerStatsEventHolder->trackingKey);
-
-				loggingTrigger = delay(SERVER_KNOBS->DATA_DISTRIBUTION_LOGGING_INTERVAL, TaskPriority::FlushTrace);
-			}
-			when(GetMetricsRequest req = waitNext(getShardMetrics.getFuture())) {
-				self.sizeChanges.add(fetchShardMetrics(&self, req));
-			}
-			when(GetTopKMetricsRequest req = waitNext(getTopKMetrics)) {
-				self.sizeChanges.add(fetchTopKShardMetrics(&self, req));
-			}
-			when(GetMetricsListRequest req = waitNext(getShardMetricsList.getFuture())) {
-				self.sizeChanges.add(fetchShardMetricsList(&self, req));
-			}
-			when(wait(self.sizeChanges.getResult())) {}
-			when(KeyRange req = waitNext(self.shardsAffectedByTeamFailure->restartShardTracker.getFuture())) {
-				restartShardTrackers(&self, req);
-			}
-		}
-	} catch (Error& e) {
-		TraceEvent(SevError, "DataDistributionTrackerError", self.distributorId).error(e);
-		throw e;
-	}
-}
-
-std::vector<KeyRange> ShardsAffectedByTeamFailure::getShardsFor(Team team) const {
-	std::vector<KeyRange> r;
-	for (auto it = team_shards.lower_bound(std::pair<Team, KeyRange>(team, KeyRangeRef()));
-	     it != team_shards.end() && it->first == team;
-	     ++it)
-		r.push_back(it->second);
-	return r;
-}
-
-bool ShardsAffectedByTeamFailure::hasShards(Team team) const {
-	auto it = team_shards.lower_bound(std::pair<Team, KeyRange>(team, KeyRangeRef()));
-	return it != team_shards.end() && it->first == team;
-}
-
-int ShardsAffectedByTeamFailure::getNumberOfShards(UID ssID) const {
-	auto it = storageServerShards.find(ssID);
-	return it == storageServerShards.end() ? 0 : it->second;
-}
-
-std::pair<std::vector<ShardsAffectedByTeamFailure::Team>, std::vector<ShardsAffectedByTeamFailure::Team>>
-ShardsAffectedByTeamFailure::getTeamsFor(KeyRangeRef keys) {
-	return shard_teams[keys.begin];
-}
-
-void ShardsAffectedByTeamFailure::erase(Team team, KeyRange const& range) {
-	if (team_shards.erase(std::pair<Team, KeyRange>(team, range)) > 0) {
-		for (auto uid = team.servers.begin(); uid != team.servers.end(); ++uid) {
-			// Safeguard against going negative after eraseServer() sets value to 0
-			if (storageServerShards[*uid] > 0) {
-				storageServerShards[*uid]--;
-			}
-		}
-	}
-}
-
-void ShardsAffectedByTeamFailure::insert(Team team, KeyRange const& range) {
-	if (team_shards.insert(std::pair<Team, KeyRange>(team, range)).second) {
-		for (auto uid = team.servers.begin(); uid != team.servers.end(); ++uid)
-			storageServerShards[*uid]++;
-	}
-}
-
-void ShardsAffectedByTeamFailure::defineShard(KeyRangeRef keys) {
-	std::vector<Team> teams;
-	std::vector<Team> prevTeams;
-	auto rs = shard_teams.intersectingRanges(keys);
-	for (auto it = rs.begin(); it != rs.end(); ++it) {
-		for (auto t = it->value().first.begin(); t != it->value().first.end(); ++t) {
-			teams.push_back(*t);
-			erase(*t, it->range());
-		}
-		for (auto t = it->value().second.begin(); t != it->value().second.end(); ++t) {
-			prevTeams.push_back(*t);
-		}
-	}
-	uniquify(teams);
-	uniquify(prevTeams);
-
-	/*TraceEvent("ShardsAffectedByTeamFailureDefine")
-	    .detail("KeyBegin", keys.begin)
-	    .detail("KeyEnd", keys.end)
-	    .detail("TeamCount", teams.size());*/
-
-	auto affectedRanges = shard_teams.getAffectedRangesAfterInsertion(keys);
-	shard_teams.insert(keys, std::make_pair(teams, prevTeams));
-
-	for (auto r = affectedRanges.begin(); r != affectedRanges.end(); ++r) {
-		auto& t = shard_teams[r->begin];
-		for (auto it = t.first.begin(); it != t.first.end(); ++it) {
-			insert(*it, *r);
-		}
-	}
-	check();
-}
-
-// Move keys to destinationTeams by updating shard_teams
-void ShardsAffectedByTeamFailure::moveShard(KeyRangeRef keys, std::vector<Team> destinationTeams) {
-	/*TraceEvent("ShardsAffectedByTeamFailureMove")
-	    .detail("KeyBegin", keys.begin)
-	    .detail("KeyEnd", keys.end)
-	    .detail("NewTeamSize", destinationTeam.size())
-	    .detail("NewTeam", describe(destinationTeam));*/
-
-	auto ranges = shard_teams.intersectingRanges(keys);
-	std::vector<std::pair<std::pair<std::vector<Team>, std::vector<Team>>, KeyRange>> modifiedShards;
-	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
-		if (keys.contains(it->range())) {
-			// erase the many teams that were associated with this one shard
-			for (auto t = it->value().first.begin(); t != it->value().first.end(); ++t) {
-				erase(*t, it->range());
-			}
-
-			// save this modification for later insertion
-			std::vector<Team> prevTeams = it->value().second;
-			prevTeams.insert(prevTeams.end(), it->value().first.begin(), it->value().first.end());
-			uniquify(prevTeams);
-
-			modifiedShards.push_back(std::pair<std::pair<std::vector<Team>, std::vector<Team>>, KeyRange>(
-			    std::make_pair(destinationTeams, prevTeams), it->range()));
-		} else {
-			// for each range that touches this move, add our team as affecting this range
-			for (auto& team : destinationTeams) {
-				insert(team, it->range());
-			}
-
-			// if we are not in the list of teams associated with this shard, add us in
-			auto& teams = it->value();
-			teams.second.insert(teams.second.end(), teams.first.begin(), teams.first.end());
-			uniquify(teams.second);
-
-			teams.first.insert(teams.first.end(), destinationTeams.begin(), destinationTeams.end());
-			uniquify(teams.first);
-		}
-	}
-
-	// we cannot modify the KeyRangeMap while iterating through it, so add saved modifications now
-	for (int i = 0; i < modifiedShards.size(); i++) {
-		for (auto& t : modifiedShards[i].first.first) {
-			insert(t, modifiedShards[i].second);
-		}
-		shard_teams.insert(modifiedShards[i].second, modifiedShards[i].first);
-	}
-
-	check();
-}
-
-void ShardsAffectedByTeamFailure::finishMove(KeyRangeRef keys) {
-	auto ranges = shard_teams.containedRanges(keys);
-	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
-		it.value().second.clear();
-	}
-}
-
-void ShardsAffectedByTeamFailure::check() const {
-	if (EXPENSIVE_VALIDATION) {
-		for (auto t = team_shards.begin(); t != team_shards.end(); ++t) {
-			auto i = shard_teams.rangeContaining(t->second.begin);
-			if (i->range() != t->second || !std::count(i->value().first.begin(), i->value().first.end(), t->first)) {
-				ASSERT(false);
-			}
-		}
-		auto rs = shard_teams.ranges();
-		for (auto i = rs.begin(); i != rs.end(); ++i) {
-			for (auto t = i->value().first.begin(); t != i->value().first.end(); ++t) {
-				if (!team_shards.count(std::make_pair(*t, i->range()))) {
-					std::string teamDesc, shards;
-					for (int k = 0; k < t->servers.size(); k++)
-						teamDesc += format("%llx ", t->servers[k].first());
-					for (auto x = team_shards.lower_bound(std::make_pair(*t, KeyRangeRef()));
-					     x != team_shards.end() && x->first == *t;
-					     ++x)
-						shards += printable(x->second.begin) + "-" + printable(x->second.end) + ",";
-					TraceEvent(SevError, "SATFInvariantError2")
-					    .detail("KB", i->begin())
-					    .detail("KE", i->end())
-					    .detail("Team", teamDesc)
-					    .detail("Shards", shards);
-					ASSERT(false);
-				}
-			}
-		}
-	}
-}
diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index 0b96f30f67..40342945b6 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -193,7 +193,7 @@ public:
 		memset(firstPages[0], 0xFF, sizeof(Page));
 		firstPages[1] = (Page*)((uintptr_t)firstPages[0] + 4096);
 		memset(firstPages[1], 0xFF, sizeof(Page));
-		stallCount.init(LiteralStringRef("RawDiskQueue.StallCount"));
+		stallCount.init("RawDiskQueue.StallCount"_sr);
 	}
 
 	Future<Void> pushAndCommit(StringRef pageData, StringBuffer* pageMem, uint64_t poppedPages) {
@@ -379,7 +379,7 @@ public:
 					    pageFloor(std::max(self->files[1].size - desiredMaxFileSize, self->fileShrinkBytes));
 					if ((maxShrink > SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_BYTES) ||
 					    (frivolouslyTruncate && deterministicRandom()->random01() < 0.3)) {
-						TEST(true); // Replacing DiskQueue file
+						CODE_PROBE(true, "Replacing DiskQueue file");
 						TraceEvent("DiskQueueReplaceFile", self->dbgid)
 						    .detail("Filename", self->files[1].f->getFilename())
 						    .detail("OldFileSize", self->files[1].size)
@@ -389,7 +389,7 @@ public:
 						waitfor.push_back(self->files[1].f->truncate(self->fileExtensionBytes));
 						self->files[1].size = self->fileExtensionBytes;
 					} else {
-						TEST(true); // Truncating DiskQueue file
+						CODE_PROBE(true, "Truncating DiskQueue file");
 						const int64_t startingSize = self->files[1].size;
 						self->files[1].size -= std::min(maxShrink, self->files[1].size);
 						self->files[1].size = std::max(self->files[1].size, self->fileExtensionBytes);
@@ -460,12 +460,12 @@ public:
 
 			wait(ready);
 
-			TEST(pageData.size() > sizeof(Page)); // push more than one page of data
+			CODE_PROBE(pageData.size() > sizeof(Page), "push more than one page of data");
 
 			Future<Void> pushed = wait(self->push(pageData, &syncFiles));
 			pushing.send(Void());
 			ASSERT(syncFiles.size() >= 1 && syncFiles.size() <= 2);
-			TEST(2 == syncFiles.size()); // push spans both files
+			CODE_PROBE(2 == syncFiles.size(), "push spans both files");
 			wait(pushed);
 
 			delete pageMem;
@@ -491,8 +491,8 @@ public:
 			committed.send(Void());
 		} catch (Error& e) {
 			delete pageMem;
-			TEST(true); // push error
-			TEST(2 == syncFiles.size()); // push spanning both files error
+			CODE_PROBE(true, "push error");
+			CODE_PROBE(2 == syncFiles.size(), "push spanning both files error");
 			TraceEvent(SevError, "RDQPushAndCommitError", dbgid)
 			    .errorUnsuppressed(e)
 			    .detail("InitialFilename0", filename);
@@ -805,7 +805,7 @@ public:
 			Standalone<StringRef> result = self->readingBuffer.pop_front(sizeof(Page));
 			return result;
 		} catch (Error& e) {
-			TEST(true); // Read next page error
+			CODE_PROBE(true, "Read next page error");
 			TraceEvent(SevError, "RDQReadNextPageError", self->dbgid)
 			    .errorUnsuppressed(e)
 			    .detail("File0Name", self->files[0].dbgFilename);
@@ -840,8 +840,8 @@ public:
 			state std::vector<Future<Void>> commits;
 			state bool swap = file == 0;
 
-			TEST(file == 0); // truncate before last read page on file 0
-			TEST(file == 1 && pos != self->files[1].size); // truncate before last read page on file 1
+			CODE_PROBE(file == 0, "truncate before last read page on file 0");
+			CODE_PROBE(file == 1 && pos != self->files[1].size, "truncate before last read page on file 1");
 
 			self->readingFile = 2;
 			self->readingBuffer.clear();
@@ -890,10 +890,10 @@ public:
 		ASSERT(recovered);
 		uint8_t const* begin = contents.begin();
 		uint8_t const* end = contents.end();
-		TEST(contents.size() && pushedPageCount()); // More than one push between commits
+		CODE_PROBE(contents.size() && pushedPageCount(), "More than one push between commits");
 
 		bool pushAtEndOfPage = contents.size() >= 4 && pushedPageCount() && backPage().remainingCapacity() < 4;
-		TEST(pushAtEndOfPage); // Push right at the end of a page, possibly splitting size
+		CODE_PROBE(pushAtEndOfPage, "Push right at the end of a page, possibly splitting size");
 		while (begin != end) {
 			if (!pushedPageCount() || !backPage().remainingCapacity())
 				addEmptyPage();
@@ -1391,7 +1391,7 @@ private:
 		int f;
 		int64_t p;
 		bool poppedNotDurable = self->lastPoppedSeq / sizeof(Page) != self->poppedSeq / sizeof(Page);
-		TEST(poppedNotDurable); // DiskQueue: Recovery popped position not fully durable
+		CODE_PROBE(poppedNotDurable, "DiskQueue: Recovery popped position not fully durable");
 		self->findPhysicalLocation(self->lastPoppedSeq, &f, &p, "lastPoppedSeq");
 		wait(self->rawQueue->setPoppedPage(f, p, pageFloor(self->lastPoppedSeq)));
 
@@ -1408,8 +1408,8 @@ private:
 		self->recovered = true;
 		ASSERT(self->poppedSeq <= self->endLocation());
 
-		TEST(result.size() == 0); // End of queue at border between reads
-		TEST(result.size() != 0); // Partial read at end of queue
+		CODE_PROBE(result.size() == 0, "End of queue at border between reads");
+		CODE_PROBE(result.size() != 0, "Partial read at end of queue");
 
 		// The next read location isn't necessarily the end of the last commit, but this is sufficient for helping us
 		// check an ASSERTion
@@ -1628,8 +1628,9 @@ public:
 			                // totally finished
 			pop(popLocation);
 			commitFuture = commitFuture && queue->commit();
-		} else
-			TEST(true); // No uncommitted data was popped
+		} else {
+			CODE_PROBE(true, "No uncommitted data was popped");
+		}
 
 		return commitFuture;
 	}
diff --git a/fdbserver/EncryptKeyProxy.actor.cpp b/fdbserver/EncryptKeyProxy.actor.cpp
index 554221dc19..c1ed8bf2e2 100644
--- a/fdbserver/EncryptKeyProxy.actor.cpp
+++ b/fdbserver/EncryptKeyProxy.actor.cpp
@@ -18,10 +18,9 @@
  * limitations under the License.
  */
 
-#include "fdbserver/EncryptKeyProxyInterface.h"
+#include "fdbclient/EncryptKeyProxyInterface.h"
 
 #include "fdbrpc/Locality.h"
-#include "fdbrpc/Stats.h"
 #include "fdbserver/KmsConnector.h"
 #include "fdbserver/KmsConnectorInterface.h"
 #include "fdbserver/Knobs.h"
@@ -53,6 +52,10 @@
 
 namespace {
 
+const std::string REST_KMS_CONNECTOR_TYPE_STR = "RESTKmsConnector";
+const std::string FDB_PREF_KMS_CONNECTOR_TYPE_STR = "FDBPerfKmsConnector";
+const std::string FDB_SIM_KMS_CONNECTOR_TYPE_STR = "SimKmsConnector";
+
 struct CipherKeyValidityTS {
 	int64_t refreshAtTS;
 	int64_t expAtTS;
@@ -137,7 +140,7 @@ CipherKeyValidityTS getCipherKeyValidityTS(Optional<int64_t> refreshInterval, Op
 
 struct EncryptBaseCipherKey {
 	EncryptCipherDomainId domainId;
-	Standalone<EncryptCipherDomainName> domainName;
+	Standalone<EncryptCipherDomainNameRef> domainName;
 	EncryptCipherBaseKeyId baseCipherId;
 	Standalone<StringRef> baseCipherKey;
 	// Timestamp after which the cached CipherKey is eligible for KMS refresh
@@ -155,13 +158,13 @@ struct EncryptBaseCipherKey {
 
 	EncryptBaseCipherKey() : domainId(0), baseCipherId(0), baseCipherKey(StringRef()), refreshAt(0), expireAt(0) {}
 	explicit EncryptBaseCipherKey(EncryptCipherDomainId dId,
-	                              EncryptCipherDomainName dName,
+	                              Standalone<EncryptCipherDomainNameRef> dName,
 	                              EncryptCipherBaseKeyId cipherId,
-	                              StringRef cipherKey,
+	                              Standalone<StringRef> cipherKey,
 	                              int64_t refAtTS,
 	                              int64_t expAtTS)
-	  : domainId(dId), domainName(Standalone<StringRef>(dName)), baseCipherId(cipherId),
-	    baseCipherKey(Standalone<StringRef>(cipherKey)), refreshAt(refAtTS), expireAt(expAtTS) {}
+	  : domainId(dId), domainName(dName), baseCipherId(cipherId), baseCipherKey(cipherKey), refreshAt(refAtTS),
+	    expireAt(expAtTS) {}
 
 	bool isValid() const {
 		int64_t currTS = (int64_t)now();
@@ -219,6 +222,10 @@ public:
 	Counter blobMetadataRefreshed;
 	Counter numBlobMetadataRefreshErrors;
 
+	LatencySample kmsLookupByIdsReqLatency;
+	LatencySample kmsLookupByDomainIdsReqLatency;
+	LatencySample kmsBlobMetadataReqLatency;
+
 	explicit EncryptKeyProxyData(UID id)
 	  : myId(id), ekpCacheMetrics("EKPMetrics", myId.toString()),
 	    baseCipherKeyIdCacheMisses("EKPCipherIdCacheMisses", ekpCacheMetrics),
@@ -231,7 +238,19 @@ public:
 	    blobMetadataCacheHits("EKPBlobMetadataCacheHits", ekpCacheMetrics),
 	    blobMetadataCacheMisses("EKPBlobMetadataCacheMisses", ekpCacheMetrics),
 	    blobMetadataRefreshed("EKPBlobMetadataRefreshed", ekpCacheMetrics),
-	    numBlobMetadataRefreshErrors("EKPBlobMetadataRefreshErrors", ekpCacheMetrics) {}
+	    numBlobMetadataRefreshErrors("EKPBlobMetadataRefreshErrors", ekpCacheMetrics),
+	    kmsLookupByIdsReqLatency("EKPKmsLookupByIdsReqLatency",
+	                             id,
+	                             SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+	                             SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+	    kmsLookupByDomainIdsReqLatency("EKPKmsLookupByDomainIdsReqLatency",
+	                                   id,
+	                                   SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+	                                   SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+	    kmsBlobMetadataReqLatency("EKPKmsBlobMetadataReqLatency",
+	                              id,
+	                              SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+	                              SERVER_KNOBS->LATENCY_SAMPLE_SIZE) {}
 
 	EncryptBaseCipherDomainIdKeyIdCacheKey getBaseCipherDomainIdKeyIdCacheKey(
 	    const EncryptCipherDomainId domainId,
@@ -240,9 +259,9 @@ public:
 	}
 
 	void insertIntoBaseDomainIdCache(const EncryptCipherDomainId domainId,
-	                                 EncryptCipherDomainName domainName,
+	                                 Standalone<EncryptCipherDomainNameRef> domainName,
 	                                 const EncryptCipherBaseKeyId baseCipherId,
-	                                 StringRef baseCipherKey,
+	                                 Standalone<StringRef> baseCipherKey,
 	                                 int64_t refreshAtTS,
 	                                 int64_t expireAtTS) {
 		// Entries in domainId cache are eligible for periodic refreshes to support 'limiting lifetime of encryption
@@ -259,9 +278,9 @@ public:
 	}
 
 	void insertIntoBaseCipherIdCache(const EncryptCipherDomainId domainId,
-	                                 EncryptCipherDomainName domainName,
+	                                 Standalone<EncryptCipherDomainNameRef> domainName,
 	                                 const EncryptCipherBaseKeyId baseCipherId,
-	                                 const StringRef baseCipherKey,
+	                                 const Standalone<StringRef> baseCipherKey,
 	                                 int64_t refreshAtTS,
 	                                 int64_t expireAtTS) {
 		// Given an cipherKey is immutable, it is OK to NOT expire cached information.
@@ -371,7 +390,9 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
 				    keysByIdsReq.arena, item.second.domainId, item.second.baseCipherId, item.second.domainName);
 			}
 			keysByIdsReq.debugId = keysByIds.debugId;
+			state double startTime = now();
 			KmsConnLookupEKsByKeyIdsRep keysByIdsRep = wait(kmsConnectorInf.ekLookupByIds.getReply(keysByIdsReq));
+			ekpProxyData->kmsLookupByIdsReqLatency.addMeasurement(now() - startTime);
 
 			for (const auto& item : keysByIdsRep.cipherKeyDetails) {
 				keyIdsReply.baseCipherDetails.emplace_back(
@@ -390,7 +411,7 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
 
 				const auto itr = lookupCipherInfoMap.find(std::make_pair(item.encryptDomainId, item.encryptKeyId));
 				if (itr == lookupCipherInfoMap.end()) {
-					TraceEvent(SevError, "GetCipherKeysByKeyIds_MappingNotFound", ekpProxyData->myId)
+					TraceEvent(SevError, "GetCipherKeysByKeyIdsMappingNotFound", ekpProxyData->myId)
 					    .detail("DomainId", item.encryptDomainId);
 					throw encrypt_keys_fetch_failed();
 				}
@@ -507,8 +528,10 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
 			}
 			keysByDomainIdReq.debugId = latestKeysReq.debugId;
 
+			state double startTime = now();
 			KmsConnLookupEKsByDomainIdsRep keysByDomainIdRep =
 			    wait(kmsConnectorInf.ekLookupByDomainIds.getReply(keysByDomainIdReq));
+			ekpProxyData->kmsLookupByDomainIdsReqLatency.addMeasurement(now() - startTime);
 
 			for (auto& item : keysByDomainIdRep.cipherKeyDetails) {
 				CipherKeyValidityTS validityTS = getCipherKeyValidityTS(item.refreshAfterSec, item.expireAfterSec);
@@ -523,7 +546,7 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
 				// Record the fetched cipher details to the local cache for the future references
 				const auto itr = lookupCipherDomains.find(item.encryptDomainId);
 				if (itr == lookupCipherDomains.end()) {
-					TraceEvent(SevError, "GetLatestCipherKeys_DomainIdNotFound", ekpProxyData->myId)
+					TraceEvent(SevError, "GetLatestCipherKeysDomainIdNotFound", ekpProxyData->myId)
 					    .detail("DomainId", item.encryptDomainId);
 					throw encrypt_keys_fetch_failed();
 				}
@@ -579,7 +602,7 @@ ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpP
                                              KmsConnectorInterface kmsConnectorInf) {
 	state UID debugId = deterministicRandom()->randomUniqueID();
 
-	state TraceEvent t("RefreshEKs_Start", ekpProxyData->myId);
+	state TraceEvent t("RefreshEKsStart", ekpProxyData->myId);
 	t.setMaxEventLength(SERVER_KNOBS->ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH);
 	t.detail("KmsConnInf", kmsConnectorInf.id());
 	t.detail("DebugId", debugId);
@@ -587,12 +610,13 @@ ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpP
 	try {
 		KmsConnLookupEKsByDomainIdsReq req;
 		req.debugId = debugId;
-		req.encryptDomainInfos.reserve(req.arena, ekpProxyData->baseCipherDomainIdCache.size());
+		// req.encryptDomainInfos.reserve(req.arena, ekpProxyData->baseCipherDomainIdCache.size());
 
 		int64_t currTS = (int64_t)now();
 		for (auto itr = ekpProxyData->baseCipherDomainIdCache.begin();
 		     itr != ekpProxyData->baseCipherDomainIdCache.end();) {
 			if (isCipherKeyEligibleForRefresh(itr->second, currTS)) {
+				TraceEvent("RefreshEKs").detail("Id", itr->first);
 				req.encryptDomainInfos.emplace_back_deep(req.arena, itr->first, itr->second.domainName);
 			}
 
@@ -604,13 +628,15 @@ ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpP
 			}
 		}
 
+		state double startTime = now();
 		KmsConnLookupEKsByDomainIdsRep rep = wait(kmsConnectorInf.ekLookupByDomainIds.getReply(req));
+		ekpProxyData->kmsLookupByDomainIdsReqLatency.addMeasurement(now() - startTime);
 		for (const auto& item : rep.cipherKeyDetails) {
 			const auto itr = ekpProxyData->baseCipherDomainIdCache.find(item.encryptDomainId);
 			if (itr == ekpProxyData->baseCipherDomainIdCache.end()) {
-				TraceEvent(SevError, "RefreshEKs_DomainIdNotFound", ekpProxyData->myId)
+				TraceEvent(SevInfo, "RefreshEKsDomainIdNotFound", ekpProxyData->myId)
 				    .detail("DomainId", item.encryptDomainId);
-				// Continue updating the cache with othe elements
+				// Continue updating the cache with other elements
 				continue;
 			}
 
@@ -633,10 +659,10 @@ ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpP
 
 		ekpProxyData->baseCipherKeysRefreshed += rep.cipherKeyDetails.size();
 
-		t.detail("nKeys", rep.cipherKeyDetails.size());
+		t.detail("NumKeys", rep.cipherKeyDetails.size());
 	} catch (Error& e) {
 		if (!canReplyWith(e)) {
-			TraceEvent(SevWarn, "RefreshEKs_Error").error(e);
+			TraceEvent(SevWarn, "RefreshEKsError").error(e);
 			throw e;
 		}
 		TraceEvent("RefreshEKs").detail("ErrorCode", e.code());
@@ -646,8 +672,8 @@ ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpP
 	return Void();
 }
 
-void refreshEncryptionKeys(Reference<EncryptKeyProxyData> ekpProxyData, KmsConnectorInterface kmsConnectorInf) {
-	Future<Void> ignored = refreshEncryptionKeysCore(ekpProxyData, kmsConnectorInf);
+Future<Void> refreshEncryptionKeys(Reference<EncryptKeyProxyData> ekpProxyData, KmsConnectorInterface kmsConnectorInf) {
+	return refreshEncryptionKeysCore(ekpProxyData, kmsConnectorInf);
 }
 
 ACTOR Future<Void> getLatestBlobMetadata(Reference<EncryptKeyProxyData> ekpProxyData,
@@ -702,7 +728,9 @@ ACTOR Future<Void> getLatestBlobMetadata(Reference<EncryptKeyProxyData> ekpProxy
 	if (!lookupDomains.empty()) {
 		try {
 			KmsConnBlobMetadataReq kmsReq(lookupDomains, req.debugId);
+			state double startTime = now();
 			KmsConnBlobMetadataRep kmsRep = wait(kmsConnectorInf.blobMetadataReq.getReply(kmsReq));
+			ekpProxyData->kmsBlobMetadataReqLatency.addMeasurement(now() - startTime);
 			metadataDetails.arena().dependsOn(kmsRep.metadataDetails.arena());
 
 			for (auto& item : kmsRep.metadataDetails) {
@@ -748,7 +776,9 @@ ACTOR Future<Void> refreshBlobMetadataCore(Reference<EncryptKeyProxyData> ekpPro
 		for (auto& item : ekpProxyData->blobMetadataDomainIdCache) {
 			req.domainIds.emplace_back(item.first);
 		}
+		state double startTime = now();
 		KmsConnBlobMetadataRep rep = wait(kmsConnectorInf.blobMetadataReq.getReply(req));
+		ekpProxyData->kmsBlobMetadataReqLatency.addMeasurement(now() - startTime);
 		for (auto& item : rep.metadataDetails) {
 			ekpProxyData->insertIntoBlobMetadataCache(item.domainId, item);
 			t.detail("BM" + std::to_string(item.domainId), "");
@@ -774,15 +804,19 @@ void refreshBlobMetadata(Reference<EncryptKeyProxyData> ekpProxyData, KmsConnect
 }
 
 void activateKmsConnector(Reference<EncryptKeyProxyData> ekpProxyData, KmsConnectorInterface kmsConnectorInf) {
-	if (g_network->isSimulated()) {
+	if (g_network->isSimulated() || (SERVER_KNOBS->KMS_CONNECTOR_TYPE.compare(FDB_PREF_KMS_CONNECTOR_TYPE_STR) == 0)) {
 		ekpProxyData->kmsConnector = std::make_unique<SimKmsConnector>();
-	} else if (SERVER_KNOBS->KMS_CONNECTOR_TYPE.compare("RESTKmsConnector")) {
+	} else if (SERVER_KNOBS->KMS_CONNECTOR_TYPE.compare(REST_KMS_CONNECTOR_TYPE_STR) == 0) {
 		ekpProxyData->kmsConnector = std::make_unique<RESTKmsConnector>();
 	} else {
 		throw not_implemented();
 	}
 
-	TraceEvent("EKP_ActiveKmsConnector", ekpProxyData->myId).detail("ConnectorType", SERVER_KNOBS->KMS_CONNECTOR_TYPE);
+	TraceEvent("EKPActiveKmsConnector", ekpProxyData->myId)
+	    .detail("ConnectorType",
+	            g_network->isSimulated() ? FDB_SIM_KMS_CONNECTOR_TYPE_STR : SERVER_KNOBS->KMS_CONNECTOR_TYPE)
+	    .detail("InfId", kmsConnectorInf.id());
+
 	ekpProxyData->addActor.send(ekpProxyData->kmsConnector->connectorCore(kmsConnectorInf));
 }
 
@@ -794,7 +828,7 @@ ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface,
 	state KmsConnectorInterface kmsConnectorInf;
 	kmsConnectorInf.initEndpoints();
 
-	TraceEvent("EKP_Start", self->myId).detail("KmsConnectorInf", kmsConnectorInf.id());
+	TraceEvent("EKPStart", self->myId).detail("KmsConnectorInf", kmsConnectorInf.id());
 
 	activateKmsConnector(self, kmsConnectorInf);
 
@@ -805,9 +839,11 @@ ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface,
 	// FLOW_KNOB->ENCRRYPTION_KEY_REFRESH_INTERVAL_SEC, allowing the interactions with external Encryption Key Manager
 	// mostly not co-inciding with FDB process encryption key refresh attempts.
 
-	self->encryptionKeyRefresher = recurring([&]() { refreshEncryptionKeys(self, kmsConnectorInf); },
-	                                         FLOW_KNOBS->ENCRYPT_KEY_REFRESH_INTERVAL,
-	                                         TaskPriority::Worker);
+	self->encryptionKeyRefresher = recurringAsync([&]() { return refreshEncryptionKeys(self, kmsConnectorInf); },
+	                                              FLOW_KNOBS->ENCRYPT_KEY_REFRESH_INTERVAL, /* interval */
+	                                              true, /* absoluteIntervalDelay */
+	                                              FLOW_KNOBS->ENCRYPT_KEY_REFRESH_INTERVAL, /* initialDelay */
+	                                              TaskPriority::Worker);
 
 	self->blobMetadataRefresher = recurring([&]() { refreshBlobMetadata(self, kmsConnectorInf); },
 	                                        SERVER_KNOBS->BLOB_METADATA_REFRESH_INTERVAL,
@@ -825,7 +861,7 @@ ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface,
 				self->addActor.send(getLatestBlobMetadata(self, kmsConnectorInf, req));
 			}
 			when(HaltEncryptKeyProxyRequest req = waitNext(ekpInterface.haltEncryptKeyProxy.getFuture())) {
-				TraceEvent("EKP_Halted", self->myId).detail("ReqID", req.requesterID);
+				TraceEvent("EKPHalted", self->myId).detail("ReqID", req.requesterID);
 				req.reply.send(Void());
 				break;
 			}
@@ -835,7 +871,7 @@ ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface,
 			}
 		}
 	} catch (Error& e) {
-		TraceEvent("EKP_Terminated", self->myId).errorUnsuppressed(e);
+		TraceEvent("EKPTerminated", self->myId).errorUnsuppressed(e);
 	}
 
 	return Void();
diff --git a/fdbserver/FDBExecHelper.actor.cpp b/fdbserver/FDBExecHelper.actor.cpp
index a8110f438e..8f76bf419a 100644
--- a/fdbserver/FDBExecHelper.actor.cpp
+++ b/fdbserver/FDBExecHelper.actor.cpp
@@ -18,6 +18,14 @@
  * limitations under the License.
  */
 
+#if !defined(_WIN32) && !defined(__APPLE__) && !defined(__INTEL_COMPILER)
+#define BOOST_SYSTEM_NO_LIB
+#define BOOST_DATE_TIME_NO_LIB
+#define BOOST_REGEX_NO_LIB
+#include <boost/process.hpp>
+#endif
+#include <boost/algorithm/string.hpp>
+
 #include "flow/TLSConfig.actor.h"
 #include "flow/Trace.h"
 #include "flow/Platform.h"
@@ -34,14 +42,6 @@
 #include "fdbserver/Knobs.h"
 #include "fdbserver/RemoteIKeyValueStore.actor.h"
 
-#if !defined(_WIN32) && !defined(__APPLE__) && !defined(__INTEL_COMPILER)
-#define BOOST_SYSTEM_NO_LIB
-#define BOOST_DATE_TIME_NO_LIB
-#define BOOST_REGEX_NO_LIB
-#include <boost/process.hpp>
-#endif
-#include <boost/algorithm/string.hpp>
-
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 ExecCmdValueString::ExecCmdValueString(StringRef pCmdValueString) {
@@ -75,7 +75,7 @@ VectorRef<StringRef> ExecCmdValueString::getBinaryArgs() const {
 void ExecCmdValueString::parseCmdValue() {
 	StringRef param = this->cmdValueString;
 	// get the binary path
-	this->binaryPath = param.eat(LiteralStringRef(" "));
+	this->binaryPath = param.eat(" "_sr);
 
 	// no arguments provided
 	if (param == StringRef()) {
@@ -84,7 +84,7 @@ void ExecCmdValueString::parseCmdValue() {
 
 	// extract the arguments
 	while (param != StringRef()) {
-		StringRef token = param.eat(LiteralStringRef(" "));
+		StringRef token = param.eat(" "_sr);
 		this->binaryArgs.push_back(this->binaryArgs.arena(), token);
 	}
 	return;
@@ -108,7 +108,7 @@ ACTOR void destoryChildProcess(Future<Void> parentSSClosed, ISimulator::ProcessI
 	wait(parentSSClosed);
 	TraceEvent(SevDebug, message.c_str()).log();
 	// This one is root cause for most failures, make sure it's okay to destory
-	g_pSimulator->destroyProcess(childInfo);
+	g_simulator->destroyProcess(childInfo);
 	// Explicitly reset the connection with the child process in case re-spawn very quickly
 	FlowTransport::transport().resetConnection(childInfo->address);
 }
@@ -118,7 +118,7 @@ ACTOR Future<int> spawnSimulated(std::vector<std::string> paramList,
                                  bool isSync,
                                  double maxSimDelayTime,
                                  IClosable* parent) {
-	state ISimulator::ProcessInfo* self = g_pSimulator->getCurrentProcess();
+	state ISimulator::ProcessInfo* self = g_simulator->getCurrentProcess();
 	state ISimulator::ProcessInfo* child;
 
 	state std::string role;
@@ -160,7 +160,7 @@ ACTOR Future<int> spawnSimulated(std::vector<std::string> paramList,
 		}
 	}
 	state int result = 0;
-	child = g_pSimulator->newProcess(
+	child = g_simulator->newProcess(
 	    "remote flow process",
 	    self->address.ip,
 	    0,
@@ -171,7 +171,7 @@ ACTOR Future<int> spawnSimulated(std::vector<std::string> paramList,
 	    self->dataFolder.c_str(),
 	    self->coordinationFolder.c_str(), // do we need to customize this coordination folder path?
 	    self->protocolVersion);
-	wait(g_pSimulator->onProcess(child));
+	wait(g_simulator->onProcess(child));
 	state Future<ISimulator::KillType> onShutdown = child->onShutdown();
 	state Future<ISimulator::KillType> parentShutdown = self->onShutdown();
 	state Future<Void> flowProcessF;
@@ -199,7 +199,7 @@ ACTOR Future<int> spawnSimulated(std::vector<std::string> paramList,
 			choose {
 				when(wait(flowProcessF)) {
 					TraceEvent(SevDebug, "ChildProcessKilled").log();
-					wait(g_pSimulator->onProcess(self));
+					wait(g_simulator->onProcess(self));
 					TraceEvent(SevDebug, "BackOnParentProcess").detail("Result", std::to_string(result));
 					destoryChildProcess(parentSSClosed, child, "StorageServerReceivedClosedMessage");
 				}
diff --git a/fdbserver/GlobalTagThrottler.actor.cpp b/fdbserver/GlobalTagThrottler.actor.cpp
index 5320f3671c..c1ec99541f 100644
--- a/fdbserver/GlobalTagThrottler.actor.cpp
+++ b/fdbserver/GlobalTagThrottler.actor.cpp
@@ -27,197 +27,458 @@
 
 #include "flow/actorcompiler.h" // must be last include
 
+// In the function names below, several terms are used repeatedly. The context-specific are defined here:
+//
+// Cost: Every read or write operation has an associated cost, determined by the number of bytes accessed.
+//       Global tag throttling quotas are specified in terms of the amount of this cost that can be consumed
+//       per second. In the global tag throttler, cost refers to the per second rate of cost consumption.
+//
+// TPS: Transactions per second. Quotas are not specified in terms of TPS, but the limits given to clients must
+//      be specified in terms of TPS because throttling is performed at the front end of transactions (before costs are
+//      known).
+//
+// Total: Refers to the total quota specified by clients through the global tag throttling API. The sum of the
+//        costs of all operations (cluster-wide) with a particular tag cannot exceed the tag's specified total quota,
+//        even if the cluster has no saturated processes.
+//
+// Desired TPS: Assuming that a tag is able to achieve its total quota, this is the TPS it would be able to perform.
+//
+// Reserved: Refers to the reserved quota specified by clients through the global tag throttling API. As long as the
+//           sum of the costs of all operations (cluster-wide) with a particular tag are not above the tag's
+//           specified reserved quota, the tag should not experience any throttling from the global tag throttler.
+//
+// Current [Cost|TPS]: Measuring the current throughput on the cluster, independent of any specified quotas.
+//
+// ThrottlingRatio: Based on the health of each storage server, a throttling ratio is provided,
+//                  informing the global tag throttler what ratio of the current throughput can be maintained.
+//
+// Limiting [Cost|TPS]: Based on the health of storage servers, a limiting throughput may be enforced.
+//
+// Target [Cost|TPS]: Based on reserved, limiting, and desired throughputs, this is the target throughput
+//                    that the global tag throttler aims to achieve (across all clients).
+//
+// PerClient TPS: Because the target throughput must be shared across multiple clients, and all clients must
+//           be given the same limits, a per-client limit is calculated based on the current and target throughputs.
+
 class GlobalTagThrottlerImpl {
-	class QuotaAndCounters {
-		Optional<ThrottleApi::TagQuotaValue> quota;
-		std::unordered_map<UID, double> ssToReadCostRate;
-		std::unordered_map<UID, double> ssToWriteCostRate;
-		Smoother totalReadCostRate;
-		Smoother totalWriteCostRate;
-		Smoother transactionCounter;
-		Smoother perClientRate;
-
-		Optional<double> getReadTPSLimit() const {
-			if (totalReadCostRate.smoothTotal() > 0) {
-				return quota.get().totalReadQuota * transactionCounter.smoothRate() / totalReadCostRate.smoothTotal();
-			} else {
-				return {};
-			}
+	template <class K, class V>
+	static Optional<V> tryGet(std::unordered_map<K, V> const& m, K const& k) {
+		auto it = m.find(k);
+		if (it == m.end()) {
+			return {};
+		} else {
+			return it->second;
 		}
+	}
 
-		Optional<double> getWriteTPSLimit() const {
-			if (totalWriteCostRate.smoothTotal() > 0) {
-				return quota.get().totalWriteQuota * transactionCounter.smoothRate() / totalWriteCostRate.smoothTotal();
-			} else {
-				return {};
-			}
+	static Optional<double> getMin(Optional<double> a, Optional<double> b) {
+		if (a.present() && b.present()) {
+			return std::min(a.get(), b.get());
+		} else if (a.present()) {
+			return a;
+		} else {
+			return b;
 		}
+	}
+
+	static Optional<double> getMax(Optional<double> a, Optional<double> b) {
+		if (a.present() && b.present()) {
+			return std::max(a.get(), b.get());
+		} else if (a.present()) {
+			return a;
+		} else {
+			return b;
+		}
+	}
+
+	enum class LimitType { RESERVED, TOTAL };
+	enum class OpType { READ, WRITE };
+
+	class ThroughputCounters {
+		Smoother readCost;
+		Smoother writeCost;
 
 	public:
-		QuotaAndCounters()
-		  : totalReadCostRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME),
-		    totalWriteCostRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME),
-		    transactionCounter(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME),
-		    perClientRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME) {}
+		ThroughputCounters()
+		  : readCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME),
+		    writeCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME) {}
 
-		void setQuota(ThrottleApi::TagQuotaValue const& quota) { this->quota = quota; }
-
-		void updateReadCostRate(UID ssId, double newReadCostRate) {
-			auto& currentReadCostRate = ssToReadCostRate[ssId];
-			auto diff = newReadCostRate - currentReadCostRate;
-			currentReadCostRate += diff;
-			totalReadCostRate.addDelta(diff);
+		void updateCost(double newCost, OpType opType) {
+			if (opType == OpType::READ) {
+				readCost.setTotal(newCost);
+			} else {
+				writeCost.setTotal(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO * newCost);
+			}
 		}
 
-		void updateWriteCostRate(UID ssId, double newWriteCostRate) {
-			auto& currentWriteCostRate = ssToWriteCostRate[ssId];
-			auto diff = newWriteCostRate - currentWriteCostRate;
-			currentWriteCostRate += diff;
-			totalWriteCostRate.addDelta(diff);
-		}
+		double getCost() const { return readCost.smoothTotal() + writeCost.smoothTotal(); }
+	};
+
+	// Track various statistics per tag, aggregated across all storage servers
+	class PerTagStatistics {
+		Optional<ThrottleApi::TagQuotaValue> quota;
+		Smoother transactionCounter;
+		Smoother perClientRate;
+		Smoother targetRate;
+
+	public:
+		explicit PerTagStatistics()
+		  : transactionCounter(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME),
+		    perClientRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME),
+		    targetRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FOLDING_TIME) {}
+
+		Optional<ThrottleApi::TagQuotaValue> getQuota() const { return quota; }
+
+		void setQuota(ThrottleApi::TagQuotaValue quota) { this->quota = quota; }
+
+		void clearQuota() { quota = {}; }
 
 		void addTransactions(int count) { transactionCounter.addDelta(count); }
 
-		Optional<double> getTargetTotalTPSLimit() const {
-			if (!quota.present())
-				return {};
-			auto readLimit = getReadTPSLimit();
-			auto writeLimit = getWriteTPSLimit();
+		double getTransactionRate() const { return transactionCounter.smoothRate(); }
 
-			// TODO: Implement expiration logic
-			if (!readLimit.present() && !writeLimit.present()) {
-				return {};
-			} else {
-				if (!readLimit.present()) {
-					return writeLimit.get();
-				} else if (!writeLimit.present()) {
-					return readLimit.get();
-				} else {
-					return std::min(readLimit.get(), writeLimit.get());
-				}
-			}
+		ClientTagThrottleLimits updateAndGetPerClientLimit(double targetTps) {
+			auto newPerClientRate = std::max(
+			    SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MIN_RATE,
+			    std::min(targetTps, (targetTps / transactionCounter.smoothRate()) * perClientRate.smoothTotal()));
+			perClientRate.setTotal(newPerClientRate);
+			return ClientTagThrottleLimits(
+			    std::max(perClientRate.smoothTotal(), SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MIN_RATE),
+			    ClientTagThrottleLimits::NO_EXPIRATION);
 		}
 
-		Optional<ClientTagThrottleLimits> updateAndGetPerClientLimit() {
-			auto targetRate = getTargetTotalTPSLimit();
-			if (targetRate.present() && transactionCounter.smoothRate() > 0) {
-				auto newPerClientRate = std::max(
-				    SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MIN_RATE,
-				    std::min(targetRate.get(),
-				             (targetRate.get() / transactionCounter.smoothRate()) * perClientRate.smoothTotal()));
-				perClientRate.setTotal(newPerClientRate);
-				return ClientTagThrottleLimits(perClientRate.getTotal(), ClientTagThrottleLimits::NO_EXPIRATION);
-			} else {
-				return {};
-			}
-		}
-
-		void processTraceEvent(TraceEvent& te) const {
-			if (quota.present()) {
-				te.detail("ProvidedReadTPSLimit", getReadTPSLimit())
-				    .detail("ProvidedWriteTPSLimit", getWriteTPSLimit())
-				    .detail("ReadCostRate", totalReadCostRate.smoothTotal())
-				    .detail("WriteCostRate", totalWriteCostRate.smoothTotal())
-				    .detail("TotalReadQuota", quota.get().totalReadQuota)
-				    .detail("ReservedReadQuota", quota.get().reservedReadQuota)
-				    .detail("TotalWriteQuota", quota.get().totalWriteQuota)
-				    .detail("ReservedWriteQuota", quota.get().reservedWriteQuota);
-			}
+		double updateAndGetTargetLimit(double targetTps) {
+			targetRate.setTotal(targetTps);
+			return targetRate.smoothTotal();
 		}
 	};
 
 	Database db;
 	UID id;
-	std::map<TransactionTag, QuotaAndCounters> trackedTags;
 	uint64_t throttledTagChangeId{ 0 };
-	Future<Void> traceActor;
+	uint32_t lastBusyTagCount{ 0 };
 
-	ACTOR static Future<Void> tracer(GlobalTagThrottlerImpl const* self) {
-		loop {
-			for (const auto& [tag, quotaAndCounters] : self->trackedTags) {
-				TraceEvent te("GlobalTagThrottling");
-				te.detail("Tag", tag);
-				quotaAndCounters.processTraceEvent(te);
+	std::unordered_map<UID, Optional<double>> throttlingRatios;
+	std::unordered_map<TransactionTag, PerTagStatistics> tagStatistics;
+	std::unordered_map<UID, std::unordered_map<TransactionTag, ThroughputCounters>> throughput;
+
+	// Returns the cost rate for the given tag on the given storage server
+	Optional<double> getCurrentCost(UID storageServerId, TransactionTag tag) const {
+		auto const tagToThroughputCounters = tryGet(throughput, storageServerId);
+		if (!tagToThroughputCounters.present()) {
+			return {};
+		}
+		auto const throughputCounter = tryGet(tagToThroughputCounters.get(), tag);
+		if (!throughputCounter.present()) {
+			return {};
+		}
+		return throughputCounter.get().getCost();
+	}
+
+	// Return the cost rate on the given storage server, summed across all tags
+	Optional<double> getCurrentCost(UID storageServerId) const {
+		auto tagToPerTagThroughput = tryGet(throughput, storageServerId);
+		if (!tagToPerTagThroughput.present()) {
+			return {};
+		}
+		double result = 0;
+		for (const auto& [tag, perTagThroughput] : tagToPerTagThroughput.get()) {
+			result += perTagThroughput.getCost();
+		}
+		return result;
+	}
+
+	// Return the cost rate for the given tag, summed across all storage servers
+	double getCurrentCost(TransactionTag tag) const {
+		double result{ 0.0 };
+		for (const auto& [id, _] : throughput) {
+			result += getCurrentCost(id, tag).orDefault(0);
+		}
+		TraceEvent("GlobalTagThrottler_GetCurrentCost").detail("Tag", printable(tag)).detail("Cost", result);
+
+		return result;
+	}
+
+	// For transactions with the provided tag, returns the average cost that gets associated with the provided storage
+	// server
+	Optional<double> getAverageTransactionCost(TransactionTag tag, UID storageServerId) const {
+		auto const cost = getCurrentCost(storageServerId, tag);
+		if (!cost.present()) {
+			return {};
+		}
+		auto const stats = tryGet(tagStatistics, tag);
+		if (!stats.present()) {
+			return {};
+		}
+		auto const transactionRate = stats.get().getTransactionRate();
+		if (transactionRate == 0.0) {
+			return {};
+		} else {
+			return std::max(1.0, cost.get() / transactionRate);
+		}
+	}
+
+	// For transactions with the provided tag, returns the average cost of all transactions
+	// accross the cluster. The minimum cost is 1.
+	double getAverageTransactionCost(TransactionTag tag) const {
+		auto const cost = getCurrentCost(tag);
+		auto const stats = tryGet(tagStatistics, tag);
+		if (!stats.present()) {
+			return 1.0;
+		}
+		auto const transactionRate = stats.get().getTransactionRate();
+		TraceEvent("GlobalTagThrottler_GetAverageTransactionCost")
+		    .detail("Tag", tag)
+		    .detail("TransactionRate", transactionRate)
+		    .detail("Cost", cost);
+		if (transactionRate == 0.0) {
+			return 1.0;
+		} else {
+			return std::max(1.0, cost / transactionRate);
+		}
+	}
+
+	// Returns the list of all tags performing meaningful work on the given storage server
+	std::vector<TransactionTag> getTagsAffectingStorageServer(UID storageServerId) const {
+		std::vector<TransactionTag> result;
+		auto const tagToThroughputCounters = tryGet(throughput, storageServerId);
+		if (!tagToThroughputCounters.present()) {
+			return {};
+		} else {
+			result.reserve(tagToThroughputCounters.get().size());
+			for (const auto& [t, _] : tagToThroughputCounters.get()) {
+				result.push_back(t);
+			}
+		}
+		return result;
+	}
+
+	Optional<double> getQuota(TransactionTag tag, LimitType limitType) const {
+		auto const stats = tryGet(tagStatistics, tag);
+		if (!stats.present()) {
+			return {};
+		}
+		auto const quota = stats.get().getQuota();
+		if (!quota.present()) {
+			return {};
+		}
+		if (limitType == LimitType::TOTAL) {
+			return quota.get().totalQuota;
+		} else {
+			return quota.get().reservedQuota;
+		}
+	}
+
+	// Of all tags meaningfully performing workload on the given storage server,
+	// returns the ratio of total quota allocated to the specified tag
+	double getQuotaRatio(TransactionTagRef tag, UID storageServerId) const {
+		double sumQuota{ 0.0 };
+		double tagQuota{ 0.0 };
+		auto const tagsAffectingStorageServer = getTagsAffectingStorageServer(storageServerId);
+		for (const auto& t : tagsAffectingStorageServer) {
+			auto const tQuota = getQuota(t, LimitType::TOTAL);
+			sumQuota += tQuota.orDefault(0);
+			if (tag.compare(tag) == 0) {
+				tagQuota = tQuota.orDefault(0);
+			}
+		}
+		if (tagQuota == 0.0) {
+			return 0;
+		}
+		ASSERT_GT(sumQuota, 0.0);
+		return tagQuota / sumQuota;
+	}
+
+	// Returns the desired cost for a storage server, based on its current
+	// cost and throttling ratio
+	Optional<double> getLimitingCost(UID storageServerId) const {
+		auto const throttlingRatio = tryGet(throttlingRatios, storageServerId);
+		auto const currentCost = getCurrentCost(storageServerId);
+		if (!throttlingRatio.present() || !currentCost.present() || !throttlingRatio.get().present()) {
+			return {};
+		}
+		return throttlingRatio.get().get() * currentCost.get();
+	}
+
+	// For a given storage server and tag combination, return the limiting transaction rate.
+	Optional<double> getLimitingTps(UID storageServerId, TransactionTag tag) const {
+		auto const quotaRatio = getQuotaRatio(tag, storageServerId);
+		auto const limitingCost = getLimitingCost(storageServerId);
+		auto const averageTransactionCost = getAverageTransactionCost(tag, storageServerId);
+		if (!limitingCost.present() || !averageTransactionCost.present()) {
+			return {};
+		}
+
+		auto const limitingCostForTag = limitingCost.get() * quotaRatio;
+		return limitingCostForTag / averageTransactionCost.get();
+	}
+
+	// Return the limiting transaction rate, aggregated across all storage servers
+	Optional<double> getLimitingTps(TransactionTag tag) const {
+		Optional<double> result;
+		for (const auto& [id, _] : throttlingRatios) {
+			auto const targetTpsForSS = getLimitingTps(id, tag);
+			result = getMin(result, targetTpsForSS);
+		}
+		return result;
+	}
+
+	Optional<double> getTps(TransactionTag tag, LimitType limitType, double averageTransactionCost) const {
+		auto const cost = getQuota(tag, limitType);
+		if (!cost.present()) {
+			return {};
+		} else {
+			return cost.get() / averageTransactionCost;
+		}
+	}
+
+	void removeUnseenQuotas(std::unordered_set<TransactionTag> const& tagsWithQuota) {
+		for (auto& [tag, stats] : tagStatistics) {
+			if (!tagsWithQuota.count(tag)) {
+				stats.clearQuota();
 			}
-			wait(delay(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TRACE_INTERVAL));
 		}
 	}
 
 	ACTOR static Future<Void> monitorThrottlingChanges(GlobalTagThrottlerImpl* self) {
+		state std::unordered_set<TransactionTag> tagsWithQuota;
+
 		loop {
 			state ReadYourWritesTransaction tr(self->db);
-
 			loop {
-				// TODO: Clean up quotas that have been removed
 				try {
 					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
+					tagsWithQuota.clear();
 					state RangeResult currentQuotas = wait(tr.getRange(tagQuotaKeys, CLIENT_KNOBS->TOO_MANY));
-					TraceEvent("GlobalTagThrottler_ReadCurrentQuotas").detail("Size", currentQuotas.size());
+					TraceEvent("GlobalTagThrottler_ReadCurrentQuotas", self->id).detail("Size", currentQuotas.size());
 					for (auto const kv : currentQuotas) {
 						auto const tag = kv.key.removePrefix(tagQuotaPrefix);
 						auto const quota = ThrottleApi::TagQuotaValue::fromValue(kv.value);
-						self->trackedTags[tag].setQuota(quota);
+						self->tagStatistics[tag].setQuota(quota);
+						tagsWithQuota.insert(tag);
 					}
-
+					self->removeUnseenQuotas(tagsWithQuota);
 					++self->throttledTagChangeId;
-					// FIXME: Should wait on watch instead
-					// wait(tr.watch(tagThrottleSignalKey));
 					wait(delay(5.0));
-					TraceEvent("GlobalTagThrottler_ChangeSignaled");
-					TEST(true); // Global tag throttler detected quota changes
 					break;
 				} catch (Error& e) {
-					TraceEvent("GlobalTagThrottlerMonitoringChangesError", self->id).error(e);
+					TraceEvent("GlobalTagThrottler_MonitoringChangesError", self->id).error(e);
 					wait(tr.onError(e));
 				}
 			}
 		}
 	}
 
+	Optional<double> getTargetTps(TransactionTag tag, bool& isBusy, TraceEvent& te) {
+		auto const limitingTps = getLimitingTps(tag);
+		auto const averageTransactionCost = getAverageTransactionCost(tag);
+		auto const desiredTps = getTps(tag, LimitType::TOTAL, averageTransactionCost);
+		if (!desiredTps.present()) {
+			return {};
+		}
+		auto reservedTps = getTps(tag, LimitType::RESERVED, averageTransactionCost);
+		auto targetTps = getMax(reservedTps, getMin(desiredTps, limitingTps));
+
+		isBusy = limitingTps.present() && limitingTps.get() < desiredTps.orDefault(0);
+
+		te.detail("Tag", printable(tag))
+		    .detail("TargetTps", targetTps)
+		    .detail("AverageTransactionCost", averageTransactionCost)
+		    .detail("LimitingTps", limitingTps)
+		    .detail("ReservedTps", reservedTps)
+		    .detail("DesiredTps", desiredTps)
+		    .detail("NumStorageServers", throughput.size());
+
+		return targetTps;
+	}
+
 public:
-	GlobalTagThrottlerImpl(Database db, UID id) : db(db), id(id) { traceActor = tracer(this); }
+	GlobalTagThrottlerImpl(Database db, UID id) : db(db), id(id) {}
 	Future<Void> monitorThrottlingChanges() { return monitorThrottlingChanges(this); }
-	void addRequests(TransactionTag tag, int count) { trackedTags[tag].addTransactions(count); }
+	void addRequests(TransactionTag tag, int count) { tagStatistics[tag].addTransactions(static_cast<double>(count)); }
 	uint64_t getThrottledTagChangeId() const { return throttledTagChangeId; }
-	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() {
-		// TODO: For now, only enforce total throttling rates.
-		// We should use reserved quotas as well.
-		PrioritizedTransactionTagMap<ClientTagThrottleLimits> result;
-		for (auto& [tag, quotaAndCounters] : trackedTags) {
+
+	PrioritizedTransactionTagMap<double> getProxyRates(int numProxies) {
+		PrioritizedTransactionTagMap<double> result;
+		lastBusyTagCount = 0;
+
+		for (auto& [tag, stats] : tagStatistics) {
 			// Currently there is no differentiation between batch priority and default priority transactions
-			auto const limit = quotaAndCounters.updateAndGetPerClientLimit();
-			if (limit.present()) {
-				result[TransactionPriority::BATCH][tag] = result[TransactionPriority::DEFAULT][tag] = limit.get();
+			TraceEvent te("GlobalTagThrottler_GotRate", id);
+			bool isBusy{ false };
+			auto const targetTps = getTargetTps(tag, isBusy, te);
+			if (isBusy) {
+				++lastBusyTagCount;
+			}
+			if (targetTps.present()) {
+				auto const smoothedTargetTps = stats.updateAndGetTargetLimit(targetTps.get());
+				result[TransactionPriority::BATCH][tag] = result[TransactionPriority::DEFAULT][tag] =
+				    smoothedTargetTps / numProxies;
+			} else {
+				te.disable();
+			}
+		}
+
+		return result;
+	}
+
+	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() {
+		PrioritizedTransactionTagMap<ClientTagThrottleLimits> result;
+		lastBusyTagCount = 0;
+
+		for (auto& [tag, stats] : tagStatistics) {
+			// Currently there is no differentiation between batch priority and default priority transactions
+			bool isBusy{ false };
+			TraceEvent te("GlobalTagThrottler_GotClientRate", id);
+			auto const targetTps = getTargetTps(tag, isBusy, te);
+
+			if (isBusy) {
+				++lastBusyTagCount;
+			}
+
+			if (targetTps.present()) {
+				auto const clientRate = stats.updateAndGetPerClientLimit(targetTps.get());
+				result[TransactionPriority::BATCH][tag] = result[TransactionPriority::DEFAULT][tag] = clientRate;
+				te.detail("ClientTps", clientRate.tpsRate);
+			} else {
+				te.disable();
 			}
 		}
 		return result;
 	}
-	int64_t autoThrottleCount() const { return trackedTags.size(); }
-	uint32_t busyReadTagCount() const {
-		// TODO: Implement
-		return 0;
+
+	int64_t autoThrottleCount() const {
+		int64_t result{ 0 };
+		for (const auto& [tag, stats] : tagStatistics) {
+			if (stats.getQuota().present()) {
+				++result;
+			}
+		}
+		return result;
 	}
-	uint32_t busyWriteTagCount() const {
-		// TODO: Implement
-		return 0;
-	}
-	int64_t manualThrottleCount() const { return trackedTags.size(); }
+	uint32_t busyReadTagCount() const { return lastBusyTagCount; }
+	uint32_t busyWriteTagCount() const { return lastBusyTagCount; }
+	int64_t manualThrottleCount() const { return 0; }
+
 	Future<Void> tryUpdateAutoThrottling(StorageQueueInfo const& ss) {
+		throttlingRatios[ss.id] = ss.getThrottlingRatio(SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER,
+		                                                SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER);
 		for (const auto& busyReadTag : ss.busiestReadTags) {
-			trackedTags[busyReadTag.tag].updateReadCostRate(ss.id, busyReadTag.rate);
+			throughput[ss.id][busyReadTag.tag].updateCost(busyReadTag.rate, OpType::READ);
 		}
 		for (const auto& busyWriteTag : ss.busiestWriteTags) {
-			trackedTags[busyWriteTag.tag].updateWriteCostRate(ss.id, busyWriteTag.rate);
+			throughput[ss.id][busyWriteTag.tag].updateCost(busyWriteTag.rate, OpType::WRITE);
 		}
-		// TODO: Call ThrottleApi::throttleTags
 		return Void();
 	}
 
 	void setQuota(TransactionTagRef tag, ThrottleApi::TagQuotaValue const& tagQuotaValue) {
-		trackedTags[tag].setQuota(tagQuotaValue);
+		tagStatistics[tag].setQuota(tagQuotaValue);
 	}
+
+	void removeQuota(TransactionTagRef tag) { tagStatistics[tag].clearQuota(); }
 };
 
 GlobalTagThrottler::GlobalTagThrottler(Database db, UID id) : impl(PImpl<GlobalTagThrottlerImpl>::create(db, id)) {}
@@ -236,6 +497,9 @@ uint64_t GlobalTagThrottler::getThrottledTagChangeId() const {
 PrioritizedTransactionTagMap<ClientTagThrottleLimits> GlobalTagThrottler::getClientRates() {
 	return impl->getClientRates();
 }
+PrioritizedTransactionTagMap<double> GlobalTagThrottler::getProxyRates(int numProxies) {
+	return impl->getProxyRates(numProxies);
+}
 int64_t GlobalTagThrottler::autoThrottleCount() const {
 	return impl->autoThrottleCount();
 }
@@ -259,8 +523,15 @@ void GlobalTagThrottler::setQuota(TransactionTagRef tag, ThrottleApi::TagQuotaVa
 	return impl->setQuota(tag, tagQuotaValue);
 }
 
+void GlobalTagThrottler::removeQuota(TransactionTagRef tag) {
+	return impl->removeQuota(tag);
+}
+
 namespace GlobalTagThrottlerTesting {
 
+enum class LimitType { RESERVED, TOTAL };
+enum class OpType { READ, WRITE };
+
 Optional<double> getTPSLimit(GlobalTagThrottler& globalTagThrottler, TransactionTag tag) {
 	auto clientRates = globalTagThrottler.getClientRates();
 	auto it1 = clientRates.find(TransactionPriority::DEFAULT);
@@ -273,7 +544,7 @@ Optional<double> getTPSLimit(GlobalTagThrottler& globalTagThrottler, Transaction
 	return {};
 }
 
-class StorageServerCollection {
+class MockStorageServer {
 	class Cost {
 		Smoother smoother;
 
@@ -286,82 +557,106 @@ class StorageServerCollection {
 		double smoothRate() const { return smoother.smoothRate(); }
 	};
 
-	std::vector<std::map<TransactionTag, Cost>> readCosts;
-	std::vector<std::map<TransactionTag, Cost>> writeCosts;
+	UID id;
+	double targetCost;
+	std::map<TransactionTag, Cost> readCosts, writeCosts;
+	Cost totalReadCost, totalWriteCost;
 
 public:
-	StorageServerCollection(size_t size) : readCosts(size), writeCosts(size) { ASSERT_GT(size, 0); }
-
+	explicit MockStorageServer(UID id, double targetCost) : id(id), targetCost(targetCost) { ASSERT_GT(targetCost, 0); }
 	void addReadCost(TransactionTag tag, double cost) {
-		auto const costPerSS = cost / readCosts.size();
-		for (auto& readCost : readCosts) {
-			readCost[tag] += costPerSS;
+		readCosts[tag] += cost;
+		totalReadCost += cost;
+	}
+	void addWriteCost(TransactionTag tag, double cost) {
+		writeCosts[tag] += cost;
+		totalWriteCost += cost;
+	}
+
+	StorageQueueInfo getStorageQueueInfo() const {
+		StorageQueueInfo result(id, LocalityData{});
+		for (const auto& [tag, readCost] : readCosts) {
+			double fractionalBusyness{ 0.0 }; // unused for global tag throttling
+			result.busiestReadTags.emplace_back(tag, readCost.smoothRate(), fractionalBusyness);
+		}
+		for (const auto& [tag, writeCost] : writeCosts) {
+			double fractionalBusyness{ 0.0 }; // unused for global tag throttling
+			result.busiestWriteTags.emplace_back(tag, writeCost.smoothRate(), fractionalBusyness);
+		}
+		result.lastReply.bytesInput = ((totalReadCost.smoothRate() + totalWriteCost.smoothRate()) / targetCost) *
+		                              SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER;
+		return result;
+	}
+};
+
+class StorageServerCollection {
+	std::vector<MockStorageServer> storageServers;
+
+public:
+	StorageServerCollection(size_t size, double targetCost) {
+		ASSERT_GT(size, 0);
+		storageServers.reserve(size);
+		for (int i = 0; i < size; ++i) {
+			storageServers.emplace_back(UID(i, i), targetCost);
 		}
 	}
 
-	void addWriteCost(TransactionTag tag, double cost) {
-		auto const costPerSS = cost / writeCosts.size();
-		for (auto& writeCost : writeCosts) {
-			writeCost[tag] += costPerSS;
+	void addCost(TransactionTag tag, double cost, std::vector<int> const& storageServerIndices, OpType opType) {
+		if (storageServerIndices.empty()) {
+			auto const costPerSS = cost / storageServers.size();
+			for (auto& storageServer : storageServers) {
+				if (opType == OpType::READ) {
+					storageServer.addReadCost(tag, costPerSS);
+				} else {
+					storageServer.addWriteCost(tag, costPerSS);
+				}
+			}
+		} else {
+			auto const costPerSS = cost / storageServerIndices.size();
+			for (auto i : storageServerIndices) {
+				if (opType == OpType::READ) {
+					storageServers[i].addReadCost(tag, costPerSS);
+				} else {
+					storageServers[i].addWriteCost(tag, costPerSS);
+				}
+			}
 		}
 	}
 
 	std::vector<StorageQueueInfo> getStorageQueueInfos() const {
 		std::vector<StorageQueueInfo> result;
-		result.reserve(readCosts.size());
-		for (int i = 0; i < readCosts.size(); ++i) {
-			StorageQueueInfo sqInfo(UID(i, i), LocalityData{});
-			for (const auto& [tag, readCost] : readCosts[i]) {
-				double fractionalBusyness{ 0.0 }; // unused for global tag throttling
-				sqInfo.busiestReadTags.emplace_back(tag, readCost.smoothRate(), fractionalBusyness);
-			}
-			for (const auto& [tag, writeCost] : writeCosts[i]) {
-				double fractionalBusyness{ 0.0 }; // unused for global tag throttling
-				sqInfo.busiestWriteTags.emplace_back(tag, writeCost.smoothRate(), fractionalBusyness);
-			}
-			result.push_back(sqInfo);
+		result.reserve(storageServers.size());
+		for (const auto& storageServer : storageServers) {
+			result.push_back(storageServer.getStorageQueueInfo());
 		}
 		return result;
 	}
 };
 
-ACTOR static Future<Void> runClient(GlobalTagThrottler* globalTagThrottler,
-                                    StorageServerCollection* storageServers,
-                                    TransactionTag tag,
-                                    double desiredTpsRate,
-                                    double costPerTransaction,
-                                    bool write) {
+ACTOR Future<Void> runClient(GlobalTagThrottler* globalTagThrottler,
+                             StorageServerCollection* storageServers,
+                             TransactionTag tag,
+                             double tpsRate,
+                             double costPerTransaction,
+                             OpType opType,
+                             std::vector<int> storageServerIndices = std::vector<int>()) {
 	loop {
 		auto tpsLimit = getTPSLimit(*globalTagThrottler, tag);
-		state double tpsRate = tpsLimit.present() ? std::min<double>(desiredTpsRate, tpsLimit.get()) : desiredTpsRate;
-		wait(delay(1 / tpsRate));
-		if (write) {
-			storageServers->addWriteCost(tag, costPerTransaction);
-		} else {
-			storageServers->addReadCost(tag, costPerTransaction);
-		}
+		state double enforcedRate = tpsLimit.present() ? std::min<double>(tpsRate, tpsLimit.get()) : tpsRate;
+		wait(delay(1 / enforcedRate));
+		storageServers->addCost(tag, costPerTransaction, storageServerIndices, opType);
 		globalTagThrottler->addRequests(tag, 1);
 	}
 }
 
-ACTOR static Future<Void> monitorClientRates(GlobalTagThrottler* globalTagThrottler,
-                                             TransactionTag tag,
-                                             double desiredTPSLimit) {
+ACTOR template <class Check>
+Future<Void> monitor(GlobalTagThrottler* globalTagThrottler, Check check) {
 	state int successes = 0;
 	loop {
 		wait(delay(1.0));
-		auto currentTPSLimit = getTPSLimit(*globalTagThrottler, tag);
-		if (currentTPSLimit.present()) {
-			TraceEvent("GlobalTagThrottling_RateMonitor")
-			    .detail("Tag", tag)
-			    .detail("CurrentTPSRate", currentTPSLimit.get())
-			    .detail("DesiredTPSRate", desiredTPSLimit);
-			if (abs(currentTPSLimit.get() - desiredTPSLimit) < 1.0) {
-				if (++successes == 3) {
-					return Void();
-				}
-			} else {
-				successes = 0;
+		if (check(*globalTagThrottler)) {
+			if (++successes == 3) {
+				return Void();
 			}
 		} else {
 			successes = 0;
@@ -369,8 +664,54 @@ ACTOR static Future<Void> monitorClientRates(GlobalTagThrottler* globalTagThrott
 	}
 }
 
-ACTOR static Future<Void> updateGlobalTagThrottler(GlobalTagThrottler* globalTagThrottler,
-                                                   StorageServerCollection const* storageServers) {
+bool isNear(double a, double b) {
+	return abs(a - b) < 3.0;
+}
+
+bool isNear(Optional<double> a, Optional<double> b) {
+	if (a.present()) {
+		return b.present() && isNear(a.get(), b.get());
+	} else {
+		return !b.present();
+	}
+}
+
+bool targetRateIsNear(GlobalTagThrottler& globalTagThrottler, TransactionTag tag, Optional<double> expected) {
+	Optional<double> rate;
+	auto targetRates = globalTagThrottler.getProxyRates(1);
+	auto it1 = targetRates.find(TransactionPriority::DEFAULT);
+	if (it1 != targetRates.end()) {
+		auto it2 = it1->second.find(tag);
+		if (it2 != it1->second.end()) {
+			rate = it2->second;
+		}
+	}
+	TraceEvent("GlobalTagThrottling_RateMonitor")
+	    .detail("Tag", tag)
+	    .detail("CurrentTPSRate", rate)
+	    .detail("ExpectedTPSRate", expected);
+	return isNear(rate, expected);
+}
+
+bool clientRateIsNear(GlobalTagThrottler& globalTagThrottler, TransactionTag tag, Optional<double> expected) {
+	Optional<double> rate;
+	auto clientRates = globalTagThrottler.getClientRates();
+	auto it1 = clientRates.find(TransactionPriority::DEFAULT);
+	if (it1 != clientRates.end()) {
+		auto it2 = it1->second.find(tag);
+		if (it2 != it1->second.end()) {
+			rate = it2->second.tpsRate;
+		}
+	}
+	TraceEvent("GlobalTagThrottling_ClientRateMonitor")
+	    .detail("Tag", tag)
+	    .detail("CurrentTPSRate", rate)
+	    .detail("ExpectedTPSRate", expected);
+	return isNear(rate, expected);
+}
+
+ACTOR Future<Void> updateGlobalTagThrottler(GlobalTagThrottler* globalTagThrottler,
+                                            StorageServerCollection const* storageServers) {
 	loop {
 		wait(delay(1.0));
 		auto const storageQueueInfos = storageServers->getStorageQueueInfos();
@@ -384,150 +725,307 @@ ACTOR static Future<Void> updateGlobalTagThrottler(GlobalTagThrottler* globalTag
 
 TEST_CASE("/GlobalTagThrottler/Simple") {
 	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
-	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10);
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 100);
 	ThrottleApi::TagQuotaValue tagQuotaValue;
 	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalReadQuota = 100.0;
+	tagQuotaValue.totalQuota = 100.0;
 	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, false);
-	state Future<Void> monitor =
-	    GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag, 100.0 / 6.0);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 5.0, 6.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag, 100.0 / 6.0);
+	});
 	state Future<Void> updater =
 	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 300.0));
+	wait(timeoutError(monitor || client || updater, 600.0));
 	return Void();
 }
 
 TEST_CASE("/GlobalTagThrottler/WriteThrottling") {
 	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
-	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10);
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 100);
 	ThrottleApi::TagQuotaValue tagQuotaValue;
 	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalWriteQuota = 100.0;
+	tagQuotaValue.totalQuota = 100.0;
 	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, true);
-	state Future<Void> monitor =
-	    GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag, 100.0 / 6.0);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 5.0, 6.0, GlobalTagThrottlerTesting::OpType::WRITE);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag, 100.0 / 6.0);
+	});
 	state Future<Void> updater =
 	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 300.0));
+	wait(timeoutError(monitor || client || updater, 600.0));
 	return Void();
 }
 
 TEST_CASE("/GlobalTagThrottler/MultiTagThrottling") {
 	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
-	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10);
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 100);
 	ThrottleApi::TagQuotaValue tagQuotaValue;
 	TransactionTag testTag1 = "sampleTag1"_sr;
 	TransactionTag testTag2 = "sampleTag2"_sr;
-	tagQuotaValue.totalReadQuota = 100.0;
+	tagQuotaValue.totalQuota = 100.0;
 	globalTagThrottler.setQuota(testTag1, tagQuotaValue);
 	globalTagThrottler.setQuota(testTag2, tagQuotaValue);
 	state std::vector<Future<Void>> futures;
 	state std::vector<Future<Void>> monitorFutures;
-	futures.push_back(
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag1, 5.0, 6.0, false));
-	futures.push_back(
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag2, 5.0, 6.0, false));
+	futures.push_back(GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag1, 5.0, 6.0, GlobalTagThrottlerTesting::OpType::READ));
+	futures.push_back(GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag2, 5.0, 6.0, GlobalTagThrottlerTesting::OpType::READ));
 	futures.push_back(GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers));
-	monitorFutures.push_back(GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag1, 100.0 / 6.0));
-	monitorFutures.push_back(GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag2, 100.0 / 6.0));
-	wait(timeoutError(waitForAny(futures) || waitForAll(monitorFutures), 300.0));
+	state Future<Void> monitor =
+	    GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag1, testTag2](auto& gtt) {
+		    return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag1, 100.0 / 6.0) &&
+		           GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag2, 100.0 / 6.0);
+	    });
+	wait(timeoutError(waitForAny(futures) || monitor, 600.0));
 	return Void();
 }
 
-TEST_CASE("/GlobalTagThrottler/ActiveThrottling") {
+TEST_CASE("/GlobalTagThrottler/AttemptWorkloadAboveQuota") {
 	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
-	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10);
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 100);
 	ThrottleApi::TagQuotaValue tagQuotaValue;
 	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalReadQuota = 100.0;
+	tagQuotaValue.totalQuota = 100.0;
 	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 20.0, 10.0, false);
-	state Future<Void> monitor = GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag, 10.0);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 20.0, 10.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag, 10.0);
+	});
 	state Future<Void> updater =
 	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 300.0));
+	wait(timeoutError(monitor || client || updater, 600.0));
 	return Void();
 }
 
 TEST_CASE("/GlobalTagThrottler/MultiClientThrottling") {
 	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
-	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10);
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 100);
 	ThrottleApi::TagQuotaValue tagQuotaValue;
 	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalReadQuota = 100.0;
+	tagQuotaValue.totalQuota = 100.0;
 	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, false);
-	state Future<Void> client2 =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, false);
-	state Future<Void> monitor =
-	    GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag, 100.0 / 6.0);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 5.0, 6.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> client2 = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 5.0, 6.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag, 100.0 / 6.0) &&
+		       GlobalTagThrottlerTesting::clientRateIsNear(gtt, testTag, 100.0 / 6.0);
+	});
 	state Future<Void> updater =
 	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 300.0));
+	wait(timeoutError(monitor || client || client2 || updater, 600.0));
 	return Void();
 }
 
-TEST_CASE("/GlobalTagThrottler/MultiClientActiveThrottling") {
+TEST_CASE("/GlobalTagThrottler/MultiClientThrottling2") {
 	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
-	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10);
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 100);
 	ThrottleApi::TagQuotaValue tagQuotaValue;
 	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalReadQuota = 100.0;
+	tagQuotaValue.totalQuota = 100.0;
 	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 20.0, 10.0, false);
-	state Future<Void> client2 =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 20.0, 10.0, false);
-	state Future<Void> monitor = GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag, 5.0);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 20.0, 10.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> client2 = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 20.0, 10.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag, 10.0) &&
+		       GlobalTagThrottlerTesting::clientRateIsNear(gtt, testTag, 5.0);
+	});
 	state Future<Void> updater =
 	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 300.0));
+	wait(timeoutError(monitor || client || updater, 600.0));
 	return Void();
 }
 
 // Global transaction rate should be 20.0, with a distribution of (5, 15) between the 2 clients
-TEST_CASE("/GlobalTagThrottler/SkewedMultiClientActiveThrottling") {
+TEST_CASE("/GlobalTagThrottler/SkewedMultiClientThrottling") {
 	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
-	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10);
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 100);
 	ThrottleApi::TagQuotaValue tagQuotaValue;
 	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalReadQuota = 100.0;
+	tagQuotaValue.totalQuota = 100.0;
 	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 5.0, false);
-	state Future<Void> client2 =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 25.0, 5.0, false);
-	state Future<Void> monitor = GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag, 15.0);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 5.0, 5.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> client2 = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 25.0, 5.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag, 20.0) &&
+		       GlobalTagThrottlerTesting::clientRateIsNear(gtt, testTag, 15.0);
+	});
 	state Future<Void> updater =
 	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 300.0));
+	wait(timeoutError(monitor || client || updater, 600.0));
 	return Void();
 }
 
 // Test that the tag throttler can reach equilibrium, then adjust to a new equilibrium once the quota is changed
 TEST_CASE("/GlobalTagThrottler/UpdateQuota") {
 	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
-	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10);
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 100);
 	state ThrottleApi::TagQuotaValue tagQuotaValue;
 	state TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalReadQuota = 100.0;
+	tagQuotaValue.totalQuota = 100.0;
 	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client =
-	    GlobalTagThrottlerTesting::runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, false);
-	state Future<Void> monitor =
-	    GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag, 100.0 / 6.0);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 5.0, 6.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, "sampleTag1"_sr, 100.0 / 6.0);
+	});
 	state Future<Void> updater =
 	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 300.0));
-	tagQuotaValue.totalReadQuota = 50.0;
+	wait(timeoutError(monitor || client || updater, 600.0));
+	tagQuotaValue.totalQuota = 50.0;
 	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	monitor = GlobalTagThrottlerTesting::monitorClientRates(&globalTagThrottler, testTag, 50.0 / 6.0);
-	wait(timeoutError(monitor || client || updater, 300.0));
+	monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, "sampleTag1"_sr, 50.0 / 6.0);
+	});
+	wait(timeoutError(monitor || client || updater, 600.0));
+	return Void();
+}
+
+TEST_CASE("/GlobalTagThrottler/RemoveQuota") {
+	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 100);
+	state ThrottleApi::TagQuotaValue tagQuotaValue;
+	state TransactionTag testTag = "sampleTag1"_sr;
+	tagQuotaValue.totalQuota = 100.0;
+	globalTagThrottler.setQuota(testTag, tagQuotaValue);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 5.0, 6.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, "sampleTag1"_sr, 100.0 / 6.0);
+	});
+	state Future<Void> updater =
+	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
+	wait(timeoutError(monitor || client || updater, 600.0));
+	globalTagThrottler.removeQuota(testTag);
+	monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, "sampleTag1"_sr, {});
+	});
+	wait(timeoutError(monitor || client || updater, 600.0));
+	return Void();
+}
+
+TEST_CASE("/GlobalTagThrottler/ActiveThrottling") {
+	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 5);
+	state ThrottleApi::TagQuotaValue tagQuotaValue;
+	TransactionTag testTag = "sampleTag1"_sr;
+	tagQuotaValue.totalQuota = 100.0;
+	globalTagThrottler.setQuota(testTag, tagQuotaValue);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 10.0, 6.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag, 50 / 6.0) && gtt.busyReadTagCount() == 1;
+	});
+	state Future<Void> updater =
+	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
+	wait(timeoutError(monitor || client || updater, 600.0));
+	return Void();
+}
+
+TEST_CASE("/GlobalTagThrottler/MultiTagActiveThrottling") {
+	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 5);
+	state ThrottleApi::TagQuotaValue tagQuotaValue1;
+	state ThrottleApi::TagQuotaValue tagQuotaValue2;
+	TransactionTag testTag1 = "sampleTag1"_sr;
+	TransactionTag testTag2 = "sampleTag2"_sr;
+	tagQuotaValue1.totalQuota = 50.0;
+	tagQuotaValue2.totalQuota = 100.0;
+	globalTagThrottler.setQuota(testTag1, tagQuotaValue1);
+	globalTagThrottler.setQuota(testTag2, tagQuotaValue2);
+	std::vector<Future<Void>> futures;
+	futures.push_back(GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag1, 10.0, 6.0, GlobalTagThrottlerTesting::OpType::READ));
+	futures.push_back(GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag2, 10.0, 6.0, GlobalTagThrottlerTesting::OpType::READ));
+	state Future<Void> monitor =
+	    GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag1, testTag2](auto& gtt) {
+		    return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag1, (50 / 6.0) / 3) &&
+		           GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag2, 2 * (50 / 6.0) / 3) &&
+		           gtt.busyReadTagCount() == 2;
+	    });
+	futures.push_back(GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers));
+	wait(timeoutError(waitForAny(futures) || monitor, 600.0));
+	return Void();
+}
+
+TEST_CASE("/GlobalTagThrottler/MultiTagActiveThrottling2") {
+	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(3, 50);
+	state ThrottleApi::TagQuotaValue tagQuotaValue1;
+	state ThrottleApi::TagQuotaValue tagQuotaValue2;
+	TransactionTag testTag1 = "sampleTag1"_sr;
+	TransactionTag testTag2 = "sampleTag2"_sr;
+	tagQuotaValue1.totalQuota = 100.0;
+	tagQuotaValue2.totalQuota = 100.0;
+	globalTagThrottler.setQuota(testTag1, tagQuotaValue1);
+	globalTagThrottler.setQuota(testTag2, tagQuotaValue2);
+	std::vector<Future<Void>> futures;
+	futures.push_back(GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag1, 10.0, 6.0, GlobalTagThrottlerTesting::OpType::READ, { 0, 1 }));
+	futures.push_back(GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag2, 10.0, 6.0, GlobalTagThrottlerTesting::OpType::READ, { 1, 2 }));
+	state Future<Void> monitor =
+	    GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag1, testTag2](auto& gtt) {
+		    return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag1, 50 / 6.0) &&
+		           GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag2, 50 / 6.0) && gtt.busyReadTagCount() == 2;
+	    });
+	futures.push_back(GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers));
+	wait(timeoutError(waitForAny(futures) || monitor, 600.0));
+	return Void();
+}
+
+TEST_CASE("/GlobalTagThrottler/MultiTagActiveThrottling3") {
+	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(3, 50);
+	state ThrottleApi::TagQuotaValue tagQuotaValue1;
+	state ThrottleApi::TagQuotaValue tagQuotaValue2;
+	TransactionTag testTag1 = "sampleTag1"_sr;
+	TransactionTag testTag2 = "sampleTag2"_sr;
+	tagQuotaValue1.totalQuota = 100.0;
+	tagQuotaValue2.totalQuota = 100.0;
+	globalTagThrottler.setQuota(testTag1, tagQuotaValue1);
+	globalTagThrottler.setQuota(testTag2, tagQuotaValue2);
+	std::vector<Future<Void>> futures;
+	futures.push_back(GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag1, 10.0, 6.0, GlobalTagThrottlerTesting::OpType::READ, { 0 }));
+	futures.push_back(GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag2, 10.0, 6.0, GlobalTagThrottlerTesting::OpType::READ, { 1, 2 }));
+	state Future<Void> monitor =
+	    GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag1, testTag2](auto& gtt) {
+		    return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag1, 50 / 6.0) &&
+		           GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag2, 100 / 6.0) && gtt.busyReadTagCount() == 1;
+	    });
+	futures.push_back(GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers));
+	wait(timeoutError(waitForAny(futures) || monitor, 600.0));
+	return Void();
+}
+
+TEST_CASE("/GlobalTagThrottler/ReservedQuota") {
+	state GlobalTagThrottler globalTagThrottler(Database{}, UID{});
+	state GlobalTagThrottlerTesting::StorageServerCollection storageServers(10, 5);
+	state ThrottleApi::TagQuotaValue tagQuotaValue;
+	TransactionTag testTag = "sampleTag1"_sr;
+	tagQuotaValue.totalQuota = 100.0;
+	tagQuotaValue.reservedQuota = 70.0;
+	globalTagThrottler.setQuota(testTag, tagQuotaValue);
+	state Future<Void> client = GlobalTagThrottlerTesting::runClient(
+	    &globalTagThrottler, &storageServers, testTag, 10.0, 6.0, GlobalTagThrottlerTesting::OpType::READ);
+	state Future<Void> monitor = GlobalTagThrottlerTesting::monitor(&globalTagThrottler, [testTag](auto& gtt) {
+		return GlobalTagThrottlerTesting::targetRateIsNear(gtt, testTag, 70 / 6.0);
+	});
+	state Future<Void> updater =
+	    GlobalTagThrottlerTesting::updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
+	wait(timeoutError(monitor || client || updater, 600.0));
 	return Void();
 }
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index 30ad98bcf1..789b2973b1 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -18,9 +18,12 @@
  * limitations under the License.
  */
 
+#include "fdbclient/ClientKnobs.h"
+#include "fdbclient/DatabaseContext.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/TransactionLineage.h"
+#include "fdbclient/Tuple.h"
 #include "fdbserver/LogSystem.h"
 #include "fdbserver/LogSystemDiskQueueAdapter.h"
 #include "fdbclient/CommitProxyInterface.h"
@@ -29,6 +32,7 @@
 #include "fdbserver/WaitFailure.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbrpc/sim_validation.h"
+#include "flow/IRandom.h"
 #include "flow/flow.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -126,12 +130,10 @@ struct GrvProxyStats {
 	                          SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
 	    recentRequests(0), lastBucketBegin(now()),
 	    bucketInterval(FLOW_KNOBS->BASIC_LOAD_BALANCE_UPDATE_RATE / FLOW_KNOBS->BASIC_LOAD_BALANCE_BUCKETS),
-	    grvConfirmEpochLiveDist(Histogram::getHistogram(LiteralStringRef("GrvProxy"),
-	                                                    LiteralStringRef("GrvConfirmEpochLive"),
-	                                                    Histogram::Unit::microseconds)),
-	    grvGetCommittedVersionRpcDist(Histogram::getHistogram(LiteralStringRef("GrvProxy"),
-	                                                          LiteralStringRef("GrvGetCommittedVersionRpc"),
-	                                                          Histogram::Unit::microseconds)) {
+	    grvConfirmEpochLiveDist(
+	        Histogram::getHistogram("GrvProxy"_sr, "GrvConfirmEpochLive"_sr, Histogram::Unit::microseconds)),
+	    grvGetCommittedVersionRpcDist(
+	        Histogram::getHistogram("GrvProxy"_sr, "GrvGetCommittedVersionRpc"_sr, Histogram::Unit::microseconds)) {
 		// The rate at which the limit(budget) is allowed to grow.
 		specialCounter(cc, "SystemGRVQueueSize", [this]() { return this->systemGRVQueueSize; });
 		specialCounter(cc, "DefaultGRVQueueSize", [this]() { return this->defaultGRVQueueSize; });
@@ -166,7 +168,7 @@ struct GrvTransactionRateInfo {
 	Smoother smoothRate;
 	Smoother smoothReleased;
 
-	GrvTransactionRateInfo(double rate)
+	GrvTransactionRateInfo(double rate = 0.0)
 	  : rate(rate), limit(0), budget(0), disabled(true), smoothRate(SERVER_KNOBS->START_TRANSACTION_RATE_WINDOW),
 	    smoothReleased(SERVER_KNOBS->START_TRANSACTION_RATE_WINDOW) {}
 
@@ -183,7 +185,7 @@ struct GrvTransactionRateInfo {
 		limit = SERVER_KNOBS->START_TRANSACTION_RATE_WINDOW * releaseRate;
 	}
 
-	bool canStart(int64_t numAlreadyStarted, int64_t count) {
+	bool canStart(int64_t numAlreadyStarted, int64_t count) const {
 		return numAlreadyStarted + count <=
 		       std::min(limit + budget, SERVER_KNOBS->START_TRANSACTION_MAX_TRANSACTIONS_TO_START);
 	}
@@ -252,6 +254,7 @@ struct GrvProxyData {
 	int updateCommitRequests;
 	NotifiedDouble lastCommitTime;
 
+	Version version;
 	Version minKnownCommittedVersion; // we should ask master for this version.
 
 	// Cache of the latest commit versions of storage servers.
@@ -284,7 +287,7 @@ struct GrvProxyData {
 	                                dbgid,
 	                                SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
 	                                SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
-	    updateCommitRequests(0), lastCommitTime(0), minKnownCommittedVersion(invalidVersion) {}
+	    updateCommitRequests(0), lastCommitTime(0), version(0), minKnownCommittedVersion(invalidVersion) {}
 };
 
 ACTOR Future<Void> healthMetricsRequestServer(GrvProxyInterface grvProxy,
@@ -302,6 +305,127 @@ ACTOR Future<Void> healthMetricsRequestServer(GrvProxyInterface grvProxy,
 	}
 }
 
+// Older FDB versions used different keys for client profiling data. This
+// function performs a one-time migration of data in these keys to the new
+// global configuration key space.
+ACTOR Future<Void> globalConfigMigrate(GrvProxyData* grvProxyData) {
+	state Key migratedKey("\xff\x02/fdbClientInfo/migrated/"_sr);
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(grvProxyData->cx);
+	try {
+		loop {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+
+			try {
+				state Optional<Value> migrated = wait(tr->get(migratedKey));
+				if (migrated.present()) {
+					// Already performed migration.
+					break;
+				}
+
+				state Optional<Value> sampleRate =
+				    wait(tr->get(Key("\xff\x02/fdbClientInfo/client_txn_sample_rate/"_sr)));
+				state Optional<Value> sizeLimit =
+				    wait(tr->get(Key("\xff\x02/fdbClientInfo/client_txn_size_limit/"_sr)));
+
+				tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+				// The value doesn't matter too much, as long as the key is set.
+				tr->set(migratedKey.contents(), "1"_sr);
+				if (sampleRate.present()) {
+					const double sampleRateDbl =
+					    BinaryReader::fromStringRef<double>(sampleRate.get().contents(), Unversioned());
+					Tuple rate = Tuple::makeTuple(sampleRateDbl);
+					tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack());
+				}
+				if (sizeLimit.present()) {
+					const int64_t sizeLimitInt =
+					    BinaryReader::fromStringRef<int64_t>(sizeLimit.get().contents(), Unversioned());
+					Tuple size = Tuple::makeTuple(sizeLimitInt);
+					tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack());
+				}
+
+				wait(tr->commit());
+				break;
+			} catch (Error& e) {
+				// Multiple GRV proxies may attempt this migration at the same
+				// time, sometimes resulting in aborts due to conflicts.
+				TraceEvent(SevInfo, "GlobalConfigRetryableMigrationError").errorUnsuppressed(e).suppressFor(1.0);
+				wait(tr->onError(e));
+			}
+		}
+	} catch (Error& e) {
+		// Catch non-retryable errors (and do nothing).
+		TraceEvent(SevWarnAlways, "GlobalConfigMigrationError").error(e);
+	}
+	return Void();
+}
+
+// Periodically refresh local copy of global configuration.
+ACTOR Future<Void> globalConfigRefresh(GrvProxyData* grvProxyData, Version* cachedVersion, RangeResult* cachedData) {
+	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(grvProxyData->cx);
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			state Future<Optional<Value>> globalConfigVersionFuture = tr->get(globalConfigVersionKey);
+			state Future<RangeResult> tmpCachedDataFuture = tr->getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY);
+			state Optional<Value> globalConfigVersion = wait(globalConfigVersionFuture);
+			RangeResult tmpCachedData = wait(tmpCachedDataFuture);
+			*cachedData = tmpCachedData;
+			if (globalConfigVersion.present()) {
+				Version parsedVersion;
+				memcpy(&parsedVersion, globalConfigVersion.get().begin(), sizeof(Version));
+				*cachedVersion = bigEndian64(parsedVersion);
+			}
+			return Void();
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+}
+
+// Handle common GlobalConfig transactions on the server side, because not all
+// clients are allowed to read system keys. Eventually, this could become its
+// own role.
+ACTOR Future<Void> globalConfigRequestServer(GrvProxyData* grvProxyData, GrvProxyInterface grvProxy) {
+	state ActorCollection actors(false);
+	state Future<Void> refreshFuture; // so there is only one running attempt
+	state Version cachedVersion = 0;
+	state RangeResult cachedData;
+
+	// Attempt to refresh the configuration database while the migration is
+	// ongoing. This is a small optimization to avoid waiting for the migration
+	// actor to complete.
+	refreshFuture = timeout(globalConfigRefresh(grvProxyData, &cachedVersion, &cachedData),
+	                        SERVER_KNOBS->GLOBAL_CONFIG_REFRESH_TIMEOUT,
+	                        Void()) &&
+	                delay(SERVER_KNOBS->GLOBAL_CONFIG_REFRESH_INTERVAL);
+
+	// Run one-time migration to support upgrades.
+	wait(success(timeout(globalConfigMigrate(grvProxyData), SERVER_KNOBS->GLOBAL_CONFIG_MIGRATE_TIMEOUT)));
+
+	loop {
+		choose {
+			when(GlobalConfigRefreshRequest refresh = waitNext(grvProxy.refreshGlobalConfig.getFuture())) {
+				// Must have an up to date copy of global configuration in
+				// order to serve it to the client (up to date from the clients
+				// point of view. The client learns the version through a
+				// ClientDBInfo update).
+				if (refresh.lastKnown <= cachedVersion) {
+					refresh.reply.send(GlobalConfigRefreshReply{ cachedData.arena(), cachedData });
+				} else {
+					refresh.reply.sendError(future_version());
+				}
+			}
+			when(wait(refreshFuture)) {
+				refreshFuture = timeout(globalConfigRefresh(grvProxyData, &cachedVersion, &cachedData),
+				                        SERVER_KNOBS->GLOBAL_CONFIG_REFRESH_TIMEOUT,
+				                        Void()) &&
+				                delay(SERVER_KNOBS->GLOBAL_CONFIG_REFRESH_INTERVAL);
+			}
+			when(wait(actors.getResult())) { ASSERT(false); }
+		}
+	}
+}
+
 // Get transaction rate info from RateKeeper.
 ACTOR Future<Void> getRate(UID myID,
                            Reference<AsyncVar<ServerDBInfo> const> db,
@@ -312,8 +436,10 @@ ACTOR Future<Void> getRate(UID myID,
                            GetHealthMetricsReply* healthMetricsReply,
                            GetHealthMetricsReply* detailedHealthMetricsReply,
                            TransactionTagMap<uint64_t>* transactionTagCounter,
-                           PrioritizedTransactionTagMap<ClientTagThrottleLimits>* throttledTags,
-                           GrvProxyStats* stats) {
+                           PrioritizedTransactionTagMap<ClientTagThrottleLimits>* clientThrottledTags,
+                           PrioritizedTransactionTagMap<GrvTransactionRateInfo>* perTagRateInfo,
+                           GrvProxyStats* stats,
+                           GrvProxyData* proxyData) {
 	state Future<Void> nextRequestTimer = Never();
 	state Future<Void> leaseTimeout = Never();
 	state Future<GetRateInfoReply> reply = Never();
@@ -338,14 +464,13 @@ ACTOR Future<Void> getRate(UID myID,
 			nextRequestTimer = Never();
 			bool detailed = now() - lastDetailedReply > SERVER_KNOBS->DETAILED_METRIC_UPDATE_RATE;
 
-			TransactionTagMap<uint64_t> tagCounts;
-			for (auto itr : *throttledTags) {
-				for (auto priorityThrottles : itr.second) {
-					tagCounts[priorityThrottles.first] = (*transactionTagCounter)[priorityThrottles.first];
-				}
-			}
-			reply = brokenPromiseToNever(db->get().ratekeeper.get().getRateInfo.getReply(GetRateInfoRequest(
-			    myID, *inTransactionCount, *inBatchTransactionCount, *transactionTagCounter, detailed)));
+			reply = brokenPromiseToNever(
+			    db->get().ratekeeper.get().getRateInfo.getReply(GetRateInfoRequest(myID,
+			                                                                       *inTransactionCount,
+			                                                                       *inBatchTransactionCount,
+			                                                                       proxyData->version,
+			                                                                       *transactionTagCounter,
+			                                                                       detailed)));
 			transactionTagCounter->clear();
 			expectingDetailedReply = detailed;
 		}
@@ -369,8 +494,16 @@ ACTOR Future<Void> getRate(UID myID,
 
 			// Replace our throttles with what was sent by ratekeeper. Because we do this,
 			// we are not required to expire tags out of the map
-			if (rep.throttledTags.present()) {
-				*throttledTags = std::move(rep.throttledTags.get());
+			if (rep.clientThrottledTags.present()) {
+				*clientThrottledTags = std::move(rep.clientThrottledTags.get());
+			}
+			if (rep.proxyThrottledTags.present()) {
+				perTagRateInfo->clear();
+				for (const auto& [priority, tagToRate] : rep.proxyThrottledTags.get()) {
+					for (const auto& [tag, rate] : tagToRate) {
+						(*perTagRateInfo)[priority][tag].setRate(rate);
+					}
+				}
 			}
 		}
 		when(wait(leaseTimeout)) {
@@ -387,7 +520,7 @@ ACTOR Future<Void> getRate(UID myID,
 // Respond with an error to the GetReadVersion request when the GRV limit is hit.
 void proxyGRVThresholdExceeded(const GetReadVersionRequest* req, GrvProxyStats* stats) {
 	++stats->txnRequestErrors;
-	req->reply.sendError(proxy_memory_limit_exceeded());
+	req->reply.sendError(grv_proxy_memory_limit_exceeded());
 	if (req->priority == TransactionPriority::IMMEDIATE) {
 		TraceEvent(SevWarnAlways, "ProxyGRVThresholdExceededSystem").suppressFor(60);
 	} else if (req->priority == TransactionPriority::DEFAULT) {
@@ -404,18 +537,20 @@ void dropRequestFromQueue(Deque<GetReadVersionRequest>* queue, GrvProxyStats* st
 }
 
 // Put a GetReadVersion request into the queue corresponding to its priority.
-ACTOR Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo> const> db,
-                                               SpannedDeque<GetReadVersionRequest>* systemQueue,
-                                               SpannedDeque<GetReadVersionRequest>* defaultQueue,
-                                               SpannedDeque<GetReadVersionRequest>* batchQueue,
-                                               FutureStream<GetReadVersionRequest> readVersionRequests,
-                                               PromiseStream<Void> GRVTimer,
-                                               double* lastGRVTime,
-                                               double* GRVBatchTime,
-                                               FutureStream<double> normalGRVLatency,
-                                               GrvProxyStats* stats,
-                                               GrvTransactionRateInfo* batchRateInfo,
-                                               TransactionTagMap<uint64_t>* transactionTagCounter) {
+ACTOR Future<Void> queueGetReadVersionRequests(
+    Reference<AsyncVar<ServerDBInfo> const> db,
+    SpannedDeque<GetReadVersionRequest>* systemQueue,
+    SpannedDeque<GetReadVersionRequest>* defaultQueue,
+    SpannedDeque<GetReadVersionRequest>* batchQueue,
+    FutureStream<GetReadVersionRequest> readVersionRequests,
+    PromiseStream<Void> GRVTimer,
+    double* lastGRVTime,
+    double* GRVBatchTime,
+    FutureStream<double> normalGRVLatency,
+    GrvProxyStats* stats,
+    GrvTransactionRateInfo* batchRateInfo,
+    TransactionTagMap<uint64_t>* transactionTagCounter,
+    PrioritizedTransactionTagMap<GrvTransactionRateInfo> const* perClientRateInfo) {
 	getCurrentLineage()->modify(&TransactionLineage::operation) =
 	    TransactionLineage::Operation::GetConsistentReadVersion;
 	loop choose {
@@ -425,7 +560,9 @@ ACTOR Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo>
 			// WARNING: this code is run at a high priority, so it needs to do as little work as possible
 			bool canBeQueued = true;
 			if (stats->txnRequestIn.getValue() - stats->txnRequestOut.getValue() >
-			    SERVER_KNOBS->START_TRANSACTION_MAX_QUEUE_SIZE) {
+			        SERVER_KNOBS->START_TRANSACTION_MAX_QUEUE_SIZE ||
+			    (g_network->isSimulated() && !g_simulator->speedUpSimulation &&
+			     deterministicRandom()->random01() < 0.01)) {
 				// When the limit is hit, try to drop requests from the lower priority queues.
 				if (req.priority == TransactionPriority::BATCH) {
 					canBeQueued = false;
@@ -579,6 +716,7 @@ ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(SpanContext parentSpan
 	}
 
 	GetRawCommittedVersionReply repFromMaster = wait(replyFromMasterFuture);
+	grvProxyData->version = std::max(grvProxyData->version, repFromMaster.version);
 	grvProxyData->minKnownCommittedVersion =
 	    std::max(grvProxyData->minKnownCommittedVersion, repFromMaster.minKnownCommittedVersion);
 	if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
@@ -619,7 +757,7 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
                                   GrvProxyData* grvProxyData,
                                   GrvProxyStats* stats,
                                   Version minKnownCommittedVersion,
-                                  PrioritizedTransactionTagMap<ClientTagThrottleLimits> throttledTags,
+                                  PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientThrottledTags,
                                   int64_t midShardSize = 0) {
 	GetReadVersionReply _reply = wait(replyFuture);
 	GetReadVersionReply reply = _reply;
@@ -655,20 +793,20 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		reply.proxyId = grvProxyData->dbgid;
 
 		if (!request.tags.empty()) {
-			auto& priorityThrottledTags = throttledTags[request.priority];
+			auto& priorityThrottledTags = clientThrottledTags[request.priority];
 			for (auto tag : request.tags) {
 				auto tagItr = priorityThrottledTags.find(tag.first);
 				if (tagItr != priorityThrottledTags.end()) {
 					if (tagItr->second.expiration > now()) {
 						if (tagItr->second.tpsRate == std::numeric_limits<double>::max()) {
-							TEST(true); // Auto TPS rate is unlimited
+							CODE_PROBE(true, "Auto TPS rate is unlimited");
 						} else {
-							TEST(true); // GRV proxy returning tag throttle
+							CODE_PROBE(true, "GRV proxy returning tag throttle");
 							reply.tagThrottleInfo[tag.first] = tagItr->second;
 						}
 					} else {
 						// This isn't required, but we might as well
-						TEST(true); // GRV proxy expiring tag throttle
+						CODE_PROBE(true, "GRV proxy expiring tag throttle");
 						priorityThrottledTags.erase(tagItr);
 					}
 				}
@@ -757,13 +895,14 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 	state int64_t batchTransactionCount = 0;
 	state GrvTransactionRateInfo normalRateInfo(10);
 	state GrvTransactionRateInfo batchRateInfo(0);
+	state PrioritizedTransactionTagMap<GrvTransactionRateInfo> perTagRateInfo;
 
 	state SpannedDeque<GetReadVersionRequest> systemQueue("GP:transactionStarterSystemQueue"_loc);
 	state SpannedDeque<GetReadVersionRequest> defaultQueue("GP:transactionStarterDefaultQueue"_loc);
 	state SpannedDeque<GetReadVersionRequest> batchQueue("GP:transactionStarterBatchQueue"_loc);
 
 	state TransactionTagMap<uint64_t> transactionTagCounter;
-	state PrioritizedTransactionTagMap<ClientTagThrottleLimits> throttledTags;
+	state PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientThrottledTags;
 
 	state PromiseStream<double> normalGRVLatency;
 	// state Span span;
@@ -782,8 +921,10 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 	                      healthMetricsReply,
 	                      detailedHealthMetricsReply,
 	                      &transactionTagCounter,
-	                      &throttledTags,
-	                      &grvProxyData->stats));
+	                      &clientThrottledTags,
+	                      &perTagRateInfo,
+	                      &grvProxyData->stats,
+	                      grvProxyData));
 	addActor.send(queueGetReadVersionRequests(db,
 	                                          &systemQueue,
 	                                          &defaultQueue,
@@ -795,7 +936,8 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 	                                          normalGRVLatency.getFuture(),
 	                                          &grvProxyData->stats,
 	                                          &batchRateInfo,
-	                                          &transactionTagCounter));
+	                                          &transactionTagCounter,
+	                                          &perTagRateInfo));
 
 	while (std::find(db->get().client.grvProxies.begin(), db->get().client.grvProxies.end(), proxy) ==
 	       db->get().client.grvProxies.end()) {
@@ -958,7 +1100,7 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 				                             grvProxyData,
 				                             &grvProxyData->stats,
 				                             grvProxyData->minKnownCommittedVersion,
-				                             throttledTags,
+				                             clientThrottledTags,
 				                             midShardSize));
 
 				// Use normal priority transaction's GRV latency to dynamically calculate transaction batching interval.
@@ -1012,6 +1154,7 @@ ACTOR Future<Void> grvProxyServerCore(GrvProxyInterface proxy,
 	addActor.send(transactionStarter(
 	    proxy, grvProxyData.db, addActor, &grvProxyData, &healthMetricsReply, &detailedHealthMetricsReply));
 	addActor.send(healthMetricsRequestServer(proxy, &healthMetricsReply, &detailedHealthMetricsReply));
+	addActor.send(globalConfigRequestServer(&grvProxyData, proxy));
 
 	if (SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION > 0) {
 		addActor.send(lastCommitUpdater(&grvProxyData, addActor));
diff --git a/fdbserver/IConfigConsumer.cpp b/fdbserver/IConfigConsumer.cpp
index 9d4c7d7e70..59e3500297 100644
--- a/fdbserver/IConfigConsumer.cpp
+++ b/fdbserver/IConfigConsumer.cpp
@@ -36,6 +36,8 @@ std::unique_ptr<IConfigConsumer> IConfigConsumer::createSimple(ServerCoordinator
 
 std::unique_ptr<IConfigConsumer> IConfigConsumer::createPaxos(ServerCoordinators const& coordinators,
                                                               double pollingInterval,
-                                                              Optional<double> compactionInterval) {
-	return std::make_unique<PaxosConfigConsumer>(coordinators, pollingInterval, compactionInterval);
+                                                              Optional<double> compactionInterval,
+                                                              bool readPreviousCoordinators) {
+	return std::make_unique<PaxosConfigConsumer>(
+	    coordinators, pollingInterval, compactionInterval, readPreviousCoordinators);
 }
diff --git a/fdbserver/KeyValueStoreCompressTestData.actor.cpp b/fdbserver/KeyValueStoreCompressTestData.actor.cpp
index 3121232b58..a5098baf4e 100644
--- a/fdbserver/KeyValueStoreCompressTestData.actor.cpp
+++ b/fdbserver/KeyValueStoreCompressTestData.actor.cpp
@@ -56,30 +56,30 @@ struct KeyValueStoreCompressTestData final : IKeyValueStore {
 	void clear(KeyRangeRef range, const Arena* arena = nullptr) override { store->clear(range, arena); }
 	Future<Void> commit(bool sequential = false) override { return store->commit(sequential); }
 
-	Future<Optional<Value>> readValue(KeyRef key, IKeyValueStore::ReadType, Optional<UID> debugID) override {
-		return doReadValue(store, key, debugID);
+	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options) override {
+		return doReadValue(store, key, options);
 	}
 
 	// Note that readValuePrefix doesn't do anything in this implementation of IKeyValueStore, so the "atomic bomb"
 	// problem is still present if you are using this storage interface, but this storage interface is not used by
 	// customers ever. However, if you want to try to test malicious atomic op workloads with compressed values for some
 	// reason, you will need to fix this.
-	Future<Optional<Value>> readValuePrefix(KeyRef key,
-	                                        int maxLength,
-	                                        IKeyValueStore::ReadType,
-	                                        Optional<UID> debugID) override {
-		return doReadValuePrefix(store, key, maxLength, debugID);
+	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override {
+		return doReadValuePrefix(store, key, maxLength, options);
 	}
 
 	// If rowLimit>=0, reads first rows sorted ascending, otherwise reads last rows sorted descending
 	// The total size of the returned value (less the last entry) will be less than byteLimit
-	Future<RangeResult> readRange(KeyRangeRef keys, int rowLimit, int byteLimit, IKeyValueStore::ReadType) override {
-		return doReadRange(store, keys, rowLimit, byteLimit);
+	Future<RangeResult> readRange(KeyRangeRef keys,
+	                              int rowLimit,
+	                              int byteLimit,
+	                              Optional<ReadOptions> options = Optional<ReadOptions>()) override {
+		return doReadRange(store, keys, rowLimit, byteLimit, options);
 	}
 
 private:
-	ACTOR static Future<Optional<Value>> doReadValue(IKeyValueStore* store, Key key, Optional<UID> debugID) {
-		Optional<Value> v = wait(store->readValue(key, IKeyValueStore::ReadType::NORMAL, debugID));
+	ACTOR static Future<Optional<Value>> doReadValue(IKeyValueStore* store, Key key, Optional<ReadOptions> options) {
+		Optional<Value> v = wait(store->readValue(key, options));
 		if (!v.present())
 			return v;
 		return unpack(v.get());
@@ -88,8 +88,8 @@ private:
 	ACTOR static Future<Optional<Value>> doReadValuePrefix(IKeyValueStore* store,
 	                                                       Key key,
 	                                                       int maxLength,
-	                                                       Optional<UID> debugID) {
-		Optional<Value> v = wait(doReadValue(store, key, debugID));
+	                                                       Optional<ReadOptions> options) {
+		Optional<Value> v = wait(doReadValue(store, key, options));
 		if (!v.present())
 			return v;
 		if (maxLength < v.get().size()) {
@@ -98,8 +98,12 @@ private:
 			return v;
 		}
 	}
-	ACTOR Future<RangeResult> doReadRange(IKeyValueStore* store, KeyRangeRef keys, int rowLimit, int byteLimit) {
-		RangeResult _vs = wait(store->readRange(keys, rowLimit, byteLimit));
+	ACTOR Future<RangeResult> doReadRange(IKeyValueStore* store,
+	                                      KeyRangeRef keys,
+	                                      int rowLimit,
+	                                      int byteLimit,
+	                                      Optional<ReadOptions> options) {
+		RangeResult _vs = wait(store->readRange(keys, rowLimit, byteLimit, options));
 		RangeResult vs = _vs; // Get rid of implicit const& from wait statement
 		Arena& a = vs.arena();
 		for (int i = 0; i < vs.size(); i++)
@@ -115,12 +119,12 @@ private:
 
 		// If the value starts with a 0-byte, then we don't compress it
 		if (c == 0)
-			return val.withPrefix(LiteralStringRef("\x00"));
+			return val.withPrefix("\x00"_sr);
 
 		for (int i = 1; i < val.size(); i++) {
 			if (val[i] != c) {
 				// The value is something other than a single repeated character, so not compressible :-)
-				return val.withPrefix(LiteralStringRef("\x00"));
+				return val.withPrefix("\x00"_sr);
 			}
 		}
 
diff --git a/fdbserver/KeyValueStoreMemory.actor.cpp b/fdbserver/KeyValueStoreMemory.actor.cpp
index 6404fe6fbf..3e6ecb8b3f 100644
--- a/fdbserver/KeyValueStoreMemory.actor.cpp
+++ b/fdbserver/KeyValueStoreMemory.actor.cpp
@@ -18,16 +18,18 @@
  * limitations under the License.
  */
 
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/SystemData.h"
 #include "fdbserver/DeltaTree.h"
-#include "fdbserver/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
 #include "fdbserver/IDiskQueue.h"
 #include "fdbserver/IKeyValueContainer.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "fdbserver/RadixTree.h"
 #include "flow/ActorCollection.h"
+#include "flow/EncryptUtils.h"
 #include "flow/Knobs.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -104,9 +106,9 @@ public:
 			TraceEvent("KVSMemSwitchingToLargeTransactionMode", id)
 			    .detail("TransactionSize", transactionSize)
 			    .detail("DataSize", committedDataSize);
-			TEST(true); // KeyValueStoreMemory switching to large transaction mode
-			TEST(committedDataSize >
-			     1e3); // KeyValueStoreMemory switching to large transaction mode with committed data
+			CODE_PROBE(true, "KeyValueStoreMemory switching to large transaction mode");
+			CODE_PROBE(committedDataSize > 1e3,
+			           "KeyValueStoreMemory switching to large transaction mode with committed data");
 		}
 
 		int64_t bytesWritten = commit_queue(queue, true);
@@ -198,11 +200,11 @@ public:
 		return c;
 	}
 
-	Future<Optional<Value>> readValue(KeyRef key, IKeyValueStore::ReadType, Optional<UID> debugID) override {
+	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options) override {
 		if (recovering.isError())
 			throw recovering.getError();
 		if (!recovering.isReady())
-			return waitAndReadValue(this, key);
+			return waitAndReadValue(this, key, options);
 
 		auto it = data.find(key);
 		if (it == data.end())
@@ -210,14 +212,11 @@ public:
 		return Optional<Value>(it.getValue());
 	}
 
-	Future<Optional<Value>> readValuePrefix(KeyRef key,
-	                                        int maxLength,
-	                                        IKeyValueStore::ReadType,
-	                                        Optional<UID> debugID) override {
+	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override {
 		if (recovering.isError())
 			throw recovering.getError();
 		if (!recovering.isReady())
-			return waitAndReadValuePrefix(this, key, maxLength);
+			return waitAndReadValuePrefix(this, key, maxLength, options);
 
 		auto it = data.find(key);
 		if (it == data.end())
@@ -232,11 +231,14 @@ public:
 
 	// If rowLimit>=0, reads first rows sorted ascending, otherwise reads last rows sorted descending
 	// The total size of the returned value (less the last entry) will be less than byteLimit
-	Future<RangeResult> readRange(KeyRangeRef keys, int rowLimit, int byteLimit, IKeyValueStore::ReadType) override {
+	Future<RangeResult> readRange(KeyRangeRef keys,
+	                              int rowLimit,
+	                              int byteLimit,
+	                              Optional<ReadOptions> options) override {
 		if (recovering.isError())
 			throw recovering.getError();
 		if (!recovering.isReady())
-			return waitAndReadRange(this, keys, rowLimit, byteLimit);
+			return waitAndReadRange(this, keys, rowLimit, byteLimit, options);
 
 		RangeResult result;
 		if (rowLimit == 0) {
@@ -489,7 +491,10 @@ private:
 			ASSERT(cipherKeys.cipherTextKey.isValid());
 			ASSERT(cipherKeys.cipherHeaderKey.isValid());
 			EncryptBlobCipherAes265Ctr cipher(
-			    cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
+			    cipherKeys.cipherTextKey,
+			    cipherKeys.cipherHeaderKey,
+			    getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
+			    BlobCipherMetrics::KV_MEMORY);
 			BlobCipherEncryptHeader cipherHeader;
 			Arena arena;
 			StringRef ciphertext =
@@ -497,7 +502,7 @@ private:
 			log->push(StringRef((const uint8_t*)&cipherHeader, BlobCipherEncryptHeader::headerSize));
 			log->push(ciphertext);
 		}
-		return log->push(LiteralStringRef("\x01")); // Changes here should be reflected in OP_DISK_OVERHEAD
+		return log->push("\x01"_sr); // Changes here should be reflected in OP_DISK_OVERHEAD
 	}
 
 	// In case the op data is not encrypted, simply read the operands and the zero fill flag.
@@ -527,8 +532,10 @@ private:
 			return data;
 		}
 		state BlobCipherEncryptHeader cipherHeader = *(BlobCipherEncryptHeader*)data.begin();
-		TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(self->db, cipherHeader));
-		DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, cipherHeader.iv);
+		TextAndHeaderCipherKeys cipherKeys =
+		    wait(getEncryptCipherKeys(self->db, cipherHeader, BlobCipherMetrics::KV_MEMORY));
+		DecryptBlobCipherAes256Ctr cipher(
+		    cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, cipherHeader.iv, BlobCipherMetrics::KV_MEMORY);
 		Arena arena;
 		StringRef plaintext = cipher
 		                          .decrypt(data.begin() + BlobCipherEncryptHeader::headerSize,
@@ -574,7 +581,7 @@ private:
 						Standalone<StringRef> data = wait(self->log->readNext(sizeof(OpHeader)));
 						if (data.size() != sizeof(OpHeader)) {
 							if (data.size()) {
-								TEST(true); // zero fill partial header in KeyValueStoreMemory
+								CODE_PROBE(true, "zero fill partial header in KeyValueStoreMemory");
 								memset(&h, 0, sizeof(OpHeader));
 								memcpy(&h, data.begin(), data.size());
 								zeroFillSize = sizeof(OpHeader) - data.size() + h.len1 + h.len2 + 1;
@@ -705,7 +712,7 @@ private:
 						ASSERT(false);
 					}
 
-					TEST(true); // Fixing a partial commit at the end of the KeyValueStoreMemory log
+					CODE_PROBE(true, "Fixing a partial commit at the end of the KeyValueStoreMemory log");
 					for (int i = 0; i < zeroFillSize; i++)
 						self->log->push(StringRef((const uint8_t*)"", 1));
 				}
@@ -825,7 +832,7 @@ private:
 
 					auto thisSnapshotEnd = self->log_op(OpSnapshotEnd, StringRef(), StringRef());
 					//TraceEvent("SnapshotEnd", self->id)
-					//	.detail("LastKey", lastKey.present() ? lastKey.get() : LiteralStringRef("<none>"))
+					//	.detail("LastKey", lastKey.present() ? lastKey.get() : "<none>"_sr)
 					//	.detail("CurrentSnapshotEndLoc", self->currentSnapshotEnd)
 					//	.detail("PreviousSnapshotEndLoc", self->previousSnapshotEnd)
 					//	.detail("ThisSnapshotEnd", thisSnapshotEnd)
@@ -926,20 +933,26 @@ private:
 		}
 	}
 
-	ACTOR static Future<Optional<Value>> waitAndReadValue(KeyValueStoreMemory* self, Key key) {
+	ACTOR static Future<Optional<Value>> waitAndReadValue(KeyValueStoreMemory* self,
+	                                                      Key key,
+	                                                      Optional<ReadOptions> options) {
 		wait(self->recovering);
-		return static_cast<IKeyValueStore*>(self)->readValue(key).get();
+		return static_cast<IKeyValueStore*>(self)->readValue(key, options).get();
 	}
-	ACTOR static Future<Optional<Value>> waitAndReadValuePrefix(KeyValueStoreMemory* self, Key key, int maxLength) {
+	ACTOR static Future<Optional<Value>> waitAndReadValuePrefix(KeyValueStoreMemory* self,
+	                                                            Key key,
+	                                                            int maxLength,
+	                                                            Optional<ReadOptions> options) {
 		wait(self->recovering);
-		return static_cast<IKeyValueStore*>(self)->readValuePrefix(key, maxLength).get();
+		return static_cast<IKeyValueStore*>(self)->readValuePrefix(key, maxLength, options).get();
 	}
 	ACTOR static Future<RangeResult> waitAndReadRange(KeyValueStoreMemory* self,
 	                                                  KeyRange keys,
 	                                                  int rowLimit,
-	                                                  int byteLimit) {
+	                                                  int byteLimit,
+	                                                  Optional<ReadOptions> options) {
 		wait(self->recovering);
-		return static_cast<IKeyValueStore*>(self)->readRange(keys, rowLimit, byteLimit).get();
+		return static_cast<IKeyValueStore*>(self)->readRange(keys, rowLimit, byteLimit, options).get();
 	}
 	ACTOR static Future<Void> waitAndCommit(KeyValueStoreMemory* self, bool sequential) {
 		wait(self->recovering);
@@ -955,7 +968,8 @@ private:
 	}
 
 	ACTOR static Future<Void> updateCipherKeys(KeyValueStoreMemory* self) {
-		TextAndHeaderCipherKeys cipherKeys = wait(getLatestSystemEncryptCipherKeys(self->db));
+		TextAndHeaderCipherKeys cipherKeys =
+		    wait(getLatestSystemEncryptCipherKeys(self->db, BlobCipherMetrics::KV_MEMORY));
 		self->cipherKeys = cipherKeys;
 		return Void();
 	}
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 5bf14efda1..9340028acb 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -20,29 +20,27 @@
 
 #ifdef SSD_ROCKSDB_EXPERIMENTAL
 
+#include <rocksdb/c.h>
 #include <rocksdb/cache.h>
 #include <rocksdb/db.h>
+#include <rocksdb/env.h>
 #include <rocksdb/filter_policy.h>
 #include <rocksdb/listener.h>
-#include <rocksdb/options.h>
 #include <rocksdb/metadata.h>
+#include <rocksdb/options.h>
+#include <rocksdb/perf_context.h>
+#include <rocksdb/rate_limiter.h>
+#include <rocksdb/slice.h>
 #include <rocksdb/slice_transform.h>
 #include <rocksdb/sst_file_reader.h>
 #include <rocksdb/sst_file_writer.h>
-#include <rocksdb/slice.h>
-#include <rocksdb/env.h>
-#include <rocksdb/options.h>
 #include <rocksdb/statistics.h>
 #include <rocksdb/table.h>
-#include <rocksdb/version.h>
 #include <rocksdb/types.h>
 #include <rocksdb/utilities/checkpoint.h>
 #include <rocksdb/utilities/table_properties_collectors.h>
 #include <rocksdb/version.h>
 
-#include <rocksdb/rate_limiter.h>
-#include <rocksdb/perf_context.h>
-#include <rocksdb/c.h>
 #if defined __has_include
 #if __has_include(<liburing.h>)
 #include <liburing.h>
@@ -79,6 +77,141 @@ static_assert((ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR == 27) ? ROCKSDB_PATCH >= 3 :
 namespace {
 using rocksdb::BackgroundErrorReason;
 
+class SharedRocksDBState {
+public:
+	SharedRocksDBState(UID id);
+
+	std::shared_ptr<LatencySample> commitLatency;
+	std::shared_ptr<LatencySample> commitQueueLatency;
+	std::shared_ptr<LatencySample> dbWriteLatency;
+
+	void setClosing() { this->closing = true; }
+	bool isClosing() const { return this->closing; }
+
+	rocksdb::DBOptions getDbOptions() const { return this->dbOptions; }
+	rocksdb::ColumnFamilyOptions getCfOptions() const { return this->cfOptions; }
+	rocksdb::Options getOptions() const { return rocksdb::Options(this->dbOptions, this->cfOptions); }
+	rocksdb::ReadOptions& getReadOptions() { return this->readOptions; }
+
+private:
+	const UID id;
+	rocksdb::ColumnFamilyOptions initialCfOptions();
+	rocksdb::DBOptions initialDbOptions();
+	rocksdb::ReadOptions initialReadOptions();
+
+	bool closing;
+	rocksdb::DBOptions dbOptions;
+	rocksdb::ColumnFamilyOptions cfOptions;
+	rocksdb::ReadOptions readOptions;
+};
+
+SharedRocksDBState::SharedRocksDBState(UID id)
+  : id(id), closing(false), dbOptions(initialDbOptions()), cfOptions(initialCfOptions()),
+    readOptions(initialReadOptions()),
+    commitLatency(std::make_shared<LatencySample>("RocksDBCommitLatency",
+                                                  id,
+                                                  SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+                                                  SERVER_KNOBS->LATENCY_SAMPLE_SIZE)),
+    commitQueueLatency(std::make_shared<LatencySample>("RocksDBCommitQueueLatency",
+                                                       id,
+                                                       SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+                                                       SERVER_KNOBS->LATENCY_SAMPLE_SIZE)),
+    dbWriteLatency(std::make_shared<LatencySample>("RocksDBWriteLatency",
+                                                   id,
+                                                   SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+                                                   SERVER_KNOBS->LATENCY_SAMPLE_SIZE)) {}
+
+rocksdb::ColumnFamilyOptions SharedRocksDBState::initialCfOptions() {
+	rocksdb::ColumnFamilyOptions options;
+	options.level_compaction_dynamic_level_bytes = SERVER_KNOBS->ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES;
+	if (SERVER_KNOBS->ROCKSDB_LEVEL_STYLE_COMPACTION) {
+		options.OptimizeLevelStyleCompaction(SERVER_KNOBS->ROCKSDB_MEMTABLE_BYTES);
+	} else {
+		options.OptimizeUniversalStyleCompaction(SERVER_KNOBS->ROCKSDB_MEMTABLE_BYTES);
+	}
+
+	if (SERVER_KNOBS->ROCKSDB_DISABLE_AUTO_COMPACTIONS) {
+		options.disable_auto_compactions = SERVER_KNOBS->ROCKSDB_DISABLE_AUTO_COMPACTIONS;
+	}
+
+	if (SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS > 0) {
+		options.periodic_compaction_seconds = SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS;
+	}
+	if (SERVER_KNOBS->ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT > 0) {
+		options.soft_pending_compaction_bytes_limit = SERVER_KNOBS->ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT;
+	}
+	if (SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT > 0) {
+		options.hard_pending_compaction_bytes_limit = SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
+	}
+
+	// Compact sstables when there's too much deleted stuff.
+	options.table_properties_collector_factories = { rocksdb::NewCompactOnDeletionCollectorFactory(128, 1) };
+
+	rocksdb::BlockBasedTableOptions bbOpts;
+	if (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0) {
+		// Prefix blooms are used during Seek.
+		options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(SERVER_KNOBS->ROCKSDB_PREFIX_LEN));
+
+		// Also turn on bloom filters in the memtable.
+		// TODO: Make a knob for this as well.
+		options.memtable_prefix_bloom_size_ratio = 0.1;
+
+		// 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned
+		// filters use a generally faster and more accurate Bloom filter
+		// implementation, with a different schema.
+		// https://github.com/facebook/rocksdb/blob/b77569f18bfc77fb1d8a0b3218f6ecf571bc4988/include/rocksdb/table.h#L391
+		bbOpts.format_version = 5;
+
+		// Create and apply a bloom filter using the 10 bits
+		// which should yield a ~1% false positive rate:
+		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#full-filters-new-format
+		bbOpts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10));
+
+		// The whole key blooms are only used for point lookups.
+		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#prefix-vs-whole-key
+		bbOpts.whole_key_filtering = false;
+	}
+
+	if (SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE > 0) {
+		bbOpts.block_cache = rocksdb::NewLRUCache(SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE);
+	}
+
+	if (SERVER_KNOBS->ROCKSDB_BLOCK_SIZE > 0) {
+		bbOpts.block_size = SERVER_KNOBS->ROCKSDB_BLOCK_SIZE;
+	}
+
+	options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbOpts));
+
+	return options;
+}
+
+rocksdb::DBOptions SharedRocksDBState::initialDbOptions() {
+	rocksdb::DBOptions options;
+	options.avoid_unnecessary_blocking_io = true;
+	options.create_if_missing = true;
+	if (SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM > 0) {
+		options.IncreaseParallelism(SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM);
+	}
+	if (SERVER_KNOBS->ROCKSDB_MAX_SUBCOMPACTIONS > 0) {
+		options.max_subcompactions = SERVER_KNOBS->ROCKSDB_MAX_SUBCOMPACTIONS;
+	}
+	if (SERVER_KNOBS->ROCKSDB_COMPACTION_READAHEAD_SIZE > 0) {
+		options.compaction_readahead_size = SERVER_KNOBS->ROCKSDB_COMPACTION_READAHEAD_SIZE;
+	}
+
+	options.statistics = rocksdb::CreateDBStatistics();
+	options.statistics->set_stats_level(rocksdb::kExceptHistogramOrTimers);
+
+	options.db_log_dir = SERVER_KNOBS->LOG_DIRECTORY;
+	return options;
+}
+
+rocksdb::ReadOptions SharedRocksDBState::initialReadOptions() {
+	rocksdb::ReadOptions options;
+	options.background_purge_on_iterator_cleanup = true;
+	return options;
+}
+
 // Returns string representation of RocksDB background error reason.
 // Error reason code:
 // https://github.com/facebook/rocksdb/blob/12d798ac06bcce36be703b057d5f5f4dab3b270c/include/rocksdb/listener.h#L125
@@ -149,28 +282,27 @@ private:
 };
 using DB = rocksdb::DB*;
 using CF = rocksdb::ColumnFamilyHandle*;
-std::shared_ptr<rocksdb::Cache> rocksdb_block_cache = nullptr;
 
 #define PERSIST_PREFIX "\xff\xff"
-const KeyRef persistVersion = LiteralStringRef(PERSIST_PREFIX "Version");
-const StringRef ROCKSDBSTORAGE_HISTOGRAM_GROUP = LiteralStringRef("RocksDBStorage");
-const StringRef ROCKSDB_COMMIT_LATENCY_HISTOGRAM = LiteralStringRef("RocksDBCommitLatency");
-const StringRef ROCKSDB_COMMIT_ACTION_HISTOGRAM = LiteralStringRef("RocksDBCommitAction");
-const StringRef ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM = LiteralStringRef("RocksDBCommitQueueWait");
-const StringRef ROCKSDB_WRITE_HISTOGRAM = LiteralStringRef("RocksDBWrite");
-const StringRef ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM = LiteralStringRef("RocksDBDeleteCompactRange");
-const StringRef ROCKSDB_READRANGE_LATENCY_HISTOGRAM = LiteralStringRef("RocksDBReadRangeLatency");
-const StringRef ROCKSDB_READVALUE_LATENCY_HISTOGRAM = LiteralStringRef("RocksDBReadValueLatency");
-const StringRef ROCKSDB_READPREFIX_LATENCY_HISTOGRAM = LiteralStringRef("RocksDBReadPrefixLatency");
-const StringRef ROCKSDB_READRANGE_ACTION_HISTOGRAM = LiteralStringRef("RocksDBReadRangeAction");
-const StringRef ROCKSDB_READVALUE_ACTION_HISTOGRAM = LiteralStringRef("RocksDBReadValueAction");
-const StringRef ROCKSDB_READPREFIX_ACTION_HISTOGRAM = LiteralStringRef("RocksDBReadPrefixAction");
-const StringRef ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM = LiteralStringRef("RocksDBReadRangeQueueWait");
-const StringRef ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM = LiteralStringRef("RocksDBReadValueQueueWait");
-const StringRef ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM = LiteralStringRef("RocksDBReadPrefixQueueWait");
-const StringRef ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM = LiteralStringRef("RocksDBReadRangeNewIterator");
-const StringRef ROCKSDB_READVALUE_GET_HISTOGRAM = LiteralStringRef("RocksDBReadValueGet");
-const StringRef ROCKSDB_READPREFIX_GET_HISTOGRAM = LiteralStringRef("RocksDBReadPrefixGet");
+const KeyRef persistVersion = PERSIST_PREFIX "Version"_sr;
+const StringRef ROCKSDBSTORAGE_HISTOGRAM_GROUP = "RocksDBStorage"_sr;
+const StringRef ROCKSDB_COMMIT_LATENCY_HISTOGRAM = "RocksDBCommitLatency"_sr;
+const StringRef ROCKSDB_COMMIT_ACTION_HISTOGRAM = "RocksDBCommitAction"_sr;
+const StringRef ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM = "RocksDBCommitQueueWait"_sr;
+const StringRef ROCKSDB_WRITE_HISTOGRAM = "RocksDBWrite"_sr;
+const StringRef ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM = "RocksDBDeleteCompactRange"_sr;
+const StringRef ROCKSDB_READRANGE_LATENCY_HISTOGRAM = "RocksDBReadRangeLatency"_sr;
+const StringRef ROCKSDB_READVALUE_LATENCY_HISTOGRAM = "RocksDBReadValueLatency"_sr;
+const StringRef ROCKSDB_READPREFIX_LATENCY_HISTOGRAM = "RocksDBReadPrefixLatency"_sr;
+const StringRef ROCKSDB_READRANGE_ACTION_HISTOGRAM = "RocksDBReadRangeAction"_sr;
+const StringRef ROCKSDB_READVALUE_ACTION_HISTOGRAM = "RocksDBReadValueAction"_sr;
+const StringRef ROCKSDB_READPREFIX_ACTION_HISTOGRAM = "RocksDBReadPrefixAction"_sr;
+const StringRef ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM = "RocksDBReadRangeQueueWait"_sr;
+const StringRef ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM = "RocksDBReadValueQueueWait"_sr;
+const StringRef ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM = "RocksDBReadPrefixQueueWait"_sr;
+const StringRef ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM = "RocksDBReadRangeNewIterator"_sr;
+const StringRef ROCKSDB_READVALUE_GET_HISTOGRAM = "RocksDBReadValueGet"_sr;
+const StringRef ROCKSDB_READPREFIX_GET_HISTOGRAM = "RocksDBReadPrefixGet"_sr;
 
 rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpoint) {
 	rocksdb::ExportImportFilesMetaData metaData;
@@ -248,91 +380,6 @@ StringRef toStringRef(rocksdb::Slice s) {
 	return StringRef(reinterpret_cast<const uint8_t*>(s.data()), s.size());
 }
 
-rocksdb::ColumnFamilyOptions getCFOptions() {
-	rocksdb::ColumnFamilyOptions options;
-	options.level_compaction_dynamic_level_bytes = true;
-	options.OptimizeLevelStyleCompaction(SERVER_KNOBS->ROCKSDB_MEMTABLE_BYTES);
-	if (SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS > 0) {
-		options.periodic_compaction_seconds = SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS;
-	}
-	if (SERVER_KNOBS->ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT > 0) {
-		options.soft_pending_compaction_bytes_limit = SERVER_KNOBS->ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT;
-	}
-	if (SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT > 0) {
-		options.hard_pending_compaction_bytes_limit = SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
-	}
-	// Compact sstables when there's too much deleted stuff.
-	options.table_properties_collector_factories = { rocksdb::NewCompactOnDeletionCollectorFactory(128, 1) };
-
-	rocksdb::BlockBasedTableOptions bbOpts;
-	// TODO: Add a knob for the block cache size. (Default is 8 MB)
-	if (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0) {
-		// Prefix blooms are used during Seek.
-		options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(SERVER_KNOBS->ROCKSDB_PREFIX_LEN));
-
-		// Also turn on bloom filters in the memtable.
-		// TODO: Make a knob for this as well.
-		options.memtable_prefix_bloom_size_ratio = 0.1;
-
-		// 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned
-		// filters use a generally faster and more accurate Bloom filter
-		// implementation, with a different schema.
-		// https://github.com/facebook/rocksdb/blob/b77569f18bfc77fb1d8a0b3218f6ecf571bc4988/include/rocksdb/table.h#L391
-		bbOpts.format_version = 5;
-
-		// Create and apply a bloom filter using the 10 bits
-		// which should yield a ~1% false positive rate:
-		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#full-filters-new-format
-		bbOpts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10));
-
-		// The whole key blooms are only used for point lookups.
-		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#prefix-vs-whole-key
-		bbOpts.whole_key_filtering = false;
-	}
-
-	if (SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE > 0) {
-		if (rocksdb_block_cache == nullptr) {
-			rocksdb_block_cache = rocksdb::NewLRUCache(SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE);
-		}
-		bbOpts.block_cache = rocksdb_block_cache;
-	}
-	if (SERVER_KNOBS->ROCKSDB_BLOCK_SIZE > 0) {
-		bbOpts.block_size = SERVER_KNOBS->ROCKSDB_BLOCK_SIZE;
-	}
-
-	options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbOpts));
-
-	return options;
-}
-
-rocksdb::Options getOptions() {
-	rocksdb::Options options({}, getCFOptions());
-	options.avoid_unnecessary_blocking_io = true;
-	options.create_if_missing = true;
-	if (SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM > 0) {
-		options.IncreaseParallelism(SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM);
-	}
-	if (SERVER_KNOBS->ROCKSDB_MAX_SUBCOMPACTIONS > 0) {
-		options.max_subcompactions = SERVER_KNOBS->ROCKSDB_MAX_SUBCOMPACTIONS;
-	}
-	if (SERVER_KNOBS->ROCKSDB_COMPACTION_READAHEAD_SIZE > 0) {
-		options.compaction_readahead_size = SERVER_KNOBS->ROCKSDB_COMPACTION_READAHEAD_SIZE;
-	}
-
-	options.statistics = rocksdb::CreateDBStatistics();
-	options.statistics->set_stats_level(rocksdb::kExceptHistogramOrTimers);
-
-	options.db_log_dir = SERVER_KNOBS->LOG_DIRECTORY;
-	return options;
-}
-
-// Set some useful defaults desired for all reads.
-rocksdb::ReadOptions getReadOptions() {
-	rocksdb::ReadOptions options;
-	options.background_purge_on_iterator_cleanup = true;
-	return options;
-}
-
 struct Counters {
 	CounterCollection cc;
 	Counter immediateThrottle;
@@ -366,8 +413,8 @@ gets deleted as the ref count becomes 0.
 */
 class ReadIteratorPool {
 public:
-	ReadIteratorPool(UID id, DB& db, CF& cf)
-	  : db(db), cf(cf), index(0), iteratorsReuseCount(0), readRangeOptions(getReadOptions()) {
+	ReadIteratorPool(UID id, DB& db, CF& cf, const rocksdb::ReadOptions readOptions)
+	  : db(db), cf(cf), index(0), iteratorsReuseCount(0), readRangeOptions(readOptions) {
 		readRangeOptions.background_purge_on_iterator_cleanup = true;
 		readRangeOptions.auto_prefix_mode = (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0);
 		TraceEvent("ReadIteratorPool", id)
@@ -737,6 +784,7 @@ ACTOR Future<Void> flowLockLogger(UID id, const FlowLock* readLock, const FlowLo
 }
 
 ACTOR Future<Void> rocksDBMetricLogger(UID id,
+                                       std::shared_ptr<SharedRocksDBState> sharedState,
                                        std::shared_ptr<rocksdb::Statistics> statistics,
                                        std::shared_ptr<PerfContextMetrics> perfContextMetrics,
                                        rocksdb::DB* db,
@@ -771,15 +819,31 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 		{ "CountWalFileSyncs", rocksdb::WAL_FILE_SYNCED, 0 },
 		{ "CountWalFileBytes", rocksdb::WAL_FILE_BYTES, 0 },
 		{ "CompactReadBytes", rocksdb::COMPACT_READ_BYTES, 0 },
+		{ "CompactReadBytesMarked", rocksdb::COMPACT_READ_BYTES_MARKED, 0 },
+		{ "CompactReadBytesPeriodic", rocksdb::COMPACT_READ_BYTES_PERIODIC, 0 },
+		{ "CompactReadBytesTtl", rocksdb::COMPACT_READ_BYTES_TTL, 0 },
 		{ "CompactWriteBytes", rocksdb::COMPACT_WRITE_BYTES, 0 },
+		{ "CompactWriteBytesMarked", rocksdb::COMPACT_WRITE_BYTES_MARKED, 0 },
+		{ "CompactWriteBytesPeriodic", rocksdb::COMPACT_WRITE_BYTES_PERIODIC, 0 },
+		{ "CompactWriteBytesTtl", rocksdb::COMPACT_WRITE_BYTES_TTL, 0 },
 		{ "FlushWriteBytes", rocksdb::FLUSH_WRITE_BYTES, 0 },
 		{ "CountBlocksCompressed", rocksdb::NUMBER_BLOCK_COMPRESSED, 0 },
 		{ "CountBlocksDecompressed", rocksdb::NUMBER_BLOCK_DECOMPRESSED, 0 },
 		{ "RowCacheHit", rocksdb::ROW_CACHE_HIT, 0 },
 		{ "RowCacheMiss", rocksdb::ROW_CACHE_MISS, 0 },
 		{ "CountIterSkippedKeys", rocksdb::NUMBER_ITER_SKIP, 0 },
-
 	};
+
+	state std::vector<std::pair<const char*, uint32_t>> histogramStats = {
+		{ "CompactionTime", rocksdb::COMPACTION_TIME },
+		{ "CompactionCPUTime", rocksdb::COMPACTION_CPU_TIME },
+		{ "CompressionTimeNanos", rocksdb::COMPRESSION_TIMES_NANOS },
+		{ "DecompressionTimeNanos", rocksdb::DECOMPRESSION_TIMES_NANOS },
+		{ "HardRateLimitDelayCount", rocksdb::HARD_RATE_LIMIT_DELAY_COUNT },
+		{ "SoftRateLimitDelayCount", rocksdb::SOFT_RATE_LIMIT_DELAY_COUNT },
+		{ "WriteStall", rocksdb::WRITE_STALL },
+	};
+
 	state std::vector<std::pair<const char*, std::string>> intPropertyStats = {
 		{ "NumImmutableMemtables", rocksdb::DB::Properties::kNumImmutableMemTable },
 		{ "NumImmutableMemtablesFlushed", rocksdb::DB::Properties::kNumImmutableMemTableFlushed },
@@ -823,6 +887,9 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 
 	loop {
 		wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
+		if (sharedState->isClosing()) {
+			break;
+		}
 		TraceEvent e("RocksDBMetrics", id);
 		uint64_t stat;
 		for (auto& [name, ticker, cum] : tickerStats) {
@@ -831,6 +898,13 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 			cum = stat;
 		}
 
+		for (auto& [name, histogram] : histogramStats) {
+			rocksdb::HistogramData histogram_data;
+			statistics->histogramData(histogram, &histogram_data);
+			e.detail(format("%s%d", name, "P95"), histogram_data.percentile95);
+			e.detail(format("%s%d", name, "P99"), histogram_data.percentile99);
+		}
+
 		for (const auto& [name, property] : intPropertyStats) {
 			stat = 0;
 			// GetAggregatedIntProperty gets the aggregated int property from all column families.
@@ -873,6 +947,8 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 			perfContextMetrics->log(true);
 		}
 	}
+
+	return Void();
 }
 
 void logRocksDBError(UID id,
@@ -919,26 +995,16 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			ThreadReturnPromise<Void> done;
 		};
 
-		DB& db;
-		CF& cf;
-		UID id;
-		std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
-		std::shared_ptr<ReadIteratorPool> readIterPool;
-		std::shared_ptr<PerfContextMetrics> perfContextMetrics;
-		int threadIndex;
-		ThreadReturnPromiseStream<std::pair<std::string, double>>* metricPromiseStream;
-		// ThreadReturnPromiseStream pair.first stores the histogram name and
-		// pair.second stores the corresponding measured latency (seconds)
-
 		explicit Writer(DB& db,
 		                CF& cf,
 		                UID id,
+		                std::shared_ptr<SharedRocksDBState> sharedState,
 		                std::shared_ptr<ReadIteratorPool> readIterPool,
 		                std::shared_ptr<PerfContextMetrics> perfContextMetrics,
 		                int threadIndex,
 		                ThreadReturnPromiseStream<std::pair<std::string, double>>* metricPromiseStream)
-		  : db(db), cf(cf), id(id), readIterPool(readIterPool), perfContextMetrics(perfContextMetrics),
-		    threadIndex(threadIndex), metricPromiseStream(metricPromiseStream),
+		  : db(db), cf(cf), id(id), sharedState(sharedState), readIterPool(readIterPool),
+		    perfContextMetrics(perfContextMetrics), threadIndex(threadIndex), metricPromiseStream(metricPromiseStream),
 		    rateLimiter(SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC > 0
 		                    ? rocksdb::NewGenericRateLimiter(
 		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, // rate_bytes_per_sec
@@ -954,12 +1020,6 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 		}
 
-		~Writer() override {
-			if (db) {
-				delete db;
-			}
-		}
-
 		void init() override {}
 
 		struct OpenAction : TypedAction<Writer, OpenAction> {
@@ -985,13 +1045,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			ASSERT(cf == nullptr);
 
 			std::vector<std::string> columnFamilies;
-			rocksdb::Options options = getOptions();
+			rocksdb::DBOptions options = sharedState->getDbOptions();
 			rocksdb::Status status = rocksdb::DB::ListColumnFamilies(options, a.path, &columnFamilies);
 			if (std::find(columnFamilies.begin(), columnFamilies.end(), "default") == columnFamilies.end()) {
 				columnFamilies.push_back("default");
 			}
 
-			rocksdb::ColumnFamilyOptions cfOptions = getCFOptions();
+			rocksdb::ColumnFamilyOptions cfOptions = sharedState->getCfOptions();
 			std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
 			for (const std::string& name : columnFamilies) {
 				descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, cfOptions });
@@ -1004,6 +1064,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 			std::vector<rocksdb::ColumnFamilyHandle*> handles;
 			status = rocksdb::DB::Open(options, a.path, descriptors, &handles, &db);
+			cfHandles.insert(handles.begin(), handles.end());
 
 			if (!status.ok()) {
 				logRocksDBError(id, status, "Open");
@@ -1020,6 +1081,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 			if (cf == nullptr) {
 				status = db->CreateColumnFamily(cfOptions, SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, &cf);
+				cfHandles.insert(cf);
 				if (!status.ok()) {
 					logRocksDBError(id, status, "Open");
 					a.done.sendError(statusToError(status));
@@ -1037,13 +1099,20 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				// The current thread and main thread are same when the code runs in simulation.
 				// blockUntilReady() is getting the thread into deadlock state, so directly calling
 				// the metricsLogger.
-				a.metrics = rocksDBMetricLogger(
-				                id, options.statistics, perfContextMetrics, db, readIterPool, &a.counters, cf) &&
-				            flowLockLogger(id, a.readLock, a.fetchLock) && refreshReadIteratorPool(readIterPool);
+				a.metrics =
+				    rocksDBMetricLogger(
+				        id, sharedState, options.statistics, perfContextMetrics, db, readIterPool, &a.counters, cf) &&
+				    flowLockLogger(id, a.readLock, a.fetchLock) && refreshReadIteratorPool(readIterPool);
 			} else {
 				onMainThread([&] {
-					a.metrics = rocksDBMetricLogger(
-					                id, options.statistics, perfContextMetrics, db, readIterPool, &a.counters, cf) &&
+					a.metrics = rocksDBMetricLogger(id,
+					                                sharedState,
+					                                options.statistics,
+					                                perfContextMetrics,
+					                                db,
+					                                readIterPool,
+					                                &a.counters,
+					                                cf) &&
 					            flowLockLogger(id, a.readLock, a.fetchLock) && refreshReadIteratorPool(readIterPool);
 					return Future<bool>(true);
 				}).blockUntilReady();
@@ -1109,6 +1178,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				perfContextMetrics->reset();
 			}
 			double commitBeginTime;
+			sharedState->commitQueueLatency->addMeasurement(commitBeginTime - a.startTime);
 			if (a.getHistograms) {
 				commitBeginTime = timer_monotonic();
 				metricPromiseStream->send(
@@ -1135,9 +1205,11 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 			s = db->Write(options, a.batchToCommit.get());
 			readIterPool->update();
+			double currTime = timer_monotonic();
+			sharedState->dbWriteLatency->addMeasurement(currTime - writeBeginTime);
 			if (a.getHistograms) {
 				metricPromiseStream->send(
-				    std::make_pair(ROCKSDB_WRITE_HISTOGRAM.toString(), timer_monotonic() - writeBeginTime));
+				    std::make_pair(ROCKSDB_WRITE_HISTOGRAM.toString(), currTime - writeBeginTime));
 			}
 
 			if (!s.ok()) {
@@ -1146,19 +1218,23 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			} else {
 				a.done.send(Void());
 
-				double compactRangeBeginTime = a.getHistograms ? timer_monotonic() : 0;
-				for (const auto& keyRange : deletes) {
-					auto begin = toSlice(keyRange.begin);
-					auto end = toSlice(keyRange.end);
-					ASSERT(db->SuggestCompactRange(cf, &begin, &end).ok());
-				}
-				if (a.getHistograms) {
-					metricPromiseStream->send(std::make_pair(ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM.toString(),
-					                                         timer_monotonic() - compactRangeBeginTime));
+				if (SERVER_KNOBS->ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE) {
+					double compactRangeBeginTime = a.getHistograms ? timer_monotonic() : 0;
+					for (const auto& keyRange : deletes) {
+						auto begin = toSlice(keyRange.begin);
+						auto end = toSlice(keyRange.end);
+
+						ASSERT(db->SuggestCompactRange(cf, &begin, &end).ok());
+					}
+					if (a.getHistograms) {
+						metricPromiseStream->send(std::make_pair(ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM.toString(),
+						                                         timer_monotonic() - compactRangeBeginTime));
+					}
 				}
 			}
+			currTime = timer_monotonic();
+			sharedState->commitLatency->addMeasurement(currTime - a.startTime);
 			if (a.getHistograms) {
-				double currTime = timer_monotonic();
 				metricPromiseStream->send(
 				    std::make_pair(ROCKSDB_COMMIT_ACTION_HISTOGRAM.toString(), currTime - commitBeginTime));
 				metricPromiseStream->send(
@@ -1182,6 +1258,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				a.done.send(Void());
 				return;
 			}
+			for (rocksdb::ColumnFamilyHandle* handle : cfHandles) {
+				if (handle != nullptr) {
+					db->DestroyColumnFamilyHandle(handle);
+				}
+			}
+			cfHandles.clear();
 			auto s = db->Close();
 			if (!s.ok()) {
 				logRocksDBError(id, s, "Close");
@@ -1191,9 +1273,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				columnFamilies.insert(SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY);
 				std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
 				for (const std::string name : columnFamilies) {
-					descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, getCFOptions() });
+					descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, sharedState->getCfOptions() });
 				}
-				s = rocksdb::DestroyDB(a.path, getOptions(), descriptors);
+				s = rocksdb::DestroyDB(a.path, sharedState->getOptions(), descriptors);
 				if (!s.ok()) {
 					logRocksDBError(id, s, "Destroy");
 				} else {
@@ -1207,12 +1289,27 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		void action(CheckpointAction& a);
 
 		void action(RestoreAction& a);
+
+		std::shared_ptr<SharedRocksDBState> sharedState;
+		DB& db;
+		CF& cf;
+		std::unordered_set<rocksdb::ColumnFamilyHandle*> cfHandles;
+		UID id;
+		std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
+		std::shared_ptr<ReadIteratorPool> readIterPool;
+		std::shared_ptr<PerfContextMetrics> perfContextMetrics;
+		int threadIndex;
+
+		// ThreadReturnPromiseStream pair.first stores the histogram name and
+		// pair.second stores the corresponding measured latency (seconds)
+		ThreadReturnPromiseStream<std::pair<std::string, double>>* metricPromiseStream;
 	};
 
 	struct Reader : IThreadPoolReceiver {
 		UID id;
 		DB& db;
 		CF& cf;
+		std::shared_ptr<SharedRocksDBState> sharedState;
 		double readValueTimeout;
 		double readValuePrefixTimeout;
 		double readRangeTimeout;
@@ -1226,12 +1323,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		explicit Reader(UID id,
 		                DB& db,
 		                CF& cf,
+		                std::shared_ptr<SharedRocksDBState> sharedState,
 		                std::shared_ptr<ReadIteratorPool> readIterPool,
 		                std::shared_ptr<PerfContextMetrics> perfContextMetrics,
 		                int threadIndex,
 		                ThreadReturnPromiseStream<std::pair<std::string, double>>* metricPromiseStream)
-		  : id(id), db(db), cf(cf), readIterPool(readIterPool), perfContextMetrics(perfContextMetrics),
-		    metricPromiseStream(metricPromiseStream), threadIndex(threadIndex) {
+		  : id(id), db(db), cf(cf), sharedState(sharedState), readIterPool(readIterPool),
+		    perfContextMetrics(perfContextMetrics), metricPromiseStream(metricPromiseStream), threadIndex(threadIndex) {
 			if (g_network->isSimulated()) {
 				// In simulation, increasing the read operation timeouts to 5 minutes, as some of the tests have
 				// very high load and single read thread cannot process all the load within the timeouts.
@@ -1293,7 +1391,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 
 			rocksdb::PinnableSlice value;
-			auto options = getReadOptions();
+			auto& options = sharedState->getReadOptions();
 			uint64_t deadlineMircos =
 			    db->GetEnv()->NowMicros() + (readValueTimeout - (readBeginTime - a.startTime)) * 1000000;
 			std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
@@ -1380,7 +1478,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 
 			rocksdb::PinnableSlice value;
-			auto options = getReadOptions();
+			auto& options = sharedState->getReadOptions();
 			uint64_t deadlineMircos =
 			    db->GetEnv()->NowMicros() + (readValuePrefixTimeout - (readBeginTime - a.startTime)) * 1000000;
 			std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
@@ -1547,35 +1645,10 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		}
 	};
 
-	DB db = nullptr;
-	std::shared_ptr<PerfContextMetrics> perfContextMetrics;
-	std::string path;
-	rocksdb::ColumnFamilyHandle* defaultFdbCF = nullptr;
-	UID id;
-	Reference<IThreadPool> writeThread;
-	Reference<IThreadPool> readThreads;
-	std::shared_ptr<RocksDBErrorListener> errorListener;
-	Future<Void> errorFuture;
-	Promise<Void> closePromise;
-	Future<Void> openFuture;
-	std::unique_ptr<rocksdb::WriteBatch> writeBatch;
-	Optional<Future<Void>> metrics;
-	FlowLock readSemaphore;
-	int numReadWaiters;
-	FlowLock fetchSemaphore;
-	int numFetchWaiters;
-	std::shared_ptr<ReadIteratorPool> readIterPool;
-	std::vector<std::unique_ptr<ThreadReturnPromiseStream<std::pair<std::string, double>>>> metricPromiseStreams;
-	// ThreadReturnPromiseStream pair.first stores the histogram name and
-	// pair.second stores the corresponding measured latency (seconds)
-	Future<Void> actorErrorListener;
-	Future<Void> collection;
-	PromiseStream<Future<Void>> addActor;
-	Counters counters;
-
 	explicit RocksDBKeyValueStore(const std::string& path, UID id)
-	  : path(path), id(id), perfContextMetrics(new PerfContextMetrics()),
-	    readIterPool(new ReadIteratorPool(id, db, defaultFdbCF)),
+	  : id(id), sharedState(std::make_shared<SharedRocksDBState>(id)), path(path),
+	    perfContextMetrics(new PerfContextMetrics()),
+	    readIterPool(new ReadIteratorPool(id, db, defaultFdbCF, sharedState->getReadOptions())),
 	    readSemaphore(SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
 	    fetchSemaphore(SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_SOFT_MAX),
 	    numReadWaiters(SERVER_KNOBS->ROCKSDB_READ_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
@@ -1613,6 +1686,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		writeThread->addThread(new Writer(db,
 		                                  defaultFdbCF,
 		                                  id,
+		                                  this->sharedState,
 		                                  readIterPool,
 		                                  perfContextMetrics,
 		                                  SERVER_KNOBS->ROCKSDB_READ_PARALLELISM,
@@ -1627,6 +1701,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			    new Reader(id,
 			               db,
 			               defaultFdbCF,
+			               this->sharedState,
 			               readIterPool,
 			               perfContextMetrics,
 			               i,
@@ -1730,6 +1805,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	Future<Void> getError() const override { return errorFuture; }
 
 	ACTOR static void doClose(RocksDBKeyValueStore* self, bool deleteOnClose) {
+		self->sharedState->setClosing();
+
 		// The metrics future retains a reference to the DB, so stop it before we delete it.
 		self->metrics.reset();
 
@@ -1740,8 +1817,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		self->writeThread->post(a);
 		wait(f);
 		wait(self->writeThread->stop());
-		if (self->closePromise.canBeSet())
+		if (self->closePromise.canBeSet()) {
 			self->closePromise.send(Void());
+		}
+		if (self->db != nullptr) {
+			delete self->db;
+		}
 		delete self;
 	}
 
@@ -1832,8 +1913,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 	// We don't throttle eager reads and reads to the FF keyspace because FDB struggles when those reads fail.
 	// Thus far, they have been low enough volume to not cause an issue.
-	static bool shouldThrottle(IKeyValueStore::ReadType type, KeyRef key) {
-		return type != IKeyValueStore::ReadType::EAGER && !(key.startsWith(systemKeys.begin));
+	static bool shouldThrottle(ReadType type, KeyRef key) {
+		return type != ReadType::EAGER && !(key.startsWith(systemKeys.begin));
 	}
 
 	ACTOR template <class Action>
@@ -1854,7 +1935,15 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		return result;
 	}
 
-	Future<Optional<Value>> readValue(KeyRef key, IKeyValueStore::ReadType type, Optional<UID> debugID) override {
+	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options) override {
+		ReadType type = ReadType::NORMAL;
+		Optional<UID> debugID;
+
+		if (options.present()) {
+			type = options.get().type;
+			debugID = options.get().debugID;
+		}
+
 		if (!shouldThrottle(type, key)) {
 			auto a = new Reader::ReadValueAction(key, debugID);
 			auto res = a->result.getFuture();
@@ -1862,18 +1951,23 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			return res;
 		}
 
-		auto& semaphore = (type == IKeyValueStore::ReadType::FETCH) ? fetchSemaphore : readSemaphore;
-		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
+		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
+		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
 		auto a = std::make_unique<Reader::ReadValueAction>(key, debugID);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 
-	Future<Optional<Value>> readValuePrefix(KeyRef key,
-	                                        int maxLength,
-	                                        IKeyValueStore::ReadType type,
-	                                        Optional<UID> debugID) override {
+	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override {
+		ReadType type = ReadType::NORMAL;
+		Optional<UID> debugID;
+
+		if (options.present()) {
+			type = options.get().type;
+			debugID = options.get().debugID;
+		}
+
 		if (!shouldThrottle(type, key)) {
 			auto a = new Reader::ReadValuePrefixAction(key, maxLength, debugID);
 			auto res = a->result.getFuture();
@@ -1881,8 +1975,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			return res;
 		}
 
-		auto& semaphore = (type == IKeyValueStore::ReadType::FETCH) ? fetchSemaphore : readSemaphore;
-		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
+		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
+		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
 		auto a = std::make_unique<Reader::ReadValuePrefixAction>(key, maxLength, debugID);
@@ -1912,7 +2006,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	Future<RangeResult> readRange(KeyRangeRef keys,
 	                              int rowLimit,
 	                              int byteLimit,
-	                              IKeyValueStore::ReadType type) override {
+	                              Optional<ReadOptions> options) override {
+		ReadType type = ReadType::NORMAL;
+
+		if (options.present()) {
+			type = options.get().type;
+		}
+
 		if (!shouldThrottle(type, keys.begin)) {
 			auto a = new Reader::ReadRangeAction(keys, rowLimit, byteLimit);
 			auto res = a->result.getFuture();
@@ -1920,8 +2020,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			return res;
 		}
 
-		auto& semaphore = (type == IKeyValueStore::ReadType::FETCH) ? fetchSemaphore : readSemaphore;
-		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
+		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
+		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
 		auto a = std::make_unique<Reader::ReadRangeAction>(keys, rowLimit, byteLimit);
@@ -1978,6 +2078,33 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		}
 		return Void();
 	}
+
+	DB db = nullptr;
+	std::shared_ptr<SharedRocksDBState> sharedState;
+	std::shared_ptr<PerfContextMetrics> perfContextMetrics;
+	std::string path;
+	rocksdb::ColumnFamilyHandle* defaultFdbCF = nullptr;
+	UID id;
+	Reference<IThreadPool> writeThread;
+	Reference<IThreadPool> readThreads;
+	std::shared_ptr<RocksDBErrorListener> errorListener;
+	Future<Void> errorFuture;
+	Promise<Void> closePromise;
+	Future<Void> openFuture;
+	std::unique_ptr<rocksdb::WriteBatch> writeBatch;
+	Optional<Future<Void>> metrics;
+	FlowLock readSemaphore;
+	int numReadWaiters;
+	FlowLock fetchSemaphore;
+	int numFetchWaiters;
+	std::shared_ptr<ReadIteratorPool> readIterPool;
+	std::vector<std::unique_ptr<ThreadReturnPromiseStream<std::pair<std::string, double>>>> metricPromiseStreams;
+	// ThreadReturnPromiseStream pair.first stores the histogram name and
+	// pair.second stores the corresponding measured latency (seconds)
+	Future<Void> actorErrorListener;
+	Future<Void> collection;
+	PromiseStream<Future<Void>> addActor;
+	Counters counters;
 };
 
 void RocksDBKeyValueStore::Writer::action(CheckpointAction& a) {
@@ -1987,7 +2114,7 @@ void RocksDBKeyValueStore::Writer::action(CheckpointAction& a) {
 	    .detail("Format", static_cast<int>(a.request.format))
 	    .detail("CheckpointDir", a.request.checkpointDir);
 
-	rocksdb::Checkpoint* checkpoint;
+	rocksdb::Checkpoint* checkpoint = nullptr;
 	rocksdb::Status s = rocksdb::Checkpoint::Create(db, &checkpoint);
 	if (!s.ok()) {
 		logRocksDBError(id, s, "Checkpoint");
@@ -1996,7 +2123,7 @@ void RocksDBKeyValueStore::Writer::action(CheckpointAction& a) {
 	}
 
 	rocksdb::PinnableSlice value;
-	rocksdb::ReadOptions readOptions = getReadOptions();
+	rocksdb::ReadOptions& readOptions = sharedState->getReadOptions();
 	s = db->Get(readOptions, cf, toSlice(persistVersion), &value);
 
 	if (!s.ok() && !s.IsNotFound()) {
@@ -2051,9 +2178,15 @@ void RocksDBKeyValueStore::Writer::action(CheckpointAction& a) {
 		    .detail("RocksSequenceNumber", debugCheckpointSeq)
 		    .detail("CheckpointDir", checkpointDir);
 	} else {
+		if (checkpoint != nullptr) {
+			delete checkpoint;
+		}
 		throw not_implemented();
 	}
 
+	if (checkpoint != nullptr) {
+		delete checkpoint;
+	}
 	res.setState(CheckpointMetaData::Complete);
 	a.reply.send(res);
 }
@@ -2081,13 +2214,16 @@ void RocksDBKeyValueStore::Writer::action(RestoreAction& a) {
 
 		if (cf != nullptr) {
 			ASSERT(db->DropColumnFamily(cf).ok());
+			db->DestroyColumnFamilyHandle(cf);
+			cfHandles.erase(cf);
 		}
 
 		rocksdb::ExportImportFilesMetaData metaData = getMetaData(a.checkpoints[0]);
 		rocksdb::ImportColumnFamilyOptions importOptions;
 		importOptions.move_files = true;
 		status = db->CreateColumnFamilyWithImport(
-		    getCFOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, importOptions, metaData, &cf);
+		    sharedState->getCfOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, importOptions, metaData, &cf);
+		cfHandles.insert(cf);
 
 		if (!status.ok()) {
 			logRocksDBError(id, status, "Restore");
@@ -2100,7 +2236,9 @@ void RocksDBKeyValueStore::Writer::action(RestoreAction& a) {
 		}
 	} else if (format == RocksDB) {
 		if (cf == nullptr) {
-			status = db->CreateColumnFamily(getCFOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, &cf);
+			status = db->CreateColumnFamily(
+			    sharedState->getCfOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, &cf);
+			cfHandles.insert(cf);
 			TraceEvent("RocksDBServeRestoreRange", id)
 			    .detail("Path", a.path)
 			    .detail("Checkpoint", describe(a.checkpoints));
@@ -2217,7 +2355,7 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBBasic") {
 	}
 
 	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
+	kvStore->dispose();
 	wait(closed);
 
 	platform::eraseDirectoryRecursive(rocksDBTestDir);
@@ -2231,11 +2369,11 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBReopen") {
 	state IKeyValueStore* kvStore = new RocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
 	wait(kvStore->init());
 
-	kvStore->set({ LiteralStringRef("foo"), LiteralStringRef("bar") });
+	kvStore->set({ "foo"_sr, "bar"_sr });
 	wait(kvStore->commit(false));
 
-	Optional<Value> val = wait(kvStore->readValue(LiteralStringRef("foo")));
-	ASSERT(Optional<Value>(LiteralStringRef("bar")) == val);
+	Optional<Value> val = wait(kvStore->readValue("foo"_sr));
+	ASSERT(Optional<Value>("bar"_sr) == val);
 
 	Future<Void> closed = kvStore->onClosed();
 	kvStore->close();
@@ -2246,12 +2384,16 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBReopen") {
 	// Confirm that `init()` is idempotent.
 	wait(kvStore->init());
 
-	Optional<Value> val = wait(kvStore->readValue(LiteralStringRef("foo")));
-	ASSERT(Optional<Value>(LiteralStringRef("bar")) == val);
+	{
+		Optional<Value> val = wait(kvStore->readValue("foo"_sr));
+		ASSERT(Optional<Value>("bar"_sr) == val);
+	}
 
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
-	wait(closed);
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->dispose();
+		wait(closed);
+	}
 
 	platform::eraseDirectoryRecursive(rocksDBTestDir);
 	return Void();
@@ -2265,11 +2407,11 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestoreColumnFamily")
 	state IKeyValueStore* kvStore = new RocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
 	wait(kvStore->init());
 
-	kvStore->set({ LiteralStringRef("foo"), LiteralStringRef("bar") });
+	kvStore->set({ "foo"_sr, "bar"_sr });
 	wait(kvStore->commit(false));
 
-	Optional<Value> val = wait(kvStore->readValue(LiteralStringRef("foo")));
-	ASSERT(Optional<Value>(LiteralStringRef("bar")) == val);
+	Optional<Value> val = wait(kvStore->readValue("foo"_sr));
+	ASSERT(Optional<Value>("bar"_sr) == val);
 
 	state std::string rocksDBRestoreDir = "rocksdb-kvstore-br-restore-db";
 	platform::eraseDirectoryRecursive(rocksDBRestoreDir);
@@ -2289,14 +2431,16 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestoreColumnFamily")
 	checkpoints.push_back(metaData);
 	wait(kvStoreCopy->restore(checkpoints));
 
-	Optional<Value> val = wait(kvStoreCopy->readValue(LiteralStringRef("foo")));
-	ASSERT(Optional<Value>(LiteralStringRef("bar")) == val);
+	{
+		Optional<Value> val = wait(kvStoreCopy->readValue("foo"_sr));
+		ASSERT(Optional<Value>("bar"_sr) == val);
+	}
 
 	std::vector<Future<Void>> closes;
 	closes.push_back(kvStore->onClosed());
 	closes.push_back(kvStoreCopy->onClosed());
-	kvStore->close();
-	kvStoreCopy->close();
+	kvStore->dispose();
+	kvStoreCopy->dispose();
 	wait(waitForAll(closes));
 
 	platform::eraseDirectoryRecursive(rocksDBTestDir);
@@ -2312,10 +2456,10 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestoreKeyValues") {
 	state IKeyValueStore* kvStore = new RocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
 	wait(kvStore->init());
 
-	kvStore->set({ LiteralStringRef("foo"), LiteralStringRef("bar") });
+	kvStore->set({ "foo"_sr, "bar"_sr });
 	wait(kvStore->commit(false));
-	Optional<Value> val = wait(kvStore->readValue(LiteralStringRef("foo")));
-	ASSERT(Optional<Value>(LiteralStringRef("bar")) == val);
+	Optional<Value> val = wait(kvStore->readValue("foo"_sr));
+	ASSERT(Optional<Value>("bar"_sr) == val);
 
 	platform::eraseDirectoryRecursive("checkpoint");
 	std::string checkpointDir = cwd + "checkpoint";
@@ -2346,7 +2490,7 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestoreKeyValues") {
 	std::vector<Future<Void>> closes;
 	closes.push_back(cpReader->close());
 	closes.push_back(kvStore->onClosed());
-	kvStore->close();
+	kvStore->dispose();
 	wait(waitForAll(closes));
 
 	platform::eraseDirectoryRecursive(rocksDBTestDir);
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index f65ce27199..07d9e06171 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -117,7 +117,7 @@ struct PageChecksumCodec {
 			crc32Sum.part1 = 0;
 			crc32Sum.part2 = crc32c_append(0xfdbeefdb, static_cast<uint8_t*>(data), dataLen);
 			if (crc32Sum == *pSumInPage) {
-				TEST(true); // Read CRC32 checksum
+				CODE_PROBE(true, "Read CRC32 checksum");
 				return true;
 			}
 		}
@@ -133,7 +133,7 @@ struct PageChecksumCodec {
 			xxHash3Sum.part1 = static_cast<uint32_t>((xxHash3 >> 32) & 0x00ffffff);
 			xxHash3Sum.part2 = static_cast<uint32_t>(xxHash3 & 0xffffffff);
 			if (xxHash3Sum == *pSumInPage) {
-				TEST(true); // Read xxHash3 checksum
+				CODE_PROBE(true, "Read xxHash3 checksum");
 				return true;
 			}
 		}
@@ -144,7 +144,7 @@ struct PageChecksumCodec {
 		hashLittle2Sum.part2 = 0x5ca1ab1e;
 		hashlittle2(pData, dataLen, &hashLittle2Sum.part1, &hashLittle2Sum.part2);
 		if (hashLittle2Sum == *pSumInPage) {
-			TEST(true); // Read HashLittle2 checksum
+			CODE_PROBE(true, "Read HashLittle2 checksum");
 			return true;
 		}
 
@@ -357,7 +357,7 @@ struct SQLiteDB : NonCopyable {
 					lineStart = lineEnd;
 				}
 			}
-			TEST(true); // BTree integrity checked
+			CODE_PROBE(true, "BTree integrity checked");
 		}
 		if (e)
 			sqlite3_free(e);
@@ -1333,7 +1333,7 @@ int SQLiteDB::checkAllPageChecksums() {
 	// then we could instead open a read cursor for the same effect, as currently tryReadEveryDbPage() requires it.
 	Statement* jm = new Statement(*this, "PRAGMA journal_mode");
 	ASSERT(jm->nextRow());
-	if (jm->column(0) != LiteralStringRef("wal")) {
+	if (jm->column(0) != "wal"_sr) {
 		TraceEvent(SevError, "JournalModeError").detail("Filename", filename).detail("Mode", jm->column(0));
 		ASSERT(false);
 	}
@@ -1423,7 +1423,7 @@ void SQLiteDB::open(bool writable) {
 			renameFile(walpath, walpath + "-old-" + deterministicRandom()->randomUniqueID().toString());
 			ASSERT_WE_THINK(false); //< This code should not be hit in FoundationDB at the moment, because worker looks
 			                        // for databases to open by listing .fdb files, not .fdb-wal files
-			// TEST(true);  // Replace a partially constructed or destructed DB
+			// CODE_PROBE(true, "Replace a partially constructed or destructed DB");
 		}
 
 		if (dbFile.isError() && walFile.isError() && writable &&
@@ -1502,7 +1502,7 @@ void SQLiteDB::open(bool writable) {
 
 	Statement jm(*this, "PRAGMA journal_mode");
 	ASSERT(jm.nextRow());
-	if (jm.column(0) != LiteralStringRef("wal")) {
+	if (jm.column(0) != "wal"_sr) {
 		TraceEvent(SevError, "JournalModeError").detail("Filename", filename).detail("Mode", jm.column(0));
 		ASSERT(false);
 	}
@@ -1589,12 +1589,12 @@ public:
 	void clear(KeyRangeRef range, const Arena* arena = nullptr) override;
 	Future<Void> commit(bool sequential = false) override;
 
-	Future<Optional<Value>> readValue(KeyRef key, IKeyValueStore::ReadType, Optional<UID> debugID) override;
-	Future<Optional<Value>> readValuePrefix(KeyRef key,
-	                                        int maxLength,
-	                                        IKeyValueStore::ReadType,
-	                                        Optional<UID> debugID) override;
-	Future<RangeResult> readRange(KeyRangeRef keys, int rowLimit, int byteLimit, IKeyValueStore::ReadType) override;
+	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> optionss) override;
+	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override;
+	Future<RangeResult> readRange(KeyRangeRef keys,
+	                              int rowLimit,
+	                              int byteLimit,
+	                              Optional<ReadOptions> options) override;
 
 	KeyValueStoreSQLite(std::string const& filename,
 	                    UID logID,
@@ -1810,7 +1810,7 @@ private:
 			cursor->set(a.kv);
 			++setsThisCommit;
 			++writesComplete;
-			if (g_network->isSimulated() && g_simulator.getCurrentProcess()->rebooting)
+			if (g_network->isSimulated() && g_simulator->getCurrentProcess()->rebooting)
 				TraceEvent("SetActionFinished", dbgid).detail("Elapsed", now() - s);
 		}
 
@@ -1824,7 +1824,7 @@ private:
 			cursor->fastClear(a.range, freeTableEmpty);
 			cursor->clear(a.range); // TODO: at most one
 			++writesComplete;
-			if (g_network->isSimulated() && g_simulator.getCurrentProcess()->rebooting)
+			if (g_network->isSimulated() && g_simulator->getCurrentProcess()->rebooting)
 				TraceEvent("ClearActionFinished", dbgid).detail("Elapsed", now() - s);
 		}
 
@@ -1864,7 +1864,7 @@ private:
 
 			diskBytesUsed = waitForAndGet(conn.dbFile->size()) + waitForAndGet(conn.walFile->size());
 
-			if (g_network->isSimulated() && g_simulator.getCurrentProcess()->rebooting)
+			if (g_network->isSimulated() && g_simulator->getCurrentProcess()->rebooting)
 				TraceEvent("CommitActionFinished", dbgid).detail("Elapsed", now() - t1);
 		}
 
@@ -1942,8 +1942,8 @@ private:
 				}
 
 				if (canDelete && (!canVacuum || deterministicRandom()->random01() < lazyDeleteBatchProbability)) {
-					TEST(canVacuum); // SQLite lazy deletion when vacuuming is active
-					TEST(!canVacuum); // SQLite lazy deletion when vacuuming is inactive
+					CODE_PROBE(canVacuum, "SQLite lazy deletion when vacuuming is active");
+					CODE_PROBE(!canVacuum, "SQLite lazy deletion when vacuuming is inactive");
 
 					int pagesToDelete = std::max(
 					    1,
@@ -1955,10 +1955,10 @@ private:
 					lazyDeleteTime += now() - begin;
 				} else {
 					ASSERT(canVacuum);
-					TEST(canDelete); // SQLite vacuuming when lazy delete is active
-					TEST(!canDelete); // SQLite vacuuming when lazy delete is inactive
-					TEST(SERVER_KNOBS->SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE !=
-					     0); // SQLite vacuuming with nonzero vacuums_per_lazy_delete_page
+					CODE_PROBE(canDelete, "SQLite vacuuming when lazy delete is active");
+					CODE_PROBE(!canDelete, "SQLite vacuuming when lazy delete is inactive");
+					CODE_PROBE(SERVER_KNOBS->SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE != 0,
+					           "SQLite vacuuming with nonzero vacuums_per_lazy_delete_page");
 
 					vacuumFinished = conn.vacuum();
 					if (!vacuumFinished) {
@@ -1973,10 +1973,10 @@ private:
 
 			freeListPages = conn.freePages();
 
-			TEST(workPerformed.lazyDeletePages > 0); // Pages lazily deleted
-			TEST(workPerformed.vacuumedPages > 0); // Pages vacuumed
-			TEST(vacuumTime > 0); // Time spent vacuuming
-			TEST(lazyDeleteTime > 0); // Time spent lazy deleting
+			CODE_PROBE(workPerformed.lazyDeletePages > 0, "Pages lazily deleted");
+			CODE_PROBE(workPerformed.vacuumedPages > 0, "Pages vacuumed");
+			CODE_PROBE(vacuumTime > 0, "Time spent vacuuming");
+			CODE_PROBE(lazyDeleteTime > 0, "Time spent lazy deleting");
 
 			++springCleaningStats.springCleaningCount;
 			springCleaningStats.lazyDeletePages += workPerformed.lazyDeletePages;
@@ -1987,7 +1987,7 @@ private:
 
 			a.result.send(workPerformed);
 			++writesComplete;
-			if (g_network->isSimulated() && g_simulator.getCurrentProcess()->rebooting)
+			if (g_network->isSimulated() && g_simulator->getCurrentProcess()->rebooting)
 				TraceEvent("SpringCleaningActionFinished", dbgid).detail("Elapsed", now() - s);
 		}
 	};
@@ -1996,7 +1996,7 @@ private:
 		state int64_t lastReadsComplete = 0;
 		state int64_t lastWritesComplete = 0;
 		loop {
-			wait(delay(FLOW_KNOBS->DISK_METRIC_LOGGING_INTERVAL));
+			wait(delay(FLOW_KNOBS->SQLITE_DISK_METRIC_LOGGING_INTERVAL));
 
 			int64_t rc = self->readsComplete, wc = self->writesComplete;
 			TraceEvent("DiskMetrics", self->logID)
@@ -2216,18 +2216,23 @@ Future<Void> KeyValueStoreSQLite::commit(bool sequential) {
 	writeThread->post(p);
 	return f;
 }
-Future<Optional<Value>> KeyValueStoreSQLite::readValue(KeyRef key, IKeyValueStore::ReadType, Optional<UID> debugID) {
+Future<Optional<Value>> KeyValueStoreSQLite::readValue(KeyRef key, Optional<ReadOptions> options) {
 	++readsRequested;
+	Optional<UID> debugID;
+	if (options.present()) {
+		debugID = options.get().debugID;
+	}
 	auto p = new Reader::ReadValueAction(key, debugID);
 	auto f = p->result.getFuture();
 	readThreads->post(p);
 	return f;
 }
-Future<Optional<Value>> KeyValueStoreSQLite::readValuePrefix(KeyRef key,
-                                                             int maxLength,
-                                                             IKeyValueStore::ReadType,
-                                                             Optional<UID> debugID) {
+Future<Optional<Value>> KeyValueStoreSQLite::readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) {
 	++readsRequested;
+	Optional<UID> debugID;
+	if (options.present()) {
+		debugID = options.get().debugID;
+	}
 	auto p = new Reader::ReadValuePrefixAction(key, maxLength, debugID);
 	auto f = p->result.getFuture();
 	readThreads->post(p);
@@ -2236,7 +2241,7 @@ Future<Optional<Value>> KeyValueStoreSQLite::readValuePrefix(KeyRef key,
 Future<RangeResult> KeyValueStoreSQLite::readRange(KeyRangeRef keys,
                                                    int rowLimit,
                                                    int byteLimit,
-                                                   IKeyValueStore::ReadType) {
+                                                   Optional<ReadOptions> options) {
 	++readsRequested;
 	auto p = new Reader::ReadRangeAction(keys, rowLimit, byteLimit);
 	auto f = p->result.getFuture();
@@ -2282,9 +2287,9 @@ ACTOR Future<Void> KVFileCheck(std::string filename, bool integrity) {
 
 	StringRef kvFile(filename);
 	KeyValueStoreType type = KeyValueStoreType::END;
-	if (kvFile.endsWith(LiteralStringRef(".fdb")))
+	if (kvFile.endsWith(".fdb"_sr))
 		type = KeyValueStoreType::SSD_BTREE_V1;
-	else if (kvFile.endsWith(LiteralStringRef(".sqlite")))
+	else if (kvFile.endsWith(".sqlite"_sr))
 		type = KeyValueStoreType::SSD_BTREE_V2;
 	ASSERT(type != KeyValueStoreType::END);
 
diff --git a/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp b/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp
index 93680682d2..7d9b84b842 100644
--- a/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp
@@ -8,6 +8,7 @@
 #include <rocksdb/db.h>
 #include <rocksdb/filter_policy.h>
 #include <rocksdb/listener.h>
+#include <rocksdb/metadata.h>
 #include <rocksdb/options.h>
 #include <rocksdb/slice_transform.h>
 #include <rocksdb/statistics.h>
@@ -48,31 +49,32 @@ static_assert((ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR == 27) ? ROCKSDB_PATCH >= 3 :
               "Unsupported rocksdb version. Update the rocksdb to 6.27.3 version");
 
 const std::string rocksDataFolderSuffix = "-data";
-const KeyRef shardMappingPrefix(LiteralStringRef("\xff\xff/ShardMapping/"));
+const KeyRef shardMappingPrefix("\xff\xff/ShardMapping/"_sr);
 // TODO: move constants to a header file.
-const StringRef ROCKSDBSTORAGE_HISTOGRAM_GROUP = LiteralStringRef("RocksDBStorage");
-const StringRef ROCKSDB_COMMIT_LATENCY_HISTOGRAM = LiteralStringRef("RocksDBCommitLatency");
-const StringRef ROCKSDB_COMMIT_ACTION_HISTOGRAM = LiteralStringRef("RocksDBCommitAction");
-const StringRef ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM = LiteralStringRef("RocksDBCommitQueueWait");
-const StringRef ROCKSDB_WRITE_HISTOGRAM = LiteralStringRef("RocksDBWrite");
-const StringRef ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM = LiteralStringRef("RocksDBDeleteCompactRange");
-const StringRef ROCKSDB_READRANGE_LATENCY_HISTOGRAM = LiteralStringRef("RocksDBReadRangeLatency");
-const StringRef ROCKSDB_READVALUE_LATENCY_HISTOGRAM = LiteralStringRef("RocksDBReadValueLatency");
-const StringRef ROCKSDB_READPREFIX_LATENCY_HISTOGRAM = LiteralStringRef("RocksDBReadPrefixLatency");
-const StringRef ROCKSDB_READRANGE_ACTION_HISTOGRAM = LiteralStringRef("RocksDBReadRangeAction");
-const StringRef ROCKSDB_READVALUE_ACTION_HISTOGRAM = LiteralStringRef("RocksDBReadValueAction");
-const StringRef ROCKSDB_READPREFIX_ACTION_HISTOGRAM = LiteralStringRef("RocksDBReadPrefixAction");
-const StringRef ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM = LiteralStringRef("RocksDBReadRangeQueueWait");
-const StringRef ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM = LiteralStringRef("RocksDBReadValueQueueWait");
-const StringRef ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM = LiteralStringRef("RocksDBReadPrefixQueueWait");
-const StringRef ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM = LiteralStringRef("RocksDBReadRangeNewIterator");
-const StringRef ROCKSDB_READVALUE_GET_HISTOGRAM = LiteralStringRef("RocksDBReadValueGet");
-const StringRef ROCKSDB_READPREFIX_GET_HISTOGRAM = LiteralStringRef("RocksDBReadPrefixGet");
+const StringRef ROCKSDBSTORAGE_HISTOGRAM_GROUP = "RocksDBStorage"_sr;
+const StringRef ROCKSDB_COMMIT_LATENCY_HISTOGRAM = "RocksDBCommitLatency"_sr;
+const StringRef ROCKSDB_COMMIT_ACTION_HISTOGRAM = "RocksDBCommitAction"_sr;
+const StringRef ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM = "RocksDBCommitQueueWait"_sr;
+const StringRef ROCKSDB_WRITE_HISTOGRAM = "RocksDBWrite"_sr;
+const StringRef ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM = "RocksDBDeleteCompactRange"_sr;
+const StringRef ROCKSDB_READRANGE_LATENCY_HISTOGRAM = "RocksDBReadRangeLatency"_sr;
+const StringRef ROCKSDB_READVALUE_LATENCY_HISTOGRAM = "RocksDBReadValueLatency"_sr;
+const StringRef ROCKSDB_READPREFIX_LATENCY_HISTOGRAM = "RocksDBReadPrefixLatency"_sr;
+const StringRef ROCKSDB_READRANGE_ACTION_HISTOGRAM = "RocksDBReadRangeAction"_sr;
+const StringRef ROCKSDB_READVALUE_ACTION_HISTOGRAM = "RocksDBReadValueAction"_sr;
+const StringRef ROCKSDB_READPREFIX_ACTION_HISTOGRAM = "RocksDBReadPrefixAction"_sr;
+const StringRef ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM = "RocksDBReadRangeQueueWait"_sr;
+const StringRef ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM = "RocksDBReadValueQueueWait"_sr;
+const StringRef ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM = "RocksDBReadPrefixQueueWait"_sr;
+const StringRef ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM = "RocksDBReadRangeNewIterator"_sr;
+const StringRef ROCKSDB_READVALUE_GET_HISTOGRAM = "RocksDBReadValueGet"_sr;
+const StringRef ROCKSDB_READPREFIX_GET_HISTOGRAM = "RocksDBReadPrefixGet"_sr;
 
 namespace {
 struct PhysicalShard;
 struct DataShard;
 struct ReadIterator;
+struct ShardedRocksDBKeyValueStore;
 
 using rocksdb::BackgroundErrorReason;
 
@@ -110,9 +112,9 @@ class RocksDBErrorListener : public rocksdb::EventListener {
 public:
 	RocksDBErrorListener(){};
 	void OnBackgroundError(rocksdb::BackgroundErrorReason reason, rocksdb::Status* bg_error) override {
-		TraceEvent(SevError, "RocksDBBGError")
+		TraceEvent(SevError, "ShardedRocksDBBGError")
 		    .detail("Reason", getErrorReason(reason))
-		    .detail("RocksDBSeverity", bg_error->severity())
+		    .detail("ShardedRocksDBSeverity", bg_error->severity())
 		    .detail("Status", bg_error->ToString());
 		std::unique_lock<std::mutex> lock(mutex);
 		if (!errorPromise.isValid())
@@ -164,30 +166,10 @@ std::string getShardMappingKey(KeyRef key, StringRef prefix) {
 	return prefix.toString() + key.toString();
 }
 
-std::vector<std::pair<KeyRange, std::string>> decodeShardMapping(const RangeResult& result, StringRef prefix) {
-	std::vector<std::pair<KeyRange, std::string>> shards;
-	KeyRef endKey;
-	std::string name;
-
-	for (const auto& kv : result) {
-		auto keyWithoutPrefix = kv.key.removePrefix(prefix);
-		if (name.size() > 0) {
-			shards.push_back({ KeyRange(KeyRangeRef(endKey, keyWithoutPrefix)), name });
-			TraceEvent(SevDebug, "DecodeShardMapping")
-			    .detail("BeginKey", endKey)
-			    .detail("EndKey", keyWithoutPrefix)
-			    .detail("Name", name);
-		}
-		endKey = keyWithoutPrefix;
-		name = kv.value.toString();
-	}
-	return shards;
-}
-
 void logRocksDBError(const rocksdb::Status& status, const std::string& method) {
 	auto level = status.IsTimedOut() ? SevWarn : SevError;
-	TraceEvent e(level, "RocksDBError");
-	e.detail("Error", status.ToString()).detail("Method", method).detail("RocksDBSeverity", status.severity());
+	TraceEvent e(level, "ShardedRocksDBError");
+	e.detail("Error", status.ToString()).detail("Method", method).detail("ShardedRocksDBSeverity", status.severity());
 	if (status.IsIOError()) {
 		e.detail("SubCode", status.subcode());
 	}
@@ -219,7 +201,7 @@ const char* ShardOpToString(ShardOp op) {
 	}
 }
 void logShardEvent(StringRef name, ShardOp op, Severity severity = SevInfo, const std::string& message = "") {
-	TraceEvent e(severity, "KVSShardEvent");
+	TraceEvent e(severity, "ShardedRocksDBKVSShardEvent");
 	e.detail("Name", name).detail("Action", ShardOpToString(op));
 	if (!message.empty()) {
 		e.detail("Message", message);
@@ -230,7 +212,7 @@ void logShardEvent(StringRef name,
                    ShardOp op,
                    Severity severity = SevInfo,
                    const std::string& message = "") {
-	TraceEvent e(severity, "KVSShardEvent");
+	TraceEvent e(severity, "ShardedRocksDBKVSShardEvent");
 	e.detail("Name", name).detail("Action", ShardOpToString(op)).detail("Begin", range.begin).detail("End", range.end);
 	if (message != "") {
 		e.detail("Message", message);
@@ -249,7 +231,7 @@ Error statusToError(const rocksdb::Status& s) {
 
 rocksdb::ColumnFamilyOptions getCFOptions() {
 	rocksdb::ColumnFamilyOptions options;
-	options.level_compaction_dynamic_level_bytes = true;
+	options.level_compaction_dynamic_level_bytes = SERVER_KNOBS->ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES;
 	options.OptimizeLevelStyleCompaction(SERVER_KNOBS->ROCKSDB_MEMTABLE_BYTES);
 	if (SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS > 0) {
 		options.periodic_compaction_seconds = SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS;
@@ -283,8 +265,8 @@ rocksdb::ColumnFamilyOptions getCFOptions() {
 		bbOpts.whole_key_filtering = false;
 	}
 
-	if (rocksdb_block_cache == nullptr) {
-		rocksdb_block_cache = rocksdb::NewLRUCache(128);
+	if (rocksdb_block_cache == nullptr && SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE > 0) {
+		rocksdb_block_cache = rocksdb::NewLRUCache(SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE);
 	}
 	bbOpts.block_cache = rocksdb_block_cache;
 
@@ -301,7 +283,15 @@ rocksdb::Options getOptions() {
 		options.IncreaseParallelism(SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM);
 	}
 
-	// TODO: enable rocksdb metrics.
+	options.delete_obsolete_files_period_micros = SERVER_KNOBS->ROCKSDB_DELETE_OBSOLETE_FILE_PERIOD * 1000000;
+	options.max_total_wal_size = SERVER_KNOBS->ROCKSDB_MAX_TOTAL_WAL_SIZE;
+	options.max_subcompactions = SERVER_KNOBS->ROCKSDB_MAX_SUBCOMPACTIONS;
+	options.max_background_jobs = SERVER_KNOBS->ROCKSDB_MAX_BACKGROUND_JOBS;
+
+	options.db_write_buffer_size = SERVER_KNOBS->ROCKSDB_WRITE_BUFFER_SIZE;
+	options.write_buffer_size = SERVER_KNOBS->ROCKSDB_CF_WRITE_BUFFER_SIZE;
+	options.statistics = rocksdb::CreateDBStatistics();
+	options.statistics->set_stats_level(rocksdb::kExceptHistogramOrTimers);
 	options.db_log_dir = SERVER_KNOBS->LOG_DIRECTORY;
 	return options;
 }
@@ -343,7 +333,7 @@ public:
 		ASSERT(cf);
 		readRangeOptions.background_purge_on_iterator_cleanup = true;
 		readRangeOptions.auto_prefix_mode = (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0);
-		TraceEvent(SevDebug, "ReadIteratorPool")
+		TraceEvent(SevVerbose, "ShardedRocksReadIteratorPool")
 		    .detail("Path", path)
 		    .detail("KnobRocksDBReadRangeReuseIterators", SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS)
 		    .detail("KnobRocksDBPrefixLen", SERVER_KNOBS->ROCKSDB_PREFIX_LEN);
@@ -425,7 +415,7 @@ private:
 ACTOR Future<Void> flowLockLogger(const FlowLock* readLock, const FlowLock* fetchLock) {
 	loop {
 		wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
-		TraceEvent e("RocksDBFlowLock");
+		TraceEvent e("ShardedRocksDBFlowLock");
 		e.detail("ReadAvailable", readLock->available());
 		e.detail("ReadActivePermits", readLock->activePermits());
 		e.detail("ReadWaiters", readLock->waiters());
@@ -505,6 +495,7 @@ struct PhysicalShard {
 	std::shared_ptr<ReadIteratorPool> readIterPool;
 	bool deletePending = false;
 	std::atomic<bool> isInitialized;
+	double deleteTimeSec;
 };
 
 int readRangeInDb(PhysicalShard* shard, const KeyRangeRef& range, int rowLimit, int byteLimit, RangeResult* result) {
@@ -581,6 +572,9 @@ public:
 	ACTOR static Future<Void> shardMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
 	                                             Future<Void> openFuture,
 	                                             ShardManager* shardManager) {
+		state std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* physicalShards =
+		    shardManager->getAllShards();
+
 		try {
 			wait(openFuture);
 			loop {
@@ -588,23 +582,46 @@ public:
 				if (rState->closing) {
 					break;
 				}
-				TraceEvent(SevInfo, "KVSPhysialShardMetrics")
+				TraceEvent(SevInfo, "ShardedRocksKVSPhysialShardMetrics")
 				    .detail("NumActiveShards", shardManager->numActiveShards())
 				    .detail("TotalPhysicalShards", shardManager->numPhysicalShards());
+
+				for (auto& [id, shard] : *physicalShards) {
+					if (!shard->initialized()) {
+						continue;
+					}
+					std::string propValue = "";
+					ASSERT(shard->db->GetProperty(shard->cf, rocksdb::DB::Properties::kCFStats, &propValue));
+					TraceEvent(SevInfo, "PhysicalShardCFStats").detail("ShardId", id).detail("Detail", propValue);
+
+					// Get compression ratio for each level.
+					rocksdb::ColumnFamilyMetaData cfMetadata;
+					shard->db->GetColumnFamilyMetaData(shard->cf, &cfMetadata);
+					TraceEvent e(SevInfo, "PhysicalShardLevelStats");
+					e.detail("ShardId", id);
+					std::string levelProp;
+					for (auto it = cfMetadata.levels.begin(); it != cfMetadata.levels.end(); ++it) {
+						std::string propValue = "";
+						ASSERT(shard->db->GetProperty(shard->cf,
+						                              rocksdb::DB::Properties::kCompressionRatioAtLevelPrefix +
+						                                  std::to_string(it->level),
+						                              &propValue));
+						e.detail("Level" + std::to_string(it->level), std::to_string(it->size) + " " + propValue);
+					}
+				}
 			}
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled) {
-				TraceEvent(SevError, "ShardMetricsLoggerError").errorUnsuppressed(e);
+				TraceEvent(SevError, "ShardedRocksShardMetricsLoggerError").errorUnsuppressed(e);
 			}
 		}
 		return Void();
 	}
 
-	rocksdb::Status init() {
+	rocksdb::Status init(rocksdb::Options options) {
 		// Open instance.
-		TraceEvent(SevVerbose, "ShardManagerInitBegin", this->logId).detail("DataPath", path);
+		TraceEvent(SevInfo, "ShardedRocksShardManagerInitBegin", this->logId).detail("DataPath", path);
 		std::vector<std::string> columnFamilies;
-		rocksdb::Options options = getOptions();
 		rocksdb::Status status = rocksdb::DB::ListColumnFamilies(options, path, &columnFamilies);
 
 		rocksdb::ColumnFamilyOptions cfOptions = getCFOptions();
@@ -632,6 +649,9 @@ public:
 		}
 
 		if (foundMetadata) {
+			TraceEvent(SevInfo, "ShardedRocksInitLoadPhysicalShards", this->logId)
+			    .detail("PhysicalShardCount", handles.size());
+
 			for (auto handle : handles) {
 				if (handle->GetName() == "kvs-metadata") {
 					metadataShard = std::make_shared<PhysicalShard>(db, "kvs-metadata", handle);
@@ -639,38 +659,90 @@ public:
 					physicalShards[handle->GetName()] = std::make_shared<PhysicalShard>(db, handle->GetName(), handle);
 				}
 				columnFamilyMap[handle->GetID()] = handle;
-				TraceEvent(SevInfo, "ShardedRocskDB").detail("FoundShard", handle->GetName()).detail("Action", "Init");
+				TraceEvent(SevVerbose, "ShardedRocksInitPhysicalShard", this->logId)
+				    .detail("PhysicalShard", handle->GetName());
 			}
-			RangeResult metadata;
-			readRangeInDb(metadataShard.get(), prefixRange(shardMappingPrefix), UINT16_MAX, UINT16_MAX, &metadata);
 
-			std::vector<std::pair<KeyRange, std::string>> mapping = decodeShardMapping(metadata, shardMappingPrefix);
+			std::set<std::string> unusedShards(columnFamilies.begin(), columnFamilies.end());
+			unusedShards.erase("kvs-metadata");
+			unusedShards.erase("default");
 
-			for (const auto& [range, name] : mapping) {
-				TraceEvent(SevDebug, "ShardedRocksLoadPhysicalShard", this->logId)
-				    .detail("Range", range)
-				    .detail("PhysicalShard", name);
-				auto it = physicalShards.find(name);
-				// Raise error if physical shard is missing.
-				if (it == physicalShards.end()) {
-					TraceEvent(SevError, "ShardedRocksDB").detail("MissingShard", name);
-					return rocksdb::Status::NotFound();
+			KeyRange keyRange = prefixRange(shardMappingPrefix);
+			while (true) {
+				RangeResult metadata;
+				const int bytes = readRangeInDb(metadataShard.get(),
+				                                keyRange,
+				                                std::max(2, SERVER_KNOBS->ROCKSDB_READ_RANGE_ROW_LIMIT),
+				                                UINT16_MAX,
+				                                &metadata);
+				if (bytes <= 0) {
+					break;
 				}
-				std::unique_ptr<DataShard> dataShard = std::make_unique<DataShard>(range, it->second.get());
-				dataShardMap.insert(range, dataShard.get());
-				it->second->dataShards[range.begin.toString()] = std::move(dataShard);
-				activePhysicalShardIds.emplace(name);
-			}
-			// TODO: remove unused column families.
 
+				ASSERT_GT(metadata.size(), 0);
+				for (int i = 0; i < metadata.size() - 1; ++i) {
+					const std::string name = metadata[i].value.toString();
+					KeyRangeRef range(metadata[i].key.removePrefix(shardMappingPrefix),
+					                  metadata[i + 1].key.removePrefix(shardMappingPrefix));
+					TraceEvent(SevVerbose, "DecodeShardMapping", this->logId)
+					    .detail("Range", range)
+					    .detail("Name", name);
+
+					// Empty name indicates the shard doesn't belong to the SS/KVS.
+					if (name.empty()) {
+						continue;
+					}
+
+					auto it = physicalShards.find(name);
+					// Raise error if physical shard is missing.
+					if (it == physicalShards.end()) {
+						TraceEvent(SevError, "ShardedRocksDB", this->logId).detail("MissingShard", name);
+						return rocksdb::Status::NotFound();
+					}
+
+					std::unique_ptr<DataShard> dataShard = std::make_unique<DataShard>(range, it->second.get());
+					dataShardMap.insert(range, dataShard.get());
+					it->second->dataShards[range.begin.toString()] = std::move(dataShard);
+					activePhysicalShardIds.emplace(name);
+					unusedShards.erase(name);
+				}
+
+				if (metadata.back().key.removePrefix(shardMappingPrefix) == specialKeys.end) {
+					TraceEvent(SevVerbose, "ShardedRocksLoadShardMappingEnd", this->logId);
+					break;
+				}
+
+				// Read from the current last key since the shard begining with it hasn't been processed.
+				if (metadata.size() == 1 && metadata.back().value.toString().empty()) {
+					// Should not happen, just being paranoid.
+					keyRange = KeyRangeRef(keyAfter(metadata.back().key), keyRange.end);
+				} else {
+					keyRange = KeyRangeRef(metadata.back().key, keyRange.end);
+				}
+			}
+
+			for (const auto& name : unusedShards) {
+				TraceEvent(SevDebug, "UnusedShardName", logId).detail("Name", name);
+				auto it = physicalShards.find(name);
+				ASSERT(it != physicalShards.end());
+				auto shard = it->second;
+				if (shard->dataShards.size() == 0) {
+					shard->deleteTimeSec = now();
+					pendingDeletionShards.push_back(name);
+				}
+			}
+			if (unusedShards.size() > 0) {
+				TraceEvent("ShardedRocksDB", logId).detail("CleanUpUnusedShards", unusedShards.size());
+			}
 		} else {
 			// DB is opened with default shard.
 			ASSERT(handles.size() == 1);
+
 			// Add SpecialKeys range. This range should not be modified.
 			std::shared_ptr<PhysicalShard> defaultShard = std::make_shared<PhysicalShard>(db, "default", handles[0]);
 			columnFamilyMap[defaultShard->cf->GetID()] = defaultShard->cf;
 			std::unique_ptr<DataShard> dataShard = std::make_unique<DataShard>(specialKeys, defaultShard.get());
-			dataShardMap.insert(dataShard->range, dataShard.get());
+			dataShardMap.insert(specialKeys, dataShard.get());
 			defaultShard->dataShards[specialKeys.begin.toString()] = std::move(dataShard);
 			physicalShards[defaultShard->id] = defaultShard;
 
@@ -688,7 +760,7 @@ public:
 				return status;
 			}
 			metadataShard->readIterPool->update();
-			TraceEvent(SevVerbose, "InitializeMetaDataShard", this->logId)
+			TraceEvent(SevInfo, "ShardedRocksInitializeMetaDataShard", this->logId)
 			    .detail("MetadataShardCF", metadataShard->cf->GetID());
 		}
 		physicalShards["kvs-metadata"] = metadataShard;
@@ -696,7 +768,7 @@ public:
 		writeBatch = std::make_unique<rocksdb::WriteBatch>();
 		dirtyShards = std::make_unique<std::set<PhysicalShard*>>();
 
-		TraceEvent(SevDebug, "ShardManagerInitEnd", this->logId).detail("DataPath", path);
+		TraceEvent(SevInfo, "ShardedRocksShardManagerInitEnd", this->logId).detail("DataPath", path);
 		return status;
 	}
 
@@ -712,7 +784,7 @@ public:
 
 		for (auto it = rangeIterator.begin(); it != rangeIterator.end(); ++it) {
 			if (it.value() == nullptr) {
-				TraceEvent(SevDebug, "ShardedRocksDB")
+				TraceEvent(SevVerbose, "ShardedRocksDB")
 				    .detail("Info", "ShardNotFound")
 				    .detail("BeginKey", range.begin)
 				    .detail("EndKey", range.end);
@@ -724,9 +796,10 @@ public:
 	}
 
 	PhysicalShard* addRange(KeyRange range, std::string id) {
-		TraceEvent(SevVerbose, "ShardedRocksAddRangeBegin", this->logId)
+		TraceEvent(SevInfo, "ShardedRocksAddRangeBegin", this->logId)
 		    .detail("Range", range)
 		    .detail("PhysicalShardID", id);
+
 		// Newly added range should not overlap with any existing range.
 		auto ranges = dataShardMap.intersectingRanges(range);
 
@@ -750,7 +823,7 @@ public:
 
 		validate();
 
-		TraceEvent(SevVerbose, "ShardedRocksAddRangeEnd", this->logId)
+		TraceEvent(SevInfo, "ShardedRocksAddRangeEnd", this->logId)
 		    .detail("Range", range)
 		    .detail("PhysicalShardID", id);
 
@@ -758,7 +831,7 @@ public:
 	}
 
 	std::vector<std::string> removeRange(KeyRange range) {
-		TraceEvent(SevVerbose, "ShardedRocksRemoveRangeBegin", this->logId).detail("Range", range);
+		TraceEvent(SevInfo, "ShardedRocksRemoveRangeBegin", this->logId).detail("Range", range);
 
 		std::vector<std::string> shardIds;
 
@@ -792,10 +865,13 @@ public:
 				if (existingShard->dataShards.size() == 0) {
 					TraceEvent(SevDebug, "ShardedRocksDB").detail("EmptyShardId", existingShard->id);
 					shardIds.push_back(existingShard->id);
+					existingShard->deleteTimeSec = now();
+					pendingDeletionShards.push_back(existingShard->id);
 					activePhysicalShardIds.erase(existingShard->id);
 				}
 				continue;
 			}
+
 			// Range modification could result in more than one segments. Remove the original segment key here.
 			existingShard->dataShards.erase(shardRange.begin.toString());
 			if (shardRange.begin < range.begin) {
@@ -826,18 +902,27 @@ public:
 
 		validate();
 
-		TraceEvent(SevVerbose, "ShardedRocksRemoveRangeEnd", this->logId).detail("Range", range);
+		TraceEvent(SevInfo, "ShardedRocksRemoveRangeEnd", this->logId).detail("Range", range);
 
 		return shardIds;
 	}
 
-	std::vector<std::shared_ptr<PhysicalShard>> cleanUpShards(const std::vector<std::string>& shardIds) {
+	std::vector<std::shared_ptr<PhysicalShard>> getPendingDeletionShards(double cleanUpDelay) {
 		std::vector<std::shared_ptr<PhysicalShard>> emptyShards;
-		for (const auto& id : shardIds) {
+		double currentTime = now();
+		while (!pendingDeletionShards.empty()) {
+			const auto& id = pendingDeletionShards.front();
 			auto it = physicalShards.find(id);
-			if (it != physicalShards.end() && it->second->dataShards.size() == 0) {
+			if (it == physicalShards.end() || it->second->dataShards.size() != 0) {
+				pendingDeletionShards.pop_front();
+				continue;
+			}
+			if (currentTime - it->second->deleteTimeSec > cleanUpDelay) {
+				pendingDeletionShards.pop_front();
 				emptyShards.push_back(it->second);
-				physicalShards.erase(it);
+				physicalShards.erase(id);
+			} else {
+				break;
 			}
 		}
 		return emptyShards;
@@ -849,7 +934,7 @@ public:
 			TraceEvent(SevError, "ShardedRocksDB").detail("Error", "write to non-exist shard").detail("WriteKey", key);
 			return;
 		}
-		TraceEvent(SevVerbose, "ShardManagerPut", this->logId)
+		TraceEvent(SevVerbose, "ShardedRocksShardManagerPut", this->logId)
 		    .detail("WriteKey", key)
 		    .detail("Value", value)
 		    .detail("MapRange", it.range())
@@ -859,7 +944,9 @@ public:
 		ASSERT(dirtyShards != nullptr);
 		writeBatch->Put(it.value()->physicalShard->cf, toSlice(key), toSlice(value));
 		dirtyShards->insert(it.value()->physicalShard);
-		TraceEvent(SevVerbose, "ShardManagerPutEnd", this->logId).detail("WriteKey", key).detail("Value", value);
+		TraceEvent(SevVerbose, "ShardedRocksShardManagerPutEnd", this->logId)
+		    .detail("WriteKey", key)
+		    .detail("Value", value);
 	}
 
 	void clear(KeyRef key) {
@@ -876,6 +963,7 @@ public:
 
 		for (auto it = rangeIterator.begin(); it != rangeIterator.end(); ++it) {
 			if (it.value() == nullptr) {
+				TraceEvent(SevDebug, "ShardedRocksDB").detail("ClearNonExistentRange", it.range());
 				continue;
 			}
 			writeBatch->DeleteRange(it.value()->physicalShard->cf, toSlice(range.begin), toSlice(range.end));
@@ -884,7 +972,7 @@ public:
 	}
 
 	void persistRangeMapping(KeyRangeRef range, bool isAdd) {
-		TraceEvent(SevDebug, "ShardedRocksDB")
+		TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
 		    .detail("Info", "RangeToPersist")
 		    .detail("BeginKey", range.begin)
 		    .detail("EndKey", range.end);
@@ -902,7 +990,7 @@ public:
 					writeBatch->Put(metadataShard->cf,
 					                getShardMappingKey(it.range().begin, shardMappingPrefix),
 					                it.value()->physicalShard->id);
-					TraceEvent(SevDebug, "ShardedRocksDB")
+					TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
 					    .detail("Action", "PersistRangeMapping")
 					    .detail("BeginKey", it.range().begin)
 					    .detail("EndKey", it.range().end)
@@ -911,7 +999,7 @@ public:
 				} else {
 					// Empty range.
 					writeBatch->Put(metadataShard->cf, getShardMappingKey(it.range().begin, shardMappingPrefix), "");
-					TraceEvent(SevDebug, "ShardedRocksDB")
+					TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
 					    .detail("Action", "PersistRangeMapping")
 					    .detail("BeginKey", it.range().begin)
 					    .detail("EndKey", it.range().end)
@@ -921,7 +1009,7 @@ public:
 			}
 		} else {
 			writeBatch->Put(metadataShard->cf, getShardMappingKey(range.begin, shardMappingPrefix), "");
-			TraceEvent(SevDebug, "ShardedRocksDB")
+			TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
 			    .detail("Action", "PersistRangeMapping")
 			    .detail("RemoveRange", "True")
 			    .detail("BeginKey", range.begin)
@@ -972,7 +1060,7 @@ public:
 		if (!s.ok()) {
 			logRocksDBError(s, "DestroyDB");
 		}
-		TraceEvent("RocksDB").detail("Info", "DBDestroyed");
+		TraceEvent("ShardedRocksDB", this->logId).detail("Info", "DBDestroyed");
 	}
 
 	rocksdb::DB* getDb() const { return db; }
@@ -997,9 +1085,9 @@ public:
 	}
 
 	void validate() {
-		TraceEvent(SevVerbose, "ValidateShardManager", this->logId);
+		TraceEvent(SevVerbose, "ShardedRocksValidateShardManager", this->logId);
 		for (auto s = dataShardMap.ranges().begin(); s != dataShardMap.ranges().end(); ++s) {
-			TraceEvent e(SevVerbose, "ValidateDataShardMap", this->logId);
+			TraceEvent e(SevVerbose, "ShardedRocksValidateDataShardMap", this->logId);
 			e.detail("Range", s->range());
 			const DataShard* shard = s->value();
 			e.detail("ShardAddress", reinterpret_cast<std::uintptr_t>(shard));
@@ -1008,6 +1096,13 @@ public:
 			} else {
 				e.detail("Shard", "Empty");
 			}
+			if (shard != nullptr) {
+				ASSERT(shard->range == static_cast<KeyRangeRef>(s->range()));
+				ASSERT(shard->physicalShard != nullptr);
+				auto it = shard->physicalShard->dataShards.find(shard->range.begin.toString());
+				ASSERT(it != shard->physicalShard->dataShards.end());
+				ASSERT(it->second.get() == shard);
+			}
 		}
 	}
 
@@ -1023,17 +1118,16 @@ private:
 	std::unique_ptr<std::set<PhysicalShard*>> dirtyShards;
 	KeyRangeMap<DataShard*> dataShardMap;
 	std::shared_ptr<PhysicalShard> metadataShard = nullptr;
+	std::deque<std::string> pendingDeletionShards;
 };
 
 class RocksDBMetrics {
 public:
-	RocksDBMetrics();
-	// Statistics
-	std::shared_ptr<rocksdb::Statistics> getStatsObjForRocksDB();
+	RocksDBMetrics(UID debugID, std::shared_ptr<rocksdb::Statistics> stats);
 	void logStats(rocksdb::DB* db);
 	// PerfContext
 	void resetPerfContext();
-	void setPerfContext(int index);
+	void collectPerfContext(int index);
 	void logPerfContext(bool ignoreZeroMetric);
 	// For Readers
 	Reference<Histogram> getReadRangeLatencyHistogram(int index);
@@ -1055,14 +1149,15 @@ public:
 	Reference<Histogram> getWriteHistogram();
 	Reference<Histogram> getDeleteCompactRangeHistogram();
 	// Stat for Memory Usage
-	void logMemUsagePerShard(std::string shardName, rocksdb::DB* db);
+	void logMemUsage(rocksdb::DB* db);
 
 private:
+	const UID debugID;
 	// Global Statistic Input to RocksDB DB instance
 	std::shared_ptr<rocksdb::Statistics> stats;
 	// Statistic Output from RocksDB
 	std::vector<std::tuple<const char*, uint32_t, uint64_t>> tickerStats;
-	std::vector<std::pair<const char*, std::string>> propertyStats;
+	std::vector<std::pair<const char*, std::string>> intPropertyStats;
 	// Iterator Pool Stats
 	std::unordered_map<std::string, uint64_t> readIteratorPoolStats;
 	// PerfContext
@@ -1144,9 +1239,8 @@ Reference<Histogram> RocksDBMetrics::getDeleteCompactRangeHistogram() {
 	return deleteCompactRangeHistogram;
 }
 
-RocksDBMetrics::RocksDBMetrics() {
-	stats = rocksdb::CreateDBStatistics();
-	stats->set_stats_level(rocksdb::kExceptHistogramOrTimers);
+RocksDBMetrics::RocksDBMetrics(UID debugID, std::shared_ptr<rocksdb::Statistics> stats)
+  : debugID(debugID), stats(stats) {
 	tickerStats = {
 		{ "StallMicros", rocksdb::STALL_MICROS, 0 },
 		{ "BytesRead", rocksdb::BYTES_READ, 0 },
@@ -1182,10 +1276,10 @@ RocksDBMetrics::RocksDBMetrics() {
 		{ "RowCacheHit", rocksdb::ROW_CACHE_HIT, 0 },
 		{ "RowCacheMiss", rocksdb::ROW_CACHE_MISS, 0 },
 		{ "CountIterSkippedKeys", rocksdb::NUMBER_ITER_SKIP, 0 },
+
 	};
-	propertyStats = {
-		// Zhe: TODO Aggregation
-		{ "NumCompactionsRunning", rocksdb::DB::Properties::kNumRunningCompactions },
+
+	intPropertyStats = {
 		{ "NumImmutableMemtables", rocksdb::DB::Properties::kNumImmutableMemTable },
 		{ "NumImmutableMemtablesFlushed", rocksdb::DB::Properties::kNumImmutableMemTableFlushed },
 		{ "IsMemtableFlushPending", rocksdb::DB::Properties::kMemTableFlushPending },
@@ -1194,27 +1288,25 @@ RocksDBMetrics::RocksDBMetrics() {
 		{ "NumRunningCompactions", rocksdb::DB::Properties::kNumRunningCompactions },
 		{ "CumulativeBackgroundErrors", rocksdb::DB::Properties::kBackgroundErrors },
 		{ "CurrentSizeActiveMemtable", rocksdb::DB::Properties::kCurSizeActiveMemTable },
-		{ "AllMemtablesBytes", rocksdb::DB::Properties::kCurSizeAllMemTables }, // for mem usage
+		{ "AllMemtablesBytes", rocksdb::DB::Properties::kCurSizeAllMemTables },
 		{ "ActiveMemtableBytes", rocksdb::DB::Properties::kSizeAllMemTables },
 		{ "CountEntriesActiveMemtable", rocksdb::DB::Properties::kNumEntriesActiveMemTable },
 		{ "CountEntriesImmutMemtables", rocksdb::DB::Properties::kNumEntriesImmMemTables },
 		{ "CountDeletesActiveMemtable", rocksdb::DB::Properties::kNumDeletesActiveMemTable },
 		{ "CountDeletesImmutMemtables", rocksdb::DB::Properties::kNumDeletesImmMemTables },
 		{ "EstimatedCountKeys", rocksdb::DB::Properties::kEstimateNumKeys },
-		{ "EstimateSstReaderBytes", rocksdb::DB::Properties::kEstimateTableReadersMem }, // for mem usage
+		{ "EstimateSstReaderBytes", rocksdb::DB::Properties::kEstimateTableReadersMem },
 		{ "CountActiveSnapshots", rocksdb::DB::Properties::kNumSnapshots },
 		{ "OldestSnapshotTime", rocksdb::DB::Properties::kOldestSnapshotTime },
 		{ "CountLiveVersions", rocksdb::DB::Properties::kNumLiveVersions },
 		{ "EstimateLiveDataSize", rocksdb::DB::Properties::kEstimateLiveDataSize },
 		{ "BaseLevel", rocksdb::DB::Properties::kBaseLevel },
 		{ "EstPendCompactBytes", rocksdb::DB::Properties::kEstimatePendingCompactionBytes },
-		{ "BlockCacheUsage", rocksdb::DB::Properties::kBlockCacheUsage }, // for mem usage
-		{ "BlockCachePinnedUsage", rocksdb::DB::Properties::kBlockCachePinnedUsage }, // for mem usage
-	};
-	std::unordered_map<std::string, uint64_t> readIteratorPoolStats = {
-		{ "NumReadIteratorsCreated", 0 },
-		{ "NumTimesReadIteratorsReused", 0 },
+		{ "BlockCacheUsage", rocksdb::DB::Properties::kBlockCacheUsage },
+		{ "BlockCachePinnedUsage", rocksdb::DB::Properties::kBlockCachePinnedUsage },
+		{ "LiveSstFilesSize", rocksdb::DB::Properties::kLiveSstFilesSize },
 	};
+
 	perfContextMetrics = {
 		{ "UserKeyComparisonCount", rocksdb_user_key_comparison_count, {} },
 		{ "BlockCacheHitCount", rocksdb_block_cache_hit_count, {} },
@@ -1329,65 +1421,48 @@ RocksDBMetrics::RocksDBMetrics() {
 	    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM, Histogram::Unit::microseconds);
 }
 
-std::shared_ptr<rocksdb::Statistics> RocksDBMetrics::getStatsObjForRocksDB() {
-	// Zhe: reserved for statistic of RocksDBMetrics per shard
-	// ASSERT(shard != nullptr && shard->stats != nullptr);
-	// return shard->stats;
-	ASSERT(stats != nullptr);
-	return stats;
-}
-
 void RocksDBMetrics::logStats(rocksdb::DB* db) {
-	TraceEvent e("RocksDBMetrics");
+	TraceEvent e(SevInfo, "ShardedRocksDBMetrics", debugID);
 	uint64_t stat;
 	for (auto& [name, ticker, cumulation] : tickerStats) {
 		stat = stats->getTickerCount(ticker);
 		e.detail(name, stat - cumulation);
 		cumulation = stat;
 	}
-	for (auto& [name, property] : propertyStats) { // Zhe: TODO aggregation
+	for (auto& [name, property] : intPropertyStats) {
 		stat = 0;
-		ASSERT(db->GetIntProperty(property, &stat));
+		ASSERT(db->GetAggregatedIntProperty(property, &stat));
 		e.detail(name, stat);
 	}
-	/*
-	stat = readIterPool->numReadIteratorsCreated();
-	e.detail("NumReadIteratorsCreated", stat - readIteratorPoolStats["NumReadIteratorsCreated"]);
-	readIteratorPoolStats["NumReadIteratorsCreated"] = stat;
-	stat = readIterPool->numTimesReadIteratorsReused();
-	e.detail("NumTimesReadIteratorsReused", stat - readIteratorPoolStats["NumTimesReadIteratorsReused"]);
-	readIteratorPoolStats["NumTimesReadIteratorsReused"] = stat;
-	*/
 }
 
-void RocksDBMetrics::logMemUsagePerShard(std::string shardName, rocksdb::DB* db) {
-	TraceEvent e("RocksDBShardMemMetrics");
+void RocksDBMetrics::logMemUsage(rocksdb::DB* db) {
+	TraceEvent e(SevInfo, "ShardedRocksDBMemMetrics", debugID);
 	uint64_t stat;
 	ASSERT(db != nullptr);
-	ASSERT(db->GetIntProperty(rocksdb::DB::Properties::kBlockCacheUsage, &stat));
+	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kBlockCacheUsage, &stat));
 	e.detail("BlockCacheUsage", stat);
-	ASSERT(db->GetIntProperty(rocksdb::DB::Properties::kEstimateTableReadersMem, &stat));
+	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimateTableReadersMem, &stat));
 	e.detail("EstimateSstReaderBytes", stat);
-	ASSERT(db->GetIntProperty(rocksdb::DB::Properties::kCurSizeAllMemTables, &stat));
+	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kCurSizeAllMemTables, &stat));
 	e.detail("AllMemtablesBytes", stat);
-	ASSERT(db->GetIntProperty(rocksdb::DB::Properties::kBlockCachePinnedUsage, &stat));
+	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kBlockCachePinnedUsage, &stat));
 	e.detail("BlockCachePinnedUsage", stat);
-	e.detail("Name", shardName);
 }
 
 void RocksDBMetrics::resetPerfContext() {
-	rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
+	rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableCount);
 	rocksdb::get_perf_context()->Reset();
 }
 
-void RocksDBMetrics::setPerfContext(int index) {
+void RocksDBMetrics::collectPerfContext(int index) {
 	for (auto& [name, metric, vals] : perfContextMetrics) {
 		vals[index] = getRocksdbPerfcontextMetric(metric);
 	}
 }
 
 void RocksDBMetrics::logPerfContext(bool ignoreZeroMetric) {
-	TraceEvent e("RocksDBPerfContextMetrics");
+	TraceEvent e(SevInfo, "ShardedRocksDBPerfContextMetrics", debugID);
 	e.setMaxEventLength(20000);
 	for (auto& [name, metric, vals] : perfContextMetrics) {
 		uint64_t s = 0;
@@ -1396,12 +1471,7 @@ void RocksDBMetrics::logPerfContext(bool ignoreZeroMetric) {
 		}
 		if (ignoreZeroMetric && s == 0)
 			continue;
-		for (int i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; i++) {
-			if (vals[i] != 0)
-				e.detail("RD" + std::to_string(i) + name, vals[i]);
-		}
-		if (vals[SERVER_KNOBS->ROCKSDB_READ_PARALLELISM] != 0)
-			e.detail("WR" + (std::string)name, vals[SERVER_KNOBS->ROCKSDB_READ_PARALLELISM]);
+		e.detail(name, s);
 	}
 }
 
@@ -1549,17 +1619,30 @@ uint64_t RocksDBMetrics::getRocksdbPerfcontextMetric(int metric) {
 	return 0;
 }
 
-ACTOR Future<Void> rocksDBAggregatedMetricsLogger(std::shared_ptr<RocksDBMetrics> rocksDBMetrics, rocksdb::DB* db) {
-	loop {
-		wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
-		/*
-		if (SERVER_KNOBS->ROCKSDB_ENABLE_STATISTIC) {
-		    rocksDBMetrics->logStats(db);
+ACTOR Future<Void> rocksDBAggregatedMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
+                                                  Future<Void> openFuture,
+                                                  std::shared_ptr<RocksDBMetrics> rocksDBMetrics,
+                                                  ShardManager* shardManager) {
+	try {
+		wait(openFuture);
+		state rocksdb::DB* db = shardManager->getDb();
+		loop {
+			wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
+			if (rState->closing) {
+				break;
+			}
+			rocksDBMetrics->logStats(db);
+			rocksDBMetrics->logMemUsage(db);
+			if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE != 0) {
+				rocksDBMetrics->logPerfContext(true);
+			}
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled) {
+			TraceEvent(SevError, "ShardedRocksDBMetricsError").errorUnsuppressed(e);
 		}
-		if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE != 0) {
-		    rocksDBMetrics->logPerfContext(true);
-		}*/
 	}
+	return Void();
 }
 
 struct ShardedRocksDBKeyValueStore : IKeyValueStore {
@@ -1604,6 +1687,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* columnFamilyMap;
 		std::shared_ptr<RocksDBMetrics> rocksDBMetrics;
 		std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
+		double sampleStartTime;
 
 		explicit Writer(UID logId,
 		                int threadIndex,
@@ -1617,14 +1701,25 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		                          10, // fairness
 		                          rocksdb::RateLimiter::Mode::kWritesOnly,
 		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE)
-		                    : nullptr) {}
+		                    : nullptr),
+		    sampleStartTime(now()) {}
 
 		~Writer() override {}
 
 		void init() override {}
 
+		void sample() {
+			if (SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL > 0 &&
+			    now() - sampleStartTime >= SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL) {
+				rocksDBMetrics->collectPerfContext(threadIndex);
+				rocksDBMetrics->resetPerfContext();
+				sampleStartTime = now();
+			}
+		}
+
 		struct OpenAction : TypedAction<Writer, OpenAction> {
 			ShardManager* shardManager;
+			rocksdb::Options dbOptions;
 			ThreadReturnPromise<Void> done;
 			Optional<Future<Void>>& metrics;
 			const FlowLock* readLock;
@@ -1632,25 +1727,27 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			std::shared_ptr<RocksDBErrorListener> errorListener;
 
 			OpenAction(ShardManager* shardManager,
+			           rocksdb::Options dbOptions,
 			           Optional<Future<Void>>& metrics,
 			           const FlowLock* readLock,
 			           const FlowLock* fetchLock,
 			           std::shared_ptr<RocksDBErrorListener> errorListener)
-			  : shardManager(shardManager), metrics(metrics), readLock(readLock), fetchLock(fetchLock),
-			    errorListener(errorListener) {}
+			  : shardManager(shardManager), dbOptions(dbOptions), metrics(metrics), readLock(readLock),
+			    fetchLock(fetchLock), errorListener(errorListener) {}
 
 			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
 		};
 
 		void action(OpenAction& a) {
-			auto status = a.shardManager->init();
+			auto status = a.shardManager->init(a.dbOptions);
+
 			if (!status.ok()) {
 				logRocksDBError(status, "Open");
 				a.done.sendError(statusToError(status));
 				return;
 			}
 
-			TraceEvent(SevInfo, "RocksDB").detail("Method", "Open");
+			TraceEvent(SevInfo, "ShardedRocksDB").detail("Method", "Open");
 			a.done.send(Void());
 		}
 
@@ -1698,7 +1795,6 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			ThreadReturnPromise<Void> done;
 			double startTime;
 			bool getHistograms;
-			bool getPerfContext;
 			bool logShardMemUsage;
 			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
 			CommitAction(rocksdb::DB* db,
@@ -1713,12 +1809,6 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				} else {
 					getHistograms = false;
 				}
-				if ((SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE != 0) &&
-				    (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE)) {
-					getPerfContext = true;
-				} else {
-					getPerfContext = false;
-				}
 			}
 		};
 
@@ -1787,9 +1877,6 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		}
 
 		void action(CommitAction& a) {
-			if (a.getPerfContext) {
-				rocksDBMetrics->resetPerfContext();
-			}
 			double commitBeginTime;
 			if (a.getHistograms) {
 				commitBeginTime = timer_monotonic();
@@ -1807,12 +1894,15 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			}
 
 			a.done.send(Void());
-			for (const auto& [id, range] : deletes) {
-				auto cf = columnFamilyMap->find(id);
-				ASSERT(cf != columnFamilyMap->end());
-				auto begin = toSlice(range.begin);
-				auto end = toSlice(range.end);
-				ASSERT(a.db->SuggestCompactRange(cf->second, &begin, &end).ok());
+			if (SERVER_KNOBS->ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE) {
+				for (const auto& [id, range] : deletes) {
+					auto cf = columnFamilyMap->find(id);
+					ASSERT(cf != columnFamilyMap->end());
+					auto begin = toSlice(range.begin);
+					auto end = toSlice(range.end);
+
+					ASSERT(a.db->SuggestCompactRange(cf->second, &begin, &end).ok());
+				}
 			}
 
 			if (a.getHistograms) {
@@ -1821,9 +1911,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				rocksDBMetrics->getCommitLatencyHistogram()->sampleSeconds(currTime - a.startTime);
 			}
 
-			if (a.getPerfContext) {
-				rocksDBMetrics->setPerfContext(threadIndex);
-			}
+			sample();
 		}
 
 		struct CloseAction : TypedAction<Writer, CloseAction> {
@@ -1841,7 +1929,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			} else {
 				a.shardManager->closeAllShards();
 			}
-			TraceEvent(SevInfo, "RocksDB").detail("Method", "Close");
+			TraceEvent(SevInfo, "ShardedRocksDB").detail("Method", "Close");
 			a.done.send(Void());
 		}
 	};
@@ -1853,51 +1941,44 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		double readRangeTimeout;
 		int threadIndex;
 		std::shared_ptr<RocksDBMetrics> rocksDBMetrics;
+		double sampleStartTime;
 
 		explicit Reader(UID logId, int threadIndex, std::shared_ptr<RocksDBMetrics> rocksDBMetrics)
-		  : logId(logId), threadIndex(threadIndex), rocksDBMetrics(rocksDBMetrics) {
-			if (g_network->isSimulated()) {
-				// In simulation, increasing the read operation timeouts to 5 minutes, as some of the tests have
-				// very high load and single read thread cannot process all the load within the timeouts.
-				readValueTimeout = 5 * 60;
-				readValuePrefixTimeout = 5 * 60;
-				readRangeTimeout = 5 * 60;
-			} else {
-				readValueTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_TIMEOUT;
-				readValuePrefixTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_PREFIX_TIMEOUT;
-				readRangeTimeout = SERVER_KNOBS->ROCKSDB_READ_RANGE_TIMEOUT;
-			}
+		  : logId(logId), threadIndex(threadIndex), rocksDBMetrics(rocksDBMetrics), sampleStartTime(now()) {
+			readValueTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_TIMEOUT;
+			readValuePrefixTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_PREFIX_TIMEOUT;
+			readRangeTimeout = SERVER_KNOBS->ROCKSDB_READ_RANGE_TIMEOUT;
 		}
 
 		void init() override {}
 
+		void sample() {
+			if (SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL > 0 &&
+			    now() - sampleStartTime >= SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL) {
+				rocksDBMetrics->collectPerfContext(threadIndex);
+				rocksDBMetrics->resetPerfContext();
+				sampleStartTime = now();
+			}
+		}
 		struct ReadValueAction : TypedAction<Reader, ReadValueAction> {
 			Key key;
 			PhysicalShard* shard;
 			Optional<UID> debugID;
 			double startTime;
 			bool getHistograms;
-			bool getPerfContext;
 			bool logShardMemUsage;
 			ThreadReturnPromise<Optional<Value>> result;
 
 			ReadValueAction(KeyRef key, PhysicalShard* shard, Optional<UID> debugID)
 			  : key(key), shard(shard), debugID(debugID), startTime(timer_monotonic()),
 			    getHistograms(
-			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false),
-			    getPerfContext(
-			        (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE != 0) &&
-			                (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE)
-			            ? true
-			            : false) {}
+			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false) {
+			}
 
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
 		};
 
 		void action(ReadValueAction& a) {
-			if (a.getPerfContext) {
-				rocksDBMetrics->resetPerfContext();
-			}
 			double readBeginTime = timer_monotonic();
 			if (a.getHistograms) {
 				rocksDBMetrics->getReadValueQueueWaitHistogram(threadIndex)->sampleSeconds(readBeginTime - a.startTime);
@@ -1908,7 +1989,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				traceBatch.get().addEvent("GetValueDebug", a.debugID.get().first(), "Reader.Before");
 			}
 			if (readBeginTime - a.startTime > readValueTimeout) {
-				TraceEvent(SevWarn, "RocksDBError")
+				TraceEvent(SevWarn, "ShardedRocksDBError")
 				    .detail("Error", "Read value request timedout")
 				    .detail("Method", "ReadValueAction")
 				    .detail("Timeout value", readValueTimeout);
@@ -1950,9 +2031,8 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				rocksDBMetrics->getReadValueActionHistogram(threadIndex)->sampleSeconds(currTime - readBeginTime);
 				rocksDBMetrics->getReadValueLatencyHistogram(threadIndex)->sampleSeconds(currTime - a.startTime);
 			}
-			if (a.getPerfContext) {
-				rocksDBMetrics->setPerfContext(threadIndex);
-			}
+
+			sample();
 		}
 
 		struct ReadValuePrefixAction : TypedAction<Reader, ReadValuePrefixAction> {
@@ -1962,26 +2042,18 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			Optional<UID> debugID;
 			double startTime;
 			bool getHistograms;
-			bool getPerfContext;
 			bool logShardMemUsage;
 			ThreadReturnPromise<Optional<Value>> result;
 
 			ReadValuePrefixAction(Key key, int maxLength, PhysicalShard* shard, Optional<UID> debugID)
 			  : key(key), maxLength(maxLength), shard(shard), debugID(debugID), startTime(timer_monotonic()),
-			    getHistograms(
-			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false),
-			    getPerfContext(
-			        (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE != 0) &&
-			                (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE)
-			            ? true
-			            : false){};
+			    getHistograms((deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE)
+			                      ? true
+			                      : false){};
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
 		};
 
 		void action(ReadValuePrefixAction& a) {
-			if (a.getPerfContext) {
-				rocksDBMetrics->resetPerfContext();
-			}
 			double readBeginTime = timer_monotonic();
 			if (a.getHistograms) {
 				rocksDBMetrics->getReadPrefixQueueWaitHistogram(threadIndex)
@@ -1995,7 +2067,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				                          "Reader.Before"); //.detail("TaskID", g_network->getCurrentTask());
 			}
 			if (readBeginTime - a.startTime > readValuePrefixTimeout) {
-				TraceEvent(SevWarn, "RocksDBError")
+				TraceEvent(SevWarn, "ShardedRocksDBError")
 				    .detail("Error", "Read value prefix request timedout")
 				    .detail("Method", "ReadValuePrefixAction")
 				    .detail("Timeout value", readValuePrefixTimeout);
@@ -2039,9 +2111,8 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				rocksDBMetrics->getReadPrefixActionHistogram(threadIndex)->sampleSeconds(currTime - readBeginTime);
 				rocksDBMetrics->getReadPrefixLatencyHistogram(threadIndex)->sampleSeconds(currTime - a.startTime);
 			}
-			if (a.getPerfContext) {
-				rocksDBMetrics->setPerfContext(threadIndex);
-			}
+
+			sample();
 		}
 
 		struct ReadRangeAction : TypedAction<Reader, ReadRangeAction>, FastAllocated<ReadRangeAction> {
@@ -2050,35 +2121,28 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			int rowLimit, byteLimit;
 			double startTime;
 			bool getHistograms;
-			bool getPerfContext;
 			bool logShardMemUsage;
 			ThreadReturnPromise<RangeResult> result;
 			ReadRangeAction(KeyRange keys, std::vector<DataShard*> shards, int rowLimit, int byteLimit)
 			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), startTime(timer_monotonic()),
 			    getHistograms(
-			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false),
-			    getPerfContext(
-			        (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE != 0) &&
-			                (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE)
-			            ? true
-			            : false) {
+			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false) {
 				for (const DataShard* shard : shards) {
-					shardRanges.emplace_back(shard->physicalShard, keys & shard->range);
+					if (shard != nullptr) {
+						shardRanges.emplace_back(shard->physicalShard, keys & shard->range);
+					}
 				}
 			}
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
 		};
 
 		void action(ReadRangeAction& a) {
-			if (a.getPerfContext) {
-				rocksDBMetrics->resetPerfContext();
-			}
 			double readBeginTime = timer_monotonic();
 			if (a.getHistograms) {
 				rocksDBMetrics->getReadRangeQueueWaitHistogram(threadIndex)->sampleSeconds(readBeginTime - a.startTime);
 			}
 			if (readBeginTime - a.startTime > readRangeTimeout) {
-				TraceEvent(SevWarn, "KVSReadTimeout")
+				TraceEvent(SevWarn, "ShardedRocksKVSReadTimeout")
 				    .detail("Error", "Read range request timedout")
 				    .detail("Method", "ReadRangeAction")
 				    .detail("Timeout value", readRangeTimeout);
@@ -2098,14 +2162,19 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				std::reverse(a.shardRanges.begin(), a.shardRanges.end());
 			}
 
-			// TODO: consider multi-thread reads. It's possible to read multiple shards in parallel. However, the number
-			// of rows to read needs to be calculated based on the previous read result. We may read more than we
-			// expected when parallel read is used when the previous result is not available. It's unlikely to get to
-			// performance improvement when the actual number of rows to read is very small.
+			// TODO: consider multi-thread reads. It's possible to read multiple shards in parallel. However, the
+			// number of rows to read needs to be calculated based on the previous read result. We may read more
+			// than we expected when parallel read is used when the previous result is not available. It's unlikely
+			// to get to performance improvement when the actual number of rows to read is very small.
 			int accumulatedBytes = 0;
 			int numShards = 0;
 			for (auto& [shard, range] : a.shardRanges) {
-				ASSERT(shard != nullptr && shard->initialized());
+				if (shard == nullptr || !shard->initialized()) {
+					TraceEvent(SevWarn, "ShardedRocksReadRangeShardNotReady", logId)
+					    .detail("Range", range)
+					    .detail("Reason", shard == nullptr ? "Not Exist" : "Not Initialized");
+					continue;
+				}
 				auto bytesRead = readRangeInDb(shard, range, rowLimit, byteLimit, &result);
 				if (bytesRead < 0) {
 					// Error reading an instance.
@@ -2120,10 +2189,6 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				}
 			}
 
-			Histogram::getHistogram(
-			    ROCKSDBSTORAGE_HISTOGRAM_GROUP, "ShardedRocksDBNumShardsInRangeRead"_sr, Histogram::Unit::countLinear)
-			    ->sample(numShards);
-
 			result.more =
 			    (result.size() == a.rowLimit) || (result.size() == -a.rowLimit) || (accumulatedBytes >= a.byteLimit);
 			if (result.more) {
@@ -2135,9 +2200,8 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				rocksDBMetrics->getReadRangeActionHistogram(threadIndex)->sampleSeconds(currTime - readBeginTime);
 				rocksDBMetrics->getReadRangeLatencyHistogram(threadIndex)->sampleSeconds(currTime - a.startTime);
 			}
-			if (a.getPerfContext) {
-				rocksDBMetrics->setPerfContext(threadIndex);
-			}
+
+			sample();
 		}
 	};
 
@@ -2158,17 +2222,19 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 	    numReadWaiters(SERVER_KNOBS->ROCKSDB_READ_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
 	    numFetchWaiters(SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_SOFT_MAX),
 	    errorListener(std::make_shared<RocksDBErrorListener>()), errorFuture(errorListener->getFuture()),
-	    shardManager(path, id), rocksDBMetrics(new RocksDBMetrics()) {
-		// In simluation, run the reader/writer threads as Coro threads (i.e. in the network thread. The storage engine
-		// is still multi-threaded as background compaction threads are still present. Reads/writes to disk will also
-		// block the network thread in a way that would be unacceptable in production but is a necessary evil here. When
-		// performing the reads in background threads in simulation, the event loop thinks there is no work to do and
-		// advances time faster than 1 sec/sec. By the time the blocking read actually finishes, simulation has advanced
-		// time by more than 5 seconds, so every read fails with a transaction_too_old error. Doing blocking IO on the
-		// main thread solves this issue. There are almost certainly better fixes, but my goal was to get a less
-		// invasive change merged first and work on a more realistic version if/when we think that would provide
-		// substantially more confidence in the correctness.
-		// TODO: Adapt the simulation framework to not advance time quickly when background reads/writes are occurring.
+	    shardManager(path, id), dbOptions(getOptions()),
+	    rocksDBMetrics(std::make_shared<RocksDBMetrics>(id, dbOptions.statistics)) {
+		// In simluation, run the reader/writer threads as Coro threads (i.e. in the network thread. The storage
+		// engine is still multi-threaded as background compaction threads are still present. Reads/writes to disk
+		// will also block the network thread in a way that would be unacceptable in production but is a necessary
+		// evil here. When performing the reads in background threads in simulation, the event loop thinks there is
+		// no work to do and advances time faster than 1 sec/sec. By the time the blocking read actually finishes,
+		// simulation has advanced time by more than 5 seconds, so every read fails with a transaction_too_old
+		// error. Doing blocking IO on the main thread solves this issue. There are almost certainly better fixes,
+		// but my goal was to get a less invasive change merged first and work on a more realistic version if/when
+		// we think that would provide substantially more confidence in the correctness.
+		// TODO: Adapt the simulation framework to not advance time quickly when background reads/writes are
+		// occurring.
 		if (g_network->isSimulated()) {
 			writeThread = CoroThreadPool::createThreadPool();
 			readThreads = CoroThreadPool::createThreadPool();
@@ -2177,7 +2243,8 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			readThreads = createGenericThreadPool();
 		}
 		writeThread->addThread(new Writer(id, 0, shardManager.getColumnFamilyMap(), rocksDBMetrics), "fdb-rocksdb-wr");
-		TraceEvent("RocksDBReadThreads").detail("KnobRocksDBReadParallelism", SERVER_KNOBS->ROCKSDB_READ_PARALLELISM);
+		TraceEvent("ShardedRocksDBReadThreads", id)
+		    .detail("KnobRocksDBReadParallelism", SERVER_KNOBS->ROCKSDB_READ_PARALLELISM);
 		for (unsigned i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; ++i) {
 			readThreads->addThread(new Reader(id, i, rocksDBMetrics), "fdb-rocksdb-re");
 		}
@@ -2190,6 +2257,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		// The metrics future retains a reference to the DB, so stop it before we delete it.
 		self->metrics.reset();
 		self->refreshHolder.cancel();
+		self->cleanUpJob.cancel();
 
 		wait(self->readThreads->stop());
 		auto a = new Writer::CloseAction(&self->shardManager, deleteOnClose);
@@ -2210,18 +2278,22 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 
 	KeyValueStoreType getType() const override { return KeyValueStoreType(KeyValueStoreType::SSD_SHARDED_ROCKSDB); }
 
+	bool shardAware() const override { return true; }
+
 	Future<Void> init() override {
 		if (openFuture.isValid()) {
 			return openFuture;
-			// Restore durable state if KVS is open. KVS will be re-initialized during rollback. To avoid the cost of
-			// opening and closing multiple rocksdb instances, we reconcile the shard map using persist shard mapping
-			// data.
+			// Restore durable state if KVS is open. KVS will be re-initialized during rollback. To avoid the cost
+			// of opening and closing multiple rocksdb instances, we reconcile the shard map using persist shard
+			// mapping data.
 		} else {
 			auto a = std::make_unique<Writer::OpenAction>(
-			    &shardManager, metrics, &readSemaphore, &fetchSemaphore, errorListener);
+			    &shardManager, dbOptions, metrics, &readSemaphore, &fetchSemaphore, errorListener);
 			openFuture = a->done.getFuture();
-			this->metrics = ShardManager::shardMetricsLogger(this->rState, openFuture, &shardManager);
+			this->metrics = ShardManager::shardMetricsLogger(this->rState, openFuture, &shardManager) &&
+			                rocksDBAggregatedMetricsLogger(this->rState, openFuture, rocksDBMetrics, &shardManager);
 			this->refreshHolder = refreshReadIteratorPools(this->rState, openFuture, shardManager.getAllShards());
+			this->cleanUpJob = emptyShardCleaner(this->rState, openFuture, &shardManager, writeThread);
 			writeThread->post(a.release());
 			return openFuture;
 		}
@@ -2267,8 +2339,8 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 
 	// We don't throttle eager reads and reads to the FF keyspace because FDB struggles when those reads fail.
 	// Thus far, they have been low enough volume to not cause an issue.
-	static bool shouldThrottle(IKeyValueStore::ReadType type, KeyRef key) {
-		return type != IKeyValueStore::ReadType::EAGER && !(key.startsWith(systemKeys.begin));
+	static bool shouldThrottle(ReadType type, KeyRef key) {
+		return type != ReadType::EAGER && !(key.startsWith(systemKeys.begin));
 	}
 
 	ACTOR template <class Action>
@@ -2289,14 +2361,24 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		return result;
 	}
 
-	Future<Optional<Value>> readValue(KeyRef key, IKeyValueStore::ReadType type, Optional<UID> debugID) override {
+	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options) override {
 		auto* shard = shardManager.getDataShard(key);
 		if (shard == nullptr || !shard->physicalShard->initialized()) {
 			// TODO: read non-exist system key range should not cause an error.
-			TraceEvent(SevError, "ShardedRocksDB").detail("Detail", "Read non-exist key range").detail("ReadKey", key);
+			TraceEvent(SevWarnAlways, "ShardedRocksDB", this->id)
+			    .detail("Detail", "Read non-exist key range")
+			    .detail("ReadKey", key);
 			return Optional<Value>();
 		}
 
+		ReadType type = ReadType::NORMAL;
+		Optional<UID> debugID;
+
+		if (options.present()) {
+			type = options.get().type;
+			debugID = options.get().debugID;
+		}
+
 		if (!shouldThrottle(type, key)) {
 			auto a = new Reader::ReadValueAction(key, shard->physicalShard, debugID);
 			auto res = a->result.getFuture();
@@ -2304,27 +2386,32 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			return res;
 		}
 
-		auto& semaphore = (type == IKeyValueStore::ReadType::FETCH) ? fetchSemaphore : readSemaphore;
-		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
+		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
+		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
 		auto a = std::make_unique<Reader::ReadValueAction>(key, shard->physicalShard, debugID);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 
-	Future<Optional<Value>> readValuePrefix(KeyRef key,
-	                                        int maxLength,
-	                                        IKeyValueStore::ReadType type,
-	                                        Optional<UID> debugID) override {
+	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override {
 		auto* shard = shardManager.getDataShard(key);
 		if (shard == nullptr || !shard->physicalShard->initialized()) {
 			// TODO: read non-exist system key range should not cause an error.
-			TraceEvent(SevWarnAlways, "ShardedRocksDB")
+			TraceEvent(SevWarnAlways, "ShardedRocksDB", this->id)
 			    .detail("Detail", "Read non-exist key range")
 			    .detail("ReadKey", key);
 			return Optional<Value>();
 		}
 
+		ReadType type = ReadType::NORMAL;
+		Optional<UID> debugID;
+
+		if (options.present()) {
+			type = options.get().type;
+			debugID = options.get().debugID;
+		}
+
 		if (!shouldThrottle(type, key)) {
 			auto a = new Reader::ReadValuePrefixAction(key, maxLength, shard->physicalShard, debugID);
 			auto res = a->result.getFuture();
@@ -2332,8 +2419,8 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			return res;
 		}
 
-		auto& semaphore = (type == IKeyValueStore::ReadType::FETCH) ? fetchSemaphore : readSemaphore;
-		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
+		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
+		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
 		auto a = std::make_unique<Reader::ReadValuePrefixAction>(key, maxLength, shard->physicalShard, debugID);
@@ -2363,14 +2450,13 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 	Future<RangeResult> readRange(KeyRangeRef keys,
 	                              int rowLimit,
 	                              int byteLimit,
-	                              IKeyValueStore::ReadType type) override {
+	                              Optional<ReadOptions> options = Optional<ReadOptions>()) override {
 		TraceEvent(SevVerbose, "ShardedRocksReadRangeBegin", this->id).detail("Range", keys);
 		auto shards = shardManager.getDataShardsByRange(keys);
 
-		for (DataShard* shard : shards) {
-			if (shard == nullptr || !shard->physicalShard->initialized()) {
-				return RangeResult();
-			}
+		ReadType type = ReadType::NORMAL;
+		if (options.present()) {
+			type = options.get().type;
 		}
 
 		if (!shouldThrottle(type, keys.begin)) {
@@ -2380,14 +2466,42 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			return res;
 		}
 
-		auto& semaphore = (type == IKeyValueStore::ReadType::FETCH) ? fetchSemaphore : readSemaphore;
-		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
+		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
+		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 		checkWaiters(semaphore, maxWaiters);
 
 		auto a = std::make_unique<Reader::ReadRangeAction>(keys, shards, rowLimit, byteLimit);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 
+	ACTOR static Future<Void> emptyShardCleaner(std::shared_ptr<ShardedRocksDBState> rState,
+	                                            Future<Void> openFuture,
+	                                            ShardManager* shardManager,
+	                                            Reference<IThreadPool> writeThread) {
+		state double cleanUpDelay = SERVER_KNOBS->ROCKSDB_PHYSICAL_SHARD_CLEAN_UP_DELAY;
+		state double cleanUpPeriod = cleanUpDelay * 2;
+		try {
+			wait(openFuture);
+			loop {
+				wait(delay(cleanUpPeriod));
+				if (rState->closing) {
+					break;
+				}
+				auto shards = shardManager->getPendingDeletionShards(cleanUpDelay);
+				auto a = new Writer::RemoveShardAction(shards);
+				Future<Void> f = a->done.getFuture();
+				writeThread->post(a);
+				TraceEvent(SevInfo, "ShardedRocksDB").detail("DeleteEmptyShards", shards.size());
+				wait(f);
+			}
+		} catch (Error& e) {
+			if (e.code() != error_code_actor_cancelled) {
+				TraceEvent(SevError, "DeleteEmptyShardsError").errorUnsuppressed(e);
+			}
+		}
+		return Void();
+	}
+
 	StorageBytes getStorageBytes() const override {
 		uint64_t live = 0;
 		ASSERT(shardManager.getDb()->GetAggregatedIntProperty(rocksdb::DB::Properties::kLiveSstFilesSize, &live));
@@ -2404,19 +2518,12 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		return shardManager.persistRangeMapping(range, isAdd);
 	}
 
-	Future<Void> cleanUpShardsIfNeeded(const std::vector<std::string>& shardIds) override {
-		auto shards = shardManager.cleanUpShards(shardIds);
-		auto a = new Writer::RemoveShardAction(shards);
-		Future<Void> res = a->done.getFuture();
-		writeThread->post(a);
-		return res;
-	}
-
 	// Used for debugging shard mapping issue.
 	std::vector<std::pair<KeyRange, std::string>> getDataMapping() { return shardManager.getDataMapping(); }
 
 	std::shared_ptr<ShardedRocksDBState> rState;
 	ShardManager shardManager;
+	rocksdb::Options dbOptions;
 	std::shared_ptr<RocksDBMetrics> rocksDBMetrics;
 	std::string path;
 	UID id;
@@ -2433,6 +2540,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 	int numFetchWaiters;
 	Counters counters;
 	Future<Void> refreshHolder;
+	Future<Void> cleanUpJob;
 };
 
 } // namespace
@@ -2447,7 +2555,7 @@ IKeyValueStore* keyValueStoreShardedRocksDB(std::string const& path,
 #ifdef SSD_ROCKSDB_EXPERIMENTAL
 	return new ShardedRocksDBKeyValueStore(path, logID);
 #else
-	TraceEvent(SevError, "RocksDBEngineInitFailure").detail("Reason", "Built without RocksDB");
+	TraceEvent(SevError, "ShardedRocksDBEngineInitFailure").detail("Reason", "Built without RocksDB");
 	ASSERT(false);
 	return nullptr;
 #endif // SSD_ROCKSDB_EXPERIMENTAL
@@ -2490,8 +2598,10 @@ TEST_CASE("noSim/ShardedRocksDB/SingleShardRead") {
 
 	Optional<Value> val = wait(kvStore->readValue("a"_sr));
 	ASSERT(Optional<Value>("foo"_sr) == val);
-	Optional<Value> val = wait(kvStore->readValue("ac"_sr));
-	ASSERT(Optional<Value>("bar"_sr) == val);
+	{
+		Optional<Value> val = wait(kvStore->readValue("ac"_sr));
+		ASSERT(Optional<Value>("bar"_sr) == val);
+	}
 
 	Future<Void> closed = kvStore->onClosed();
 	kvStore->dispose();
@@ -2511,9 +2621,10 @@ TEST_CASE("noSim/ShardedRocksDB/RangeOps") {
 	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("0"_sr, "3"_sr), "shard-1"));
 	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("4"_sr, "7"_sr), "shard-2"));
 
-	kvStore->persistRangeMapping(KeyRangeRef("0"_sr, "7"_sr), true);
 	wait(waitForAll(addRangeFutures));
 
+	kvStore->persistRangeMapping(KeyRangeRef("0"_sr, "7"_sr), true);
+
 	// write to shard 1
 	state RangeResult expectedRows;
 	for (int i = 0; i < 30; ++i) {
@@ -2547,27 +2658,28 @@ TEST_CASE("noSim/ShardedRocksDB/RangeOps") {
 
 	// Range read
 	// Read forward full range.
-	RangeResult result =
-	    wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000, IKeyValueStore::ReadType::NORMAL));
+	RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000));
 	ASSERT_EQ(result.size(), expectedRows.size());
 	for (int i = 0; i < expectedRows.size(); ++i) {
 		ASSERT(result[i] == expectedRows[i]);
 	}
 
 	// Read backward full range.
-	RangeResult result =
-	    wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), -1000, 10000, IKeyValueStore::ReadType::NORMAL));
-	ASSERT_EQ(result.size(), expectedRows.size());
-	for (int i = 0; i < expectedRows.size(); ++i) {
-		ASSERT(result[i] == expectedRows[59 - i]);
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), -1000, 10000));
+		ASSERT_EQ(result.size(), expectedRows.size());
+		for (int i = 0; i < expectedRows.size(); ++i) {
+			ASSERT(result[i] == expectedRows[59 - i]);
+		}
 	}
 
 	// Forward with row limit.
-	RangeResult result =
-	    wait(kvStore->readRange(KeyRangeRef("2"_sr, "6"_sr), 10, 10000, IKeyValueStore::ReadType::NORMAL));
-	ASSERT_EQ(result.size(), 10);
-	for (int i = 0; i < 10; ++i) {
-		ASSERT(result[i] == expectedRows[20 + i]);
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("2"_sr, "6"_sr), 10, 10000));
+		ASSERT_EQ(result.size(), 10);
+		for (int i = 0; i < 10; ++i) {
+			ASSERT(result[i] == expectedRows[20 + i]);
+		}
 	}
 
 	// Add another range on shard-1.
@@ -2583,35 +2695,40 @@ TEST_CASE("noSim/ShardedRocksDB/RangeOps") {
 
 	wait(kvStore->commit(false));
 
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
-	wait(closed);
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
 	kvStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
 	wait(kvStore->init());
 
 	// Read all values.
-	RangeResult result =
-	    wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000, IKeyValueStore::ReadType::NORMAL));
-	ASSERT_EQ(result.size(), expectedRows.size());
-	for (int i = 0; i < expectedRows.size(); ++i) {
-		ASSERT(result[i] == expectedRows[i]);
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000));
+		ASSERT_EQ(result.size(), expectedRows.size());
+		for (int i = 0; i < expectedRows.size(); ++i) {
+			ASSERT(result[i] == expectedRows[i]);
+		}
 	}
 
 	// Read partial range with row limit
-	RangeResult result =
-	    wait(kvStore->readRange(KeyRangeRef("5"_sr, ":"_sr), 35, 10000, IKeyValueStore::ReadType::NORMAL));
-	ASSERT_EQ(result.size(), 35);
-	for (int i = 0; i < result.size(); ++i) {
-		ASSERT(result[i] == expectedRows[40 + i]);
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("5"_sr, ":"_sr), 35, 10000));
+		ASSERT_EQ(result.size(), 35);
+		for (int i = 0; i < result.size(); ++i) {
+			ASSERT(result[i] == expectedRows[40 + i]);
+		}
 	}
 
 	// Clear a range on a single shard.
 	kvStore->clear(KeyRangeRef("40"_sr, "45"_sr));
 	wait(kvStore->commit(false));
 
-	RangeResult result =
-	    wait(kvStore->readRange(KeyRangeRef("4"_sr, "5"_sr), 20, 10000, IKeyValueStore::ReadType::NORMAL));
-	ASSERT_EQ(result.size(), 5);
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("4"_sr, "5"_sr), 20, 10000));
+		ASSERT_EQ(result.size(), 5);
+	}
 
 	// Clear a single value.
 	kvStore->clear(KeyRangeRef("01"_sr, keyAfter("01"_sr)));
@@ -2624,23 +2741,29 @@ TEST_CASE("noSim/ShardedRocksDB/RangeOps") {
 	kvStore->clear(KeyRangeRef("1"_sr, "8"_sr));
 	wait(kvStore->commit(false));
 
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
-	wait(closed);
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
 	kvStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
 	wait(kvStore->init());
 
-	RangeResult result =
-	    wait(kvStore->readRange(KeyRangeRef("1"_sr, "8"_sr), 1000, 10000, IKeyValueStore::ReadType::NORMAL));
-	ASSERT_EQ(result.size(), 0);
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("1"_sr, "8"_sr), 1000, 10000));
+		ASSERT_EQ(result.size(), 0);
+	}
 
-	RangeResult result =
-	    wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000, IKeyValueStore::ReadType::NORMAL));
-	ASSERT_EQ(result.size(), 19);
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000));
+		ASSERT_EQ(result.size(), 19);
+	}
 
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->dispose();
-	wait(closed);
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->dispose();
+		wait(closed);
+	}
 
 	return Void();
 }
@@ -2655,17 +2778,21 @@ TEST_CASE("noSim/ShardedRocksDB/ShardOps") {
 	wait(kvStore->init());
 
 	// Add some ranges.
-	std::vector<Future<Void>> addRangeFutures;
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("a"_sr, "c"_sr), "shard-1"));
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("c"_sr, "f"_sr), "shard-2"));
+	{
+		std::vector<Future<Void>> addRangeFutures;
+		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("a"_sr, "c"_sr), "shard-1"));
+		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("c"_sr, "f"_sr), "shard-2"));
 
-	wait(waitForAll(addRangeFutures));
+		wait(waitForAll(addRangeFutures));
+	}
 
-	std::vector<Future<Void>> addRangeFutures;
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("x"_sr, "z"_sr), "shard-1"));
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("l"_sr, "n"_sr), "shard-3"));
+	{
+		std::vector<Future<Void>> addRangeFutures;
+		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("x"_sr, "z"_sr), "shard-1"));
+		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("l"_sr, "n"_sr), "shard-3"));
 
-	wait(waitForAll(addRangeFutures));
+		wait(waitForAll(addRangeFutures));
+	}
 
 	// Remove single range.
 	std::vector<std::string> shardsToCleanUp;
@@ -2701,6 +2828,7 @@ TEST_CASE("noSim/ShardedRocksDB/ShardOps") {
 	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("m"_sr, "n"_sr)), "shard-3"));
 	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("u"_sr, "v"_sr)), "shard-3"));
 	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("x"_sr, "z"_sr)), "shard-1"));
+	mapping.push_back(std::make_pair(specialKeys, "default"));
 
 	for (auto it = dataMap.begin(); it != dataMap.end(); ++it) {
 		std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
@@ -2720,30 +2848,37 @@ TEST_CASE("noSim/ShardedRocksDB/ShardOps") {
 	kvStore = rocksdbStore;
 	wait(kvStore->init());
 
-	auto dataMap = rocksdbStore->getDataMapping();
-	for (auto it = dataMap.begin(); it != dataMap.end(); ++it) {
-		std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
-		          << it->second << "\n";
+	{
+		auto dataMap = rocksdbStore->getDataMapping();
+		for (auto it = dataMap.begin(); it != dataMap.end(); ++it) {
+			std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
+			          << it->second << "\n";
+		}
+		ASSERT(dataMap == mapping);
 	}
-	ASSERT(dataMap == mapping);
 
 	// Remove all the ranges.
-	state std::vector<std::string> shardsToCleanUp = kvStore->removeRange(KeyRangeRef("a"_sr, "z"_sr));
-	ASSERT_EQ(shardsToCleanUp.size(), 3);
+	{
+		state std::vector<std::string> shardsToCleanUp = kvStore->removeRange(KeyRangeRef("a"_sr, "z"_sr));
+		ASSERT_EQ(shardsToCleanUp.size(), 3);
 
-	// Add another range to shard-2.
-	wait(kvStore->addRange(KeyRangeRef("h"_sr, "i"_sr), "shard-2"));
+		// Add another range to shard-2.
+		wait(kvStore->addRange(KeyRangeRef("h"_sr, "i"_sr), "shard-2"));
 
-	// Clean up shards. Shard-2 should not be removed.
-	wait(kvStore->cleanUpShardsIfNeeded(shardsToCleanUp));
+		// Clean up shards. Shard-2 should not be removed.
+		wait(kvStore->cleanUpShardsIfNeeded(shardsToCleanUp));
+	}
+	{
+		auto dataMap = rocksdbStore->getDataMapping();
+		ASSERT_EQ(dataMap.size(), 2);
+		ASSERT(dataMap[0].second == "shard-2");
+	}
 
-	auto dataMap = rocksdbStore->getDataMapping();
-	ASSERT_EQ(dataMap.size(), 1);
-	ASSERT(dataMap[0].second == "shard-2");
-
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->dispose();
-	wait(closed);
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->dispose();
+		wait(closed);
+	}
 	return Void();
 }
 
@@ -2764,8 +2899,10 @@ TEST_CASE("noSim/ShardedRocksDB/Metadata") {
 	kvStore->set(KeyValueRef(testSpecialKey, testSpecialValue));
 	wait(kvStore->commit(false));
 
-	Optional<Value> val = wait(kvStore->readValue(testSpecialKey));
-	ASSERT(val.get() == testSpecialValue);
+	{
+		Optional<Value> val = wait(kvStore->readValue(testSpecialKey));
+		ASSERT(val.get() == testSpecialValue);
+	}
 
 	// Add some ranges.
 	std::vector<Future<Void>> addRangeFutures;
@@ -2792,10 +2929,14 @@ TEST_CASE("noSim/ShardedRocksDB/Metadata") {
 	}
 
 	// Read value back.
-	Optional<Value> val = wait(kvStore->readValue("a1"_sr));
-	ASSERT(val == Optional<Value>("foo"_sr));
-	Optional<Value> val = wait(kvStore->readValue("d1"_sr));
-	ASSERT(val == Optional<Value>("bar"_sr));
+	{
+		Optional<Value> val = wait(kvStore->readValue("a1"_sr));
+		ASSERT(val == Optional<Value>("foo"_sr));
+	}
+	{
+		Optional<Value> val = wait(kvStore->readValue("d1"_sr));
+		ASSERT(val == Optional<Value>("bar"_sr));
+	}
 
 	// Remove range containing a1.
 	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "b"_sr), false);
@@ -2803,22 +2944,30 @@ TEST_CASE("noSim/ShardedRocksDB/Metadata") {
 	wait(kvStore->commit(false));
 
 	// Read a1.
-	Optional<Value> val = wait(kvStore->readValue("a1"_sr));
-	ASSERT(!val.present());
+	{
+		Optional<Value> val = wait(kvStore->readValue("a1"_sr));
+		ASSERT(!val.present());
+	}
 
 	// Restart.
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
-	wait(closed);
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
 	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
 	kvStore = rocksdbStore;
 	wait(kvStore->init());
 
 	// Read again.
-	Optional<Value> val = wait(kvStore->readValue("a1"_sr));
-	ASSERT(!val.present());
-	Optional<Value> val = wait(kvStore->readValue("d1"_sr));
-	ASSERT(val == Optional<Value>("bar"_sr));
+	{
+		Optional<Value> val = wait(kvStore->readValue("a1"_sr));
+		ASSERT(!val.present());
+	}
+	{
+		Optional<Value> val = wait(kvStore->readValue("d1"_sr));
+		ASSERT(val == Optional<Value>("bar"_sr));
+	}
 
 	auto mapping = rocksdbStore->getDataMapping();
 	ASSERT(mapping.size() == 3);
@@ -2829,16 +2978,20 @@ TEST_CASE("noSim/ShardedRocksDB/Metadata") {
 	ASSERT(mapping.size() == 1);
 
 	// Restart.
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
-	wait(closed);
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
 	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
 	kvStore = rocksdbStore;
 	wait(kvStore->init());
 
 	// Because range metadata was not committed, ranges should be restored.
-	auto mapping = rocksdbStore->getDataMapping();
-	ASSERT(mapping.size() == 3);
+	{
+		auto mapping = rocksdbStore->getDataMapping();
+		ASSERT(mapping.size() == 3);
+	}
 
 	// Remove ranges again.
 	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "f"_sr), false);
@@ -2850,25 +3003,31 @@ TEST_CASE("noSim/ShardedRocksDB/Metadata") {
 	wait(kvStore->commit(false));
 
 	// Restart.
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
-	wait(closed);
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
 
 	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
 	kvStore = rocksdbStore;
 	wait(kvStore->init());
 
 	// No range available.
-	auto mapping = rocksdbStore->getDataMapping();
-	for (auto it = mapping.begin(); it != mapping.end(); ++it) {
-		std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
-		          << it->second << "\n";
+	{
+		auto mapping = rocksdbStore->getDataMapping();
+		for (auto it = mapping.begin(); it != mapping.end(); ++it) {
+			std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
+			          << it->second << "\n";
+		}
+		ASSERT(mapping.size() == 1);
 	}
-	ASSERT(mapping.size() == 1);
 
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->dispose();
-	wait(closed);
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->dispose();
+		wait(closed);
+	}
 
 	return Void();
 }
diff --git a/fdbserver/KnobProtectiveGroups.cpp b/fdbserver/KnobProtectiveGroups.cpp
index 2f251035fe..a460f0966d 100644
--- a/fdbserver/KnobProtectiveGroups.cpp
+++ b/fdbserver/KnobProtectiveGroups.cpp
@@ -22,6 +22,7 @@
 
 #include <array>
 
+#include "flow/Knobs.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/ServerKnobCollection.h"
 #include "fdbserver/Knobs.h"
@@ -52,6 +53,9 @@ void KnobProtectiveGroup::snapshotOriginalKnobs() {
 		if (std::get_if<NoKnobFound>(&value)) {
 			value = SERVER_KNOBS->getKnob(name);
 		}
+		if (std::get_if<NoKnobFound>(&value)) {
+			value = FLOW_KNOBS->getKnob(name);
+		}
 		if (std::get_if<NoKnobFound>(&value)) {
 			ASSERT(false);
 		}
@@ -70,4 +74,4 @@ void KnobProtectiveGroup::assignKnobs(const KnobKeyValuePairs& overrideKnobs) {
 		ASSERT(mutableServerKnobs.trySetKnob(name, valueRef));
 		TraceEvent(SevInfo, "AssignKnobValue").detail("KnobName", name).detail("KnobValue", valueRef.toString());
 	}
-}
\ No newline at end of file
+}
diff --git a/fdbserver/LeaderElection.actor.cpp b/fdbserver/LeaderElection.actor.cpp
index d7eedd561f..40c093093e 100644
--- a/fdbserver/LeaderElection.actor.cpp
+++ b/fdbserver/LeaderElection.actor.cpp
@@ -206,7 +206,7 @@ ACTOR Future<Void> tryBecomeLeaderInternal(ServerCoordinators coordinators,
 			choose {
 				when(wait(nomineeChange.onTrigger())) {}
 				when(wait(badCandidateTimeout.isValid() ? badCandidateTimeout : Never())) {
-					TEST(true); // Bad candidate timeout
+					CODE_PROBE(true, "Bad candidate timeout");
 					TraceEvent("LeaderBadCandidateTimeout", myInfo.changeID).log();
 					break;
 				}
diff --git a/fdbserver/LocalConfiguration.actor.cpp b/fdbserver/LocalConfiguration.actor.cpp
index fbc3380f9e..7a8e04e76b 100644
--- a/fdbserver/LocalConfiguration.actor.cpp
+++ b/fdbserver/LocalConfiguration.actor.cpp
@@ -65,7 +65,7 @@ class LocalConfigurationImpl {
 					configClassToKnobToValue[configPath.back()] = {};
 				}
 			} else {
-				TEST(true); // Invalid configuration path
+				CODE_PROBE(true, "Invalid configuration path");
 				if (!g_network->isSimulated()) {
 					fprintf(stderr, "WARNING: Invalid configuration path: `%s'\n", paramString.c_str());
 				}
@@ -88,7 +88,7 @@ class LocalConfigurationImpl {
 					knobCollection.setKnob(knobName.toString(), knobValue);
 				} catch (Error& e) {
 					if (e.code() == error_code_invalid_option_value) {
-						TEST(true); // invalid knob in configuration database
+						CODE_PROBE(true, "invalid knob in configuration database");
 						TraceEvent(SevWarnAlways, "InvalidKnobOptionValue")
 						    .detail("KnobName", knobName)
 						    .detail("KnobValue", knobValue.toString());
@@ -126,10 +126,10 @@ class LocalConfigurationImpl {
 					this->overrides[stringToKeyRef(knobName)] = knobValue;
 				} catch (Error& e) {
 					if (e.code() == error_code_invalid_option) {
-						TEST(true); // Attempted to manually set invalid knob option
+						CODE_PROBE(true, "Attempted to manually set invalid knob option");
 						TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(knobName));
 					} else if (e.code() == error_code_invalid_option_value) {
-						TEST(true); // Invalid manually set knob value
+						CODE_PROBE(true, "Invalid manually set knob value");
 						TraceEvent(SevWarnAlways, "InvalidKnobValue")
 						    .detail("Knob", printable(knobName))
 						    .detail("Value", printable(knobValueString));
@@ -198,7 +198,7 @@ class LocalConfigurationImpl {
 		state ConfigKnobOverrides storedConfigPath =
 		    BinaryReader::fromStringRef<ConfigKnobOverrides>(storedConfigPathValue.get(), IncludeVersion());
 		if (!storedConfigPath.hasSameConfigPath(self->configKnobOverrides)) {
-			TEST(true); // All local information is outdated
+			CODE_PROBE(true, "All local information is outdated");
 			wait(clearKVStore(self));
 			wait(saveConfigPath(self));
 			self->updateInMemoryState(lastSeenVersion);
@@ -228,7 +228,8 @@ class LocalConfigurationImpl {
 
 	ACTOR static Future<Void> setSnapshot(LocalConfigurationImpl* self,
 	                                      std::map<ConfigKey, KnobValue> snapshot,
-	                                      Version snapshotVersion) {
+	                                      Version snapshotVersion,
+	                                      double restartDelay) {
 		if (snapshotVersion <= self->lastSeenVersion) {
 			TraceEvent(SevWarnAlways, "LocalConfigGotOldSnapshot", self->id)
 			    .detail("NewSnapshotVersion", snapshotVersion)
@@ -249,6 +250,9 @@ class LocalConfigurationImpl {
 		self->kvStore->set(KeyValueRef(lastSeenVersionKey, BinaryWriter::toValue(snapshotVersion, IncludeVersion())));
 		wait(self->kvStore->commit());
 		if (restartRequired) {
+			if (restartDelay > 0) {
+				wait(delay(restartDelay));
+			}
 			throw local_config_changed();
 		}
 		self->updateInMemoryState(snapshotVersion);
@@ -257,7 +261,8 @@ class LocalConfigurationImpl {
 
 	ACTOR static Future<Void> addChanges(LocalConfigurationImpl* self,
 	                                     Standalone<VectorRef<VersionedConfigMutationRef>> changes,
-	                                     Version mostRecentVersion) {
+	                                     Version mostRecentVersion,
+	                                     double restartDelay) {
 		// TODO: Concurrency control?
 		++self->changeRequestsFetched;
 		state bool restartRequired = false;
@@ -271,7 +276,7 @@ class LocalConfigurationImpl {
 			++self->mutations;
 			const auto& mutation = versionedMutation.mutation;
 			{
-				TraceEvent te(SevDebug, "LocalConfigAddingChange", self->id);
+				TraceEvent te(SevInfo, "LocalConfigAddingChange", self->id);
 				te.detail("ConfigClass", mutation.getConfigClass())
 				    .detail("Version", versionedMutation.version)
 				    .detail("KnobName", mutation.getKnobName());
@@ -293,6 +298,9 @@ class LocalConfigurationImpl {
 		self->kvStore->set(KeyValueRef(lastSeenVersionKey, BinaryWriter::toValue(mostRecentVersion, IncludeVersion())));
 		wait(self->kvStore->commit());
 		if (restartRequired) {
+			if (restartDelay > 0) {
+				wait(delay(restartDelay));
+			}
 			throw local_config_changed();
 		}
 		self->updateInMemoryState(mostRecentVersion);
@@ -304,11 +312,12 @@ class LocalConfigurationImpl {
 		loop {
 			choose {
 				when(state ConfigBroadcastSnapshotRequest snapshotReq = waitNext(broadcaster.snapshot.getFuture())) {
-					wait(setSnapshot(self, std::move(snapshotReq.snapshot), snapshotReq.version));
+					wait(setSnapshot(
+					    self, std::move(snapshotReq.snapshot), snapshotReq.version, snapshotReq.restartDelay));
 					snapshotReq.reply.send(ConfigBroadcastSnapshotReply{});
 				}
 				when(state ConfigBroadcastChangesRequest req = waitNext(broadcaster.changes.getFuture())) {
-					wait(self->addChanges(req.changes, req.mostRecentVersion));
+					wait(self->addChanges(req.changes, req.mostRecentVersion, req.restartDelay));
 					req.reply.send(ConfigBroadcastChangesReply{});
 				}
 			}
@@ -342,8 +351,10 @@ public:
 		    "LocalConfigurationMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "LocalConfigurationMetrics");
 	}
 
-	Future<Void> addChanges(Standalone<VectorRef<VersionedConfigMutationRef>> changes, Version mostRecentVersion) {
-		return addChanges(this, changes, mostRecentVersion);
+	Future<Void> addChanges(Standalone<VectorRef<VersionedConfigMutationRef>> changes,
+	                        Version mostRecentVersion,
+	                        double restartDelay) {
+		return addChanges(this, changes, mostRecentVersion, restartDelay);
 	}
 
 	FlowKnobs const& getFlowKnobs() const { return getKnobs().getFlowKnobs(); }
@@ -447,8 +458,9 @@ Future<Void> LocalConfiguration::consume(ConfigBroadcastInterface const& broadca
 }
 
 Future<Void> LocalConfiguration::addChanges(Standalone<VectorRef<VersionedConfigMutationRef>> changes,
-                                            Version mostRecentVersion) {
-	return impl->addChanges(changes, mostRecentVersion);
+                                            Version mostRecentVersion,
+                                            double restartDelay) {
+	return impl->addChanges(changes, mostRecentVersion, restartDelay);
 }
 
 void LocalConfiguration::close() {
diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index aa3e159be5..2ee606e6fd 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -137,9 +137,7 @@ struct LogRouterData {
 	  : dbgid(dbgid), logSystem(new AsyncVar<Reference<ILogSystem>>()), version(req.startVersion - 1), minPopped(0),
 	    startVersion(req.startVersion), minKnownCommittedVersion(0), poppedVersion(0), routerTag(req.routerTag),
 	    allowPops(false), foundEpochEnd(false), generation(req.recoveryCount),
-	    peekLatencyDist(Histogram::getHistogram(LiteralStringRef("LogRouter"),
-	                                            LiteralStringRef("PeekTLogLatency"),
-	                                            Histogram::Unit::microseconds)),
+	    peekLatencyDist(Histogram::getHistogram("LogRouter"_sr, "PeekTLogLatency"_sr, Histogram::Unit::microseconds)),
 	    cc("LogRouter", dbgid.toString()), getMoreCount("GetMoreCount", cc),
 	    getMoreBlockedCount("GetMoreBlockedCount", cc) {
 		// setup just enough of a logSet to be able to call getPushLocations
@@ -367,7 +365,7 @@ ACTOR Future<Void> pullAsyncData(LogRouterData* self) {
 
 				if (!foundMessage) {
 					ver--; // ver is the next possible version we will get data for
-					if (ver > self->version.get()) {
+					if (ver > self->version.get() && ver >= r->popped()) {
 						wait(waitForVersion(self, ver));
 
 						self->version.set(ver);
@@ -592,7 +590,7 @@ Future<Void> logRouterPeekMessages(PromiseType replyPromise,
 		}
 		if (sequenceData.isSet()) {
 			if (sequenceData.getFuture().get().first != reply.end) {
-				TEST(true); // tlog peek second attempt ended at a different version
+				CODE_PROBE(true, "tlog peek second attempt ended at a different version");
 				replyPromise.sendError(operation_obsolete());
 				return Void();
 			}
diff --git a/fdbserver/LogSystem.cpp b/fdbserver/LogSystem.cpp
index 6444b23234..81ca7300f1 100644
--- a/fdbserver/LogSystem.cpp
+++ b/fdbserver/LogSystem.cpp
@@ -290,7 +290,7 @@ void LogPushData::addTxsTag() {
 }
 
 void LogPushData::addTransactionInfo(SpanContext const& context) {
-	TEST(!spanContext.isValid()); // addTransactionInfo with invalid SpanContext
+	CODE_PROBE(!spanContext.isValid(), "addTransactionInfo with invalid SpanContext");
 	spanContext = context;
 	writtenLocations.clear();
 }
@@ -352,7 +352,7 @@ bool LogPushData::writeTransactionInfo(int location, uint32_t subseq) {
 		return false;
 	}
 
-	TEST(true); // Wrote SpanContextMessage to a transaction log
+	CODE_PROBE(true, "Wrote SpanContextMessage to a transaction log");
 	writtenLocations.insert(location);
 
 	BinaryWriter& wr = messagesWriter[location];
@@ -375,10 +375,10 @@ bool LogPushData::writeTransactionInfo(int location, uint32_t subseq) {
 		// parent->child.
 		SpanContextMessage contextMessage;
 		if (spanContext.isSampled()) {
-			TEST(true); // Converting OTELSpanContextMessage to traced SpanContextMessage
+			CODE_PROBE(true, "Converting OTELSpanContextMessage to traced SpanContextMessage");
 			contextMessage = SpanContextMessage(UID(spanContext.traceID.first(), spanContext.traceID.second()));
 		} else {
-			TEST(true); // Converting OTELSpanContextMessage to untraced SpanContextMessage
+			CODE_PROBE(true, "Converting OTELSpanContextMessage to untraced SpanContextMessage");
 			contextMessage = SpanContextMessage(UID(0, 0));
 		}
 		wr << contextMessage;
diff --git a/fdbserver/LogSystemDiskQueueAdapter.actor.cpp b/fdbserver/LogSystemDiskQueueAdapter.actor.cpp
index 029e551e7a..12e64a5f1e 100644
--- a/fdbserver/LogSystemDiskQueueAdapter.actor.cpp
+++ b/fdbserver/LogSystemDiskQueueAdapter.actor.cpp
@@ -101,7 +101,7 @@ public:
 					TraceEvent(SevWarnAlways, "DiskQueueAdapterReset")
 					    .detail("Version", self->cursor->popped())
 					    .detail("PeekTypeSwitch", self->peekTypeSwitches % 3);
-					TEST(true); // disk adapter reset
+					CODE_PROBE(true, "disk adapter reset");
 					if (self->cursor->popped() != 0) {
 						self->recoveryLoc = self->cursor->popped();
 					} else {
diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp
index a7bf08bdee..ab2ee955ba 100644
--- a/fdbserver/LogSystemPeekCursor.actor.cpp
+++ b/fdbserver/LogSystemPeekCursor.actor.cpp
@@ -317,7 +317,7 @@ ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self,
 				//
 				// A cursor for a log router can be delayed indefinitely during a network partition, so only fail
 				// simulation tests sufficiently far after we finish simulating network partitions.
-				TEST(e.code() == error_code_timed_out); // peek cursor timed out
+				CODE_PROBE(e.code() == error_code_timed_out, "peek cursor timed out");
 				if (now() >= FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS + SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME) {
 					ASSERT_WE_THINK(e.code() == error_code_operation_obsolete ||
 					                SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME < 10);
@@ -382,7 +382,8 @@ ACTOR Future<Void> serverPeekStreamGetMore(ILogSystem::ServerPeekCursor* self, T
 			DebugLogTraceEvent(SevDebug, "SPC_GetMoreB_Error", self->randomID)
 			    .errorUnsuppressed(e)
 			    .detail("Tag", self->tag);
-			if (e.code() == error_code_connection_failed || e.code() == error_code_operation_obsolete) {
+			if (e.code() == error_code_connection_failed || e.code() == error_code_operation_obsolete ||
+			    e.code() == error_code_request_maybe_delivered) {
 				// NOTE: delay in order to avoid the endless retry loop block other tasks
 				self->peekReplyStream.reset();
 				wait(delay(0));
@@ -653,7 +654,7 @@ void ILogSystem::MergedPeekCursor::updateMessage(bool usePolicy) {
 			c->advanceTo(messageVersion);
 			if (start <= messageVersion && messageVersion < c->version()) {
 				advancedPast = true;
-				TEST(true); // Merge peek cursor advanced past desired sequence
+				CODE_PROBE(true, "Merge peek cursor advanced past desired sequence");
 			}
 		}
 
@@ -965,7 +966,7 @@ void ILogSystem::SetPeekCursor::updateMessage(int logIdx, bool usePolicy) {
 				c->advanceTo(messageVersion);
 				if (start <= messageVersion && messageVersion < c->version()) {
 					advancedPast = true;
-					TEST(true); // Merge peek cursor with logIdx advanced past desired sequence
+					CODE_PROBE(true, "Merge peek cursor with logIdx advanced past desired sequence");
 				}
 			}
 		}
diff --git a/fdbserver/MetricLogger.actor.cpp b/fdbserver/MetricLogger.actor.cpp
index 933c1f5d85..ef6a2601b1 100644
--- a/fdbserver/MetricLogger.actor.cpp
+++ b/fdbserver/MetricLogger.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include <cmath>
+#include "flow/ApiVersion.h"
 #include "flow/UnitTest.h"
 #include "flow/TDMetric.actor.h"
 #include "fdbclient/DatabaseContext.h"
@@ -40,13 +41,7 @@ struct MetricsRule {
 	int minLevel;
 
 	Tuple pack() const {
-		return Tuple()
-		    .append(namePattern)
-		    .append(typePattern)
-		    .append(addressPattern)
-		    .append(idPattern)
-		    .append(enabled ? 1 : 0)
-		    .append(minLevel);
+		return Tuple::makeTuple(namePattern, typePattern, addressPattern, idPattern, enabled ? 1 : 0, minLevel);
 	}
 
 	static inline MetricsRule unpack(Tuple const& t) {
@@ -95,18 +90,16 @@ struct MetricsRule {
 
 struct MetricsConfig {
 	MetricsConfig(Key prefix = KeyRef())
-	  : space(prefix), ruleMap(space.get(LiteralStringRef("Rules")).key()),
-	    addressMap(space.get(LiteralStringRef("Enum")).get(LiteralStringRef("Address")).key()),
-	    nameAndTypeMap(space.get(LiteralStringRef("Enum")).get(LiteralStringRef("NameType")).key()),
-	    ruleChangeKey(space.get(LiteralStringRef("RulesChanged")).key()),
-	    enumsChangeKey(space.get(LiteralStringRef("EnumsChanged")).key()),
-	    fieldChangeKey(space.get(LiteralStringRef("FieldsChanged")).key()) {}
+	  : space(prefix), ruleMap(space.get("Rules"_sr).key()), addressMap(space.get("Enum"_sr).get("Address"_sr).key()),
+	    nameAndTypeMap(space.get("Enum"_sr).get("NameType"_sr).key()),
+	    ruleChangeKey(space.get("RulesChanged"_sr).key()), enumsChangeKey(space.get("EnumsChanged"_sr).key()),
+	    fieldChangeKey(space.get("FieldsChanged"_sr).key()) {}
 
 	Subspace space;
 
 	typedef KeyBackedMap<int64_t, MetricsRule> RuleMapT;
 	RuleMapT ruleMap;
-	RuleMapT::PairsType rules;
+	RuleMapT::RangeResultType rules;
 
 	KeyBackedMap<Key, int64_t> addressMap;
 	KeyBackedMap<std::pair<Key, Key>, int64_t> nameAndTypeMap;
@@ -152,11 +145,11 @@ ACTOR Future<Void> metricRuleUpdater(Database cx, MetricsConfig* config, TDMetri
 		state Future<Void> newMetric = collection->metricAdded.onTrigger();
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			MetricsConfig::RuleMapT::PairsType rules = wait(config->ruleMap.getRange(tr, 0, {}, 1e6));
+			MetricsConfig::RuleMapT::RangeResultType rules = wait(config->ruleMap.getRange(tr, 0, {}, 1e6));
 
 			for (auto& it : collection->metricMap) {
 				it.value->setConfig(false);
-				for (auto i = rules.rbegin(); !(i == rules.rend()); ++i)
+				for (auto i = rules.results.rbegin(); !(i == rules.results.rend()); ++i)
 					if (i->second.applyTo(it.value.getPtr(), collection->address))
 						break;
 			}
@@ -423,8 +416,8 @@ TEST_CASE("/fdbserver/metrics/TraceEvents") {
 	}
 	fprintf(stdout, "Using environment variables METRICS_CONNFILE and METRICS_PREFIX.\n");
 
-	state Database metricsDb = Database::createDatabase(metricsConnFile, Database::API_VERSION_LATEST);
-	TDMetricCollection::getTDMetrics()->address = LiteralStringRef("0.0.0.0:0");
+	state Database metricsDb = Database::createDatabase(metricsConnFile, ApiVersion::LATEST_VERSION);
+	TDMetricCollection::getTDMetrics()->address = "0.0.0.0:0"_sr;
 	state Future<Void> metrics = runMetrics(metricsDb, KeyRef(metricsPrefix));
 	state int64_t x = 0;
 
@@ -443,9 +436,9 @@ TEST_CASE("/fdbserver/metrics/TraceEvents") {
 	fprintf(stdout, "  d is always present, is a string, and rotates through the values 'one', 'two', and ''.\n");
 	fprintf(stdout, "  Plotting j on the x axis and k on the y axis should look like x=sin(2t), y=sin(3t)\n");
 
-	state Int64MetricHandle intMetric = Int64MetricHandle(LiteralStringRef("DummyInt"));
-	state BoolMetricHandle boolMetric = BoolMetricHandle(LiteralStringRef("DummyBool"));
-	state StringMetricHandle stringMetric = StringMetricHandle(LiteralStringRef("DummyString"));
+	state Int64MetricHandle intMetric = Int64MetricHandle("DummyInt"_sr);
+	state BoolMetricHandle boolMetric = BoolMetricHandle("DummyBool"_sr);
+	state StringMetricHandle stringMetric = StringMetricHandle("DummyString"_sr);
 
 	static const char* dStrings[] = { "one", "two", "" };
 	state const char** d = dStrings;
diff --git a/fdbserver/MockGlobalState.cpp b/fdbserver/MockGlobalState.cpp
new file mode 100644
index 0000000000..1229ac6f32
--- /dev/null
+++ b/fdbserver/MockGlobalState.cpp
@@ -0,0 +1,111 @@
+/*
+ * MockGlobalState.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/MockGlobalState.h"
+
+bool MockStorageServer::allShardStatusEqual(KeyRangeRef range, MockShardStatus status) {
+	auto ranges = serverKeys.intersectingRanges(range);
+	ASSERT(!ranges.empty()); // at least the range is allKeys
+
+	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+		if (it->cvalue().status != status)
+			return false;
+	}
+	return true;
+}
+
+void MockGlobalState::initializeAsEmptyDatabaseMGS(const DatabaseConfiguration& conf, uint64_t defaultDiskSpace) {
+	ASSERT(conf.storageTeamSize > 0);
+	configuration = conf;
+	std::vector<UID> serverIds;
+	for (int i = 1; i <= conf.storageTeamSize; ++i) {
+		UID id = indexToUID(i);
+		serverIds.push_back(id);
+		allServers[id] = MockStorageServer(id, defaultDiskSpace);
+		allServers[id].serverKeys.insert(allKeys, { MockShardStatus::COMPLETED, 0 });
+	}
+	shardMapping->assignRangeToTeams(allKeys, { Team(serverIds, true) });
+}
+
+void MockGlobalState::addStorageServer(StorageServerInterface server, uint64_t diskSpace) {
+	allServers[server.id()] = MockStorageServer(server, diskSpace);
+}
+
+bool MockGlobalState::serverIsSourceForShard(const UID& serverId, KeyRangeRef shard, bool inFlightShard) {
+	if (!allServers.count(serverId))
+		return false;
+
+	// check serverKeys
+	auto& mss = allServers.at(serverId);
+	if (!mss.allShardStatusEqual(shard, MockShardStatus::COMPLETED)) {
+		return false;
+	}
+
+	// check keyServers
+	auto teams = shardMapping->getTeamsFor(shard);
+	if (inFlightShard) {
+		return std::any_of(teams.second.begin(), teams.second.end(), [&serverId](const Team& team) {
+			return team.hasServer(serverId);
+		});
+	}
+	return std::any_of(
+	    teams.first.begin(), teams.first.end(), [&serverId](const Team& team) { return team.hasServer(serverId); });
+}
+
+bool MockGlobalState::serverIsDestForShard(const UID& serverId, KeyRangeRef shard) {
+	if (!allServers.count(serverId))
+		return false;
+
+	// check serverKeys
+	auto& mss = allServers.at(serverId);
+	if (!mss.allShardStatusEqual(shard, MockShardStatus::INFLIGHT)) {
+		return false;
+	}
+
+	// check keyServers
+	auto teams = shardMapping->getTeamsFor(shard);
+	return !teams.second.empty() && std::any_of(teams.first.begin(), teams.first.end(), [&serverId](const Team& team) {
+		return team.hasServer(serverId);
+	});
+}
+
+bool MockGlobalState::allShardRemovedFromServer(const UID& serverId) {
+	return allServers.count(serverId) && shardMapping->getNumberOfShards(serverId) == 0;
+}
+
+TEST_CASE("/MockGlobalState/initializeAsEmptyDatabaseMGS/SimpleThree") {
+	BasicTestConfig testConfig;
+	testConfig.simpleConfig = true;
+	testConfig.minimumReplication = 3;
+	testConfig.logAntiQuorum = 0;
+	DatabaseConfiguration dbConfig = generateNormalDatabaseConfiguration(testConfig);
+	TraceEvent("UnitTestDbConfig").detail("Config", dbConfig.toString());
+
+	auto mgs = std::make_shared<MockGlobalState>();
+	mgs->initializeAsEmptyDatabaseMGS(dbConfig);
+	for (int i = 1; i <= dbConfig.storageTeamSize; ++i) {
+		auto id = MockGlobalState::indexToUID(i);
+		std::cout << "Check server " << i << "\n";
+		ASSERT(mgs->serverIsSourceForShard(id, allKeys));
+		ASSERT(mgs->allServers.at(id).serverKeys.sumRange(allKeys.begin, allKeys.end) == 0);
+	}
+
+	return Void();
+}
diff --git a/fdbserver/MoveKeys.actor.cpp b/fdbserver/MoveKeys.actor.cpp
index 52e0c364e9..259b60e2b2 100644
--- a/fdbserver/MoveKeys.actor.cpp
+++ b/fdbserver/MoveKeys.actor.cpp
@@ -27,6 +27,7 @@
 #include "fdbclient/SystemData.h"
 #include "fdbserver/MoveKeys.actor.h"
 #include "fdbserver/Knobs.h"
+#include "fdbclient/ReadYourWrites.h"
 #include "fdbserver/TSSMappingUtil.actor.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -184,6 +185,34 @@ bool DDEnabledState::setDDEnabled(bool status, UID snapUID) {
 	return true;
 }
 
+ACTOR Future<Void> readMoveKeysLock(Transaction* tr, MoveKeysLock* lock) {
+	{
+		Optional<Value> readVal = wait(tr->get(moveKeysLockOwnerKey));
+		lock->prevOwner = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
+	}
+	{
+		Optional<Value> readVal = wait(tr->get(moveKeysLockWriteKey));
+		lock->prevWrite = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
+	}
+	return Void();
+}
+
+ACTOR Future<MoveKeysLock> readMoveKeysLock(Database cx) {
+	state Transaction tr(cx);
+	loop {
+		try {
+			state MoveKeysLock lock;
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			wait(readMoveKeysLock(&tr, &lock));
+			return lock;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+			CODE_PROBE(true, "readMoveKeysLock retry");
+		}
+	}
+}
+
 ACTOR Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId) {
 	state Transaction tr(cx);
 	loop {
@@ -196,16 +225,7 @@ ACTOR Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId) {
 				txnId = deterministicRandom()->randomUniqueID();
 				tr.debugTransaction(txnId);
 			}
-			{
-				Optional<Value> readVal = wait(tr.get(moveKeysLockOwnerKey));
-				lock.prevOwner =
-				    readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
-			}
-			{
-				Optional<Value> readVal = wait(tr.get(moveKeysLockWriteKey));
-				lock.prevWrite =
-				    readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
-			}
+			wait(readMoveKeysLock(&tr, &lock));
 			lock.myOwner = deterministicRandom()->randomUniqueID();
 			tr.set(moveKeysLockOwnerKey, BinaryWriter::toValue(lock.myOwner, Unversioned()));
 			wait(tr.commit());
@@ -217,7 +237,7 @@ ACTOR Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId) {
 			return lock;
 		} catch (Error& e) {
 			wait(tr.onError(e));
-			TEST(true); // takeMoveKeysLock retry
+			CODE_PROBE(true, "takeMoveKeysLock retry");
 		}
 	}
 }
@@ -239,7 +259,7 @@ ACTOR static Future<Void> checkMoveKeysLock(Transaction* tr,
 		Optional<Value> readVal = wait(tr->get(moveKeysLockWriteKey));
 		UID lastWrite = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
 		if (lastWrite != lock.prevWrite) {
-			TEST(true); // checkMoveKeysLock: Conflict with previous owner
+			CODE_PROBE(true, "checkMoveKeysLock: Conflict with previous owner");
 			throw movekeys_conflict();
 		}
 
@@ -272,7 +292,7 @@ ACTOR static Future<Void> checkMoveKeysLock(Transaction* tr,
 
 		return Void();
 	} else {
-		TEST(true); // checkMoveKeysLock: Conflict with new owner
+		CODE_PROBE(true, "checkMoveKeysLock: Conflict with new owner");
 		throw movekeys_conflict();
 	}
 }
@@ -281,6 +301,7 @@ Future<Void> checkMoveKeysLockReadOnly(Transaction* tr, MoveKeysLock lock, const
 	return checkMoveKeysLock(tr, lock, ddEnabledState, false);
 }
 
+namespace {
 ACTOR Future<Optional<UID>> checkReadWrite(Future<ErrorOr<GetShardStateReply>> fReply, UID uid, Version version) {
 	ErrorOr<GetShardStateReply> reply = wait(fReply);
 	if (!reply.present() || reply.get().first < version)
@@ -295,7 +316,7 @@ ACTOR Future<Void> cleanUpSingleShardDataMove(Database occ,
                                               FlowLock* cleanUpDataMoveParallelismLock,
                                               UID dataMoveId,
                                               const DDEnabledState* ddEnabledState) {
-	ASSERT(CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
 	TraceEvent(SevInfo, "CleanUpSingleShardDataMoveBegin", dataMoveId).detail("Range", keys);
 
 	loop {
@@ -591,7 +612,7 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 		// This process can be split up into multiple transactions if there are too many existing overlapping shards
 		// In that case, each iteration of this loop will have begin set to the end of the last processed shard
 		while (begin < keys.end) {
-			TEST(begin > keys.begin); // Multi-transactional startMoveKeys
+			CODE_PROBE(begin > keys.begin, "Multi-transactional startMoveKeys");
 			batches++;
 
 			// RYW to optimize re-reading the same key ranges
@@ -631,7 +652,7 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 							// Attempt to move onto a server that isn't in serverList (removed or never added to the
 							// database) This can happen (why?) and is handled by the data distribution algorithm
 							// FIXME: Answer why this can happen?
-							TEST(true); // start move keys moving to a removed server
+							CODE_PROBE(true, "start move keys moving to a removed server");
 							throw move_to_removed_server();
 						}
 					}
@@ -825,7 +846,7 @@ ACTOR Future<Void> checkFetchingState(Database cx,
 			for (int s = 0; s < serverListValues.size(); s++) {
 				if (!serverListValues[s].present()) {
 					// FIXME: Is this the right behavior?  dataMovementComplete will never be sent!
-					TEST(true); // check fetching state moved to removed server
+					CODE_PROBE(true, "check fetching state moved to removed server");
 					throw move_to_removed_server();
 				}
 				auto si = decodeServerListValue(serverListValues[s].get());
@@ -897,7 +918,7 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 		// This process can be split up into multiple transactions if there are too many existing overlapping shards
 		// In that case, each iteration of this loop will have begin set to the end of the last processed shard
 		while (begin < keys.end) {
-			TEST(begin > keys.begin); // Multi-transactional finishMoveKeys
+			CODE_PROBE(begin > keys.begin, "Multi-transactional finishMoveKeys");
 
 			state Transaction tr(occ);
 
@@ -994,7 +1015,8 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 						} else if (alreadyMoved) {
 							dest.clear();
 							src.clear();
-							TEST(true); // FinishMoveKeys first key in iteration sub-range has already been processed
+							CODE_PROBE(true,
+							           "FinishMoveKeys first key in iteration sub-range has already been processed");
 						}
 					}
 
@@ -1029,8 +1051,9 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 						}
 					}
 					if (!dest.size()) {
-						TEST(true); // A previous finishMoveKeys for this range committed just as it was cancelled to
-						            // start this one?
+						CODE_PROBE(true,
+						           "A previous finishMoveKeys for this range committed just as it was cancelled to "
+						           "start this one?");
 						TraceEvent("FinishMoveKeysNothingToDo", relocationIntervalId)
 						    .detail("KeyBegin", keys.begin)
 						    .detail("KeyEnd", keys.end)
@@ -1223,7 +1246,7 @@ ACTOR static Future<Void> startMoveShards(Database occ,
                                           UID relocationIntervalId,
                                           const DDEnabledState* ddEnabledState,
                                           CancelConflictingDataMoves cancelConflictingDataMoves) {
-	ASSERT(CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
 	state Future<Void> warningLogger = logWarningAfter("StartMoveShardsTooLong", 600, servers);
 
 	wait(startMoveKeysLock->take(TaskPriority::DataDistributionLaunch));
@@ -1394,7 +1417,6 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 							physicalShardMap[ssId].emplace_back(rangeIntersectKeys, srcId);
 						}
 
-						const UID checkpontId = deterministicRandom()->randomUniqueID();
 						for (const UID& ssId : src) {
 							dataMove.src.insert(ssId);
 							// TODO(psm): Create checkpoint for the range.
@@ -1559,7 +1581,7 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
                                            UID relocationIntervalId,
                                            std::map<UID, StorageServerInterface> tssMapping,
                                            const DDEnabledState* ddEnabledState) {
-	ASSERT(CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
 	state KeyRange keys = targetKeys;
 	state Future<Void> warningLogger = logWarningAfter("FinishMoveShardsTooLong", 600, destinationTeam);
 	state int retries = 0;
@@ -1710,15 +1732,19 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 				             Void(),
 				             TaskPriority::MoveKeys));
 
-				int count = 0;
+				std::vector<UID> readyServers;
 				for (int s = 0; s < serverReady.size(); ++s) {
-					count += serverReady[s].isReady() && !serverReady[s].isError();
+					if (serverReady[s].isReady() && !serverReady[s].isError()) {
+						readyServers.push_back(storageServerInterfaces[s].uniqueID);
+					}
 				}
 
 				TraceEvent(SevVerbose, "FinishMoveShardsWaitedServers", relocationIntervalId)
-				    .detail("ReadyServers", count);
+				    .detail("DataMoveID", dataMoveId)
+				    .detail("ReadyServers", describe(readyServers));
+
+				if (readyServers.size() == newDestinations.size()) {
 
-				if (count == newDestinations.size()) {
 					std::vector<Future<Void>> actors;
 					actors.push_back(krmSetRangeCoalescing(
 					    &tr, keyServersPrefix, range, allKeys, keyServersValue(destServers, {}, dataMoveId, UID())));
@@ -1789,6 +1815,8 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 	return Void();
 }
 
+}; // anonymous namespace
+
 ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServerInterface server) {
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
 	state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
@@ -2021,8 +2049,9 @@ ACTOR Future<Void> removeStorageServer(Database cx,
 
 			state bool canRemove = wait(canRemoveStorageServer(tr, serverID));
 			if (!canRemove) {
-				TEST(true); // The caller had a transaction in flight that assigned keys to the server.  Wait for it to
-				            // reverse its mistake.
+				CODE_PROBE(true,
+				           "The caller had a transaction in flight that assigned keys to the server.  Wait for it to "
+				           "reverse its mistake.");
 				TraceEvent(SevWarn, "NoCanRemove").detail("Count", noCanRemoveCount++).detail("ServerID", serverID);
 				wait(delayJittered(SERVER_KNOBS->REMOVE_RETRY_DELAY, TaskPriority::DataDistributionLaunch));
 				tr->reset();
@@ -2039,7 +2068,7 @@ ACTOR Future<Void> removeStorageServer(Database cx,
 
 				if (!fListKey.get().present()) {
 					if (retry) {
-						TEST(true); // Storage server already removed after retrying transaction
+						CODE_PROBE(true, "Storage server already removed after retrying transaction");
 						return Void();
 					}
 					TraceEvent(SevError, "RemoveInvalidServer").detail("ServerID", serverID);
@@ -2218,7 +2247,7 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
 						const UID shardId = newShardId(deterministicRandom()->randomUInt64(), AssignEmptyRange::True);
 
 						// Assign the shard to teamForDroppedRange in keyServer space.
-						if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+						if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
 							tr.set(keyServersKey(it.key), keyServersValue(teamForDroppedRange, {}, shardId, UID()));
 						} else {
 							tr.set(keyServersKey(it.key), keyServersValue(UIDtoTagMap, teamForDroppedRange));
@@ -2235,7 +2264,7 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
 						// Assign the shard to the new team as an empty range.
 						// Note, there could be data loss.
 						for (const UID& id : teamForDroppedRange) {
-							if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+							if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
 								actors.push_back(krmSetRangeCoalescing(
 								    &tr, serverKeysPrefixFor(id), range, allKeys, serverKeysValue(shardId)));
 							} else {
@@ -2440,76 +2469,75 @@ ACTOR Future<Void> cleanUpDataMove(Database occ,
 	return Void();
 }
 
-ACTOR Future<Void> moveKeys(Database cx,
-                            UID dataMoveId,
-                            KeyRange keys,
-                            std::vector<UID> destinationTeam,
-                            std::vector<UID> healthyDestinations,
-                            MoveKeysLock lock,
-                            Promise<Void> dataMovementComplete,
-                            FlowLock* startMoveKeysParallelismLock,
-                            FlowLock* finishMoveKeysParallelismLock,
-                            bool hasRemote,
-                            UID relocationIntervalId,
-                            const DDEnabledState* ddEnabledState,
-                            CancelConflictingDataMoves cancelConflictingDataMoves) {
-	ASSERT(destinationTeam.size());
-	std::sort(destinationTeam.begin(), destinationTeam.end());
+Future<Void> startMovement(Database occ, MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
+	if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+		return startMoveShards(std::move(occ),
+		                       params.dataMoveId,
+		                       params.keys,
+		                       params.destinationTeam,
+		                       params.lock,
+		                       params.startMoveKeysParallelismLock,
+		                       params.relocationIntervalId,
+		                       params.ddEnabledState,
+		                       params.cancelConflictingDataMoves);
+	}
+	return startMoveKeys(std::move(occ),
+	                     params.keys,
+	                     params.destinationTeam,
+	                     params.lock,
+	                     params.startMoveKeysParallelismLock,
+	                     params.relocationIntervalId,
+	                     &tssMapping,
+	                     params.ddEnabledState);
+}
+
+Future<Void> finishMovement(Database occ,
+                            MoveKeysParams& params,
+                            const std::map<UID, StorageServerInterface>& tssMapping) {
+	if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+		return finishMoveShards(std::move(occ),
+		                        params.dataMoveId,
+		                        params.keys,
+		                        params.destinationTeam,
+		                        params.lock,
+		                        params.finishMoveKeysParallelismLock,
+		                        params.hasRemote,
+		                        params.relocationIntervalId,
+		                        tssMapping,
+		                        params.ddEnabledState);
+	}
+	return finishMoveKeys(std::move(occ),
+	                      params.keys,
+	                      params.destinationTeam,
+	                      params.lock,
+	                      params.finishMoveKeysParallelismLock,
+	                      params.hasRemote,
+	                      params.relocationIntervalId,
+	                      tssMapping,
+	                      params.ddEnabledState);
+}
+
+ACTOR Future<Void> moveKeys(Database occ, MoveKeysParams params) {
+	ASSERT(params.destinationTeam.size());
+	std::sort(params.destinationTeam.begin(), params.destinationTeam.end());
 
 	state std::map<UID, StorageServerInterface> tssMapping;
 
-	if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
-		wait(startMoveShards(cx,
-		                     dataMoveId,
-		                     keys,
-		                     destinationTeam,
-		                     lock,
-		                     startMoveKeysParallelismLock,
-		                     relocationIntervalId,
-		                     ddEnabledState,
-		                     cancelConflictingDataMoves));
+	wait(startMovement(occ, params, tssMapping));
 
-	} else {
-		wait(startMoveKeys(cx,
-		                   keys,
-		                   destinationTeam,
-		                   lock,
-		                   startMoveKeysParallelismLock,
-		                   relocationIntervalId,
-		                   &tssMapping,
-		                   ddEnabledState));
-	}
+	state Future<Void> completionSignaller = checkFetchingState(occ,
+	                                                            params.healthyDestinations,
+	                                                            params.keys,
+	                                                            params.dataMovementComplete,
+	                                                            params.relocationIntervalId,
+	                                                            tssMapping);
 
-	state Future<Void> completionSignaller =
-	    checkFetchingState(cx, healthyDestinations, keys, dataMovementComplete, relocationIntervalId, tssMapping);
-
-	if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
-		wait(finishMoveShards(cx,
-		                      dataMoveId,
-		                      keys,
-		                      destinationTeam,
-		                      lock,
-		                      finishMoveKeysParallelismLock,
-		                      hasRemote,
-		                      relocationIntervalId,
-		                      tssMapping,
-		                      ddEnabledState));
-	} else {
-		wait(finishMoveKeys(cx,
-		                    keys,
-		                    destinationTeam,
-		                    lock,
-		                    finishMoveKeysParallelismLock,
-		                    hasRemote,
-		                    relocationIntervalId,
-		                    tssMapping,
-		                    ddEnabledState));
-	}
+	wait(finishMovement(occ, params, tssMapping));
 
 	// This is defensive, but make sure that we always say that the movement is complete before moveKeys completes
 	completionSignaller.cancel();
-	if (!dataMovementComplete.isSet())
-		dataMovementComplete.send(Void());
+	if (!params.dataMovementComplete.isSet())
+		params.dataMovementComplete.send(Void());
 
 	return Void();
 }
@@ -2566,7 +2594,7 @@ void seedShardServers(Arena& arena, CommitTransactionRef& tr, std::vector<Storag
 	// We have to set this range in two blocks, because the master tracking of "keyServersLocations" depends on a change
 	// to a specific
 	//   key (keyServersKeyServersKey)
-	if (CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+	if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
 		const UID teamId = deterministicRandom()->randomUniqueID();
 		ksValue = keyServersValue(serverSrcUID, /*dest=*/std::vector<UID>(), teamId, UID());
 		krmSetPreviouslyEmptyRange(tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), allKeys.end), ksValue, Value());
diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp
index b0db6b1ed5..ab3bb5ee53 100644
--- a/fdbserver/MutationTracking.cpp
+++ b/fdbserver/MutationTracking.cpp
@@ -21,7 +21,6 @@
 #include <algorithm>
 #include <vector>
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/EncryptedMutationMessage.h"
 #include "fdbserver/MutationTracking.h"
 #include "fdbserver/LogProtocolMessage.h"
 #include "fdbserver/SpanContextMessage.h"
@@ -99,12 +98,10 @@ TraceEvent debugTagsAndMessageEnabled(const char* context, Version version, Stri
 			SpanContextMessage scm;
 			br >> scm;
 		} else if (OTELSpanContextMessage::startsOTELSpanContextMessage(mutationType)) {
-			TEST(true); // MutationTracking reading OTELSpanContextMessage
+			CODE_PROBE(true, "MutationTracking reading OTELSpanContextMessage");
 			BinaryReader br(mutationData, AssumeVersion(rdr.protocolVersion()));
 			OTELSpanContextMessage scm;
 			br >> scm;
-		} else if (EncryptedMutationMessage::startsEncryptedMutationMessage(mutationType)) {
-			throw encrypt_unsupported();
 		} else {
 			MutationRef m;
 			BinaryReader br(mutationData, AssumeVersion(rdr.protocolVersion()));
diff --git a/fdbserver/OldTLogServer_4_6.actor.cpp b/fdbserver/OldTLogServer_4_6.actor.cpp
index 7fa6573a53..0ecc22032a 100644
--- a/fdbserver/OldTLogServer_4_6.actor.cpp
+++ b/fdbserver/OldTLogServer_4_6.actor.cpp
@@ -182,7 +182,7 @@ private:
 			Standalone<StringRef> h = wait(self->queue->readNext(sizeof(uint32_t)));
 			if (h.size() != sizeof(uint32_t)) {
 				if (h.size()) {
-					TEST(true); // Zero fill within size field
+					CODE_PROBE(true, "Zero fill within size field");
 					int payloadSize = 0;
 					memcpy(&payloadSize, h.begin(), h.size());
 					zeroFillSize = sizeof(uint32_t) - h.size(); // zero fill the size itself
@@ -196,7 +196,7 @@ private:
 
 			Standalone<StringRef> e = wait(self->queue->readNext(payloadSize + 1));
 			if (e.size() != payloadSize + 1) {
-				TEST(true); // Zero fill within payload
+				CODE_PROBE(true, "Zero fill within payload");
 				zeroFillSize = payloadSize + 1 - e.size();
 				break;
 			}
@@ -210,7 +210,7 @@ private:
 			}
 		}
 		if (zeroFillSize) {
-			TEST(true); // Fixing a partial commit at the end of the tlog queue
+			CODE_PROBE(true, "Fixing a partial commit at the end of the tlog queue");
 			for (int i = 0; i < zeroFillSize; i++)
 				self->queue->push(StringRef((const uint8_t*)"", 1));
 		}
@@ -221,17 +221,14 @@ private:
 ////// Persistence format (for self->persistentData)
 
 // Immutable keys
-static const KeyValueRef persistFormat(LiteralStringRef("Format"), LiteralStringRef("FoundationDB/LogServer/2/3"));
-static const KeyRangeRef persistFormatReadableRange(LiteralStringRef("FoundationDB/LogServer/2/3"),
-                                                    LiteralStringRef("FoundationDB/LogServer/2/4"));
-static const KeyRangeRef persistRecoveryCountKeys =
-    KeyRangeRef(LiteralStringRef("DbRecoveryCount/"), LiteralStringRef("DbRecoveryCount0"));
+static const KeyValueRef persistFormat("Format"_sr, "FoundationDB/LogServer/2/3"_sr);
+static const KeyRangeRef persistFormatReadableRange("FoundationDB/LogServer/2/3"_sr, "FoundationDB/LogServer/2/4"_sr);
+static const KeyRangeRef persistRecoveryCountKeys = KeyRangeRef("DbRecoveryCount/"_sr, "DbRecoveryCount0"_sr);
 
 // Updated on updatePersistentData()
-static const KeyRangeRef persistCurrentVersionKeys =
-    KeyRangeRef(LiteralStringRef("version/"), LiteralStringRef("version0"));
-static const KeyRange persistTagMessagesKeys = prefixRange(LiteralStringRef("TagMsg/"));
-static const KeyRange persistTagPoppedKeys = prefixRange(LiteralStringRef("TagPop/"));
+static const KeyRangeRef persistCurrentVersionKeys = KeyRangeRef("version/"_sr, "version0"_sr);
+static const KeyRange persistTagMessagesKeys = prefixRange("TagMsg/"_sr);
+static const KeyRange persistTagPoppedKeys = prefixRange("TagPop/"_sr);
 
 static Key persistTagMessagesKey(UID id, OldTag tag, Version version) {
 	BinaryWriter wr(Unversioned());
@@ -450,32 +447,14 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 		          "Restored");
 		addActor.send(traceRole(Role::TRANSACTION_LOG, interf.id()));
 
-		persistentDataVersion.init(LiteralStringRef("TLog.PersistentDataVersion"), cc.id);
-		persistentDataDurableVersion.init(LiteralStringRef("TLog.PersistentDataDurableVersion"), cc.id);
-		version.initMetric(LiteralStringRef("TLog.Version"), cc.id);
-		queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id);
+		persistentDataVersion.init("TLog.PersistentDataVersion"_sr, cc.id);
+		persistentDataDurableVersion.init("TLog.PersistentDataDurableVersion"_sr, cc.id);
+		version.initMetric("TLog.Version"_sr, cc.id);
+		queueCommittedVersion.initMetric("TLog.QueueCommittedVersion"_sr, cc.id);
 
 		specialCounter(cc, "Version", [this]() { return this->version.get(); });
 		specialCounter(cc, "SharedBytesInput", [tLogData]() { return tLogData->bytesInput; });
 		specialCounter(cc, "SharedBytesDurable", [tLogData]() { return tLogData->bytesDurable; });
-		specialCounter(
-		    cc, "KvstoreBytesUsed", [tLogData]() { return tLogData->persistentData->getStorageBytes().used; });
-		specialCounter(
-		    cc, "KvstoreBytesFree", [tLogData]() { return tLogData->persistentData->getStorageBytes().free; });
-		specialCounter(cc, "KvstoreBytesAvailable", [tLogData]() {
-			return tLogData->persistentData->getStorageBytes().available;
-		});
-		specialCounter(
-		    cc, "KvstoreBytesTotal", [tLogData]() { return tLogData->persistentData->getStorageBytes().total; });
-		specialCounter(
-		    cc, "QueueDiskBytesUsed", [tLogData]() { return tLogData->rawPersistentQueue->getStorageBytes().used; });
-		specialCounter(
-		    cc, "QueueDiskBytesFree", [tLogData]() { return tLogData->rawPersistentQueue->getStorageBytes().free; });
-		specialCounter(cc, "QueueDiskBytesAvailable", [tLogData]() {
-			return tLogData->rawPersistentQueue->getStorageBytes().available;
-		});
-		specialCounter(
-		    cc, "QueueDiskBytesTotal", [tLogData]() { return tLogData->rawPersistentQueue->getStorageBytes().total; });
 		specialCounter(cc, "ActivePeekStreams", [tLogData]() { return tLogData->activePeekStreams; });
 	}
 
@@ -507,9 +486,9 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 ACTOR Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply, Reference<LogData> logData) {
 	state Version stopVersion = logData->version.get();
 
-	TEST(true); // TLog stopped by recovering master
-	TEST(logData->stopped); // LogData already stopped
-	TEST(!logData->stopped); // LogData not yet stopped
+	CODE_PROBE(true, "TLog stopped by recovering master");
+	CODE_PROBE(logData->stopped, "LogData already stopped");
+	CODE_PROBE(!logData->stopped, "LogData not yet stopped");
 
 	TraceEvent("TLogStop", logData->logId)
 	    .detail("Ver", stopVersion)
@@ -611,7 +590,7 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 	// Now that the changes we made to persistentData are durable, erase the data we moved from memory and the queue,
 	// increase bytesDurable accordingly, and update persistentDataDurableVersion.
 
-	TEST(anyData); // TLog moved data to persistentData
+	CODE_PROBE(anyData, "TLog moved data to persistentData");
 	logData->persistentDataDurableVersion = newPersistentDataVersion;
 
 	for (tag = logData->tag_data.begin(); tag != logData->tag_data.end(); ++tag) {
@@ -834,7 +813,7 @@ void commitMessages(Reference<LogData> self,
 		// Fill up the rest of this block
 		int bytes = (uint8_t*)r.getLengthPtr() - messages.begin();
 		if (bytes) {
-			TEST(true); // Splitting commit messages across multiple blocks
+			CODE_PROBE(true, "Splitting commit messages across multiple blocks");
 			messages1 = StringRef(block.end(), bytes);
 			block.append(block.arena(), messages.begin(), bytes);
 			self->messageBlocks.emplace_back(version, block);
@@ -1047,7 +1026,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 			}
 			if (sequenceData.isSet()) {
 				if (sequenceData.getFuture().get() != rep.end) {
-					TEST(true); // tlog peek second attempt ended at a different version
+					CODE_PROBE(true, "tlog peek second attempt ended at a different version");
 					replyPromise.sendError(operation_obsolete());
 					return Void();
 				}
@@ -1120,7 +1099,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 		auto& sequenceData = trackerData.sequence_version[sequence + 1];
 		if (sequenceData.isSet()) {
 			if (sequenceData.getFuture().get() != reply.end) {
-				TEST(true); // tlog peek second attempt ended at a different version (2)
+				CODE_PROBE(true, "tlog peek second attempt ended at a different version (2)");
 				replyPromise.sendError(operation_obsolete());
 				return Void();
 			}
@@ -1424,7 +1403,23 @@ ACTOR Future<Void> tLogCore(TLogData* self, Reference<LogData> logData) {
 	                                     logData->logId,
 	                                     SERVER_KNOBS->STORAGE_LOGGING_DELAY,
 	                                     &logData->cc,
-	                                     logData->logId.toString() + "/TLogMetrics"));
+	                                     logData->logId.toString() + "/TLogMetrics",
+	                                     [self = self](TraceEvent& te) {
+		                                     StorageBytes sbTlog = self->persistentData->getStorageBytes();
+		                                     te.detail("KvstoreBytesUsed", sbTlog.used);
+		                                     te.detail("KvstoreBytesFree", sbTlog.free);
+		                                     te.detail("KvstoreBytesAvailable", sbTlog.available);
+		                                     te.detail("KvstoreBytesTotal", sbTlog.total);
+		                                     te.detail("KvstoreBytesTemp", sbTlog.temp);
+
+		                                     StorageBytes sbQueue = self->rawPersistentQueue->getStorageBytes();
+		                                     te.detail("QueueDiskBytesUsed", sbQueue.used);
+		                                     te.detail("QueueDiskBytesFree", sbQueue.free);
+		                                     te.detail("QueueDiskBytesAvailable", sbQueue.available);
+		                                     te.detail("QueueDiskBytesTotal", sbQueue.total);
+		                                     te.detail("QueueDiskBytesTemp", sbQueue.temp);
+	                                     }));
+
 	logData->addActor.send(serveTLogInterface(self, logData->tli, logData, warningCollectorInput));
 
 	try {
@@ -1465,9 +1460,9 @@ ACTOR Future<Void> restorePersistentState(TLogData* self, LocalityData locality)
 	}
 
 	if (!fFormat.get().present()) {
-		RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), LiteralStringRef("\xff")), 1));
+		RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), "\xff"_sr), 1));
 		if (!v.size()) {
-			TEST(true); // The DB is completely empty, so it was never initialized.  Delete it.
+			CODE_PROBE(true, "The DB is completely empty, so it was never initialized.  Delete it.");
 			throw worker_removed();
 		} else {
 			// This should never happen
@@ -1553,7 +1548,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self, LocalityData locality)
 	try {
 		loop {
 			if (allRemoved.isReady()) {
-				TEST(true); // all tlogs removed during queue recovery
+				CODE_PROBE(true, "all tlogs removed during queue recovery");
 				throw worker_removed();
 			}
 			choose {
@@ -1586,7 +1581,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self, LocalityData locality)
 							logData->queueCommittedVersion.set(qe.version);
 
 							while (self->bytesInput - self->bytesDurable >= recoverMemoryLimit) {
-								TEST(true); // Flush excess data during TLog queue recovery
+								CODE_PROBE(true, "Flush excess data during TLog queue recovery");
 								TraceEvent("FlushLargeQueueDuringRecovery", self->dbgid)
 								    .detail("BytesInput", self->bytesInput)
 								    .detail("BytesDurable", self->bytesDurable)
@@ -1610,7 +1605,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self, LocalityData locality)
 	}
 
 	TraceEvent("TLogRestorePersistentStateDone", self->dbgid).detail("Took", now() - startt);
-	TEST(now() - startt >= 1.0); // TLog recovery took more than 1 second
+	CODE_PROBE(now() - startt >= 1.0, "TLog recovery took more than 1 second");
 
 	for (auto it : self->id_data) {
 		if (it.second->queueCommittedVersion.get() == 0) {
diff --git a/fdbserver/OldTLogServer_6_0.actor.cpp b/fdbserver/OldTLogServer_6_0.actor.cpp
index 2d016d4a6a..b2a31e09ff 100644
--- a/fdbserver/OldTLogServer_6_0.actor.cpp
+++ b/fdbserver/OldTLogServer_6_0.actor.cpp
@@ -148,7 +148,7 @@ private:
 			Standalone<StringRef> h = wait(self->queue->readNext(sizeof(uint32_t)));
 			if (h.size() != sizeof(uint32_t)) {
 				if (h.size()) {
-					TEST(true); // Zero fill within size field
+					CODE_PROBE(true, "Zero fill within size field");
 					int payloadSize = 0;
 					memcpy(&payloadSize, h.begin(), h.size());
 					zeroFillSize = sizeof(uint32_t) - h.size(); // zero fill the size itself
@@ -162,7 +162,7 @@ private:
 
 			Standalone<StringRef> e = wait(self->queue->readNext(payloadSize + 1));
 			if (e.size() != payloadSize + 1) {
-				TEST(true); // Zero fill within payload
+				CODE_PROBE(true, "Zero fill within payload");
 				zeroFillSize = payloadSize + 1 - e.size();
 				break;
 			}
@@ -176,7 +176,7 @@ private:
 			}
 		}
 		if (zeroFillSize) {
-			TEST(true); // Fixing a partial commit at the end of the tlog queue
+			CODE_PROBE(true, "Fixing a partial commit at the end of the tlog queue");
 			for (int i = 0; i < zeroFillSize; i++)
 				self->queue->push(StringRef((const uint8_t*)"", 1));
 		}
@@ -187,24 +187,18 @@ private:
 ////// Persistence format (for self->persistentData)
 
 // Immutable keys
-static const KeyValueRef persistFormat(LiteralStringRef("Format"), LiteralStringRef("FoundationDB/LogServer/2/4"));
-static const KeyRangeRef persistFormatReadableRange(LiteralStringRef("FoundationDB/LogServer/2/3"),
-                                                    LiteralStringRef("FoundationDB/LogServer/2/5"));
-static const KeyRangeRef persistRecoveryCountKeys =
-    KeyRangeRef(LiteralStringRef("DbRecoveryCount/"), LiteralStringRef("DbRecoveryCount0"));
+static const KeyValueRef persistFormat("Format"_sr, "FoundationDB/LogServer/2/4"_sr);
+static const KeyRangeRef persistFormatReadableRange("FoundationDB/LogServer/2/3"_sr, "FoundationDB/LogServer/2/5"_sr);
+static const KeyRangeRef persistRecoveryCountKeys = KeyRangeRef("DbRecoveryCount/"_sr, "DbRecoveryCount0"_sr);
 
 // Updated on updatePersistentData()
-static const KeyRangeRef persistCurrentVersionKeys =
-    KeyRangeRef(LiteralStringRef("version/"), LiteralStringRef("version0"));
-static const KeyRangeRef persistKnownCommittedVersionKeys =
-    KeyRangeRef(LiteralStringRef("knownCommitted/"), LiteralStringRef("knownCommitted0"));
-static const KeyRangeRef persistLocalityKeys =
-    KeyRangeRef(LiteralStringRef("Locality/"), LiteralStringRef("Locality0"));
-static const KeyRangeRef persistLogRouterTagsKeys =
-    KeyRangeRef(LiteralStringRef("LogRouterTags/"), LiteralStringRef("LogRouterTags0"));
-static const KeyRangeRef persistTxsTagsKeys = KeyRangeRef(LiteralStringRef("TxsTags/"), LiteralStringRef("TxsTags0"));
-static const KeyRange persistTagMessagesKeys = prefixRange(LiteralStringRef("TagMsg/"));
-static const KeyRange persistTagPoppedKeys = prefixRange(LiteralStringRef("TagPop/"));
+static const KeyRangeRef persistCurrentVersionKeys = KeyRangeRef("version/"_sr, "version0"_sr);
+static const KeyRangeRef persistKnownCommittedVersionKeys = KeyRangeRef("knownCommitted/"_sr, "knownCommitted0"_sr);
+static const KeyRangeRef persistLocalityKeys = KeyRangeRef("Locality/"_sr, "Locality0"_sr);
+static const KeyRangeRef persistLogRouterTagsKeys = KeyRangeRef("LogRouterTags/"_sr, "LogRouterTags0"_sr);
+static const KeyRangeRef persistTxsTagsKeys = KeyRangeRef("TxsTags/"_sr, "TxsTags0"_sr);
+static const KeyRange persistTagMessagesKeys = prefixRange("TagMsg/"_sr);
+static const KeyRange persistTagPoppedKeys = prefixRange("TagPop/"_sr);
 
 static Key persistTagMessagesKey(UID id, Tag tag, Version version) {
 	BinaryWriter wr(Unversioned());
@@ -539,10 +533,10 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 		          context);
 		addActor.send(traceRole(Role::TRANSACTION_LOG, interf.id()));
 
-		persistentDataVersion.init(LiteralStringRef("TLog.PersistentDataVersion"), cc.id);
-		persistentDataDurableVersion.init(LiteralStringRef("TLog.PersistentDataDurableVersion"), cc.id);
-		version.initMetric(LiteralStringRef("TLog.Version"), cc.id);
-		queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id);
+		persistentDataVersion.init("TLog.PersistentDataVersion"_sr, cc.id);
+		persistentDataDurableVersion.init("TLog.PersistentDataDurableVersion"_sr, cc.id);
+		version.initMetric("TLog.Version"_sr, cc.id);
+		queueCommittedVersion.initMetric("TLog.QueueCommittedVersion"_sr, cc.id);
 
 		specialCounter(cc, "Version", [this]() { return this->version.get(); });
 		specialCounter(cc, "QueueCommittedVersion", [this]() { return this->queueCommittedVersion.get(); });
@@ -554,24 +548,6 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 		specialCounter(cc, "SharedBytesDurable", [tLogData]() { return tLogData->bytesDurable; });
 		specialCounter(cc, "SharedOverheadBytesInput", [tLogData]() { return tLogData->overheadBytesInput; });
 		specialCounter(cc, "SharedOverheadBytesDurable", [tLogData]() { return tLogData->overheadBytesDurable; });
-		specialCounter(
-		    cc, "KvstoreBytesUsed", [tLogData]() { return tLogData->persistentData->getStorageBytes().used; });
-		specialCounter(
-		    cc, "KvstoreBytesFree", [tLogData]() { return tLogData->persistentData->getStorageBytes().free; });
-		specialCounter(cc, "KvstoreBytesAvailable", [tLogData]() {
-			return tLogData->persistentData->getStorageBytes().available;
-		});
-		specialCounter(
-		    cc, "KvstoreBytesTotal", [tLogData]() { return tLogData->persistentData->getStorageBytes().total; });
-		specialCounter(
-		    cc, "QueueDiskBytesUsed", [tLogData]() { return tLogData->rawPersistentQueue->getStorageBytes().used; });
-		specialCounter(
-		    cc, "QueueDiskBytesFree", [tLogData]() { return tLogData->rawPersistentQueue->getStorageBytes().free; });
-		specialCounter(cc, "QueueDiskBytesAvailable", [tLogData]() {
-			return tLogData->rawPersistentQueue->getStorageBytes().available;
-		});
-		specialCounter(
-		    cc, "QueueDiskBytesTotal", [tLogData]() { return tLogData->rawPersistentQueue->getStorageBytes().total; });
 		specialCounter(cc, "ActivePeekStreams", [tLogData]() { return tLogData->activePeekStreams; });
 	}
 
@@ -653,9 +629,9 @@ void TLogQueue::updateVersionSizes(const TLogQueueEntry& result, TLogData* tLog)
 ACTOR Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply, Reference<LogData> logData) {
 	state Version stopVersion = logData->version.get();
 
-	TEST(true); // TLog stopped by recovering master
-	TEST(logData->stopped); // logData already stopped
-	TEST(!logData->stopped); // logData not yet stopped
+	CODE_PROBE(true, "TLog stopped by recovering master");
+	CODE_PROBE(logData->stopped, "logData already stopped");
+	CODE_PROBE(!logData->stopped, "logData not yet stopped");
 
 	TraceEvent("TLogStop", logData->logId)
 	    .detail("Ver", stopVersion)
@@ -769,7 +745,7 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 	// Now that the changes we made to persistentData are durable, erase the data we moved from memory and the queue,
 	// increase bytesDurable accordingly, and update persistentDataDurableVersion.
 
-	TEST(anyData); // TLog moved data to persistentData
+	CODE_PROBE(anyData, "TLog moved data to persistentData");
 	logData->persistentDataDurableVersion = newPersistentDataVersion;
 
 	for (tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
@@ -1341,7 +1317,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 			}
 			if (sequenceData.isSet()) {
 				if (sequenceData.getFuture().get().first != rep.end) {
-					TEST(true); // tlog peek second attempt ended at a different version
+					CODE_PROBE(true, "tlog peek second attempt ended at a different version");
 					replyPromise.sendError(operation_obsolete());
 					return Void();
 				}
@@ -1439,7 +1415,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 		if (sequenceData.isSet()) {
 			trackerData.duplicatePeeks++;
 			if (sequenceData.getFuture().get().first != reply.end) {
-				TEST(true); // tlog peek second attempt ended at a different version (2)
+				CODE_PROBE(true, "tlog peek second attempt ended at a different version (2)");
 				replyPromise.sendError(operation_obsolete());
 				return Void();
 			}
@@ -1509,7 +1485,7 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 
 	wait(ioDegradedOrTimeoutError(
 	    c, SERVER_KNOBS->MAX_STORAGE_COMMIT_TIME, self->degraded, SERVER_KNOBS->TLOG_DEGRADED_DURATION));
-	if (g_network->isSimulated() && !g_simulator.speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
+	if (g_network->isSimulated() && !g_simulator->speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
 		wait(delay(6.0));
 	}
 	wait(self->queueCommitEnd.whenAtLeast(commitNumber - 1));
@@ -1546,7 +1522,7 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 		    .detail("LogId", logData->logId)
 		    .detail("Version", it->version.get())
 		    .detail("QueueVer", it->queueCommittedVersion.get());
-		TEST(true); // A TLog was replaced before having a chance to commit its queue
+		CODE_PROBE(true, "A TLog was replaced before having a chance to commit its queue");
 		it->queueCommittedVersion.set(it->version.get());
 	}
 	return Void();
@@ -2007,7 +1983,7 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 		when(TLogCommitRequest req = waitNext(tli.commit.getFuture())) {
 			//TraceEvent("TLogCommitReq", logData->logId).detail("Ver", req.version).detail("PrevVer", req.prevVersion).detail("LogVer", logData->version.get());
 			ASSERT(logData->isPrimary);
-			TEST(logData->stopped); // TLogCommitRequest while stopped
+			CODE_PROBE(logData->stopped, "TLogCommitRequest while stopped");
 			if (!logData->stopped)
 				logData->addActor.send(tLogCommit(self, req, logData, warningCollectorInput));
 			else
@@ -2241,7 +2217,23 @@ ACTOR Future<Void> tLogCore(TLogData* self,
 	                                     logData->logId,
 	                                     SERVER_KNOBS->STORAGE_LOGGING_DELAY,
 	                                     &logData->cc,
-	                                     logData->logId.toString() + "/TLogMetrics"));
+	                                     logData->logId.toString() + "/TLogMetrics",
+	                                     [self = self](TraceEvent& te) {
+		                                     StorageBytes sbTlog = self->persistentData->getStorageBytes();
+		                                     te.detail("KvstoreBytesUsed", sbTlog.used);
+		                                     te.detail("KvstoreBytesFree", sbTlog.free);
+		                                     te.detail("KvstoreBytesAvailable", sbTlog.available);
+		                                     te.detail("KvstoreBytesTotal", sbTlog.total);
+		                                     te.detail("KvstoreBytesTemp", sbTlog.temp);
+
+		                                     StorageBytes sbQueue = self->rawPersistentQueue->getStorageBytes();
+		                                     te.detail("QueueDiskBytesUsed", sbQueue.used);
+		                                     te.detail("QueueDiskBytesFree", sbQueue.free);
+		                                     te.detail("QueueDiskBytesAvailable", sbQueue.available);
+		                                     te.detail("QueueDiskBytesTotal", sbQueue.total);
+		                                     te.detail("QueueDiskBytesTemp", sbQueue.temp);
+	                                     }));
+
 	logData->addActor.send(serveTLogInterface(self, tli, logData, warningCollectorInput));
 	logData->addActor.send(cleanupPeekTrackers(logData.getPtr()));
 	logData->addActor.send(logPeekTrackers(logData.getPtr()));
@@ -2331,9 +2323,9 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 	}
 
 	if (!fFormat.get().present()) {
-		RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), LiteralStringRef("\xff")), 1));
+		RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), "\xff"_sr), 1));
 		if (!v.size()) {
-			TEST(true); // The DB is completely empty, so it was never initialized.  Delete it.
+			CODE_PROBE(true, "The DB is completely empty, so it was never initialized.  Delete it.");
 			throw worker_removed();
 		} else {
 			// This should never happen
@@ -2345,7 +2337,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 
 	state std::vector<Future<ErrorOr<Void>>> removed;
 
-	if (fFormat.get().get() == LiteralStringRef("FoundationDB/LogServer/2/3")) {
+	if (fFormat.get().get() == "FoundationDB/LogServer/2/3"_sr) {
 		// FIXME: need for upgrades from 5.X to 6.0, remove once this upgrade path is no longer needed
 		if (recovered.canBeSet())
 			recovered.send(Void());
@@ -2473,7 +2465,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 	try {
 		loop {
 			if (allRemoved.isReady()) {
-				TEST(true); // all tlogs removed during queue recovery
+				CODE_PROBE(true, "all tlogs removed during queue recovery");
 				throw worker_removed();
 			}
 			choose {
@@ -2503,7 +2495,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 							logData->queueCommittedVersion.set(qe.version);
 
 							while (self->bytesInput - self->bytesDurable >= recoverMemoryLimit) {
-								TEST(true); // Flush excess data during TLog queue recovery
+								CODE_PROBE(true, "Flush excess data during TLog queue recovery");
 								TraceEvent("FlushLargeQueueDuringRecovery", self->dbgid)
 								    .detail("BytesInput", self->bytesInput)
 								    .detail("BytesDurable", self->bytesDurable)
@@ -2527,7 +2519,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 	}
 
 	TraceEvent("TLogRestorePersistentStateDone", self->dbgid).detail("Took", now() - startt);
-	TEST(now() - startt >= 1.0); // TLog recovery took more than 1 second
+	CODE_PROBE(now() - startt >= 1.0, "TLog recovery took more than 1 second");
 
 	for (auto it : self->id_data) {
 		if (it.second->queueCommittedVersion.get() == 0) {
@@ -2770,8 +2762,10 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 	req.reply.send(recruited);
 
 	TraceEvent("TLogReady", logData->logId)
-	    .detail("AllTags", describe(req.allTags))
-	    .detail("Locality", logData->locality);
+	    .detail("Locality", logData->locality)
+	    .setMaxEventLength(11000)
+	    .setMaxFieldLength(10000)
+	    .detail("AllTags", describe(req.allTags));
 
 	updater = Void();
 	wait(tLogCore(self, logData, recruited, pulledRecoveryVersions));
diff --git a/fdbserver/OldTLogServer_6_2.actor.cpp b/fdbserver/OldTLogServer_6_2.actor.cpp
index ffb449aff5..7caeea9aba 100644
--- a/fdbserver/OldTLogServer_6_2.actor.cpp
+++ b/fdbserver/OldTLogServer_6_2.actor.cpp
@@ -156,7 +156,7 @@ private:
 			Standalone<StringRef> h = wait(self->queue->readNext(sizeof(uint32_t)));
 			if (h.size() != sizeof(uint32_t)) {
 				if (h.size()) {
-					TEST(true); // Zero fill within size field
+					CODE_PROBE(true, "Zero fill within size field");
 					int payloadSize = 0;
 					memcpy(&payloadSize, h.begin(), h.size());
 					zeroFillSize = sizeof(uint32_t) - h.size(); // zero fill the size itself
@@ -170,7 +170,7 @@ private:
 
 			Standalone<StringRef> e = wait(self->queue->readNext(payloadSize + 1));
 			if (e.size() != payloadSize + 1) {
-				TEST(true); // Zero fill within payload
+				CODE_PROBE(true, "Zero fill within payload");
 				zeroFillSize = payloadSize + 1 - e.size();
 				break;
 			}
@@ -186,7 +186,7 @@ private:
 			}
 		}
 		if (zeroFillSize) {
-			TEST(true); // Fixing a partial commit at the end of the tlog queue
+			CODE_PROBE(true, "Fixing a partial commit at the end of the tlog queue");
 			for (int i = 0; i < zeroFillSize; i++)
 				self->queue->push(StringRef((const uint8_t*)"", 1));
 		}
@@ -199,28 +199,21 @@ private:
 // Immutable keys
 // persistFormat has been mostly invalidated by TLogVersion, and can probably be removed when
 // 4.6's TLog code is removed.
-static const KeyValueRef persistFormat(LiteralStringRef("Format"), LiteralStringRef("FoundationDB/LogServer/3/0"));
-static const KeyRangeRef persistFormatReadableRange(LiteralStringRef("FoundationDB/LogServer/3/0"),
-                                                    LiteralStringRef("FoundationDB/LogServer/4/0"));
-static const KeyRangeRef persistProtocolVersionKeys(LiteralStringRef("ProtocolVersion/"),
-                                                    LiteralStringRef("ProtocolVersion0"));
-static const KeyRangeRef persistRecoveryCountKeys =
-    KeyRangeRef(LiteralStringRef("DbRecoveryCount/"), LiteralStringRef("DbRecoveryCount0"));
+static const KeyValueRef persistFormat("Format"_sr, "FoundationDB/LogServer/3/0"_sr);
+static const KeyRangeRef persistFormatReadableRange("FoundationDB/LogServer/3/0"_sr, "FoundationDB/LogServer/4/0"_sr);
+static const KeyRangeRef persistProtocolVersionKeys("ProtocolVersion/"_sr, "ProtocolVersion0"_sr);
+static const KeyRangeRef persistRecoveryCountKeys = KeyRangeRef("DbRecoveryCount/"_sr, "DbRecoveryCount0"_sr);
 
 // Updated on updatePersistentData()
-static const KeyRangeRef persistCurrentVersionKeys =
-    KeyRangeRef(LiteralStringRef("version/"), LiteralStringRef("version0"));
-static const KeyRangeRef persistKnownCommittedVersionKeys =
-    KeyRangeRef(LiteralStringRef("knownCommitted/"), LiteralStringRef("knownCommitted0"));
-static const KeyRef persistRecoveryLocationKey = KeyRef(LiteralStringRef("recoveryLocation"));
-static const KeyRangeRef persistLocalityKeys =
-    KeyRangeRef(LiteralStringRef("Locality/"), LiteralStringRef("Locality0"));
-static const KeyRangeRef persistLogRouterTagsKeys =
-    KeyRangeRef(LiteralStringRef("LogRouterTags/"), LiteralStringRef("LogRouterTags0"));
-static const KeyRangeRef persistTxsTagsKeys = KeyRangeRef(LiteralStringRef("TxsTags/"), LiteralStringRef("TxsTags0"));
-static const KeyRange persistTagMessagesKeys = prefixRange(LiteralStringRef("TagMsg/"));
-static const KeyRange persistTagMessageRefsKeys = prefixRange(LiteralStringRef("TagMsgRef/"));
-static const KeyRange persistTagPoppedKeys = prefixRange(LiteralStringRef("TagPop/"));
+static const KeyRangeRef persistCurrentVersionKeys = KeyRangeRef("version/"_sr, "version0"_sr);
+static const KeyRangeRef persistKnownCommittedVersionKeys = KeyRangeRef("knownCommitted/"_sr, "knownCommitted0"_sr);
+static const KeyRef persistRecoveryLocationKey = KeyRef("recoveryLocation"_sr);
+static const KeyRangeRef persistLocalityKeys = KeyRangeRef("Locality/"_sr, "Locality0"_sr);
+static const KeyRangeRef persistLogRouterTagsKeys = KeyRangeRef("LogRouterTags/"_sr, "LogRouterTags0"_sr);
+static const KeyRangeRef persistTxsTagsKeys = KeyRangeRef("TxsTags/"_sr, "TxsTags0"_sr);
+static const KeyRange persistTagMessagesKeys = prefixRange("TagMsg/"_sr);
+static const KeyRange persistTagMessageRefsKeys = prefixRange("TagMsgRef/"_sr);
+static const KeyRange persistTagPoppedKeys = prefixRange("TagPop/"_sr);
 
 static Key persistTagMessagesKey(UID id, Tag tag, Version version) {
 	BinaryWriter wr(Unversioned());
@@ -623,10 +616,10 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 		          context);
 		addActor.send(traceRole(Role::TRANSACTION_LOG, interf.id()));
 
-		persistentDataVersion.init(LiteralStringRef("TLog.PersistentDataVersion"), cc.id);
-		persistentDataDurableVersion.init(LiteralStringRef("TLog.PersistentDataDurableVersion"), cc.id);
-		version.initMetric(LiteralStringRef("TLog.Version"), cc.id);
-		queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id);
+		persistentDataVersion.init("TLog.PersistentDataVersion"_sr, cc.id);
+		persistentDataDurableVersion.init("TLog.PersistentDataDurableVersion"_sr, cc.id);
+		version.initMetric("TLog.Version"_sr, cc.id);
+		queueCommittedVersion.initMetric("TLog.QueueCommittedVersion"_sr, cc.id);
 
 		specialCounter(cc, "Version", [this]() { return this->version.get(); });
 		specialCounter(cc, "QueueCommittedVersion", [this]() { return this->queueCommittedVersion.get(); });
@@ -641,24 +634,6 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 		specialCounter(cc, "SharedBytesDurable", [tLogData]() { return tLogData->bytesDurable; });
 		specialCounter(cc, "SharedOverheadBytesInput", [tLogData]() { return tLogData->overheadBytesInput; });
 		specialCounter(cc, "SharedOverheadBytesDurable", [tLogData]() { return tLogData->overheadBytesDurable; });
-		specialCounter(
-		    cc, "KvstoreBytesUsed", [tLogData]() { return tLogData->persistentData->getStorageBytes().used; });
-		specialCounter(
-		    cc, "KvstoreBytesFree", [tLogData]() { return tLogData->persistentData->getStorageBytes().free; });
-		specialCounter(cc, "KvstoreBytesAvailable", [tLogData]() {
-			return tLogData->persistentData->getStorageBytes().available;
-		});
-		specialCounter(
-		    cc, "KvstoreBytesTotal", [tLogData]() { return tLogData->persistentData->getStorageBytes().total; });
-		specialCounter(
-		    cc, "QueueDiskBytesUsed", [tLogData]() { return tLogData->rawPersistentQueue->getStorageBytes().used; });
-		specialCounter(
-		    cc, "QueueDiskBytesFree", [tLogData]() { return tLogData->rawPersistentQueue->getStorageBytes().free; });
-		specialCounter(cc, "QueueDiskBytesAvailable", [tLogData]() {
-			return tLogData->rawPersistentQueue->getStorageBytes().available;
-		});
-		specialCounter(
-		    cc, "QueueDiskBytesTotal", [tLogData]() { return tLogData->rawPersistentQueue->getStorageBytes().total; });
 		specialCounter(cc, "PeekMemoryReserved", [tLogData]() { return tLogData->peekMemoryLimiter.activePermits(); });
 		specialCounter(cc, "PeekMemoryRequestsStalled", [tLogData]() { return tLogData->peekMemoryLimiter.waiters(); });
 		specialCounter(cc, "ActivePeekStreams", [tLogData]() { return tLogData->activePeekStreams; });
@@ -756,9 +731,9 @@ void TLogQueue::updateVersionSizes(const TLogQueueEntry& result,
 ACTOR Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply, Reference<LogData> logData) {
 	state Version stopVersion = logData->version.get();
 
-	TEST(true); // TLog stopped by recovering master
-	TEST(logData->stopped); // logData already stopped
-	TEST(!logData->stopped); // logData not yet stopped
+	CODE_PROBE(true, "TLog stopped by recovering master");
+	CODE_PROBE(logData->stopped, "logData already stopped");
+	CODE_PROBE(!logData->stopped, "logData not yet stopped");
 
 	TraceEvent("TLogStop", logData->logId)
 	    .detail("Ver", stopVersion)
@@ -1042,7 +1017,7 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 	// Now that the changes we made to persistentData are durable, erase the data we moved from memory and the queue,
 	// increase bytesDurable accordingly, and update persistentDataDurableVersion.
 
-	TEST(anyData); // TLog moved data to persistentData
+	CODE_PROBE(anyData, "TLog moved data to persistentData");
 	logData->persistentDataDurableVersion = newPersistentDataVersion;
 
 	for (tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
@@ -1680,7 +1655,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 			}
 			if (sequenceData.isSet()) {
 				if (sequenceData.getFuture().get().first != rep.end) {
-					TEST(true); // tlog peek second attempt ended at a different version
+					CODE_PROBE(true, "tlog peek second attempt ended at a different version");
 					replyPromise.sendError(operation_obsolete());
 					return Void();
 				}
@@ -1868,7 +1843,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 		if (sequenceData.isSet()) {
 			trackerData.duplicatePeeks++;
 			if (sequenceData.getFuture().get().first != reply.end) {
-				TEST(true); // tlog peek second attempt ended at a different version (2)
+				CODE_PROBE(true, "tlog peek second attempt ended at a different version (2)");
 				replyPromise.sendError(operation_obsolete());
 				return Void();
 			}
@@ -1923,14 +1898,14 @@ ACTOR Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Ref
 }
 
 ACTOR Future<Void> watchDegraded(TLogData* self) {
-	if (g_network->isSimulated() && g_simulator.speedUpSimulation) {
+	if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
 		return Void();
 	}
 
 	wait(lowPriorityDelay(SERVER_KNOBS->TLOG_DEGRADED_DURATION));
 
 	TraceEvent(SevWarnAlways, "TLogDegraded", self->dbgid).log();
-	TEST(true); // TLog degraded
+	CODE_PROBE(true, "TLog degraded");
 	self->degraded->set(true);
 	return Void();
 }
@@ -1950,7 +1925,7 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 
 	state Future<Void> degraded = watchDegraded(self);
 	wait(c);
-	if (g_network->isSimulated() && !g_simulator.speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
+	if (g_network->isSimulated() && !g_simulator->speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
 		wait(delay(6.0));
 	}
 	degraded.cancel();
@@ -1988,7 +1963,7 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 		    .detail("LogId", logData->logId)
 		    .detail("Version", it->version.get())
 		    .detail("QueueVer", it->queueCommittedVersion.get());
-		TEST(true); // A TLog was replaced before having a chance to commit its queue
+		CODE_PROBE(true, "A TLog was replaced before having a chance to commit its queue");
 		it->queueCommittedVersion.set(it->version.get());
 	}
 	return Void();
@@ -2452,7 +2427,7 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 		when(TLogCommitRequest req = waitNext(tli.commit.getFuture())) {
 			//TraceEvent("TLogCommitReq", logData->logId).detail("Ver", req.version).detail("PrevVer", req.prevVersion).detail("LogVer", logData->version.get());
 			ASSERT(logData->isPrimary);
-			TEST(logData->stopped); // TLogCommitRequest while stopped
+			CODE_PROBE(logData->stopped, "TLogCommitRequest while stopped");
 			if (!logData->stopped)
 				logData->addActor.send(tLogCommit(self, req, logData, warningCollectorInput));
 			else
@@ -2701,7 +2676,23 @@ ACTOR Future<Void> tLogCore(TLogData* self,
 	                                     logData->logId,
 	                                     SERVER_KNOBS->STORAGE_LOGGING_DELAY,
 	                                     &logData->cc,
-	                                     logData->logId.toString() + "/TLogMetrics"));
+	                                     logData->logId.toString() + "/TLogMetrics",
+	                                     [self = self](TraceEvent& te) {
+		                                     StorageBytes sbTlog = self->persistentData->getStorageBytes();
+		                                     te.detail("KvstoreBytesUsed", sbTlog.used);
+		                                     te.detail("KvstoreBytesFree", sbTlog.free);
+		                                     te.detail("KvstoreBytesAvailable", sbTlog.available);
+		                                     te.detail("KvstoreBytesTotal", sbTlog.total);
+		                                     te.detail("KvstoreBytesTemp", sbTlog.temp);
+
+		                                     StorageBytes sbQueue = self->rawPersistentQueue->getStorageBytes();
+		                                     te.detail("QueueDiskBytesUsed", sbQueue.used);
+		                                     te.detail("QueueDiskBytesFree", sbQueue.free);
+		                                     te.detail("QueueDiskBytesAvailable", sbQueue.available);
+		                                     te.detail("QueueDiskBytesTotal", sbQueue.total);
+		                                     te.detail("QueueDiskBytesTemp", sbQueue.temp);
+	                                     }));
+
 	logData->addActor.send(serveTLogInterface(self, tli, logData, warningCollectorInput));
 	logData->addActor.send(cleanupPeekTrackers(logData.getPtr()));
 	logData->addActor.send(logPeekTrackers(logData.getPtr()));
@@ -2799,9 +2790,9 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 	}
 
 	if (!fFormat.get().present()) {
-		RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), LiteralStringRef("\xff")), 1));
+		RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), "\xff"_sr), 1));
 		if (!v.size()) {
-			TEST(true); // The DB is completely empty, so it was never initialized.  Delete it.
+			CODE_PROBE(true, "The DB is completely empty, so it was never initialized.  Delete it.");
 			throw worker_removed();
 		} else {
 			// This should never happen
@@ -2813,7 +2804,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 
 	state std::vector<Future<ErrorOr<Void>>> removed;
 
-	ASSERT(fFormat.get().get() == LiteralStringRef("FoundationDB/LogServer/3/0"));
+	ASSERT(fFormat.get().get() == "FoundationDB/LogServer/3/0"_sr);
 
 	ASSERT(fVers.get().size() == fRecoverCounts.get().size());
 
@@ -2949,7 +2940,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 			throw end_of_stream();
 		loop {
 			if (allRemoved.isReady()) {
-				TEST(true); // all tlogs removed during queue recovery
+				CODE_PROBE(true, "all tlogs removed during queue recovery");
 				throw worker_removed();
 			}
 			choose {
@@ -2980,7 +2971,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 							logData->queueCommittedVersion.set(qe.version);
 
 							while (self->bytesInput - self->bytesDurable >= recoverMemoryLimit) {
-								TEST(true); // Flush excess data during TLog queue recovery
+								CODE_PROBE(true, "Flush excess data during TLog queue recovery");
 								TraceEvent("FlushLargeQueueDuringRecovery", self->dbgid)
 								    .detail("LogId", logData->logId)
 								    .detail("BytesInput", self->bytesInput)
@@ -3010,7 +3001,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 	}
 
 	TraceEvent("TLogRestorePersistentStateDone", self->dbgid).detail("Took", now() - startt);
-	TEST(now() - startt >= 1.0); // TLog recovery took more than 1 second
+	CODE_PROBE(now() - startt >= 1.0, "TLog recovery took more than 1 second");
 
 	for (auto it : self->id_data) {
 		if (it.second->queueCommittedVersion.get() == 0) {
@@ -3261,8 +3252,10 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 	req.reply.send(recruited);
 
 	TraceEvent("TLogReady", logData->logId)
-	    .detail("AllTags", describe(req.allTags))
-	    .detail("Locality", logData->locality);
+	    .detail("Locality", logData->locality)
+	    .setMaxEventLength(11000)
+	    .setMaxFieldLength(10000)
+	    .detail("AllTags", describe(req.allTags));
 
 	updater = Void();
 	wait(tLogCore(self, logData, recruited, pulledRecoveryVersions));
diff --git a/fdbserver/PaxosConfigConsumer.actor.cpp b/fdbserver/PaxosConfigConsumer.actor.cpp
index bbe5fd7e8e..3515c1db5e 100644
--- a/fdbserver/PaxosConfigConsumer.actor.cpp
+++ b/fdbserver/PaxosConfigConsumer.actor.cpp
@@ -53,10 +53,18 @@ class GetCommittedVersionQuorum {
 	Version largestCompactedResponse{ 0 };
 	// Last durably committed version.
 	Version lastSeenVersion;
+	// Largest compacted version on the existing ConfigNodes.
+	Version largestCompacted;
 	size_t totalRepliesReceived{ 0 };
 	size_t maxAgreement{ 0 };
+	// Stores the largest live version out of all the responses.
+	Version largestLive{ 0 };
 	// Stores the largest committed version out of all responses.
 	Version largestCommitted{ 0 };
+	bool allowSpecialCaseRollforward_{ false };
+	// True if a quorum has zero as their committed version. See explanation
+	// comment below.
+	bool specialZeroQuorum{ false };
 
 	// Sends rollback/rollforward messages to any nodes that are not up to date
 	// with the latest committed version as determined by the quorum. Should
@@ -67,9 +75,18 @@ class GetCommittedVersionQuorum {
 	                                     Version lastCompacted,
 	                                     ConfigFollowerInterface cfi) {
 		state Version target = quorumVersion.lastCommitted;
+		// TraceEvent("ConsumerUpdateNodeStart")
+		// 		.detail("NodeAddress", cfi.address())
+		// 		.detail("Target", target)
+		// 		.detail("NodeVersionLastCommitted", nodeVersion.lastCommitted)
+		// 		.detail("NodeVersionSecondToLastCommitted", nodeVersion.secondToLastCommitted)
+		// 		.detail("QuorumVersionLastCommitted", quorumVersion.lastCommitted)
+		// 		.detail("QuorumVersionSecondToLastCommitted", quorumVersion.secondToLastCommitted)
+		// 		.detail("LargestCompacted", self->largestCompacted);
 		if (nodeVersion.lastCommitted == target) {
 			return Void();
 		}
+
 		if (nodeVersion.lastCommitted < target) {
 			state Optional<Version> rollback;
 			if (nodeVersion.lastCommitted > quorumVersion.secondToLastCommitted) {
@@ -83,7 +100,7 @@ class GetCommittedVersionQuorum {
 				// On the other hand, if the node is on an older committed
 				// version, it's possible the version it is on was never made
 				// durable. To be safe, roll it back by one version.
-				rollback = std::max(nodeVersion.lastCommitted - 1, Version{ 0 });
+				rollback = std::max(nodeVersion.lastCommitted - 1, self->largestCompacted);
 			}
 
 			if (rollback.present()) {
@@ -118,18 +135,34 @@ class GetCommittedVersionQuorum {
 				                                       ConfigFollowerGetChangesRequest{ lastSeenVersion, target }),
 				                      SERVER_KNOBS->GET_COMMITTED_VERSION_TIMEOUT));
 
+				// TraceEvent("ConsumerUpdateNodeSendingRollforward")
+				// 		.detail("NodeAddress", cfi.address())
+				// 		.detail("RollbackTo", rollback)
+				// 		.detail("LastKnownCommitted", nodeVersion.lastCommitted)
+				// 		.detail("Target", target)
+				// 		.detail("ChangesSize", reply.changes.size())
+				// 		.detail("AnnotationsSize", reply.annotations.size())
+				// 		.detail("LargestCompacted", self->largestCompactedResponse)
+				// 		.detail("SpecialZeroQuorum", self->specialZeroQuorum);
 				if (cfi.hostname.present()) {
 					wait(timeoutError(
-					    retryGetReplyFromHostname(
-					        ConfigFollowerRollforwardRequest{
-					            rollback, nodeVersion.lastCommitted, target, reply.changes, reply.annotations },
-					        cfi.hostname.get(),
-					        WLTOKEN_CONFIGFOLLOWER_ROLLFORWARD),
+					    retryGetReplyFromHostname(ConfigFollowerRollforwardRequest{ rollback,
+					                                                                nodeVersion.lastCommitted,
+					                                                                target,
+					                                                                reply.changes,
+					                                                                reply.annotations,
+					                                                                self->specialZeroQuorum },
+					                              cfi.hostname.get(),
+					                              WLTOKEN_CONFIGFOLLOWER_ROLLFORWARD),
 					    SERVER_KNOBS->GET_COMMITTED_VERSION_TIMEOUT));
 				} else {
 					wait(timeoutError(
-					    cfi.rollforward.getReply(ConfigFollowerRollforwardRequest{
-					        rollback, nodeVersion.lastCommitted, target, reply.changes, reply.annotations }),
+					    cfi.rollforward.getReply(ConfigFollowerRollforwardRequest{ rollback,
+					                                                               nodeVersion.lastCommitted,
+					                                                               target,
+					                                                               reply.changes,
+					                                                               reply.annotations,
+					                                                               self->specialZeroQuorum }),
 					    SERVER_KNOBS->GET_COMMITTED_VERSION_TIMEOUT));
 				}
 			} catch (Error& e) {
@@ -139,7 +172,7 @@ class GetCommittedVersionQuorum {
 					// one of these errors in response to a get changes or
 					// rollforward request. The retry loop should handle this
 					// case.
-					TraceEvent(SevInfo, "ConfigNodeRollforwardError").error(e);
+					TraceEvent(SevInfo, "ConsumerConfigNodeRollforwardError").error(e);
 				} else {
 					throw;
 				}
@@ -163,9 +196,19 @@ class GetCommittedVersionQuorum {
 				    SERVER_KNOBS->GET_COMMITTED_VERSION_TIMEOUT));
 			}
 
+			if (!reply.registered) {
+				// ConfigNodes serve their GetCommittedVersion interface before
+				// being registered to allow them to be rolled forward.
+				// However, their responses should not count towards the
+				// quorum.
+				throw future_version();
+			}
+
 			++self->totalRepliesReceived;
 			self->largestCompactedResponse = std::max(self->largestCompactedResponse, reply.lastCompacted);
 			state Version lastCompacted = reply.lastCompacted;
+			self->committed[cfi.address()] = reply.lastCommitted;
+			self->largestLive = std::max(self->largestLive, reply.lastLive);
 			self->largestCommitted = std::max(self->largestCommitted, reply.lastCommitted);
 			state CommittedVersions committedVersions = CommittedVersions{ self->lastSeenVersion, reply.lastCommitted };
 			if (self->priorVersions.find(committedVersions.lastCommitted) == self->priorVersions.end()) {
@@ -174,7 +217,59 @@ class GetCommittedVersionQuorum {
 			auto& nodes = self->replies[committedVersions.lastCommitted];
 			nodes.push_back(cfi);
 			self->maxAgreement = std::max(nodes.size(), self->maxAgreement);
+			// TraceEvent("ConsumerGetCommittedVersionReply")
+			// 	.detail("From", cfi.address())
+			// 	.detail("LastCompactedVersion", lastCompacted)
+			// 	.detail("LastCommittedVersion", reply.lastCommitted)
+			// 	.detail("LastSeenVersion", self->lastSeenVersion)
+			// 	.detail("Replies", self->totalRepliesReceived)
+			// 	.detail("RepliesMatchingVersion", nodes.size())
+			// 	.detail("Coordinators", self->cfis.size())
+			// 	.detail("AllowSpecialCaseRollforward", self->allowSpecialCaseRollforward_);
 			if (nodes.size() >= self->cfis.size() / 2 + 1) {
+				// A quorum at version 0 should use any higher committed
+				// version seen instead of 0. Imagine the following scenario
+				// with three coordinators:
+				//
+				//      t0   t1   t2  t3
+				//   A  1    1    |   1
+				//   B  1   dies  |   0
+				//   C  0    0    |   0
+				//
+				// At t0, a value at version 1 is committed to A and B. At t1,
+				// B dies, and now the value only exists on A. At t2, a change
+				// coordinators command is executed by a client, causing a
+				// recovery. When the ConfigBroadcaster comes online and
+				// attempts to read the state of the previous coordinators (at
+				// time t3) so it can transfer it to the new coordinators, 2/3
+				// ConfigNodes are unregistered and only know about version 0.
+				// Quorum logic dictates the committed version is, thus,
+				// version 0. But we know a majority committed version 1. This
+				// is a special case error where a ConfigNode losing data is
+				// immediately followed by a coordinator change and recovery,
+				// and 0 is a special case. Imagine the following if C instead
+				// has had some values committed:
+				//
+				//      t0   t1   t2   t3  t4
+				//   A  1    2    2    |   2
+				//   B  1    2   dies  |   0
+				//   C  1    1    1    |   1
+				//
+				// In this case, there is no quorum, and so all nodes would
+				// (correctly) be rolled forward to version 2. Since a node
+				// losing data is equivalent to saying it has a committed
+				// version of 0, we must treat a quorum of nodes at version 0
+				// as a special case, and instead use the largest committed
+				// version we've seen as the quorum version. This does not
+				// affect correctness because version 0 means nothing was
+				// committed, so there shouldn't be an issue rolling those
+				// nodes forward.
+				if (self->allowSpecialCaseRollforward_ && committedVersions.lastCommitted == 0 &&
+				    self->largestCommitted > 0) {
+					self->specialZeroQuorum = true;
+					committedVersions = CommittedVersions{ 0, self->largestCommitted };
+				}
+
 				// A quorum of ConfigNodes agree on the latest committed version.
 				if (self->quorumVersion.canBeSet()) {
 					self->quorumVersion.send(QuorumVersion{ committedVersions, true });
@@ -186,7 +281,8 @@ class GetCommittedVersionQuorum {
 				// but the node we just got a reply from is not one of them. We may
 				// need to roll it forward or back.
 				QuorumVersion quorumVersion = wait(self->quorumVersion.getFuture());
-				ASSERT(committedVersions.lastCommitted != quorumVersion.versions.lastCommitted);
+				ASSERT(committedVersions.lastCommitted != quorumVersion.versions.lastCommitted ||
+				       self->specialZeroQuorum);
 				wait(self->updateNode(self, committedVersions, quorumVersion.versions, lastCompacted, cfi));
 			} else if (self->maxAgreement + (self->cfis.size() - self->totalRepliesReceived) <
 			           (self->cfis.size() / 2 + 1)) {
@@ -213,13 +309,18 @@ class GetCommittedVersionQuorum {
 		} catch (Error& e) {
 			// Count a timeout as a reply.
 			++self->totalRepliesReceived;
+			// TraceEvent("ConsumerGetCommittedVersionError").error(e)
+			// 	.detail("From", cfi.address())
+			// 	.detail("Replies", self->totalRepliesReceived)
+			// 	.detail("Coordinators", self->cfis.size());
 			if (e.code() == error_code_version_already_compacted) {
 				if (self->quorumVersion.canBeSet()) {
 					// Calling sendError could delete self
 					auto local = self->quorumVersion;
 					local.sendError(e);
 				}
-			} else if (e.code() != error_code_timed_out && e.code() != error_code_broken_promise) {
+			} else if (e.code() != error_code_timed_out && e.code() != error_code_future_version &&
+			           e.code() != error_code_broken_promise) {
 				if (self->quorumVersion.canBeSet()) {
 					// Calling sendError could delete self
 					auto local = self->quorumVersion;
@@ -231,6 +332,7 @@ class GetCommittedVersionQuorum {
 				    std::accumulate(self->replies.begin(), self->replies.end(), 0, [](int value, auto const& p) {
 					    return value + p.second.size();
 				    });
+
 				if (nonTimeoutReplies >= self->cfis.size() / 2 + 1) {
 					// Make sure to trigger the quorumVersion if a timeout
 					// occurred, a quorum disagree on the committed version,
@@ -239,6 +341,14 @@ class GetCommittedVersionQuorum {
 					// back the largest committed version seen.
 					self->quorumVersion.send(
 					    QuorumVersion{ CommittedVersions{ self->lastSeenVersion, self->largestCommitted }, false });
+
+					if (e.code() == error_code_future_version) {
+						wait(self->updateNode(self,
+						                      CommittedVersions{ self->lastSeenVersion, self->largestCommitted },
+						                      self->quorumVersion.getFuture().get().versions,
+						                      self->largestCompactedResponse,
+						                      cfi));
+					}
 				} else if (!self->quorumVersion.isSet()) {
 					// Otherwise, if a quorum agree on the committed version,
 					// some other occurred. Notify the caller of it.
@@ -253,8 +363,10 @@ class GetCommittedVersionQuorum {
 	}
 
 public:
-	explicit GetCommittedVersionQuorum(std::vector<ConfigFollowerInterface> const& cfis, Version lastSeenVersion)
-	  : cfis(cfis), lastSeenVersion(lastSeenVersion) {}
+	explicit GetCommittedVersionQuorum(std::vector<ConfigFollowerInterface> const& cfis,
+	                                   Version lastSeenVersion,
+	                                   Version largestCompacted)
+	  : cfis(cfis), lastSeenVersion(lastSeenVersion), largestCompacted(largestCompacted) {}
 	Future<QuorumVersion> getCommittedVersion() {
 		ASSERT(!isReady()); // ensures this function is not accidentally called before resetting state
 		for (const auto& cfi : cfis) {
@@ -273,6 +385,7 @@ public:
 		ASSERT(isReady());
 		return replies.at(quorumVersion.getFuture().get().versions.lastCommitted);
 	}
+	Version getLargestLive() const { return largestLive; }
 	Version getSmallestCommitted() const {
 		if (committed.size() == cfis.size()) {
 			Version smallest = MAX_VERSION;
@@ -283,6 +396,8 @@ public:
 		}
 		return ::invalidVersion;
 	}
+	void allowSpecialCaseRollforward() { allowSpecialCaseRollforward_ = true; }
+	bool isSpecialZeroQuorum() const { return specialZeroQuorum; }
 	Future<Void> complete() const { return waitForAll(actors); }
 };
 
@@ -293,9 +408,14 @@ class PaxosConfigConsumerImpl {
 	Version compactionVersion{ 0 };
 	double pollingInterval;
 	Optional<double> compactionInterval;
+	bool allowSpecialCaseRollforward_{ false };
+	bool readPreviousCoordinators{ false };
 	UID id;
 
 	ACTOR static Future<Version> getCommittedVersion(PaxosConfigConsumerImpl* self) {
+		if (self->allowSpecialCaseRollforward_) {
+			self->getCommittedVersionQuorum.allowSpecialCaseRollforward();
+		}
 		QuorumVersion quorumVersion = wait(self->getCommittedVersionQuorum.getCommittedVersion());
 		if (!quorumVersion.isQuorum) {
 			throw failed_to_reach_quorum();
@@ -357,29 +477,37 @@ class PaxosConfigConsumerImpl {
 				                                       &ConfigFollowerInterface::getSnapshotAndChanges,
 				                                       ConfigFollowerGetSnapshotAndChangesRequest{ committedVersion }),
 				                      SERVER_KNOBS->GET_SNAPSHOT_AND_CHANGES_TIMEOUT));
+				Version smallestCommitted = self->getCommittedVersionQuorum.getSmallestCommitted();
 				TraceEvent(SevDebug, "ConfigConsumerGotSnapshotAndChanges", self->id)
 				    .detail("SnapshotVersion", reply.snapshotVersion)
 				    .detail("SnapshotSize", reply.snapshot.size())
 				    .detail("ChangesVersion", committedVersion)
 				    .detail("ChangesSize", reply.changes.size())
-				    .detail("AnnotationsSize", reply.annotations.size());
+				    .detail("AnnotationsSize", reply.annotations.size())
+				    .detail("LargestLiveVersion", self->getCommittedVersionQuorum.getLargestLive())
+				    .detail("SmallestCommitted", smallestCommitted);
 				ASSERT_GE(committedVersion, self->lastSeenVersion);
 				self->lastSeenVersion = committedVersion;
-				Version smallestCommitted = self->getCommittedVersionQuorum.getSmallestCommitted();
 				self->compactionVersion = std::max(self->compactionVersion, smallestCommitted);
 				broadcaster->applySnapshotAndChanges(std::move(reply.snapshot),
 				                                     reply.snapshotVersion,
 				                                     reply.changes,
 				                                     committedVersion,
 				                                     reply.annotations,
-				                                     self->getCommittedVersionQuorum.getReadReplicas());
+				                                     self->getCommittedVersionQuorum.getReadReplicas(),
+				                                     self->getCommittedVersionQuorum.getLargestLive(),
+				                                     self->readPreviousCoordinators);
 				wait(self->getCommittedVersionQuorum.complete());
+				if (self->allowSpecialCaseRollforward_) {
+					self->allowSpecialCaseRollforward_ = false;
+				}
 				break;
 			} catch (Error& e) {
 				if (e.code() == error_code_failed_to_reach_quorum) {
 					wait(self->getCommittedVersionQuorum.complete());
 				} else if (e.code() != error_code_timed_out && e.code() != error_code_broken_promise &&
-				           e.code() != error_code_version_already_compacted && e.code() != error_code_process_behind) {
+				           e.code() != error_code_version_already_compacted && e.code() != error_code_process_behind &&
+				           e.code() != error_code_future_version) {
 					throw;
 				}
 				wait(delayJittered(0.1));
@@ -404,7 +532,8 @@ class PaxosConfigConsumerImpl {
 				// ConfigNodes changes to 1, 1, 2, the committed version
 				// returned would be 1.
 				if (committedVersion > self->lastSeenVersion) {
-					ASSERT(self->getCommittedVersionQuorum.getReadReplicas().size() >= self->cfis.size() / 2 + 1);
+					ASSERT(self->getCommittedVersionQuorum.getReadReplicas().size() >= self->cfis.size() / 2 + 1 ||
+					       self->getCommittedVersionQuorum.isSpecialZeroQuorum());
 					state std::vector<ConfigFollowerInterface> readReplicas =
 					    self->getCommittedVersionQuorum.getReadReplicas();
 					std::vector<Future<Void>> fs;
@@ -448,8 +577,8 @@ class PaxosConfigConsumerImpl {
 			} catch (Error& e) {
 				if (e.code() == error_code_version_already_compacted || e.code() == error_code_timed_out ||
 				    e.code() == error_code_failed_to_reach_quorum || e.code() == error_code_version_already_compacted ||
-				    e.code() == error_code_process_behind) {
-					TEST(true); // PaxosConfigConsumer get version_already_compacted error
+				    e.code() == error_code_process_behind || e.code() == error_code_future_version) {
+					CODE_PROBE(true, "PaxosConfigConsumer fetch error");
 					if (e.code() == error_code_failed_to_reach_quorum) {
 						try {
 							wait(self->getCommittedVersionQuorum.complete());
@@ -483,39 +612,58 @@ class PaxosConfigConsumerImpl {
 	}
 
 	void resetCommittedVersionQuorum() {
-		getCommittedVersionQuorum = GetCommittedVersionQuorum{ cfis, lastSeenVersion };
+		getCommittedVersionQuorum = GetCommittedVersionQuorum{ cfis, lastSeenVersion, compactionVersion };
 	}
 
 public:
+	Future<Void> readSnapshot(ConfigBroadcaster& broadcaster) { return getSnapshotAndChanges(this, &broadcaster); }
+
 	Future<Void> consume(ConfigBroadcaster& broadcaster) {
 		return fetchChanges(this, &broadcaster) || compactor(this, &broadcaster);
 	}
 
+	void allowSpecialCaseRollforward() { this->allowSpecialCaseRollforward_ = true; }
+
 	UID getID() const { return id; }
 
 	PaxosConfigConsumerImpl(std::vector<ConfigFollowerInterface> const& cfis,
 	                        double pollingInterval,
-	                        Optional<double> compactionInterval)
-	  : cfis(cfis), getCommittedVersionQuorum(cfis, 0), pollingInterval(pollingInterval),
-	    compactionInterval(compactionInterval), id(deterministicRandom()->randomUniqueID()) {}
+	                        Optional<double> compactionInterval,
+	                        bool readPreviousCoordinators)
+	  : cfis(cfis), getCommittedVersionQuorum(cfis, 0, 0), pollingInterval(pollingInterval),
+	    compactionInterval(compactionInterval), readPreviousCoordinators(readPreviousCoordinators),
+	    id(deterministicRandom()->randomUniqueID()) {}
 };
 
 PaxosConfigConsumer::PaxosConfigConsumer(std::vector<ConfigFollowerInterface> const& cfis,
                                          double pollingInterval,
-                                         Optional<double> compactionInterval)
-  : impl(PImpl<PaxosConfigConsumerImpl>::create(cfis, pollingInterval, compactionInterval)) {}
+                                         Optional<double> compactionInterval,
+                                         bool readPreviousCoordinators)
+  : impl(PImpl<PaxosConfigConsumerImpl>::create(cfis, pollingInterval, compactionInterval, readPreviousCoordinators)) {}
 
 PaxosConfigConsumer::PaxosConfigConsumer(ServerCoordinators const& coordinators,
                                          double pollingInterval,
-                                         Optional<double> compactionInterval)
-  : impl(PImpl<PaxosConfigConsumerImpl>::create(coordinators.configServers, pollingInterval, compactionInterval)) {}
+                                         Optional<double> compactionInterval,
+                                         bool readPreviousCoordinators)
+  : impl(PImpl<PaxosConfigConsumerImpl>::create(coordinators.configServers,
+                                                pollingInterval,
+                                                compactionInterval,
+                                                readPreviousCoordinators)) {}
 
 PaxosConfigConsumer::~PaxosConfigConsumer() = default;
 
+Future<Void> PaxosConfigConsumer::readSnapshot(ConfigBroadcaster& broadcaster) {
+	return impl->readSnapshot(broadcaster);
+}
+
 Future<Void> PaxosConfigConsumer::consume(ConfigBroadcaster& broadcaster) {
 	return impl->consume(broadcaster);
 }
 
+void PaxosConfigConsumer::allowSpecialCaseRollforward() {
+	impl->allowSpecialCaseRollforward();
+}
+
 UID PaxosConfigConsumer::getID() const {
 	return impl->getID();
 }
diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/QuietDatabase.actor.cpp
index e6d5976578..878e5eb554 100644
--- a/fdbserver/QuietDatabase.actor.cpp
+++ b/fdbserver/QuietDatabase.actor.cpp
@@ -111,8 +111,8 @@ ACTOR Future<WorkerInterface> getDataDistributorWorker(Database cx, Reference<As
 ACTOR Future<int64_t> getDataInFlight(Database cx, WorkerInterface distributorWorker) {
 	try {
 		TraceEvent("DataInFlight").detail("Stage", "ContactingDataDistributor");
-		TraceEventFields md = wait(timeoutError(
-		    distributorWorker.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("TotalDataInFlight"))), 1.0));
+		TraceEventFields md = wait(
+		    timeoutError(distributorWorker.eventLogRequest.getReply(EventLogRequest("TotalDataInFlight"_sr)), 1.0));
 		int64_t dataInFlight = boost::lexical_cast<int64_t>(md.getValue("TotalBytes"));
 		return dataInFlight;
 	} catch (Error& e) {
@@ -235,7 +235,9 @@ ACTOR Future<std::pair<int64_t, int64_t>> getTLogQueueInfo(Database cx,
 }
 
 // Returns a vector of blob worker interfaces which have been persisted under the system key space
-ACTOR Future<std::vector<BlobWorkerInterface>> getBlobWorkers(Database cx, bool use_system_priority = false) {
+ACTOR Future<std::vector<BlobWorkerInterface>> getBlobWorkers(Database cx,
+                                                              bool use_system_priority = false,
+                                                              Version* grv = nullptr) {
 	state Transaction tr(cx);
 	loop {
 		if (use_system_priority) {
@@ -252,6 +254,9 @@ ACTOR Future<std::vector<BlobWorkerInterface>> getBlobWorkers(Database cx, bool
 			for (int i = 0; i < blobWorkersList.size(); i++) {
 				blobWorkers.push_back(decodeBlobWorkerListValue(blobWorkersList[i].value));
 			}
+			if (grv) {
+				*grv = tr.getReadVersion().get();
+			}
 			return blobWorkers;
 		} catch (Error& e) {
 			wait(tr.onError(e));
@@ -295,7 +300,7 @@ getStorageWorkers(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo, b
 	    wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
 		    tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		    tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		    return tr->get(LiteralStringRef("usable_regions").withPrefix(configKeysPrefix));
+		    return tr->get("usable_regions"_sr.withPrefix(configKeysPrefix));
 	    }));
 	int usableRegions = 1;
 	if (regionsValue.present()) {
@@ -435,8 +440,8 @@ ACTOR Future<int64_t> getDataDistributionQueueSize(Database cx,
 	try {
 		TraceEvent("DataDistributionQueueSize").detail("Stage", "ContactingDataDistributor");
 
-		TraceEventFields movingDataMessage = wait(timeoutError(
-		    distributorWorker.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("MovingData"))), 1.0));
+		TraceEventFields movingDataMessage =
+		    wait(timeoutError(distributorWorker.eventLogRequest.getReply(EventLogRequest("MovingData"_sr)), 1.0));
 
 		TraceEvent("DataDistributionQueueSize").detail("Stage", "GotString");
 
@@ -478,8 +483,7 @@ ACTOR Future<bool> getTeamCollectionValid(Database cx, WorkerInterface dataDistr
 			TraceEvent("GetTeamCollectionValid").detail("Stage", "ContactingMaster");
 
 			TraceEventFields teamCollectionInfoMessage = wait(timeoutError(
-			    dataDistributorWorker.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("TeamCollectionInfo"))),
-			    1.0));
+			    dataDistributorWorker.eventLogRequest.getReply(EventLogRequest("TeamCollectionInfo"_sr)), 1.0));
 
 			TraceEvent("GetTeamCollectionValid").detail("Stage", "GotString");
 
@@ -586,8 +590,8 @@ ACTOR Future<bool> getDataDistributionActive(Database cx, WorkerInterface distri
 	try {
 		TraceEvent("DataDistributionActive").detail("Stage", "ContactingDataDistributor");
 
-		TraceEventFields activeMessage = wait(timeoutError(
-		    distributorWorker.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("DDTrackerStarting"))), 1.0));
+		TraceEventFields activeMessage = wait(
+		    timeoutError(distributorWorker.eventLogRequest.getReply(EventLogRequest("DDTrackerStarting"_sr)), 1.0));
 
 		return activeMessage.getValue("State") == "Active";
 	} catch (Error& e) {
@@ -652,9 +656,9 @@ ACTOR Future<int64_t> getVersionOffset(Database cx,
 ACTOR Future<Void> repairDeadDatacenter(Database cx,
                                         Reference<AsyncVar<ServerDBInfo> const> dbInfo,
                                         std::string context) {
-	if (g_network->isSimulated() && g_simulator.usableRegions > 1) {
-		bool primaryDead = g_simulator.datacenterDead(g_simulator.primaryDcId);
-		bool remoteDead = g_simulator.datacenterDead(g_simulator.remoteDcId);
+	if (g_network->isSimulated() && g_simulator->usableRegions > 1) {
+		bool primaryDead = g_simulator->datacenterDead(g_simulator->primaryDcId);
+		bool remoteDead = g_simulator->datacenterDead(g_simulator->remoteDcId);
 
 		// FIXME: the primary and remote can both be considered dead because excludes are not handled properly by the
 		// datacenterDead function
@@ -668,10 +672,10 @@ ACTOR Future<Void> repairDeadDatacenter(Database cx,
 			    .detail("Stage", "Repopulate")
 			    .detail("RemoteDead", remoteDead)
 			    .detail("PrimaryDead", primaryDead);
-			g_simulator.usableRegions = 1;
+			g_simulator->usableRegions = 1;
 			wait(success(ManagementAPI::changeConfig(
 			    cx.getReference(),
-			    (primaryDead ? g_simulator.disablePrimary : g_simulator.disableRemote) + " repopulate_anti_quorum=1",
+			    (primaryDead ? g_simulator->disablePrimary : g_simulator->disableRemote) + " repopulate_anti_quorum=1",
 			    true)));
 			while (dbInfo->get().recoveryState < RecoveryState::STORAGE_RECOVERED) {
 				wait(dbInfo->onChange());
@@ -763,7 +767,7 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
                                         int64_t maxDataDistributionQueueSize = 0,
                                         int64_t maxPoppedVersionLag = 30e6,
                                         int64_t maxVersionOffset = 1e6) {
-	state QuietDatabaseChecker checker(isBuggifyEnabled(BuggifyType::General) ? 3600.0 : 1000.0);
+	state QuietDatabaseChecker checker(isBuggifyEnabled(BuggifyType::General) ? 4000.0 : 1000.0);
 	state Future<Void> reconfig =
 	    reconfigureAfter(cx, 100 + (deterministicRandom()->random01() * 100), dbInfo, "QuietDatabase");
 	state Future<int64_t> dataInFlight;
diff --git a/fdbserver/RESTKmsConnector.actor.cpp b/fdbserver/RESTKmsConnector.actor.cpp
index dbb1271ebb..c1f0fbbc23 100644
--- a/fdbserver/RESTKmsConnector.actor.cpp
+++ b/fdbserver/RESTKmsConnector.actor.cpp
@@ -21,7 +21,7 @@
 #include "fdbserver/RESTKmsConnector.h"
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/HTTP.h"
+#include "fdbrpc/HTTP.h"
 #include "flow/IAsyncFile.h"
 #include "fdbserver/KmsConnectorInterface.h"
 #include "fdbserver/Knobs.h"
@@ -276,7 +276,7 @@ ACTOR Future<Void> discoverKmsUrls(Reference<RESTKmsConnectorCtx> ctx, bool refr
 void parseKmsResponse(Reference<RESTKmsConnectorCtx> ctx,
                       Reference<HTTP::Response> resp,
                       Arena* arena,
-                      VectorRef<EncryptCipherKeyDetails>* outCipherKeyDetails) {
+                      VectorRef<EncryptCipherKeyDetailsRef>* outCipherKeyDetails) {
 	// Acceptable response payload json format:
 	//
 	// response_json_payload {
@@ -542,7 +542,7 @@ ACTOR
 Future<Void> fetchEncryptionKeys_impl(Reference<RESTKmsConnectorCtx> ctx,
                                       StringRef requestBodyRef,
                                       Arena* arena,
-                                      VectorRef<EncryptCipherKeyDetails>* outCipherKeyDetails) {
+                                      VectorRef<EncryptCipherKeyDetailsRef>* outCipherKeyDetails) {
 	state Reference<HTTP::Response> resp;
 
 	// Follow 2-phase scheme:
@@ -880,7 +880,7 @@ std::shared_ptr<platform::TmpFile> prepareTokenFile(const uint8_t* buff, const i
 
 std::shared_ptr<platform::TmpFile> prepareTokenFile(const int tokenLen) {
 	Standalone<StringRef> buff = makeString(tokenLen);
-	generateRandomData(mutateString(buff), tokenLen);
+	deterministicRandom()->randomBytes(mutateString(buff), tokenLen);
 
 	return prepareTokenFile(buff.begin(), tokenLen);
 }
@@ -941,7 +941,7 @@ ACTOR Future<Void> testValidationFileTokenPayloadTooLarge(Reference<RESTKmsConne
 	                        SERVER_KNOBS->REST_KMS_CONNECTOR_VALIDATION_TOKEN_MAX_SIZE +
 	                    2;
 	Standalone<StringRef> buff = makeString(tokenLen);
-	generateRandomData(mutateString(buff), tokenLen);
+	deterministicRandom()->randomBytes(mutateString(buff), tokenLen);
 
 	std::string details;
 	state std::vector<std::shared_ptr<platform::TmpFile>> tokenfiles;
@@ -972,7 +972,7 @@ ACTOR Future<Void> testMultiValidationFileTokenFiles(Reference<RESTKmsConnectorC
 	state std::unordered_map<std::string, std::string> tokenNameValueMap;
 	state std::string tokenDetailsStr;
 
-	generateRandomData(mutateString(buff), tokenLen);
+	deterministicRandom()->randomBytes(mutateString(buff), tokenLen);
 
 	for (int i = 1; i <= numFiles; i++) {
 		std::string tokenName = std::to_string(i);
@@ -1096,15 +1096,14 @@ void validateKmsUrls(Reference<RESTKmsConnectorCtx> ctx) {
 	ASSERT_EQ(urlCtx->url.compare(KMS_URL_NAME_TEST), 0);
 }
 
-void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, Arena arena) {
+void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, Arena& arena) {
 	KmsConnLookupEKsByKeyIdsReq req;
 	std::unordered_map<EncryptCipherBaseKeyId, EncryptCipherDomainId> keyMap;
 	const int nKeys = deterministicRandom()->randomInt(7, 8);
 	for (int i = 1; i < nKeys; i++) {
 		EncryptCipherDomainId domainId = getRandomDomainId();
-		EncryptCipherDomainName domainName = domainId < 0
-		                                         ? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME))
-		                                         : StringRef(arena, std::to_string(domainId));
+		EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
+		                                                     : StringRef(arena, std::to_string(domainId));
 		req.encryptKeyInfos.emplace_back_deep(req.arena, domainId, i, domainName);
 		keyMap[i] = domainId;
 	}
@@ -1121,7 +1120,7 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
 	getFakeKmsResponse(requestBodyRef, true, httpResp);
 	TraceEvent("FetchKeysByKeyIds", ctx->uid).setMaxFieldLength(100000).detail("HttpRespStr", httpResp->content);
 
-	VectorRef<EncryptCipherKeyDetails> cipherDetails;
+	VectorRef<EncryptCipherKeyDetailsRef> cipherDetails;
 	parseKmsResponse(ctx, httpResp, &arena, &cipherDetails);
 	ASSERT_EQ(cipherDetails.size(), keyMap.size());
 	for (const auto& detail : cipherDetails) {
@@ -1135,16 +1134,15 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
 	}
 }
 
-void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, Arena arena) {
+void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, Arena& arena) {
 	KmsConnLookupEKsByDomainIdsReq req;
-	std::unordered_map<EncryptCipherDomainId, KmsConnLookupDomainIdsReqInfo> domainInfoMap;
+	std::unordered_map<EncryptCipherDomainId, KmsConnLookupDomainIdsReqInfoRef> domainInfoMap;
 	const int nKeys = deterministicRandom()->randomInt(7, 25);
 	for (int i = 1; i < nKeys; i++) {
 		EncryptCipherDomainId domainId = getRandomDomainId();
-		EncryptCipherDomainName domainName = domainId < 0
-		                                         ? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME))
-		                                         : StringRef(arena, std::to_string(domainId));
-		KmsConnLookupDomainIdsReqInfo reqInfo(req.arena, domainId, domainName);
+		EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
+		                                                     : StringRef(arena, std::to_string(domainId));
+		KmsConnLookupDomainIdsReqInfoRef reqInfo(req.arena, domainId, domainName);
 		if (domainInfoMap.insert({ domainId, reqInfo }).second) {
 			req.encryptDomainInfos.push_back(req.arena, reqInfo);
 		}
@@ -1159,7 +1157,7 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
 	getFakeKmsResponse(jsonReqRef, false, httpResp);
 	TraceEvent("FetchKeysByDomainIds", ctx->uid).detail("HttpRespStr", httpResp->content);
 
-	VectorRef<EncryptCipherKeyDetails> cipherDetails;
+	VectorRef<EncryptCipherKeyDetailsRef> cipherDetails;
 	parseKmsResponse(ctx, httpResp, &arena, &cipherDetails);
 	ASSERT_EQ(domainInfoMap.size(), cipherDetails.size());
 	for (const auto& detail : cipherDetails) {
@@ -1174,7 +1172,7 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
 
 void testMissingCipherDetailsTag(Reference<RESTKmsConnectorCtx> ctx) {
 	Arena arena;
-	VectorRef<EncryptCipherKeyDetails> cipherDetails;
+	VectorRef<EncryptCipherKeyDetailsRef> cipherDetails;
 
 	rapidjson::Document doc;
 	doc.SetObject();
@@ -1201,7 +1199,7 @@ void testMissingCipherDetailsTag(Reference<RESTKmsConnectorCtx> ctx) {
 
 void testMalformedCipherDetails(Reference<RESTKmsConnectorCtx> ctx) {
 	Arena arena;
-	VectorRef<EncryptCipherKeyDetails> cipherDetails;
+	VectorRef<EncryptCipherKeyDetailsRef> cipherDetails;
 
 	rapidjson::Document doc;
 	doc.SetObject();
@@ -1228,7 +1226,7 @@ void testMalformedCipherDetails(Reference<RESTKmsConnectorCtx> ctx) {
 
 void testMalfromedCipherDetailObj(Reference<RESTKmsConnectorCtx> ctx) {
 	Arena arena;
-	VectorRef<EncryptCipherKeyDetails> cipherDetails;
+	VectorRef<EncryptCipherKeyDetailsRef> cipherDetails;
 
 	rapidjson::Document doc;
 	doc.SetObject();
@@ -1260,7 +1258,7 @@ void testMalfromedCipherDetailObj(Reference<RESTKmsConnectorCtx> ctx) {
 
 void testKMSErrorResponse(Reference<RESTKmsConnectorCtx> ctx) {
 	Arena arena;
-	VectorRef<EncryptCipherKeyDetails> cipherDetails;
+	VectorRef<EncryptCipherKeyDetailsRef> cipherDetails;
 
 	rapidjson::Document doc;
 	doc.SetObject();
@@ -1350,7 +1348,7 @@ TEST_CASE("/KmsConnector/REST/ParseKmsDiscoveryUrls") {
 	state Arena arena;
 
 	// initialize cipher key used for testing
-	generateRandomData(&BASE_CIPHER_KEY_TEST[0], 32);
+	deterministicRandom()->randomBytes(&BASE_CIPHER_KEY_TEST[0], 32);
 
 	wait(testParseDiscoverKmsUrlFileNotFound(ctx));
 	wait(testParseDiscoverKmsUrlFile(ctx));
@@ -1363,7 +1361,7 @@ TEST_CASE("/KmsConnector/REST/ParseValidationTokenFile") {
 	state Arena arena;
 
 	// initialize cipher key used for testing
-	generateRandomData(&BASE_CIPHER_KEY_TEST[0], 32);
+	deterministicRandom()->randomBytes(&BASE_CIPHER_KEY_TEST[0], 32);
 
 	wait(testEmptyValidationFileDetails(ctx));
 	wait(testMalformedFileValidationTokenDetails(ctx));
@@ -1380,7 +1378,7 @@ TEST_CASE("/KmsConnector/REST/ParseKmsResponse") {
 	state Arena arena;
 
 	// initialize cipher key used for testing
-	generateRandomData(&BASE_CIPHER_KEY_TEST[0], 32);
+	deterministicRandom()->randomBytes(&BASE_CIPHER_KEY_TEST[0], 32);
 
 	testMissingCipherDetailsTag(ctx);
 	testMalformedCipherDetails(ctx);
@@ -1394,7 +1392,7 @@ TEST_CASE("/KmsConnector/REST/GetEncryptionKeyOps") {
 	state Arena arena;
 
 	// initialize cipher key used for testing
-	generateRandomData(&BASE_CIPHER_KEY_TEST[0], 32);
+	deterministicRandom()->randomBytes(&BASE_CIPHER_KEY_TEST[0], 32);
 
 	// Prepare KmsConnector context details
 	wait(testParseDiscoverKmsUrlFile(ctx));
diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 73345b2e75..a7d06dc533 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -18,10 +18,14 @@
  * limitations under the License.
  */
 
+#include "fdbclient/ClientKnobs.h"
 #include "fdbserver/DataDistribution.actor.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/Ratekeeper.h"
 #include "fdbserver/TagThrottler.h"
 #include "fdbserver/WaitFailure.h"
+#include "fdbserver/QuietDatabase.h"
+#include "flow/OwningResource.h"
 
 #include "flow/actorcompiler.h" // must be last include
 
@@ -36,7 +40,9 @@ const char* limitReasonName[] = { "workload",
 	                              "log_server_min_free_space",
 	                              "log_server_min_free_space_ratio",
 	                              "storage_server_durability_lag",
-	                              "storage_server_list_fetch_failed" };
+	                              "storage_server_list_fetch_failed",
+	                              "blob_worker_lag",
+	                              "blob_worker_missing" };
 static_assert(sizeof(limitReasonName) / sizeof(limitReasonName[0]) == limitReason_t_end, "limitReasonDesc table size");
 
 int limitReasonEnd = limitReason_t_end;
@@ -54,7 +60,9 @@ const char* limitReasonDesc[] = { "Workload or read performance.",
 	                              "Log server running out of space (approaching 100MB limit).",
 	                              "Log server running out of space (approaching 5% limit).",
 	                              "Storage server durable version falling behind.",
-	                              "Unable to fetch storage server list." };
+	                              "Unable to fetch storage server list.",
+	                              "Blob worker granule version falling behind.",
+	                              "No blob workers are reporting metrics." };
 
 static_assert(sizeof(limitReasonDesc) / sizeof(limitReasonDesc[0]) == limitReason_t_end, "limitReasonDesc table size");
 
@@ -147,9 +155,9 @@ public:
 		}
 	}
 
-	ACTOR static Future<Void> trackStorageServerQueueInfo(Ratekeeper* self, StorageServerInterface ssi) {
-		self->storageQueueInfo.insert(mapPair(ssi.id(), StorageQueueInfo(ssi.id(), ssi.locality)));
-		state Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
+	ACTOR static Future<Void> trackStorageServerQueueInfo(ActorWeakSelfRef<Ratekeeper> self,
+	                                                      StorageServerInterface ssi) {
+		self->storageQueueInfo.insert(mapPair(ssi.id(), StorageQueueInfo(self->id, ssi.id(), ssi.locality)));
 		TraceEvent("RkTracking", self->id)
 		    .detail("StorageServer", ssi.id())
 		    .detail("Locality", ssi.locality.toString());
@@ -157,6 +165,7 @@ public:
 			loop {
 				ErrorOr<StorageQueuingMetricsReply> reply = wait(ssi.getQueuingMetrics.getReplyUnlessFailedFor(
 				    StorageQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
+				Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
 				if (reply.present()) {
 					myQueueInfo->value.update(reply.get(), self->smoothTotalDurableBytes);
 					myQueueInfo->value.acceptingRequests = ssi.isAcceptingRequests();
@@ -173,7 +182,8 @@ public:
 			}
 		} catch (...) {
 			// including cancellation
-			self->storageQueueInfo.erase(myQueueInfo);
+			self->storageQueueInfo.erase(ssi.id());
+			self->storageServerInterfaces.erase(ssi.id());
 			throw;
 		}
 	}
@@ -207,28 +217,133 @@ public:
 	}
 
 	ACTOR static Future<Void> trackEachStorageServer(
-	    Ratekeeper* self,
+	    ActorWeakSelfRef<Ratekeeper> self,
 	    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-		state Map<UID, Future<Void>> actors;
+
+		state std::unordered_map<UID, Future<Void>> storageServerTrackers;
 		state Promise<Void> err;
+
 		loop choose {
 			when(state std::pair<UID, Optional<StorageServerInterface>> change = waitNext(serverChanges)) {
 				wait(delay(0)); // prevent storageServerTracker from getting cancelled while on the call stack
+
+				const UID& id = change.first;
 				if (change.second.present()) {
 					if (!change.second.get().isTss()) {
-						auto& a = actors[change.first];
+
+						auto& a = storageServerTrackers[change.first];
 						a = Future<Void>();
 						a = splitError(trackStorageServerQueueInfo(self, change.second.get()), err);
+
+						self->storageServerInterfaces[id] = change.second.get();
 					}
-				} else
-					actors.erase(change.first);
+				} else {
+					storageServerTrackers.erase(id);
+
+					self->storageServerInterfaces.erase(id);
+				}
 			}
 			when(wait(err.getFuture())) {}
 		}
 	}
 
+	ACTOR static Future<Void> monitorBlobWorkers(Ratekeeper* self, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+		state std::vector<BlobWorkerInterface> blobWorkers;
+		state int workerFetchCount = 0;
+		state double lastStartTime = 0;
+		state double startTime = 0;
+		state bool blobWorkerDead = false;
+		state double lastLoggedTime = 0;
+
+		loop {
+			while (!self->configuration.blobGranulesEnabled) {
+				wait(delay(SERVER_KNOBS->SERVER_LIST_DELAY));
+			}
+
+			state Version grv;
+			state Future<Void> blobWorkerDelay =
+			    delay(SERVER_KNOBS->METRIC_UPDATE_RATE * FLOW_KNOBS->DELAY_JITTER_OFFSET);
+			int fetchAmount = SERVER_KNOBS->BW_FETCH_WORKERS_INTERVAL /
+			                  (SERVER_KNOBS->METRIC_UPDATE_RATE * FLOW_KNOBS->DELAY_JITTER_OFFSET);
+			if (++workerFetchCount == fetchAmount || blobWorkerDead) {
+				workerFetchCount = 0;
+				std::vector<BlobWorkerInterface> _blobWorkers = wait(getBlobWorkers(self->db, true, &grv));
+				blobWorkers = _blobWorkers;
+			} else {
+				grv = self->maxVersion;
+			}
+
+			lastStartTime = startTime;
+			startTime = now();
+
+			if (blobWorkers.size() > 0) {
+				state Future<Optional<BlobManagerBlockedReply>> blockedAssignments;
+				if (dbInfo->get().blobManager.present()) {
+					blockedAssignments =
+					    timeout(brokenPromiseToNever(dbInfo->get().blobManager.get().blobManagerBlockedReq.getReply(
+					                BlobManagerBlockedRequest())),
+					            SERVER_KNOBS->BLOB_WORKER_TIMEOUT);
+				}
+				state std::vector<Future<Optional<MinBlobVersionReply>>> aliveVersions;
+				aliveVersions.reserve(blobWorkers.size());
+				for (auto& it : blobWorkers) {
+					MinBlobVersionRequest req;
+					req.grv = grv;
+					aliveVersions.push_back(timeout(brokenPromiseToNever(it.minBlobVersionRequest.getReply(req)),
+					                                SERVER_KNOBS->BLOB_WORKER_TIMEOUT));
+				}
+				if (blockedAssignments.isValid()) {
+					wait(success(blockedAssignments));
+					if (blockedAssignments.get().present() && blockedAssignments.get().get().blockedAssignments == 0) {
+						self->unblockedAssignmentTime = now();
+					}
+				}
+				wait(waitForAll(aliveVersions));
+				Version minVer = grv;
+				blobWorkerDead = false;
+				int minIdx = 0;
+				for (int i = 0; i < blobWorkers.size(); i++) {
+					if (aliveVersions[i].get().present()) {
+						if (aliveVersions[i].get().get().version < minVer) {
+							minVer = aliveVersions[i].get().get().version;
+							minIdx = i;
+						}
+					} else {
+						blobWorkerDead = true;
+						minVer = 0;
+						minIdx = i;
+						break;
+					}
+				}
+				if (minVer > 0 && blobWorkers.size() > 0 &&
+				    now() - self->unblockedAssignmentTime < SERVER_KNOBS->BW_MAX_BLOCKED_INTERVAL) {
+					while (!self->blobWorkerVersionHistory.empty() &&
+					       minVer < self->blobWorkerVersionHistory.back().second) {
+						self->blobWorkerVersionHistory.pop_back();
+					}
+					self->blobWorkerVersionHistory.push_back(std::make_pair(now(), minVer));
+				}
+				while (self->blobWorkerVersionHistory.size() > SERVER_KNOBS->MIN_BW_HISTORY &&
+				       self->blobWorkerVersionHistory[1].first <
+				           self->blobWorkerVersionHistory.back().first - SERVER_KNOBS->BW_ESTIMATION_INTERVAL) {
+					self->blobWorkerVersionHistory.pop_front();
+				}
+				if (now() - lastLoggedTime > SERVER_KNOBS->BW_RW_LOGGING_INTERVAL) {
+					lastLoggedTime = now();
+					TraceEvent("RkMinBlobWorkerVersion")
+					    .detail("BWVersion", minVer)
+					    .detail("MaxVer", self->maxVersion)
+					    .detail("MinId", blobWorkers.size() > 0 ? blobWorkers[minIdx].id() : UID());
+				}
+			}
+			wait(blobWorkerDelay);
+		}
+	}
+
 	ACTOR static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-		state Ratekeeper self(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True));
+		state ActorOwningSelfRef<Ratekeeper> pSelf(
+		    new Ratekeeper(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True)));
+		state Ratekeeper& self = *pSelf;
 		state Future<Void> timeout = Void();
 		state std::vector<Future<Void>> tlogTrackers;
 		state std::vector<TLogInterface> tlogInterfs;
@@ -241,10 +356,13 @@ public:
 
 		PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges;
 		self.addActor.send(self.monitorServerListChange(serverChanges));
-		self.addActor.send(self.trackEachStorageServer(serverChanges.getFuture()));
+		self.addActor.send(RatekeeperImpl::trackEachStorageServer(pSelf, serverChanges.getFuture()));
 		self.addActor.send(traceRole(Role::RATEKEEPER, rkInterf.id()));
 
 		self.addActor.send(self.monitorThrottlingChanges());
+		if (SERVER_KNOBS->BW_THROTTLING_ENABLED) {
+			self.addActor.send(self.monitorBlobWorkers(dbInfo));
+		}
 		self.addActor.send(self.refreshStorageServerCommitCosts());
 
 		TraceEvent("RkTLogQueueSizeParameters", rkInterf.id())
@@ -274,10 +392,55 @@ public:
 
 		self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
 
+		state bool recovering = dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS;
+		state Version recoveryVersion = std::numeric_limits<Version>::max();
+		if (recovering) {
+			self.version_recovery[recoveryVersion] = std::make_pair(now(), Optional<double>());
+		}
+
 		try {
 			state bool lastLimited = false;
 			loop choose {
 				when(wait(timeout)) {
+					double actualTps = self.smoothReleasedTransactions.smoothRate();
+					actualTps =
+					    std::max(std::max(1.0, actualTps),
+					             self.smoothTotalDurableBytes.smoothRate() / CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
+
+					if (self.actualTpsHistory.size() > SERVER_KNOBS->MAX_TPS_HISTORY_SAMPLES) {
+						self.actualTpsHistory.pop_front();
+					}
+					self.actualTpsHistory.push_back(actualTps);
+
+					if (self.configuration.blobGranulesEnabled && SERVER_KNOBS->BW_THROTTLING_ENABLED) {
+						Version maxVersion = 0;
+						int64_t totalReleased = 0;
+						int64_t batchReleased = 0;
+						for (auto& it : self.grvProxyInfo) {
+							maxVersion = std::max(maxVersion, it.second.version);
+							totalReleased += it.second.totalTransactions;
+							batchReleased += it.second.batchTransactions;
+						}
+						self.version_transactions[maxVersion] =
+						    Ratekeeper::VersionInfo(totalReleased, batchReleased, now());
+
+						loop {
+							auto secondEntry = self.version_transactions.begin();
+							++secondEntry;
+							if (secondEntry != self.version_transactions.end() &&
+							    secondEntry->second.created < now() - (2 * SERVER_KNOBS->TARGET_BW_LAG) &&
+							    (self.blobWorkerVersionHistory.empty() ||
+							     secondEntry->first < self.blobWorkerVersionHistory.front().second)) {
+								self.version_transactions.erase(self.version_transactions.begin());
+							} else {
+								break;
+							}
+						}
+					}
+					while (self.version_recovery.size() > CLIENT_KNOBS->MAX_GENERATIONS) {
+						self.version_recovery.erase(self.version_recovery.begin());
+					}
+
 					self.updateRate(&self.normalLimits);
 					self.updateRate(&self.batchLimits);
 
@@ -311,6 +474,17 @@ public:
 
 					p.totalTransactions = req.totalReleasedTransactions;
 					p.batchTransactions = req.batchReleasedTransactions;
+					p.version = req.version;
+					self.maxVersion = std::max(self.maxVersion, req.version);
+
+					if (recoveryVersion == std::numeric_limits<Version>::max() &&
+					    self.version_recovery.count(recoveryVersion)) {
+						recoveryVersion = self.maxVersion;
+						self.version_recovery[recoveryVersion] =
+						    self.version_recovery[std::numeric_limits<Version>::max()];
+						self.version_recovery.erase(std::numeric_limits<Version>::max());
+					}
+
 					p.lastUpdateTime = now();
 
 					reply.transactionRate = self.normalLimits.tpsLimit / self.grvProxyInfo.size();
@@ -322,10 +496,17 @@ public:
 						p.lastThrottledTagChangeId = self.tagThrottler->getThrottledTagChangeId();
 						p.lastTagPushTime = now();
 
-						reply.throttledTags = self.tagThrottler->getClientRates();
-						bool returningTagsToProxy =
-						    reply.throttledTags.present() && reply.throttledTags.get().size() > 0;
-						TEST(returningTagsToProxy); // Returning tag throttles to a proxy
+						bool returningTagsToProxy{ false };
+						if (SERVER_KNOBS->ENFORCE_TAG_THROTTLING_ON_PROXIES) {
+							reply.proxyThrottledTags = self.tagThrottler->getProxyRates(self.grvProxyInfo.size());
+							returningTagsToProxy =
+							    reply.proxyThrottledTags.present() && reply.proxyThrottledTags.get().size() > 0;
+						} else {
+							reply.clientThrottledTags = self.tagThrottler->getClientRates();
+							returningTagsToProxy =
+							    reply.clientThrottledTags.present() && reply.clientThrottledTags.get().size() > 0;
+						}
+						CODE_PROBE(returningTagsToProxy, "Returning tag throttles to a proxy");
 					}
 
 					reply.healthMetrics.update(self.healthMetrics, true, req.detailed);
@@ -346,6 +527,27 @@ public:
 				}
 				when(wait(err.getFuture())) {}
 				when(wait(dbInfo->onChange())) {
+					if (!recovering && dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+						recovering = true;
+						recoveryVersion = self.maxVersion;
+						if (recoveryVersion == 0) {
+							recoveryVersion = std::numeric_limits<Version>::max();
+						}
+						if (self.version_recovery.count(recoveryVersion)) {
+							auto& it = self.version_recovery[recoveryVersion];
+							double existingEnd = it.second.present() ? it.second.get() : now();
+							double existingDuration = existingEnd - it.first;
+							self.version_recovery[recoveryVersion] =
+							    std::make_pair(now() - existingDuration, Optional<double>());
+						} else {
+							self.version_recovery[recoveryVersion] = std::make_pair(now(), Optional<double>());
+						}
+					}
+					if (recovering && dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
+						recovering = false;
+						self.version_recovery[recoveryVersion].second = now();
+					}
+
 					if (tlogInterfs != dbInfo->get().logSystemConfig.allLocalLogs()) {
 						tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
 						tlogTrackers = std::vector<Future<Void>>();
@@ -367,17 +569,23 @@ public:
 
 	ACTOR static Future<Void> refreshStorageServerCommitCosts(Ratekeeper* self) {
 		state double lastBusiestCommitTagPick;
+		state std::vector<Future<Void>> replies;
 		loop {
 			lastBusiestCommitTagPick = now();
 			wait(delay(SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL));
+
+			replies.clear();
+
 			double elapsed = now() - lastBusiestCommitTagPick;
 			// for each SS, select the busiest commit tag from ssTrTagCommitCost
 			for (auto& [ssId, ssQueueInfo] : self->storageQueueInfo) {
-				ssQueueInfo.refreshCommitCost(elapsed);
+				// NOTE: In some cases, for unknown reason SS will not respond to the updateCommitCostRequest. Since the
+				// information is not time-sensitive, we do not wait for the replies.
+				replies.push_back(self->storageServerInterfaces[ssId].updateCommitCostRequest.getReply(
+				    ssQueueInfo.refreshCommitCost(elapsed)));
 			}
 		}
 	}
-
 }; // class RatekeeperImpl
 
 Future<Void> Ratekeeper::configurationMonitor() {
@@ -389,15 +597,6 @@ Future<Void> Ratekeeper::monitorServerListChange(
 	return RatekeeperImpl::monitorServerListChange(this, serverChanges);
 }
 
-Future<Void> Ratekeeper::trackEachStorageServer(
-    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-	return RatekeeperImpl::trackEachStorageServer(this, serverChanges);
-}
-
-Future<Void> Ratekeeper::trackStorageServerQueueInfo(StorageServerInterface ssi) {
-	return RatekeeperImpl::trackStorageServerQueueInfo(this, ssi);
-}
-
 Future<Void> Ratekeeper::trackTLogQueueInfo(TLogInterface tli) {
 	return RatekeeperImpl::trackTLogQueueInfo(this, tli);
 }
@@ -406,6 +605,10 @@ Future<Void> Ratekeeper::monitorThrottlingChanges() {
 	return tagThrottler->monitorThrottlingChanges();
 }
 
+Future<Void> Ratekeeper::monitorBlobWorkers(Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	return RatekeeperImpl::monitorBlobWorkers(this, dbInfo);
+}
+
 Future<Void> Ratekeeper::run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
 	return RatekeeperImpl::run(rkInterf, dbInfo);
 }
@@ -413,16 +616,16 @@ Future<Void> Ratekeeper::run(RatekeeperInterface rkInterf, Reference<AsyncVar<Se
 Ratekeeper::Ratekeeper(UID id, Database db)
   : id(id), db(db), smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
     smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT),
-    smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
-    actualTpsMetric(LiteralStringRef("Ratekeeper.ActualTPS")), lastWarning(0), lastSSListFetchedTimestamp(now()),
-    normalLimits(TransactionPriority::DEFAULT,
-                 "",
-                 SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER,
-                 SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER,
-                 SERVER_KNOBS->TARGET_BYTES_PER_TLOG,
-                 SERVER_KNOBS->SPRING_BYTES_TLOG,
-                 SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE,
-                 SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS),
+    smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT), actualTpsMetric("Ratekeeper.ActualTPS"_sr),
+    lastWarning(0), lastSSListFetchedTimestamp(now()), normalLimits(TransactionPriority::DEFAULT,
+                                                                    "",
+                                                                    SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER,
+                                                                    SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER,
+                                                                    SERVER_KNOBS->TARGET_BYTES_PER_TLOG,
+                                                                    SERVER_KNOBS->SPRING_BYTES_TLOG,
+                                                                    SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE,
+                                                                    SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS,
+                                                                    SERVER_KNOBS->TARGET_BW_LAG),
     batchLimits(TransactionPriority::BATCH,
                 "Batch",
                 SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER_BATCH,
@@ -430,7 +633,9 @@ Ratekeeper::Ratekeeper(UID id, Database db)
                 SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH,
                 SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH,
                 SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH,
-                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH) {
+                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH,
+                SERVER_KNOBS->TARGET_BW_LAG_BATCH),
+    maxVersion(0), blobWorkerTime(now()), unblockedAssignmentTime(now()) {
 	if (SERVER_KNOBS->GLOBAL_TAG_THROTTLING) {
 		tagThrottler = std::make_unique<GlobalTagThrottler>(db, id);
 	} else {
@@ -455,16 +660,11 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 
 	double actualTps = smoothReleasedTransactions.smoothRate();
 	actualTpsMetric = (int64_t)actualTps;
-	// SOMEDAY: Remove the max( 1.0, ... ) since the below calculations _should_ be able to recover back up from this
-	// value
+	// SOMEDAY: Remove the max( 1.0, ... ) since the below calculations _should_ be able to recover back
+	// up from this value
 	actualTps =
 	    std::max(std::max(1.0, actualTps), smoothTotalDurableBytes.smoothRate() / CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
 
-	if (actualTpsHistory.size() > SERVER_KNOBS->MAX_TPS_HISTORY_SAMPLES) {
-		actualTpsHistory.pop_front();
-	}
-	actualTpsHistory.push_back(actualTps);
-
 	limits->tpsLimit = std::numeric_limits<double>::infinity();
 	UID reasonID = UID();
 	limitReason_t limitReason = limitReason_t::unlimited;
@@ -485,7 +685,8 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 	    SERVER_KNOBS->RATEKEEPER_PRINT_LIMIT_REASON &&
 	    (deterministicRandom()->random01() < SERVER_KNOBS->RATEKEEPER_LIMIT_REASON_SAMPLE_RATE);
 
-	// Look at each storage server's write queue and local rate, compute and store the desired rate ratio
+	// Look at each storage server's write queue and local rate, compute and store the desired rate
+	// ratio
 	for (auto i = storageQueueInfo.begin(); i != storageQueueInfo.end(); ++i) {
 		auto const& ss = i->value;
 		if (!ss.valid || !ss.acceptingRequests || (remoteDC.present() && ss.locality.dcId() == remoteDC))
@@ -698,6 +899,139 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 		break;
 	}
 
+	if (configuration.blobGranulesEnabled && SERVER_KNOBS->BW_THROTTLING_ENABLED) {
+		Version lastBWVer = 0;
+		auto lastIter = version_transactions.end();
+		if (!blobWorkerVersionHistory.empty()) {
+			lastBWVer = blobWorkerVersionHistory.back().second;
+			lastIter = version_transactions.lower_bound(lastBWVer);
+			if (lastIter != version_transactions.end()) {
+				blobWorkerTime = lastIter->second.created;
+			} else {
+				blobWorkerTime = std::max(blobWorkerTime,
+				                          now() - (maxVersion - lastBWVer) / (double)SERVER_KNOBS->VERSIONS_PER_SECOND);
+			}
+		}
+		double blobWorkerLag = (now() - blobWorkerTime) - getRecoveryDuration(lastBWVer);
+		if (blobWorkerLag > limits->bwLagTarget / 2 && !blobWorkerVersionHistory.empty()) {
+			double elapsed = blobWorkerVersionHistory.back().first - blobWorkerVersionHistory.front().first;
+			Version firstBWVer = blobWorkerVersionHistory.front().second;
+			ASSERT(lastBWVer >= firstBWVer);
+			if (elapsed > SERVER_KNOBS->BW_ESTIMATION_INTERVAL / 2) {
+				auto firstIter = version_transactions.upper_bound(firstBWVer);
+				if (lastIter != version_transactions.end() && firstIter != version_transactions.begin()) {
+					--firstIter;
+					double targetRateRatio;
+					if (blobWorkerLag > 3 * limits->bwLagTarget) {
+						targetRateRatio = 0;
+						ASSERT(!g_network->isSimulated() || limits->bwLagTarget != SERVER_KNOBS->TARGET_BW_LAG ||
+						       now() < FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS + SERVER_KNOBS->BW_RK_SIM_QUIESCE_DELAY);
+					} else if (blobWorkerLag > limits->bwLagTarget) {
+						targetRateRatio = SERVER_KNOBS->BW_LAG_DECREASE_AMOUNT;
+					} else {
+						targetRateRatio = SERVER_KNOBS->BW_LAG_INCREASE_AMOUNT;
+					}
+					int64_t totalTransactions =
+					    lastIter->second.totalTransactions - firstIter->second.totalTransactions;
+					int64_t batchTransactions =
+					    lastIter->second.batchTransactions - firstIter->second.batchTransactions;
+					int64_t normalTransactions = totalTransactions - batchTransactions;
+					double bwTPS;
+					if (limits->bwLagTarget == SERVER_KNOBS->TARGET_BW_LAG) {
+						bwTPS = targetRateRatio * (totalTransactions) / elapsed;
+					} else {
+						bwTPS = std::max(0.0, ((targetRateRatio * (totalTransactions)) - normalTransactions) / elapsed);
+					}
+
+					if (bwTPS < limits->tpsLimit) {
+						if (printRateKeepLimitReasonDetails) {
+							TraceEvent("RatekeeperLimitReasonDetails")
+							    .detail("Reason", limitReason_t::blob_worker_lag)
+							    .detail("BWLag", blobWorkerLag)
+							    .detail("BWRate", bwTPS)
+							    .detail("Ratio", targetRateRatio)
+							    .detail("Released", totalTransactions)
+							    .detail("Elapsed", elapsed)
+							    .detail("LastVer", lastBWVer)
+							    .detail("RecoveryDuration", getRecoveryDuration(lastBWVer));
+						}
+						limits->tpsLimit = bwTPS;
+						limitReason = limitReason_t::blob_worker_lag;
+					}
+				} else if (blobWorkerLag > limits->bwLagTarget) {
+					double maxTps = 0;
+					for (int i = 0; i < actualTpsHistory.size(); i++) {
+						maxTps = std::max(maxTps, actualTpsHistory[i]);
+					}
+					double bwProgress =
+					    std::min(elapsed, (lastBWVer - firstBWVer) / (double)SERVER_KNOBS->VERSIONS_PER_SECOND);
+					double bwTPS = maxTps * bwProgress / elapsed;
+
+					if (blobWorkerLag > 3 * limits->bwLagTarget) {
+						limits->tpsLimit = 0.0;
+						if (printRateKeepLimitReasonDetails) {
+							TraceEvent("RatekeeperLimitReasonDetails")
+							    .detail("Reason", limitReason_t::blob_worker_missing)
+							    .detail("LastValid", lastIter != version_transactions.end())
+							    .detail("FirstValid", firstIter != version_transactions.begin())
+							    .detail("FirstVersion",
+							            version_transactions.size() ? version_transactions.begin()->first : -1)
+							    .detail("FirstBWVer", firstBWVer)
+							    .detail("LastBWVer", lastBWVer)
+							    .detail("VerTransactions", version_transactions.size())
+							    .detail("RecoveryDuration", getRecoveryDuration(lastBWVer));
+						}
+						limitReason = limitReason_t::blob_worker_missing;
+						ASSERT(!g_network->isSimulated() || limits->bwLagTarget != SERVER_KNOBS->TARGET_BW_LAG ||
+						       now() < FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS + SERVER_KNOBS->BW_RK_SIM_QUIESCE_DELAY);
+					} else if (bwTPS < limits->tpsLimit) {
+						if (printRateKeepLimitReasonDetails) {
+							TraceEvent("RatekeeperLimitReasonDetails")
+							    .detail("Reason", limitReason_t::blob_worker_lag)
+							    .detail("BWLag", blobWorkerLag)
+							    .detail("BWRate", bwTPS)
+							    .detail("MaxTPS", maxTps)
+							    .detail("Progress", bwProgress)
+							    .detail("Elapsed", elapsed);
+						}
+						limits->tpsLimit = bwTPS;
+						limitReason = limitReason_t::blob_worker_lag;
+					}
+				}
+			} else if (blobWorkerLag > 3 * limits->bwLagTarget) {
+				limits->tpsLimit = 0.0;
+				if (printRateKeepLimitReasonDetails) {
+					TraceEvent("RatekeeperLimitReasonDetails")
+					    .detail("Reason", limitReason_t::blob_worker_missing)
+					    .detail("Elapsed", elapsed)
+					    .detail("LastVer", lastBWVer)
+					    .detail("FirstVer", firstBWVer)
+					    .detail("BWLag", blobWorkerLag)
+					    .detail("RecoveryDuration", getRecoveryDuration(lastBWVer));
+					;
+				}
+				limitReason = limitReason_t::blob_worker_missing;
+				ASSERT(!g_network->isSimulated() || limits->bwLagTarget != SERVER_KNOBS->TARGET_BW_LAG ||
+				       now() < FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS + SERVER_KNOBS->BW_RK_SIM_QUIESCE_DELAY);
+			}
+		} else if (blobWorkerLag > 3 * limits->bwLagTarget) {
+			limits->tpsLimit = 0.0;
+			if (printRateKeepLimitReasonDetails) {
+				TraceEvent("RatekeeperLimitReasonDetails")
+				    .detail("Reason", limitReason_t::blob_worker_missing)
+				    .detail("BWLag", blobWorkerLag)
+				    .detail("RecoveryDuration", getRecoveryDuration(lastBWVer))
+				    .detail("HistorySize", blobWorkerVersionHistory.size());
+			}
+			limitReason = limitReason_t::blob_worker_missing;
+			ASSERT(!g_network->isSimulated() || limits->bwLagTarget != SERVER_KNOBS->TARGET_BW_LAG ||
+			       now() < FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS + SERVER_KNOBS->BW_RK_SIM_QUIESCE_DELAY);
+		}
+	} else {
+		blobWorkerTime = now();
+		unblockedAssignmentTime = now();
+	}
+
 	healthMetrics.worstStorageQueue = worstStorageQueueStorageServer;
 	healthMetrics.limitingStorageQueue = limitingStorageQueueStorageServer;
 	healthMetrics.worstStorageDurabilityLag = worstDurabilityLag;
@@ -732,7 +1066,8 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 		}
 
 		if (minSSVer != std::numeric_limits<Version>::max() && maxTLVer != std::numeric_limits<Version>::min()) {
-			// writeToReadLatencyLimit: 0 = infinite speed; 1 = TL durable speed ; 2 = half TL durable speed
+			// writeToReadLatencyLimit: 0 = infinite speed; 1 = TL durable speed ; 2 = half TL durable
+			// speed
 			writeToReadLatencyLimit =
 			    ((maxTLVer - minLimitingSSVer) - limits->maxVersionDifference / 2) / (limits->maxVersionDifference / 4);
 			worstVersionLag = std::max((Version)0, maxTLVer - minSSVer);
@@ -872,7 +1207,7 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 
 	limits->tpsLimit = std::max(limits->tpsLimit, 0.0);
 
-	if (g_network->isSimulated() && g_simulator.speedUpSimulation) {
+	if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
 		limits->tpsLimit = std::max(limits->tpsLimit, 100.0);
 	}
 
@@ -900,6 +1235,14 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 	limits->tpsLimitMetric = std::min(limits->tpsLimit, 1e6);
 	limits->reasonMetric = limitReason;
 
+	if (limits->priority == TransactionPriority::DEFAULT) {
+		limits->tpsLimit = std::max(limits->tpsLimit, SERVER_KNOBS->RATEKEEPER_MIN_RATE);
+		limits->tpsLimit = std::min(limits->tpsLimit, SERVER_KNOBS->RATEKEEPER_MAX_RATE);
+	} else if (limits->priority == TransactionPriority::BATCH) {
+		limits->tpsLimit = std::max(limits->tpsLimit, SERVER_KNOBS->RATEKEEPER_BATCH_MIN_RATE);
+		limits->tpsLimit = std::min(limits->tpsLimit, SERVER_KNOBS->RATEKEEPER_BATCH_MAX_RATE);
+	}
+
 	if (deterministicRandom()->random01() < 0.1) {
 		const std::string& name = limits->rkUpdateEventCacheHolder.getPtr()->trackingKey;
 		TraceEvent(name.c_str(), id)
@@ -940,17 +1283,19 @@ ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<S
 	return Void();
 }
 
-StorageQueueInfo::StorageQueueInfo(UID id, LocalityData locality)
-  : busiestWriteTagEventHolder(makeReference<EventCacheHolder>(id.toString() + "/BusiestWriteTag")), valid(false),
-    id(id), locality(locality), acceptingRequests(false), smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
-    smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
-    smoothDurableVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothLatestVersion(SERVER_KNOBS->SMOOTHING_AMOUNT),
-    smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT),
-    limitReason(limitReason_t::unlimited) {
+StorageQueueInfo::StorageQueueInfo(const UID& ratekeeperID_, const UID& id_, const LocalityData& locality_)
+  : valid(false), ratekeeperID(ratekeeperID_), id(id_), locality(locality_), acceptingRequests(false),
+    smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
+    verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT), smoothDurableVersion(SERVER_KNOBS->SMOOTHING_AMOUNT),
+    smoothLatestVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT),
+    smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), limitReason(limitReason_t::unlimited) {
 	// FIXME: this is a tacky workaround for a potential uninitialized use in trackStorageServerQueueInfo
 	lastReply.instanceID = -1;
 }
 
+StorageQueueInfo::StorageQueueInfo(const UID& id_, const LocalityData& locality_)
+  : StorageQueueInfo(UID(), id_, locality_) {}
+
 void StorageQueueInfo::addCommitCost(TransactionTagRef tagName, TransactionCommitCostEstimation const& cost) {
 	tagCostEst[tagName] += cost;
 	totalWriteCosts += cost.getCostSum();
@@ -983,7 +1328,7 @@ void StorageQueueInfo::update(StorageQueuingMetricsReply const& reply, Smoother&
 	busiestReadTags = reply.busiestTags;
 }
 
-void StorageQueueInfo::refreshCommitCost(double elapsed) {
+UpdateCommitCostRequest StorageQueueInfo::refreshCommitCost(double elapsed) {
 	busiestWriteTags.clear();
 	TransactionTag busiestTag;
 	TransactionCommitCostEstimation maxCost;
@@ -1003,27 +1348,40 @@ void StorageQueueInfo::refreshCommitCost(double elapsed) {
 		busiestWriteTags.emplace_back(busiestTag, maxRate, maxBusyness);
 	}
 
-	TraceEvent("BusiestWriteTag", id)
-	    .detail("Elapsed", elapsed)
-	    .detail("Tag", printable(busiestTag))
-	    .detail("TagOps", maxCost.getOpsSum())
-	    .detail("TagCost", maxCost.getCostSum())
-	    .detail("TotalCost", totalWriteCosts)
-	    .detail("Reported", !busiestWriteTags.empty())
-	    .trackLatest(busiestWriteTagEventHolder->trackingKey);
+	UpdateCommitCostRequest updateCommitCostRequest{ ratekeeperID,
+		                                             now(),
+		                                             elapsed,
+		                                             busiestTag,
+		                                             maxCost.getOpsSum(),
+		                                             maxCost.getCostSum(),
+		                                             totalWriteCosts,
+		                                             !busiestWriteTags.empty(),
+		                                             ReplyPromise<Void>() };
 
 	// reset statistics
 	tagCostEst.clear();
 	totalWriteOps = 0;
 	totalWriteCosts = 0;
+
+	return updateCommitCostRequest;
+}
+
+Optional<double> StorageQueueInfo::getThrottlingRatio(int64_t storageTargetBytes, int64_t storageSpringBytes) const {
+	auto const storageQueue = getStorageQueueBytes();
+	if (storageQueue < storageTargetBytes - storageSpringBytes) {
+		return {};
+	} else {
+		return std::max(
+		    0.0, static_cast<double>((storageTargetBytes + storageSpringBytes) - storageQueue) / storageSpringBytes);
+	}
 }
 
 TLogQueueInfo::TLogQueueInfo(UID id)
   : valid(false), id(id), smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
     smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
     smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT) {
-	// FIXME: this is a tacky workaround for a potential uninitialized use in trackTLogQueueInfo (copied from
-	// storageQueueInfO)
+	// FIXME: this is a tacky workaround for a potential uninitialized use in trackTLogQueueInfo (copied
+	// from storageQueueInfO)
 	lastReply.instanceID = -1;
 }
 
@@ -1054,14 +1412,16 @@ RatekeeperLimits::RatekeeperLimits(TransactionPriority priority,
                                    int64_t logTargetBytes,
                                    int64_t logSpringBytes,
                                    double maxVersionDifference,
-                                   int64_t durabilityLagTargetVersions)
+                                   int64_t durabilityLagTargetVersions,
+                                   double bwLagTarget)
   : tpsLimit(std::numeric_limits<double>::infinity()), tpsLimitMetric(StringRef("Ratekeeper.TPSLimit" + context)),
     reasonMetric(StringRef("Ratekeeper.Reason" + context)), storageTargetBytes(storageTargetBytes),
     storageSpringBytes(storageSpringBytes), logTargetBytes(logTargetBytes), logSpringBytes(logSpringBytes),
     maxVersionDifference(maxVersionDifference),
-    durabilityLagTargetVersions(
-        durabilityLagTargetVersions +
-        SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS), // The read transaction life versions are expected to not
+    durabilityLagTargetVersions(durabilityLagTargetVersions +
+                                SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS), // The read transaction life versions
+                                                                                   // are expected to not
     // be durable on the storage servers
-    lastDurabilityLag(0), durabilityLagLimit(std::numeric_limits<double>::infinity()), priority(priority),
-    context(context), rkUpdateEventCacheHolder(makeReference<EventCacheHolder>("RkUpdate" + context)) {}
+    lastDurabilityLag(0), durabilityLagLimit(std::numeric_limits<double>::infinity()), bwLagTarget(bwLagTarget),
+    priority(priority), context(context),
+    rkUpdateEventCacheHolder(makeReference<EventCacheHolder>("RkUpdate" + context)) {}
diff --git a/fdbserver/RemoteIKeyValueStore.actor.cpp b/fdbserver/RemoteIKeyValueStore.actor.cpp
index bead82267a..63471825e8 100644
--- a/fdbserver/RemoteIKeyValueStore.actor.cpp
+++ b/fdbserver/RemoteIKeyValueStore.actor.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "fdbserver/IKeyValueStore.h"
 #include "flow/ActorCollection.h"
 #include "flow/Error.h"
 #include "flow/Platform.h"
@@ -99,8 +100,7 @@ ACTOR Future<Void> runIKVS(OpenKVStoreRequest openReq, IKVSInterface ikvsInterfa
 		try {
 			choose {
 				when(IKVSGetValueRequest getReq = waitNext(ikvsInterface.getValue.getFuture())) {
-					actors.add(cancellableForwardPromise(getReq.reply,
-					                                     kvStore->readValue(getReq.key, getReq.type, getReq.debugID)));
+					actors.add(cancellableForwardPromise(getReq.reply, kvStore->readValue(getReq.key, getReq.options)));
 				}
 				when(IKVSSetRequest req = waitNext(ikvsInterface.set.getFuture())) { kvStore->set(req.keyValue); }
 				when(IKVSClearRequest req = waitNext(ikvsInterface.clear.getFuture())) { kvStore->clear(req.range); }
@@ -110,16 +110,16 @@ ACTOR Future<Void> runIKVS(OpenKVStoreRequest openReq, IKVSInterface ikvsInterfa
 				when(IKVSReadValuePrefixRequest readPrefixReq = waitNext(ikvsInterface.readValuePrefix.getFuture())) {
 					actors.add(cancellableForwardPromise(
 					    readPrefixReq.reply,
-					    kvStore->readValuePrefix(
-					        readPrefixReq.key, readPrefixReq.maxLength, readPrefixReq.type, readPrefixReq.debugID)));
+					    kvStore->readValuePrefix(readPrefixReq.key, readPrefixReq.maxLength, readPrefixReq.options)));
 				}
 				when(IKVSReadRangeRequest readRangeReq = waitNext(ikvsInterface.readRange.getFuture())) {
 					actors.add(cancellableForwardPromise(
 					    readRangeReq.reply,
-					    fmap(
-					        [](const RangeResult& result) { return IKVSReadRangeReply(result); },
-					        kvStore->readRange(
-					            readRangeReq.keys, readRangeReq.rowLimit, readRangeReq.byteLimit, readRangeReq.type))));
+					    fmap([](const RangeResult& result) { return IKVSReadRangeReply(result); },
+					         kvStore->readRange(readRangeReq.keys,
+					                            readRangeReq.rowLimit,
+					                            readRangeReq.byteLimit,
+					                            readRangeReq.options))));
 				}
 				when(IKVSGetStorageByteRequest req = waitNext(ikvsInterface.getStorageBytes.getFuture())) {
 					StorageBytes storageBytes = kvStore->getStorageBytes();
@@ -178,6 +178,8 @@ ACTOR static Future<int> flowProcessRunner(RemoteIKeyValueStore* self, Promise<V
 		                              SERVER_KNOBS->CONN_FILE,
 		                              "--logdir",
 		                              SERVER_KNOBS->LOG_DIRECTORY,
+		                              "--trace-format",
+		                              getTraceFormatExtension(),
 		                              "-p",
 		                              flowProcessAddr,
 		                              "--process-name",
@@ -227,12 +229,6 @@ IKeyValueStore* openRemoteKVStore(KeyValueStoreType storeType,
 	return self;
 }
 
-ACTOR static Future<Void> delayFlowProcessRunAction(FlowProcess* self, double time) {
-	wait(delay(time));
-	wait(self->run());
-	return Void();
-}
-
 Future<Void> runFlowProcess(std::string const& name, Endpoint endpoint) {
 	TraceEvent(SevInfo, "RunFlowProcessStart").log();
 	FlowProcess* self = IProcessFactory::create(name.c_str());
@@ -242,5 +238,5 @@ Future<Void> runFlowProcess(std::string const& name, Endpoint endpoint) {
 	req.flowProcessInterface = self->serializedInterface();
 	registerProcess.send(req);
 	TraceEvent(SevDebug, "FlowProcessInitFinished").log();
-	return delayFlowProcessRunAction(self, g_network->isSimulated() ? 0 : SERVER_KNOBS->REMOTE_KV_STORE_INIT_DELAY);
+	return self->run();
 }
diff --git a/fdbserver/ResolutionBalancer.actor.cpp b/fdbserver/ResolutionBalancer.actor.cpp
index 5556bc5a1f..f56aec8e09 100644
--- a/fdbserver/ResolutionBalancer.actor.cpp
+++ b/fdbserver/ResolutionBalancer.actor.cpp
@@ -39,7 +39,7 @@ void ResolutionBalancer::setChangesInReply(UID requestingProxy, GetCommitVersion
 		rep.resolverChangesVersion = resolverChangesVersion;
 		resolverNeedingChanges.erase(requestingProxy);
 
-		TEST(!rep.resolverChanges.empty()); // resolution balancing moves keyranges
+		CODE_PROBE(!rep.resolverChanges.empty(), "resolution balancing moves keyranges");
 		if (resolverNeedingChanges.empty())
 			resolverChanges.set(Standalone<VectorRef<ResolverMoveRef>>());
 	}
diff --git a/fdbserver/Resolver.actor.cpp b/fdbserver/Resolver.actor.cpp
index 0987e94054..00c69106f2 100644
--- a/fdbserver/Resolver.actor.cpp
+++ b/fdbserver/Resolver.actor.cpp
@@ -26,6 +26,7 @@
 #include "fdbclient/SystemData.h"
 #include "fdbserver/ApplyMetadataMutation.h"
 #include "fdbserver/ConflictSet.h"
+#include "fdbserver/EncryptionOpsUtils.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/LogSystem.h"
@@ -193,7 +194,9 @@ struct Resolver : ReferenceCounted<Resolver> {
 };
 } // namespace
 
-ACTOR Future<Void> resolveBatch(Reference<Resolver> self, ResolveTransactionBatchRequest req) {
+ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
+                                ResolveTransactionBatchRequest req,
+                                Reference<AsyncVar<ServerDBInfo> const> db) {
 	state Optional<UID> debugID;
 	state Span span("R:resolveBatch"_loc, req.spanContext);
 
@@ -350,9 +353,9 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self, ResolveTransactionBatc
 				SpanContext spanContext =
 				    req.transactions[t].spanContext.present() ? req.transactions[t].spanContext.get() : SpanContext();
 
-				applyMetadataMutations(spanContext, *resolverData, req.transactions[t].mutations);
+				applyMetadataMutations(spanContext, *resolverData, req.transactions[t].mutations, db);
 			}
-			TEST(self->forceRecovery); // Resolver detects forced recovery
+			CODE_PROBE(self->forceRecovery, "Resolver detects forced recovery");
 		}
 
 		self->resolvedStateTransactions += req.txnStateTransactions.size();
@@ -364,7 +367,7 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self, ResolveTransactionBatc
 		ASSERT(req.version >= firstUnseenVersion);
 		ASSERT(firstUnseenVersion >= self->debugMinRecentStateVersion);
 
-		TEST(firstUnseenVersion == req.version); // Resolver first unseen version is current version
+		CODE_PROBE(firstUnseenVersion == req.version, "Resolver first unseen version is current version");
 
 		// If shardChanged at or before this commit version, the proxy may have computed
 		// the wrong set of groups. Then we need to broadcast to all groups below.
@@ -402,13 +405,14 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self, ResolveTransactionBatc
 			}
 		}
 
-		TEST(oldestProxyVersion == req.version); // The proxy that sent this request has the oldest current version
-		TEST(oldestProxyVersion !=
-		     req.version); // The proxy that sent this request does not have the oldest current version
+		CODE_PROBE(oldestProxyVersion == req.version,
+		           "The proxy that sent this request has the oldest current version");
+		CODE_PROBE(oldestProxyVersion != req.version,
+		           "The proxy that sent this request does not have the oldest current version");
 
 		bool anyPopped = false;
 		if (firstUnseenVersion <= oldestProxyVersion && self->proxyInfoMap.size() == self->commitProxyCount + 1) {
-			TEST(true); // Deleting old state transactions
+			CODE_PROBE(true, "Deleting old state transactions");
 			int64_t erasedBytes = self->recentStateTransactionsInfo.eraseUpTo(oldestProxyVersion);
 			self->debugMinRecentStateVersion = oldestProxyVersion + 1;
 			anyPopped = erasedBytes > 0;
@@ -447,7 +451,7 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self, ResolveTransactionBatc
 		if (req.debugID.present())
 			g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "Resolver.resolveBatch.After");
 	} else {
-		TEST(true); // Duplicate resolve batch request
+		CODE_PROBE(true, "Duplicate resolve batch request");
 		//TraceEvent("DupResolveBatchReq", self->dbgid).detail("From", proxyAddress);
 	}
 
@@ -458,13 +462,13 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self, ResolveTransactionBatc
 		if (batchItr != proxyInfoItr->second.outstandingBatches.end()) {
 			req.reply.send(batchItr->second);
 		} else {
-			TEST(true); // No outstanding batches for version on proxy
+			CODE_PROBE(true, "No outstanding batches for version on proxy");
 			req.reply.send(Never());
 		}
 	} else {
 		ASSERT_WE_THINK(false); // The first non-duplicate request with this proxyAddress, including this one, should
 		                        // have inserted this item in the map!
-		// TEST(true); // No prior proxy requests
+		// CODE_PROBE(true, "No prior proxy requests");
 		req.reply.send(Never());
 	}
 
@@ -504,7 +508,8 @@ struct TransactionStateResolveContext {
 	}
 };
 
-ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolveContext* pContext) {
+ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolveContext* pContext,
+                                                          Reference<AsyncVar<ServerDBInfo> const> db) {
 	state KeyRange txnKeys = allKeys;
 	state std::map<Tag, UID> tag_uid;
 
@@ -571,8 +576,7 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
 		bool confChanges; // Ignore configuration changes for initial commits.
 		ResolverData resolverData(
 		    pContext->pResolverData->dbgid, pContext->pTxnStateStore, &pContext->pResolverData->keyInfo, confChanges);
-
-		applyMetadataMutations(SpanContext(), resolverData, mutations);
+		applyMetadataMutations(SpanContext(), resolverData, mutations, db);
 	} // loop
 
 	auto lockedKey = pContext->pTxnStateStore->readValue(databaseLockedKey).get();
@@ -585,7 +589,8 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
 }
 
 ACTOR Future<Void> processTransactionStateRequestPart(TransactionStateResolveContext* pContext,
-                                                      TxnStateRequest request) {
+                                                      TxnStateRequest request,
+                                                      Reference<AsyncVar<ServerDBInfo> const> db) {
 	ASSERT(pContext->pResolverData.getPtr() != nullptr);
 	ASSERT(pContext->pActors != nullptr);
 
@@ -612,7 +617,7 @@ ACTOR Future<Void> processTransactionStateRequestPart(TransactionStateResolveCon
 	if (pContext->receivedSequences.size() == pContext->maxSequence) {
 		// Received all components of the txnStateRequest
 		ASSERT(!pContext->processed);
-		wait(processCompleteTransactionStateRequest(pContext));
+		wait(processCompleteTransactionStateRequest(pContext, db));
 		pContext->processed = true;
 	}
 
@@ -650,8 +655,15 @@ ACTOR Future<Void> resolverCore(ResolverInterface resolver,
 	state TransactionStateResolveContext transactionStateResolveContext;
 	if (SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS) {
 		self->logAdapter = new LogSystemDiskQueueAdapter(self->logSystem, Reference<AsyncVar<PeekTxsInfo>>(), 1, false);
-		self->txnStateStore = keyValueStoreLogSystem(
-		    self->logAdapter, db, resolver.id(), 2e9, true, true, true, SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION);
+		self->txnStateStore =
+		    keyValueStoreLogSystem(self->logAdapter,
+		                           db,
+		                           resolver.id(),
+		                           2e9,
+		                           true,
+		                           true,
+		                           true,
+		                           isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION, db->get().client));
 
 		// wait for txnStateStore recovery
 		wait(success(self->txnStateStore->readValue(StringRef())));
@@ -668,7 +680,7 @@ ACTOR Future<Void> resolverCore(ResolverInterface resolver,
 
 	loop choose {
 		when(ResolveTransactionBatchRequest batch = waitNext(resolver.resolve.getFuture())) {
-			actors.add(resolveBatch(self, batch));
+			actors.add(resolveBatch(self, batch, db));
 		}
 		when(ResolutionMetricsRequest req = waitNext(resolver.metrics.getFuture())) {
 			++self->metricsRequests;
@@ -691,7 +703,7 @@ ACTOR Future<Void> resolverCore(ResolverInterface resolver,
 		}
 		when(TxnStateRequest request = waitNext(resolver.txnState.getFuture())) {
 			if (SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS) {
-				addActor.send(processTransactionStateRequestPart(&transactionStateResolveContext, request));
+				addActor.send(processTransactionStateRequestPart(&transactionStateResolveContext, request, db));
 			} else {
 				ASSERT(false);
 			}
diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index 7b361af4a7..afa1487cd1 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -286,7 +286,7 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 		ASSERT(!g_network->isSimulated());
 		int i = 0;
 		for (auto& key : incompleteStagingKeys) {
-			MutationRef m(MutationRef::SetValue, key.first, LiteralStringRef("0"));
+			MutationRef m(MutationRef::SetValue, key.first, "0"_sr);
 			key.second->second.add(m, LogMessageVersion(1));
 			key.second->second.precomputeResult("GetAndComputeStagingKeys", applierID, batchIndex);
 			i++;
diff --git a/fdbserver/RestoreCommon.actor.cpp b/fdbserver/RestoreCommon.actor.cpp
index fbaaaf8ef5..54dfc0769b 100644
--- a/fdbserver/RestoreCommon.actor.cpp
+++ b/fdbserver/RestoreCommon.actor.cpp
@@ -38,58 +38,58 @@
 // RestoreCommon.actor.cpp
 
 KeyBackedProperty<ERestoreState> RestoreConfigFR::stateEnum() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 Future<StringRef> RestoreConfigFR::stateText(Reference<ReadYourWritesTransaction> tr) {
 	return map(stateEnum().getD(tr), [](ERestoreState s) -> StringRef { return FileBackupAgent::restoreStateText(s); });
 }
 KeyBackedProperty<Key> RestoreConfigFR::addPrefix() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 KeyBackedProperty<Key> RestoreConfigFR::removePrefix() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 // XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges
 KeyBackedProperty<KeyRange> RestoreConfigFR::restoreRange() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 KeyBackedProperty<std::vector<KeyRange>> RestoreConfigFR::restoreRanges() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 KeyBackedProperty<Key> RestoreConfigFR::batchFuture() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 KeyBackedProperty<Version> RestoreConfigFR::restoreVersion() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 
 KeyBackedProperty<Reference<IBackupContainer>> RestoreConfigFR::sourceContainer() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 // Get the source container as a bare URL, without creating a container instance
 KeyBackedProperty<Value> RestoreConfigFR::sourceContainerURL() {
-	return configSpace.pack(LiteralStringRef("sourceContainer"));
+	return configSpace.pack("sourceContainer"_sr);
 }
 
 // Total bytes written by all log and range restore tasks.
 KeyBackedBinaryValue<int64_t> RestoreConfigFR::bytesWritten() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 // File blocks that have had tasks created for them by the Dispatch task
 KeyBackedBinaryValue<int64_t> RestoreConfigFR::filesBlocksDispatched() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 // File blocks whose tasks have finished
 KeyBackedBinaryValue<int64_t> RestoreConfigFR::fileBlocksFinished() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 // Total number of files in the fileMap
 KeyBackedBinaryValue<int64_t> RestoreConfigFR::fileCount() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 // Total number of file blocks in the fileMap
 KeyBackedBinaryValue<int64_t> RestoreConfigFR::fileBlockCount() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 
 Future<std::vector<KeyRange>> RestoreConfigFR::getRestoreRangesOrDefault(Reference<ReadYourWritesTransaction> tr) {
@@ -108,7 +108,7 @@ ACTOR Future<std::vector<KeyRange>> RestoreConfigFR::getRestoreRangesOrDefault_i
 }
 
 KeyBackedSet<RestoreConfigFR::RestoreFile> RestoreConfigFR::fileSet() {
-	return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	return configSpace.pack(__FUNCTION__sr);
 }
 
 Future<bool> RestoreConfigFR::isRunnable(Reference<ReadYourWritesTransaction> tr) {
diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp
index 8092ebe39a..99273dfa25 100644
--- a/fdbserver/RestoreController.actor.cpp
+++ b/fdbserver/RestoreController.actor.cpp
@@ -48,7 +48,8 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
 ACTOR static Future<Void> buildRangeVersions(KeyRangeMap<Version>* pRangeVersions,
                                              std::vector<RestoreFileFR>* pRangeFiles,
                                              Key url,
-                                             Optional<std::string> proxy);
+                                             Optional<std::string> proxy,
+                                             Optional<Database> cx);
 
 ACTOR static Future<Version> processRestoreRequest(Reference<RestoreControllerData> self,
                                                    Database cx,
@@ -335,7 +336,7 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreControllerDa
 	// Build range versions: version of key ranges in range file
 	state KeyRangeMap<Version> rangeVersions(minRangeVersion, allKeys.end);
 	if (SERVER_KNOBS->FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE) {
-		wait(buildRangeVersions(&rangeVersions, &rangeFiles, request.url, request.proxy));
+		wait(buildRangeVersions(&rangeVersions, &rangeFiles, request.url, request.proxy, cx));
 	} else {
 		// Debug purpose, dump range versions
 		auto ranges = rangeVersions.ranges();
@@ -657,12 +658,12 @@ void splitKeyRangeForAppliers(Reference<ControllerBatchData> batchData,
 	    .detail("SlotSize", slotSize);
 
 	std::set<Key> keyrangeSplitter; // unique key to split key range for appliers
-	keyrangeSplitter.insert(normalKeys.begin); // First slot
+	keyrangeSplitter.insert(allKeys.begin); // First slot
 	TraceEvent("FastRestoreControllerPhaseCalculateApplierKeyRanges")
 	    .detail("BatchIndex", batchIndex)
 	    .detail("CumulativeSize", cumulativeSize)
 	    .detail("Slot", 0)
-	    .detail("LowerBoundKey", normalKeys.begin);
+	    .detail("LowerBoundKey", allKeys.begin);
 	int slotIdx = 1;
 	while (cumulativeSize < batchData->samplesSize) {
 		IndexedSet<Key, int64_t>::iterator lowerBound = batchData->samples.index(cumulativeSize);
@@ -774,7 +775,7 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
 
 	state VectorRef<KeyRangeRef> restoreRanges;
 	restoreRanges.add(request.range);
-	Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(request.targetVersion, restoreRanges));
+	Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(request.targetVersion, cx, restoreRanges));
 
 	if (!restorable.present()) {
 		TraceEvent(SevWarn, "FastRestoreControllerPhaseCollectBackupFiles")
@@ -843,12 +844,13 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
 // set (beginKey, endKey) and file->version to pRangeVersions
 ACTOR static Future<Void> insertRangeVersion(KeyRangeMap<Version>* pRangeVersions,
                                              RestoreFileFR* file,
-                                             Reference<IBackupContainer> bc) {
+                                             Reference<IBackupContainer> bc,
+                                             Optional<Database> cx) {
 	TraceEvent("FastRestoreControllerDecodeRangeVersion").detail("File", file->toString());
 	RangeFile rangeFile = { file->version, (uint32_t)file->blockSize, file->fileName, file->fileSize };
 
 	// First and last key are the range for this file: endKey is exclusive
-	KeyRange fileRange = wait(bc->getSnapshotFileKeyRange(rangeFile));
+	KeyRange fileRange = wait(bc->getSnapshotFileKeyRange(rangeFile, cx));
 	TraceEvent("FastRestoreControllerInsertRangeVersion")
 	    .detail("DecodedRangeFile", file->fileName)
 	    .detail("KeyRange", fileRange)
@@ -883,7 +885,8 @@ ACTOR static Future<Void> insertRangeVersion(KeyRangeMap<Version>* pRangeVersion
 ACTOR static Future<Void> buildRangeVersions(KeyRangeMap<Version>* pRangeVersions,
                                              std::vector<RestoreFileFR>* pRangeFiles,
                                              Key url,
-                                             Optional<std::string> proxy) {
+                                             Optional<std::string> proxy,
+                                             Optional<Database> cx) {
 	if (!g_network->isSimulated()) {
 		TraceEvent(SevError, "ExpensiveBuildRangeVersions")
 		    .detail("Reason", "Parsing all range files is slow and memory intensive");
@@ -896,7 +899,7 @@ ACTOR static Future<Void> buildRangeVersions(KeyRangeMap<Version>* pRangeVersion
 	state int fileIndex = 0;
 	state std::vector<Future<Void>> fInsertRangeVersions;
 	for (; fileIndex < pRangeFiles->size(); ++fileIndex) {
-		fInsertRangeVersions.push_back(insertRangeVersion(pRangeVersions, &pRangeFiles->at(fileIndex), bc));
+		fInsertRangeVersions.push_back(insertRangeVersion(pRangeVersions, &pRangeFiles->at(fileIndex), bc, cx));
 	}
 
 	wait(waitForAll(fInsertRangeVersions));
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 736e73920f..964045c153 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -21,10 +21,12 @@
 // This file implements the functions and actors used by the RestoreLoader role.
 // The RestoreLoader role starts with the restoreLoaderCore actor
 
+#include "fdbclient/BlobCipher.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupAgent.actor.h"
-#include "fdbserver/EncryptedMutationMessage.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/DatabaseContext.h"
 #include "fdbserver/RestoreLoader.actor.h"
 #include "fdbserver/RestoreRoleCommon.actor.h"
 #include "fdbserver/MutationTracking.h"
@@ -45,17 +47,19 @@ void splitMutation(const KeyRangeMap<UID>& krMap,
                    VectorRef<MutationRef>& mvector,
                    Arena& nodeIDs_arena,
                    VectorRef<UID>& nodeIDs);
-void _parseSerializedMutation(KeyRangeMap<Version>* pRangeVersions,
-                              std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
-                              SerializedMutationListMap* mutationMap,
-                              std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
-                              LoaderCounters* cc,
-                              const RestoreAsset& asset);
+ACTOR Future<Void> _parseSerializedMutation(KeyRangeMap<Version>* pRangeVersions,
+                                            std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
+                                            SerializedMutationListMap* mutationMap,
+                                            std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
+                                            LoaderCounters* cc,
+                                            RestoreAsset asset,
+                                            Database cx);
 
 void handleRestoreSysInfoRequest(const RestoreSysInfoRequest& req, Reference<RestoreLoaderData> self);
-ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self);
+ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self, Database cx);
 ACTOR Future<Void> handleSendMutationsRequest(RestoreSendMutationsToAppliersRequest req,
-                                              Reference<RestoreLoaderData> self);
+                                              Reference<RestoreLoaderData> self,
+                                              Database cx);
 ACTOR Future<Void> sendMutationsToApplier(
     std::priority_queue<RestoreLoaderSchedSendLoadParamRequest>* sendLoadParamQueue,
     std::map<int, int>* inflightSendLoadParamReqs,
@@ -65,7 +69,8 @@ ACTOR Future<Void> sendMutationsToApplier(
     RestoreAsset asset,
     bool isRangeFile,
     std::map<Key, UID>* pRangeToApplier,
-    std::map<UID, RestoreApplierInterface>* pApplierInterfaces);
+    std::map<UID, RestoreApplierInterface>* pApplierInterfaces,
+    Database cx);
 ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(NotifiedVersion* pProcessedFileOffset,
                                                            SerializedMutationListMap* mutationMap,
                                                            Reference<IBackupContainer> bc,
@@ -80,13 +85,14 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
     LoaderCounters* cc,
     Reference<IBackupContainer> bc,
     Version version,
-    RestoreAsset asset);
+    RestoreAsset asset,
+    Optional<Database> cx);
 ACTOR Future<Void> handleFinishVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreLoaderData> self);
 
 // Dispatch requests based on node's business (i.e, cpu usage for now) and requests' priorities
 // Requests for earlier version batches are preferred; which is equivalent to
 // sendMuttionsRequests are preferred than loadingFileRequests
-ACTOR Future<Void> dispatchRequests(Reference<RestoreLoaderData> self) {
+ACTOR Future<Void> dispatchRequests(Reference<RestoreLoaderData> self, Database cx) {
 	try {
 		state int curVBInflightReqs = 0;
 		state int sendLoadParams = 0;
@@ -140,7 +146,7 @@ ACTOR Future<Void> dispatchRequests(Reference<RestoreLoaderData> self) {
 				// Dispatch the request if it is the next version batch to process or if cpu usage is low
 				if (req.batchIndex - 1 == self->finishedSendingVB ||
 				    self->cpuUsage < SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT) {
-					self->addActor.send(handleSendMutationsRequest(req, self));
+					self->addActor.send(handleSendMutationsRequest(req, self, cx));
 					self->sendingQueue.pop();
 				}
 			}
@@ -205,7 +211,7 @@ ACTOR Future<Void> dispatchRequests(Reference<RestoreLoaderData> self) {
 					self->loadingQueue.pop();
 					ASSERT(false); // Check if this ever happens easily
 				} else {
-					self->addActor.send(handleLoadFileRequest(req, self));
+					self->addActor.send(handleLoadFileRequest(req, self, cx));
 					self->loadingQueue.pop();
 					lastLoadReqs++;
 				}
@@ -245,7 +251,7 @@ ACTOR Future<Void> restoreLoaderCore(RestoreLoaderInterface loaderInterf,
 	actors.add(updateProcessMetrics(self));
 	actors.add(traceProcessMetrics(self, "RestoreLoader"));
 
-	self->addActor.send(dispatchRequests(self));
+	self->addActor.send(dispatchRequests(self, cx));
 
 	loop {
 		state std::string requestTypeStr = "[Init]";
@@ -362,6 +368,18 @@ void handleRestoreSysInfoRequest(const RestoreSysInfoRequest& req, Reference<Res
 	req.reply.send(RestoreCommonReply(self->id()));
 }
 
+ACTOR static Future<MutationRef> _decryptMutation(MutationRef mutation, Database cx, Arena* arena) {
+	ASSERT(mutation.isEncrypted());
+	Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
+	state const BlobCipherEncryptHeader* header = mutation.encryptionHeader();
+	std::unordered_set<BlobCipherDetails> cipherDetails;
+	cipherDetails.insert(header->cipherHeaderDetails);
+	cipherDetails.insert(header->cipherTextDetails);
+	std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> getCipherKeysResult =
+	    wait(getEncryptCipherKeys(dbInfo, cipherDetails, BlobCipherMetrics::BACKUP));
+	return mutation.decrypt(getCipherKeysResult, *arena, BlobCipherMetrics::BACKUP);
+}
+
 // Parse a data block in a partitioned mutation log file and store mutations
 // into "kvOpsIter" and samples into "samplesIter".
 ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
@@ -371,7 +389,8 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
     std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
     LoaderCounters* cc,
     Reference<IBackupContainer> bc,
-    RestoreAsset asset) {
+    RestoreAsset asset,
+    Database cx) {
 	state Standalone<StringRef> buf = makeString(asset.len);
 	state Reference<IAsyncFile> file = wait(bc->readFile(asset.filename));
 	int rLen = wait(file->read(mutateString(buf), asset.len, asset.offset));
@@ -390,21 +409,21 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 	wait(processedFileOffset->whenAtLeast(asset.offset));
 	ASSERT(processedFileOffset->get() == asset.offset);
 
-	Arena tempArena;
-	StringRefReader reader(buf, restore_corrupted_data());
+	state Arena tempArena;
+	state StringRefReader reader(buf, restore_corrupted_data());
 	try {
 		// Read block header
 		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION)
 			throw restore_unsupported_file_version();
 
-		VersionedMutationsMap& kvOps = kvOpsIter->second;
+		state VersionedMutationsMap* kvOps = &kvOpsIter->second;
 		while (1) {
 			// If eof reached or first key len bytes is 0xFF then end of block was reached.
 			if (reader.eof() || *reader.rptr == 0xFF)
 				break;
 
 			// Deserialize messages written in saveMutationsToFile().
-			LogMessageVersion msgVersion;
+			state LogMessageVersion msgVersion;
 			msgVersion.version = reader.consumeNetworkUInt64();
 			msgVersion.sub = reader.consumeNetworkUInt32();
 			int msgSize = reader.consumeNetworkInt32();
@@ -414,20 +433,21 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 			if (!asset.isInVersionRange(msgVersion.version))
 				continue;
 
-			VersionedMutationsMap::iterator it;
+			state VersionedMutationsMap::iterator it;
 			bool inserted;
-			std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec());
+			std::tie(it, inserted) = kvOps->emplace(msgVersion, MutationsVec());
 			// A clear mutation can be split into multiple mutations with the same (version, sub).
 			// See saveMutationsToFile(). Current tests only use one key range per backup, thus
 			// only one clear mutation is generated (i.e., always inserted).
 			ASSERT(inserted);
 
 			ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion()));
-			if (EncryptedMutationMessage::isNextIn(rd)) {
-				throw encrypt_unsupported();
-			}
-			MutationRef mutation;
+			state MutationRef mutation;
 			rd >> mutation;
+			if (mutation.isEncrypted()) {
+				MutationRef decryptedMutation = wait(_decryptMutation(mutation, cx, &tempArena));
+				mutation = decryptedMutation;
+			}
 
 			// Skip mutation whose commitVesion < range kv's version
 			if (logMutationTooOld(pRangeVersions, mutation, msgVersion.version)) {
@@ -501,12 +521,13 @@ ACTOR static Future<Void> parsePartitionedLogFileOnLoader(
     std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
     LoaderCounters* cc,
     Reference<IBackupContainer> bc,
-    RestoreAsset asset) {
+    RestoreAsset asset,
+    Database cx) {
 	state int readFileRetries = 0;
 	loop {
 		try {
 			wait(_parsePartitionedLogFileOnLoader(
-			    pRangeVersions, processedFileOffset, kvOpsIter, samplesIter, cc, bc, asset));
+			    pRangeVersions, processedFileOffset, kvOpsIter, samplesIter, cc, bc, asset, cx));
 			break;
 		} catch (Error& e) {
 			if (e.code() == error_code_restore_bad_read || e.code() == error_code_restore_unsupported_file_version ||
@@ -533,7 +554,8 @@ ACTOR Future<Void> _processLoadingParam(KeyRangeMap<Version>* pRangeVersions,
                                         LoadingParam param,
                                         Reference<LoaderBatchData> batchData,
                                         UID loaderID,
-                                        Reference<IBackupContainer> bc) {
+                                        Reference<IBackupContainer> bc,
+                                        Database cx) {
 	// Temporary data structure for parsing log files into (version, <K, V, mutationType>)
 	// Must use StandAlone to save mutations, otherwise, the mutationref memory will be corrupted
 	// mutationMap: Key is the unique identifier for a batch of mutation logs at the same version
@@ -563,7 +585,7 @@ ACTOR Future<Void> _processLoadingParam(KeyRangeMap<Version>* pRangeVersions,
 		subAsset.len = std::min<int64_t>(param.blockSize, param.asset.len - j);
 		if (param.isRangeFile) {
 			fileParserFutures.push_back(_parseRangeFileToMutationsOnLoader(
-			    kvOpsPerLPIter, samplesIter, &batchData->counters, bc, param.rangeVersion.get(), subAsset));
+			    kvOpsPerLPIter, samplesIter, &batchData->counters, bc, param.rangeVersion.get(), subAsset, cx));
 		} else {
 			// TODO: Sanity check the log file's range is overlapped with the restored version range
 			if (param.isPartitionedLog()) {
@@ -573,7 +595,8 @@ ACTOR Future<Void> _processLoadingParam(KeyRangeMap<Version>* pRangeVersions,
 				                                                            samplesIter,
 				                                                            &batchData->counters,
 				                                                            bc,
-				                                                            subAsset));
+				                                                            subAsset,
+				                                                            cx));
 			} else {
 				fileParserFutures.push_back(
 				    parseLogFileToMutationsOnLoader(&processedFileOffset, &mutationMap, bc, subAsset));
@@ -583,8 +606,8 @@ ACTOR Future<Void> _processLoadingParam(KeyRangeMap<Version>* pRangeVersions,
 	wait(waitForAll(fileParserFutures));
 
 	if (!param.isRangeFile && !param.isPartitionedLog()) {
-		_parseSerializedMutation(
-		    pRangeVersions, kvOpsPerLPIter, &mutationMap, samplesIter, &batchData->counters, param.asset);
+		wait(_parseSerializedMutation(
+		    pRangeVersions, kvOpsPerLPIter, &mutationMap, samplesIter, &batchData->counters, param.asset, cx));
 	}
 
 	TraceEvent("FastRestoreLoaderProcessLoadingParamDone", loaderID)
@@ -595,7 +618,7 @@ ACTOR Future<Void> _processLoadingParam(KeyRangeMap<Version>* pRangeVersions,
 }
 
 // A loader can process multiple RestoreLoadFileRequest in parallel.
-ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self) {
+ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self, Database cx) {
 	state Reference<LoaderBatchData> batchData = self->batch[req.batchIndex];
 	state bool isDuplicated = true;
 	state bool printTrace = false;
@@ -624,7 +647,7 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
 		    .detail("ProcessLoadParam", req.param.toString());
 		ASSERT(batchData->sampleMutations.find(req.param) == batchData->sampleMutations.end());
 		batchData->processedFileParams[req.param] =
-		    _processLoadingParam(&self->rangeVersions, req.param, batchData, self->id(), self->bc);
+		    _processLoadingParam(&self->rangeVersions, req.param, batchData, self->id(), self->bc, cx);
 		self->inflightLoadingReqs++;
 		isDuplicated = false;
 	} else {
@@ -683,7 +706,8 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
 // Send buffered mutations to appliers.
 // Do not need to block on low memory usage because this actor should not increase memory usage.
 ACTOR Future<Void> handleSendMutationsRequest(RestoreSendMutationsToAppliersRequest req,
-                                              Reference<RestoreLoaderData> self) {
+                                              Reference<RestoreLoaderData> self,
+                                              Database cx) {
 	state Reference<LoaderBatchData> batchData;
 	state Reference<LoaderBatchStatus> batchStatus;
 	state bool isDuplicated = true;
@@ -760,7 +784,8 @@ ACTOR Future<Void> handleSendMutationsRequest(RestoreSendMutationsToAppliersRequ
 				                                                loadParam.asset,
 				                                                loadParam.isRangeFile,
 				                                                &batchData->rangeToApplier,
-				                                                &self->appliersInterf));
+				                                                &self->appliersInterf,
+				                                                cx));
 			}
 		}
 		wait(waitForAll(fSendMutations));
@@ -795,7 +820,7 @@ void buildApplierRangeMap(KeyRangeMap<UID>* krMap, std::map<Key, UID>* pRangeToA
 		endKey++;
 	}
 	if (beginKey != pRangeToApplier->end()) {
-		krMap->insert(KeyRangeRef(beginKey->first, normalKeys.end), beginKey->second);
+		krMap->insert(KeyRangeRef(beginKey->first, allKeys.end), beginKey->second);
 	}
 }
 
@@ -813,7 +838,8 @@ ACTOR Future<Void> sendMutationsToApplier(
     RestoreAsset asset,
     bool isRangeFile,
     std::map<Key, UID>* pRangeToApplier,
-    std::map<UID, RestoreApplierInterface>* pApplierInterfaces) {
+    std::map<UID, RestoreApplierInterface>* pApplierInterfaces,
+    Database cx) {
 	state VersionedMutationsMap& kvOps = *pkvOps;
 	state VersionedMutationsMap::iterator kvOp = kvOps.begin();
 	state int kvCount = 0;
@@ -821,6 +847,7 @@ ACTOR Future<Void> sendMutationsToApplier(
 	state Version msgIndex = 1; // Monotonically increased index for send message, must start at 1
 	state std::vector<UID> applierIDs = getApplierIDs(*pRangeToApplier);
 	state double msgSize = 0; // size of mutations in the message
+	state Arena arena;
 
 	// Wait for scheduler to kick it off
 	Promise<Void> toSched;
@@ -864,14 +891,18 @@ ACTOR Future<Void> sendMutationsToApplier(
 	for (auto& applierID : applierIDs) {
 		applierVersionedMutationsBuffer[applierID] = VersionedMutationsVec();
 	}
-	KeyRangeMap<UID> krMap;
+	state KeyRangeMap<UID> krMap;
 	buildApplierRangeMap(&krMap, pRangeToApplier);
 	for (kvOp = kvOps.begin(); kvOp != kvOps.end(); kvOp++) {
 		commitVersion = kvOp->first;
 		ASSERT(commitVersion.version >= asset.beginVersion);
 		ASSERT(commitVersion.version <= asset.endVersion); // endVersion is an empty commit to ensure progress
 		for (mIndex = 0; mIndex < kvOp->second.size(); mIndex++) {
-			MutationRef& kvm = kvOp->second[mIndex];
+			state MutationRef kvm = kvOp->second[mIndex];
+			if (kvm.isEncrypted()) {
+				MutationRef decryptedMutation = wait(_decryptMutation(kvm, cx, &arena));
+				kvm = decryptedMutation;
+			}
 			// Send the mutation to applier
 			if (isRangeMutation(kvm)) {
 				MutationsVec mvector;
@@ -1083,31 +1114,35 @@ bool concatenateBackupMutationForLogFile(SerializedMutationListMap* pMutationMap
 // we may not get the entire mutation list for the version encoded_list_of_mutations:
 // [mutation1][mutation2]...[mutationk], where
 //	a mutation is encoded as [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][keyContent][valueContent]
-void _parseSerializedMutation(KeyRangeMap<Version>* pRangeVersions,
-                              std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
-                              SerializedMutationListMap* pmutationMap,
-                              std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
-                              LoaderCounters* cc,
-                              const RestoreAsset& asset) {
-	VersionedMutationsMap& kvOps = kvOpsIter->second;
-	SampledMutationsVec& samples = samplesIter->second;
-	SerializedMutationListMap& mutationMap = *pmutationMap;
+ACTOR Future<Void> _parseSerializedMutation(KeyRangeMap<Version>* pRangeVersions,
+                                            std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
+                                            SerializedMutationListMap* pmutationMap,
+                                            std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
+                                            LoaderCounters* cc,
+                                            RestoreAsset asset,
+                                            Database cx) {
+	state VersionedMutationsMap* kvOps = &kvOpsIter->second;
+	state SampledMutationsVec* samples = &samplesIter->second;
+	state SerializedMutationListMap::iterator mutationMapIterator = pmutationMap->begin();
 
 	TraceEvent(SevFRMutationInfo, "FastRestoreLoaderParseSerializedLogMutation")
 	    .detail("BatchIndex", asset.batchIndex)
 	    .detail("RestoreAsset", asset.toString());
 
-	Arena tempArena;
-	for (auto& m : mutationMap) {
-		StringRef k = m.first.contents();
-		StringRef val = m.second.first.contents();
+	state Arena tempArena;
+	loop {
+		if (mutationMapIterator == pmutationMap->end()) {
+			break;
+		}
+		StringRef k = mutationMapIterator->first.contents();
+		state StringRef val = mutationMapIterator->second.first.contents();
 
 		StringRefReader kReader(k, restore_corrupted_data());
-		uint64_t commitVersion = kReader.consume<uint64_t>(); // Consume little Endian data
+		state uint64_t commitVersion = kReader.consume<uint64_t>(); // Consume little Endian data
 		// We have already filter the commit not in [beginVersion, endVersion) when we concatenate kv pair in log file
 		ASSERT_WE_THINK(asset.isInVersionRange(commitVersion));
 
-		StringRefReader vReader(val, restore_corrupted_data());
+		state StringRefReader vReader(val, restore_corrupted_data());
 		vReader.consume<uint64_t>(); // Consume the includeVersion
 		// TODO(xumengpanda): verify the protocol version is compatible and raise error if needed
 
@@ -1115,72 +1150,79 @@ void _parseSerializedMutation(KeyRangeMap<Version>* pRangeVersions,
 		uint32_t val_length_decoded = vReader.consume<uint32_t>();
 		ASSERT(val_length_decoded == val.size() - sizeof(uint64_t) - sizeof(uint32_t));
 
-		int sub = 0;
-		while (1) {
+		state int sub = 0;
+		loop {
 			// stop when reach the end of the string
 			if (vReader.eof()) { //|| *reader.rptr == 0xFF
 				break;
 			}
 
-			uint32_t type = vReader.consume<uint32_t>();
-			uint32_t kLen = vReader.consume<uint32_t>();
-			uint32_t vLen = vReader.consume<uint32_t>();
-			const uint8_t* k = vReader.consume(kLen);
-			const uint8_t* v = vReader.consume(vLen);
+			state uint32_t type = vReader.consume<uint32_t>();
+			state uint32_t kLen = vReader.consume<uint32_t>();
+			state uint32_t vLen = vReader.consume<uint32_t>();
+			state const uint8_t* k = vReader.consume(kLen);
+			state const uint8_t* v = vReader.consume(vLen);
 
-			MutationRef mutation((MutationRef::Type)type, KeyRef(k, kLen), KeyRef(v, vLen));
+			state MutationRef mutation((MutationRef::Type)type, KeyRef(k, kLen), KeyRef(v, vLen));
+			if (mutation.isEncrypted()) {
+				MutationRef decryptedMutation = wait(_decryptMutation(mutation, cx, &tempArena));
+				mutation = decryptedMutation;
+			}
 			// Should this mutation be skipped?
 			// Skip mutation whose commitVesion < range kv's version
 			if (logMutationTooOld(pRangeVersions, mutation, commitVersion)) {
 				cc->oldLogMutations += 1;
-				continue;
-			}
-
-			if (mutation.param1 >= asset.range.end ||
-			    (isRangeMutation(mutation) && mutation.param2 < asset.range.begin) ||
-			    (!isRangeMutation(mutation) && mutation.param1 < asset.range.begin)) {
-				continue;
-			}
-			// Only apply mutation within the asset.range and apply removePrefix and addPrefix
-			ASSERT(asset.removePrefix.size() == 0);
-			if (isRangeMutation(mutation)) {
-				mutation.param1 = mutation.param1 >= asset.range.begin ? mutation.param1 : asset.range.begin;
-				mutation.param2 = mutation.param2 < asset.range.end ? mutation.param2 : asset.range.end;
-				// Remove prefix or add prefix if we restore data to a new key space
-				if (asset.hasPrefix()) { // Avoid creating new Key
-					mutation.param1 =
-					    mutation.param1.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
-					mutation.param2 =
-					    mutation.param2.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
-				}
 			} else {
-				if (asset.hasPrefix()) { // Avoid creating new Key
-					mutation.param1 =
-					    mutation.param1.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
+				if (mutation.param1 >= asset.range.end ||
+				    (isRangeMutation(mutation) && mutation.param2 < asset.range.begin) ||
+				    (!isRangeMutation(mutation) && mutation.param1 < asset.range.begin)) {
+				} else {
+					// Only apply mutation within the asset.range and apply removePrefix and addPrefix
+					ASSERT(asset.removePrefix.size() == 0);
+					if (isRangeMutation(mutation)) {
+						mutation.param1 = mutation.param1 >= asset.range.begin ? mutation.param1 : asset.range.begin;
+						mutation.param2 = mutation.param2 < asset.range.end ? mutation.param2 : asset.range.end;
+						// Remove prefix or add prefix if we restore data to a new key space
+						if (asset.hasPrefix()) { // Avoid creating new Key
+							mutation.param1 =
+							    mutation.param1.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
+							mutation.param2 =
+							    mutation.param2.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
+						}
+					} else {
+						if (asset.hasPrefix()) { // Avoid creating new Key
+							mutation.param1 =
+							    mutation.param1.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
+						}
+					}
+
+					cc->loadedLogBytes += mutation.totalSize();
+
+					TraceEvent(SevFRMutationInfo, "FastRestoreDecodeLogFile")
+					    .detail("CommitVersion", commitVersion)
+					    .detail("ParsedMutation", mutation.toString());
+
+					auto it = kvOps->insert(std::make_pair(LogMessageVersion(commitVersion, sub++), MutationsVec()));
+					ASSERT(it.second); // inserted is true
+					ASSERT(sub <
+					       std::numeric_limits<int32_t>::max()); // range file mutation uses int32_max as subversion
+					it.first->second.push_back_deep(it.first->second.arena(), mutation);
+
+					// Sampling data similar to how SS sample bytes
+					ByteSampleInfo sampleInfo = isKeyValueInSample(KeyValueRef(mutation.param1, mutation.param2));
+					if (sampleInfo.inSample) {
+						cc->sampledLogBytes += sampleInfo.sampledSize;
+						samples->push_back_deep(samples->arena(),
+						                        SampledMutation(mutation.param1, sampleInfo.sampledSize));
+					}
+					ASSERT_WE_THINK(kLen >= 0 && kLen < val.size());
+					ASSERT_WE_THINK(vLen >= 0 && vLen < val.size());
 				}
 			}
-
-			cc->loadedLogBytes += mutation.totalSize();
-
-			TraceEvent(SevFRMutationInfo, "FastRestoreDecodeLogFile")
-			    .detail("CommitVersion", commitVersion)
-			    .detail("ParsedMutation", mutation.toString());
-
-			auto it = kvOps.insert(std::make_pair(LogMessageVersion(commitVersion, sub++), MutationsVec()));
-			ASSERT(it.second); // inserted is true
-			ASSERT(sub < std::numeric_limits<int32_t>::max()); // range file mutation uses int32_max as subversion
-			it.first->second.push_back_deep(it.first->second.arena(), mutation);
-
-			// Sampling data similar to how SS sample bytes
-			ByteSampleInfo sampleInfo = isKeyValueInSample(KeyValueRef(mutation.param1, mutation.param2));
-			if (sampleInfo.inSample) {
-				cc->sampledLogBytes += sampleInfo.sampledSize;
-				samples.push_back_deep(samples.arena(), SampledMutation(mutation.param1, sampleInfo.sampledSize));
-			}
-			ASSERT_WE_THINK(kLen >= 0 && kLen < val.size());
-			ASSERT_WE_THINK(vLen >= 0 && vLen < val.size());
 		}
+		mutationMapIterator++;
 	}
+	return Void();
 }
 
 // Parsing the data blocks in a range file
@@ -1195,7 +1237,8 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
     LoaderCounters* cc,
     Reference<IBackupContainer> bc,
     Version version,
-    RestoreAsset asset) {
+    RestoreAsset asset,
+    Optional<Database> cx) {
 	state VersionedMutationsMap& kvOps = kvOpsIter->second;
 	state SampledMutationsVec& sampleMutations = samplesIter->second;
 
@@ -1218,7 +1261,7 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 			// version
 			Reference<IAsyncFile> inFile = wait(bc->readFile(asset.filename));
 			Standalone<VectorRef<KeyValueRef>> kvs =
-			    wait(fileBackup::decodeRangeFileBlock(inFile, asset.offset, asset.len));
+			    wait(fileBackup::decodeRangeFileBlock(inFile, asset.offset, asset.len, cx));
 			TraceEvent("FastRestoreLoaderDecodedRangeFile")
 			    .detail("BatchIndex", asset.batchIndex)
 			    .detail("Filename", asset.filename)
@@ -1466,7 +1509,7 @@ void oldSplitMutation(std::map<Key, UID>* pRangeToApplier,
 		itApplier = itlow;
 		itlow++;
 		if (itlow == itup) {
-			ASSERT(m.param2 <= normalKeys.end);
+			ASSERT(m.param2 <= allKeys.end);
 			curm.param2 = m.param2;
 		} else if (m.param2 < itlow->first) {
 			UNREACHABLE();
@@ -1491,7 +1534,7 @@ TEST_CASE("/FastRestore/RestoreLoader/splitMutation") {
 	Standalone<VectorRef<UID>> nodeIDs;
 
 	// Prepare RangeToApplier
-	rangeToApplier.emplace(normalKeys.begin, deterministicRandom()->randomUniqueID());
+	rangeToApplier.emplace(allKeys.begin, deterministicRandom()->randomUniqueID());
 	int numAppliers = deterministicRandom()->randomInt(1, 50);
 	for (int i = 0; i < numAppliers; ++i) {
 		Key k = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(1, 1000)));
diff --git a/fdbserver/RestoreUtil.actor.cpp b/fdbserver/RestoreUtil.actor.cpp
index e25b665fd2..622fe98b19 100644
--- a/fdbserver/RestoreUtil.actor.cpp
+++ b/fdbserver/RestoreUtil.actor.cpp
@@ -29,7 +29,7 @@ int numRoles = RestoreRoleStr.size();
 
 // Similar to debugMutation(), we use debugFRMutation to track mutations for fast restore systems only.
 #if CENABLED(0, NOT_IN_CLEAN)
-StringRef debugFRKey = LiteralStringRef("\xff\xff\xff\xff");
+StringRef debugFRKey = "\xff\xff\xff\xff"_sr;
 
 // Track any mutation in fast restore that has overlap with debugFRKey
 bool debugFRMutation(const char* context, Version version, MutationRef const& mutation) {
diff --git a/fdbserver/RestoreWorker.actor.cpp b/fdbserver/RestoreWorker.actor.cpp
index 38200a3ca9..db724ee2b9 100644
--- a/fdbserver/RestoreWorker.actor.cpp
+++ b/fdbserver/RestoreWorker.actor.cpp
@@ -31,6 +31,7 @@
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/MutationList.h"
 #include "fdbclient/BackupContainer.h"
+#include "flow/ApiVersion.h"
 #include "flow/IAsyncFile.h"
 #include "fdbrpc/simulator.h"
 #include "flow/genericactors.actor.h"
@@ -365,13 +366,13 @@ ACTOR Future<Void> _restoreWorker(Database cx, LocalityData locality) {
 	// Protect restore worker from being killed in simulation;
 	// Future: Remove the protection once restore can tolerate failure
 	if (g_network->isSimulated()) {
-		auto addresses = g_simulator.getProcessByAddress(myWorkerInterf.address())->addresses;
+		auto addresses = g_simulator->getProcessByAddress(myWorkerInterf.address())->addresses;
 
-		g_simulator.protectedAddresses.insert(addresses.address);
+		g_simulator->protectedAddresses.insert(addresses.address);
 		if (addresses.secondaryAddress.present()) {
-			g_simulator.protectedAddresses.insert(addresses.secondaryAddress.get());
+			g_simulator->protectedAddresses.insert(addresses.secondaryAddress.get());
 		}
-		ISimulator::ProcessInfo* p = g_simulator.getProcessByAddress(myWorkerInterf.address());
+		ISimulator::ProcessInfo* p = g_simulator->getProcessByAddress(myWorkerInterf.address());
 		TraceEvent("ProtectRestoreWorker")
 		    .detail("Address", addresses.toString())
 		    .detail("IsReliable", p->isReliable())
@@ -410,7 +411,7 @@ ACTOR Future<Void> restoreWorker(Reference<IClusterConnectionRecord> connRecord,
                                  LocalityData locality,
                                  std::string coordFolder) {
 	try {
-		Database cx = Database::createDatabase(connRecord, Database::API_VERSION_LATEST, IsInternal::True, locality);
+		Database cx = Database::createDatabase(connRecord, ApiVersion::LATEST_VERSION, IsInternal::True, locality);
 		wait(reportErrors(_restoreWorker(cx, locality), "RestoreWorker"));
 	} catch (Error& e) {
 		TraceEvent("FastRestoreWorker").detail("Error", e.what());
diff --git a/fdbserver/RkTagThrottleCollection.cpp b/fdbserver/RkTagThrottleCollection.cpp
index d0e8cb9892..1c543f9908 100644
--- a/fdbserver/RkTagThrottleCollection.cpp
+++ b/fdbserver/RkTagThrottleCollection.cpp
@@ -48,7 +48,7 @@ Optional<double> RkTagThrottleCollection::RkTagThrottleData::updateAndGetClientR
 		ASSERT_GE(rate, 0);
 		return rate;
 	} else {
-		TEST(true); // Get throttle rate for expired throttle
+		CODE_PROBE(true, "Get throttle rate for expired throttle");
 		rateSet = false;
 		return Optional<double>();
 	}
@@ -92,14 +92,14 @@ Optional<double> RkTagThrottleCollection::autoThrottleTag(UID id,
 	bool present = (itr != autoThrottledTags.end());
 	if (!present) {
 		if (autoThrottledTags.size() >= SERVER_KNOBS->MAX_AUTO_THROTTLED_TRANSACTION_TAGS) {
-			TEST(true); // Reached auto-throttle limit
+			CODE_PROBE(true, "Reached auto-throttle limit");
 			return Optional<double>();
 		}
 
 		itr = autoThrottledTags.try_emplace(tag).first;
 		initializeTag(tag);
 	} else if (itr->second.limits.expiration <= now()) {
-		TEST(true); // Re-throttling expired tag that hasn't been cleaned up
+		CODE_PROBE(true, "Re-throttling expired tag that hasn't been cleaned up");
 		present = false;
 		itr->second = RkTagThrottleData();
 	}
@@ -113,7 +113,7 @@ Optional<double> RkTagThrottleCollection::autoThrottleTag(UID id,
 				return Optional<double>();
 			}
 		} else if (now() <= throttle.lastUpdated + SERVER_KNOBS->AUTO_TAG_THROTTLE_UPDATE_FREQUENCY) {
-			TEST(true); // Tag auto-throttled too quickly
+			CODE_PROBE(true, "Tag auto-throttled too quickly");
 			return Optional<double>();
 		} else {
 			tpsRate = computeTargetTpsRate(fractionalBusyness,
@@ -121,7 +121,7 @@ Optional<double> RkTagThrottleCollection::autoThrottleTag(UID id,
 			                               tagData[tag].requestRate.smoothRate());
 
 			if (throttle.limits.expiration > now() && tpsRate.get() >= throttle.limits.tpsRate) {
-				TEST(true); // Tag auto-throttle rate increase attempt while active
+				CODE_PROBE(true, "Tag auto-throttle rate increase attempt while active");
 				return Optional<double>();
 			}
 
@@ -176,14 +176,14 @@ void RkTagThrottleCollection::manualThrottleTag(UID id,
 	result.first->second.limits.expiration = expiration;
 
 	if (!oldLimits.present()) {
-		TEST(true); // Transaction tag manually throttled
+		CODE_PROBE(true, "Transaction tag manually throttled");
 		TraceEvent("RatekeeperAddingManualThrottle", id)
 		    .detail("Tag", tag)
 		    .detail("Rate", tpsRate)
 		    .detail("Priority", transactionPriorityToString(priority))
 		    .detail("SecondsToExpiration", expiration - now());
 	} else if (oldLimits.get().tpsRate != tpsRate || oldLimits.get().expiration != expiration) {
-		TEST(true); // Manual transaction tag throttle updated
+		CODE_PROBE(true, "Manual transaction tag throttle updated");
 		TraceEvent("RatekeeperUpdatingManualThrottle", id)
 		    .detail("Tag", tag)
 		    .detail("Rate", tpsRate)
@@ -225,14 +225,14 @@ PrioritizedTransactionTagMap<ClientTagThrottleLimits> RkTagThrottleCollection::g
 				if (priorityItr != manualItr->second.end()) {
 					Optional<double> priorityClientRate = priorityItr->second.updateAndGetClientRate(requestRate);
 					if (!priorityClientRate.present()) {
-						TEST(true); // Manual priority throttle expired
+						CODE_PROBE(true, "Manual priority throttle expired");
 						priorityItr = manualItr->second.erase(priorityItr);
 					} else {
 						if (!manualClientRate.present() || manualClientRate.get().tpsRate > priorityClientRate.get()) {
 							manualClientRate = ClientTagThrottleLimits(priorityClientRate.get(),
 							                                           priorityItr->second.limits.expiration);
 						} else {
-							TEST(true); // Manual throttle overriden by higher priority
+							CODE_PROBE(true, "Manual throttle overriden by higher priority");
 						}
 
 						++priorityItr;
@@ -241,13 +241,13 @@ PrioritizedTransactionTagMap<ClientTagThrottleLimits> RkTagThrottleCollection::g
 
 				if (manualClientRate.present()) {
 					tagPresent = true;
-					TEST(true); // Using manual throttle
+					CODE_PROBE(true, "Using manual throttle");
 					clientRates[*priority][tagItr->first] = manualClientRate.get();
 				}
 			}
 
 			if (manualItr->second.empty()) {
-				TEST(true); // All manual throttles expired
+				CODE_PROBE(true, "All manual throttles expired");
 				manualThrottledTags.erase(manualItr);
 				break;
 			}
@@ -261,7 +261,7 @@ PrioritizedTransactionTagMap<ClientTagThrottleLimits> RkTagThrottleCollection::g
 				double rampStartTime = autoItr->second.lastReduced + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION -
 				                       SERVER_KNOBS->AUTO_TAG_THROTTLE_RAMP_UP_TIME;
 				if (now() >= rampStartTime && adjustedRate != std::numeric_limits<double>::max()) {
-					TEST(true); // Tag auto-throttle ramping up
+					CODE_PROBE(true, "Tag auto-throttle ramping up");
 
 					double targetBusyness = SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS;
 					if (targetBusyness == 0) {
@@ -280,14 +280,14 @@ PrioritizedTransactionTagMap<ClientTagThrottleLimits> RkTagThrottleCollection::g
 					if (!result.second && result.first->second.tpsRate > adjustedRate) {
 						result.first->second = ClientTagThrottleLimits(adjustedRate, autoItr->second.limits.expiration);
 					} else {
-						TEST(true); // Auto throttle overriden by manual throttle
+						CODE_PROBE(true, "Auto throttle overriden by manual throttle");
 					}
 					clientRates[TransactionPriority::BATCH][tagItr->first] =
 					    ClientTagThrottleLimits(0, autoItr->second.limits.expiration);
 				}
 			} else {
 				ASSERT(autoItr->second.limits.expiration <= now());
-				TEST(true); // Auto throttle expired
+				CODE_PROBE(true, "Auto throttle expired");
 				if (BUGGIFY) { // Temporarily extend the window between expiration and cleanup
 					tagPresent = true;
 				} else {
@@ -297,7 +297,7 @@ PrioritizedTransactionTagMap<ClientTagThrottleLimits> RkTagThrottleCollection::g
 		}
 
 		if (!tagPresent) {
-			TEST(true); // All tag throttles expired
+			CODE_PROBE(true, "All tag throttles expired");
 			tagItr = tagData.erase(tagItr);
 		} else {
 			++tagItr;
@@ -309,7 +309,7 @@ PrioritizedTransactionTagMap<ClientTagThrottleLimits> RkTagThrottleCollection::g
 
 void RkTagThrottleCollection::addRequests(TransactionTag const& tag, int requests) {
 	if (requests > 0) {
-		TEST(true); // Requests reported for throttled tag
+		CODE_PROBE(true, "Requests reported for throttled tag");
 
 		auto tagItr = tagData.try_emplace(tag);
 		tagItr.first->second.requestRate.addDelta(requests);
@@ -353,6 +353,8 @@ void RkTagThrottleCollection::incrementBusyTagCount(TagThrottledReason reason) {
 	} else if (reason == TagThrottledReason::BUSY_WRITE) {
 		++busyWriteTagCount;
 	} else {
-		ASSERT(false);
+		ASSERT(reason == TagThrottledReason::UNSET);
+		// Tag throttled reason is unset, probably because of upgrading
+		TraceEvent(SevWarn, "UnsetTagThrottledReason");
 	}
 }
diff --git a/fdbserver/RocksDBCheckpointUtils.actor.cpp b/fdbserver/RocksDBCheckpointUtils.actor.cpp
index 9edc6923fd..a60cf5e691 100644
--- a/fdbserver/RocksDBCheckpointUtils.actor.cpp
+++ b/fdbserver/RocksDBCheckpointUtils.actor.cpp
@@ -109,9 +109,9 @@ public:
 
 	Future<RangeResult> nextKeyValues(const int rowLimit, const int byteLimit) override;
 
-	Future<Standalone<StringRef>> nextChunk(const int byteLimit) { throw not_implemented(); }
+	Future<Standalone<StringRef>> nextChunk(const int byteLimit) override { throw not_implemented(); }
 
-	Future<Void> close() { return doClose(this); }
+	Future<Void> close() override { return doClose(this); }
 
 private:
 	struct Reader : IThreadPoolReceiver {
@@ -162,6 +162,7 @@ private:
 		CF cf;
 		Key begin;
 		Key end;
+		std::vector<rocksdb::ColumnFamilyHandle*> handles;
 		double readRangeTimeout;
 		std::unique_ptr<rocksdb::Iterator> cursor;
 	};
@@ -233,7 +234,6 @@ void RocksDBCheckpointReader::Reader::action(RocksDBCheckpointReader::Reader::Op
 		descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, cfOptions });
 	}
 
-	std::vector<rocksdb::ColumnFamilyHandle*> handles;
 	status = rocksdb::DB::OpenForReadOnly(options, a.path, descriptors, &handles, &db);
 
 	if (!status.ok()) {
@@ -288,6 +288,14 @@ void RocksDBCheckpointReader::Reader::action(RocksDBCheckpointReader::Reader::Cl
 		return;
 	}
 
+	for (rocksdb::ColumnFamilyHandle* handle : handles) {
+		if (handle != nullptr) {
+			TraceEvent("RocksDBCheckpointReaderDestroyCF").detail("Path", a.path).detail("CF", handle->GetName());
+			db->DestroyColumnFamilyHandle(handle);
+		}
+	}
+	handles.clear();
+
 	rocksdb::Status s = db->Close();
 	if (!s.ok()) {
 		logRocksDBError(s, "Close");
@@ -385,6 +393,9 @@ ACTOR Future<Void> RocksDBCheckpointReader::doClose(RocksDBCheckpointReader* sel
 	}
 
 	if (self != nullptr) {
+		if (self->db != nullptr) {
+			delete self->db;
+		}
 		delete self;
 	}
 
@@ -515,8 +526,11 @@ ACTOR Future<Void> fetchCheckpointFile(Database cx,
 	}
 
 	state int attempt = 0;
+	state int64_t offset = 0;
+	state Reference<IAsyncFile> asyncFile;
 	loop {
 		try {
+			asyncFile = Reference<IAsyncFile>();
 			++attempt;
 			TraceEvent("FetchCheckpointFileBegin")
 			    .detail("RemoteFile", remoteFile)
@@ -528,8 +542,7 @@ ACTOR Future<Void> fetchCheckpointFile(Database cx,
 			wait(IAsyncFileSystem::filesystem()->deleteFile(localFile, true));
 			const int64_t flags = IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE |
 			                      IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO;
-			state int64_t offset = 0;
-			state Reference<IAsyncFile> asyncFile = wait(IAsyncFileSystem::filesystem()->open(localFile, flags, 0666));
+			asyncFile = wait(IAsyncFileSystem::filesystem()->open(localFile, flags, 0666));
 
 			state ReplyPromiseStream<FetchCheckpointReply> stream =
 			    ssi.fetchCheckpoint.getReplyStream(FetchCheckpointRequest(metaData->checkpointID, remoteFile));
diff --git a/fdbserver/ShardsAffectedByTeamFailure.cpp b/fdbserver/ShardsAffectedByTeamFailure.cpp
new file mode 100644
index 0000000000..d834febaf7
--- /dev/null
+++ b/fdbserver/ShardsAffectedByTeamFailure.cpp
@@ -0,0 +1,238 @@
+/*
+ * ShardsAffectedByTeamFailure.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/ShardsAffectedByTeamFailure.h"
+
+std::vector<KeyRange> ShardsAffectedByTeamFailure::getShardsFor(Team team) const {
+	std::vector<KeyRange> r;
+	for (auto it = team_shards.lower_bound(std::pair<Team, KeyRange>(team, KeyRangeRef()));
+	     it != team_shards.end() && it->first == team;
+	     ++it)
+		r.push_back(it->second);
+	return r;
+}
+
+bool ShardsAffectedByTeamFailure::hasShards(Team team) const {
+	auto it = team_shards.lower_bound(std::pair<Team, KeyRange>(team, KeyRangeRef()));
+	return it != team_shards.end() && it->first == team;
+}
+
+int ShardsAffectedByTeamFailure::getNumberOfShards(UID ssID) const {
+	auto it = storageServerShards.find(ssID);
+	return it == storageServerShards.end() ? 0 : it->second;
+}
+
+std::pair<std::vector<ShardsAffectedByTeamFailure::Team>, std::vector<ShardsAffectedByTeamFailure::Team>>
+ShardsAffectedByTeamFailure::getTeamsFor(KeyRangeRef keys) {
+	return shard_teams[keys.begin];
+}
+
+void ShardsAffectedByTeamFailure::erase(Team team, KeyRange const& range) {
+	DisabledTraceEvent(SevDebug, "ShardsAffectedByTeamFailureErase")
+	    .detail("Range", range)
+	    .detail("Team", team.toString());
+	if (team_shards.erase(std::pair<Team, KeyRange>(team, range)) > 0) {
+		for (auto uid = team.servers.begin(); uid != team.servers.end(); ++uid) {
+			// Safeguard against going negative after eraseServer() sets value to 0
+			if (storageServerShards[*uid] > 0) {
+				storageServerShards[*uid]--;
+			}
+		}
+	}
+}
+
+void ShardsAffectedByTeamFailure::insert(Team team, KeyRange const& range) {
+	DisabledTraceEvent(SevDebug, "ShardsAffectedByTeamFailureInsert")
+	    .detail("Range", range)
+	    .detail("Team", team.toString());
+	if (team_shards.insert(std::pair<Team, KeyRange>(team, range)).second) {
+		for (auto uid = team.servers.begin(); uid != team.servers.end(); ++uid)
+			storageServerShards[*uid]++;
+	}
+}
+
+void ShardsAffectedByTeamFailure::defineShard(KeyRangeRef keys) {
+	std::vector<Team> teams;
+	std::vector<Team> prevTeams;
+	auto rs = shard_teams.intersectingRanges(keys);
+	for (auto it = rs.begin(); it != rs.end(); ++it) {
+		for (auto t = it->value().first.begin(); t != it->value().first.end(); ++t) {
+			teams.push_back(*t);
+			erase(*t, it->range());
+		}
+		for (auto t = it->value().second.begin(); t != it->value().second.end(); ++t) {
+			prevTeams.push_back(*t);
+		}
+	}
+	uniquify(teams);
+	uniquify(prevTeams);
+
+	/*TraceEvent("ShardsAffectedByTeamFailureDefine")
+	    .detail("KeyBegin", keys.begin)
+	    .detail("KeyEnd", keys.end)
+	    .detail("TeamCount", teams.size());*/
+
+	auto affectedRanges = shard_teams.getAffectedRangesAfterInsertion(keys);
+	shard_teams.insert(keys, std::make_pair(teams, prevTeams));
+
+	for (auto r = affectedRanges.begin(); r != affectedRanges.end(); ++r) {
+		auto& t = shard_teams[r->begin];
+		for (auto it = t.first.begin(); it != t.first.end(); ++it) {
+			insert(*it, *r);
+		}
+	}
+	check();
+}
+
+// Move keys to destinationTeams by updating shard_teams
+void ShardsAffectedByTeamFailure::moveShard(KeyRangeRef keys, std::vector<Team> destinationTeams) {
+	/*TraceEvent("ShardsAffectedByTeamFailureMove")
+	    .detail("KeyBegin", keys.begin)
+	    .detail("KeyEnd", keys.end)
+	    .detail("NewTeamSize", destinationTeam.size())
+	    .detail("NewTeam", describe(destinationTeam));*/
+
+	auto ranges = shard_teams.intersectingRanges(keys);
+	std::vector<std::pair<std::pair<std::vector<Team>, std::vector<Team>>, KeyRange>> modifiedShards;
+	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+		if (keys.contains(it->range())) {
+			// erase the many teams that were associated with this one shard
+			for (auto t = it->value().first.begin(); t != it->value().first.end(); ++t) {
+				erase(*t, it->range());
+			}
+
+			// save this modification for later insertion
+			std::vector<Team> prevTeams = it->value().second;
+			prevTeams.insert(prevTeams.end(), it->value().first.begin(), it->value().first.end());
+			uniquify(prevTeams);
+
+			modifiedShards.push_back(std::pair<std::pair<std::vector<Team>, std::vector<Team>>, KeyRange>(
+			    std::make_pair(destinationTeams, prevTeams), it->range()));
+		} else {
+			// for each range that touches this move, add our team as affecting this range
+			for (auto& team : destinationTeams) {
+				insert(team, it->range());
+			}
+
+			// if we are not in the list of teams associated with this shard, add us in
+			auto& teams = it->value();
+			teams.second.insert(teams.second.end(), teams.first.begin(), teams.first.end());
+			uniquify(teams.second);
+
+			teams.first.insert(teams.first.end(), destinationTeams.begin(), destinationTeams.end());
+			uniquify(teams.first);
+		}
+	}
+
+	// we cannot modify the KeyRangeMap while iterating through it, so add saved modifications now
+	for (int i = 0; i < modifiedShards.size(); i++) {
+		for (auto& t : modifiedShards[i].first.first) {
+			insert(t, modifiedShards[i].second);
+		}
+		shard_teams.insert(modifiedShards[i].second, modifiedShards[i].first);
+	}
+
+	check();
+}
+
+void ShardsAffectedByTeamFailure::finishMove(KeyRangeRef keys) {
+	auto ranges = shard_teams.containedRanges(keys);
+	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+		it.value().second.clear();
+	}
+}
+
+void ShardsAffectedByTeamFailure::setCheckMode(CheckMode mode) {
+	checkMode = mode;
+}
+
+void ShardsAffectedByTeamFailure::check() const {
+	if (checkMode == CheckMode::ForceNoCheck)
+		return;
+	if (EXPENSIVE_VALIDATION || checkMode == CheckMode::ForceCheck) {
+		for (auto t = team_shards.begin(); t != team_shards.end(); ++t) {
+			auto i = shard_teams.rangeContaining(t->second.begin);
+			if (i->range() != t->second || !std::count(i->value().first.begin(), i->value().first.end(), t->first)) {
+				ASSERT(false);
+			}
+		}
+		auto rs = shard_teams.ranges();
+		for (auto i = rs.begin(); i != rs.end(); ++i) {
+			for (auto t = i->value().first.begin(); t != i->value().first.end(); ++t) {
+				if (!team_shards.count(std::make_pair(*t, i->range()))) {
+					std::string teamDesc, shards;
+					for (int k = 0; k < t->servers.size(); k++)
+						teamDesc += format("%llx ", t->servers[k].first());
+					for (auto x = team_shards.lower_bound(std::make_pair(*t, KeyRangeRef()));
+					     x != team_shards.end() && x->first == *t;
+					     ++x)
+						shards += printable(x->second.begin) + "-" + printable(x->second.end) + ",";
+					TraceEvent(SevError, "SATFInvariantError2")
+					    .detail("KB", i->begin())
+					    .detail("KE", i->end())
+					    .detail("Team", teamDesc)
+					    .detail("Shards", shards);
+					ASSERT(false);
+				}
+			}
+		}
+	}
+}
+
+size_t ShardsAffectedByTeamFailure::getNumberOfShards() const {
+	return shard_teams.size();
+}
+
+auto ShardsAffectedByTeamFailure::getAllRanges() const -> decltype(shard_teams)::ConstRanges {
+	return shard_teams.ranges();
+}
+
+void ShardsAffectedByTeamFailure::assignRangeToTeams(KeyRangeRef keys, const std::vector<Team>& destinationTeam) {
+	defineShard(keys);
+	moveShard(keys, destinationTeam);
+	finishMove(keys);
+}
+
+bool ShardsAffectedByTeamFailure::removeFailedServerForSingleRange(ShardsAffectedByTeamFailure::Team& team,
+                                                                   const UID& id,
+                                                                   KeyRangeRef keys) {
+	if (team.hasServer(id)) {
+		erase(team, keys);
+		team.removeServer(id);
+		insert(team, keys);
+		return true;
+	}
+	return false;
+}
+
+void ShardsAffectedByTeamFailure::removeFailedServerForRange(KeyRangeRef keys, const UID& serverID) {
+	auto rs = shard_teams.intersectingRanges(keys);
+	for (auto it = rs.begin(); it != rs.end(); ++it) {
+		// first team vector
+		for (std::vector<Team>::iterator t = it->value().first.begin(); t != it->value().first.end(); ++t) {
+			removeFailedServerForSingleRange(*t, serverID, it->range());
+		}
+		// second team vector
+		for (std::vector<Team>::iterator t = it->value().second.begin(); t != it->value().second.end(); ++t) {
+			removeFailedServerForSingleRange(*t, serverID, it->range());
+		}
+	}
+	check();
+}
diff --git a/fdbserver/SimKmsConnector.actor.cpp b/fdbserver/SimKmsConnector.actor.cpp
index 91d843345a..da903870bd 100644
--- a/fdbserver/SimKmsConnector.actor.cpp
+++ b/fdbserver/SimKmsConnector.actor.cpp
@@ -22,11 +22,11 @@
 
 #include "fmt/format.h"
 #include "fdbrpc/sim_validation.h"
+#include "fdbclient/BlobCipher.h"
 #include "fdbserver/KmsConnectorInterface.h"
 #include "fdbserver/Knobs.h"
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
-#include "flow/BlobCipher.h"
 #include "flow/EncryptUtils.h"
 #include "flow/Error.h"
 #include "flow/FastRef.h"
@@ -66,28 +66,33 @@ struct SimKmsConnectorContext : NonCopyable, ReferenceCounted<SimKmsConnectorCon
 		// Construct encryption keyStore.
 		// Note the keys generated must be the same after restart.
 		for (int i = 1; i <= maxEncryptionKeys; i++) {
-			Arena arena;
-			StringRef digest = computeAuthToken(
-			    reinterpret_cast<const unsigned char*>(&i), sizeof(i), SHA_KEY, AES_256_KEY_LENGTH, arena);
-			simEncryptKeyStore[i] =
-			    std::make_unique<SimEncryptKeyCtx>(i, reinterpret_cast<const char*>(digest.begin()));
+			uint8_t digest[AUTH_TOKEN_HMAC_SHA_SIZE];
+			computeAuthToken({ { reinterpret_cast<const uint8_t*>(&i), sizeof(i) } },
+			                 SHA_KEY,
+			                 AES_256_KEY_LENGTH,
+			                 &digest[0],
+			                 EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
+			                 AUTH_TOKEN_HMAC_SHA_SIZE);
+			simEncryptKeyStore[i] = std::make_unique<SimEncryptKeyCtx>(i, reinterpret_cast<const char*>(&digest[0]));
 		}
 	}
 };
 
 namespace {
-Optional<int64_t> getRefreshInterval(int64_t now, int64_t defaultTtl) {
+Optional<int64_t> getRefreshInterval(const int64_t now, const int64_t defaultTtl) {
 	if (BUGGIFY) {
-		return Optional<int64_t>(now + defaultTtl);
+		return Optional<int64_t>(now);
 	}
-	return Optional<int64_t>();
+	return Optional<int64_t>(now + defaultTtl);
 }
 
-Optional<int64_t> getExpireInterval(Optional<int64_t> refTS) {
+Optional<int64_t> getExpireInterval(Optional<int64_t> refTS, const int64_t defaultTtl) {
+	ASSERT(refTS.present());
+
 	if (BUGGIFY) {
 		return Optional<int64_t>(-1);
 	}
-	return refTS;
+	return (refTS.get() + defaultTtl);
 }
 } // namespace
 
@@ -105,11 +110,17 @@ ACTOR Future<Void> ekLookupByIds(Reference<SimKmsConnectorContext> ctx,
 	}
 
 	// Lookup corresponding EncryptKeyCtx for input keyId
+	const int64_t currTS = (int64_t)now();
+	// Fetch default TTL to avoid BUGGIFY giving different value per invocation causing refTS > expTS
+	const int64_t defaultTtl = FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL;
+	Optional<int64_t> refAtTS = getRefreshInterval(currTS, defaultTtl);
+	Optional<int64_t> expAtTS = getExpireInterval(refAtTS, defaultTtl);
+	TraceEvent("SimKms.EKLookupById").detail("RefreshAt", refAtTS).detail("ExpireAt", expAtTS);
 	for (const auto& item : req.encryptKeyInfos) {
 		const auto& itr = ctx->simEncryptKeyStore.find(item.baseCipherId);
 		if (itr != ctx->simEncryptKeyStore.end()) {
 			rep.cipherKeyDetails.emplace_back_deep(
-			    rep.arena, item.domainId, itr->first, StringRef(itr->second.get()->key));
+			    rep.arena, item.domainId, itr->first, StringRef(itr->second.get()->key), refAtTS, expAtTS);
 
 			if (dbgKIdTrace.present()) {
 				// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
@@ -145,11 +156,12 @@ ACTOR Future<Void> ekLookupByDomainIds(Reference<SimKmsConnectorContext> ctx,
 	// Map encryptionDomainId to corresponding EncryptKeyCtx element using a modulo operation. This
 	// would mean multiple domains gets mapped to the same encryption key which is fine, the
 	// EncryptKeyStore guarantees that keyId -> plaintext encryptKey mapping is idempotent.
-	int64_t currTS = (int64_t)now();
+	const int64_t currTS = (int64_t)now();
 	// Fetch default TTL to avoid BUGGIFY giving different value per invocation causing refTS > expTS
-	int64_t defaultTtl = FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL;
+	const int64_t defaultTtl = FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL;
 	Optional<int64_t> refAtTS = getRefreshInterval(currTS, defaultTtl);
-	Optional<int64_t> expAtTS = getExpireInterval(refAtTS);
+	Optional<int64_t> expAtTS = getExpireInterval(refAtTS, defaultTtl);
+	TraceEvent("SimKms.EKLookupByDomainId").detail("RefreshAt", refAtTS).detail("ExpireAt", expAtTS);
 	for (const auto& info : req.encryptDomainInfos) {
 		EncryptCipherBaseKeyId keyId = 1 + abs(info.domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS;
 		const auto& itr = ctx->simEncryptKeyStore.find(keyId);
@@ -286,7 +298,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
 		for (i = 0; i < maxDomainIds; i++) {
 			// domainIdsReq.encryptDomainIds.push_back(i);
 			EncryptCipherDomainId domainId = i;
-			EncryptCipherDomainName domainName = StringRef(domainIdsReq.arena, std::to_string(domainId));
+			EncryptCipherDomainNameRef domainName = StringRef(domainIdsReq.arena, std::to_string(domainId));
 			domainIdsReq.encryptDomainInfos.emplace_back(domainIdsReq.arena, i, domainName);
 		}
 		KmsConnLookupEKsByDomainIdsRep domainIdsRep = wait(inf.ekLookupByDomainIds.getReply(domainIdsReq));
diff --git a/fdbserver/SimpleConfigConsumer.actor.cpp b/fdbserver/SimpleConfigConsumer.actor.cpp
index ddad99c80d..7241ffe48d 100644
--- a/fdbserver/SimpleConfigConsumer.actor.cpp
+++ b/fdbserver/SimpleConfigConsumer.actor.cpp
@@ -109,7 +109,7 @@ class SimpleConfigConsumerImpl {
 			} catch (Error& e) {
 				++self->failedChangeRequest;
 				if (e.code() == error_code_version_already_compacted) {
-					TEST(true); // SimpleConfigConsumer get version_already_compacted error
+					CODE_PROBE(true, "SimpleConfigConsumer get version_already_compacted error");
 					wait(getSnapshotAndChanges(self, broadcaster));
 				} else {
 					throw e;
@@ -145,7 +145,8 @@ class SimpleConfigConsumerImpl {
 		                                     reply.changes,
 		                                     committedVersion,
 		                                     reply.annotations,
-		                                     { self->cfi });
+		                                     { self->cfi },
+		                                     committedVersion);
 		return Void();
 	}
 
@@ -186,6 +187,11 @@ SimpleConfigConsumer::SimpleConfigConsumer(ServerCoordinators const& coordinator
                                            Optional<double> compactionInterval)
   : impl(PImpl<SimpleConfigConsumerImpl>::create(coordinators, pollingInterval, compactionInterval)) {}
 
+Future<Void> SimpleConfigConsumer::readSnapshot(ConfigBroadcaster& broadcaster) {
+	ASSERT(false);
+	return Void();
+}
+
 Future<Void> SimpleConfigConsumer::consume(ConfigBroadcaster& broadcaster) {
 	return impl->consume(broadcaster);
 }
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 90b5fea0fe..2380131be8 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -24,6 +24,7 @@
 #include <sstream>
 #include <string_view>
 #include <toml.hpp>
+#include "fdbclient/FDBTypes.h"
 #include "fdbrpc/Locality.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/IPAllowList.h"
@@ -41,12 +42,15 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/versions.h"
+#include "flow/MkCert.h"
 #include "fdbrpc/WellKnownEndpoints.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/network.h"
 #include "flow/TypeTraits.h"
 #include "flow/FaultInjection.h"
+#include "flow/CodeProbeUtils.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/SimulatedCluster.h"
 
 #undef max
 #undef min
@@ -56,6 +60,27 @@ extern const char* getSourceVersion();
 
 using namespace std::literals;
 
+namespace probe {
+
+namespace assert {
+
+struct HasRocksDB {
+	constexpr static AnnotationType type = AnnotationType::Assertion;
+	bool operator()(ICodeProbe const* self) const {
+#ifdef SSD_ROCKSDB_EXPERIMENTAL
+		return true;
+#else
+		return false;
+#endif
+	}
+};
+
+constexpr HasRocksDB hasRocksDB;
+
+} // namespace assert
+
+} // namespace probe
+
 // TODO: Defining these here is just asking for ODR violations.
 template <>
 std::string describe(bool const& val) {
@@ -79,10 +104,10 @@ bool destructed = false;
 
 // Configuration details specified in workload test files that change the simulation
 // environment details
-class TestConfig {
+class TestConfig : public BasicTestConfig {
 	class ConfigBuilder {
 		using value_type = toml::basic_value<toml::discard_comments>;
-		using base_variant = std::variant<int, bool, std::string, std::vector<int>, ConfigDBType>;
+		using base_variant = std::variant<int, float, double, bool, std::string, std::vector<int>, ConfigDBType>;
 		using types =
 		    variant_map<variant_concat<base_variant, variant_map<base_variant, Optional>>, std::add_pointer_t>;
 		std::unordered_map<std::string_view, types> confMap;
@@ -92,6 +117,10 @@ class TestConfig {
 			visitor(const value_type& v) : value(v) {}
 			void operator()(int* val) const { *val = value.as_integer(); }
 			void operator()(Optional<int>* val) const { *val = value.as_integer(); }
+			void operator()(float* val) const { *val = value.as_floating(); }
+			void operator()(Optional<float>* val) const { *val = value.as_floating(); }
+			void operator()(double* val) const { *val = value.as_floating(); }
+			void operator()(Optional<double>* val) const { *val = value.as_floating(); }
 			void operator()(bool* val) const { *val = value.as_boolean(); }
 			void operator()(Optional<bool>* val) const { *val = value.as_boolean(); }
 			void operator()(std::string* val) const { *val = value.as_string(); }
@@ -213,8 +242,12 @@ class TestConfig {
 			std::string attrib = removeWhitespace(line.substr(0, found));
 			std::string value = removeWhitespace(line.substr(found + 1));
 
-			if (attrib == "extraDB") {
-				sscanf(value.c_str(), "%d", &extraDB);
+			if (attrib == "extraDatabaseMode") {
+				extraDatabaseMode = ISimulator::stringToExtraDatabaseMode(value);
+			}
+
+			if (attrib == "extraDatabaseCount") {
+				sscanf(value.c_str(), "%d", &extraDatabaseCount);
 			}
 
 			if (attrib == "minimumReplication") {
@@ -268,7 +301,7 @@ class TestConfig {
 			}
 			if (attrib == "configDBType") {
 				if (value == "random") {
-					configDBType = deterministicRandom()->coinflip() ? ConfigDBType::SIMPLE : ConfigDBType::PAXOS;
+					configDBType = deterministicRandom()->random01() < 0.1 ? ConfigDBType::SIMPLE : ConfigDBType::PAXOS;
 				} else {
 					configDBType = configDBTypeFromString(value);
 				}
@@ -279,6 +312,13 @@ class TestConfig {
 			if (attrib == "blobGranulesEnabled") {
 				blobGranulesEnabled = strcmp(value.c_str(), "true") == 0;
 			}
+			if (attrib == "injectSSTargetedRestart") {
+				injectTargetedSSRestart = strcmp(value.c_str(), "true") == 0;
+			}
+
+			if (attrib == "injectSSDelay") {
+				injectSSDelay = strcmp(value.c_str(), "true") == 0;
+			}
 		}
 
 		ifs.close();
@@ -288,11 +328,13 @@ class TestConfig {
 
 public:
 	int extraDB = 0;
+	ISimulator::ExtraDatabaseMode extraDatabaseMode = ISimulator::ExtraDatabaseMode::Disabled;
+	// The number of extra database used if the database mode is MULTIPLE
+	int extraDatabaseCount = 1;
 	int minimumReplication = 0;
 	int minimumRegions = 0;
 	bool configureLocked = false;
 	bool startIncompatibleProcess = false;
-	int logAntiQuorum = -1;
 	bool isFirstTestInRestart = false;
 	// 7.0 cannot be downgraded to 6.3 after enabling TSS, so disable TSS for 6.3 downgrade tests
 	bool disableTss = false;
@@ -311,21 +353,24 @@ public:
 	//	5 = "ssd-sharded-rocksdb"
 	// Requires a comma-separated list of numbers WITHOUT whitespaces
 	std::vector<int> storageEngineExcludeTypes;
+	Optional<int> datacenters, stderrSeverity, processesPerMachine;
 	// Set the maximum TLog version that can be selected for a test
 	// Refer to FDBTypes.h::TLogVersion. Defaults to the maximum supported version.
 	int maxTLogVersion = TLogVersion::MAX_SUPPORTED;
-	// Set true to simplify simulation configs for easier debugging
-	bool simpleConfig = false;
 	int extraMachineCountDC = 0;
+
 	Optional<bool> generateFearless, buggify;
-	Optional<int> datacenters, desiredTLogCount, commitProxyCount, grvProxyCount, resolverCount, storageEngineType,
-	    stderrSeverity, machineCount, processesPerMachine, coordinators;
-	bool blobGranulesEnabled = false;
 	Optional<std::string> config;
+	bool blobGranulesEnabled = false;
 	bool randomlyRenameZoneId = false;
 
 	bool allowDefaultTenant = true;
 	bool allowDisablingTenants = true;
+	bool allowCreatingTenants = true;
+	bool injectTargetedSSRestart = false;
+	bool injectSSDelay = false;
+	std::string testClass; // unused -- used in TestHarness
+	float testPriority; // unused -- used in TestHarness
 
 	ConfigDBType getConfigDBType() const { return configDBType; }
 
@@ -351,8 +396,12 @@ public:
 			loadIniFile(testFile);
 			return;
 		}
+		std::string extraDatabaseModeStr;
 		ConfigBuilder builder;
-		builder.add("extraDB", &extraDB)
+		builder.add("testClass", &testClass)
+		    .add("testPriority", &testPriority)
+		    .add("extraDatabaseMode", &extraDatabaseModeStr)
+		    .add("extraDatabaseCount", &extraDatabaseCount)
 		    .add("minimumReplication", &minimumReplication)
 		    .add("minimumRegions", &minimumRegions)
 		    .add("configureLocked", &configureLocked)
@@ -383,7 +432,10 @@ public:
 		    .add("blobGranulesEnabled", &blobGranulesEnabled)
 		    .add("allowDefaultTenant", &allowDefaultTenant)
 		    .add("allowDisablingTenants", &allowDisablingTenants)
-		    .add("randomlyRenameZoneId", &randomlyRenameZoneId);
+		    .add("allowCreatingTenants", &allowCreatingTenants)
+		    .add("randomlyRenameZoneId", &randomlyRenameZoneId)
+		    .add("injectTargetedSSRestart", &injectTargetedSSRestart)
+		    .add("injectSSDelay", &injectSSDelay);
 		try {
 			auto file = toml::parse(testFile);
 			if (file.contains("configuration") && toml::find(file, "configuration").is_table()) {
@@ -405,6 +457,9 @@ public:
 			if (!isFirstTestInRestart) {
 				isFirstTestInRestart = tomlKeyPresent(file, "restartInfoLocation");
 			}
+			if (!extraDatabaseModeStr.empty()) {
+				extraDatabaseMode = ISimulator::stringToExtraDatabaseMode(extraDatabaseModeStr);
+			}
 		} catch (std::exception& e) {
 			std::cerr << e.what() << std::endl;
 			TraceEvent("TOMLParseError").detail("Error", printable(e.what()));
@@ -418,6 +473,9 @@ public:
 			}
 		}
 	}
+
+	TestConfig() = default;
+	TestConfig(const BasicTestConfig& config) : BasicTestConfig(config) {}
 };
 
 template <class T>
@@ -433,18 +491,18 @@ T simulate(const T& in) {
 ACTOR Future<Void> runBackup(Reference<IClusterConnectionRecord> connRecord) {
 	state std::vector<Future<Void>> agentFutures;
 
-	while (g_simulator.backupAgents == ISimulator::BackupAgentType::WaitForType) {
+	while (g_simulator->backupAgents == ISimulator::BackupAgentType::WaitForType) {
 		wait(delay(1.0));
 	}
 
-	if (g_simulator.backupAgents == ISimulator::BackupAgentType::BackupToFile) {
-		Database cx = Database::createDatabase(connRecord, -1);
+	if (g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) {
+		Database cx = Database::createDatabase(connRecord, ApiVersion::LATEST_VERSION);
 
 		state FileBackupAgent fileAgent;
 		agentFutures.push_back(fileAgent.run(
 		    cx, 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
 
-		while (g_simulator.backupAgents == ISimulator::BackupAgentType::BackupToFile) {
+		while (g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) {
 			wait(delay(1.0));
 		}
 
@@ -460,29 +518,29 @@ ACTOR Future<Void> runBackup(Reference<IClusterConnectionRecord> connRecord) {
 ACTOR Future<Void> runDr(Reference<IClusterConnectionRecord> connRecord) {
 	state std::vector<Future<Void>> agentFutures;
 
-	while (g_simulator.drAgents == ISimulator::BackupAgentType::WaitForType) {
+	while (g_simulator->drAgents == ISimulator::BackupAgentType::WaitForType) {
 		wait(delay(1.0));
 	}
 
-	if (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) {
-		Database cx = Database::createDatabase(connRecord, -1);
+	if (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
+		ASSERT(g_simulator->extraDatabases.size() == 1);
+		Database cx = Database::createDatabase(connRecord, ApiVersion::LATEST_VERSION);
 
-		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-		state Database extraDB = Database::createDatabase(extraFile, -1);
+		state Database drDatabase = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
 
 		TraceEvent("StartingDrAgents")
 		    .detail("ConnectionString", connRecord->getConnectionString().toString())
-		    .detail("ExtraString", extraFile->getConnectionString().toString());
+		    .detail("ExtraString", g_simulator->extraDatabases[0]);
 
 		state DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx);
-		state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(extraDB);
+		state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(drDatabase);
 
 		auto drPollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
 
 		agentFutures.push_back(extraAgent.run(cx, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
-		agentFutures.push_back(dbAgent.run(extraDB, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
+		agentFutures.push_back(dbAgent.run(drDatabase, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
 
-		while (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) {
+		while (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
 			wait(delay(1.0));
 		}
 
@@ -518,7 +576,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
                                                          std::string whitelistBinPaths,
                                                          ProtocolVersion protocolVersion,
                                                          ConfigDBType configDBType) {
-	state ISimulator::ProcessInfo* simProcess = g_simulator.getCurrentProcess();
+	state ISimulator::ProcessInfo* simProcess = g_simulator->getCurrentProcess();
 	state UID randomId = nondeterministicRandom()->randomUniqueID();
 	state int cycles = 0;
 	state IPAllowList allowList;
@@ -541,17 +599,17 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 
 		wait(delay(waitTime));
 
-		state ISimulator::ProcessInfo* process = g_simulator.newProcess("Server",
-		                                                                ip,
-		                                                                port,
-		                                                                sslEnabled,
-		                                                                listenPerProcess,
-		                                                                localities,
-		                                                                processClass,
-		                                                                dataFolder->c_str(),
-		                                                                coordFolder->c_str(),
-		                                                                protocolVersion);
-		wait(g_simulator.onProcess(
+		state ISimulator::ProcessInfo* process = g_simulator->newProcess("Server",
+		                                                                 ip,
+		                                                                 port,
+		                                                                 sslEnabled,
+		                                                                 listenPerProcess,
+		                                                                 localities,
+		                                                                 processClass,
+		                                                                 dataFolder->c_str(),
+		                                                                 coordFolder->c_str(),
+		                                                                 protocolVersion);
+		wait(g_simulator->onProcess(
 		    process,
 		    TaskPriority::DefaultYield)); // Now switch execution to the process on which we will run
 		state Future<ISimulator::KillType> onShutdown = process->onShutdown();
@@ -586,6 +644,9 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 				                              1,
 				                              WLTOKEN_RESERVED_COUNT,
 				                              &allowList);
+				for (const auto& p : g_simulator->authKeys) {
+					FlowTransport::transport().addPublicKey(p.first, p.second.toPublic());
+				}
 				Sim2FileSystem::newFileSystem();
 
 				std::vector<Future<Void>> futures;
@@ -634,7 +695,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 				if (e.code() != error_code_actor_cancelled)
 					printf("SimulatedFDBDTerminated: %s\n", e.what());
 				ASSERT(destructed ||
-				       g_simulator.getCurrentProcess() == process); // simulatedFDBD catch called on different process
+				       g_simulator->getCurrentProcess() == process); // simulatedFDBD catch called on different process
 				TraceEvent(e.code() == error_code_actor_cancelled || e.code() == error_code_file_not_found ||
 				                   e.code() == error_code_incompatible_software_version || destructed
 				               ? SevInfo
@@ -662,7 +723,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 			onShutdown = e;
 		}
 
-		ASSERT(destructed || g_simulator.getCurrentProcess() == process);
+		ASSERT(destructed || g_simulator->getCurrentProcess() == process);
 
 		if (!process->shutdownSignal.isSet() && !destructed) {
 			process->rebooting = true;
@@ -675,11 +736,11 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 		    .detail("Excluded", process->excluded)
 		    .detail("Rebooting", process->rebooting)
 		    .detail("ZoneId", localities.zoneId());
-		wait(g_simulator.onProcess(simProcess));
+		wait(g_simulator->onProcess(simProcess));
 
 		wait(delay(0.00001 + FLOW_KNOBS->MAX_BUGGIFIED_DELAY)); // One last chance for the process to clean up?
 
-		g_simulator.destroyProcess(
+		g_simulator->destroyProcess(
 		    process); // Leak memory here; the process may be used in other parts of the simulation
 
 		auto shutdownResult = onShutdown.get();
@@ -802,7 +863,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 				const int listenPort = i * listenPerProcess + 1;
 				AgentMode agentMode =
 				    runBackupAgents == AgentOnly ? (i == ips.size() - 1 ? AgentOnly : AgentNone) : runBackupAgents;
-				if (g_simulator.hasDiffProtocolProcess && !g_simulator.setDiffProtocol && agentMode == AgentNone) {
+				if (g_simulator->hasDiffProtocolProcess && !g_simulator->setDiffProtocol && agentMode == AgentNone) {
 					processes.push_back(simulatedFDBDRebooter(clusterFile,
 					                                          ips[i],
 					                                          sslEnabled,
@@ -819,7 +880,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 					                                          whitelistBinPaths,
 					                                          protocolVersion,
 					                                          configDBType));
-					g_simulator.setDiffProtocol = true;
+					g_simulator->setDiffProtocol = true;
 				} else {
 					processes.push_back(simulatedFDBDRebooter(clusterFile,
 					                                          ips[i],
@@ -845,9 +906,9 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 				    .detail("Folder", myFolders[i]);
 			}
 
-			TEST(bootCount >= 1); // Simulated machine rebooted
-			TEST(bootCount >= 2); // Simulated machine rebooted twice
-			TEST(bootCount >= 3); // Simulated machine rebooted three times
+			CODE_PROBE(bootCount >= 1, "Simulated machine rebooted");
+			CODE_PROBE(bootCount >= 2, "Simulated machine rebooted twice");
+			CODE_PROBE(bootCount >= 3, "Simulated machine rebooted three times");
 			++bootCount;
 
 			TraceEvent("SimulatedMachineStart", randomId)
@@ -875,7 +936,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 
 			{
 				// Kill all open files, which may cause them to write invalid data.
-				auto& machineCache = g_simulator.getMachineById(localities.machineId())->openFiles;
+				auto& machineCache = g_simulator->getMachineById(localities.machineId())->openFiles;
 
 				// Copy the file pointers to a vector because the map may be modified while we are killing files
 				std::vector<AsyncFileNonDurable*> files;
@@ -893,14 +954,14 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 
 			state std::set<std::string> filenames;
 			state std::string closingStr;
-			auto& machineCache = g_simulator.getMachineById(localities.machineId())->openFiles;
+			auto& machineCache = g_simulator->getMachineById(localities.machineId())->openFiles;
 			for (auto it : machineCache) {
 				filenames.insert(it.first);
 				closingStr += it.first + ", ";
 				ASSERT(it.second.get().canGet());
 			}
 
-			for (auto it : g_simulator.getMachineById(localities.machineId())->deletingOrClosingFiles) {
+			for (auto it : g_simulator->getMachineById(localities.machineId())->deletingOrClosingFiles) {
 				filenames.insert(it);
 				closingStr += it + ", ";
 			}
@@ -913,9 +974,9 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			    .detail("ZoneId", localities.zoneId())
 			    .detail("DataHall", localities.dataHallId());
 
-			ISimulator::MachineInfo* machine = g_simulator.getMachineById(localities.machineId());
+			ISimulator::MachineInfo* machine = g_simulator->getMachineById(localities.machineId());
 			machine->closingFiles = filenames;
-			g_simulator.getMachineById(localities.machineId())->openFiles.clear();
+			g_simulator->getMachineById(localities.machineId())->openFiles.clear();
 
 			// During a reboot:
 			//   The process is expected to close all files and be inactive in zero time, but not necessarily
@@ -925,7 +986,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			state int shutdownDelayCount = 0;
 			state double backoff = 0;
 			loop {
-				auto& machineCache = g_simulator.getMachineById(localities.machineId())->closingFiles;
+				auto& machineCache = g_simulator->getMachineById(localities.machineId())->closingFiles;
 
 				if (!machineCache.empty()) {
 					std::string openFiles;
@@ -957,7 +1018,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			    .detail("ZoneId", localities.zoneId())
 			    .detail("DataHall", localities.dataHallId());
 
-			g_simulator.destroyMachine(localities.machineId());
+			g_simulator->destroyMachine(localities.machineId());
 
 			// SOMEDAY: when processes can be rebooted, this check will be needed
 			// ASSERT( this machine is rebooting );
@@ -967,10 +1028,10 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			for (int i = 1; i < ips.size(); i++)
 				killType = std::max(processes[i].get(), killType);
 
-			TEST(true); // Simulated machine has been rebooted
+			CODE_PROBE(true, "Simulated machine has been rebooted");
 
 			state bool swap = killType == ISimulator::Reboot && BUGGIFY_WITH_PROB(0.75) &&
-			                  g_simulator.canSwapToMachine(localities.zoneId());
+			                  g_simulator->canSwapToMachine(localities.zoneId());
 			if (swap)
 				availableFolders[localities.dcId()].push_back(myFolders);
 
@@ -995,7 +1056,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 				avail.pop_back();
 
 				if (myFolders != toRebootFrom) {
-					TEST(true); // Simulated machine swapped data folders
+					CODE_PROBE(true, "Simulated machine swapped data folders");
 					TraceEvent("SimulatedMachineFolderSwap", randomId)
 					    .detail("OldFolder0", myFolders[0])
 					    .detail("NewFolder0", toRebootFrom[0])
@@ -1020,13 +1081,13 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 					}
 				}
 
-				TEST(true); // Simulated machine rebooted with data loss
+				CODE_PROBE(true, "Simulated machine rebooted with data loss");
 			}
 
 			// this machine is rebooting = false;
 		}
 	} catch (Error& e) {
-		g_simulator.getMachineById(localities.machineId())->openFiles.clear();
+		g_simulator->getMachineById(localities.machineId())->openFiles.clear();
 		throw;
 	}
 }
@@ -1067,7 +1128,7 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 	// Randomly change data center id names to test that localities
 	// can be modified on cluster restart
 	bool renameZoneIds = testConfig.randomlyRenameZoneId ? deterministicRandom()->random01() < 0.1 : false;
-	TEST(renameZoneIds); // Zone ID names altered in restart test
+	CODE_PROBE(renameZoneIds, "Zone ID names altered in restart test");
 
 	// allows multiple ipAddr entries
 	ini.SetMultiKey();
@@ -1084,12 +1145,12 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 		int testerCount = atoi(ini.GetValue("META", "testerCount"));
 		auto tssModeStr = ini.GetValue("META", "tssMode");
 		if (tssModeStr != nullptr) {
-			g_simulator.tssMode = (ISimulator::TSSMode)atoi(tssModeStr);
+			g_simulator->tssMode = (ISimulator::TSSMode)atoi(tssModeStr);
 		}
-		bool enableExtraDB = (testConfig.extraDB == 3);
 		ClusterConnectionString conn(ini.GetValue("META", "connectionString"));
-		if (enableExtraDB) {
-			g_simulator.extraDB = new ClusterConnectionString(ini.GetValue("META", "connectionString"));
+		if (testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Local) {
+			g_simulator->extraDatabases.clear();
+			g_simulator->extraDatabases.push_back(conn.toString());
 		}
 		if (!testConfig.disableHostname) {
 			auto mockDNSStr = ini.GetValue("META", "mockDNS");
@@ -1105,6 +1166,8 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 		if (testConfig.disableEncryption) {
 			g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false }));
 			g_knobs.setKnob("enable_tlog_encryption", KnobValueRef::create(bool{ false }));
+			g_knobs.setKnob("enable_storage_server_encryption", KnobValueRef::create(bool{ false }));
+			g_knobs.setKnob("enable_blob_granule_encryption", KnobValueRef::create(bool{ false }));
 			TraceEvent(SevDebug, "DisableEncryption");
 		}
 		*pConnString = conn;
@@ -1206,8 +1269,8 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 			    processClass == ProcessClass::TesterClass ? "SimulatedTesterMachine" : "SimulatedMachine"));
 		}
 
-		g_simulator.desiredCoordinators = desiredCoordinators;
-		g_simulator.processesPerMachine = processesPerMachine;
+		g_simulator->desiredCoordinators = desiredCoordinators;
+		g_simulator->processesPerMachine = processesPerMachine;
 
 		uniquify(dcIds);
 		if (!BUGGIFY && dcIds.size() == 2 && dcIds[0] != "" && dcIds[1] != "") {
@@ -1237,11 +1300,11 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 			    json_spirit::write_string(json_spirit::mValue(regionArr), json_spirit::Output_options::none);
 		}
 
-		g_simulator.restarted = true;
+		g_simulator->restarted = true;
 
 		TraceEvent("RestartSimulatorSettings")
-		    .detail("DesiredCoordinators", g_simulator.desiredCoordinators)
-		    .detail("ProcessesPerMachine", g_simulator.processesPerMachine)
+		    .detail("DesiredCoordinators", g_simulator->desiredCoordinators)
+		    .detail("ProcessesPerMachine", g_simulator->processesPerMachine)
 		    .detail("ListenersPerProcess", listenersPerProcess);
 	} catch (Error& e) {
 		TraceEvent(SevError, "RestartSimulationError").error(e);
@@ -1255,7 +1318,8 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 // Configuration details compiled in a structure used when setting up a simulated cluster
 struct SimulationConfig {
 	explicit SimulationConfig(const TestConfig& testConfig);
-	int extraDB;
+	ISimulator::ExtraDatabaseMode extraDatabaseMode;
+	int extraDatabaseCount;
 	bool generateFearless;
 
 	DatabaseConfiguration db;
@@ -1284,7 +1348,8 @@ private:
 	void generateNormalConfig(const TestConfig& testConfig);
 };
 
-SimulationConfig::SimulationConfig(const TestConfig& testConfig) : extraDB(testConfig.extraDB) {
+SimulationConfig::SimulationConfig(const TestConfig& testConfig)
+  : extraDatabaseMode(testConfig.extraDatabaseMode), extraDatabaseCount(testConfig.extraDatabaseCount) {
 	generateNormalConfig(testConfig);
 }
 
@@ -1356,6 +1421,7 @@ void SimulationConfig::setSpecificConfig(const TestConfig& testConfig) {
 	if (testConfig.resolverCount.present()) {
 		db.resolverCount = testConfig.resolverCount.get();
 	}
+	db.blobGranulesEnabled = testConfig.blobGranulesEnabled;
 }
 
 // Sets generateFearless and number of dataCenters based on testConfig details
@@ -1383,7 +1449,7 @@ void SimulationConfig::setDatacenters(const TestConfig& testConfig) {
 void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
 	// Using [0, 4) to disable the RocksDB storage engine.
 	// TODO: Figure out what is broken with the RocksDB engine in simulation.
-	int storage_engine_type = deterministicRandom()->randomInt(0, 4);
+	int storage_engine_type = deterministicRandom()->randomInt(0, 6);
 	if (testConfig.storageEngineType.present()) {
 		storage_engine_type = testConfig.storageEngineType.get();
 	} else {
@@ -1391,33 +1457,33 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
 		while (std::find(testConfig.storageEngineExcludeTypes.begin(),
 		                 testConfig.storageEngineExcludeTypes.end(),
 		                 storage_engine_type) != testConfig.storageEngineExcludeTypes.end()) {
-			storage_engine_type = deterministicRandom()->randomInt(0, 5);
+			storage_engine_type = deterministicRandom()->randomInt(0, 6);
 		}
 	}
 
 	switch (storage_engine_type) {
 	case 0: {
-		TEST(true); // Simulated cluster using ssd storage engine
+		CODE_PROBE(true, "Simulated cluster using ssd storage engine");
 		set_config("ssd");
 		break;
 	}
 	case 1: {
-		TEST(true); // Simulated cluster using default memory storage engine
+		CODE_PROBE(true, "Simulated cluster using default memory storage engine");
 		set_config("memory");
 		break;
 	}
 	case 2: {
-		TEST(true); // Simulated cluster using radix-tree storage engine
+		CODE_PROBE(true, "Simulated cluster using radix-tree storage engine");
 		set_config("memory-radixtree-beta");
 		break;
 	}
 	case 3: {
-		TEST(true); // Simulated cluster using redwood storage engine
+		CODE_PROBE(true, "Simulated cluster using redwood storage engine");
 		set_config("ssd-redwood-1-experimental");
 		break;
 	}
 	case 4: {
-		TEST(true); // Simulated cluster using RocksDB storage engine
+		CODE_PROBE(true, "Simulated cluster using RocksDB storage engine", probe::assert::hasRocksDB);
 		set_config("ssd-rocksdb-v1");
 		// Tests using the RocksDB engine are necessarily non-deterministic because of RocksDB
 		// background threads.
@@ -1427,7 +1493,7 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
 		break;
 	}
 	case 5: {
-		TEST(true); // Simulated cluster using Sharded RocksDB storage engine
+		CODE_PROBE(true, "Simulated cluster using Sharded RocksDB storage engine", probe::assert::hasRocksDB);
 		set_config("ssd-sharded-rocksdb");
 		// Tests using the RocksDB engine are necessarily non-deterministic because of RocksDB
 		// background threads.
@@ -1453,7 +1519,7 @@ void SimulationConfig::setReplicationType(const TestConfig& testConfig) {
 	} else {
 		switch (replication_type) {
 		case 0: {
-			TEST(true); // Simulated cluster using custom redundancy mode
+			CODE_PROBE(true, "Simulated cluster using custom redundancy mode");
 			int storage_servers = deterministicRandom()->randomInt(1, generateFearless ? 4 : 5);
 			// FIXME: log replicas must be more than storage replicas because otherwise better master exists will not
 			// recognize it needs to change dcs
@@ -1472,21 +1538,21 @@ void SimulationConfig::setReplicationType(const TestConfig& testConfig) {
 			break;
 		}
 		case 1: {
-			TEST(true); // Simulated cluster running in single redundancy mode
+			CODE_PROBE(true, "Simulated cluster running in single redundancy mode");
 			set_config("single");
 			break;
 		}
 		case 2: {
-			TEST(true); // Simulated cluster running in double redundancy mode
+			CODE_PROBE(true, "Simulated cluster running in double redundancy mode");
 			set_config("double");
 			break;
 		}
 		case 3: {
 			if (datacenters <= 2 || generateFearless) {
-				TEST(true); // Simulated cluster running in triple redundancy mode
+				CODE_PROBE(true, "Simulated cluster running in triple redundancy mode");
 				set_config("triple");
 			} else if (datacenters == 3) {
-				TEST(true); // Simulated cluster running in 3 data-hall mode
+				CODE_PROBE(true, "Simulated cluster running in 3 data-hall mode");
 				set_config("three_data_hall");
 			} else {
 				ASSERT(false);
@@ -1537,17 +1603,17 @@ void SimulationConfig::setRegions(const TestConfig& testConfig) {
 			int satellite_replication_type = deterministicRandom()->randomInt(0, 3);
 			switch (satellite_replication_type) {
 			case 0: {
-				TEST(true); // Simulated cluster using no satellite redundancy mode (>4 datacenters)
+				CODE_PROBE(true, "Simulated cluster using no satellite redundancy mode (>4 datacenters)");
 				break;
 			}
 			case 1: {
-				TEST(true); // Simulated cluster using two satellite fast redundancy mode
+				CODE_PROBE(true, "Simulated cluster using two satellite fast redundancy mode");
 				primaryObj["satellite_redundancy_mode"] = "two_satellite_fast";
 				remoteObj["satellite_redundancy_mode"] = "two_satellite_fast";
 				break;
 			}
 			case 2: {
-				TEST(true); // Simulated cluster using two satellite safe redundancy mode
+				CODE_PROBE(true, "Simulated cluster using two satellite safe redundancy mode");
 				primaryObj["satellite_redundancy_mode"] = "two_satellite_safe";
 				remoteObj["satellite_redundancy_mode"] = "two_satellite_safe";
 				break;
@@ -1560,27 +1626,27 @@ void SimulationConfig::setRegions(const TestConfig& testConfig) {
 			switch (satellite_replication_type) {
 			case 0: {
 				// FIXME: implement
-				TEST(true); // Simulated cluster using custom satellite redundancy mode
+				CODE_PROBE(true, "Simulated cluster using custom satellite redundancy mode");
 				break;
 			}
 			case 1: {
-				TEST(true); // Simulated cluster using no satellite redundancy mode (<4 datacenters)
+				CODE_PROBE(true, "Simulated cluster using no satellite redundancy mode (<4 datacenters)");
 				break;
 			}
 			case 2: {
-				TEST(true); // Simulated cluster using single satellite redundancy mode
+				CODE_PROBE(true, "Simulated cluster using single satellite redundancy mode");
 				primaryObj["satellite_redundancy_mode"] = "one_satellite_single";
 				remoteObj["satellite_redundancy_mode"] = "one_satellite_single";
 				break;
 			}
 			case 3: {
-				TEST(true); // Simulated cluster using double satellite redundancy mode
+				CODE_PROBE(true, "Simulated cluster using double satellite redundancy mode");
 				primaryObj["satellite_redundancy_mode"] = "one_satellite_double";
 				remoteObj["satellite_redundancy_mode"] = "one_satellite_double";
 				break;
 			}
 			case 4: {
-				TEST(true); // Simulated cluster using triple satellite redundancy mode
+				CODE_PROBE(true, "Simulated cluster using triple satellite redundancy mode");
 				primaryObj["satellite_redundancy_mode"] = "one_satellite_triple";
 				remoteObj["satellite_redundancy_mode"] = "one_satellite_triple";
 				break;
@@ -1600,10 +1666,10 @@ void SimulationConfig::setRegions(const TestConfig& testConfig) {
 		if (testConfig.minimumRegions <= 1 &&
 		    (deterministicRandom()->random01() < 0.25 ||
 		     SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS < SERVER_KNOBS->VERSIONS_PER_SECOND)) {
-			TEST(true); // Simulated cluster using one region
+			CODE_PROBE(true, "Simulated cluster using one region");
 			needsRemote = false;
 		} else {
-			TEST(true); // Simulated cluster using two regions
+			CODE_PROBE(true, "Simulated cluster using two regions");
 			db.usableRegions = 2;
 		}
 
@@ -1611,25 +1677,25 @@ void SimulationConfig::setRegions(const TestConfig& testConfig) {
 		switch (remote_replication_type) {
 		case 0: {
 			// FIXME: implement
-			TEST(true); // Simulated cluster using custom remote redundancy mode
+			CODE_PROBE(true, "Simulated cluster using custom remote redundancy mode");
 			break;
 		}
 		case 1: {
-			TEST(true); // Simulated cluster using default remote redundancy mode
+			CODE_PROBE(true, "Simulated cluster using default remote redundancy mode");
 			break;
 		}
 		case 2: {
-			TEST(true); // Simulated cluster using single remote redundancy mode
+			CODE_PROBE(true, "Simulated cluster using single remote redundancy mode");
 			set_config("remote_single");
 			break;
 		}
 		case 3: {
-			TEST(true); // Simulated cluster using double remote redundancy mode
+			CODE_PROBE(true, "Simulated cluster using double remote redundancy mode");
 			set_config("remote_double");
 			break;
 		}
 		case 4: {
-			TEST(true); // Simulated cluster using triple remote redundancy mode
+			CODE_PROBE(true, "Simulated cluster using triple remote redundancy mode");
 			set_config("remote_triple");
 			break;
 		}
@@ -1692,18 +1758,18 @@ void SimulationConfig::setRegions(const TestConfig& testConfig) {
 	}
 
 	if (needsRemote) {
-		g_simulator.originalRegions =
+		g_simulator->originalRegions =
 		    "regions=" + json_spirit::write_string(json_spirit::mValue(regionArr), json_spirit::Output_options::none);
 
 		StatusArray disablePrimary = regionArr;
 		disablePrimary[0].get_obj()["datacenters"].get_array()[0].get_obj()["priority"] = -1;
-		g_simulator.disablePrimary = "regions=" + json_spirit::write_string(json_spirit::mValue(disablePrimary),
-		                                                                    json_spirit::Output_options::none);
+		g_simulator->disablePrimary = "regions=" + json_spirit::write_string(json_spirit::mValue(disablePrimary),
+		                                                                     json_spirit::Output_options::none);
 
 		StatusArray disableRemote = regionArr;
 		disableRemote[1].get_obj()["datacenters"].get_array()[0].get_obj()["priority"] = -1;
-		g_simulator.disableRemote = "regions=" + json_spirit::write_string(json_spirit::mValue(disableRemote),
-		                                                                   json_spirit::Output_options::none);
+		g_simulator->disableRemote = "regions=" + json_spirit::write_string(json_spirit::mValue(disableRemote),
+		                                                                    json_spirit::Output_options::none);
 	} else {
 		// In order to generate a starting configuration with the remote disabled, do not apply the region
 		// configuration to the DatabaseConfiguration until after creating the starting conf string.
@@ -1730,7 +1796,9 @@ void SimulationConfig::setMachineCount(const TestConfig& testConfig) {
 		machine_count = std::max(datacenters + 2,
 		                         ((db.minDatacentersRequired() > 0) ? datacenters : 1) *
 		                             std::max(3, db.minZonesRequiredPerDatacenter()));
-		machine_count = deterministicRandom()->randomInt(machine_count, std::max(machine_count + 1, extraDB ? 6 : 10));
+		machine_count = deterministicRandom()->randomInt(
+		    machine_count,
+		    std::max(machine_count + 1, extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 10 : 6));
 		// generateMachineTeamTestConfig set up the number of servers per machine and the number of machines such that
 		// if we do not remove the surplus server and machine teams, the simulation test will report error.
 		// This is needed to make sure the number of server (and machine) teams is no larger than the desired number.
@@ -1740,7 +1808,9 @@ void SimulationConfig::setMachineCount(const TestConfig& testConfig) {
 			// while the max possible machine team number is 10.
 			// If machine_count > 5, we can still test the effectivenss of machine teams
 			// Note: machine_count may be much larger than 5 because we may have a big replication factor
-			machine_count = std::max(machine_count, deterministicRandom()->randomInt(5, extraDB ? 6 : 10));
+			machine_count = std::max(machine_count,
+			                         deterministicRandom()->randomInt(
+			                             5, extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 10 : 6));
 		}
 	}
 	machine_count += datacenters * testConfig.extraMachineCountDC;
@@ -1767,7 +1837,8 @@ void SimulationConfig::setProcessesPerMachine(const TestConfig& testConfig) {
 	} else if (generateFearless) {
 		processes_per_machine = 1;
 	} else {
-		processes_per_machine = deterministicRandom()->randomInt(1, (extraDB ? 14 : 28) / machine_count + 2);
+		processes_per_machine = deterministicRandom()->randomInt(
+		    1, (extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 28 : 14) / machine_count + 2);
 	}
 }
 
@@ -1776,7 +1847,7 @@ void SimulationConfig::setProcessesPerMachine(const TestConfig& testConfig) {
 void SimulationConfig::setTss(const TestConfig& testConfig) {
 	int tssCount = 0;
 	// TODO: Support TSS in SHARD_ENCODE_LOCATION_METADATA mode.
-	if (!testConfig.simpleConfig && !testConfig.disableTss && !CLIENT_KNOBS->SHARD_ENCODE_LOCATION_METADATA &&
+	if (!testConfig.simpleConfig && !testConfig.disableTss && !SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA &&
 	    deterministicRandom()->random01() < 0.25) {
 		// 1 or 2 tss
 		tssCount = deterministicRandom()->randomInt(1, 3);
@@ -1792,21 +1863,23 @@ void SimulationConfig::setTss(const TestConfig& testConfig) {
 		double tssRandom = deterministicRandom()->random01();
 		if (tssRandom > 0.5 || !faultInjectionActivated) {
 			// normal tss mode
-			g_simulator.tssMode = ISimulator::TSSMode::EnabledNormal;
+			g_simulator->tssMode = ISimulator::TSSMode::EnabledNormal;
 		} else if (tssRandom < 0.25 && !testConfig.isFirstTestInRestart) {
 			// fault injection - don't enable in first test in restart because second test won't know it intentionally
 			// lost data
-			g_simulator.tssMode = ISimulator::TSSMode::EnabledDropMutations;
+			g_simulator->tssMode = ISimulator::TSSMode::EnabledDropMutations;
 		} else {
 			// delay injection
-			g_simulator.tssMode = ISimulator::TSSMode::EnabledAddDelay;
+			g_simulator->tssMode = ISimulator::TSSMode::EnabledAddDelay;
 		}
-		printf("enabling tss for simulation in mode %d: %s\n", g_simulator.tssMode, confStr.c_str());
+		printf("enabling tss for simulation in mode %d: %s\n", g_simulator->tssMode, confStr.c_str());
 	}
 }
 
 void setConfigDB(TestConfig const& testConfig) {
-	g_simulator.configDBType = testConfig.getConfigDBType();
+	if (g_simulator) {
+		g_simulator->configDBType = testConfig.getConfigDBType();
+	}
 }
 
 // Generates and sets an appropriate configuration for the database according to
@@ -1865,6 +1938,9 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	}
 
 	simconfig.db.tenantMode = tenantMode;
+	simconfig.db.encryptionAtRestMode = EncryptionAtRestMode::DISABLED;
+
+	g_simulator->blobGranulesEnabled = simconfig.db.blobGranulesEnabled;
 
 	StatusObject startingConfigJSON = simconfig.db.toJSON(true);
 	std::string startingConfigString = "new";
@@ -1879,6 +1955,8 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	if (testConfig.disableEncryption) {
 		g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false }));
 		g_knobs.setKnob("enable_tlog_encryption", KnobValueRef::create(bool{ false }));
+		g_knobs.setKnob("enable_storage_server_encryption", KnobValueRef::create(bool{ false }));
+		g_knobs.setKnob("enable_blob_granule_encryption", KnobValueRef::create(bool{ false }));
 		TraceEvent(SevDebug, "DisableEncryption");
 	}
 	auto configDBType = testConfig.getConfigDBType();
@@ -1897,7 +1975,8 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		if (kv.second.type() == json_spirit::int_type) {
 			startingConfigString += kv.first + ":=" + format("%d", kv.second.get_int());
 		} else if (kv.second.type() == json_spirit::str_type) {
-			if ("storage_migration_type" == kv.first || "tenant_mode" == kv.first) {
+			if ("storage_migration_type" == kv.first || "tenant_mode" == kv.first ||
+			    "encryption_at_rest_mode" == kv.first) {
 				startingConfigString += kv.first + "=" + kv.second.get_str();
 			} else {
 				startingConfigString += kv.second.get_str();
@@ -1917,57 +1996,12 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		startingConfigString += format(" tss_storage_engine:=%d", simconfig.db.testingStorageServerStoreType);
 	}
 
-	if (g_simulator.originalRegions != "") {
-		simconfig.set_config(g_simulator.originalRegions);
-		g_simulator.startingDisabledConfiguration = startingConfigString + " " + g_simulator.disableRemote;
-		startingConfigString += " " + g_simulator.originalRegions;
+	if (g_simulator->originalRegions != "") {
+		simconfig.set_config(g_simulator->originalRegions);
+		g_simulator->startingDisabledConfiguration = startingConfigString + " " + g_simulator->disableRemote;
+		startingConfigString += " " + g_simulator->originalRegions;
 	}
 
-	g_simulator.storagePolicy = simconfig.db.storagePolicy;
-	g_simulator.tLogPolicy = simconfig.db.tLogPolicy;
-	g_simulator.tLogWriteAntiQuorum = simconfig.db.tLogWriteAntiQuorum;
-	g_simulator.remoteTLogPolicy = simconfig.db.getRemoteTLogPolicy();
-	g_simulator.usableRegions = simconfig.db.usableRegions;
-
-	if (simconfig.db.regions.size() > 0) {
-		g_simulator.primaryDcId = simconfig.db.regions[0].dcId;
-		g_simulator.hasSatelliteReplication = simconfig.db.regions[0].satelliteTLogReplicationFactor > 0;
-		if (simconfig.db.regions[0].satelliteTLogUsableDcsFallback > 0) {
-			g_simulator.satelliteTLogPolicyFallback = simconfig.db.regions[0].satelliteTLogPolicyFallback;
-			g_simulator.satelliteTLogWriteAntiQuorumFallback =
-			    simconfig.db.regions[0].satelliteTLogWriteAntiQuorumFallback;
-		} else {
-			g_simulator.satelliteTLogPolicyFallback = simconfig.db.regions[0].satelliteTLogPolicy;
-			g_simulator.satelliteTLogWriteAntiQuorumFallback = simconfig.db.regions[0].satelliteTLogWriteAntiQuorum;
-		}
-		g_simulator.satelliteTLogPolicy = simconfig.db.regions[0].satelliteTLogPolicy;
-		g_simulator.satelliteTLogWriteAntiQuorum = simconfig.db.regions[0].satelliteTLogWriteAntiQuorum;
-
-		for (auto s : simconfig.db.regions[0].satellites) {
-			g_simulator.primarySatelliteDcIds.push_back(s.dcId);
-		}
-	} else {
-		g_simulator.hasSatelliteReplication = false;
-		g_simulator.satelliteTLogWriteAntiQuorum = 0;
-	}
-
-	if (simconfig.db.regions.size() == 2) {
-		g_simulator.remoteDcId = simconfig.db.regions[1].dcId;
-		ASSERT((!simconfig.db.regions[0].satelliteTLogPolicy && !simconfig.db.regions[1].satelliteTLogPolicy) ||
-		       simconfig.db.regions[0].satelliteTLogPolicy->info() ==
-		           simconfig.db.regions[1].satelliteTLogPolicy->info());
-
-		for (auto s : simconfig.db.regions[1].satellites) {
-			g_simulator.remoteSatelliteDcIds.push_back(s.dcId);
-		}
-	}
-
-	if (g_simulator.usableRegions < 2 || !g_simulator.hasSatelliteReplication) {
-		g_simulator.allowLogSetKills = false;
-	}
-
-	ASSERT(g_simulator.storagePolicy && g_simulator.tLogPolicy);
-	ASSERT(!g_simulator.hasSatelliteReplication || g_simulator.satelliteTLogPolicy);
 	TraceEvent("SimulatorConfig").setMaxFieldLength(10000).detail("ConfigString", StringRef(startingConfigString));
 
 	const int dataCenters = simconfig.datacenters;
@@ -1982,26 +2016,39 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	bool sslEnabled = deterministicRandom()->random01() < 0.10;
 	bool sslOnly = sslEnabled && deterministicRandom()->coinflip();
 	bool isTLS = sslEnabled && sslOnly;
-	g_simulator.listenersPerProcess = sslEnabled && !sslOnly ? 2 : 1;
-	TEST(sslEnabled); // SSL enabled
-	TEST(!sslEnabled); // SSL disabled
+	g_simulator->listenersPerProcess = sslEnabled && !sslOnly ? 2 : 1;
+	CODE_PROBE(sslEnabled, "SSL enabled");
+	CODE_PROBE(!sslEnabled, "SSL disabled");
 
 	// Use IPv6 25% of the time
 	bool useIPv6 = deterministicRandom()->random01() < 0.25;
-	TEST(useIPv6); // Use IPv6
-	TEST(!useIPv6); // Use IPv4
+	CODE_PROBE(useIPv6, "Use IPv6");
+	CODE_PROBE(!useIPv6, "Use IPv4");
 
 	// Use hostname 25% of the time, unless it is disabled
 	bool useHostname = !testConfig.disableHostname && deterministicRandom()->random01() < 0.25;
-	TEST(useHostname); // Use hostname
-	TEST(!useHostname); // Use IP address
+	CODE_PROBE(useHostname, "Use hostname");
+	CODE_PROBE(!useHostname, "Use IP address");
 	NetworkAddressFromHostname fromHostname =
 	    useHostname ? NetworkAddressFromHostname::True : NetworkAddressFromHostname::False;
 
+	int extraDatabaseCount = 0;
+	bool useLocalDatabase = (testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::LocalOrSingle && BUGGIFY) ||
+	                        testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Local;
+	if (!useLocalDatabase && testConfig.extraDatabaseMode != ISimulator::ExtraDatabaseMode::Disabled) {
+		extraDatabaseCount =
+		    testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Multiple && testConfig.extraDatabaseCount > 0
+		        ? testConfig.extraDatabaseCount
+		        : 1;
+	}
+
 	std::vector<NetworkAddress> coordinatorAddresses;
 	std::vector<Hostname> coordinatorHostnames;
-	std::vector<NetworkAddress> extraCoordinatorAddresses; // Used by extra DB if the DR db is a new one
-	std::vector<Hostname> extraCoordinatorHostnames;
+
+	// A list of coordinators for each extra database being created. The Nth vector in the outer vector
+	// contains the coordinators for the Nth extra database.
+	std::vector<std::vector<NetworkAddress>> extraCoordinatorAddresses(extraDatabaseCount);
+	std::vector<std::vector<Hostname>> extraCoordinatorHostnames(extraDatabaseCount);
 
 	if (testConfig.minimumRegions > 1) {
 		// do not put coordinators in the primary region so that we can kill that region safely
@@ -2013,21 +2060,29 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 				uint16_t port = sslEnabled && !sslOnly ? 2 : 1;
 				NetworkAddress coordinator(ip, port, true, isTLS, fromHostname);
 				coordinatorAddresses.push_back(coordinator);
-				auto extraIp = makeIPAddressForSim(useIPv6, { 4, dc, 1, m });
-				NetworkAddress extraCoordinator(extraIp, port, true, isTLS, fromHostname);
-				extraCoordinatorAddresses.push_back(extraCoordinator);
 
 				if (useHostname) {
 					std::string hostname = "fakeCoordinatorDC" + std::to_string(dc) + "M" + std::to_string(m);
 					Hostname coordinatorHostname(hostname, std::to_string(port), isTLS);
 					coordinatorHostnames.push_back(coordinatorHostname);
 					INetworkConnections::net()->addMockTCPEndpoint(hostname, std::to_string(port), { coordinator });
-					hostname = "fakeExtraCoordinatorDC" + std::to_string(dc) + "M" + std::to_string(m);
-					Hostname extraCoordinatorHostname(hostname, std::to_string(port), isTLS);
-					extraCoordinatorHostnames.push_back(extraCoordinatorHostname);
-					INetworkConnections::net()->addMockTCPEndpoint(
-					    hostname, std::to_string(port), { extraCoordinator });
 				}
+
+				for (int edb = 0; edb < extraDatabaseCount; ++edb) {
+					auto extraIp = makeIPAddressForSim(useIPv6, { 4 + edb, dc, 1, m });
+					NetworkAddress extraCoordinator(extraIp, port, true, isTLS, fromHostname);
+					extraCoordinatorAddresses[edb].push_back(extraCoordinator);
+
+					if (useHostname) {
+						std::string hostname = "fakeExtraCoordinatorDC" + std::to_string(dc) + "M" + std::to_string(m) +
+						                       "C" + std::to_string(edb);
+						Hostname extraCoordinatorHostname(hostname, std::to_string(port), isTLS);
+						extraCoordinatorHostnames[edb].push_back(extraCoordinatorHostname);
+						INetworkConnections::net()->addMockTCPEndpoint(
+						    hostname, std::to_string(port), { extraCoordinator });
+					}
+				}
+
 				TraceEvent("SelectedCoordinator")
 				    .detail("Hostname", useHostname ? coordinatorHostnames.back().toString().c_str() : "N/A")
 				    .detail("Address", coordinatorAddresses.back());
@@ -2059,20 +2114,26 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 					uint16_t port = sslEnabled && !sslOnly ? 2 : 1;
 					NetworkAddress coordinator(ip, port, true, isTLS, fromHostname);
 					coordinatorAddresses.push_back(coordinator);
-					auto extraIp = makeIPAddressForSim(useIPv6, { 4, dc, 1, m });
-					NetworkAddress extraCoordinator(extraIp, port, true, isTLS, fromHostname);
-					extraCoordinatorAddresses.push_back(extraCoordinator);
+
 					if (useHostname) {
 						std::string hostname = "fakeCoordinatorDC" + std::to_string(dc) + "M" + std::to_string(m);
 						Hostname coordinatorHostname(hostname, std::to_string(port), isTLS);
 						coordinatorHostnames.push_back(coordinatorHostname);
 						INetworkConnections::net()->addMockTCPEndpoint(hostname, std::to_string(port), { coordinator });
-						hostname = "fakeExtraCoordinatorDC" + std::to_string(dc) + "M" + std::to_string(m);
+					}
+
+					for (int edb = 0; edb < extraDatabaseCount; ++edb) {
+						auto extraIp = makeIPAddressForSim(useIPv6, { 4 + edb, dc, 1, m });
+						NetworkAddress extraCoordinator(extraIp, port, true, isTLS, fromHostname);
+						extraCoordinatorAddresses[edb].push_back(extraCoordinator);
+						std::string hostname = "fakeExtraCoordinatorDC" + std::to_string(dc) + "M" + std::to_string(m) +
+						                       "C" + std::to_string(edb);
 						Hostname extraCoordinatorHostname(hostname, std::to_string(port), isTLS);
-						extraCoordinatorHostnames.push_back(extraCoordinatorHostname);
+						extraCoordinatorHostnames[edb].push_back(extraCoordinatorHostname);
 						INetworkConnections::net()->addMockTCPEndpoint(
 						    hostname, std::to_string(port), { extraCoordinator });
 					}
+
 					TraceEvent("SelectedCoordinator")
 					    .detail("Hostname", useHostname ? coordinatorHostnames.back().toString().c_str() : "N/A")
 					    .detail("Address", coordinatorAddresses.back())
@@ -2092,15 +2153,19 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	}
 
 	ASSERT(coordinatorAddresses.size() > 0);
+
+	// Mark a random majority of the coordinators as protected, so
+	// we won't accidently kill off a quorum and render the
+	// cluster unrecoverable.
 	deterministicRandom()->randomShuffle(coordinatorAddresses);
 	for (int i = 0; i < (coordinatorAddresses.size() / 2) + 1; i++) {
 		TraceEvent("ProtectCoordinator")
 		    .detail("Address", coordinatorAddresses[i])
 		    .detail("Coordinators", describe(coordinatorAddresses));
-		g_simulator.protectedAddresses.insert(NetworkAddress(
+		g_simulator->protectedAddresses.insert(NetworkAddress(
 		    coordinatorAddresses[i].ip, coordinatorAddresses[i].port, true, coordinatorAddresses[i].isTLS()));
 		if (coordinatorAddresses[i].port == 2) {
-			g_simulator.protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip, 1, true, true));
+			g_simulator->protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip, 1, true, true));
 		}
 	}
 	deterministicRandom()->randomShuffle(coordinatorAddresses);
@@ -2111,22 +2176,19 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		conn = ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr);
 	}
 
-	// If extraDB==0, leave g_simulator.extraDB as null because the test does not use DR.
-	if (testConfig.extraDB == 1) {
-		// The DR database can be either a new database or itself
-		g_simulator.extraDB =
-		    BUGGIFY ? (useHostname ? new ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr)
-		                           : new ClusterConnectionString(coordinatorAddresses, "TestCluster:0"_sr))
-		            : (useHostname ? new ClusterConnectionString(extraCoordinatorHostnames, "ExtraCluster:0"_sr)
-		                           : new ClusterConnectionString(extraCoordinatorAddresses, "ExtraCluster:0"_sr));
-	} else if (testConfig.extraDB == 2) {
-		// The DR database is a new database
-		g_simulator.extraDB = useHostname ? new ClusterConnectionString(extraCoordinatorHostnames, "ExtraCluster:0"_sr)
-		                                  : new ClusterConnectionString(extraCoordinatorAddresses, "ExtraCluster:0"_sr);
-	} else if (testConfig.extraDB == 3) {
-		// The DR database is the same database
-		g_simulator.extraDB = useHostname ? new ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr)
-		                                  : new ClusterConnectionString(coordinatorAddresses, "TestCluster:0"_sr);
+	if (useLocalDatabase) {
+		g_simulator->extraDatabases.push_back(
+		    useHostname ? ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr).toString()
+		                : ClusterConnectionString(coordinatorAddresses, "TestCluster:0"_sr).toString());
+	} else if (testConfig.extraDatabaseMode != ISimulator::ExtraDatabaseMode::Disabled) {
+		for (int i = 0; i < extraDatabaseCount; ++i) {
+			g_simulator->extraDatabases.push_back(
+			    useHostname
+			        ? ClusterConnectionString(extraCoordinatorHostnames[i], StringRef(format("ExtraCluster%04d:0", i)))
+			              .toString()
+			        : ClusterConnectionString(extraCoordinatorAddresses[i], StringRef(format("ExtraCluster%04d:0", i)))
+			              .toString());
+		}
 	}
 
 	*pConnString = conn;
@@ -2135,7 +2197,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	    .detail("String", conn.toString())
 	    .detail("ConfigString", startingConfigString);
 
-	bool requiresExtraDBMachines = testConfig.extraDB && g_simulator.extraDB->toString() != conn.toString();
+	bool requiresExtraDBMachines = !g_simulator->extraDatabases.empty() && !useLocalDatabase;
 	int assignedMachines = 0, nonVersatileMachines = 0;
 	bool gradualMigrationPossible = true;
 	std::vector<ProcessClass::ClassType> processClassesSubSet = { ProcessClass::UnsetClass,
@@ -2243,31 +2305,35 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 			                                     "SimulatedMachine"));
 
 			if (requiresExtraDBMachines) {
-				std::vector<IPAddress> extraIps;
-				extraIps.reserve(processesPerMachine);
-				for (int i = 0; i < processesPerMachine; i++) {
-					extraIps.push_back(
-					    makeIPAddressForSim(useIPv6, { 4, dc, deterministicRandom()->randomInt(1, i + 2), machine }));
+				int cluster = 4;
+				for (auto extraDatabase : g_simulator->extraDatabases) {
+					std::vector<IPAddress> extraIps;
+					extraIps.reserve(processesPerMachine);
+					for (int i = 0; i < processesPerMachine; i++) {
+						extraIps.push_back(makeIPAddressForSim(
+						    useIPv6, { cluster, dc, deterministicRandom()->randomInt(1, i + 2), machine }));
+					}
+
+					Standalone<StringRef> newMachineId(deterministicRandom()->randomUniqueID().toString());
+
+					LocalityData localities(Optional<Standalone<StringRef>>(), newZoneId, newMachineId, dcUID);
+					localities.set("data_hall"_sr, dcUID);
+					systemActors->push_back(reportErrors(simulatedMachine(ClusterConnectionString(extraDatabase),
+					                                                      extraIps,
+					                                                      sslEnabled,
+					                                                      localities,
+					                                                      processClass,
+					                                                      baseFolder,
+					                                                      false,
+					                                                      machine == useSeedForMachine,
+					                                                      AgentNone,
+					                                                      sslOnly,
+					                                                      whitelistBinPaths,
+					                                                      protocolVersion,
+					                                                      configDBType),
+					                                     "SimulatedMachine"));
+					++cluster;
 				}
-
-				Standalone<StringRef> newMachineId(deterministicRandom()->randomUniqueID().toString());
-
-				LocalityData localities(Optional<Standalone<StringRef>>(), newZoneId, newMachineId, dcUID);
-				localities.set("data_hall"_sr, dcUID);
-				systemActors->push_back(reportErrors(simulatedMachine(*g_simulator.extraDB,
-				                                                      extraIps,
-				                                                      sslEnabled,
-				                                                      localities,
-				                                                      processClass,
-				                                                      baseFolder,
-				                                                      false,
-				                                                      machine == useSeedForMachine,
-				                                                      AgentNone,
-				                                                      sslOnly,
-				                                                      whitelistBinPaths,
-				                                                      protocolVersion,
-				                                                      configDBType),
-				                                     "SimulatedMachine"));
 			}
 
 			assignedMachines++;
@@ -2278,14 +2344,14 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		}
 	}
 
-	g_simulator.desiredCoordinators = coordinatorCount;
-	g_simulator.physicalDatacenters = dataCenters;
-	g_simulator.processesPerMachine = processesPerMachine;
+	g_simulator->desiredCoordinators = coordinatorCount;
+	g_simulator->physicalDatacenters = dataCenters;
+	g_simulator->processesPerMachine = processesPerMachine;
 
 	TraceEvent("SetupSimulatorSettings")
-	    .detail("DesiredCoordinators", g_simulator.desiredCoordinators)
-	    .detail("PhysicalDatacenters", g_simulator.physicalDatacenters)
-	    .detail("ProcessesPerMachine", g_simulator.processesPerMachine);
+	    .detail("DesiredCoordinators", g_simulator->desiredCoordinators)
+	    .detail("PhysicalDatacenters", g_simulator->physicalDatacenters)
+	    .detail("ProcessesPerMachine", g_simulator->processesPerMachine);
 
 	// SOMEDAY: add locality for testers to simulate network topology
 	// FIXME: Start workers with tester class instead, at least sometimes run tests with the testers-only flag
@@ -2314,16 +2380,16 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		                 "SimulatedTesterMachine"));
 	}
 
-	if (g_simulator.setDiffProtocol) {
+	if (g_simulator->setDiffProtocol) {
 		--(*pTesterCount);
 	}
 
 	*pStartingConfiguration = startingConfigString;
 
 	// save some state that we only need when restarting the simulator.
-	g_simulator.connectionString = conn.toString();
-	g_simulator.testerCount = testerCount;
-	g_simulator.allowStorageMigrationTypeChange = gradualMigrationPossible;
+	g_simulator->connectionString = conn.toString();
+	g_simulator->testerCount = testerCount;
+	g_simulator->allowStorageMigrationTypeChange = gradualMigrationPossible;
 
 	TraceEvent("SimulatedClusterStarted")
 	    .detail("DataCenters", dataCenters)
@@ -2361,21 +2427,33 @@ ACTOR void setupAndRun(std::string dataFolder,
 	state TestConfig testConfig;
 	state IPAllowList allowList;
 	testConfig.readFromConfig(testFile);
-	g_simulator.hasDiffProtocolProcess = testConfig.startIncompatibleProcess;
-	g_simulator.setDiffProtocol = false;
+	g_simulator->hasDiffProtocolProcess = testConfig.startIncompatibleProcess;
+	g_simulator->setDiffProtocol = false;
+	if (testConfig.injectTargetedSSRestart && deterministicRandom()->random01() < 0.25) {
+		g_simulator->injectTargetedSSRestartTime = 60.0 + 340.0 * deterministicRandom()->random01();
+	}
+
+	if (testConfig.injectSSDelay && deterministicRandom()->random01() < 0.25) {
+		g_simulator->injectSSDelayTime = 60.0 + 240.0 * deterministicRandom()->random01();
+	}
 
 	// Build simulator allow list
 	allowList.addTrustedSubnet("0.0.0.0/2"sv);
 	allowList.addTrustedSubnet("abcd::/16"sv);
 	state bool allowDefaultTenant = testConfig.allowDefaultTenant;
 	state bool allowDisablingTenants = testConfig.allowDisablingTenants;
-	state bool allowCreatingTenants = true;
+	state bool allowCreatingTenants = testConfig.allowCreatingTenants;
+
+	if (!SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+		testConfig.storageEngineExcludeTypes.push_back(5);
+	}
 
 	// The RocksDB storage engine does not support the restarting tests because you cannot consistently get a clean
 	// snapshot of the storage engine without a snapshotting file system.
 	// https://github.com/apple/foundationdb/issues/5155
 	if (std::string_view(testFile).find("restarting") != std::string_view::npos) {
 		testConfig.storageEngineExcludeTypes.push_back(4);
+		testConfig.storageEngineExcludeTypes.push_back(5);
 
 		// Disable the default tenant in restarting tests for now
 		// TODO: persist the chosen default tenant in the restartInfo.ini file for the second test
@@ -2388,49 +2466,44 @@ ACTOR void setupAndRun(std::string dataFolder,
 	// Re-enable the backup and restore related simulation tests when the tests are passing again.
 	if (std::string_view(testFile).find("Backup") != std::string_view::npos) {
 		testConfig.storageEngineExcludeTypes.push_back(4);
-	}
-
-	// Disable the default tenant in backup and DR tests for now. This is because backup does not currently duplicate
-	// the tenant map and related state.
-	// TODO: reenable when backup/DR or BlobGranule supports tenants.
-	if (std::string_view(testFile).find("Backup") != std::string_view::npos || testConfig.extraDB != 0) {
-		allowDefaultTenant = false;
+		testConfig.storageEngineExcludeTypes.push_back(5);
 	}
 
 	// The RocksDB engine is not always built with the rest of fdbserver. Don't try to use it if it is not included
 	// in the build.
 	if (!rocksDBEnabled) {
 		testConfig.storageEngineExcludeTypes.push_back(4);
+		testConfig.storageEngineExcludeTypes.push_back(5);
 	}
 
-	state ProtocolVersion protocolVersion = currentProtocolVersion;
+	state ProtocolVersion protocolVersion = currentProtocolVersion();
 	if (testConfig.startIncompatibleProcess) {
 		// isolates right most 1 bit of compatibleProtocolVersionMask to make this protocolVersion incompatible
 		uint64_t minAddToMakeIncompatible =
 		    ProtocolVersion::compatibleProtocolVersionMask & ~(ProtocolVersion::compatibleProtocolVersionMask - 1);
-		protocolVersion = ProtocolVersion(currentProtocolVersion.version() + minAddToMakeIncompatible);
+		protocolVersion = ProtocolVersion(currentProtocolVersion().version() + minAddToMakeIncompatible);
 	}
 
 	// TODO (IPv6) Use IPv6?
 	auto testSystem =
-	    g_simulator.newProcess("TestSystem",
-	                           IPAddress(0x01010101),
-	                           1,
-	                           false,
-	                           1,
-	                           LocalityData(Optional<Standalone<StringRef>>(),
-	                                        Standalone<StringRef>(deterministicRandom()->randomUniqueID().toString()),
-	                                        Standalone<StringRef>(deterministicRandom()->randomUniqueID().toString()),
-	                                        Optional<Standalone<StringRef>>()),
-	                           ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource),
-	                           "",
-	                           "",
-	                           currentProtocolVersion);
+	    g_simulator->newProcess("TestSystem",
+	                            IPAddress(0x01010101),
+	                            1,
+	                            false,
+	                            1,
+	                            LocalityData(Optional<Standalone<StringRef>>(),
+	                                         Standalone<StringRef>(deterministicRandom()->randomUniqueID().toString()),
+	                                         Standalone<StringRef>(deterministicRandom()->randomUniqueID().toString()),
+	                                         Optional<Standalone<StringRef>>()),
+	                            ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource),
+	                            "",
+	                            "",
+	                            currentProtocolVersion());
 	testSystem->excludeFromRestarts = true;
-	wait(g_simulator.onProcess(testSystem, TaskPriority::DefaultYield));
+	wait(g_simulator->onProcess(testSystem, TaskPriority::DefaultYield));
 	Sim2FileSystem::newFileSystem();
 	FlowTransport::createInstance(true, 1, WLTOKEN_RESERVED_COUNT, &allowList);
-	TEST(true); // Simulation start
+	CODE_PROBE(true, "Simulation start");
 
 	state Optional<TenantName> defaultTenant;
 	state Standalone<VectorRef<TenantNameRef>> tenantsToCreate;
@@ -2492,7 +2565,25 @@ ACTOR void setupAndRun(std::string dataFolder,
 		std::string clusterFileDir = joinPath(dataFolder, deterministicRandom()->randomUniqueID().toString());
 		platform::createDirectory(clusterFileDir);
 		writeFile(joinPath(clusterFileDir, "fdb.cluster"), connectionString.get().toString());
-		wait(timeoutError(runTests(makeReference<ClusterConnectionFile>(joinPath(clusterFileDir, "fdb.cluster")),
+		state Reference<ClusterConnectionFile> connFile =
+		    makeReference<ClusterConnectionFile>(joinPath(clusterFileDir, "fdb.cluster"));
+		if (rebooting) {
+			// protect coordinators for restarting tests
+			std::vector<NetworkAddress> coordinatorAddresses =
+			    wait(connFile->getConnectionString().tryResolveHostnames());
+			ASSERT(coordinatorAddresses.size() > 0);
+			for (int i = 0; i < (coordinatorAddresses.size() / 2) + 1; i++) {
+				TraceEvent("ProtectCoordinator")
+				    .detail("Address", coordinatorAddresses[i])
+				    .detail("Coordinators", describe(coordinatorAddresses));
+				g_simulator->protectedAddresses.insert(NetworkAddress(
+				    coordinatorAddresses[i].ip, coordinatorAddresses[i].port, true, coordinatorAddresses[i].isTLS()));
+				if (coordinatorAddresses[i].port == 2) {
+					g_simulator->protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip, 1, true, true));
+				}
+			}
+		}
+		wait(timeoutError(runTests(connFile,
 		                           TEST_TYPE_FROM_FILE,
 		                           TEST_ON_TESTERS,
 		                           testerCount,
@@ -2501,15 +2592,28 @@ ACTOR void setupAndRun(std::string dataFolder,
 		                           LocalityData(),
 		                           UnitTestParameters(),
 		                           defaultTenant,
-		                           tenantsToCreate),
+		                           tenantsToCreate,
+		                           rebooting),
 		                  isBuggifyEnabled(BuggifyType::General) ? 36000.0 : 5400.0));
 	} catch (Error& e) {
 		TraceEvent(SevError, "SetupAndRunError").error(e);
 	}
 
+	TraceEvent("TracingMissingCodeProbes").log();
+	probe::traceMissedProbes(probe::ExecutionContext::Simulation);
 	TraceEvent("SimulatedSystemDestruct").log();
-	g_simulator.stop();
+	g_simulator->stop();
 	destructed = true;
 	wait(Never());
 	ASSERT(false);
 }
+
+DatabaseConfiguration generateNormalDatabaseConfiguration(const BasicTestConfig& testConfig) {
+	TestConfig config(testConfig);
+	if (!rocksDBEnabled) {
+		config.storageEngineExcludeTypes.push_back(4);
+		config.storageEngineExcludeTypes.push_back(5);
+	}
+	SimulationConfig simConf(config);
+	return simConf.db;
+}
diff --git a/fdbserver/SkipList.cpp b/fdbserver/SkipList.cpp
index 180dc13dfa..3e4e31eeb5 100644
--- a/fdbserver/SkipList.cpp
+++ b/fdbserver/SkipList.cpp
@@ -1046,32 +1046,32 @@ void miniConflictSetTest() {
 
 void operatorLessThanTest() {
 	{ // Longer strings before shorter strings.
-		KeyInfo a(LiteralStringRef("hello"), /*begin=*/false, /*write=*/true, 0, nullptr);
-		KeyInfo b(LiteralStringRef("hello\0"), /*begin=*/false, /*write=*/false, 0, nullptr);
+		KeyInfo a("hello"_sr, /*begin=*/false, /*write=*/true, 0, nullptr);
+		KeyInfo b("hello\0"_sr, /*begin=*/false, /*write=*/false, 0, nullptr);
 		ASSERT(a < b);
 		ASSERT(!(b < a));
 		ASSERT(!(a == b));
 	}
 
 	{ // Reads before writes.
-		KeyInfo a(LiteralStringRef("hello"), /*begin=*/false, /*write=*/false, 0, nullptr);
-		KeyInfo b(LiteralStringRef("hello"), /*begin=*/false, /*write=*/true, 0, nullptr);
+		KeyInfo a("hello"_sr, /*begin=*/false, /*write=*/false, 0, nullptr);
+		KeyInfo b("hello"_sr, /*begin=*/false, /*write=*/true, 0, nullptr);
 		ASSERT(a < b);
 		ASSERT(!(b < a));
 		ASSERT(!(a == b));
 	}
 
 	{ // Begin reads after writes.
-		KeyInfo a(LiteralStringRef("hello"), /*begin=*/false, /*write=*/true, 0, nullptr);
-		KeyInfo b(LiteralStringRef("hello"), /*begin=*/true, /*write=*/false, 0, nullptr);
+		KeyInfo a("hello"_sr, /*begin=*/false, /*write=*/true, 0, nullptr);
+		KeyInfo b("hello"_sr, /*begin=*/true, /*write=*/false, 0, nullptr);
 		ASSERT(a < b);
 		ASSERT(!(b < a));
 		ASSERT(!(a == b));
 	}
 
 	{ // Begin writes after writes.
-		KeyInfo a(LiteralStringRef("hello"), /*begin=*/false, /*write=*/true, 0, nullptr);
-		KeyInfo b(LiteralStringRef("hello"), /*begin=*/true, /*write=*/true, 0, nullptr);
+		KeyInfo a("hello"_sr, /*begin=*/false, /*write=*/true, 0, nullptr);
+		KeyInfo b("hello"_sr, /*begin=*/true, /*write=*/true, 0, nullptr);
 		ASSERT(a < b);
 		ASSERT(!(b < a));
 		ASSERT(!(a == b));
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 2d58b812bf..224728d8ae 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include <cinttypes>
+#include "fdbclient/BackupAgent.actor.h"
 #include "fmt/format.h"
 #include "fdbclient/BlobWorkerInterface.h"
 #include "fdbclient/KeyBackedTypes.h"
@@ -34,6 +35,7 @@
 #include "fdbserver/ClusterRecovery.actor.h"
 #include "fdbserver/CoordinationInterface.h"
 #include "fdbserver/DataDistribution.actor.h"
+#include "fdbclient/ConsistencyScanInterface.actor.h"
 #include "flow/UnitTest.h"
 #include "fdbserver/QuietDatabase.h"
 #include "fdbserver/RecoveryState.h"
@@ -283,10 +285,20 @@ static JsonBuilderObject getError(const TraceEventFields& errorFields) {
 	return statusObj;
 }
 
-static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
-                                              std::vector<WorkerDetails> workers,
-                                              Optional<DatabaseConfiguration> configuration,
-                                              std::set<std::string>* incomplete_reasons) {
+namespace {
+
+void reportCgroupCpuStat(JsonBuilderObject& object, const TraceEventFields& eventFields) {
+	JsonBuilderObject cgroupCpuStatObj;
+	cgroupCpuStatObj.setKeyRawNumber("nr_periods", eventFields.getValue("NrPeriods"));
+	cgroupCpuStatObj.setKeyRawNumber("nr_throttled", eventFields.getValue("NrThrottled"));
+	cgroupCpuStatObj.setKeyRawNumber("throttled_time", eventFields.getValue("ThrottledTime"));
+	object["cgroup_cpu_stat"] = cgroupCpuStatObj;
+}
+
+JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
+                                       std::vector<WorkerDetails> workers,
+                                       Optional<DatabaseConfiguration> configuration,
+                                       std::set<std::string>* incomplete_reasons) {
 	JsonBuilderObject machineMap;
 	double metric;
 	int failed = 0;
@@ -338,6 +350,10 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
 				memoryObj.setKeyRawNumber("free_bytes", event.getValue("AvailableMemory"));
 				statusObj["memory"] = memoryObj;
 
+#ifdef __linux__
+				reportCgroupCpuStat(statusObj, event);
+#endif // __linux__
+
 				JsonBuilderObject cpuObj;
 				double cpuSeconds = event.getDouble("CPUSeconds");
 				double elapsed = event.getDouble("Elapsed");
@@ -401,6 +417,8 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
 	return machineMap;
 }
 
+} // anonymous namespace
+
 JsonBuilderObject getLagObject(int64_t versions) {
 	JsonBuilderObject lag;
 	lag["versions"] = versions;
@@ -487,6 +505,11 @@ struct RolesInfo {
 		double dataLagSeconds = -1.0;
 		obj["id"] = iface.id().shortString();
 		obj["role"] = role;
+		if (iface.metadata.present()) {
+			obj["storage_metadata"] = iface.metadata.get().toJSON();
+			// printf("%s\n", metadataObj.getJson().c_str());
+		}
+
 		try {
 			TraceEventFields const& storageMetrics = metrics.at("StorageMetrics");
 
@@ -549,23 +572,11 @@ struct RolesInfo {
 			TraceEventFields const& busiestReadTag = metrics.at("BusiestReadTag");
 			if (busiestReadTag.size()) {
 				int64_t tagCost = busiestReadTag.getInt64("TagCost");
-
 				if (tagCost > 0) {
 					JsonBuilderObject busiestReadTagObj;
-
-					int64_t totalSampledCost = busiestReadTag.getInt64("TotalSampledCost");
-					ASSERT(totalSampledCost > 0);
-
 					busiestReadTagObj["tag"] = busiestReadTag.getValue("Tag");
-					busiestReadTagObj["fractional_cost"] = (double)tagCost / totalSampledCost;
-
-					double elapsed = busiestReadTag.getDouble("Elapsed");
-					if (CLIENT_KNOBS->READ_TAG_SAMPLE_RATE > 0 && elapsed > 0) {
-						JsonBuilderObject estimatedCostObj;
-						estimatedCostObj["hz"] = tagCost / CLIENT_KNOBS->READ_TAG_SAMPLE_RATE / elapsed;
-						busiestReadTagObj["estimated_cost"] = estimatedCostObj;
-					}
-
+					busiestReadTagObj["cost"] = tagCost;
+					busiestReadTagObj["fractional_cost"] = busiestReadTag.getValue("FractionalBusyness");
 					obj["busiest_read_tag"] = busiestReadTagObj;
 				}
 			}
@@ -594,14 +605,12 @@ struct RolesInfo {
 				}
 			}
 
-			if (iface.metadata.present()) {
-				obj["storage_metadata"] = iface.metadata.get().toJSON();
-				// printf("%s\n", metadataObj.getJson().c_str());
-			}
-
 		} catch (Error& e) {
 			if (e.code() != error_code_attribute_not_found)
 				throw e;
+			else {
+				TraceEvent(SevWarnAlways, "StorageServerStatusJson").error(e);
+			}
 		}
 
 		if (pDataLagSeconds) {
@@ -742,6 +751,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
     std::vector<std::pair<GrvProxyInterface, EventMap>> grvProxies,
     std::vector<BlobWorkerInterface> blobWorkers,
     ServerCoordinators coordinators,
+    std::vector<NetworkAddress> coordinatorAddresses,
     Database cx,
     Optional<DatabaseConfiguration> configuration,
     Optional<Key> healthyZone,
@@ -817,6 +827,10 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 		roles.addRole("blob_manager", db->get().blobManager.get());
 	}
 
+	if (db->get().consistencyScan.present()) {
+		roles.addRole("consistency_scan", db->get().consistencyScan.get());
+	}
+
 	if (SERVER_KNOBS->ENABLE_ENCRYPTION && db->get().encryptKeyProxy.present()) {
 		roles.addRole("encrypt_key_proxy", db->get().encryptKeyProxy.get());
 	}
@@ -839,8 +853,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 		}
 	}
 
-	std::vector<NetworkAddress> addressVec = wait(coordinators.ccr->getConnectionString().tryResolveHostnames());
-	for (const auto& coordinator : addressVec) {
+	for (const auto& coordinator : coordinatorAddresses) {
 		roles.addCoordinatorRole(coordinator);
 	}
 
@@ -1087,76 +1100,61 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 	return processMap;
 }
 
-struct ClientStats {
-	int count;
-	std::set<std::pair<NetworkAddress, Key>> examples;
-
-	ClientStats() : count(0) {}
-};
-
 static JsonBuilderObject clientStatusFetcher(
     std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>>* clientStatusMap) {
 	JsonBuilderObject clientStatus;
 
 	int64_t clientCount = 0;
-	std::map<Key, ClientStats> issues;
-	std::map<Standalone<ClientVersionRef>, ClientStats> supportedVersions;
-	std::map<Key, ClientStats> maxSupportedProtocol;
+	// Here we handle versions and maxSupportedProtocols, the issues will be handled in getClientIssuesAsMessages
+	std::map<Standalone<ClientVersionRef>, OpenDatabaseRequest::Samples> supportedVersions;
+	std::map<Key, OpenDatabaseRequest::Samples> maxSupportedProtocol;
 
 	for (auto iter = clientStatusMap->begin(); iter != clientStatusMap->end();) {
-		if (now() - iter->second.first < 2 * SERVER_KNOBS->COORDINATOR_REGISTER_INTERVAL) {
-			clientCount += iter->second.second.clientCount;
-			for (auto& it : iter->second.second.issues) {
-				auto& issue = issues[it.item];
-				issue.count += it.count;
-				issue.examples.insert(it.examples.begin(), it.examples.end());
-			}
-			for (auto& it : iter->second.second.supportedVersions) {
-				auto& version = supportedVersions[it.item];
-				version.count += it.count;
-				version.examples.insert(it.examples.begin(), it.examples.end());
-			}
-			for (auto& it : iter->second.second.maxProtocolSupported) {
-				auto& protocolVersion = maxSupportedProtocol[it.item];
-				protocolVersion.count += it.count;
-				protocolVersion.examples.insert(it.examples.begin(), it.examples.end());
-			}
-			++iter;
-		} else {
+		if (now() - iter->second.first >= 2 * SERVER_KNOBS->COORDINATOR_REGISTER_INTERVAL) {
 			iter = clientStatusMap->erase(iter);
+			continue;
 		}
+
+		clientCount += iter->second.second.clientCount;
+		for (const auto& [version, samples] : iter->second.second.supportedVersions) {
+			supportedVersions[version] += samples;
+		}
+		for (const auto& [protocol, samples] : iter->second.second.maxProtocolSupported) {
+			maxSupportedProtocol[protocol] += samples;
+		}
+		++iter;
 	}
 
 	clientStatus["count"] = clientCount;
 
 	JsonBuilderArray versionsArray = JsonBuilderArray();
-	for (auto& cv : supportedVersions) {
+	for (const auto& [clientVersionRef, samples] : supportedVersions) {
 		JsonBuilderObject ver;
-		ver["count"] = (int64_t)cv.second.count;
-		ver["client_version"] = cv.first.clientVersion.toString();
-		ver["protocol_version"] = cv.first.protocolVersion.toString();
-		ver["source_version"] = cv.first.sourceVersion.toString();
+		ver["count"] = (int64_t)samples.count;
+		ver["client_version"] = clientVersionRef.clientVersion.toString();
+		ver["protocol_version"] = clientVersionRef.protocolVersion.toString();
+		ver["source_version"] = clientVersionRef.sourceVersion.toString();
 
 		JsonBuilderArray clients = JsonBuilderArray();
-		for (auto& client : cv.second.examples) {
+		for (const auto& [networkAddress, trackLogGroup] : samples.samples) {
 			JsonBuilderObject cli;
-			cli["address"] = client.first.toString();
-			cli["log_group"] = client.second.toString();
+			cli["address"] = networkAddress.toString();
+			cli["log_group"] = trackLogGroup.toString();
 			clients.push_back(cli);
 		}
 
-		auto iter = maxSupportedProtocol.find(cv.first.protocolVersion);
-		if (iter != maxSupportedProtocol.end()) {
+		auto iter = maxSupportedProtocol.find(clientVersionRef.protocolVersion);
+		if (iter != std::end(maxSupportedProtocol)) {
 			JsonBuilderArray maxClients = JsonBuilderArray();
-			for (auto& client : iter->second.examples) {
+			for (const auto& [networkAddress, trackLogGroup] : iter->second.samples) {
 				JsonBuilderObject cli;
-				cli["address"] = client.first.toString();
-				cli["log_group"] = client.second.toString();
+				cli["address"] = networkAddress.toString();
+				cli["log_group"] = trackLogGroup.toString();
 				maxClients.push_back(cli);
 			}
 			ver["max_protocol_count"] = iter->second.count;
 			ver["max_protocol_clients"] = maxClients;
-			maxSupportedProtocol.erase(cv.first.protocolVersion);
+			maxSupportedProtocol.erase(clientVersionRef.protocolVersion);
 		}
 
 		ver["connected_clients"] = clients;
@@ -1288,7 +1286,7 @@ ACTOR static Future<double> doReadProbe(Future<double> grvProbe, Transaction* tr
 	loop {
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		try {
-			Optional<Standalone<StringRef>> _ = wait(tr->get(LiteralStringRef("\xff/StatusJsonTestKey62793")));
+			Optional<Standalone<StringRef>> _ = wait(tr->get("\xff/StatusJsonTestKey62793"_sr));
 			return g_network->timer_monotonic() - start;
 		} catch (Error& e) {
 			wait(tr->onError(e));
@@ -1500,11 +1498,15 @@ ACTOR static Future<Void> logRangeWarningFetcher(Database cx,
 					if (loggingRanges.count(LogRangeAndUID(range, logUid))) {
 						std::pair<Key, Key> rangePair = std::make_pair(range.begin, range.end);
 						if (existingRanges.count(rangePair)) {
+							std::string rangeDescription = (range == getDefaultBackupSharedRange())
+							                                   ? "the default backup set"
+							                                   : format("`%s` - `%s`",
+							                                            printable(range.begin).c_str(),
+							                                            printable(range.end).c_str());
 							messages->push_back(JsonString::makeMessage(
 							    "duplicate_mutation_streams",
-							    format("Backup and DR are not sharing the same stream of mutations for `%s` - `%s`",
-							           printable(range.begin).c_str(),
-							           printable(range.end).c_str())
+							    format("Backup and DR are not sharing the same stream of mutations for %s",
+							           rangeDescription.c_str())
 							        .c_str()));
 							break;
 						}
@@ -1542,10 +1544,10 @@ struct ProtocolVersionData {
 	ProtocolVersion runningProtocolVersion;
 	ProtocolVersion newestProtocolVersion;
 	ProtocolVersion lowestCompatibleProtocolVersion;
-	ProtocolVersionData() : runningProtocolVersion(currentProtocolVersion) {}
+	ProtocolVersionData() : runningProtocolVersion(currentProtocolVersion()) {}
 
 	ProtocolVersionData(uint64_t newestProtocolVersionValue, uint64_t lowestCompatibleProtocolVersionValue)
-	  : runningProtocolVersion(currentProtocolVersion), newestProtocolVersion(newestProtocolVersionValue),
+	  : runningProtocolVersion(currentProtocolVersion()), newestProtocolVersion(newestProtocolVersionValue),
 	    lowestCompatibleProtocolVersion(lowestCompatibleProtocolVersionValue) {}
 };
 
@@ -1721,17 +1723,16 @@ ACTOR static Future<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
 		std::vector<Future<TraceEventFields>> futures;
 
 		// TODO:  Should this be serial?
-		futures.push_back(timeoutError(
-		    ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("DDTrackerStarting"))), 1.0));
-		futures.push_back(timeoutError(
-		    ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("DDTrackerStats"))), 1.0));
-		futures.push_back(timeoutError(
-		    ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("MovingData"))), 1.0));
-		futures.push_back(timeoutError(
-		    ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("TotalDataInFlight"))), 1.0));
-		futures.push_back(timeoutError(
-		    ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("TotalDataInFlightRemote"))),
-		    1.0));
+		futures.push_back(
+		    timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest("DDTrackerStarting"_sr)), 1.0));
+		futures.push_back(
+		    timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest("DDTrackerStats"_sr)), 1.0));
+		futures.push_back(
+		    timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest("MovingData"_sr)), 1.0));
+		futures.push_back(
+		    timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest("TotalDataInFlight"_sr)), 1.0));
+		futures.push_back(
+		    timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest("TotalDataInFlightRemote"_sr)), 1.0));
 
 		std::vector<TraceEventFields> dataInfo = wait(getAll(futures));
 
@@ -1909,27 +1910,6 @@ static Future<std::vector<std::pair<iface, EventMap>>> getServerMetrics(
 	return results;
 }
 
-ACTOR template <class iface>
-static Future<std::vector<TraceEventFields>> getServerBusiestWriteTags(
-    std::vector<iface> servers,
-    std::unordered_map<NetworkAddress, WorkerInterface> address_workers,
-    WorkerDetails rkWorker) {
-	state std::vector<Future<Optional<TraceEventFields>>> futures;
-	futures.reserve(servers.size());
-	for (const auto& s : servers) {
-		futures.push_back(latestEventOnWorker(rkWorker.interf, s.id().toString() + "/BusiestWriteTag"));
-	}
-	wait(waitForAll(futures));
-
-	std::vector<TraceEventFields> result(servers.size());
-	for (int i = 0; i < servers.size(); ++i) {
-		if (futures[i].get().present()) {
-			result[i] = futures[i].get().get();
-		}
-	}
-	return result;
-}
-
 ACTOR
 static Future<std::vector<StorageServerStatusInfo>> readStorageInterfaceAndMetadata(Database cx,
                                                                                     bool use_system_priority) {
@@ -1967,6 +1947,16 @@ static Future<std::vector<StorageServerStatusInfo>> readStorageInterfaceAndMetad
 	return servers;
 }
 
+namespace {
+
+const std::vector<std::string> STORAGE_SERVER_METRICS_LIST{ "StorageMetrics",
+	                                                        "ReadLatencyMetrics",
+	                                                        "ReadLatencyBands",
+	                                                        "BusiestReadTag",
+	                                                        "BusiestWriteTag" };
+
+} // namespace
+
 ACTOR static Future<std::vector<StorageServerStatusInfo>> getStorageServerStatusInfos(
     Database cx,
     std::unordered_map<NetworkAddress, WorkerInterface> address_workers,
@@ -1974,18 +1964,9 @@ ACTOR static Future<std::vector<StorageServerStatusInfo>> getStorageServerStatus
 	state std::vector<StorageServerStatusInfo> servers =
 	    wait(timeoutError(readStorageInterfaceAndMetadata(cx, true), 5.0));
 	state std::vector<std::pair<StorageServerStatusInfo, EventMap>> results;
-	state std::vector<TraceEventFields> busiestWriteTags;
-	wait(store(results,
-	           getServerMetrics(servers,
-	                            address_workers,
-	                            std::vector<std::string>{
-	                                "StorageMetrics", "ReadLatencyMetrics", "ReadLatencyBands", "BusiestReadTag" })) &&
-	     store(busiestWriteTags, getServerBusiestWriteTags(servers, address_workers, rkWorker)));
-
-	ASSERT(busiestWriteTags.size() == results.size());
+	wait(store(results, getServerMetrics(servers, address_workers, STORAGE_SERVER_METRICS_LIST)));
 	for (int i = 0; i < results.size(); ++i) {
 		servers[i].eventMap = std::move(results[i].second);
-		servers[i].eventMap.emplace("BusiestWriteTag", busiestWriteTags[i]);
 	}
 	return servers;
 }
@@ -2123,8 +2104,7 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
 			auto worker = getWorker(workersMap, p.address());
 			if (worker.present())
 				commitProxyStatFutures.push_back(timeoutError(
-				    worker.get().interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("ProxyMetrics"))),
-				    1.0));
+				    worker.get().interf.eventLogRequest.getReply(EventLogRequest("ProxyMetrics"_sr)), 1.0));
 			else
 				throw all_alternatives_failed(); // We need data from all proxies for this result to be trustworthy
 		}
@@ -2132,8 +2112,7 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
 			auto worker = getWorker(workersMap, p.address());
 			if (worker.present())
 				grvProxyStatFutures.push_back(timeoutError(
-				    worker.get().interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("GrvProxyMetrics"))),
-				    1.0));
+				    worker.get().interf.eventLogRequest.getReply(EventLogRequest("GrvProxyMetrics"_sr)), 1.0));
 			else
 				throw all_alternatives_failed(); // We need data from all proxies for this result to be trustworthy
 		}
@@ -2199,10 +2178,13 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
 
 	// Transactions
 	try {
-		state TraceEventFields ratekeeper = wait(
-		    timeoutError(rkWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("RkUpdate"))), 1.0));
-		TraceEventFields batchRatekeeper = wait(timeoutError(
-		    rkWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("RkUpdateBatch"))), 1.0));
+		state Future<TraceEventFields> f1 =
+		    timeoutError(rkWorker.interf.eventLogRequest.getReply(EventLogRequest("RkUpdate"_sr)), 1.0);
+		state Future<TraceEventFields> f2 =
+		    timeoutError(rkWorker.interf.eventLogRequest.getReply(EventLogRequest("RkUpdateBatch"_sr)), 1.0);
+		wait(success(f1) && success(f2));
+		TraceEventFields ratekeeper = f1.get();
+		TraceEventFields batchRatekeeper = f2.get();
 
 		bool autoThrottlingEnabled = ratekeeper.getInt("AutoThrottlingEnabled");
 		double tpsLimit = ratekeeper.getDouble("TPSLimit");
@@ -2387,6 +2369,53 @@ ACTOR static Future<JsonBuilderObject> clusterSummaryStatisticsFetcher(
 	return statusObj;
 }
 
+ACTOR static Future<JsonBuilderObject> blobWorkerStatusFetcher(
+    std::vector<BlobWorkerInterface> servers,
+    std::unordered_map<NetworkAddress, WorkerInterface> addressWorkersMap,
+    std::set<std::string>* incompleteReason) {
+
+	state JsonBuilderObject statusObj;
+	state std::vector<Future<Optional<TraceEventFields>>> futures;
+
+	statusObj["number_of_blob_workers"] = static_cast<int>(servers.size());
+
+	try {
+		for (auto& intf : servers) {
+			auto workerIntf = addressWorkersMap[intf.address()];
+			futures.push_back(latestEventOnWorker(workerIntf, "BlobWorkerMetrics"));
+		}
+
+		wait(waitForAll(futures));
+
+		state int totalRanges = 0;
+		for (auto future : futures) {
+			if (future.get().present()) {
+				auto latestTrace = future.get().get();
+				int numRanges = latestTrace.getInt("NumRangesAssigned");
+				totalRanges += numRanges;
+
+				JsonBuilderObject workerStatusObj;
+				workerStatusObj["number_of_key_ranges"] = numRanges;
+				workerStatusObj["put_requests"] = StatusCounter(latestTrace.getValue("S3PutReqs")).getStatus();
+				workerStatusObj["get_requests"] = StatusCounter(latestTrace.getValue("S3GetReqs")).getStatus();
+				workerStatusObj["delete_requests"] = StatusCounter(latestTrace.getValue("S3DeleteReqs")).getStatus();
+				workerStatusObj["bytes_buffered"] = latestTrace.getInt64("MutationBytesBuffered");
+				workerStatusObj["compression_bytes_raw"] =
+				    StatusCounter(latestTrace.getValue("CompressionBytesRaw")).getStatus();
+				workerStatusObj["compression_bytes_final"] =
+				    StatusCounter(latestTrace.getValue("CompressionBytesFinal")).getStatus();
+				statusObj[latestTrace.getValue("ID")] = workerStatusObj;
+			}
+		}
+		statusObj["number_of_key_ranges"] = totalRanges;
+	} catch (Error& e) {
+		if (e.code() == error_code_actor_cancelled)
+			throw;
+		incompleteReason->insert("Unknown blob worker stats");
+	}
+	return statusObj;
+}
+
 static JsonBuilderObject tlogFetcher(int* logFaultTolerance,
                                      const std::vector<TLogSet>& tLogs,
                                      std::unordered_map<NetworkAddress, WorkerInterface> const& address_workers) {
@@ -2533,7 +2562,7 @@ static JsonBuilderObject faultToleranceStatusFetcher(DatabaseConfiguration confi
 
 	std::map<NetworkAddress, StringRef> workerZones;
 	for (const auto& worker : workers) {
-		workerZones[worker.interf.address()] = worker.interf.locality.zoneId().orDefault(LiteralStringRef(""));
+		workerZones[worker.interf.address()] = worker.interf.locality.zoneId().orDefault(""_sr);
 	}
 	std::map<StringRef, int> coordinatorZoneCounts;
 	for (const auto& coordinator : coordinatorAddresses) {
@@ -2615,18 +2644,19 @@ static JsonBuilderArray getClientIssuesAsMessages(
 		std::map<std::string, std::pair<int, std::vector<std::string>>> deduplicatedIssues;
 
 		for (auto iter = clientStatusMap->begin(); iter != clientStatusMap->end();) {
-			if (now() - iter->second.first < 2 * SERVER_KNOBS->COORDINATOR_REGISTER_INTERVAL) {
-				for (auto& issue : iter->second.second.issues) {
-					auto& t = deduplicatedIssues[issue.item.toString()];
-					t.first += issue.count;
-					for (auto& example : issue.examples) {
-						t.second.push_back(formatIpPort(example.first.ip, example.first.port));
-					}
-				}
-				++iter;
-			} else {
+			if (now() - iter->second.first >= 2 * SERVER_KNOBS->COORDINATOR_REGISTER_INTERVAL) {
 				iter = clientStatusMap->erase(iter);
+				continue;
 			}
+
+			for (const auto& [issueKey, samples] : iter->second.second.issues) {
+				auto& t = deduplicatedIssues[issueKey.toString()];
+				t.first += samples.count;
+				for (const auto& sample : samples.samples) {
+					t.second.push_back(formatIpPort(sample.first.ip, sample.first.port));
+				}
+			}
+			++iter;
 		}
 
 		// FIXME: add the log_group in addition to the network address
@@ -2751,6 +2781,9 @@ ACTOR Future<JsonBuilderObject> lockedStatusFetcher(Reference<AsyncVar<ServerDBI
 			try {
 				wait(tr.onError(e));
 			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled)
+					throw;
+
 				incomplete_reasons->insert(format("Unable to determine if database is locked (%s).", e.what()));
 				break;
 			}
@@ -2861,6 +2894,26 @@ ACTOR Future<JsonBuilderObject> storageWigglerStatsFetcher(Optional<DataDistribu
 	}
 	return res;
 }
+
+// read consistencyScanInfo through Read-only tx
+ACTOR Future<Optional<Value>> consistencyScanInfoFetcher(Database cx) {
+	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	state Optional<Value> val;
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			wait(store(val, ConsistencyScanInfo::getInfo(tr)));
+			wait(tr->commit());
+			break;
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+
+	TraceEvent("ConsistencyScanInfoFetcher").log();
+	return val.get();
+}
+
 // constructs the cluster section of the json status output
 ACTOR Future<StatusReply> clusterGetStatus(
     Reference<AsyncVar<ServerDBInfo>> db,
@@ -2880,6 +2933,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 	state WorkerDetails ccWorker; // Cluster-Controller worker
 	state WorkerDetails ddWorker; // DataDistributor worker
 	state WorkerDetails rkWorker; // Ratekeeper worker
+	state WorkerDetails csWorker; // ConsistencyScan worker
 
 	try {
 		// Get the master Worker interface
@@ -2926,6 +2980,19 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			rkWorker = _rkWorker.get();
 		}
 
+		// Get the ConsistencyScan worker interface
+		Optional<WorkerDetails> _csWorker;
+		if (db->get().consistencyScan.present()) {
+			_csWorker = getWorker(workers, db->get().consistencyScan.get().address());
+		}
+
+		if (!db->get().consistencyScan.present() || !_csWorker.present()) {
+			messages.push_back(JsonString::makeMessage("unreachable_consistencyScan_worker",
+			                                           "Unable to locate the consistencyScan worker."));
+		} else {
+			csWorker = _csWorker.get();
+		}
+
 		// Get latest events for various event types from ALL workers
 		// WorkerEvents is a map of worker's NetworkAddress to its event string
 		// The pair represents worker responses and a set of worker NetworkAddress strings which did not respond.
@@ -3011,6 +3078,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 		state Optional<DatabaseConfiguration> configuration;
 		state Optional<LoadConfigurationResult> loadResult;
+		state std::unordered_map<NetworkAddress, WorkerInterface> address_workers;
 
 		if (statusCode != RecoveryStatus::configuration_missing) {
 			std::pair<Optional<DatabaseConfiguration>, Optional<LoadConfigurationResult>> loadResults =
@@ -3041,6 +3109,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 		statusObj["machines"] = machineStatusFetcher(mMetrics, workers, configuration, &status_incomplete_reasons);
 
+		state std::vector<NetworkAddress> coordinatorAddresses;
 		if (configuration.present()) {
 			// Do the latency probe by itself to avoid interference from other status activities
 			state bool isAvailable = true;
@@ -3063,7 +3132,6 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			// Start getting storage servers now (using system priority) concurrently.  Using sys priority because
 			// having storage servers in status output is important to give context to error messages in status that
 			// reference a storage server role ID.
-			state std::unordered_map<NetworkAddress, WorkerInterface> address_workers;
 			for (auto const& worker : workers) {
 				address_workers[worker.interf.address()] = worker.interf;
 			}
@@ -3133,8 +3201,9 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			state std::vector<JsonBuilderObject> workerStatuses = wait(getAll(futures2));
 			wait(success(primaryDCFO));
 
-			std::vector<NetworkAddress> coordinatorAddresses =
-			    wait(coordinators.ccr->getConnectionString().tryResolveHostnames());
+			std::vector<NetworkAddress> addresses =
+			    wait(timeoutError(coordinators.ccr->getConnectionString().tryResolveHostnames(), 5.0));
+			coordinatorAddresses = std::move(addresses);
 
 			int logFaultTolerance = 100;
 			if (db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
@@ -3275,6 +3344,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		                              grvProxies,
 		                              blobWorkers,
 		                              coordinators,
+		                              coordinatorAddresses,
 		                              cx,
 		                              configuration,
 		                              loadResult.present() ? loadResult.get().healthyZone : Optional<Key>(),
@@ -3282,6 +3352,12 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		statusObj["processes"] = processStatus;
 		statusObj["clients"] = clientStatusFetcher(clientStatus);
 
+		if (configuration.present() && configuration.get().blobGranulesEnabled) {
+			JsonBuilderObject blobGranuelsStatus =
+			    wait(blobWorkerStatusFetcher(blobWorkers, address_workers, &status_incomplete_reasons));
+			statusObj["blob_granules"] = blobGranuelsStatus;
+		}
+
 		JsonBuilderArray incompatibleConnectionsArray;
 		for (auto it : incompatibleConnections) {
 			incompatibleConnectionsArray.push_back(it.toString());
@@ -3331,6 +3407,26 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			messages.push_back(clientIssueMessage);
 		}
 
+		// Fetch Consistency Scan Information
+		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		state Optional<Value> val;
+		loop {
+			try {
+				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				wait(store(val, ConsistencyScanInfo::getInfo(tr)));
+				wait(tr->commit());
+				break;
+			} catch (Error& e) {
+				wait(tr->onError(e));
+			}
+		}
+		if (val.present()) {
+			ConsistencyScanInfo consistencyScanInfo =
+			    ObjectReader::fromStringRef<ConsistencyScanInfo>(val.get(), IncludeVersion());
+			TraceEvent("StatusConsistencyScanGotVal").log();
+			statusObj["consistency_scan_info"] = consistencyScanInfo.toJSON();
+		}
+
 		// Create the status_incomplete message if there were any reasons that the status is incomplete.
 		if (!status_incomplete_reasons.empty()) {
 			JsonBuilderObject incomplete_message =
diff --git a/fdbserver/StorageCache.actor.cpp b/fdbserver/StorageCache.actor.cpp
index 46e2c90a3a..0ac15840d2 100644
--- a/fdbserver/StorageCache.actor.cpp
+++ b/fdbserver/StorageCache.actor.cpp
@@ -18,13 +18,14 @@
  * limitations under the License.
  */
 
+#include "fdbclient/BlobCipher.h"
+#include "fdbclient/FDBTypes.h"
 #include "fdbserver/OTELSpanContextMessage.h"
 #include "flow/Arena.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/EncryptedMutationMessage.h"
-#include "fdbserver/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/ServerDBInfo.h"
 #include "fdbclient/StorageServerInterface.h"
@@ -247,9 +248,9 @@ public:
 	    lastTLogVersion(0), lastVersionWithData(0), peekVersion(0), compactionInProgress(Void()),
 	    fetchKeysParallelismLock(SERVER_KNOBS->FETCH_KEYS_PARALLELISM_BYTES), debug_inApplyUpdate(false),
 	    debug_lastValidateTime(0), versionLag(0), behind(false), counters(this) {
-		version.initMetric(LiteralStringRef("StorageCacheData.Version"), counters.cc.id);
-		desiredOldestVersion.initMetric(LiteralStringRef("StorageCacheData.DesriedOldestVersion"), counters.cc.id);
-		oldestVersion.initMetric(LiteralStringRef("StorageCacheData.OldestVersion"), counters.cc.id);
+		version.initMetric("StorageCacheData.Version"_sr, counters.cc.id);
+		desiredOldestVersion.initMetric("StorageCacheData.DesriedOldestVersion"_sr, counters.cc.id);
+		oldestVersion.initMetric("StorageCacheData.OldestVersion"_sr, counters.cc.id);
 
 		newestAvailableVersion.insert(allKeys, invalidVersion);
 		newestDirtyVersion.insert(allKeys, invalidVersion);
@@ -282,7 +283,7 @@ public:
 	void checkChangeCounter(uint64_t oldCacheRangeChangeCounter, KeyRef const& key) {
 		if (oldCacheRangeChangeCounter != cacheRangeChangeCounter &&
 		    cachedRangeMap[key]->changeCounter > oldCacheRangeChangeCounter) {
-			TEST(true); // CacheRange change during getValueQ
+			// CODE_PROBE(true, "CacheRange change during getValueQ");
 			// TODO: should we throw the cold_cache_server() error here instead?
 			throw wrong_shard_server();
 		}
@@ -293,8 +294,8 @@ public:
 			auto sh = cachedRangeMap.intersectingRanges(keys);
 			for (auto i = sh.begin(); i != sh.end(); ++i)
 				if (i->value()->changeCounter > oldCacheRangeChangeCounter) {
-					TEST(true); // CacheRange change during range operation
-					// TODO: should we throw the cold_cache_server() error here instead?
+					// CODE_PROBE(true, "CacheRange change during range operation");
+					//  TODO: should we throw the cold_cache_server() error here instead?
 					throw wrong_shard_server();
 				}
 		}
@@ -455,9 +456,7 @@ ACTOR Future<Version> waitForVersionNoTooOld(StorageCacheData* data, Version ver
 	if (version <= data->version.get())
 		return version;
 	choose {
-		when(wait(data->version.whenAtLeast(version))) {
-			return version;
-		}
+		when(wait(data->version.whenAtLeast(version))) { return version; }
 		when(wait(delay(SERVER_KNOBS->FUTURE_VERSION_DELAY))) {
 			if (deterministicRandom()->random01() < 0.001)
 				TraceEvent(SevWarn, "CacheServerFutureVersion1000x", data->thisServerID)
@@ -474,7 +473,6 @@ ACTOR Future<Void> getValueQ(StorageCacheData* data, GetValueRequest req) {
 	try {
 		++data->counters.getValueQueries;
 		++data->counters.allQueries;
-		//++data->readQueueSizeMetric;
 		// TODO later
 		// data->maxQueryQueue = std::max<int>( data->maxQueryQueue, data->counters.allQueries.getValue() -
 		// data->counters.finishedQueries.getValue());
@@ -484,18 +482,18 @@ ACTOR Future<Void> getValueQ(StorageCacheData* data, GetValueRequest req) {
 		// TODO what's this?
 		wait(delay(0, TaskPriority::DefaultEndpoint));
 
-		if (req.debugID.present()) {
+		if (req.options.present() && req.options.get().debugID.present()) {
 			g_traceBatch.addEvent("GetValueDebug",
-			                      req.debugID.get().first(),
+			                      req.options.get().debugID.get().first(),
 			                      "getValueQ.DoRead"); //.detail("TaskID", g_network->getCurrentTask());
 			// FIXME
 		}
 
 		state Optional<Value> v;
 		state Version version = wait(waitForVersion(data, req.version));
-		if (req.debugID.present())
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("GetValueDebug",
-			                      req.debugID.get().first(),
+			                      req.options.get().debugID.get().first(),
 			                      "getValueQ.AfterVersion"); //.detail("TaskID", g_network->getCurrentTask());
 
 		state uint64_t changeCounter = data->cacheRangeChangeCounter;
@@ -519,9 +517,9 @@ ACTOR Future<Void> getValueQ(StorageCacheData* data, GetValueRequest req) {
 		}
 
 		// DEBUG_MUTATION("CacheGetValue", version, MutationRef(MutationRef::DebugKey, req.key,
-		// v.present()?v.get():LiteralStringRef("<null>"))); DEBUG_MUTATION("CacheGetPath", version,
+		// v.present()?v.get():"<null>"_sr)); DEBUG_MUTATION("CacheGetPath", version,
 		// MutationRef(MutationRef::DebugKey, req.key,
-		// path==0?LiteralStringRef("0"):path==1?LiteralStringRef("1"):LiteralStringRef("2")));
+		// path==0?"0"_sr:path==1?"1"_sr:"2"_sr));
 
 		if (v.present()) {
 			++data->counters.rowsQueried;
@@ -530,9 +528,9 @@ ACTOR Future<Void> getValueQ(StorageCacheData* data, GetValueRequest req) {
 			//TraceEvent(SevDebug, "SCGetValueQPresent", data->thisServerID).detail("ResultSize",resultSize).detail("Version", version).detail("ReqKey",req.key).detail("Value",v);
 		}
 
-		if (req.debugID.present())
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("GetValueDebug",
-			                      req.debugID.get().first(),
+			                      req.options.get().debugID.get().first(),
 			                      "getValueQ.AfterRead"); //.detail("TaskID", g_network->getCurrentTask());
 
 		GetValueReply reply(v, true);
@@ -546,7 +544,6 @@ ACTOR Future<Void> getValueQ(StorageCacheData* data, GetValueRequest req) {
 	}
 
 	++data->counters.finishedQueries;
-	//--data->readQueueSizeMetric;
 	// if(data->latencyBandConfig.present()) {
 	//	int maxReadBytes =
 	// data->latencyBandConfig.get().readConfig.maxReadBytes.orDefault(std::numeric_limits<int>::max());
@@ -667,7 +664,7 @@ Key findKey(StorageCacheData* data, KeySelectorRef sel, Version version, KeyRang
 	// If we get only one result in the reverse direction as a result of the data being too large, we could get stuck in
 	// a loop
 	if (more && !forward && rep.data.size() == 1) {
-		TEST(true); // Reverse key selector returned only one result in range read
+		// CODE_PROBE(true, "Reverse key selector returned only one result in range read");
 		maxBytes = std::numeric_limits<int>::max();
 		GetKeyValuesReply rep2 =
 		    readRange(data, version, KeyRangeRef(range.begin, keyAfter(sel.getKey())), -2, &maxBytes);
@@ -690,7 +687,7 @@ Key findKey(StorageCacheData* data, KeySelectorRef sel, Version version, KeyRang
 			*pOffset = -*pOffset;
 
 		if (more) {
-			TEST(true); // Key selector read range had more results
+			// CODE_PROBE(true, "Key selector read range had more results");
 
 			ASSERT(rep.data.size());
 			Key returnKey = forward ? keyAfter(rep.data.back().key) : rep.data.back().key;
@@ -730,14 +727,13 @@ ACTOR Future<Void> getKeyValues(StorageCacheData* data, GetKeyValuesRequest req)
 	++data->counters.getRangeQueries;
 	++data->counters.allQueries;
 	// printf("\nSCGetKeyValues\n");
-	//++data->readQueueSizeMetric;
 	// data->maxQueryQueue = std::max<int>( data->maxQueryQueue, data->counters.allQueries.getValue() -
 	// data->counters.finishedQueries.getValue());
 
 	// Active load balancing runs at a very high priority (to obtain accurate queue lengths)
 	// so we need to downgrade here
 	TaskPriority taskType = TaskPriority::DefaultEndpoint;
-	if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.isFetchKeys) {
+	if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.options.present() && req.options.get().type == ReadType::FETCH) {
 		taskType = TaskPriority::FetchKeys;
 		// } else if (false) {
 		// 	// Placeholder for up-prioritizing fetches for important requests
@@ -746,17 +742,18 @@ ACTOR Future<Void> getKeyValues(StorageCacheData* data, GetKeyValuesRequest req)
 	wait(delay(0, taskType));
 
 	try {
-		if (req.debugID.present())
-			g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storagecache.getKeyValues.Before");
+		if (req.options.present() && req.options.get().debugID.present())
+			g_traceBatch.addEvent(
+			    "TransactionDebug", req.options.get().debugID.get().first(), "storagecache.getKeyValues.Before");
 		state Version version = wait(waitForVersion(data, req.version));
 
 		state uint64_t changeCounter = data->cacheRangeChangeCounter;
 
 		state KeyRange cachedKeyRange = getCachedKeyRange(data, req.begin);
 
-		if (req.debugID.present())
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent(
-			    "TransactionDebug", req.debugID.get().first(), "storagecache.getKeyValues.AfterVersion");
+			    "TransactionDebug", req.options.get().debugID.get().first(), "storagecache.getKeyValues.AfterVersion");
 		//.detail("CacheRangeBegin", cachedKeyRange.begin).detail("CacheRangeEnd", cachedKeyRange.end);
 
 		if (!selectorInRange(req.end, cachedKeyRange) &&
@@ -774,8 +771,9 @@ ACTOR Future<Void> getKeyValues(StorageCacheData* data, GetKeyValuesRequest req)
 		                      : findKey(data, req.begin, version, cachedKeyRange, &offset1);
 		state Key end = req.end.isFirstGreaterOrEqual() ? req.end.getKey()
 		                                                : findKey(data, req.end, version, cachedKeyRange, &offset2);
-		if (req.debugID.present())
-			g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storagecache.getKeyValues.AfterKeys");
+		if (req.options.present() && req.options.get().debugID.present())
+			g_traceBatch.addEvent(
+			    "TransactionDebug", req.options.get().debugID.get().first(), "storagecache.getKeyValues.AfterKeys");
 		//.detail("Off1",offset1).detail("Off2",offset2).detail("ReqBegin",req.begin.getKey()).detail("ReqEnd",req.end.getKey());
 
 		// Offsets of zero indicate begin/end keys in this cachedKeyRange, which obviously means we can answer the query
@@ -783,11 +781,11 @@ ACTOR Future<Void> getKeyValues(StorageCacheData* data, GetKeyValuesRequest req)
 		// cachedKeyRange is the end the last actual key returned must be from this cachedKeyRange. A begin offset of 1
 		// is also OK because then either begin is past end or equal to end (so the result is definitely empty)
 		if ((offset1 && offset1 != 1) || (offset2 && offset2 != 1)) {
-			TEST(true); // wrong_cache_server due to offset
-			// We could detect when offset1 takes us off the beginning of the database or offset2 takes us off the end,
-			// and return a clipped range rather than an error (since that is what the NativeAPI.getRange will do anyway
-			// via its "slow path"), but we would have to add some flags to the response to encode whether we went off
-			// the beginning and the end, since it needs that information.
+			// CODE_PROBE(true, "wrong_cache_server due to offset");
+			//  We could detect when offset1 takes us off the beginning of the database or offset2 takes us off the end,
+			//  and return a clipped range rather than an error (since that is what the NativeAPI.getRange will do
+			//  anyway via its "slow path"), but we would have to add some flags to the response to encode whether we
+			//  went off the beginning and the end, since it needs that information.
 			//TraceEvent(SevDebug, "WrongCacheRangeServer2", data->thisServerID).detail("Begin", req.begin.toString()).detail("End", req.end.toString()).detail("Version", version).
 			// detail("CacheRangeBegin", cachedKeyRange.begin).detail("CacheRangeEnd", cachedKeyRange.end).detail("In",
 			// "getKeyValues>checkOffsets"). detail("BeginKey", begin).detail("EndKey", end).detail("BeginOffset",
@@ -800,8 +798,9 @@ ACTOR Future<Void> getKeyValues(StorageCacheData* data, GetKeyValuesRequest req)
 		// offset1).detail("EndOffset", offset2);
 
 		if (begin >= end) {
-			if (req.debugID.present())
-				g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storagecache.getKeyValues.Send");
+			if (req.options.present() && req.options.get().debugID.present())
+				g_traceBatch.addEvent(
+				    "TransactionDebug", req.options.get().debugID.get().first(), "storagecache.getKeyValues.Send");
 			//.detail("Begin",begin).detail("End",end);
 
 			GetKeyValuesReply none;
@@ -817,9 +816,10 @@ ACTOR Future<Void> getKeyValues(StorageCacheData* data, GetKeyValuesRequest req)
 			GetKeyValuesReply _r = readRange(data, version, KeyRangeRef(begin, end), req.limit, &remainingLimitBytes);
 			GetKeyValuesReply r = _r;
 
-			if (req.debugID.present())
-				g_traceBatch.addEvent(
-				    "TransactionDebug", req.debugID.get().first(), "storagecache.getKeyValues.AfterReadRange");
+			if (req.options.present() && req.options.get().debugID.present())
+				g_traceBatch.addEvent("TransactionDebug",
+				                      req.options.get().debugID.get().first(),
+				                      "storagecache.getKeyValues.AfterReadRange");
 			data->checkChangeCounter(
 			    changeCounter,
 			    KeyRangeRef(std::min<KeyRef>(begin, std::min<KeyRef>(req.begin.getKey(), req.end.getKey())),
@@ -945,7 +945,7 @@ bool expandMutation(MutationRef& m, StorageCacheData::VersionedData const& data,
 		if (it != data.atLatest().end() && it->isValue() && it.key() == m.param1)
 			oldVal = it->getValue();
 		else if (it != data.atLatest().end() && it->isClearTo() && it->getEndKey() > m.param1) {
-			TEST(true); // Atomic op right after a clear.
+			// CODE_PROBE(true, "Atomic op right after a clear.");
 		}
 
 		switch (m.type) {
@@ -1075,7 +1075,7 @@ void splitMutation(StorageCacheData* data, KeyRangeMap<T>& map, MutationRef cons
 }
 
 void rollback(StorageCacheData* data, Version rollbackVersion, Version nextVersion) {
-	TEST(true); // call to cacheRange rollback
+	// CODE_PROBE(true, "call to cacheRange rollback");
 	// FIXME: enable when debugKeyRange is active
 	// debugKeyRange("Rollback", rollbackVersion, allKeys);
 
@@ -1188,6 +1188,7 @@ ACTOR Future<RangeResult> tryFetchRange(Database cx,
 	state RangeResult output;
 	state KeySelectorRef begin = firstGreaterOrEqual(keys.begin);
 	state KeySelectorRef end = firstGreaterOrEqual(keys.end);
+	state ReadOptions options = ReadOptions(Optional<UID>(), ReadType::FETCH);
 
 	if (*isTooOld)
 		throw transaction_too_old();
@@ -1195,6 +1196,7 @@ ACTOR Future<RangeResult> tryFetchRange(Database cx,
 	ASSERT(!cx->switchable);
 	tr.setVersion(version);
 	tr.trState->taskID = TaskPriority::FetchKeys;
+	tr.trState->readOptions = options;
 	limits.minRows = 0;
 
 	try {
@@ -1281,7 +1283,7 @@ ACTOR Future<Void> fetchKeys(StorageCacheData* data, AddingCacheRange* cacheRang
 			lastAvailable = std::max(lastAvailable, r->value());
 
 		if (lastAvailable != invalidVersion && lastAvailable >= data->oldestVersion.get()) {
-			TEST(true); // wait for oldest version
+			// CODE_PROBE(true, "wait for oldest version");
 			wait(data->oldestVersion.whenAtLeast(lastAvailable + 1));
 		}
 
@@ -1320,7 +1322,7 @@ ACTOR Future<Void> fetchKeys(StorageCacheData* data, AddingCacheRange* cacheRang
 
 		loop {
 			try {
-				TEST(true); // Fetching keys for transferred cacheRange
+				// CODE_PROBE(true, "Fetching keys for transferred cacheRange");
 
 				state RangeResult this_block =
 				    wait(tryFetchRange(data->cx,
@@ -1384,7 +1386,7 @@ ACTOR Future<Void> fetchKeys(StorageCacheData* data, AddingCacheRange* cacheRang
 				    .suppressFor(1.0)
 				    .detail("FKID", interval.pairID);
 				if (e.code() == error_code_transaction_too_old) {
-					TEST(true); // A storage server has forgotten the history data we are fetching
+					// CODE_PROBE(true, "A storage server has forgotten the history data we are fetching");
 					Version lastFV = fetchVersion;
 					fetchVersion = data->version.get();
 					isTooOld = false;
@@ -1411,8 +1413,9 @@ ACTOR Future<Void> fetchKeys(StorageCacheData* data, AddingCacheRange* cacheRang
 						    .detail("E", data->version.get());
 					}
 				} else if (e.code() == error_code_future_version || e.code() == error_code_process_behind) {
-					TEST(true); // fetchKeys got future_version or process_behind, so there must be a huge storage lag
-					            // somewhere.  Keep trying.
+					// CODE_PROBE(true,
+					//           "fetchKeys got future_version or process_behind, so there must be a huge storage lag "
+					//           "somewhere.  Keep trying.");
 				} else {
 					throw;
 				}
@@ -1472,7 +1475,7 @@ ACTOR Future<Void> fetchKeys(StorageCacheData* data, AddingCacheRange* cacheRang
 		}
 
 		int startSize = batch->changes.size();
-		TEST(startSize); // Adding fetch data to a batch which already has changes
+		// CODE_PROBE(startSize, "Adding fetch data to a batch which already has changes");
 		batch->changes.resize(batch->changes.size() + cacheRange->updates.size());
 
 		// FIXME: pass the deque back rather than copy the data
@@ -1635,7 +1638,7 @@ void cacheWarmup(StorageCacheData* data, const KeyRangeRef& keys, bool nowAssign
 		else {
 			ASSERT(ranges[i].value->adding);
 			data->addCacheRange(CacheRangeInfo::newAdding(data, ranges[i]));
-			TEST(true); // cacheWarmup reFetchKeys
+			// CODE_PROBE(true, "cacheWarmup reFetchKeys");
 		}
 	}
 
@@ -1774,7 +1777,7 @@ private:
 			br >> rollbackVersion;
 
 			if (rollbackVersion < fromVersion && rollbackVersion > data->oldestVersion.get()) {
-				TEST(true); // CacheRangeApplyPrivateData cacheRange rollback
+				// CODE_PROBE(true, "CacheRangeApplyPrivateData cacheRange rollback");
 				TraceEvent(SevWarn, "Rollback", data->thisServerID)
 				    .detail("FromVersion", fromVersion)
 				    .detail("ToVersion", rollbackVersion)
@@ -1799,7 +1802,7 @@ ACTOR Future<Void> compactCache(StorageCacheData* data) {
 	loop {
 		// TODO understand this, should we add delay here?
 		// if (g_network->isSimulated()) {
-		//	double endTime = g_simulator.checkDisabled(format("%s/compactCache",
+		//	double endTime = g_simulator->checkDisabled(format("%s/compactCache",
 		// data->thisServerID.toString().c_str())); 	if(endTime > now()) { 		wait(delay(endTime - now(),
 		// TaskPriority::CompactCache));
 		//	}
@@ -1845,9 +1848,7 @@ ACTOR Future<Void> pullAsyncData(StorageCacheData* data) {
 
 	loop {
 		loop choose {
-			when(wait(cursor ? cursor->getMore(TaskPriority::TLogCommit) : Never())) {
-				break;
-			}
+			when(wait(cursor ? cursor->getMore(TaskPriority::TLogCommit) : Never())) { break; }
 			when(wait(dbInfoChange)) {
 				dbInfoChange = data->db->onChange();
 				if (data->db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
@@ -1915,25 +1916,19 @@ ACTOR Future<Void> pullAsyncData(StorageCacheData* data) {
 					           OTELSpanContextMessage::isNextIn(cloneReader)) {
 						OTELSpanContextMessage scm;
 						cloneReader >> scm;
-					} else if (cloneReader.protocolVersion().hasEncryptionAtRest() &&
-					           EncryptedMutationMessage::isNextIn(cloneReader) && !cipherKeys.present()) {
-						// Encrypted mutation found, but cipher keys haven't been fetch.
-						// Collect cipher details to fetch cipher keys in one batch.
-						EncryptedMutationMessage emm;
-						cloneReader >> emm;
-						cipherDetails.insert(emm.header.cipherTextDetails);
-						cipherDetails.insert(emm.header.cipherHeaderDetails);
-						collectingCipherKeys = true;
 					} else {
 						MutationRef msg;
-						if (cloneReader.protocolVersion().hasEncryptionAtRest() &&
-						    EncryptedMutationMessage::isNextIn(cloneReader)) {
-							assert(cipherKeys.present());
-							msg = EncryptedMutationMessage::decrypt(cloneReader, cloneReader.arena(), cipherKeys.get());
-						} else {
-							cloneReader >> msg;
+						cloneReader >> msg;
+						if (msg.isEncrypted()) {
+							if (!cipherKeys.present()) {
+								const BlobCipherEncryptHeader* header = msg.encryptionHeader();
+								cipherDetails.insert(header->cipherTextDetails);
+								cipherDetails.insert(header->cipherHeaderDetails);
+								collectingCipherKeys = true;
+							} else {
+								msg = msg.decrypt(cipherKeys.get(), cloneReader.arena(), BlobCipherMetrics::TLOG);
+							}
 						}
-
 						if (!collectingCipherKeys) {
 							if (firstMutation && msg.param1.startsWith(systemKeys.end))
 								hasPrivateData = true;
@@ -1952,7 +1947,7 @@ ACTOR Future<Void> pullAsyncData(StorageCacheData* data) {
 
 				if (collectingCipherKeys) {
 					std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> result =
-					    wait(getEncryptCipherKeys(data->db, cipherDetails));
+					    wait(getEncryptCipherKeys(data->db, cipherDetails, BlobCipherMetrics::TLOG));
 					cipherKeys = result;
 					collectingCipherKeys = false;
 				} else {
@@ -1966,8 +1961,10 @@ ACTOR Future<Void> pullAsyncData(StorageCacheData* data) {
 					}
 					if (data->cacheRangeChangeCounter == changeCounter)
 						break;
-					// TEST(true); // A fetchKeys completed while we were doing this, so eager might be outdated.  Read
-					// it again.
+					// CODE_PROBE(
+					//     true,
+					//     "A fetchKeys completed while we were doing this, so eager might be outdated. Read it
+					//     again.");
 				}
 			}
 
@@ -2018,15 +2015,14 @@ ACTOR Future<Void> pullAsyncData(StorageCacheData* data) {
 					SpanContextMessage scm;
 					reader >> scm;
 				} else if (reader.protocolVersion().hasOTELSpanContext() && OTELSpanContextMessage::isNextIn(reader)) {
-					TEST(true); // StorageCache reading OTELSpanContextMessage
+					// CODE_PROBE(true, "StorageCache reading OTELSpanContextMessage");
 					OTELSpanContextMessage oscm;
 					reader >> oscm;
 				} else {
 					MutationRef msg;
-					if (reader.protocolVersion().hasEncryptionAtRest() && EncryptedMutationMessage::isNextIn(reader)) {
-						msg = EncryptedMutationMessage::decrypt(reader, reader.arena(), cipherKeys.get());
-					} else {
-						reader >> msg;
+					reader >> msg;
+					if (msg.isEncrypted()) {
+						msg = msg.decrypt(cipherKeys.get(), reader.arena(), BlobCipherMetrics::TLOG);
 					}
 
 					if (ver != invalidVersion) // This change belongs to a version < minVersion
@@ -2255,21 +2251,13 @@ ACTOR Future<Void> storageCacheServer(StorageServerInterface ssi,
 				// actors.add(self->readGuard(req , getValueQ));
 				actors.add(getValueQ(&self, req));
 			}
-			when(WatchValueRequest req = waitNext(ssi.watchValue.getFuture())) {
-				ASSERT(false);
-			}
-			when(GetKeyRequest req = waitNext(ssi.getKey.getFuture())) {
-				actors.add(getKey(&self, req));
-			}
+			when(WatchValueRequest req = waitNext(ssi.watchValue.getFuture())) { ASSERT(false); }
+			when(GetKeyRequest req = waitNext(ssi.getKey.getFuture())) { actors.add(getKey(&self, req)); }
 			when(GetKeyValuesRequest req = waitNext(ssi.getKeyValues.getFuture())) {
 				actors.add(getKeyValues(&self, req));
 			}
-			when(GetShardStateRequest req = waitNext(ssi.getShardState.getFuture())) {
-				ASSERT(false);
-			}
-			when(StorageQueuingMetricsRequest req = waitNext(ssi.getQueuingMetrics.getFuture())) {
-				ASSERT(false);
-			}
+			when(GetShardStateRequest req = waitNext(ssi.getShardState.getFuture())) { ASSERT(false); }
+			when(StorageQueuingMetricsRequest req = waitNext(ssi.getQueuingMetrics.getFuture())) { ASSERT(false); }
 			// when( ReplyPromise<Version> reply = waitNext(ssi.getVersion.getFuture()) ) {
 			//	ASSERT(false);
 			//}
@@ -2277,39 +2265,21 @@ ACTOR Future<Void> storageCacheServer(StorageServerInterface ssi,
 				ASSERT(false);
 			}
 
-			when(GetMappedKeyValuesRequest req = waitNext(ssi.getMappedKeyValues.getFuture())) {
-				ASSERT(false);
-			}
-			when(WaitMetricsRequest req = waitNext(ssi.waitMetrics.getFuture())) {
-				ASSERT(false);
-			}
-			when(SplitMetricsRequest req = waitNext(ssi.splitMetrics.getFuture())) {
-				ASSERT(false);
-			}
-			when(GetStorageMetricsRequest req = waitNext(ssi.getStorageMetrics.getFuture())) {
-				ASSERT(false);
-			}
-			when(ReadHotSubRangeRequest req = waitNext(ssi.getReadHotRanges.getFuture())) {
-				ASSERT(false);
-			}
-			when(SplitRangeRequest req = waitNext(ssi.getRangeSplitPoints.getFuture())) {
-				ASSERT(false);
-			}
-			when(GetKeyValuesStreamRequest req = waitNext(ssi.getKeyValuesStream.getFuture())) {
-				ASSERT(false);
-			}
-			when(ChangeFeedStreamRequest req = waitNext(ssi.changeFeedStream.getFuture())) {
-				ASSERT(false);
-			}
+			when(GetMappedKeyValuesRequest req = waitNext(ssi.getMappedKeyValues.getFuture())) { ASSERT(false); }
+			when(WaitMetricsRequest req = waitNext(ssi.waitMetrics.getFuture())) { ASSERT(false); }
+			when(SplitMetricsRequest req = waitNext(ssi.splitMetrics.getFuture())) { ASSERT(false); }
+			when(GetStorageMetricsRequest req = waitNext(ssi.getStorageMetrics.getFuture())) { ASSERT(false); }
+			when(ReadHotSubRangeRequest req = waitNext(ssi.getReadHotRanges.getFuture())) { ASSERT(false); }
+			when(SplitRangeRequest req = waitNext(ssi.getRangeSplitPoints.getFuture())) { ASSERT(false); }
+			when(GetKeyValuesStreamRequest req = waitNext(ssi.getKeyValuesStream.getFuture())) { ASSERT(false); }
+			when(ChangeFeedStreamRequest req = waitNext(ssi.changeFeedStream.getFuture())) { ASSERT(false); }
 			when(OverlappingChangeFeedsRequest req = waitNext(ssi.overlappingChangeFeeds.getFuture())) {
 				// Simulate endpoint not found so that the requester will try another endpoint
 				// This is a workaround to the fact that storage servers do not have an easy way to enforce this
 				// request goes only to other storage servers, and in simulation we manage to trigger this behavior
 				req.reply.sendError(broken_promise());
 			}
-			when(ChangeFeedPopRequest req = waitNext(ssi.changeFeedPop.getFuture())) {
-				ASSERT(false);
-			}
+			when(ChangeFeedPopRequest req = waitNext(ssi.changeFeedPop.getFuture())) { ASSERT(false); }
 			when(ChangeFeedVersionUpdateRequest req = waitNext(ssi.changeFeedVersionUpdate.getFuture())) {
 				ASSERT(false);
 			}
diff --git a/fdbserver/StorageMetrics.actor.cpp b/fdbserver/StorageMetrics.actor.cpp
index d559380f16..16efb5411a 100644
--- a/fdbserver/StorageMetrics.actor.cpp
+++ b/fdbserver/StorageMetrics.actor.cpp
@@ -22,20 +22,551 @@
 #include "fdbserver/StorageMetrics.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+int64_t StorageMetricSample::getEstimate(KeyRangeRef keys) const {
+	return sample.sumRange(keys.begin, keys.end);
+}
+
+KeyRef StorageMetricSample::splitEstimate(KeyRangeRef range, int64_t offset, bool front) const {
+	auto fwd_split = sample.index(front ? sample.sumTo(sample.lower_bound(range.begin)) + offset
+	                                    : sample.sumTo(sample.lower_bound(range.end)) - offset);
+
+	if (fwd_split == sample.end() || *fwd_split >= range.end)
+		return range.end;
+
+	if (!front && *fwd_split <= range.begin)
+		return range.begin;
+
+	auto bck_split = fwd_split;
+
+	// Butterfly search - start at midpoint then go in both directions.
+	while ((fwd_split != sample.end() && *fwd_split < range.end) ||
+	       (bck_split != sample.begin() && *bck_split > range.begin)) {
+		if (bck_split != sample.begin() && *bck_split > range.begin) {
+			auto it = bck_split;
+			bck_split.decrementNonEnd();
+
+			KeyRef split = keyBetween(KeyRangeRef(
+			    bck_split != sample.begin() ? std::max<KeyRef>(*bck_split, range.begin) : range.begin, *it));
+			if (!front || (getEstimate(KeyRangeRef(range.begin, split)) > 0 &&
+			               split.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT))
+				return split;
+		}
+
+		if (fwd_split != sample.end() && *fwd_split < range.end) {
+			auto it = fwd_split;
+			++it;
+
+			KeyRef split =
+			    keyBetween(KeyRangeRef(*fwd_split, it != sample.end() ? std::min<KeyRef>(*it, range.end) : range.end));
+			if (front ||
+			    (getEstimate(KeyRangeRef(split, range.end)) > 0 && split.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT))
+				return split;
+
+			fwd_split = it;
+		}
+	}
+
+	// If we didn't return above, we didn't find anything.
+	TraceEvent(SevWarn, "CannotSplitLastSampleKey").detail("Range", range).detail("Offset", offset);
+	return front ? range.end : range.begin;
+}
+
+// Get the current estimated metrics for the given keys
+StorageMetrics StorageServerMetrics::getMetrics(KeyRangeRef const& keys) const {
+	StorageMetrics result;
+	result.bytes = byteSample.getEstimate(keys);
+	result.bytesPerKSecond =
+	    bandwidthSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+	result.iosPerKSecond = iopsSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+	result.bytesReadPerKSecond =
+	    bytesReadSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+	return result;
+}
+
+// Called when metrics should change (IO for a given key)
+// Notifies waiting WaitMetricsRequests through waitMetricsMap, and updates metricsAverageQueue and metricsSampleMap
+void StorageServerMetrics::notify(KeyRef key, StorageMetrics& metrics) {
+	ASSERT(metrics.bytes == 0); // ShardNotifyMetrics
+	if (g_network->isSimulated()) {
+		CODE_PROBE(metrics.bytesPerKSecond != 0, "ShardNotifyMetrics bytes");
+		CODE_PROBE(metrics.iosPerKSecond != 0, "ShardNotifyMetrics ios");
+		CODE_PROBE(metrics.bytesReadPerKSecond != 0, "ShardNotifyMetrics bytesRead");
+	}
+
+	double expire = now() + SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL;
+
+	StorageMetrics notifyMetrics;
+
+	if (metrics.bytesPerKSecond)
+		notifyMetrics.bytesPerKSecond = bandwidthSample.addAndExpire(key, metrics.bytesPerKSecond, expire) *
+		                                SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+	if (metrics.iosPerKSecond)
+		notifyMetrics.iosPerKSecond = iopsSample.addAndExpire(key, metrics.iosPerKSecond, expire) *
+		                              SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+	if (metrics.bytesReadPerKSecond)
+		notifyMetrics.bytesReadPerKSecond = bytesReadSample.addAndExpire(key, metrics.bytesReadPerKSecond, expire) *
+		                                    SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+	if (!notifyMetrics.allZero()) {
+		auto& v = waitMetricsMap[key];
+		for (int i = 0; i < v.size(); i++) {
+			if (g_network->isSimulated()) {
+				CODE_PROBE(true, "shard notify metrics");
+			}
+			// ShardNotifyMetrics
+			v[i].send(notifyMetrics);
+		}
+	}
+}
+
+// Due to the fact that read sampling will be called on all reads, use this specialized function to avoid overhead
+// around branch misses and unnecessary stack allocation which eventually addes up under heavy load.
+void StorageServerMetrics::notifyBytesReadPerKSecond(KeyRef key, int64_t in) {
+	double expire = now() + SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL;
+	int64_t bytesReadPerKSecond =
+	    bytesReadSample.addAndExpire(key, in, expire) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+	if (bytesReadPerKSecond > 0) {
+		StorageMetrics notifyMetrics;
+		notifyMetrics.bytesReadPerKSecond = bytesReadPerKSecond;
+		auto& v = waitMetricsMap[key];
+		for (int i = 0; i < v.size(); i++) {
+			CODE_PROBE(true, "ShardNotifyMetrics");
+			v[i].send(notifyMetrics);
+		}
+	}
+}
+
+// Called by StorageServerDisk when the size of a key in byteSample changes, to notify WaitMetricsRequest
+// Should not be called for keys past allKeys.end
+void StorageServerMetrics::notifyBytes(
+    RangeMap<Key, std::vector<PromiseStream<StorageMetrics>>, KeyRangeRef>::iterator shard,
+    int64_t bytes) {
+	ASSERT(shard.end() <= allKeys.end);
+
+	StorageMetrics notifyMetrics;
+	notifyMetrics.bytes = bytes;
+	for (int i = 0; i < shard.value().size(); i++) {
+		CODE_PROBE(true, "notifyBytes");
+		shard.value()[i].send(notifyMetrics);
+	}
+}
+
+// Called by StorageServerDisk when the size of a key in byteSample changes, to notify WaitMetricsRequest
+void StorageServerMetrics::notifyBytes(KeyRef key, int64_t bytes) {
+	if (key >= allKeys.end) // Do not notify on changes to internal storage server state
+		return;
+
+	notifyBytes(waitMetricsMap.rangeContaining(key), bytes);
+}
+
+// Called when a range of keys becomes unassigned (and therefore not readable), to notify waiting
+// WaitMetricsRequests (also other types of wait
+//   requests in the future?)
+void StorageServerMetrics::notifyNotReadable(KeyRangeRef keys) {
+	auto rs = waitMetricsMap.intersectingRanges(keys);
+	for (auto r = rs.begin(); r != rs.end(); ++r) {
+		auto& v = r->value();
+		CODE_PROBE(v.size(), "notifyNotReadable() sending errors to intersecting ranges");
+		for (int n = 0; n < v.size(); n++)
+			v[n].sendError(wrong_shard_server());
+	}
+}
+
+// Called periodically (~1 sec intervals) to remove older IOs from the averages
+// Removes old entries from metricsAverageQueue, updates metricsSampleMap accordingly, and notifies
+//   WaitMetricsRequests through waitMetricsMap.
+void StorageServerMetrics::poll() {
+	{
+		StorageMetrics m;
+		m.bytesPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+		bandwidthSample.poll(waitMetricsMap, m);
+	}
+	{
+		StorageMetrics m;
+		m.iosPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+		iopsSample.poll(waitMetricsMap, m);
+	}
+	{
+		StorageMetrics m;
+		m.bytesReadPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+		bytesReadSample.poll(waitMetricsMap, m);
+	}
+	// bytesSample doesn't need polling because we never call addExpire() on it
+}
+
+// This function can run on untrusted user data.  We must validate all divisions carefully.
+KeyRef StorageServerMetrics::getSplitKey(int64_t remaining,
+                                         int64_t estimated,
+                                         int64_t limits,
+                                         int64_t used,
+                                         int64_t infinity,
+                                         bool isLastShard,
+                                         const StorageMetricSample& sample,
+                                         double divisor,
+                                         KeyRef const& lastKey,
+                                         KeyRef const& key,
+                                         bool hasUsed) const {
+	ASSERT(remaining >= 0);
+	ASSERT(limits > 0);
+	ASSERT(divisor > 0);
+
+	if (limits < infinity / 2) {
+		int64_t expectedSize;
+		if (isLastShard || remaining > estimated) {
+			double remaining_divisor = (double(remaining) / limits) + 0.5;
+			expectedSize = remaining / remaining_divisor;
+		} else {
+			// If we are here, then estimated >= remaining >= 0
+			double estimated_divisor = (double(estimated) / limits) + 0.5;
+			expectedSize = remaining / estimated_divisor;
+		}
+
+		if (remaining > expectedSize) {
+			// This does the conversion from native units to bytes using the divisor.
+			double offset = (expectedSize - used) / divisor;
+			if (offset <= 0)
+				return hasUsed ? lastKey : key;
+			return sample.splitEstimate(
+			    KeyRangeRef(lastKey, key),
+			    offset * ((1.0 - SERVER_KNOBS->SPLIT_JITTER_AMOUNT) +
+			              2 * deterministicRandom()->random01() * SERVER_KNOBS->SPLIT_JITTER_AMOUNT));
+		}
+	}
+
+	return key;
+}
+
+void StorageServerMetrics::splitMetrics(SplitMetricsRequest req) const {
+	int minSplitBytes = req.minSplitBytes.present() ? req.minSplitBytes.get() : SERVER_KNOBS->MIN_SHARD_BYTES;
+	try {
+		SplitMetricsReply reply;
+		KeyRef lastKey = req.keys.begin;
+		StorageMetrics used = req.used;
+		StorageMetrics estimated = req.estimated;
+		StorageMetrics remaining = getMetrics(req.keys) + used;
+
+		//TraceEvent("SplitMetrics").detail("Begin", req.keys.begin).detail("End", req.keys.end).detail("Remaining", remaining.bytes).detail("Used", used.bytes).detail("MinSplitBytes", minSplitBytes);
+
+		while (true) {
+			if (remaining.bytes < 2 * minSplitBytes)
+				break;
+			KeyRef key = req.keys.end;
+			bool hasUsed = used.bytes != 0 || used.bytesPerKSecond != 0 || used.iosPerKSecond != 0;
+			key = getSplitKey(remaining.bytes,
+			                  estimated.bytes,
+			                  req.limits.bytes,
+			                  used.bytes,
+			                  req.limits.infinity,
+			                  req.isLastShard,
+			                  byteSample,
+			                  1,
+			                  lastKey,
+			                  key,
+			                  hasUsed);
+			if (used.bytes < minSplitBytes)
+				key = std::max(
+				    key, byteSample.splitEstimate(KeyRangeRef(lastKey, req.keys.end), minSplitBytes - used.bytes));
+			key = getSplitKey(remaining.iosPerKSecond,
+			                  estimated.iosPerKSecond,
+			                  req.limits.iosPerKSecond,
+			                  used.iosPerKSecond,
+			                  req.limits.infinity,
+			                  req.isLastShard,
+			                  iopsSample,
+			                  SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS,
+			                  lastKey,
+			                  key,
+			                  hasUsed);
+			key = getSplitKey(remaining.bytesPerKSecond,
+			                  estimated.bytesPerKSecond,
+			                  req.limits.bytesPerKSecond,
+			                  used.bytesPerKSecond,
+			                  req.limits.infinity,
+			                  req.isLastShard,
+			                  bandwidthSample,
+			                  SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS,
+			                  lastKey,
+			                  key,
+			                  hasUsed);
+			ASSERT(key != lastKey || hasUsed);
+			if (key == req.keys.end)
+				break;
+			reply.splits.push_back_deep(reply.splits.arena(), key);
+
+			StorageMetrics diff = (getMetrics(KeyRangeRef(lastKey, key)) + used);
+			remaining -= diff;
+			estimated -= diff;
+
+			used = StorageMetrics();
+			lastKey = key;
+		}
+
+		reply.used = getMetrics(KeyRangeRef(lastKey, req.keys.end)) + used;
+		req.reply.send(reply);
+	} catch (Error& e) {
+		req.reply.sendError(e);
+	}
+}
+
+void StorageServerMetrics::getStorageMetrics(GetStorageMetricsRequest req,
+                                             StorageBytes sb,
+                                             double bytesInputRate,
+                                             int64_t versionLag,
+                                             double lastUpdate) const {
+	GetStorageMetricsReply rep;
+
+	// SOMEDAY: make bytes dynamic with hard disk space
+	rep.load = getMetrics(allKeys);
+
+	if (sb.free < 1e9) {
+		TraceEvent(SevWarn, "PhysicalDiskMetrics")
+		    .suppressFor(60.0)
+		    .detail("Free", sb.free)
+		    .detail("Total", sb.total)
+		    .detail("Available", sb.available)
+		    .detail("Load", rep.load.bytes);
+	}
+
+	rep.available.bytes = sb.available;
+	rep.available.iosPerKSecond = 10e6;
+	rep.available.bytesPerKSecond = 100e9;
+	rep.available.bytesReadPerKSecond = 100e9;
+
+	rep.capacity.bytes = sb.total;
+	rep.capacity.iosPerKSecond = 10e6;
+	rep.capacity.bytesPerKSecond = 100e9;
+	rep.capacity.bytesReadPerKSecond = 100e9;
+
+	rep.bytesInputRate = bytesInputRate;
+
+	rep.versionLag = versionLag;
+	rep.lastUpdate = lastUpdate;
+
+	req.reply.send(rep);
+}
+
+// Given a read hot shard, this function will divide the shard into chunks and find those chunks whose
+// readBytes/sizeBytes exceeds the `readDensityRatio`. Please make sure to run unit tests
+// `StorageMetricsSampleTests.txt` after change made.
+std::vector<ReadHotRangeWithMetrics> StorageServerMetrics::getReadHotRanges(
+    KeyRangeRef shard,
+    double readDensityRatio,
+    int64_t baseChunkSize,
+    int64_t minShardReadBandwidthPerKSeconds) const {
+	std::vector<ReadHotRangeWithMetrics> toReturn;
+
+	double shardSize = (double)byteSample.getEstimate(shard);
+	int64_t shardReadBandwidth = bytesReadSample.getEstimate(shard);
+	if (shardReadBandwidth * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS <=
+	    minShardReadBandwidthPerKSeconds) {
+		return toReturn;
+	}
+	if (shardSize <= baseChunkSize) {
+		// Shard is small, use it as is
+		if (bytesReadSample.getEstimate(shard) > (readDensityRatio * shardSize)) {
+			toReturn.emplace_back(shard,
+			                      bytesReadSample.getEstimate(shard) / shardSize,
+			                      bytesReadSample.getEstimate(shard) / SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL);
+		}
+		return toReturn;
+	}
+	KeyRef beginKey = shard.begin;
+	auto endKey =
+	    byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + baseChunkSize);
+	while (endKey != byteSample.sample.end()) {
+		if (*endKey > shard.end) {
+			endKey = byteSample.sample.lower_bound(shard.end);
+			if (*endKey == beginKey) {
+				// No need to increment endKey since otherwise it would stuck here forever.
+				break;
+			}
+		}
+		if (*endKey == beginKey) {
+			++endKey;
+			continue;
+		}
+		if (bytesReadSample.getEstimate(KeyRangeRef(beginKey, *endKey)) >
+		    (readDensityRatio * std::max(baseChunkSize, byteSample.getEstimate(KeyRangeRef(beginKey, *endKey))))) {
+			auto range = KeyRangeRef(beginKey, *endKey);
+			if (!toReturn.empty() && toReturn.back().keys.end == range.begin) {
+				// in case two consecutive chunks both are over the ratio, merge them.
+				range = KeyRangeRef(toReturn.back().keys.begin, *endKey);
+				toReturn.pop_back();
+			}
+			toReturn.emplace_back(range,
+			                      (double)bytesReadSample.getEstimate(range) /
+			                          std::max(baseChunkSize, byteSample.getEstimate(range)),
+			                      bytesReadSample.getEstimate(range) / SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL);
+		}
+		beginKey = *endKey;
+		endKey =
+		    byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + baseChunkSize);
+	}
+	return toReturn;
+}
+
+void StorageServerMetrics::getReadHotRanges(ReadHotSubRangeRequest req) const {
+	ReadHotSubRangeReply reply;
+	auto _ranges = getReadHotRanges(req.keys,
+	                                SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO,
+	                                SERVER_KNOBS->READ_HOT_SUB_RANGE_CHUNK_SIZE,
+	                                SERVER_KNOBS->SHARD_READ_HOT_BANDWIDTH_MIN_PER_KSECONDS);
+	reply.readHotRanges = VectorRef(_ranges.data(), _ranges.size());
+	req.reply.send(reply);
+}
+
+void StorageServerMetrics::getSplitPoints(SplitRangeRequest req, Optional<Key> prefix) const {
+	SplitRangeReply reply;
+	KeyRangeRef range = req.keys;
+	if (prefix.present()) {
+		range = range.withPrefix(prefix.get(), req.arena);
+	}
+	std::vector<KeyRef> points = getSplitPoints(range, req.chunkSize, prefix);
+
+	reply.splitPoints.append_deep(reply.splitPoints.arena(), points.data(), points.size());
+	req.reply.send(reply);
+}
+
+std::vector<KeyRef> StorageServerMetrics::getSplitPoints(KeyRangeRef range,
+                                                         int64_t chunkSize,
+                                                         Optional<Key> prefixToRemove) const {
+	std::vector<KeyRef> toReturn;
+	KeyRef beginKey = range.begin;
+	IndexedSet<Key, int64_t>::const_iterator endKey =
+	    byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + chunkSize);
+	while (endKey != byteSample.sample.end()) {
+		if (*endKey > range.end) {
+			break;
+		}
+		if (*endKey == beginKey) {
+			++endKey;
+			continue;
+		}
+		KeyRef splitPoint = *endKey;
+		if (prefixToRemove.present()) {
+			splitPoint = splitPoint.removePrefix(prefixToRemove.get());
+		}
+		toReturn.push_back(splitPoint);
+		beginKey = *endKey;
+		endKey = byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + chunkSize);
+	}
+	return toReturn;
+}
+
+void StorageServerMetrics::collapse(KeyRangeMap<int>& map, KeyRef const& key) {
+	auto range = map.rangeContaining(key);
+	if (range == map.ranges().begin() || range == map.ranges().end())
+		return;
+	int value = range->value();
+	auto prev = range;
+	--prev;
+	if (prev->value() != value)
+		return;
+	KeyRange keys = KeyRangeRef(prev->begin(), range->end());
+	map.insert(keys, value);
+}
+
+void StorageServerMetrics::add(KeyRangeMap<int>& map, KeyRangeRef const& keys, int delta) {
+	auto rs = map.modify(keys);
+	for (auto r = rs.begin(); r != rs.end(); ++r)
+		r->value() += delta;
+	collapse(map, keys.begin);
+	collapse(map, keys.end);
+}
+
+// Returns the sampled metric value (possibly 0, possibly increased by the sampling factor)
+int64_t TransientStorageMetricSample::addAndExpire(KeyRef key, int64_t metric, double expiration) {
+	int64_t x = add(key, metric);
+	if (x)
+		queue.emplace_back(expiration, std::make_pair(*sample.find(key), -x));
+	return x;
+}
+
+// FIXME: both versions of erase are broken, because they do not remove items in the queue with will subtract a
+// metric from the value sometime in the future
+int64_t TransientStorageMetricSample::erase(KeyRef key) {
+	auto it = sample.find(key);
+	if (it == sample.end())
+		return 0;
+	int64_t x = sample.getMetric(it);
+	sample.erase(it);
+	return x;
+}
+
+void TransientStorageMetricSample::erase(KeyRangeRef keys) {
+	sample.erase(keys.begin, keys.end);
+}
+
+bool TransientStorageMetricSample::roll(KeyRef key, int64_t metric) const {
+	return deterministicRandom()->random01() < (double)metric / metricUnitsPerSample; //< SOMEDAY: Better randomInt64?
+}
+
+void TransientStorageMetricSample::poll(KeyRangeMap<std::vector<PromiseStream<StorageMetrics>>>& waitMap,
+                                        StorageMetrics m) {
+	double now = ::now();
+	while (queue.size() && queue.front().first <= now) {
+		KeyRef key = queue.front().second.first;
+		int64_t delta = queue.front().second.second;
+		ASSERT(delta != 0);
+
+		if (sample.addMetric(key, delta) == 0)
+			sample.erase(key);
+
+		StorageMetrics deltaM = m * delta;
+		auto v = waitMap[key];
+		for (int i = 0; i < v.size(); i++) {
+			CODE_PROBE(true, "TransientStorageMetricSample poll update");
+			v[i].send(deltaM);
+		}
+
+		queue.pop_front();
+	}
+}
+
+void TransientStorageMetricSample::poll() {
+	double now = ::now();
+	while (queue.size() && queue.front().first <= now) {
+		KeyRef key = queue.front().second.first;
+		int64_t delta = queue.front().second.second;
+		ASSERT(delta != 0);
+
+		if (sample.addMetric(key, delta) == 0)
+			sample.erase(key);
+
+		queue.pop_front();
+	}
+}
+
+int64_t TransientStorageMetricSample::add(KeyRef key, int64_t metric) {
+	if (!metric)
+		return 0;
+	int64_t mag = metric < 0 ? -metric : metric;
+
+	if (mag < metricUnitsPerSample) {
+		if (!roll(key, mag))
+			return 0;
+		metric = metric < 0 ? -metricUnitsPerSample : metricUnitsPerSample;
+	}
+
+	if (sample.addMetric(key, metric) == 0)
+		sample.erase(key);
+
+	return metric;
+}
+
 TEST_CASE("/fdbserver/StorageMetricSample/simple") {
 	StorageMetricSample s(1000);
-	s.sample.insert(LiteralStringRef("Apple"), 1000);
-	s.sample.insert(LiteralStringRef("Banana"), 2000);
-	s.sample.insert(LiteralStringRef("Cat"), 1000);
-	s.sample.insert(LiteralStringRef("Cathode"), 1000);
-	s.sample.insert(LiteralStringRef("Dog"), 1000);
+	s.sample.insert("Apple"_sr, 1000);
+	s.sample.insert("Banana"_sr, 2000);
+	s.sample.insert("Cat"_sr, 1000);
+	s.sample.insert("Cathode"_sr, 1000);
+	s.sample.insert("Dog"_sr, 1000);
 
-	ASSERT(s.getEstimate(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("D"))) == 5000);
-	ASSERT(s.getEstimate(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("E"))) == 6000);
-	ASSERT(s.getEstimate(KeyRangeRef(LiteralStringRef("B"), LiteralStringRef("C"))) == 2000);
+	ASSERT(s.getEstimate(KeyRangeRef("A"_sr, "D"_sr)) == 5000);
+	ASSERT(s.getEstimate(KeyRangeRef("A"_sr, "E"_sr)) == 6000);
+	ASSERT(s.getEstimate(KeyRangeRef("B"_sr, "C"_sr)) == 2000);
 
-	// ASSERT(s.splitEstimate(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("D")), 3500) ==
-	// LiteralStringRef("Cat"));
+	// ASSERT(s.splitEstimate(KeyRangeRef("A"_sr, "D"_sr), 3500) ==
+	// "Cat"_sr);
 
 	return Void();
 }
@@ -45,19 +576,18 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/simple") {
 	int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
 	StorageServerMetrics ssm;
 
-	ssm.byteSample.sample.insert(LiteralStringRef("A"), 200 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 800 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 200 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit);
+	ssm.byteSample.sample.insert("A"_sr, 200 * sampleUnit);
+	ssm.byteSample.sample.insert("Absolute"_sr, 800 * sampleUnit);
+	ssm.byteSample.sample.insert("Apple"_sr, 1000 * sampleUnit);
+	ssm.byteSample.sample.insert("Bah"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Banana"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Bob"_sr, 200 * sampleUnit);
+	ssm.byteSample.sample.insert("But"_sr, 100 * sampleUnit);
+	ssm.byteSample.sample.insert("Cat"_sr, 300 * sampleUnit);
 
-	std::vector<KeyRef> t = ssm.getSplitPoints(
-	    KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 2000 * sampleUnit, Optional<Key>());
+	std::vector<KeyRef> t = ssm.getSplitPoints(KeyRangeRef("A"_sr, "C"_sr), 2000 * sampleUnit, Optional<Key>());
 
-	ASSERT(t.size() == 1 && t[0] == LiteralStringRef("Bah"));
+	ASSERT(t.size() == 1 && t[0] == "Bah"_sr);
 
 	return Void();
 }
@@ -67,20 +597,18 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/multipleReturnedPoint
 	int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
 	StorageServerMetrics ssm;
 
-	ssm.byteSample.sample.insert(LiteralStringRef("A"), 200 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 800 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 200 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit);
+	ssm.byteSample.sample.insert("A"_sr, 200 * sampleUnit);
+	ssm.byteSample.sample.insert("Absolute"_sr, 800 * sampleUnit);
+	ssm.byteSample.sample.insert("Apple"_sr, 1000 * sampleUnit);
+	ssm.byteSample.sample.insert("Bah"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Banana"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Bob"_sr, 200 * sampleUnit);
+	ssm.byteSample.sample.insert("But"_sr, 100 * sampleUnit);
+	ssm.byteSample.sample.insert("Cat"_sr, 300 * sampleUnit);
 
-	std::vector<KeyRef> t = ssm.getSplitPoints(
-	    KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 600 * sampleUnit, Optional<Key>());
+	std::vector<KeyRef> t = ssm.getSplitPoints(KeyRangeRef("A"_sr, "C"_sr), 600 * sampleUnit, Optional<Key>());
 
-	ASSERT(t.size() == 3 && t[0] == LiteralStringRef("Absolute") && t[1] == LiteralStringRef("Apple") &&
-	       t[2] == LiteralStringRef("Bah"));
+	ASSERT(t.size() == 3 && t[0] == "Absolute"_sr && t[1] == "Apple"_sr && t[2] == "Bah"_sr);
 
 	return Void();
 }
@@ -90,17 +618,16 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/noneSplitable") {
 	int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
 	StorageServerMetrics ssm;
 
-	ssm.byteSample.sample.insert(LiteralStringRef("A"), 200 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 800 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 200 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit);
+	ssm.byteSample.sample.insert("A"_sr, 200 * sampleUnit);
+	ssm.byteSample.sample.insert("Absolute"_sr, 800 * sampleUnit);
+	ssm.byteSample.sample.insert("Apple"_sr, 1000 * sampleUnit);
+	ssm.byteSample.sample.insert("Bah"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Banana"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Bob"_sr, 200 * sampleUnit);
+	ssm.byteSample.sample.insert("But"_sr, 100 * sampleUnit);
+	ssm.byteSample.sample.insert("Cat"_sr, 300 * sampleUnit);
 
-	std::vector<KeyRef> t = ssm.getSplitPoints(
-	    KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 10000 * sampleUnit, Optional<Key>());
+	std::vector<KeyRef> t = ssm.getSplitPoints(KeyRangeRef("A"_sr, "C"_sr), 10000 * sampleUnit, Optional<Key>());
 
 	ASSERT(t.size() == 0);
 
@@ -112,17 +639,16 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/chunkTooLarge") {
 	int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
 	StorageServerMetrics ssm;
 
-	ssm.byteSample.sample.insert(LiteralStringRef("A"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 10 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("But"), 10 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 30 * sampleUnit);
+	ssm.byteSample.sample.insert("A"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Absolute"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Apple"_sr, 10 * sampleUnit);
+	ssm.byteSample.sample.insert("Bah"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Banana"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Bob"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("But"_sr, 10 * sampleUnit);
+	ssm.byteSample.sample.insert("Cat"_sr, 30 * sampleUnit);
 
-	std::vector<KeyRef> t = ssm.getSplitPoints(
-	    KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 1000 * sampleUnit, Optional<Key>());
+	std::vector<KeyRef> t = ssm.getSplitPoints(KeyRangeRef("A"_sr, "C"_sr), 1000 * sampleUnit, Optional<Key>());
 
 	ASSERT(t.size() == 0);
 
@@ -134,26 +660,25 @@ TEST_CASE("/fdbserver/StorageMetricSample/readHotDetect/simple") {
 	int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
 	StorageServerMetrics ssm;
 
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Banana"), 2000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Cat"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Cathode"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Dog"), 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Apple"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Banana"_sr, 2000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Cat"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Cathode"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Dog"_sr, 1000 * sampleUnit);
 
-	ssm.byteSample.sample.insert(LiteralStringRef("A"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 200 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit);
+	ssm.byteSample.sample.insert("A"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Absolute"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Apple"_sr, 1000 * sampleUnit);
+	ssm.byteSample.sample.insert("Bah"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Banana"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Bob"_sr, 200 * sampleUnit);
+	ssm.byteSample.sample.insert("But"_sr, 100 * sampleUnit);
+	ssm.byteSample.sample.insert("Cat"_sr, 300 * sampleUnit);
 
 	std::vector<ReadHotRangeWithMetrics> t =
-	    ssm.getReadHotRanges(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 2.0, 200 * sampleUnit, 0);
+	    ssm.getReadHotRanges(KeyRangeRef("A"_sr, "C"_sr), 2.0, 200 * sampleUnit, 0);
 
-	ASSERT(t.size() == 1 && (*t.begin()).keys.begin == LiteralStringRef("Bah") &&
-	       (*t.begin()).keys.end == LiteralStringRef("Bob"));
+	ASSERT(t.size() == 1 && (*t.begin()).keys.begin == "Bah"_sr && (*t.begin()).keys.end == "Bob"_sr);
 
 	return Void();
 }
@@ -163,29 +688,28 @@ TEST_CASE("/fdbserver/StorageMetricSample/readHotDetect/moreThanOneRange") {
 	int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
 	StorageServerMetrics ssm;
 
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Banana"), 2000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Cat"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Cathode"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Dog"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Final"), 2000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Apple"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Banana"_sr, 2000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Cat"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Cathode"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Dog"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Final"_sr, 2000 * sampleUnit);
 
-	ssm.byteSample.sample.insert(LiteralStringRef("A"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 200 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Dah"), 300 * sampleUnit);
+	ssm.byteSample.sample.insert("A"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Absolute"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Apple"_sr, 1000 * sampleUnit);
+	ssm.byteSample.sample.insert("Bah"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Banana"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Bob"_sr, 200 * sampleUnit);
+	ssm.byteSample.sample.insert("But"_sr, 100 * sampleUnit);
+	ssm.byteSample.sample.insert("Cat"_sr, 300 * sampleUnit);
+	ssm.byteSample.sample.insert("Dah"_sr, 300 * sampleUnit);
 
 	std::vector<ReadHotRangeWithMetrics> t =
-	    ssm.getReadHotRanges(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("D")), 2.0, 200 * sampleUnit, 0);
+	    ssm.getReadHotRanges(KeyRangeRef("A"_sr, "D"_sr), 2.0, 200 * sampleUnit, 0);
 
-	ASSERT(t.size() == 2 && (*t.begin()).keys.begin == LiteralStringRef("Bah") &&
-	       (*t.begin()).keys.end == LiteralStringRef("Bob"));
-	ASSERT(t.at(1).keys.begin == LiteralStringRef("Cat") && t.at(1).keys.end == LiteralStringRef("Dah"));
+	ASSERT(t.size() == 2 && (*t.begin()).keys.begin == "Bah"_sr && (*t.begin()).keys.end == "Bob"_sr);
+	ASSERT(t.at(1).keys.begin == "Cat"_sr && t.at(1).keys.end == "Dah"_sr);
 
 	return Void();
 }
@@ -195,30 +719,29 @@ TEST_CASE("/fdbserver/StorageMetricSample/readHotDetect/consecutiveRanges") {
 	int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
 	StorageServerMetrics ssm;
 
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Banana"), 2000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Bucket"), 2000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Cat"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Cathode"), 1000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Dog"), 5000 * sampleUnit);
-	ssm.bytesReadSample.sample.insert(LiteralStringRef("Final"), 2000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Apple"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Banana"_sr, 2000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Bucket"_sr, 2000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Cat"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Cathode"_sr, 1000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Dog"_sr, 5000 * sampleUnit);
+	ssm.bytesReadSample.sample.insert("Final"_sr, 2000 * sampleUnit);
 
-	ssm.byteSample.sample.insert(LiteralStringRef("A"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 200 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit);
-	ssm.byteSample.sample.insert(LiteralStringRef("Dah"), 300 * sampleUnit);
+	ssm.byteSample.sample.insert("A"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Absolute"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Apple"_sr, 1000 * sampleUnit);
+	ssm.byteSample.sample.insert("Bah"_sr, 20 * sampleUnit);
+	ssm.byteSample.sample.insert("Banana"_sr, 80 * sampleUnit);
+	ssm.byteSample.sample.insert("Bob"_sr, 200 * sampleUnit);
+	ssm.byteSample.sample.insert("But"_sr, 100 * sampleUnit);
+	ssm.byteSample.sample.insert("Cat"_sr, 300 * sampleUnit);
+	ssm.byteSample.sample.insert("Dah"_sr, 300 * sampleUnit);
 
 	std::vector<ReadHotRangeWithMetrics> t =
-	    ssm.getReadHotRanges(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("D")), 2.0, 200 * sampleUnit, 0);
+	    ssm.getReadHotRanges(KeyRangeRef("A"_sr, "D"_sr), 2.0, 200 * sampleUnit, 0);
 
-	ASSERT(t.size() == 2 && (*t.begin()).keys.begin == LiteralStringRef("Bah") &&
-	       (*t.begin()).keys.end == LiteralStringRef("But"));
-	ASSERT(t.at(1).keys.begin == LiteralStringRef("Cat") && t.at(1).keys.end == LiteralStringRef("Dah"));
+	ASSERT(t.size() == 2 && (*t.begin()).keys.begin == "Bah"_sr && (*t.begin()).keys.end == "But"_sr);
+	ASSERT(t.at(1).keys.begin == "Cat"_sr && t.at(1).keys.end == "Dah"_sr);
 
 	return Void();
 }
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 921000cad2..a502847e52 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -157,7 +157,7 @@ private:
 			Standalone<StringRef> h = wait(self->queue->readNext(sizeof(uint32_t)));
 			if (h.size() != sizeof(uint32_t)) {
 				if (h.size()) {
-					TEST(true); // Zero fill within size field
+					CODE_PROBE(true, "Zero fill within size field");
 					int payloadSize = 0;
 					memcpy(&payloadSize, h.begin(), h.size());
 					zeroFillSize = sizeof(uint32_t) - h.size(); // zero fill the size itself
@@ -171,7 +171,7 @@ private:
 
 			Standalone<StringRef> e = wait(self->queue->readNext(payloadSize + 1));
 			if (e.size() != payloadSize + 1) {
-				TEST(true); // Zero fill within payload
+				CODE_PROBE(true, "Zero fill within payload");
 				zeroFillSize = payloadSize + 1 - e.size();
 				break;
 			}
@@ -187,7 +187,7 @@ private:
 			}
 		}
 		if (zeroFillSize) {
-			TEST(true); // Fixing a partial commit at the end of the tlog queue
+			CODE_PROBE(true, "Fixing a partial commit at the end of the tlog queue");
 			for (int i = 0; i < zeroFillSize; i++)
 				self->queue->push(StringRef((const uint8_t*)"", 1));
 		}
@@ -200,32 +200,24 @@ private:
 // Immutable keys
 // persistFormat has been mostly invalidated by TLogVersion, and can probably be removed when
 // 4.6's TLog code is removed.
-static const KeyValueRef persistFormat(LiteralStringRef("Format"), LiteralStringRef("FoundationDB/LogServer/3/0"));
-static const KeyRangeRef persistFormatReadableRange(LiteralStringRef("FoundationDB/LogServer/3/0"),
-                                                    LiteralStringRef("FoundationDB/LogServer/4/0"));
-static const KeyRangeRef persistProtocolVersionKeys(LiteralStringRef("ProtocolVersion/"),
-                                                    LiteralStringRef("ProtocolVersion0"));
-static const KeyRangeRef persistTLogSpillTypeKeys(LiteralStringRef("TLogSpillType/"),
-                                                  LiteralStringRef("TLogSpillType0"));
-static const KeyRangeRef persistRecoveryCountKeys =
-    KeyRangeRef(LiteralStringRef("DbRecoveryCount/"), LiteralStringRef("DbRecoveryCount0"));
+static const KeyValueRef persistFormat("Format"_sr, "FoundationDB/LogServer/3/0"_sr);
+static const KeyRangeRef persistFormatReadableRange("FoundationDB/LogServer/3/0"_sr, "FoundationDB/LogServer/4/0"_sr);
+static const KeyRangeRef persistProtocolVersionKeys("ProtocolVersion/"_sr, "ProtocolVersion0"_sr);
+static const KeyRangeRef persistTLogSpillTypeKeys("TLogSpillType/"_sr, "TLogSpillType0"_sr);
+static const KeyRangeRef persistRecoveryCountKeys = KeyRangeRef("DbRecoveryCount/"_sr, "DbRecoveryCount0"_sr);
 
 // Updated on updatePersistentData()
-static const KeyRangeRef persistCurrentVersionKeys =
-    KeyRangeRef(LiteralStringRef("version/"), LiteralStringRef("version0"));
-static const KeyRangeRef persistKnownCommittedVersionKeys =
-    KeyRangeRef(LiteralStringRef("knownCommitted/"), LiteralStringRef("knownCommitted0"));
-static const KeyRef persistRecoveryLocationKey = KeyRef(LiteralStringRef("recoveryLocation"));
-static const KeyRangeRef persistLocalityKeys =
-    KeyRangeRef(LiteralStringRef("Locality/"), LiteralStringRef("Locality0"));
-static const KeyRangeRef persistLogRouterTagsKeys =
-    KeyRangeRef(LiteralStringRef("LogRouterTags/"), LiteralStringRef("LogRouterTags0"));
-static const KeyRangeRef persistTxsTagsKeys = KeyRangeRef(LiteralStringRef("TxsTags/"), LiteralStringRef("TxsTags0"));
-static const KeyRange persistTagMessagesKeys = prefixRange(LiteralStringRef("TagMsg/"));
-static const KeyRange persistTagMessageRefsKeys = prefixRange(LiteralStringRef("TagMsgRef/"));
-static const KeyRange persistTagPoppedKeys = prefixRange(LiteralStringRef("TagPop/"));
+static const KeyRangeRef persistCurrentVersionKeys = KeyRangeRef("version/"_sr, "version0"_sr);
+static const KeyRangeRef persistKnownCommittedVersionKeys = KeyRangeRef("knownCommitted/"_sr, "knownCommitted0"_sr);
+static const KeyRef persistRecoveryLocationKey = KeyRef("recoveryLocation"_sr);
+static const KeyRangeRef persistLocalityKeys = KeyRangeRef("Locality/"_sr, "Locality0"_sr);
+static const KeyRangeRef persistLogRouterTagsKeys = KeyRangeRef("LogRouterTags/"_sr, "LogRouterTags0"_sr);
+static const KeyRangeRef persistTxsTagsKeys = KeyRangeRef("TxsTags/"_sr, "TxsTags0"_sr);
+static const KeyRange persistTagMessagesKeys = prefixRange("TagMsg/"_sr);
+static const KeyRange persistTagMessageRefsKeys = prefixRange("TagMsgRef/"_sr);
+static const KeyRange persistTagPoppedKeys = prefixRange("TagPop/"_sr);
 
-static const KeyRef persistClusterIdKey = LiteralStringRef("clusterId");
+static const KeyRef persistClusterIdKey = "clusterId"_sr;
 
 static Key persistTagMessagesKey(UID id, Tag tag, Version version) {
 	BinaryWriter wr(Unversioned());
@@ -386,9 +378,8 @@ struct TLogData : NonCopyable {
 	    targetVolatileBytes(SERVER_KNOBS->TLOG_SPILL_THRESHOLD), overheadBytesInput(0), overheadBytesDurable(0),
 	    peekMemoryLimiter(SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES),
 	    concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS), ignorePopDeadline(0), dataFolder(folder),
-	    degraded(degraded), commitLatencyDist(Histogram::getHistogram(LiteralStringRef("tLog"),
-	                                                                  LiteralStringRef("commit"),
-	                                                                  Histogram::Unit::microseconds)) {
+	    degraded(degraded),
+	    commitLatencyDist(Histogram::getHistogram("tLog"_sr, "commit"_sr, Histogram::Unit::microseconds)) {
 		cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
 	}
 };
@@ -497,7 +488,8 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 	*/
 
 	AsyncTrigger stopCommit;
-	bool stopped, initialized;
+	bool initialized;
+	Promise<Void> stoppedPromise;
 	DBRecoveryCount recoveryCount;
 
 	// If persistentDataVersion != persistentDurableDataVersion,
@@ -518,7 +510,8 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 
 	Deque<std::pair<Version, Standalone<VectorRef<uint8_t>>>> messageBlocks;
 	std::vector<std::vector<Reference<TagData>>> tag_data; // tag.locality | tag.id
-	int unpoppedRecoveredTags;
+	int unpoppedRecoveredTagCount;
+	std::set<Tag> unpoppedRecoveredTags;
 	std::map<Tag, Promise<Void>> waitingTags;
 
 	Reference<TagData> getTagData(Tag tag) {
@@ -640,10 +633,10 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 	                 TLogSpillType logSpillType,
 	                 std::vector<Tag> tags,
 	                 std::string context)
-	  : stopped(false), initialized(false), queueCommittingVersion(0), knownCommittedVersion(0),
-	    durableKnownCommittedVersion(0), minKnownCommittedVersion(0), queuePoppedVersion(0), minPoppedTagVersion(0),
-	    minPoppedTag(invalidTag), unpoppedRecoveredTags(0), cc("TLog", interf.id().toString()),
-	    bytesInput("BytesInput", cc), bytesDurable("BytesDurable", cc), blockingPeeks("BlockingPeeks", cc),
+	  : initialized(false), queueCommittingVersion(0), knownCommittedVersion(0), durableKnownCommittedVersion(0),
+	    minKnownCommittedVersion(0), queuePoppedVersion(0), minPoppedTagVersion(0), minPoppedTag(invalidTag),
+	    unpoppedRecoveredTagCount(0), cc("TLog", interf.id().toString()), bytesInput("BytesInput", cc),
+	    bytesDurable("BytesDurable", cc), blockingPeeks("BlockingPeeks", cc),
 	    blockingPeekTimeouts("BlockingPeekTimeouts", cc), emptyPeeks("EmptyPeeks", cc),
 	    nonEmptyPeeks("NonEmptyPeeks", cc), logId(interf.id()), protocolVersion(protocolVersion),
 	    newPersistentDataVersion(invalidVersion), tLogData(tLogData), unrecoveredBefore(1), recoveredAt(1),
@@ -659,10 +652,10 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 		          context);
 		addActor.send(traceRole(Role::TRANSACTION_LOG, interf.id()));
 
-		persistentDataVersion.init(LiteralStringRef("TLog.PersistentDataVersion"), cc.id);
-		persistentDataDurableVersion.init(LiteralStringRef("TLog.PersistentDataDurableVersion"), cc.id);
-		version.initMetric(LiteralStringRef("TLog.Version"), cc.id);
-		queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id);
+		persistentDataVersion.init("TLog.PersistentDataVersion"_sr, cc.id);
+		persistentDataDurableVersion.init("TLog.PersistentDataDurableVersion"_sr, cc.id);
+		version.initMetric("TLog.Version"_sr, cc.id);
+		queueCommittedVersion.initMetric("TLog.QueueCommittedVersion"_sr, cc.id);
 
 		specialCounter(cc, "Version", [this]() { return this->version.get(); });
 		specialCounter(cc, "QueueCommittedVersion", [this]() { return this->queueCommittedVersion.get(); });
@@ -754,6 +747,15 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 			waitingTags.clear();
 		}
 	}
+
+	bool stopped() const { return stoppedPromise.isSet(); }
+
+	void stop() {
+		if (stoppedPromise.canBeSet()) {
+			TraceEvent(SevDebug, "StoppingTLog", tLogData->dbgid).detail("LogId", logId);
+			stoppedPromise.send(Void());
+		}
+	}
 };
 
 template <class T>
@@ -805,16 +807,16 @@ void TLogQueue::updateVersionSizes(const TLogQueueEntry& result,
 ACTOR Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply, Reference<LogData> logData) {
 	state Version stopVersion = logData->version.get();
 
-	TEST(true); // TLog stopped by recovering cluster-controller
-	TEST(logData->stopped); // logData already stopped
-	TEST(!logData->stopped); // logData not yet stopped
+	CODE_PROBE(true, "TLog stopped by recovering cluster-controller");
+	CODE_PROBE(logData->stopped(), "logData already stopped");
+	CODE_PROBE(!logData->stopped(), "logData not yet stopped");
 
 	TraceEvent("TLogStop", logData->logId)
 	    .detail("Ver", stopVersion)
-	    .detail("IsStopped", logData->stopped)
+	    .detail("IsStopped", logData->stopped())
 	    .detail("QueueCommitted", logData->queueCommittedVersion.get());
 
-	logData->stopped = true;
+	logData->stop();
 	logData->unblockWaitingPeeks();
 	if (!logData->recoveryComplete.isSet()) {
 		logData->recoveryComplete.sendError(end_of_stream());
@@ -834,7 +836,7 @@ ACTOR Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply,
 	TraceEvent("TLogStop2", self->dbgid)
 	    .detail("LogId", logData->logId)
 	    .detail("Ver", stopVersion)
-	    .detail("IsStopped", logData->stopped)
+	    .detail("IsStopped", logData->stopped())
 	    .detail("QueueCommitted", logData->queueCommittedVersion.get())
 	    .detail("KnownCommitted", result.knownCommittedVersion);
 
@@ -1091,13 +1093,16 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 	    BinaryWriter::toValue(logData->knownCommittedVersion, Unversioned())));
 	logData->persistentDataVersion = newPersistentDataVersion;
 
-	wait(self->persistentData->commit()); // SOMEDAY: This seems to be running pretty often, should we slow it down???
+	// SOMEDAY: This seems to be running pretty often, should we slow it down???
+	// This needs a timeout since nothing prevents I/O operations from hanging indefinitely.
+	wait(ioTimeoutError(self->persistentData->commit(), SERVER_KNOBS->TLOG_MAX_CREATE_DURATION));
+
 	wait(delay(0, TaskPriority::UpdateStorage));
 
 	// Now that the changes we made to persistentData are durable, erase the data we moved from memory and the queue,
 	// increase bytesDurable accordingly, and update persistentDataDurableVersion.
 
-	TEST(anyData); // TLog moved data to persistentData
+	CODE_PROBE(anyData, "TLog moved data to persistentData");
 	logData->persistentDataDurableVersion = newPersistentDataVersion;
 	for (tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
 		for (tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
@@ -1196,14 +1201,20 @@ ACTOR Future<Void> tLogPopCore(TLogData* self, Tag inputTag, Version to, Referen
 
 		if (tagData->unpoppedRecovered && upTo > logData->recoveredAt) {
 			tagData->unpoppedRecovered = false;
-			logData->unpoppedRecoveredTags--;
+			logData->unpoppedRecoveredTagCount--;
+			logData->unpoppedRecoveredTags.erase(tag);
 			TraceEvent("TLogPoppedTag", logData->logId)
-			    .detail("Tags", logData->unpoppedRecoveredTags)
+			    .detail("Tags", logData->unpoppedRecoveredTagCount)
 			    .detail("Tag", tag.toString())
 			    .detail("DurableKCVer", logData->durableKnownCommittedVersion)
-			    .detail("RecoveredAt", logData->recoveredAt);
-			if (logData->unpoppedRecoveredTags == 0 && logData->durableKnownCommittedVersion >= logData->recoveredAt &&
-			    logData->recoveryComplete.canBeSet()) {
+			    .detail("RecoveredAt", logData->recoveredAt)
+			    .detail("UnpoppedTags", describe(logData->unpoppedRecoveredTags));
+			if (logData->unpoppedRecoveredTagCount == 0 &&
+			    logData->durableKnownCommittedVersion >= logData->recoveredAt && logData->recoveryComplete.canBeSet()) {
+				TraceEvent("TLogRecoveryComplete", logData->logId)
+				    .detail("Tags", logData->unpoppedRecoveredTagCount)
+				    .detail("DurableKCVer", logData->durableKnownCommittedVersion)
+				    .detail("RecoveredAt", logData->recoveredAt);
 				logData->recoveryComplete.send(Void());
 			}
 		}
@@ -1248,7 +1259,7 @@ ACTOR Future<Void> processPopRequests(TLogData* self, Reference<LogData> logData
 		TraceEvent("PlayIgnoredPop", logData->logId).detail("Tag", tag.toString()).detail("Version", version);
 		ignoredPops.push_back(tLogPopCore(self, tag, version, logData));
 		if (++ignoredPopsPlayed % SERVER_KNOBS->TLOG_POP_BATCH_SIZE == 0) {
-			TEST(true); // Yielding while processing pop requests
+			CODE_PROBE(true, "Yielding while processing pop requests");
 			wait(yield());
 		}
 	}
@@ -1308,7 +1319,7 @@ ACTOR Future<Void> updateStorage(TLogData* self) {
 	// tag; which is not intended to ever happen.
 	Optional<Version> cachePopVersion;
 	for (auto& it : self->id_data) {
-		if (!it.second->stopped) {
+		if (!it.second->stopped()) {
 			if (it.second->version.get() - it.second->unrecoveredBefore >
 			    SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT + SERVER_KNOBS->MAX_CACHE_VERSIONS) {
 				cachePopVersion = it.second->version.get() - SERVER_KNOBS->MAX_CACHE_VERSIONS;
@@ -1325,7 +1336,7 @@ ACTOR Future<Void> updateStorage(TLogData* self) {
 		wait(waitForAll(cachePopFutures));
 	}
 
-	if (logData->stopped) {
+	if (logData->stopped()) {
 		if (self->bytesInput - self->bytesDurable >= self->targetVolatileBytes) {
 			while (logData->persistentDataDurableVersion != logData->version.get()) {
 				totalSize = 0;
@@ -1756,19 +1767,20 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 		wait(logData->version.whenAtLeast(reqBegin));
 		wait(delay(SERVER_KNOBS->TLOG_PEEK_DELAY, g_network->getCurrentTask()));
 	}
-	if (!logData->stopped && reqTag.locality != tagLocalityTxs && reqTag != txsTag &&
+	if (!logData->stopped() && reqTag.locality != tagLocalityTxs && reqTag != txsTag &&
 	    logData->version.get() < logData->recoveryTxnVersion) {
 		DebugLogTraceEvent("TLogPeekMessages1", self->dbgid)
 		    .detail("LogId", logData->logId)
 		    .detail("Tag", reqTag.toString())
 		    .detail("ReqBegin", reqBegin)
 		    .detail("Version", logData->version.get())
+		    .detail("RecoveryTxnVersion", logData->recoveryTxnVersion)
 		    .detail("RecoveredAt", logData->recoveredAt);
 		// Make sure the peek reply has the recovery txn for the current TLog.
 		// Older generation TLog has been stopped and doesn't wait here.
 		// Similarly during recovery, reading transaction state store
 		// doesn't wait here.
-		wait(logData->version.whenAtLeast(logData->recoveryTxnVersion));
+		wait(logData->version.whenAtLeast(logData->recoveryTxnVersion) || logData->stoppedPromise.getFuture());
 	}
 
 	if (logData->locality != tagLocalitySatellite && reqTag.locality == tagLocalityLogRouter) {
@@ -1836,7 +1848,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 			}
 			if (sequenceData.isSet()) {
 				if (sequenceData.getFuture().get().first != rep.end) {
-					TEST(true); // tlog peek second attempt ended at a different version
+					CODE_PROBE(true, "tlog peek second attempt ended at a different version");
 					replyPromise.sendError(operation_obsolete());
 					return Void();
 				}
@@ -2069,7 +2081,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 		if (sequenceData.isSet()) {
 			trackerData.duplicatePeeks++;
 			if (sequenceData.getFuture().get().first != reply.end) {
-				TEST(true); // tlog peek second attempt ended at a different version (2)
+				CODE_PROBE(true, "tlog peek second attempt ended at a different version (2)");
 				replyPromise.sendError(operation_obsolete());
 				return Void();
 			}
@@ -2140,7 +2152,7 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 
 	wait(ioDegradedOrTimeoutError(
 	    c, SERVER_KNOBS->MAX_STORAGE_COMMIT_TIME, self->degraded, SERVER_KNOBS->TLOG_DEGRADED_DURATION));
-	if (g_network->isSimulated() && !g_simulator.speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
+	if (g_network->isSimulated() && !g_simulator->speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
 		wait(delay(6.0));
 	}
 	wait(self->queueCommitEnd.whenAtLeast(commitNumber - 1));
@@ -2153,10 +2165,10 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 	ASSERT(ver > logData->queueCommittedVersion.get());
 
 	logData->durableKnownCommittedVersion = knownCommittedVersion;
-	if (logData->unpoppedRecoveredTags == 0 && knownCommittedVersion >= logData->recoveredAt &&
+	if (logData->unpoppedRecoveredTagCount == 0 && knownCommittedVersion >= logData->recoveredAt &&
 	    logData->recoveryComplete.canBeSet()) {
 		TraceEvent("TLogRecoveryComplete", logData->logId)
-		    .detail("Tags", logData->unpoppedRecoveredTags)
+		    .detail("Tags", logData->unpoppedRecoveredTagCount)
 		    .detail("DurableKCVer", logData->durableKnownCommittedVersion)
 		    .detail("RecoveredAt", logData->recoveredAt);
 		logData->recoveryComplete.send(Void());
@@ -2177,7 +2189,7 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 		    .detail("LogId", logData->logId)
 		    .detail("Version", it->version.get())
 		    .detail("QueueVer", it->queueCommittedVersion.get());
-		TEST(true); // A TLog was replaced before having a chance to commit its queue
+		CODE_PROBE(true, "A TLog was replaced before having a chance to commit its queue");
 		it->queueCommittedVersion.set(it->version.get());
 	}
 	return Void();
@@ -2190,7 +2202,7 @@ ACTOR Future<Void> commitQueue(TLogData* self) {
 	loop {
 		int foundCount = 0;
 		for (auto it : self->id_data) {
-			if (!it.second->stopped) {
+			if (!it.second->stopped()) {
 				logData = it.second;
 				foundCount++;
 			} else if (it.second->version.get() >
@@ -2215,8 +2227,8 @@ ACTOR Future<Void> commitQueue(TLogData* self) {
 		}
 
 		loop {
-			if (logData->stopped && logData->version.get() == std::max(logData->queueCommittingVersion,
-			                                                           logData->queueCommittedVersion.get())) {
+			if (logData->stopped() && logData->version.get() == std::max(logData->queueCommittingVersion,
+			                                                             logData->queueCommittedVersion.get())) {
 				wait(logData->queueCommittedVersion.whenAtLeast(logData->version.get()));
 				break;
 			}
@@ -2267,7 +2279,7 @@ ACTOR Future<Void> tLogCommit(TLogData* self,
 	}
 
 	state double waitStartT = 0;
-	while (self->bytesInput - self->bytesDurable >= SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES && !logData->stopped) {
+	while (self->bytesInput - self->bytesDurable >= SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES && !logData->stopped()) {
 		if (now() - waitStartT >= 1) {
 			TraceEvent(SevWarn, "TLogUpdateLag", logData->logId)
 			    .detail("Version", logData->version.get())
@@ -2278,7 +2290,7 @@ ACTOR Future<Void> tLogCommit(TLogData* self,
 		wait(delayJittered(.005, TaskPriority::TLogCommit));
 	}
 
-	if (logData->stopped) {
+	if (logData->stopped()) {
 		req.reply.sendError(tlog_stopped());
 		return Void();
 	}
@@ -2327,7 +2339,7 @@ ACTOR Future<Void> tLogCommit(TLogData* self,
 	    timeoutWarning(logData->queueCommittedVersion.whenAtLeast(req.version) || stopped, 0.1, warningCollectorInput));
 
 	if (stopped.isReady()) {
-		ASSERT(logData->stopped);
+		ASSERT(logData->stopped());
 		req.reply.sendError(tlog_stopped());
 		return Void();
 	}
@@ -2347,10 +2359,7 @@ ACTOR Future<Void> initPersistentState(TLogData* self, Reference<LogData> logDat
 	wait(self->persistentDataCommitLock.take());
 	state FlowLock::Releaser commitLockReleaser(self->persistentDataCommitLock);
 
-	// PERSIST: Initial setup of persistentData for a brand new tLog for a new database
 	state IKeyValueStore* storage = self->persistentData;
-	wait(ioTimeoutError(storage->init(), SERVER_KNOBS->TLOG_MAX_CREATE_DURATION));
-	storage->set(persistFormat);
 	storage->set(
 	    KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistCurrentVersionKeys.begin),
 	                BinaryWriter::toValue(logData->version.get(), Unversioned())));
@@ -2380,8 +2389,8 @@ ACTOR Future<Void> initPersistentState(TLogData* self, Reference<LogData> logDat
 		updatePersistentPopped(self, logData, logData->getTagData(tag));
 	}
 
-	TraceEvent("TLogInitCommit", logData->logId).log();
-	wait(ioTimeoutError(self->persistentData->commit(), SERVER_KNOBS->TLOG_MAX_CREATE_DURATION));
+	TraceEvent("TLogInitCommit", logData->logId);
+	wait(self->persistentData->commit());
 	return Void();
 }
 
@@ -2403,9 +2412,11 @@ ACTOR Future<UID> getClusterId(TLogData* self) {
 	}
 }
 
+// send stopped promise instead of LogData* to avoid reference cycles
 ACTOR Future<Void> rejoinClusterController(TLogData* self,
                                            TLogInterface tli,
                                            DBRecoveryCount recoveryCount,
+                                           Promise<Void> stoppedPromise,
                                            Future<Void> registerWithCC,
                                            bool isPrimary) {
 	state LifetimeToken lastMasterLifetime;
@@ -2446,6 +2457,13 @@ ACTOR Future<Void> rejoinClusterController(TLogData* self,
 			if (BUGGIFY)
 				wait(delay(SERVER_KNOBS->BUGGIFY_WORKER_REMOVED_MAX_LAG * deterministicRandom()->random01()));
 			throw worker_removed();
+		} else if (inf.recoveryCount > recoveryCount && stoppedPromise.canBeSet()) {
+			CODE_PROBE(true, "Stopping tlog because new dbinfo has a higher recovery count");
+			TraceEvent("StoppingTLog", self->dbgid)
+			    .detail("LogId", tli.id())
+			    .detail("NewRecoveryCount", inf.recoveryCount)
+			    .detail("MyRecoveryCount", recoveryCount);
+			stoppedPromise.send(Void());
 		}
 
 		if (registerWithCC.isReady()) {
@@ -2594,6 +2612,27 @@ ACTOR Future<Void> tLogEnablePopReq(TLogEnablePopRequest enablePopReq, TLogData*
 	return Void();
 }
 
+ACTOR Future<Void> updateDurableClusterID(TLogData* self) {
+	loop {
+		// Persist cluster ID once cluster has recovered.
+		if (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED) {
+			ASSERT(!self->durableClusterId.isValid());
+			state UID ccClusterId = self->dbInfo->get().client.clusterId;
+			self->durableClusterId = ccClusterId;
+			ASSERT(ccClusterId.isValid());
+
+			wait(self->persistentDataCommitLock.take());
+			state FlowLock::Releaser commitLockReleaser(self->persistentDataCommitLock);
+			self->persistentData->set(
+			    KeyValueRef(persistClusterIdKey, BinaryWriter::toValue(ccClusterId, Unversioned())));
+			wait(self->persistentData->commit());
+
+			return Void();
+		}
+		wait(self->dbInfo->onChange());
+	}
+}
+
 ACTOR Future<Void> serveTLogInterface(TLogData* self,
                                       TLogInterface tli,
                                       Reference<LogData> logData,
@@ -2621,24 +2660,13 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 					                               logData->locality);
 				}
 
-				if (!logData->isPrimary && logData->stopped) {
+				if (!logData->isPrimary && logData->stopped()) {
 					TraceEvent("TLogAlreadyStopped", self->dbgid).detail("LogId", logData->logId);
 					logData->removed = logData->removed && logData->logSystem->get()->endEpoch();
 				}
 			} else {
 				logData->logSystem->set(Reference<ILogSystem>());
 			}
-
-			// Persist cluster ID once cluster has recovered.
-			auto ccClusterId = self->dbInfo->get().client.clusterId;
-			if (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED &&
-			    !self->durableClusterId.isValid()) {
-				ASSERT(ccClusterId.isValid());
-				self->durableClusterId = ccClusterId;
-				self->persistentData->set(
-				    KeyValueRef(persistClusterIdKey, BinaryWriter::toValue(ccClusterId, Unversioned())));
-				wait(self->persistentData->commit());
-			}
 		}
 		when(TLogPeekStreamRequest req = waitNext(tli.peekStreamMessages.getFuture())) {
 			TraceEvent(SevDebug, "TLogPeekStream", logData->logId)
@@ -2655,8 +2683,8 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 		when(TLogCommitRequest req = waitNext(tli.commit.getFuture())) {
 			//TraceEvent("TLogCommitReq", logData->logId).detail("Ver", req.version).detail("PrevVer", req.prevVersion).detail("LogVer", logData->version.get());
 			ASSERT(logData->isPrimary);
-			TEST(logData->stopped); // TLogCommitRequest while stopped
-			if (!logData->stopped)
+			CODE_PROBE(logData->stopped(), "TLogCommitRequest while stopped");
+			if (!logData->stopped())
 				logData->addActor.send(tLogCommit(self, req, logData, warningCollectorInput));
 			else
 				req.reply.sendError(tlog_stopped());
@@ -2673,7 +2701,7 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 				g_traceBatch.addAttach("TransactionAttachID", req.debugID.get().first(), tlogDebugID.first());
 				g_traceBatch.addEvent("TransactionDebug", tlogDebugID.first(), "TLogServer.TLogConfirmRunningRequest");
 			}
-			if (!logData->stopped)
+			if (!logData->stopped())
 				req.reply.send(Void());
 			else
 				req.reply.sendError(tlog_stopped());
@@ -2706,7 +2734,7 @@ void removeLog(TLogData* self, Reference<LogData> logData) {
 	    .detail("LogId", logData->logId)
 	    .detail("Input", logData->bytesInput.getValue())
 	    .detail("Durable", logData->bytesDurable.getValue());
-	logData->stopped = true;
+	logData->stop();
 	logData->unblockWaitingPeeks();
 	if (!logData->recoveryComplete.isSet()) {
 		logData->recoveryComplete.sendError(end_of_stream());
@@ -2766,7 +2794,7 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 		}
 
 		state double waitStartT = 0;
-		while (self->bytesInput - self->bytesDurable >= SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES && !logData->stopped) {
+		while (self->bytesInput - self->bytesDurable >= SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES && !logData->stopped()) {
 			if (now() - waitStartT >= 1) {
 				TraceEvent(SevWarn, "TLogUpdateLag", logData->logId)
 				    .detail("Version", logData->version.get())
@@ -2785,7 +2813,7 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 			if (!foundMessage || r->version().version != ver) {
 				ASSERT(r->version().version > lastVer);
 				if (ver) {
-					if (logData->stopped || (endVersion.present() && ver > endVersion.get())) {
+					if (logData->stopped() || (endVersion.present() && ver > endVersion.get())) {
 						return Void();
 					}
 
@@ -2826,7 +2854,7 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 				if (!foundMessage) {
 					ver--;
 					if (ver > logData->version.get()) {
-						if (logData->stopped || (endVersion.present() && ver > endVersion.get())) {
+						if (logData->stopped() || (endVersion.present() && ver > endVersion.get())) {
 							return Void();
 						}
 
@@ -2960,10 +2988,20 @@ ACTOR Future<Void> checkEmptyQueue(TLogData* self) {
 	}
 }
 
-ACTOR Future<Void> checkRecovered(TLogData* self) {
-	TraceEvent("TLogCheckRecoveredBegin", self->dbgid).log();
-	Optional<Value> v = wait(self->persistentData->readValue(StringRef()));
-	TraceEvent("TLogCheckRecoveredEnd", self->dbgid).log();
+ACTOR Future<Void> initPersistentStorage(TLogData* self) {
+	TraceEvent("TLogInitPersistentStorageStart", self->dbgid);
+
+	wait(self->persistentDataCommitLock.take());
+	state FlowLock::Releaser commitLockReleaser(self->persistentDataCommitLock);
+
+	// PERSIST: Initial setup of persistentData for a brand new tLog for a new database
+	state IKeyValueStore* storage = self->persistentData;
+	wait(storage->init());
+	storage->set(persistFormat);
+
+	wait(storage->commit());
+
+	TraceEvent("TLogInitPersistentStorageDone", self->dbgid);
 	return Void();
 }
 
@@ -3024,9 +3062,9 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 	}
 
 	if (!fFormat.get().present()) {
-		RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), LiteralStringRef("\xff")), 1));
+		RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), "\xff"_sr), 1));
 		if (!v.size()) {
-			TEST(true); // The DB is completely empty, so it was never initialized.  Delete it.
+			CODE_PROBE(true, "The DB is completely empty, so it was never initialized.  Delete it.");
 			throw worker_removed();
 		} else {
 			// This should never happen
@@ -3038,7 +3076,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 
 	state std::vector<Future<ErrorOr<Void>>> removed;
 
-	ASSERT(fFormat.get().get() == LiteralStringRef("FoundationDB/LogServer/3/0"));
+	ASSERT(fFormat.get().get() == "FoundationDB/LogServer/3/0"_sr);
 
 	ASSERT(fVers.get().size() == fRecoverCounts.get().size());
 
@@ -3119,7 +3157,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 		                                 std::vector<Tag>(),
 		                                 "Restored");
 		logData->locality = id_locality[id1];
-		logData->stopped = true;
+		logData->stop();
 		logData->unblockWaitingPeeks();
 		self->id_data[id1] = logData;
 		id_interf[id1] = recruited;
@@ -3131,8 +3169,8 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 		logData->version.set(ver);
 		logData->recoveryCount =
 		    BinaryReader::fromStringRef<DBRecoveryCount>(fRecoverCounts.get()[idx].value, Unversioned());
-		logData->removed =
-		    rejoinClusterController(self, recruited, logData->recoveryCount, registerWithCC.getFuture(), false);
+		logData->removed = rejoinClusterController(
+		    self, recruited, logData->recoveryCount, logData->stoppedPromise, registerWithCC.getFuture(), false);
 		removed.push_back(errorOr(logData->removed));
 		logsByVersion.emplace_back(ver, id1);
 
@@ -3183,7 +3221,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 			throw end_of_stream();
 		loop {
 			if (allRemoved.isReady()) {
-				TEST(true); // all tlogs removed during queue recovery
+				CODE_PROBE(true, "all tlogs removed during queue recovery");
 				throw worker_removed();
 			}
 			choose {
@@ -3214,7 +3252,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 							logData->queueCommittedVersion.set(qe.version);
 
 							while (self->bytesInput - self->bytesDurable >= recoverMemoryLimit) {
-								TEST(true); // Flush excess data during TLog queue recovery
+								CODE_PROBE(true, "Flush excess data during TLog queue recovery");
 								TraceEvent("FlushLargeQueueDuringRecovery", self->dbgid)
 								    .detail("LogId", logData->logId)
 								    .detail("BytesInput", self->bytesInput)
@@ -3244,7 +3282,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 	}
 
 	TraceEvent("TLogRestorePersistentStateDone", self->dbgid).detail("Took", now() - startt);
-	TEST(now() - startt >= 1.0); // TLog recovery took more than 1 second
+	CODE_PROBE(now() - startt >= 1.0, "TLog recovery took more than 1 second");
 
 	for (auto it : self->id_data) {
 		if (it.second->queueCommittedVersion.get() == 0) {
@@ -3325,7 +3363,7 @@ ACTOR Future<Void> updateLogSystem(TLogData* self,
 
 void stopAllTLogs(TLogData* self, UID newLogId) {
 	for (auto it : self->id_data) {
-		if (!it.second->stopped) {
+		if (!it.second->stopped()) {
 			TraceEvent("TLogStoppedByNewRecruitment", self->dbgid)
 			    .detail("LogId", it.second->logId)
 			    .detail("StoppedId", it.first.toString())
@@ -3338,7 +3376,7 @@ void stopAllTLogs(TLogData* self, UID newLogId) {
 				it.second->committingQueue.sendError(worker_removed());
 			}
 		}
-		it.second->stopped = true;
+		it.second->stop();
 		it.second->unblockWaitingPeeks();
 		if (!it.second->recoveryComplete.isSet()) {
 			it.second->recoveryComplete.sendError(end_of_stream());
@@ -3383,7 +3421,8 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 	logData->locality = req.locality;
 	logData->recoveryCount = req.epoch;
 	logData->recoveryTxnVersion = req.recoveryTransactionVersion;
-	logData->removed = rejoinClusterController(self, recruited, req.epoch, Future<Void>(Void()), req.isPrimary);
+	logData->removed = rejoinClusterController(
+	    self, recruited, req.epoch, logData->stoppedPromise, Future<Void>(Void()), req.isPrimary);
 	self->popOrder.push_back(recruited.id());
 	self->spillOrder.push_back(recruited.id());
 
@@ -3408,8 +3447,10 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 			logData->queueCommittedVersion.set(logData->unrecoveredBefore - 1);
 			logData->version.set(logData->unrecoveredBefore - 1);
 
-			logData->unpoppedRecoveredTags = req.allTags.size();
-			wait(initPersistentState(self, logData) || logData->removed);
+			logData->unpoppedRecoveredTagCount = req.allTags.size();
+			logData->unpoppedRecoveredTags = std::set<Tag>(req.allTags.begin(), req.allTags.end());
+			wait(ioTimeoutError(initPersistentState(self, logData) || logData->removed,
+			                    SERVER_KNOBS->TLOG_MAX_CREATE_DURATION));
 
 			TraceEvent("TLogRecover", self->dbgid)
 			    .detail("LogId", logData->logId)
@@ -3429,7 +3470,7 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 			logData->initialized = true;
 			self->newLogData.trigger();
 
-			if ((req.isPrimary || req.recoverFrom.logRouterTags == 0) && !logData->stopped &&
+			if ((req.isPrimary || req.recoverFrom.logRouterTags == 0) && !logData->stopped() &&
 			    logData->unrecoveredBefore <= recoverAt) {
 				if (req.recoverFrom.logRouterTags > 0 && req.locality != tagLocalitySatellite) {
 					logData->logRouterPopToVersion = recoverAt;
@@ -3450,7 +3491,7 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 			state Version lastVersionPrevEpoch = req.recoverAt;
 
 			if ((req.isPrimary || req.recoverFrom.logRouterTags == 0) &&
-			    logData->version.get() < lastVersionPrevEpoch && !logData->stopped) {
+			    logData->version.get() < lastVersionPrevEpoch && !logData->stopped()) {
 				// Log the changes to the persistent queue, to be committed by commitQueue()
 				TLogQueueEntryRef qe;
 				qe.version = lastVersionPrevEpoch;
@@ -3473,7 +3514,8 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 			logData->addActor.send(respondToRecovered(recruited, logData->recoveryComplete));
 		} else {
 			// Brand new tlog, initialization has already been done by caller
-			wait(initPersistentState(self, logData) || logData->removed);
+			wait(ioTimeoutError(initPersistentState(self, logData) || logData->removed,
+			                    SERVER_KNOBS->TLOG_MAX_CREATE_DURATION));
 
 			if (logData->recoveryComplete.isSet()) {
 				throw worker_removed();
@@ -3502,8 +3544,10 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 	req.reply.send(recruited);
 
 	TraceEvent("TLogReady", logData->logId)
-	    .detail("AllTags", describe(req.allTags))
-	    .detail("Locality", logData->locality);
+	    .detail("Locality", logData->locality)
+	    .setMaxEventLength(11000)
+	    .setMaxFieldLength(10000)
+	    .detail("AllTags", describe(req.allTags));
 
 	updater = Void();
 	wait(tLogCore(self, logData, recruited, pulledRecoveryVersions));
@@ -3540,13 +3584,13 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 	state TLogData self(tlogId, workerID, persistentData, persistentQueue, db, degraded, folder);
 	state Future<Void> error = actorCollection(self.sharedActors.getFuture());
 
-	TraceEvent("SharedTlog", tlogId).log();
+	TraceEvent("SharedTlog", tlogId);
 	try {
 		try {
 			if (restoreFromDisk) {
 				wait(restorePersistentState(&self, locality, oldLog, recovered, tlogRequests));
 			} else {
-				wait(ioTimeoutError(checkEmptyQueue(&self) && checkRecovered(&self),
+				wait(ioTimeoutError(checkEmptyQueue(&self) && initPersistentStorage(&self),
 				                    SERVER_KNOBS->TLOG_MAX_CREATE_DURATION));
 			}
 
@@ -3556,6 +3600,9 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 			if (recovered.canBeSet())
 				recovered.send(Void());
 
+			if (!self.durableClusterId.isValid()) {
+				self.sharedActors.send(updateDurableClusterID(&self));
+			}
 			self.sharedActors.send(commitQueue(&self));
 			self.sharedActors.send(updateStorageLoop(&self));
 			self.sharedActors.send(traceRole(Role::SHARED_TRANSACTION_LOG, tlogId));
@@ -3564,21 +3611,6 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 			loop {
 				choose {
 					when(state InitializeTLogRequest req = waitNext(tlogRequests.getFuture())) {
-						ASSERT(req.clusterId.isValid());
-						// Durably persist the cluster ID if it is not already
-						// durable and the cluster has progressed far enough
-						// through recovery. To avoid different partitions from
-						// persisting different cluster IDs, we need to wait
-						// until a single cluster ID has been persisted in the
-						// txnStateStore before finally writing it to disk.
-						auto recoveryState = self.dbInfo->get().recoveryState;
-						if (!self.durableClusterId.isValid() && recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
-							self.durableClusterId = req.clusterId;
-							// Will let commit loop durably write the cluster ID.
-							self.persistentData->set(
-							    KeyValueRef(persistClusterIdKey, BinaryWriter::toValue(req.clusterId, Unversioned())));
-						}
-
 						if (!self.tlogCache.exists(req.recruitmentID)) {
 							self.tlogCache.set(req.recruitmentID, req.reply.getFuture());
 							self.sharedActors.send(
diff --git a/fdbserver/TSSMappingUtil.actor.cpp b/fdbserver/TSSMappingUtil.actor.cpp
index e501b58998..408138dbbc 100644
--- a/fdbserver/TSSMappingUtil.actor.cpp
+++ b/fdbserver/TSSMappingUtil.actor.cpp
@@ -26,14 +26,15 @@
 ACTOR Future<Void> readTSSMappingRYW(Reference<ReadYourWritesTransaction> tr,
                                      std::map<UID, StorageServerInterface>* tssMapping) {
 	KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
-	state std::vector<std::pair<UID, UID>> uidMapping =
+	state KeyBackedMap<UID, UID>::RangeResultType uidMapping =
 	    wait(tssMapDB.getRange(tr, UID(), Optional<UID>(), CLIENT_KNOBS->TOO_MANY));
-	ASSERT(uidMapping.size() < CLIENT_KNOBS->TOO_MANY);
+	ASSERT(uidMapping.results.size() < CLIENT_KNOBS->TOO_MANY);
 
 	state std::map<UID, StorageServerInterface> mapping;
-	for (auto& it : uidMapping) {
-		state UID ssId = it.first;
-		Optional<Value> v = wait(tr->get(serverListKeyFor(it.second)));
+	state std::vector<std::pair<UID, UID>>::iterator mapItr;
+	for (mapItr = uidMapping.results.begin(); mapItr != uidMapping.results.end(); ++mapItr) {
+		state UID ssId = mapItr->first;
+		Optional<Value> v = wait(tr->get(serverListKeyFor(mapItr->second)));
 		(*tssMapping)[ssId] = decodeServerListValue(v.get());
 	}
 	return Void();
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 7f19ffdf4a..5b34c5b68e 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -2167,7 +2167,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 		}
 	}
 
-	TEST(true); // Master recovery from pre-existing database
+	CODE_PROBE(true, "Master recovery from pre-existing database");
 
 	// trackRejoins listens for rejoin requests from the tLogs that we are recovering from, to learn their
 	// TLogInterfaces
@@ -2228,7 +2228,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 			}
 			if (!lockedLocalities.count(log->locality)) {
 				TraceEvent("EpochEndLockExtra").detail("Locality", log->locality);
-				TEST(true); // locking old generations for version information
+				CODE_PROBE(true, "locking old generations for version information");
 				lockedLocalities.insert(log->locality);
 				LogLockInfo lockResult;
 				lockResult.epochEnd = old.epochEnd;
@@ -2312,7 +2312,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 			changes.push_back(TagPartitionedLogSystem::getDurableVersionChanged(lockResults[log], logFailed[log]));
 		}
 		if (maxEnd > 0 && (!lastEnd.present() || maxEnd < lastEnd.get())) {
-			TEST(lastEnd.present()); // Restarting recovery at an earlier point
+			CODE_PROBE(lastEnd.present(), "Restarting recovery at an earlier point");
 
 			auto logSystem = makeReference<TagPartitionedLogSystem>(dbgid, locality, prevState.recoveryCount);
 
@@ -2614,6 +2614,8 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 		req.tLogLocalities = localities;
 		req.tLogPolicy = logSet->tLogPolicy;
 		req.locality = remoteLocality;
+		TraceEvent("RemoteTLogRouterReplies", self->dbgid)
+		    .detail("WorkerID", remoteWorkers.logRouters[i % remoteWorkers.logRouters.size()].id());
 		logRouterInitializationReplies.push_back(transformErrors(
 		    throwErrorOr(
 		        remoteWorkers.logRouters[i % remoteWorkers.logRouters.size()].logRouter.getReplyUnlessFailedFor(
@@ -2693,11 +2695,13 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 	}
 
 	remoteTLogInitializationReplies.reserve(remoteWorkers.remoteTLogs.size());
-	for (int i = 0; i < remoteWorkers.remoteTLogs.size(); i++)
+	for (int i = 0; i < remoteWorkers.remoteTLogs.size(); i++) {
+		TraceEvent("RemoteTLogReplies", self->dbgid).detail("WorkerID", remoteWorkers.remoteTLogs[i].id());
 		remoteTLogInitializationReplies.push_back(transformErrors(
 		    throwErrorOr(remoteWorkers.remoteTLogs[i].tLog.getReplyUnlessFailedFor(
 		        remoteTLogReqs[i], SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)),
 		    cluster_recovery_failed()));
+	}
 
 	TraceEvent("RemoteLogRecruitment_InitializingRemoteLogs")
 	    .detail("StartVersion", logSet->startVersion)
@@ -2966,11 +2970,13 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 	}
 
 	initializationReplies.reserve(recr.tLogs.size());
-	for (int i = 0; i < recr.tLogs.size(); i++)
+	for (int i = 0; i < recr.tLogs.size(); i++) {
+		TraceEvent("PrimaryTLogReplies", logSystem->getDebugID()).detail("WorkerID", recr.tLogs[i].id());
 		initializationReplies.push_back(transformErrors(
 		    throwErrorOr(recr.tLogs[i].tLog.getReplyUnlessFailedFor(
 		        reqs[i], SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)),
 		    cluster_recovery_failed()));
+	}
 
 	state std::vector<Future<Void>> recoveryComplete;
 
@@ -3034,11 +3040,14 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		}
 
 		satelliteInitializationReplies.reserve(recr.satelliteTLogs.size());
-		for (int i = 0; i < recr.satelliteTLogs.size(); i++)
+		for (int i = 0; i < recr.satelliteTLogs.size(); i++) {
+			TraceEvent("PrimarySatelliteTLogReplies", logSystem->getDebugID())
+			    .detail("WorkerID", recr.satelliteTLogs[i].id());
 			satelliteInitializationReplies.push_back(transformErrors(
 			    throwErrorOr(recr.satelliteTLogs[i].tLog.getReplyUnlessFailedFor(
 			        sreqs[i], SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)),
 			    cluster_recovery_failed()));
+		}
 
 		wait(waitForAll(satelliteInitializationReplies) || oldRouterRecruitment);
 
@@ -3068,7 +3077,7 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 
 	// Don't force failure of recovery if it took us a long time to recover. This avoids multiple long running
 	// recoveries causing tests to timeout
-	if (BUGGIFY && now() - startTime < 300 && g_network->isSimulated() && g_simulator.speedUpSimulation)
+	if (BUGGIFY && now() - startTime < 300 && g_network->isSimulated() && g_simulator->speedUpSimulation)
 		throw cluster_recovery_failed();
 
 	for (int i = 0; i < logSystem->tLogs[0]->logServers.size(); i++)
diff --git a/fdbserver/TagThrottler.actor.cpp b/fdbserver/TagThrottler.actor.cpp
index 432aa459b1..36e11e7e9b 100644
--- a/fdbserver/TagThrottler.actor.cpp
+++ b/fdbserver/TagThrottler.actor.cpp
@@ -52,30 +52,27 @@ class TagThrottlerImpl {
 
 					wait(success(throttledTagKeys) && success(autoThrottlingEnabled));
 
-					if (autoThrottlingEnabled.get().present() &&
-					    autoThrottlingEnabled.get().get() == LiteralStringRef("0")) {
-						TEST(true); // Auto-throttling disabled
+					if (autoThrottlingEnabled.get().present() && autoThrottlingEnabled.get().get() == "0"_sr) {
+						CODE_PROBE(true, "Auto-throttling disabled");
 						if (self->autoThrottlingEnabled) {
 							TraceEvent("AutoTagThrottlingDisabled", self->id).log();
 						}
 						self->autoThrottlingEnabled = false;
-					} else if (autoThrottlingEnabled.get().present() &&
-					           autoThrottlingEnabled.get().get() == LiteralStringRef("1")) {
-						TEST(true); // Auto-throttling enabled
+					} else if (autoThrottlingEnabled.get().present() && autoThrottlingEnabled.get().get() == "1"_sr) {
+						CODE_PROBE(true, "Auto-throttling enabled");
 						if (!self->autoThrottlingEnabled) {
 							TraceEvent("AutoTagThrottlingEnabled", self->id).log();
 						}
 						self->autoThrottlingEnabled = true;
 					} else {
-						TEST(true); // Auto-throttling unspecified
+						CODE_PROBE(true, "Auto-throttling unspecified");
 						if (autoThrottlingEnabled.get().present()) {
 							TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue", self->id)
 							    .detail("Value", autoThrottlingEnabled.get().get());
 						}
 						self->autoThrottlingEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
 						if (!committed)
-							tr.set(tagThrottleAutoEnabledKey,
-							       LiteralStringRef(self->autoThrottlingEnabled ? "1" : "0"));
+							tr.set(tagThrottleAutoEnabledKey, self->autoThrottlingEnabled ? "1"_sr : "0"_sr);
 					}
 
 					RkTagThrottleCollection updatedTagThrottles;
@@ -90,7 +87,7 @@ class TagThrottlerImpl {
 
 						if (tagValue.expirationTime == 0 ||
 						    tagValue.expirationTime > now() + tagValue.initialDuration) {
-							TEST(true); // Converting tag throttle duration to absolute time
+							CODE_PROBE(true, "Converting tag throttle duration to absolute time");
 							tagValue.expirationTime = now() + tagValue.initialDuration;
 							BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
 							wr << tagValue;
@@ -128,7 +125,7 @@ class TagThrottlerImpl {
 
 					wait(watchFuture);
 					TraceEvent("RatekeeperThrottleSignaled", self->id).log();
-					TEST(true); // Tag throttle changes detected
+					CODE_PROBE(true, "Tag throttle changes detected");
 					break;
 				} catch (Error& e) {
 					TraceEvent("RatekeeperMonitorThrottlingChangesError", self->id).error(e);
@@ -142,7 +139,7 @@ class TagThrottlerImpl {
 		// NOTE: before the comparison with MIN_TAG_COST, the busiest tag rate also compares with MIN_TAG_PAGES_RATE
 		// currently MIN_TAG_PAGES_RATE > MIN_TAG_COST in our default knobs.
 		if (busyness > SERVER_KNOBS->AUTO_THROTTLE_TARGET_TAG_BUSYNESS && rate > SERVER_KNOBS->MIN_TAG_COST) {
-			TEST(true); // Transaction tag auto-throttled
+			CODE_PROBE(true, "Transaction tag auto-throttled");
 			Optional<double> clientRate = throttledTags.autoThrottleTag(id, tag, busyness);
 			// TODO: Increment tag throttle counts here?
 			if (clientRate.present()) {
diff --git a/fdbserver/TenantCache.actor.cpp b/fdbserver/TenantCache.actor.cpp
index d3b8d64c4a..b226d016e1 100644
--- a/fdbserver/TenantCache.actor.cpp
+++ b/fdbserver/TenantCache.actor.cpp
@@ -18,23 +18,28 @@
  * limitations under the License.
  */
 
+#include "fdbclient/SystemData.h"
+#include "fdbclient/FDBTypes.h"
 #include "fdbserver/DDTeamCollection.h"
 #include "fdbserver/TenantCache.h"
+#include "flow/flow.h"
 #include <limits>
 #include <string>
 #include "flow/actorcompiler.h"
 
 class TenantCacheImpl {
 
-	ACTOR static Future<RangeResult> getTenantList(TenantCache* tenantCache, Transaction* tr) {
+	ACTOR static Future<std::vector<std::pair<TenantName, TenantMapEntry>>> getTenantList(TenantCache* tenantCache,
+	                                                                                      Transaction* tr) {
 		tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 
-		state Future<RangeResult> tenantList = tr->getRange(tenantMapKeys, CLIENT_KNOBS->TOO_MANY);
-		wait(success(tenantList));
-		ASSERT(!tenantList.get().more && tenantList.get().size() < CLIENT_KNOBS->TOO_MANY);
+		KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenantList =
+		    wait(TenantMetadata::tenantMap().getRange(
+		        tr, Optional<TenantName>(), Optional<TenantName>(), CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1));
+		ASSERT(tenantList.results.size() <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER && !tenantList.more);
 
-		return tenantList.get();
+		return tenantList.results;
 	}
 
 public:
@@ -44,18 +49,15 @@ public:
 		TraceEvent(SevInfo, "BuildingTenantCache", tenantCache->id()).log();
 
 		try {
-			state RangeResult tenantList = wait(getTenantList(tenantCache, &tr));
+			state std::vector<std::pair<TenantName, TenantMapEntry>> tenantList = wait(getTenantList(tenantCache, &tr));
 
 			for (int i = 0; i < tenantList.size(); i++) {
-				TenantName tname = tenantList[i].key.removePrefix(tenantMapPrefix);
-				TenantMapEntry t = TenantMapEntry::decode(tenantList[i].value);
-
-				tenantCache->insert(tname, t);
+				tenantCache->insert(tenantList[i].first, tenantList[i].second);
 
 				TraceEvent(SevInfo, "TenantFound", tenantCache->id())
-				    .detail("TenantName", tname)
-				    .detail("TenantID", t.id)
-				    .detail("TenantPrefix", t.prefix);
+				    .detail("TenantName", tenantList[i].first)
+				    .detail("TenantID", tenantList[i].second.id)
+				    .detail("TenantPrefix", tenantList[i].second.prefix);
 			}
 		} catch (Error& e) {
 			wait(tr.onError(e));
@@ -79,17 +81,17 @@ public:
 					TraceEvent(SevWarn, "TenantListRefreshDelay", tenantCache->id()).log();
 				}
 
-				state RangeResult tenantList = wait(getTenantList(tenantCache, &tr));
+				state std::vector<std::pair<TenantName, TenantMapEntry>> tenantList =
+				    wait(getTenantList(tenantCache, &tr));
 
 				tenantCache->startRefresh();
 				bool tenantListUpdated = false;
 
 				for (int i = 0; i < tenantList.size(); i++) {
-					TenantName tname = tenantList[i].key.removePrefix(tenantMapPrefix);
-					TenantMapEntry t = TenantMapEntry::decode(tenantList[i].value);
-
-					if (tenantCache->update(tname, t)) {
+					if (tenantCache->update(tenantList[i].first, tenantList[i].second)) {
 						tenantListUpdated = true;
+						TenantCacheTenantCreated req(tenantList[i].second.prefix);
+						tenantCache->tenantCreationSignal.send(req);
 					}
 				}
 
@@ -120,7 +122,7 @@ void TenantCache::insert(TenantName& tenantName, TenantMapEntry& tenant) {
 	KeyRef tenantPrefix(tenant.prefix.begin(), tenant.prefix.size());
 	ASSERT(tenantCache.find(tenantPrefix) == tenantCache.end());
 
-	TenantInfo tenantInfo(tenantName, tenant.id);
+	TenantInfo tenantInfo(tenantName, Optional<Standalone<StringRef>>(), tenant.id);
 	tenantCache[tenantPrefix] = makeReference<TCTenantInfo>(tenantInfo, tenant.prefix);
 	tenantCache[tenantPrefix]->updateCacheGeneration(generation);
 }
@@ -177,7 +179,7 @@ std::string TenantCache::desc() const {
 			s += ", ";
 		}
 
-		s += "Name: " + tenant->name().toString() + " Prefix: " + tenantPrefix.printable();
+		s += "Name: " + tenant->name().toString() + " Prefix: " + tenantPrefix.toString();
 		count++;
 	}
 
@@ -197,10 +199,23 @@ bool TenantCache::isTenantKey(KeyRef key) const {
 	return true;
 }
 
-Future<Void> TenantCache::build(Database cx) {
+Future<Void> TenantCache::build() {
 	return TenantCacheImpl::build(this);
 }
 
+Optional<Reference<TCTenantInfo>> TenantCache::tenantOwning(KeyRef key) const {
+	auto it = tenantCache.lastLessOrEqual(key);
+	if (it == tenantCache.end()) {
+		return {};
+	}
+
+	if (!key.startsWith(it->key)) {
+		return {};
+	}
+
+	return it->value;
+}
+
 Future<Void> TenantCache::monitorTenantMap() {
 	return TenantCacheImpl::monitorTenantMap(this);
 }
@@ -220,7 +235,7 @@ public:
 
 		for (uint16_t i = 0; i < tenantCount; i++) {
 			TenantName tenantName(format("%s_%08d", "ddtc_test_tenant", tenantNumber + i));
-			TenantMapEntry tenant(tenantNumber + i, KeyRef());
+			TenantMapEntry tenant(tenantNumber + i, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION);
 
 			tenantCache.insert(tenantName, tenant);
 		}
@@ -248,7 +263,7 @@ public:
 
 		for (uint16_t i = 0; i < tenantCount; i++) {
 			TenantName tenantName(format("%s_%08d", "ddtc_test_tenant", tenantNumber + i));
-			TenantMapEntry tenant(tenantNumber + i, KeyRef());
+			TenantMapEntry tenant(tenantNumber + i, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION);
 
 			tenantCache.insert(tenantName, tenant);
 		}
@@ -262,7 +277,7 @@ public:
 
 			if (tenantOrdinal % staleTenantFraction != 0) {
 				TenantName tenantName(format("%s_%08d", "ddtc_test_tenant", tenantOrdinal));
-				TenantMapEntry tenant(tenantOrdinal, KeyRef());
+				TenantMapEntry tenant(tenantOrdinal, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION);
 				bool newTenant = tenantCache.update(tenantName, tenant);
 				ASSERT(!newTenant);
 				keepCount++;
diff --git a/fdbserver/TransactionTagCounter.cpp b/fdbserver/TransactionTagCounter.cpp
index 7b7829f676..17e7539bad 100644
--- a/fdbserver/TransactionTagCounter.cpp
+++ b/fdbserver/TransactionTagCounter.cpp
@@ -90,7 +90,8 @@ class TransactionTagCounterImpl {
 	std::vector<StorageQueuingMetricsReply::TagInfo> previousBusiestTags;
 	Reference<EventCacheHolder> busiestReadTagEventHolder;
 
-	static int64_t costFunction(int64_t bytes) { return bytes / SERVER_KNOBS->READ_COST_BYTE_FACTOR + 1; }
+	// Round up to the nearest page size
+	static int64_t costFunction(int64_t bytes) { return (bytes - 1) / CLIENT_KNOBS->READ_COST_BYTE_FACTOR + 1; }
 
 public:
 	TransactionTagCounterImpl(UID thisServerID)
@@ -99,7 +100,7 @@ public:
 
 	void addRequest(Optional<TagSet> const& tags, int64_t bytes) {
 		if (tags.present()) {
-			TEST(true); // Tracking transaction tag in counter
+			CODE_PROBE(true, "Tracking transaction tag in counter");
 			double cost = costFunction(bytes);
 			for (auto& tag : tags.get()) {
 				int64_t& count = intervalCounts[TransactionTag(tag, tags.get().getArena())];
@@ -117,13 +118,29 @@ public:
 		if (intervalStart > 0 && CLIENT_KNOBS->READ_TAG_SAMPLE_RATE > 0 && elapsed > 0) {
 			previousBusiestTags = topTags.getBusiestTags(elapsed, intervalTotalSampledCount);
 
-			TraceEvent("BusiestReadTag", thisServerID)
-			    .detail("Elapsed", elapsed)
-			    //.detail("Tag", printable(busiestTag))
-			    //.detail("TagCost", busiestTagCount)
-			    .detail("TotalSampledCost", intervalTotalSampledCount)
-			    .detail("Reported", previousBusiestTags.size())
-			    .trackLatest(busiestReadTagEventHolder->trackingKey);
+			// For status, report the busiest tag:
+			if (previousBusiestTags.empty()) {
+				TraceEvent("BusiestReadTag", thisServerID).detail("TagCost", 0);
+			} else {
+				auto busiestTagInfo = previousBusiestTags[0];
+				for (int i = 1; i < previousBusiestTags.size(); ++i) {
+					auto const& tagInfo = previousBusiestTags[i];
+					if (tagInfo.rate > busiestTagInfo.rate) {
+						busiestTagInfo = tagInfo;
+					}
+				}
+				TraceEvent("BusiestReadTag", thisServerID)
+				    .detail("Tag", printable(busiestTagInfo.tag))
+				    .detail("TagCost", busiestTagInfo.rate)
+				    .detail("FractionalBusyness", busiestTagInfo.fractionalBusyness);
+			}
+
+			for (const auto& tagInfo : previousBusiestTags) {
+				TraceEvent("BusyReadTag", thisServerID)
+				    .detail("Tag", printable(tagInfo.tag))
+				    .detail("TagCost", tagInfo.rate)
+				    .detail("FractionalBusyness", tagInfo.fractionalBusyness);
+			}
 		}
 
 		intervalCounts.clear();
diff --git a/fdbserver/VFSAsync.cpp b/fdbserver/VFSAsync.cpp
index 9ea9007203..36aa65588e 100644
--- a/fdbserver/VFSAsync.cpp
+++ b/fdbserver/VFSAsync.cpp
@@ -731,7 +731,7 @@ static int asyncSleep(sqlite3_vfs* pVfs, int microseconds) {
 	try {
 		Future<Void> simCancel = Never();
 		if (g_network->isSimulated())
-			simCancel = success(g_simulator.getCurrentProcess()->shutdownSignal.getFuture());
+			simCancel = success(g_simulator->getCurrentProcess()->shutdownSignal.getFuture());
 		if (simCancel.isReady()) {
 			waitFor(delay(FLOW_KNOBS->MAX_BUGGIFIED_DELAY));
 			return 0;
diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp
index 98bd0f684a..cd34ea49d1 100644
--- a/fdbserver/VersionedBTree.actor.cpp
+++ b/fdbserver/VersionedBTree.actor.cpp
@@ -18,37 +18,40 @@
  * limitations under the License.
  */
 
-#include "fmt/format.h"
+#include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
+#include "fdbclient/Tuple.h"
+#include "fdbrpc/ContinuousSample.h"
+#include "fdbrpc/simulator.h"
+#include "fdbserver/DeltaTree.h"
+#include "fdbserver/IKeyValueStore.h"
+#include "fdbserver/IPager.h"
 #include "fdbserver/Knobs.h"
+#include "fdbserver/WorkerInterface.actor.h"
+#include "flow/ActorCollection.h"
 #include "flow/Error.h"
+#include "flow/FastRef.h"
+#include "flow/flow.h"
+#include "flow/genericactors.actor.h"
+#include "flow/Histogram.h"
+#include "flow/IAsyncFile.h"
 #include "flow/IRandom.h"
 #include "flow/Knobs.h"
 #include "flow/ObjectSerializer.h"
-#include "flow/Trace.h"
-#include "flow/flow.h"
-#include "flow/Histogram.h"
-#include <limits>
-#include <random>
-#include "fdbrpc/ContinuousSample.h"
-#include "fdbrpc/simulator.h"
-#include "fdbserver/IPager.h"
-#include "fdbclient/Tuple.h"
 #include "flow/serialize.h"
-#include "flow/genericactors.actor.h"
+#include "flow/Trace.h"
 #include "flow/UnitTest.h"
-#include "flow/IAsyncFile.h"
-#include "flow/ActorCollection.h"
+#include "fmt/format.h"
+
+#include <boost/intrusive/list.hpp>
+#include <cinttypes>
+#include <limits>
 #include <map>
+#include <random>
 #include <string>
 #include <unordered_map>
 #include <vector>
-#include "fdbclient/CommitTransaction.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/DeltaTree.h"
-#include <string.h>
-#include <cinttypes>
-#include <boost/intrusive/list.hpp>
+
 #include "flow/actorcompiler.h" // must be last include
 
 #define REDWOOD_DEBUG 0
@@ -2192,15 +2195,15 @@ public:
 	          int64_t remapCleanupWindowBytes,
 	          int concurrentExtentReads,
 	          bool memoryOnly,
-	          std::shared_ptr<IEncryptionKeyProvider> keyProvider,
+	          Reference<IPageEncryptionKeyProvider> keyProvider,
 	          Promise<Void> errorPromise = {})
 	  : keyProvider(keyProvider), ioLock(FLOW_KNOBS->MAX_OUTSTANDING, ioMaxPriority, FLOW_KNOBS->MAX_OUTSTANDING / 2),
 	    pageCacheBytes(pageCacheSizeBytes), desiredPageSize(desiredPageSize), desiredExtentSize(desiredExtentSize),
 	    filename(filename), memoryOnly(memoryOnly), errorPromise(errorPromise),
 	    remapCleanupWindowBytes(remapCleanupWindowBytes), concurrentExtentReads(new FlowLock(concurrentExtentReads)) {
 
-		if (keyProvider == nullptr) {
-			keyProvider = std::make_shared<NullKeyProvider>();
+		if (!keyProvider) {
+			keyProvider = makeReference<NullKeyProvider>();
 		}
 
 		// This sets the page cache size for all PageCacheT instances using the same evictor
@@ -2963,11 +2966,8 @@ public:
 		             page->rawData(),
 		             header);
 
-		int readBytes = wait(readPhysicalBlock(self,
-		                                       page->rawData(),
-		                                       page->rawSize(),
-		                                       (int64_t)pageID * page->rawSize(),
-		                                       std::min(priority, ioMaxPriority)));
+		int readBytes = wait(
+		    readPhysicalBlock(self, page->rawData(), page->rawSize(), (int64_t)pageID * page->rawSize(), priority));
 		debug_printf("DWALPager(%s) op=readPhysicalDiskReadComplete %s ptr=%p bytes=%d\n",
 		             self->filename.c_str(),
 		             toString(pageID).c_str(),
@@ -2977,7 +2977,7 @@ public:
 		try {
 			page->postReadHeader(pageID);
 			if (page->isEncrypted()) {
-				EncryptionKey k = wait(self->keyProvider->getSecrets(page->encryptionKey));
+				ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader()));
 				page->encryptionKey = k;
 			}
 			page->postReadPayload(pageID);
@@ -2987,7 +2987,7 @@ public:
 			             page->rawData());
 		} catch (Error& e) {
 			Error err = e;
-			if (g_network->isSimulated() && g_simulator.checkInjectedCorruption()) {
+			if (g_network->isSimulated() && g_simulator->checkInjectedCorruption()) {
 				err = err.asInjectedFault();
 			}
 
@@ -3045,7 +3045,7 @@ public:
 		try {
 			page->postReadHeader(pageIDs.front());
 			if (page->isEncrypted()) {
-				EncryptionKey k = wait(self->keyProvider->getSecrets(page->encryptionKey));
+				ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader()));
 				page->encryptionKey = k;
 			}
 			page->postReadPayload(pageIDs.front());
@@ -3958,7 +3958,7 @@ private:
 	int physicalExtentSize;
 	int pagesPerExtent;
 
-	std::shared_ptr<IEncryptionKeyProvider> keyProvider;
+	Reference<IPageEncryptionKeyProvider> keyProvider;
 
 	PriorityMultiLock ioLock;
 
@@ -4205,7 +4205,6 @@ std::string toString(BTreeNodeLinkRef id) {
 	return std::string("BTreePageID") + toString(id.begin(), id.end());
 }
 
-#define STR(x) LiteralStringRef(x)
 struct RedwoodRecordRef {
 	typedef uint8_t byte;
 
@@ -4772,6 +4771,8 @@ struct DecodeBoundaryVerifier {
 	struct DecodeBoundaries {
 		Key lower;
 		Key upper;
+		unsigned int height;
+		Optional<int64_t> domainId;
 		bool empty() const { return lower.empty() && upper.empty(); }
 	};
 
@@ -4780,16 +4781,23 @@ struct DecodeBoundaryVerifier {
 	std::vector<Key> boundarySamples;
 	int boundarySampleSize = 1000;
 	int boundaryPopulation = 0;
+	Reference<IPageEncryptionKeyProvider> keyProvider;
+
+	// Sample rate of pages to be scanned to verify if all entries in the page meet domain prefix requirement.
+	double domainPrefixScanProbability = 0.01;
+	uint64_t domainPrefixScanCount = 0;
 
 	static DecodeBoundaryVerifier* getVerifier(std::string name) {
 		static std::map<std::string, DecodeBoundaryVerifier> verifiers;
 		// Only use verifier in a non-restarted simulation so that all page writes are captured
-		if (g_network->isSimulated() && !g_simulator.restarted) {
+		if (g_network->isSimulated() && !g_simulator->restarted) {
 			return &verifiers[name];
 		}
 		return nullptr;
 	}
 
+	void setKeyProvider(Reference<IPageEncryptionKeyProvider> kp) { keyProvider = kp; }
+
 	void sampleBoundary(Key b) {
 		if (boundaryPopulation <= boundarySampleSize) {
 			boundarySamples.push_back(b);
@@ -4806,21 +4814,53 @@ struct DecodeBoundaryVerifier {
 		return boundarySamples[deterministicRandom()->randomInt(0, boundarySamples.size())];
 	}
 
-	void update(BTreeNodeLinkRef id, Version v, Key lowerBound, Key upperBound) {
+	bool update(BTreeNodeLinkRef id,
+	            Version v,
+	            Key lowerBound,
+	            Key upperBound,
+	            unsigned int height,
+	            Optional<int64_t> domainId) {
 		sampleBoundary(lowerBound);
 		sampleBoundary(upperBound);
-		debug_printf("decodeBoundariesUpdate %s %s '%s' to '%s'\n",
+		debug_printf("decodeBoundariesUpdate %s %s '%s' to '%s', %u, %s\n",
 		             ::toString(id).c_str(),
 		             ::toString(v).c_str(),
 		             lowerBound.printable().c_str(),
-		             upperBound.printable().c_str());
+		             upperBound.printable().c_str(),
+		             height,
+		             Traceable<decltype(domainId)>::toString(domainId).c_str());
+
+		if (domainId.present()) {
+			ASSERT(keyProvider && keyProvider->enableEncryptionDomain());
+			// Temporarily disabling the check, since if a tenant is removed, where the key provider
+			// would not find the domain, the data for the tenant may still be in Redwood and being read.
+			// TODO(yiwu): re-enable the check.
+			/*
+			if (domainId.get() != keyProvider->getDefaultEncryptionDomainId() &&
+			    !keyProvider->keyFitsInDomain(domainId.get(), lowerBound, false)) {
+			    fprintf(stderr,
+			            "Page lower bound not in domain: %s %s, domain id %s, lower bound '%s'\n",
+			            ::toString(id).c_str(),
+			            ::toString(v).c_str(),
+			            ::toString(domainId).c_str(),
+			            lowerBound.printable().c_str());
+			    return false;
+			}
+			*/
+		}
 
 		auto& b = boundariesByPageID[id.front()][v];
 		ASSERT(b.empty());
-		b = { lowerBound, upperBound };
+		b = { lowerBound, upperBound, height, domainId };
+		return true;
 	}
 
-	bool verify(LogicalPageID id, Version v, Key lowerBound, Key upperBound) {
+	bool verify(LogicalPageID id,
+	            Version v,
+	            Key lowerBound,
+	            Key upperBound,
+	            Optional<int64_t> domainId,
+	            BTreePage::BinaryTree::Cursor& cursor) {
 		auto i = boundariesByPageID.find(id);
 		ASSERT(i != boundariesByPageID.end());
 		ASSERT(!i->second.empty());
@@ -4839,10 +4879,66 @@ struct DecodeBoundaryVerifier {
 			        b->second.upper.printable().c_str());
 			return false;
 		}
+
+		if (!b->second.domainId.present()) {
+			ASSERT(!keyProvider || !keyProvider->enableEncryptionDomain());
+			ASSERT(!domainId.present());
+		} else {
+			ASSERT(keyProvider->enableEncryptionDomain());
+			if (b->second.domainId != domainId) {
+				fprintf(stderr,
+				        "Page encrypted with incorrect domain: %s %s, using %s, written %s\n",
+				        ::toString(id).c_str(),
+				        ::toString(v).c_str(),
+				        ::toString(domainId).c_str(),
+				        ::toString(b->second.domainId).c_str());
+				return false;
+			}
+			// Temporarily disabling the check, since if a tenant is removed, where the key provider
+			// would not find the domain, the data for the tenant may still be in Redwood and being read.
+			// TODO(yiwu): re-enable the check.
+			/*
+			ASSERT(domainId.present());
+			auto checkKeyFitsInDomain = [&]() -> bool {
+			    if (!keyProvider->keyFitsInDomain(domainId.get(), cursor.get().key, b->second.height > 1)) {
+			        fprintf(stderr,
+			                "Encryption domain mismatch on %s, %s, domain: %s, key %s\n",
+			                ::toString(id).c_str(),
+			                ::toString(v).c_str(),
+			                ::toString(domainId).c_str(),
+			                cursor.get().key.printable().c_str());
+			        return false;
+			    }
+			    return true;
+			};
+			if (domainId.get() != keyProvider->getDefaultEncryptionDomainId()) {
+			    cursor.moveFirst();
+			    if (cursor.valid() && !checkKeyFitsInDomain()) {
+			        return false;
+			    }
+			    cursor.moveLast();
+			    if (cursor.valid() && !checkKeyFitsInDomain()) {
+			        return false;
+			    }
+			} else {
+			    if (deterministicRandom()->random01() < domainPrefixScanProbability) {
+			        cursor.moveFirst();
+			        while (cursor.valid()) {
+			            if (!checkKeyFitsInDomain()) {
+			                return false;
+			            }
+			            cursor.moveNext();
+			        }
+			        domainPrefixScanCount++;
+			    }
+			}
+			*/
+		}
+
 		return true;
 	}
 
-	void update(Version v, LogicalPageID oldID, LogicalPageID newID) {
+	void updatePageId(Version v, LogicalPageID oldID, LogicalPageID newID) {
 		auto& old = boundariesByPageID[oldID];
 		ASSERT(!old.empty());
 		auto i = old.end();
@@ -4954,6 +5050,9 @@ public:
 		constexpr static FileIdentifier file_identifier = 10847329;
 		constexpr static unsigned int FORMAT_VERSION = 17;
 
+		// Maximum size of the root pointer
+		constexpr static int maxRootPointerSize = 3000 / sizeof(LogicalPageID);
+
 		// This serves as the format version for the entire tree, individual pages will not be versioned
 		uint32_t formatVersion;
 		EncodingType encodingType;
@@ -5036,7 +5135,7 @@ public:
 	VersionedBTree(IPager2* pager,
 	               std::string name,
 	               EncodingType defaultEncodingType,
-	               std::shared_ptr<IEncryptionKeyProvider> keyProvider)
+	               Reference<IPageEncryptionKeyProvider> keyProvider)
 	  : m_pager(pager), m_encodingType(defaultEncodingType), m_enforceEncodingType(false), m_keyProvider(keyProvider),
 	    m_pBuffer(nullptr), m_mutationCount(0), m_name(name) {
 
@@ -5044,16 +5143,20 @@ public:
 		// This prevents an attack where an encrypted page is replaced by an attacker with an unencrypted page
 		// or an encrypted page fabricated using a compromised scheme.
 		if (ArenaPage::isEncodingTypeEncrypted(m_encodingType)) {
-			ASSERT(keyProvider != nullptr);
+			ASSERT(keyProvider.isValid());
 			m_enforceEncodingType = true;
 		}
 
 		// If key provider isn't given, instantiate the null provider
-		if (m_keyProvider == nullptr) {
-			m_keyProvider = std::make_shared<NullKeyProvider>();
+		if (!m_keyProvider) {
+			m_keyProvider = makeReference<NullKeyProvider>();
 		}
 
 		m_pBoundaryVerifier = DecodeBoundaryVerifier::getVerifier(name);
+		if (m_pBoundaryVerifier != nullptr) {
+			m_pBoundaryVerifier->setKeyProvider(m_keyProvider);
+		}
+
 		m_pDecodeCacheMemory = m_pager->getPageCachePenaltySource();
 		m_lazyClearActor = 0;
 		m_init = init_impl(this);
@@ -5064,7 +5167,7 @@ public:
 		state Reference<ArenaPage> page = self->m_pager->newPageBuffer();
 		page->init(self->m_encodingType, PageType::BTreeNode, 1);
 		if (page->isEncrypted()) {
-			EncryptionKey k = wait(self->m_keyProvider->getByRange(dbBegin.key, dbEnd.key));
+			ArenaPage::EncryptionKey k = wait(self->m_keyProvider->getLatestDefaultEncryptionKey());
 			page->encryptionKey = k;
 		}
 
@@ -5236,6 +5339,17 @@ public:
 			self->m_lazyClearQueue.recover(self->m_pager, self->m_header.lazyDeleteQueue, "LazyClearQueueRecovered");
 			debug_printf("BTree recovered.\n");
 
+			if (ArenaPage::isEncodingTypeEncrypted(self->m_header.encodingType) &&
+			    self->m_encodingType == EncodingType::XXHash64) {
+				// On restart the encryption config of the cluster could be unknown. In that case if we find the Redwood
+				// instance is encrypted, we should use the same encryption encoding.
+				self->m_encodingType = self->m_header.encodingType;
+				self->m_enforceEncodingType = true;
+				TraceEvent("RedwoodBTreeNodeForceEncryption")
+				    .detail("InstanceName", self->m_pager->getName())
+				    .detail("EncodingFound", self->m_header.encodingType)
+				    .detail("EncodingDesired", self->m_encodingType);
+			}
 			if (self->m_header.encodingType != self->m_encodingType) {
 				TraceEvent(SevWarn, "RedwoodBTreeNodeEncodingMismatch")
 				    .detail("InstanceName", self->m_pager->getName())
@@ -5532,7 +5646,7 @@ private:
 	IPager2* m_pager;
 	EncodingType m_encodingType;
 	bool m_enforceEncodingType;
-	std::shared_ptr<IEncryptionKeyProvider> m_keyProvider;
+	Reference<IPageEncryptionKeyProvider> m_keyProvider;
 
 	// Counter to update with DecodeCache memory usage
 	int64_t* m_pDecodeCacheMemory = nullptr;
@@ -5564,16 +5678,27 @@ private:
 
 	// Describes a range of a vector of records that should be built into a single BTreePage
 	struct PageToBuild {
-		PageToBuild(int index, int blockSize, EncodingType t)
+		PageToBuild(int index,
+		            int blockSize,
+		            EncodingType encodingType,
+		            unsigned int height,
+		            bool useEncryptionDomain,
+		            bool splitByDomain,
+		            IPageEncryptionKeyProvider* keyProvider)
 		  : startIndex(index), count(0), pageSize(blockSize),
 		    largeDeltaTree(pageSize > BTreePage::BinaryTree::SmallSizeLimit), blockSize(blockSize), blockCount(1),
-		    kvBytes(0) {
+		    kvBytes(0), encodingType(encodingType), height(height), useEncryptionDomain(useEncryptionDomain),
+		    splitByDomain(splitByDomain), keyProvider(keyProvider) {
 
 			// Subtrace Page header overhead, BTreePage overhead, and DeltaTree (BTreePage::BinaryTree) overhead.
-			bytesLeft = ArenaPage::getUsableSize(blockSize, t) - sizeof(BTreePage) - sizeof(BTreePage::BinaryTree);
+			bytesLeft =
+			    ArenaPage::getUsableSize(blockSize, encodingType) - sizeof(BTreePage) - sizeof(BTreePage::BinaryTree);
 		}
 
-		PageToBuild next(EncodingType t) { return PageToBuild(endIndex(), blockSize, t); }
+		PageToBuild next() {
+			return PageToBuild(
+			    endIndex(), blockSize, encodingType, height, useEncryptionDomain, splitByDomain, keyProvider);
+		}
 
 		int startIndex; // Index of the first record
 		int count; // Number of records added to the page
@@ -5585,6 +5710,16 @@ private:
 		int blockCount; // The number of blocks in pageSize
 		int kvBytes; // The amount of user key/value bytes added to the page
 
+		EncodingType encodingType;
+		unsigned int height;
+		bool useEncryptionDomain;
+		bool splitByDomain;
+		IPageEncryptionKeyProvider* keyProvider;
+
+		Optional<int64_t> domainId = Optional<int64_t>();
+		size_t domainPrefixLength = 0;
+		bool canUseDefaultDomain = false;
+
 		// Number of bytes used by the generated/serialized BTreePage, including all headers
 		int usedBytes() const { return pageSize - bytesLeft; }
 
@@ -5604,17 +5739,18 @@ private:
 		int endIndex() const { return startIndex + count; }
 
 		std::string toString() const {
-			return format(
-			    "{start=%d count=%d used %d/%d bytes (%.2f%% slack) kvBytes=%d blocks=%d blockSize=%d large=%d}",
-			    startIndex,
-			    count,
-			    usedBytes(),
-			    pageSize,
-			    slackFraction() * 100,
-			    kvBytes,
-			    blockCount,
-			    blockSize,
-			    largeDeltaTree);
+			return format("{start=%d count=%d used %d/%d bytes (%.2f%% slack) kvBytes=%d blocks=%d blockSize=%d "
+			              "large=%d, domain=%s}",
+			              startIndex,
+			              count,
+			              usedBytes(),
+			              pageSize,
+			              slackFraction() * 100,
+			              kvBytes,
+			              blockCount,
+			              blockSize,
+			              largeDeltaTree,
+			              ::toString(domainId).c_str());
 		}
 
 		// Move an item from a to b if a has 2 or more items and the item fits in b
@@ -5646,7 +5782,8 @@ private:
 		// Try to add a record of the given delta size to the page.
 		// If force is true, the page will be expanded to make the record fit if needed.
 		// Return value is whether or not the record was added to the page.
-		bool addRecord(const RedwoodRecordRef& rec, int deltaSize, bool force) {
+		bool addRecord(const RedwoodRecordRef& rec, const RedwoodRecordRef& nextRecord, int deltaSize, bool force) {
+
 			int nodeSize = deltaSize + BTreePage::BinaryTree::Node::headerSize(largeDeltaTree);
 
 			// If the record doesn't fit and the page can't be expanded then return false
@@ -5654,6 +5791,53 @@ private:
 				return false;
 			}
 
+			if (useEncryptionDomain) {
+				int64_t defaultDomainId = keyProvider->getDefaultEncryptionDomainId();
+				int64_t currentDomainId;
+				size_t prefixLength;
+				if (count == 0 || (splitByDomain && count > 0)) {
+					std::tie(currentDomainId, prefixLength) = keyProvider->getEncryptionDomain(rec.key, domainId);
+				}
+				if (count == 0) {
+					domainId = currentDomainId;
+					domainPrefixLength = prefixLength;
+					canUseDefaultDomain =
+					    (height > 1 && (currentDomainId == defaultDomainId || prefixLength == rec.key.size()));
+				} else if (splitByDomain) {
+					ASSERT(domainId.present());
+					if (domainId == currentDomainId) {
+						// The new record falls in the same domain as the rest of the page.
+						// Since this is not the first record, the key must contain a non-prefix portion,
+						// so we cannot use the default domain the encrypt the page (unless domainId is the default
+						// domain).
+						if (domainId != defaultDomainId) {
+							ASSERT(prefixLength < rec.key.size());
+							canUseDefaultDomain = false;
+						}
+					} else if (canUseDefaultDomain &&
+					           (currentDomainId == defaultDomainId ||
+					            (prefixLength == rec.key.size() &&
+					             (nextRecord.key.empty() ||
+					              !nextRecord.key.startsWith(rec.key.substr(0, prefixLength)))))) {
+						// The new record meets one of the following conditions:
+						//   1. it falls in the default domain, or
+						//   2. its key contain only the domain prefix, and
+						//     2a. the following record doesn't fall in the same domain.
+						// In this case switch to use the default domain to encrypt the page.
+						// Condition 2a is needed, because if there are multiple records from the same domain,
+						// they need to form their own page(s).
+						domainId = defaultDomainId;
+						domainPrefixLength = 0;
+					} else {
+						// The new record doesn't fit in the same domain as the existing page.
+						return false;
+					}
+				} else {
+					ASSERT(domainPrefixLength < rec.key.size());
+					canUseDefaultDomain = false;
+				}
+			}
+
 			++count;
 			bytesLeft -= nodeSize;
 			kvBytes += rec.kvBytes();
@@ -5676,6 +5860,12 @@ private:
 			}
 			return true;
 		}
+
+		void finish() {
+			if (useEncryptionDomain && canUseDefaultDomain) {
+				domainId = keyProvider->getDefaultEncryptionDomainId();
+			}
+		}
 	};
 
 	// Scans a vector of records and decides on page split points, returning a vector of 1+ pages to build
@@ -5695,8 +5885,25 @@ private:
 		// Leaves can have just one record if it's large, but internal pages should have at least 4
 		int minRecords = height == 1 ? 1 : 4;
 		double maxSlack = SERVER_KNOBS->REDWOOD_PAGE_REBUILD_MAX_SLACK;
+		RedwoodRecordRef emptyRecord;
 		std::vector<PageToBuild> pages;
 
+		// Whether encryption is used and we need to set encryption domain for a page.
+		bool useEncryptionDomain =
+		    ArenaPage::isEncodingTypeEncrypted(m_encodingType) && m_keyProvider->enableEncryptionDomain();
+		// Whether we may need to split by encryption domain. It is mean to be an optimization to avoid
+		// unnecessary domain check and may not be exhaust all cases.
+		bool splitByDomain = false;
+		if (useEncryptionDomain && records.size() > 1) {
+			int64_t firstDomain = std::get<0>(m_keyProvider->getEncryptionDomain(records[0].key));
+			int64_t lastDomain = std::get<0>(m_keyProvider->getEncryptionDomain(records[records.size() - 1].key));
+			// If the two record falls in the same non-default domain, we know all the records fall in the
+			// same domain. Otherwise we may need to split pages by domain.
+			if (firstDomain != lastDomain || firstDomain == m_keyProvider->getDefaultEncryptionDomainId()) {
+				splitByDomain = true;
+			}
+		}
+
 		// deltaSizes contains pair-wise delta sizes for [lowerBound, records..., upperBound]
 		std::vector<int> deltaSizes(records.size() + 1);
 		deltaSizes.front() = records.front().deltaSize(*lowerBound, prefixLen, true);
@@ -5705,28 +5912,34 @@ private:
 			deltaSizes[i] = records[i].deltaSize(records[i - 1], prefixLen, true);
 		}
 
-		PageToBuild p(0, m_blockSize, m_encodingType);
+		PageToBuild p(
+		    0, m_blockSize, m_encodingType, height, useEncryptionDomain, splitByDomain, m_keyProvider.getPtr());
 
-		for (int i = 0; i < records.size(); ++i) {
+		for (int i = 0; i < records.size();) {
 			bool force = p.count < minRecords || p.slackFraction() > maxSlack;
-			debug_printf(
-			    "  before addRecord  i=%d  records=%d  deltaSize=%d  kvSize=%d  force=%d  pageToBuild=%s  record=%s",
-			    i,
-			    records.size(),
-			    deltaSizes[i],
-			    records[i].kvBytes(),
-			    force,
-			    p.toString().c_str(),
-			    records[i].toString(height == 1).c_str());
+			if (i == 0 || p.count > 0) {
+				debug_printf("  before addRecord  i=%d  records=%d  deltaSize=%d  kvSize=%d  force=%d  pageToBuild=%s  "
+				             "record=%s",
+				             i,
+				             records.size(),
+				             deltaSizes[i],
+				             records[i].kvBytes(),
+				             force,
+				             p.toString().c_str(),
+				             records[i].toString(height == 1).c_str());
+			}
 
-			if (!p.addRecord(records[i], deltaSizes[i], force)) {
+			if (!p.addRecord(records[i], i + 1 < records.size() ? records[i + 1] : emptyRecord, deltaSizes[i], force)) {
+				p.finish();
 				pages.push_back(p);
-				p = p.next(m_encodingType);
-				p.addRecord(records[i], deltaSizes[i], true);
+				p = p.next();
+			} else {
+				i++;
 			}
 		}
 
 		if (p.count > 0) {
+			p.finish();
 			pages.push_back(p);
 		}
 
@@ -5739,15 +5952,20 @@ private:
 			PageToBuild& a = pages[pages.size() - 2];
 			PageToBuild& b = pages.back();
 
-			// While the last page page has too much slack and the second to last page
-			// has more than the minimum record count, shift a record from the second
-			// to last page to the last page.
-			while (b.slackFraction() > maxSlack && a.count > minRecords) {
-				int i = a.lastIndex();
-				if (!PageToBuild::shiftItem(a, b, deltaSizes[i], records[i].kvBytes())) {
-					break;
+			// We can rebalance the two pages only if they are in the same encryption domain.
+			ASSERT(!useEncryptionDomain || (a.domainId.present() && b.domainId.present()));
+			if (!useEncryptionDomain || a.domainId.get() == b.domainId.get()) {
+
+				// While the last page page has too much slack and the second to last page
+				// has more than the minimum record count, shift a record from the second
+				// to last page to the last page.
+				while (b.slackFraction() > maxSlack && a.count > minRecords) {
+					int i = a.lastIndex();
+					if (!PageToBuild::shiftItem(a, b, deltaSizes[i], records[i].kvBytes())) {
+						break;
+					}
+					debug_printf("  After shifting i=%d: a=%s b=%s\n", i, a.toString().c_str(), b.toString().c_str());
 				}
-				debug_printf("  After shifting i=%d: a=%s b=%s\n", i, a.toString().c_str(), b.toString().c_str());
 			}
 		}
 
@@ -5770,8 +5988,13 @@ private:
 		// All records share the prefix shared by the lower and upper boundaries
 		state int prefixLen = lowerBound->getCommonPrefixLen(*upperBound);
 
+		// Whether encryption is used and we need to set encryption domain for a page.
+		state bool useEncryptionDomain =
+		    ArenaPage::isEncodingTypeEncrypted(self->m_encodingType) && self->m_keyProvider->enableEncryptionDomain();
+
 		state std::vector<PageToBuild> pagesToBuild =
 		    self->splitPages(lowerBound, upperBound, prefixLen, entries, height);
+		ASSERT(pagesToBuild.size() > 0);
 		debug_printf("splitPages returning %s\n", toString(pagesToBuild).c_str());
 
 		// Lower bound of the page being added to
@@ -5781,6 +6004,18 @@ private:
 
 		state int pageIndex;
 
+		if (useEncryptionDomain) {
+			ASSERT(pagesToBuild[0].domainId.present());
+			int64_t domainId = pagesToBuild[0].domainId.get();
+			// We need to make sure we use the domain prefix as the page lower bound, for the first page
+			// of a non-default domain on a level. That way we ensure that pages for a domain form a full subtree
+			// (i.e. have a single root) in the B-tree.
+			if (domainId != self->m_keyProvider->getDefaultEncryptionDomainId() &&
+			    !self->m_keyProvider->keyFitsInDomain(domainId, pageLowerBound.key, false)) {
+				pageLowerBound = RedwoodRecordRef(entries[0].key.substr(0, pagesToBuild[0].domainPrefixLength));
+			}
+		}
+
 		for (pageIndex = 0; pageIndex < pagesToBuild.size(); ++pageIndex) {
 			debug_printf("building page %d of %zu %s\n",
 			             pageIndex + 1,
@@ -5788,6 +6023,30 @@ private:
 			             pagesToBuild[pageIndex].toString().c_str());
 			ASSERT(pagesToBuild[pageIndex].count != 0);
 
+			// Use the next entry as the upper bound, or upperBound if there are no more entries beyond this page
+			int endIndex = pagesToBuild[pageIndex].endIndex();
+			bool lastPage = endIndex == entries.size();
+			pageUpperBound = lastPage ? upperBound->withoutValue() : entries[endIndex].withoutValue();
+
+			if (!lastPage) {
+				PageToBuild& p = pagesToBuild[pageIndex];
+				PageToBuild& nextPage = pagesToBuild[pageIndex + 1];
+				if (height == 1) {
+					// If this is a leaf page, and not the last one to be written, shorten the upper boundary)
+					int commonPrefix = pageUpperBound.getCommonPrefixLen(entries[endIndex - 1], prefixLen);
+					pageUpperBound.truncate(commonPrefix + 1);
+				}
+				if (useEncryptionDomain) {
+					ASSERT(p.domainId.present());
+					ASSERT(nextPage.domainId.present());
+					if (p.domainId.get() != nextPage.domainId.get() &&
+					    nextPage.domainId.get() != self->m_keyProvider->getDefaultEncryptionDomainId()) {
+						pageUpperBound =
+						    RedwoodRecordRef(entries[nextPage.startIndex].key.substr(0, nextPage.domainPrefixLength));
+					}
+				}
+			}
+
 			// For internal pages, skip first entry if child link is null.  Such links only exist
 			// to maintain a borrow-able prefix for the previous subtree after a subtree deletion.
 			// If the null link falls on a new page post-split, then the pageLowerBound of the page
@@ -5801,38 +6060,34 @@ private:
 				--p.count;
 				debug_printf("Skipping first null record, new count=%d\n", p.count);
 
-				// If the page is now empty then it must be the last page in pagesToBuild, otherwise there would
-				// be more than 1 item since internal pages need to have multiple children. While there is no page
-				// to be built here, a record must be added to the output set because the upper boundary of the last
+				// In case encryption or encryption domain is not enabled, if the page is now empty then it must be the
+				// last page in pagesToBuild, otherwise there would be more than 1 item since internal pages need to
+				// have multiple children. In case encryption and encryption domain is enabled, however, because of the
+				// page split by encryption domain, it may not be the last page.
+				//
+				// Either way, a record must be added to the output set because the upper boundary of the last
 				// page built does not match the upper boundary of the original page that this call to writePages() is
 				// replacing.  Put another way, the upper boundary of the rightmost page of the page set that was just
 				// built does not match the upper boundary of the original page that the page set is replacing, so
 				// adding the extra null link fixes this.
 				if (p.count == 0) {
-					ASSERT(pageIndex == pagesToBuild.size() - 1);
-					records.push_back_deep(records.arena(), pageUpperBound);
-					break;
+					ASSERT(useEncryptionDomain || lastPage);
+					records.push_back_deep(records.arena(), pageLowerBound);
+					pageLowerBound = pageUpperBound;
+					continue;
 				}
 			}
 
-			// Use the next entry as the upper bound, or upperBound if there are no more entries beyond this page
-			int endIndex = pagesToBuild[pageIndex].endIndex();
-			bool lastPage = endIndex == entries.size();
-			pageUpperBound = lastPage ? upperBound->withoutValue() : entries[endIndex].withoutValue();
-
-			// If this is a leaf page, and not the last one to be written, shorten the upper boundary
-			if (!lastPage && height == 1) {
-				int commonPrefix = pageUpperBound.getCommonPrefixLen(entries[endIndex - 1], prefixLen);
-				pageUpperBound.truncate(commonPrefix + 1);
-			}
-
 			// Create and init page here otherwise many variables must become state vars
 			state Reference<ArenaPage> page = self->m_pager->newPageBuffer(pagesToBuild[pageIndex].blockCount);
 			page->init(self->m_encodingType,
 			           (pagesToBuild[pageIndex].blockCount == 1) ? PageType::BTreeNode : PageType::BTreeSuperNode,
 			           height);
 			if (page->isEncrypted()) {
-				EncryptionKey k = wait(self->m_keyProvider->getByRange(pageLowerBound.key, pageUpperBound.key));
+				ArenaPage::EncryptionKey k =
+				    wait(useEncryptionDomain
+				             ? self->m_keyProvider->getLatestEncryptionKey(pagesToBuild[pageIndex].domainId.get())
+				             : self->m_keyProvider->getLatestDefaultEncryptionKey());
 				page->encryptionKey = k;
 			}
 
@@ -5925,7 +6180,8 @@ private:
 			}
 
 			if (self->m_pBoundaryVerifier != nullptr) {
-				self->m_pBoundaryVerifier->update(childPageID, v, pageLowerBound.key, pageUpperBound.key);
+				ASSERT(self->m_pBoundaryVerifier->update(
+				    childPageID, v, pageLowerBound.key, pageUpperBound.key, height, pagesToBuild[pageIndex].domainId));
 			}
 
 			if (++sinceYield > 100) {
@@ -5961,15 +6217,39 @@ private:
 		return records;
 	}
 
-	ACTOR static Future<Standalone<VectorRef<RedwoodRecordRef>>> buildNewRoot(
+	// Takes a list of records commitSubtree() on the root and builds new root pages
+	// until there is only 1 root records which is small enough to fit in the BTree commit header.
+	ACTOR static Future<Standalone<VectorRef<RedwoodRecordRef>>> buildNewRootsIfNeeded(
 	    VersionedBTree* self,
 	    Version version,
 	    Standalone<VectorRef<RedwoodRecordRef>> records,
 	    unsigned int height) {
 		debug_printf("buildNewRoot start version %" PRId64 ", %d records\n", version, records.size());
 
-		// While there are multiple child pages for this version we must write new tree levels.
-		while (records.size() > 1) {
+		// While there are multiple child records or there is only one but it is too large to fit in the BTree
+		// commit record, build a new root page and update records to be a link to that new page.
+		// Root pointer size is limited because the pager commit header is limited to smallestPhysicalBlock in
+		// size.
+		//
+		// There's another case. When encryption domain is enabled, we want to make sure the root node is encrypted
+		// using the default encryption domain. An indication that's not true is when the first record is not using
+		// dbBegin as key.
+		while (records.size() > 1 ||
+		       records.front().getChildPage().size() > (BUGGIFY ? 1 : BTreeCommitHeader::maxRootPointerSize) ||
+		       records[0].key != dbBegin.key) {
+			CODE_PROBE(records.size() == 1, "Writing a new root because the current root pointer would be too large");
+			if (records[0].key != dbBegin.key) {
+				ASSERT(self->m_keyProvider.isValid() && self->m_keyProvider->enableEncryption() &&
+				       self->m_keyProvider->enableEncryptionDomain());
+				int64_t domainId;
+				size_t prefixLength;
+				std::tie(domainId, prefixLength) = self->m_keyProvider->getEncryptionDomain(records[0].key);
+				ASSERT(domainId != self->m_keyProvider->getDefaultEncryptionDomainId());
+				ASSERT(records[0].key.size() == prefixLength);
+				CODE_PROBE(true,
+				           "Writing a new root because the current root is encrypted with non-default encryption "
+				           "domain cipher key");
+			}
 			self->m_header.height = ++height;
 			ASSERT(height < std::numeric_limits<int8_t>::max());
 			Standalone<VectorRef<RedwoodRecordRef>> newRecords = wait(
@@ -6148,7 +6428,7 @@ private:
 		self->m_pager->updatePage(PagerEventReasons::Commit, height, newID, page);
 
 		if (self->m_pBoundaryVerifier != nullptr) {
-			self->m_pBoundaryVerifier->update(writeVersion, oldID.front(), newID.front());
+			self->m_pBoundaryVerifier->updatePageId(writeVersion, oldID.front(), newID.front());
 		}
 
 		self->freeBTreePage(height, oldID, writeVersion);
@@ -6310,8 +6590,14 @@ private:
 
 	struct InternalPageModifier {
 		InternalPageModifier() {}
-		InternalPageModifier(Reference<const ArenaPage> p, bool alreadyCloned, bool updating, ParentInfo* parentInfo)
-		  : updating(updating), page(p), clonedPage(alreadyCloned), changesMade(false), parentInfo(parentInfo) {}
+		InternalPageModifier(Reference<const ArenaPage> p,
+		                     bool alreadyCloned,
+		                     bool updating,
+		                     ParentInfo* parentInfo,
+		                     Reference<IPageEncryptionKeyProvider> keyProvider,
+		                     Optional<int64_t> pageDomainId)
+		  : updating(updating), page(p), clonedPage(alreadyCloned), changesMade(false), parentInfo(parentInfo),
+		    keyProvider(keyProvider), pageDomainId(pageDomainId) {}
 
 		// Whether updating the existing page is allowed
 		bool updating;
@@ -6326,6 +6612,9 @@ private:
 		bool changesMade;
 		ParentInfo* parentInfo;
 
+		Reference<IPageEncryptionKeyProvider> keyProvider;
+		Optional<int64_t> pageDomainId;
+
 		BTreePage* btPage() const { return (BTreePage*)page->mutateData(); }
 
 		bool empty() const {
@@ -6348,6 +6637,7 @@ private:
 		void insert(BTreePage::BinaryTree::Cursor end, const VectorRef<RedwoodRecordRef>& recs) {
 			int i = 0;
 			if (updating) {
+				cloneForUpdate();
 				// Update must be done in the new tree, not the original tree where the end cursor will be from
 				end.switchTree(btPage()->tree());
 
@@ -6356,7 +6646,18 @@ private:
 					const RedwoodRecordRef& rec = recs[i];
 					debug_printf("internal page (updating) insert: %s\n", rec.toString(false).c_str());
 
-					if (!end.insert(rec)) {
+					// Fail if the inserted record does not belong to the same encryption domain as the existing page
+					// data.
+					bool canInsert = true;
+					if (page->isEncrypted() && keyProvider->enableEncryptionDomain()) {
+						ASSERT(keyProvider && pageDomainId.present());
+						canInsert = keyProvider->keyFitsInDomain(pageDomainId.get(), rec.key, true);
+					}
+					if (canInsert) {
+						canInsert = end.insert(rec);
+					}
+
+					if (!canInsert) {
 						debug_printf("internal page: failed to insert %s, switching to rebuild\n",
 						             rec.toString(false).c_str());
 
@@ -6415,14 +6716,11 @@ private:
 
 			// If the children changed, replace [cBegin, cEnd) with newLinks
 			if (u.childrenChanged) {
+				cloneForUpdate();
 				if (updating) {
 					auto c = u.cBegin;
-
-					if (c != u.cEnd) {
-						cloneForUpdate();
-						// must point c to the tree to erase from
-						c.switchTree(btPage()->tree());
-					}
+					// must point c to the tree to erase from
+					c.switchTree(btPage()->tree());
 
 					while (c != u.cEnd) {
 						debug_printf("applyUpdate (updating) erasing: %s\n", c.get().toString(false).c_str());
@@ -6472,6 +6770,55 @@ private:
 		}
 	};
 
+	ACTOR static Future<Void> buildNewSubtree(VersionedBTree* self,
+	                                          Version version,
+	                                          LogicalPageID parentID,
+	                                          unsigned int height,
+	                                          MutationBuffer::const_iterator mBegin,
+	                                          MutationBuffer::const_iterator mEnd,
+	                                          InternalPageSliceUpdate* update) {
+		ASSERT(height > 1);
+		debug_printf(
+		    "buildNewSubtree start version %" PRId64 ", height %u, %s\n'", version, height, update->toString().c_str());
+		state Standalone<VectorRef<RedwoodRecordRef>> records;
+		while (mBegin != mEnd && mBegin.key() < update->subtreeLowerBound.key) {
+			++mBegin;
+		}
+		while (mBegin != mEnd) {
+			if (mBegin.mutation().boundarySet()) {
+				RedwoodRecordRef rec(mBegin.key(), mBegin.mutation().boundaryValue.get());
+				records.push_back_deep(records.arena(), rec);
+				if (REDWOOD_DEBUG) {
+					debug_printf("      Added %s", rec.toString().c_str());
+				}
+			}
+			++mBegin;
+		}
+		if (records.empty()) {
+			update->cleared();
+		} else {
+			state unsigned int h = 1;
+			debug_printf("buildNewSubtree at level %u\n", h);
+			while (h < height) {
+				// Only the parentID at the root is known as we are building the subtree bottom-up.
+				// We use the parentID for all levels, since the parentID is currently used for
+				// debug use only.
+				Standalone<VectorRef<RedwoodRecordRef>> newRecords = wait(writePages(self,
+				                                                                     &update->subtreeLowerBound,
+				                                                                     &update->subtreeUpperBound,
+				                                                                     records,
+				                                                                     h,
+				                                                                     version,
+				                                                                     BTreeNodeLinkRef(),
+				                                                                     parentID));
+				records = newRecords;
+				h++;
+			}
+			update->rebuilt(records);
+		}
+		return Void();
+	}
+
 	ACTOR static Future<Void> commitSubtree(
 	    VersionedBTree* self,
 	    CommitBatch* batch,
@@ -6528,6 +6875,12 @@ private:
 		// TryToUpdate indicates insert and erase operations should be tried on the existing page first
 		state bool tryToUpdate = btPage->tree()->numItems > 0 && update->boundariesNormal();
 
+		state bool useEncryptionDomain = page->isEncrypted() && self->m_keyProvider->enableEncryptionDomain();
+		state Optional<int64_t> pageDomainId;
+		if (useEncryptionDomain) {
+			pageDomainId = page->getEncryptionDomainId();
+		}
+
 		debug_printf("%s tryToUpdate=%d\n", context.c_str(), tryToUpdate);
 		debug_print(addPrefix(context,
 		                      btPage->toString("commitSubtreeStart",
@@ -6545,7 +6898,9 @@ private:
 				ASSERT(self->m_pBoundaryVerifier->verify(rootID.front(),
 				                                         batch->snapshot->getVersion(),
 				                                         update->cBegin.get().key,
-				                                         update->cBegin.next().getOrUpperBound().key));
+				                                         update->cBegin.next().getOrUpperBound().key,
+				                                         pageDomainId,
+				                                         cursor));
 			}
 		}
 
@@ -6644,8 +6999,16 @@ private:
 
 						// If updating, first try to add the record to the page
 						if (updatingDeltaTree) {
-							copyForUpdate();
-							if (cursor.insert(rec, update->skipLen, maxHeightAllowed)) {
+							bool canInsert = true;
+							if (useEncryptionDomain) {
+								ASSERT(pageDomainId.present());
+								canInsert = self->m_keyProvider->keyFitsInDomain(pageDomainId.get(), rec.key, false);
+							}
+							if (canInsert) {
+								copyForUpdate();
+								canInsert = cursor.insert(rec, update->skipLen, maxHeightAllowed);
+							}
+							if (canInsert) {
 								btPage->kvBytes += rec.kvBytes();
 								debug_printf("%s Inserted %s [mutation, boundary start]\n",
 								             context.c_str(),
@@ -6864,6 +7227,25 @@ private:
 
 			bool first = true;
 
+			if (useEncryptionDomain && cursor.valid() && update->subtreeLowerBound.key < cursor.get().key) {
+				mEnd = batch->mutations->lower_bound(cursor.get().key);
+				first = false;
+				if (mBegin != mEnd) {
+					slices.emplace_back(new InternalPageSliceUpdate());
+					InternalPageSliceUpdate& u = *slices.back();
+					u.cBegin = cursor;
+					u.cEnd = cursor;
+					u.subtreeLowerBound = update->subtreeLowerBound;
+					u.decodeLowerBound = u.subtreeLowerBound;
+					u.subtreeUpperBound = cursor.get();
+					u.decodeUpperBound = u.subtreeUpperBound;
+					u.expectedUpperBound = u.subtreeUpperBound;
+					u.skipLen = 0;
+					recursions.push_back(
+					    self->buildNewSubtree(self, batch->writeVersion, parentID, height, mBegin, mEnd, &u));
+				}
+			}
+
 			while (cursor.valid()) {
 				slices.emplace_back(new InternalPageSliceUpdate());
 				InternalPageSliceUpdate& u = *slices.back();
@@ -7046,7 +7428,8 @@ private:
 			// which to build new page(s) if modification is not possible or not allowed.
 			// If pageCopy is already set it was initialized to page above so the modifier doesn't need
 			// to copy it
-			state InternalPageModifier modifier(page, pageCopy.isValid(), tryToUpdate, parentInfo);
+			state InternalPageModifier modifier(
+			    page, pageCopy.isValid(), tryToUpdate, parentInfo, self->m_keyProvider, pageDomainId);
 
 			// Apply the possible changes for each subtree range recursed to, except the last one.
 			// For each range, the expected next record, if any, is checked against the first boundary
@@ -7065,8 +7448,11 @@ private:
 			             modifier.changesMade);
 			debug_print(addPrefix(context, update->toString()));
 
+			// TODO(yiwu): check whether we can pass decodeUpperBound as nextBoundary when the last slice
+			// have childenChanged=true.
 			modifier.applyUpdate(*slices.back(),
-			                     modifier.changesMade ? &update->subtreeUpperBound : &update->decodeUpperBound);
+			                     modifier.changesMade || slices.back()->childrenChanged ? &update->subtreeUpperBound
+			                                                                            : &update->decodeUpperBound);
 
 			state bool detachChildren = (parentInfo->count > 2);
 			state bool forceUpdate = false;
@@ -7129,7 +7515,8 @@ private:
 											if (newID != invalidPhysicalPageID) {
 												debug_printf("%s Detach updated %u -> %u\n", context.c_str(), p, newID);
 												if (self->m_pBoundaryVerifier != nullptr) {
-													self->m_pBoundaryVerifier->update(batch->writeVersion, p, newID);
+													self->m_pBoundaryVerifier->updatePageId(
+													    batch->writeVersion, p, newID);
 												}
 												p = newID;
 												++stats.metrics.detachChild;
@@ -7195,7 +7582,8 @@ private:
 												rec.setChildPage(newPages);
 												debug_printf("%s Detach updated %u -> %u\n", context.c_str(), p, newID);
 												if (self->m_pBoundaryVerifier != nullptr) {
-													self->m_pBoundaryVerifier->update(batch->writeVersion, p, newID);
+													self->m_pBoundaryVerifier->updatePageId(
+													    batch->writeVersion, p, newID);
 												}
 												++stats.metrics.detachChild;
 											}
@@ -7205,7 +7593,6 @@ private:
 							}
 							parentInfo->clear();
 						}
-
 						Standalone<VectorRef<RedwoodRecordRef>> newChildEntries =
 						    wait(writePages(self,
 						                    &update->subtreeLowerBound,
@@ -7299,14 +7686,10 @@ private:
 				self->m_pager->updatePage(PagerEventReasons::Commit, self->m_header.height, rootNodeLink, page);
 			} else {
 				Standalone<VectorRef<RedwoodRecordRef>> newRootRecords(all.newLinks, all.newLinks.arena());
-				if (newRootRecords.size() == 1) {
-					rootNodeLink = newRootRecords.front().getChildPage();
-				} else {
-					// If the new root level's size is not 1 then build new root level(s)
-					Standalone<VectorRef<RedwoodRecordRef>> newRootPage =
-					    wait(buildNewRoot(self, batch.writeVersion, newRootRecords, self->m_header.height));
-					rootNodeLink = newRootPage.front().getChildPage();
-				}
+				// Build new root levels if there are multiple new root records or if the root pointer is too large
+				Standalone<VectorRef<RedwoodRecordRef>> newRootPage =
+				    wait(buildNewRootsIfNeeded(self, batch.writeVersion, newRootRecords, self->m_header.height));
+				rootNodeLink = newRootPage.front().getChildPage();
 			}
 		}
 
@@ -7349,6 +7732,7 @@ public:
 
 	private:
 		PagerEventReasons reason;
+		Optional<ReadOptions> options;
 		VersionedBTree* btree;
 		Reference<IPagerSnapshot> pager;
 		bool valid;
@@ -7414,19 +7798,26 @@ public:
 			                    link.get().getChildPage(),
 			                    ioMaxPriority,
 			                    false,
-			                    true),
+			                    !options.present() || options.get().cacheResult || path.back().btPage()->height != 2),
 			           [=](Reference<const ArenaPage> p) {
+				           BTreePage::BinaryTree::Cursor cursor = btree->getCursor(p.getPtr(), link);
 #if REDWOOD_DEBUG
-				           path.push_back({ p, btree->getCursor(p.getPtr(), link), link.get().getChildPage() });
+				           path.push_back({ p, cursor, link.get().getChildPage() });
 #else
-							path.push_back({ p, btree->getCursor(p.getPtr(), link) });
+							path.push_back({ p, cursor });
 #endif
 
 				           if (btree->m_pBoundaryVerifier != nullptr) {
+					           Optional<int64_t> domainId;
+					           if (p->isEncrypted() && btree->m_keyProvider->enableEncryptionDomain()) {
+						           domainId = p->getEncryptionDomainId();
+					           }
 					           ASSERT(btree->m_pBoundaryVerifier->verify(link.get().getChildPage().front(),
 					                                                     pager->getVersion(),
 					                                                     link.get().key,
-					                                                     link.next().getOrUpperBound().key));
+					                                                     link.next().getOrUpperBound().key,
+					                                                     domainId,
+					                                                     cursor));
 				           }
 				           return Void();
 			           });
@@ -7448,10 +7839,12 @@ public:
 		// Initialize or reinitialize cursor
 		Future<Void> init(VersionedBTree* btree_in,
 		                  PagerEventReasons reason_in,
+		                  Optional<ReadOptions> options_in,
 		                  Reference<IPagerSnapshot> pager_in,
 		                  BTreeNodeLink root) {
 			btree = btree_in;
 			reason = reason_in;
+			options = options_in;
 			pager = pager_in;
 			path.clear();
 			path.reserve(6);
@@ -7646,7 +8039,10 @@ public:
 		Future<Void> movePrev() { return path.empty() ? Void() : move_impl(this, false); }
 	};
 
-	Future<Void> initBTreeCursor(BTreeCursor* cursor, Version snapshotVersion, PagerEventReasons reason) {
+	Future<Void> initBTreeCursor(BTreeCursor* cursor,
+	                             Version snapshotVersion,
+	                             PagerEventReasons reason,
+	                             Optional<ReadOptions> options = Optional<ReadOptions>()) {
 		Reference<IPagerSnapshot> snapshot = m_pager->getReadSnapshot(snapshotVersion);
 
 		BTreeNodeLinkRef root;
@@ -7663,18 +8059,18 @@ public:
 			root = *snapshot->extra.getPtr<BTreeNodeLink>();
 		}
 
-		return cursor->init(this, reason, snapshot, root);
+		return cursor->init(this, reason, options, snapshot, root);
 	}
 };
 
 #include "fdbserver/art_impl.h"
 
-RedwoodRecordRef VersionedBTree::dbBegin(LiteralStringRef(""));
-RedwoodRecordRef VersionedBTree::dbEnd(LiteralStringRef("\xff\xff\xff\xff\xff"));
+RedwoodRecordRef VersionedBTree::dbBegin(""_sr);
+RedwoodRecordRef VersionedBTree::dbEnd("\xff\xff\xff\xff\xff"_sr);
 
 class KeyValueStoreRedwood : public IKeyValueStore {
 public:
-	KeyValueStoreRedwood(std::string filename, UID logID)
+	KeyValueStoreRedwood(std::string filename, UID logID, Reference<IPageEncryptionKeyProvider> encryptionKeyProvider)
 	  : m_filename(filename), m_concurrentReads(SERVER_KNOBS->REDWOOD_KVSTORE_CONCURRENT_READS, 0),
 	    prefetch(SERVER_KNOBS->REDWOOD_KVSTORE_RANGE_PREFETCH) {
 
@@ -7697,10 +8093,20 @@ public:
 
 		EncodingType encodingType = EncodingType::XXHash64;
 
-		// Deterministically enable encryption based on uid
-		if (g_network->isSimulated() && logID.hash() % 2 == 0) {
-			encodingType = EncodingType::XOREncryption;
-			m_keyProvider = std::make_shared<XOREncryptionKeyProvider>(filename);
+		// When reopening Redwood on restart, the cluser encryption config could be unknown at this point,
+		// for which shouldEnableEncryption will return false. In that case, if the Redwood instance was encrypted
+		// before, the encoding type in the header page will be used instead.
+		//
+		// TODO(yiwu): When the cluster encryption config is available later, fail if the cluster is configured to
+		// enable encryption, but the Redwood instance is unencrypted.
+		if (encryptionKeyProvider && encryptionKeyProvider->enableEncryption()) {
+			ASSERT(encryptionKeyProvider->expectedEncodingType() == EncodingType::AESEncryptionV1);
+			encodingType = EncodingType::AESEncryptionV1;
+			m_keyProvider = encryptionKeyProvider;
+		} else if (g_network->isSimulated() && logID.hash() % 2 == 0) {
+			// Simulation only. Deterministically enable encryption based on uid
+			encodingType = EncodingType::XOREncryption_TestOnly;
+			m_keyProvider = makeReference<XOREncryptionKeyProvider_TestOnly>(filename);
 		}
 
 		IPager2* pager = new DWALPager(pageSize,
@@ -7792,18 +8198,26 @@ public:
 		m_tree->set(keyValue);
 	}
 
-	Future<RangeResult> readRange(KeyRangeRef keys, int rowLimit, int byteLimit, IKeyValueStore::ReadType) override {
+	Future<RangeResult> readRange(KeyRangeRef keys,
+	                              int rowLimit,
+	                              int byteLimit,
+	                              Optional<ReadOptions> options) override {
 		debug_printf("READRANGE %s\n", printable(keys).c_str());
-		return catchError(readRange_impl(this, keys, rowLimit, byteLimit));
+		return catchError(readRange_impl(this, keys, rowLimit, byteLimit, options));
 	}
 
 	ACTOR static Future<RangeResult> readRange_impl(KeyValueStoreRedwood* self,
 	                                                KeyRange keys,
 	                                                int rowLimit,
-	                                                int byteLimit) {
+	                                                int byteLimit,
+	                                                Optional<ReadOptions> options) {
+		state PagerEventReasons reason = PagerEventReasons::RangeRead;
 		state VersionedBTree::BTreeCursor cur;
-		wait(
-		    self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion(), PagerEventReasons::RangeRead));
+		if (options.present() && options.get().type == ReadType::FETCH) {
+			reason = PagerEventReasons::FetchRange;
+		}
+		wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion(), reason, options));
+
 		state PriorityMultiLock::Lock lock;
 		state Future<Void> f;
 		++g_redwoodMetrics.metric.opGetRange;
@@ -7819,10 +8233,10 @@ public:
 		if (rowLimit > 0) {
 			f = cur.seekGTE(keys.begin);
 			if (f.isReady()) {
-				TEST(true); // Cached forward range read seek
+				CODE_PROBE(true, "Cached forward range read seek");
 				f.get();
 			} else {
-				TEST(true); // Uncached forward range read seek
+				CODE_PROBE(true, "Uncached forward range read seek");
 				wait(store(lock, self->m_concurrentReads.lock()));
 				wait(f);
 			}
@@ -7875,10 +8289,10 @@ public:
 		} else {
 			f = cur.seekLT(keys.end);
 			if (f.isReady()) {
-				TEST(true); // Cached reverse range read seek
+				CODE_PROBE(true, "Cached reverse range read seek");
 				f.get();
 			} else {
-				TEST(true); // Uncached reverse range read seek
+				CODE_PROBE(true, "Uncached reverse range read seek");
 				wait(store(lock, self->m_concurrentReads.lock()));
 				wait(f);
 			}
@@ -7939,10 +8353,12 @@ public:
 		return result;
 	}
 
-	ACTOR static Future<Optional<Value>> readValue_impl(KeyValueStoreRedwood* self, Key key, Optional<UID> debugID) {
+	ACTOR static Future<Optional<Value>> readValue_impl(KeyValueStoreRedwood* self,
+	                                                    Key key,
+	                                                    Optional<ReadOptions> options) {
 		state VersionedBTree::BTreeCursor cur;
-		wait(
-		    self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion(), PagerEventReasons::PointRead));
+		wait(self->m_tree->initBTreeCursor(
+		    &cur, self->m_tree->getLastCommittedVersion(), PagerEventReasons::PointRead, options));
 
 		// Not locking for point reads, instead relying on IO priority lock
 		// state PriorityMultiLock::Lock lock = wait(self->m_concurrentReads.lock());
@@ -7961,15 +8377,12 @@ public:
 		return Optional<Value>();
 	}
 
-	Future<Optional<Value>> readValue(KeyRef key, IKeyValueStore::ReadType, Optional<UID> debugID) override {
-		return catchError(readValue_impl(this, key, debugID));
+	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options) override {
+		return catchError(readValue_impl(this, key, options));
 	}
 
-	Future<Optional<Value>> readValuePrefix(KeyRef key,
-	                                        int maxLength,
-	                                        IKeyValueStore::ReadType,
-	                                        Optional<UID> debugID) override {
-		return catchError(map(readValue_impl(this, key, debugID), [maxLength](Optional<Value> v) {
+	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override {
+		return catchError(map(readValue_impl(this, key, options), [maxLength](Optional<Value> v) {
 			if (v.present() && v.get().size() > maxLength) {
 				v.get().contents() = v.get().substr(0, maxLength);
 			}
@@ -7988,7 +8401,7 @@ private:
 	PriorityMultiLock m_concurrentReads;
 	bool prefetch;
 	Version m_nextCommitVersion;
-	std::shared_ptr<IEncryptionKeyProvider> m_keyProvider;
+	Reference<IPageEncryptionKeyProvider> m_keyProvider;
 	Future<Void> m_lastCommit = Void();
 
 	template <typename T>
@@ -7997,8 +8410,10 @@ private:
 	}
 };
 
-IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename, UID logID) {
-	return new KeyValueStoreRedwood(filename, logID);
+IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename,
+                                       UID logID,
+                                       Reference<IPageEncryptionKeyProvider> encryptionKeyProvider) {
+	return new KeyValueStoreRedwood(filename, logID, encryptionKeyProvider);
 }
 
 int randomSize(int max) {
@@ -8315,8 +8730,8 @@ ACTOR Future<Void> verify(VersionedBTree* btree,
 			wait(btree->initBTreeCursor(&cur, v, PagerEventReasons::RangeRead));
 
 			debug_printf("Verifying entire key range at version %" PRId64 "\n", v);
-			state Future<Void> fRangeAll = verifyRangeBTreeCursor(
-			    btree, LiteralStringRef(""), LiteralStringRef("\xff\xff"), v, written, pRecordsRead);
+			state Future<Void> fRangeAll =
+			    verifyRangeBTreeCursor(btree, ""_sr, "\xff\xff"_sr, v, written, pRecordsRead);
 			if (serial) {
 				wait(fRangeAll);
 			}
@@ -8641,35 +9056,26 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") {
 		ASSERT(r2.getChildPage().begin() != id.begin());
 	}
 
-	deltaTest(RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")),
-	          RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")));
+	deltaTest(RedwoodRecordRef(""_sr, ""_sr), RedwoodRecordRef(""_sr, ""_sr));
 
-	deltaTest(RedwoodRecordRef(LiteralStringRef("abc"), LiteralStringRef("")),
-	          RedwoodRecordRef(LiteralStringRef("abc"), LiteralStringRef("")));
+	deltaTest(RedwoodRecordRef("abc"_sr, ""_sr), RedwoodRecordRef("abc"_sr, ""_sr));
 
-	deltaTest(RedwoodRecordRef(LiteralStringRef("abc"), LiteralStringRef("")),
-	          RedwoodRecordRef(LiteralStringRef("abcd"), LiteralStringRef("")));
+	deltaTest(RedwoodRecordRef("abc"_sr, ""_sr), RedwoodRecordRef("abcd"_sr, ""_sr));
 
-	deltaTest(RedwoodRecordRef(LiteralStringRef("abcd"), LiteralStringRef("")),
-	          RedwoodRecordRef(LiteralStringRef("abc"), LiteralStringRef("")));
+	deltaTest(RedwoodRecordRef("abcd"_sr, ""_sr), RedwoodRecordRef("abc"_sr, ""_sr));
 
 	deltaTest(RedwoodRecordRef(std::string(300, 'k'), std::string(1e6, 'v')),
-	          RedwoodRecordRef(std::string(300, 'k'), LiteralStringRef("")));
+	          RedwoodRecordRef(std::string(300, 'k'), ""_sr));
 
-	deltaTest(RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")),
-	          RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")));
+	deltaTest(RedwoodRecordRef(""_sr, ""_sr), RedwoodRecordRef(""_sr, ""_sr));
 
-	deltaTest(RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")),
-	          RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")));
+	deltaTest(RedwoodRecordRef(""_sr, ""_sr), RedwoodRecordRef(""_sr, ""_sr));
 
-	deltaTest(RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")),
-	          RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")));
+	deltaTest(RedwoodRecordRef(""_sr, ""_sr), RedwoodRecordRef(""_sr, ""_sr));
 
-	deltaTest(RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")),
-	          RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")));
+	deltaTest(RedwoodRecordRef(""_sr, ""_sr), RedwoodRecordRef(""_sr, ""_sr));
 
-	deltaTest(RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")),
-	          RedwoodRecordRef(LiteralStringRef(""), LiteralStringRef("")));
+	deltaTest(RedwoodRecordRef(""_sr, ""_sr), RedwoodRecordRef(""_sr, ""_sr));
 
 	Arena mem;
 	double start;
@@ -8706,8 +9112,8 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") {
 	RedwoodRecordRef rec1;
 	RedwoodRecordRef rec2;
 
-	rec1.key = LiteralStringRef("alksdfjaklsdfjlkasdjflkasdjfklajsdflk;ajsdflkajdsflkjadsf1");
-	rec2.key = LiteralStringRef("alksdfjaklsdfjlkasdjflkasdjfklajsdflk;ajsdflkajdsflkjadsf234");
+	rec1.key = "alksdfjaklsdfjlkasdjflkasdjfklajsdflk;ajsdflkajdsflkjadsf1"_sr;
+	rec2.key = "alksdfjaklsdfjlkasdjflkasdjfklajsdflk;ajsdflkajdsflkjadsf234"_sr;
 
 	start = timer();
 	total = 0;
@@ -8773,7 +9179,7 @@ TEST_CASE("Lredwood/correctness/unit/deltaTree/RedwoodRecordRef") {
 	const int N = deterministicRandom()->randomInt(200, 1000);
 
 	RedwoodRecordRef prev;
-	RedwoodRecordRef next(LiteralStringRef("\xff\xff\xff\xff"));
+	RedwoodRecordRef next("\xff\xff\xff\xff"_sr);
 
 	Arena arena;
 	std::set<RedwoodRecordRef> uniqueItems;
@@ -8950,7 +9356,7 @@ TEST_CASE("Lredwood/correctness/unit/deltaTree/RedwoodRecordRef2") {
 	const int N = deterministicRandom()->randomInt(200, 1000);
 
 	RedwoodRecordRef prev;
-	RedwoodRecordRef next(LiteralStringRef("\xff\xff\xff\xff"));
+	RedwoodRecordRef next("\xff\xff\xff\xff"_sr);
 
 	Arena arena;
 	std::set<RedwoodRecordRef> uniqueItems;
@@ -9728,8 +10134,13 @@ TEST_CASE("Lredwood/correctness/btree") {
 	state bool serialTest = params.getInt("serialTest").orDefault(deterministicRandom()->random01() < 0.25);
 	state bool shortTest = params.getInt("shortTest").orDefault(deterministicRandom()->random01() < 0.25);
 
+	state int encoding =
+	    params.getInt("encodingType").orDefault(deterministicRandom()->randomInt(0, EncodingType::MAX_ENCODING_TYPE));
+	state unsigned int encryptionDomainMode =
+	    params.getInt("domainMode")
+	        .orDefault(deterministicRandom()->randomInt(0, RandomEncryptionKeyProvider::EncryptionDomainMode::MAX));
 	state int pageSize =
-	    shortTest ? 200 : (deterministicRandom()->coinflip() ? 4096 : deterministicRandom()->randomInt(200, 400));
+	    shortTest ? 250 : (deterministicRandom()->coinflip() ? 4096 : deterministicRandom()->randomInt(250, 400));
 	state int extentSize =
 	    params.getInt("extentSize")
 	        .orDefault(deterministicRandom()->coinflip() ? SERVER_KNOBS->REDWOOD_DEFAULT_EXTENT_SIZE
@@ -9740,7 +10151,9 @@ TEST_CASE("Lredwood/correctness/btree") {
 	state int maxValueSize = params.getInt("maxValueSize").orDefault(randomSize(pageSize * 25));
 	state int maxCommitSize =
 	    params.getInt("maxCommitSize")
-	        .orDefault(shortTest ? 1000 : randomSize(std::min<int>((maxKeySize + maxValueSize) * 20000, 10e6)));
+	        .orDefault(shortTest
+	                       ? 1000
+	                       : randomSize((int)std::min<int64_t>((maxKeySize + maxValueSize) * int64_t(20000), 10e6)));
 	state double setExistingKeyProbability =
 	    params.getDouble("setExistingKeyProbability").orDefault(deterministicRandom()->random01() * .5);
 	state double clearProbability =
@@ -9776,23 +10189,25 @@ TEST_CASE("Lredwood/correctness/btree") {
 	// Max number of records in the BTree or the versioned written map to visit
 	state int64_t maxRecordsRead = params.getInt("maxRecordsRead").orDefault(300e6);
 
-	state EncodingType encodingType = EncodingType::XXHash64;
-	state std::shared_ptr<IEncryptionKeyProvider> keyProvider;
-
-	if (deterministicRandom()->coinflip()) {
-		encodingType = EncodingType::XOREncryption;
-		keyProvider = std::make_shared<XOREncryptionKeyProvider>(file);
+	state EncodingType encodingType = static_cast<EncodingType>(encoding);
+	state Reference<IPageEncryptionKeyProvider> keyProvider;
+	if (encodingType == EncodingType::AESEncryptionV1) {
+		keyProvider = makeReference<RandomEncryptionKeyProvider>(
+		    RandomEncryptionKeyProvider::EncryptionDomainMode(encryptionDomainMode));
+	} else if (encodingType == EncodingType::XOREncryption_TestOnly) {
+		keyProvider = makeReference<XOREncryptionKeyProvider_TestOnly>(file);
 	}
 
 	printf("\n");
 	printf("file: %s\n", file.c_str());
-	printf("encodingType: %d\n", encodingType);
 	printf("maxPageOps: %" PRId64 "\n", maxPageOps);
 	printf("maxVerificationMapEntries: %d\n", maxVerificationMapEntries);
 	printf("maxRecordsRead: %" PRId64 "\n", maxRecordsRead);
 	printf("pagerMemoryOnly: %d\n", pagerMemoryOnly);
 	printf("serialTest: %d\n", serialTest);
 	printf("shortTest: %d\n", shortTest);
+	printf("encodingType: %d\n", encodingType);
+	printf("domainMode: %d\n", encryptionDomainMode);
 	printf("pageSize: %d\n", pageSize);
 	printf("extentSize: %d\n", extentSize);
 	printf("maxKeySize: %d\n", maxKeySize);
@@ -10263,7 +10678,7 @@ TEST_CASE(":/redwood/performance/extentQueue") {
 		                      remapCleanupWindowBytes,
 		                      concurrentExtentReads,
 		                      false,
-		                      nullptr);
+		                      Reference<IPageEncryptionKeyProvider>());
 
 		wait(success(pager->init()));
 
@@ -10273,7 +10688,7 @@ TEST_CASE(":/redwood/performance/extentQueue") {
 
 		state int v;
 		state ExtentQueueEntry<16> e;
-		generateRandomData(e.entry, 16);
+		deterministicRandom()->randomBytes(e.entry, 16);
 		state int sinceYield = 0;
 		for (v = 1; v <= numEntries; ++v) {
 			// Sometimes do a commit
@@ -10314,8 +10729,14 @@ TEST_CASE(":/redwood/performance/extentQueue") {
 	}
 
 	printf("Reopening pager file from disk.\n");
-	pager = new DWALPager(
-	    pageSize, extentSize, fileName, cacheSizeBytes, remapCleanupWindowBytes, concurrentExtentReads, false, nullptr);
+	pager = new DWALPager(pageSize,
+	                      extentSize,
+	                      fileName,
+	                      cacheSizeBytes,
+	                      remapCleanupWindowBytes,
+	                      concurrentExtentReads,
+	                      false,
+	                      Reference<IPageEncryptionKeyProvider>());
 	wait(success(pager->init()));
 
 	printf("Starting ExtentQueue FastPath Recovery from Disk.\n");
@@ -10460,8 +10881,9 @@ TEST_CASE(":/redwood/performance/set") {
 	                                 remapCleanupWindowBytes,
 	                                 concurrentExtentReads,
 	                                 pagerMemoryOnly,
-	                                 nullptr);
-	state VersionedBTree* btree = new VersionedBTree(pager, file, EncodingType::XXHash64, nullptr);
+	                                 Reference<IPageEncryptionKeyProvider>());
+	state VersionedBTree* btree =
+	    new VersionedBTree(pager, file, EncodingType::XXHash64, Reference<IPageEncryptionKeyProvider>());
 	wait(btree->init());
 	printf("Initialized.  StorageBytes=%s\n", btree->getStorageBytes().toString().c_str());
 
@@ -10839,7 +11261,7 @@ ACTOR Future<Void> prefixClusteredInsert(IKeyValueStore* kvs,
 	if (clearAfter) {
 		printf("Clearing all keys\n");
 		intervalStart = timer();
-		kvs->clear(KeyRangeRef(LiteralStringRef(""), LiteralStringRef("\xff")));
+		kvs->clear(KeyRangeRef(""_sr, "\xff"_sr));
 		state StorageBytes sbClear = wait(getStableStorageBytes(kvs));
 		printf("Cleared all keys in %.2f seconds, final storageByte: %s\n",
 		       timer() - intervalStart,
@@ -10989,7 +11411,9 @@ ACTOR Future<Void> randomRangeScans(IKeyValueStore* kvs,
                                     int valueSize,
                                     int recordCountTarget,
                                     bool singlePrefix,
-                                    int rowLimit) {
+                                    int rowLimit,
+                                    int byteLimit,
+                                    Optional<ReadOptions> options = Optional<ReadOptions>()) {
 	fmt::print("\nstoreType: {}\n", static_cast<int>(kvs->getType()));
 	fmt::print("prefixSource: {}\n", source.toString());
 	fmt::print("suffixSize: {}\n", suffixSize);
@@ -11022,7 +11446,7 @@ ACTOR Future<Void> randomRangeScans(IKeyValueStore* kvs,
 		KeyRangeRef range = source.getKeyRangeRef(singlePrefix, suffixSize);
 		int rowLim = (deterministicRandom()->randomInt(0, 2) != 0) ? rowLimit : -rowLimit;
 
-		RangeResult result = wait(kvs->readRange(range, rowLim));
+		RangeResult result = wait(kvs->readRange(range, rowLim, byteLimit, options));
 
 		recordsRead += result.size();
 		bytesRead += result.size() * recordSize;
@@ -11045,6 +11469,7 @@ TEST_CASE(":/redwood/performance/randomRangeScans") {
 	state int prefixLen = 30;
 	state int suffixSize = 12;
 	state int valueSize = 100;
+	state int maxByteLimit = std::numeric_limits<int>::max();
 
 	// TODO change to 100e8 after figuring out no-disk redwood mode
 	state int writeRecordCountTarget = 1e6;
@@ -11060,102 +11485,35 @@ TEST_CASE(":/redwood/performance/randomRangeScans") {
 	    redwood, suffixSize, valueSize, source, writeRecordCountTarget, writePrefixesInOrder, false));
 
 	// divide targets for tiny queries by 10 because they are much slower
-	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget / 10, true, 10));
-	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget, true, 1000));
-	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget / 10, false, 100));
-	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget, false, 10000));
-	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget, false, 1000000));
+	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget / 10, true, 10, maxByteLimit));
+	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget, true, 1000, maxByteLimit));
+	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget / 10, false, 100, maxByteLimit));
+	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget, false, 10000, maxByteLimit));
+	wait(randomRangeScans(redwood, suffixSize, source, valueSize, queryRecordTarget, false, 1000000, maxByteLimit));
 	wait(closeKVS(redwood));
 	printf("\n");
 	return Void();
 }
 
-TEST_CASE(":/redwood/performance/histogramThroughput") {
-	std::default_random_engine generator;
-	std::uniform_int_distribution<uint32_t> distribution(0, UINT32_MAX);
-	state size_t inputSize = pow(10, 8);
-	state std::vector<uint32_t> uniform;
-	for (int i = 0; i < inputSize; i++) {
-		uniform.push_back(distribution(generator));
+TEST_CASE(":/redwood/performance/histograms") {
+	// Time needed to log 33 histograms.
+	std::vector<Reference<Histogram>> histograms;
+	for (int i = 0; i < 33; i++) {
+		std::string levelString = "L" + std::to_string(i);
+		histograms.push_back(Histogram::getHistogram("histogramTest"_sr, "levelString"_sr, Histogram::Unit::bytes));
 	}
-	std::cout << "size of input: " << uniform.size() << std::endl;
-	{
-		// Time needed to log 33 histograms.
-		std::vector<Reference<Histogram>> histograms;
-		for (int i = 0; i < 33; i++) {
-			std::string levelString = "L" + std::to_string(i);
-			histograms.push_back(Histogram::getHistogram(
-			    LiteralStringRef("histogramTest"), LiteralStringRef("levelString"), Histogram::Unit::bytes));
+	for (int i = 0; i < 33; i++) {
+		for (int j = 0; j < 32; j++) {
+			histograms[i]->sample(std::pow(2, j));
 		}
-		for (int i = 0; i < 33; i++) {
-			for (int j = 0; j < 32; j++) {
-				histograms[i]->sample(std::pow(2, j));
-			}
-		}
-		auto t_start = std::chrono::high_resolution_clock::now();
-		for (int i = 0; i < 33; i++) {
-			histograms[i]->writeToLog(30.0);
-		}
-		auto t_end = std::chrono::high_resolution_clock::now();
-		double elapsed_time_ms = std::chrono::duration<double, std::milli>(t_end - t_start).count();
-		std::cout << "Time needed to log 33 histograms (millisecond): " << elapsed_time_ms << std::endl;
 	}
-	{
-		std::cout << "Histogram Unit bytes" << std::endl;
-		auto t_start = std::chrono::high_resolution_clock::now();
-		Reference<Histogram> h = Histogram::getHistogram(
-		    LiteralStringRef("histogramTest"), LiteralStringRef("counts"), Histogram::Unit::bytes);
-		ASSERT(uniform.size() == inputSize);
-		for (size_t i = 0; i < uniform.size(); i++) {
-			h->sample(uniform[i]);
-		}
-		auto t_end = std::chrono::high_resolution_clock::now();
-		std::cout << h->drawHistogram();
-		double elapsed_time_ms = std::chrono::duration<double, std::milli>(t_end - t_start).count();
-		std::cout << "Time in millisecond: " << elapsed_time_ms << std::endl;
+	auto t_start = std::chrono::high_resolution_clock::now();
+	for (int i = 0; i < 33; i++) {
+		histograms[i]->writeToLog(30.0);
+	}
+	auto t_end = std::chrono::high_resolution_clock::now();
+	double elapsed_time_ms = std::chrono::duration<double, std::milli>(t_end - t_start).count();
+	std::cout << "Time needed to log 33 histograms (millisecond): " << elapsed_time_ms << std::endl;
 
-		Reference<Histogram> hCopy = Histogram::getHistogram(
-		    LiteralStringRef("histogramTest"), LiteralStringRef("counts"), Histogram::Unit::bytes);
-		std::cout << hCopy->drawHistogram();
-		GetHistogramRegistry().logReport();
-	}
-	{
-		std::cout << "Histogram Unit percentage: " << std::endl;
-		auto t_start = std::chrono::high_resolution_clock::now();
-		Reference<Histogram> h = Histogram::getHistogram(
-		    LiteralStringRef("histogramTest"), LiteralStringRef("counts"), Histogram::Unit::percentageLinear);
-		ASSERT(uniform.size() == inputSize);
-		for (size_t i = 0; i < uniform.size(); i++) {
-			h->samplePercentage((double)uniform[i] / UINT32_MAX);
-		}
-		auto t_end = std::chrono::high_resolution_clock::now();
-		std::cout << h->drawHistogram();
-		GetHistogramRegistry().logReport();
-		double elapsed_time_ms = std::chrono::duration<double, std::milli>(t_end - t_start).count();
-		std::cout << "Time in millisecond: " << elapsed_time_ms << std::endl;
-	}
-	return Void();
-}
-TEST_CASE(":/redwood/performance/continuousSmapleThroughput") {
-	std::default_random_engine generator;
-	std::uniform_int_distribution<uint32_t> distribution(0, UINT32_MAX);
-	state size_t inputSize = pow(10, 8);
-	state std::vector<uint32_t> uniform;
-	for (int i = 0; i < inputSize; i++) {
-		uniform.push_back(distribution(generator));
-	}
-
-	{
-		ContinuousSample<uint32_t> s = ContinuousSample<uint32_t>(pow(10, 3));
-		auto t_start = std::chrono::high_resolution_clock::now();
-		ASSERT(uniform.size() == inputSize);
-		for (size_t i = 0; i < uniform.size(); i++) {
-			s.addSample(uniform[i]);
-		}
-		auto t_end = std::chrono::high_resolution_clock::now();
-		double elapsed_time_ms = std::chrono::duration<double, std::milli>(t_end - t_start).count();
-		std::cout << "size of input: " << uniform.size() << std::endl;
-		std::cout << "Time in millisecond: " << elapsed_time_ms << std::endl;
-	}
 	return Void();
 }
diff --git a/fdbserver/WaitFailure.actor.cpp b/fdbserver/WaitFailure.actor.cpp
index d1ff0f0a4e..62e514ecab 100644
--- a/fdbserver/WaitFailure.actor.cpp
+++ b/fdbserver/WaitFailure.actor.cpp
@@ -30,7 +30,7 @@ ACTOR Future<Void> waitFailureServer(FutureStream<ReplyPromise<Void>> waitFailur
 		ReplyPromise<Void> P = waitNext(waitFailure);
 		queue.push_back(P);
 		if (queue.size() > SERVER_KNOBS->MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS) {
-			TEST(true); // wait server queue full
+			CODE_PROBE(true, "wait server queue full");
 			queue.front().send(Void());
 			queue.pop_front();
 		}
diff --git a/fdbserver/coroimpl/CoroFlow.actor.cpp b/fdbserver/coroimpl/CoroFlow.actor.cpp
index 1af12ffda2..d4b4e954a8 100644
--- a/fdbserver/coroimpl/CoroFlow.actor.cpp
+++ b/fdbserver/coroimpl/CoroFlow.actor.cpp
@@ -103,7 +103,7 @@ protected:
 		blocked = Promise<Void>();
 		double before = now();
 		CoroThreadPool::waitFor(blocked.getFuture());
-		if (g_network->isSimulated() && g_simulator.getCurrentProcess()->rebooting)
+		if (g_network->isSimulated() && g_simulator->getCurrentProcess()->rebooting)
 			TraceEvent("CoroUnblocked").detail("After", now() - before);
 	}
 
diff --git a/fdbserver/coroimpl/CoroFlowCoro.actor.cpp b/fdbserver/coroimpl/CoroFlowCoro.actor.cpp
index 93c64b6e3a..5c366099a7 100644
--- a/fdbserver/coroimpl/CoroFlowCoro.actor.cpp
+++ b/fdbserver/coroimpl/CoroFlowCoro.actor.cpp
@@ -73,7 +73,7 @@ protected:
 		blocked = Promise<Void>();
 		double before = now();
 		CoroThreadPool::waitFor(blocked.getFuture());
-		if (g_network->isSimulated() && g_simulator.getCurrentProcess()->rebooting)
+		if (g_network->isSimulated() && g_simulator->getCurrentProcess()->rebooting)
 			TraceEvent("CoroUnblocked").detail("After", now() - before);
 	}
 
@@ -265,7 +265,7 @@ ACTOR void coroSwitcher(Future<Void> what, TaskPriority taskID, Coro* coro) {
 	try {
 		// state double t = now();
 		wait(what);
-		// if (g_network->isSimulated() && g_simulator.getCurrentProcess()->rebooting && now()!=t)
+		// if (g_network->isSimulated() && g_simulator->getCurrentProcess()->rebooting && now()!=t)
 		//	TraceEvent("NonzeroWaitDuringReboot").detail("TaskID", taskID).detail("Elapsed", now()-t).backtrace("Flow");
 	} catch (Error&) {
 	}
@@ -280,7 +280,7 @@ void CoroThreadPool::waitFor(Future<Void> what) {
 	// double t = now();
 	coroSwitcher(what, g_network->getCurrentTask(), current_coro);
 	Coro_switchTo_(swapCoro(main_coro), main_coro);
-	// if (g_network->isSimulated() && g_simulator.getCurrentProcess()->rebooting && now()!=t)
+	// if (g_network->isSimulated() && g_simulator->getCurrentProcess()->rebooting && now()!=t)
 	//	TraceEvent("NonzeroWaitDuringReboot").detail("TaskID", currentTaskID).detail("Elapsed",
 	// now()-t).backtrace("Coro");
 	ASSERT(what.isReady());
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 28b93ff008..e60a3e33b9 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -69,6 +69,7 @@
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbserver/pubsub.h"
+#include "fdbserver/OnDemandStore.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/ArgParseUtil.h"
 #include "flow/DeterministicRandom.h"
@@ -87,7 +88,11 @@
 #if defined(__linux__) || defined(__FreeBSD__)
 #include <execinfo.h>
 #include <signal.h>
+#if defined(__linux__)
 #include <sys/prctl.h>
+#elif defined(__FreeBSD__)
+#include <sys/procctl.h>
+#endif
 #ifdef ALLOC_INSTRUMENTATION
 #include <cxxabi.h>
 #endif
@@ -108,10 +113,11 @@ enum {
 	OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_TRACER, OPT_NEWCONSOLE,
 	OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_VMEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID,
 	OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR,
-	OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
+	OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_PRINT_CODE_PROBES, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
 	OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
-	OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE, OPT_CONFIG_PATH, OPT_USE_TEST_CONFIG_DB, OPT_FAULT_INJECTION, OPT_PROFILER, OPT_PRINT_SIMTIME,
-	OPT_FLOW_PROCESS_NAME, OPT_FLOW_PROCESS_ENDPOINT, OPT_IP_TRUSTED_MASK, OPT_KMS_CONN_DISCOVERY_URL_FILE, OPT_KMS_CONN_VALIDATION_TOKEN_DETAILS, OPT_KMS_CONN_GET_ENCRYPTION_KEYS_ENDPOINT
+	OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE, OPT_CONFIG_PATH, OPT_USE_TEST_CONFIG_DB, OPT_NO_CONFIG_DB, OPT_FAULT_INJECTION, OPT_PROFILER, OPT_PRINT_SIMTIME,
+	OPT_FLOW_PROCESS_NAME, OPT_FLOW_PROCESS_ENDPOINT, OPT_IP_TRUSTED_MASK, OPT_KMS_CONN_DISCOVERY_URL_FILE, OPT_KMS_CONNECTOR_TYPE, OPT_KMS_CONN_VALIDATION_TOKEN_DETAILS,
+	OPT_KMS_CONN_GET_ENCRYPTION_KEYS_ENDPOINT, OPT_NEW_CLUSTER_KEY, OPT_AUTHZ_PUBLIC_KEY_FILE, OPT_USE_FUTURE_PROTOCOL_VERSION
 };
 
 CSimpleOpt::SOption g_rgOptions[] = {
@@ -126,8 +132,8 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_LISTEN,                "-l",                          SO_REQ_SEP },
 	{ OPT_LISTEN,                "--listen-address",            SO_REQ_SEP },
 #ifdef __linux__
-	{ OPT_FILESYSTEM,           "--data-filesystem",           SO_REQ_SEP },
-	{ OPT_PROFILER_RSS_SIZE,    "--rsssize",                   SO_REQ_SEP },
+	{ OPT_FILESYSTEM,           "--data-filesystem",            SO_REQ_SEP },
+	{ OPT_PROFILER_RSS_SIZE,    "--rsssize",                    SO_REQ_SEP },
 #endif
 	{ OPT_DATAFOLDER,            "-d",                          SO_REQ_SEP },
 	{ OPT_DATAFOLDER,            "--datadir",                   SO_REQ_SEP },
@@ -183,6 +189,7 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_HELP,                  "-h",                          SO_NONE },
 	{ OPT_HELP,                  "--help",                      SO_NONE },
 	{ OPT_DEVHELP,               "--dev-help",                  SO_NONE },
+	{ OPT_PRINT_CODE_PROBES,     "--code-probes",               SO_REQ_SEP },
 	{ OPT_KNOB,                  "--knob-",                     SO_REQ_SEP },
 	{ OPT_UNITTESTPARAM,         "--test-",                     SO_REQ_SEP },
 	{ OPT_LOCALITY,              "--locality-",                 SO_REQ_SEP },
@@ -197,6 +204,7 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_BLOB_CREDENTIAL_FILE,  "--blob-credential-file",      SO_REQ_SEP },
 	{ OPT_CONFIG_PATH,           "--config-path",               SO_REQ_SEP },
 	{ OPT_USE_TEST_CONFIG_DB,    "--use-test-config-db",        SO_NONE },
+	{ OPT_NO_CONFIG_DB,          "--no-config-db",              SO_NONE },
 	{ OPT_FAULT_INJECTION,       "-fi",                         SO_REQ_SEP },
 	{ OPT_FAULT_INJECTION,       "--fault-injection",           SO_REQ_SEP },
 	{ OPT_PROFILER,	             "--profiler-",                 SO_REQ_SEP },
@@ -204,9 +212,13 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_FLOW_PROCESS_NAME,     "--process-name",              SO_REQ_SEP },
 	{ OPT_FLOW_PROCESS_ENDPOINT, "--process-endpoint",          SO_REQ_SEP },
 	{ OPT_IP_TRUSTED_MASK,       "--trusted-subnet-",           SO_REQ_SEP },
-	{ OPT_KMS_CONN_DISCOVERY_URL_FILE,           "--discover-kms-conn-url-file",            SO_REQ_SEP},
-	{ OPT_KMS_CONN_VALIDATION_TOKEN_DETAILS,     "--kms-conn-validation-token-details",     SO_REQ_SEP},
-	{ OPT_KMS_CONN_GET_ENCRYPTION_KEYS_ENDPOINT, "--kms-conn-get-encryption-keys-endpoint", SO_REQ_SEP},
+	{ OPT_NEW_CLUSTER_KEY,       "--new-cluster-key",           SO_REQ_SEP },
+	{ OPT_AUTHZ_PUBLIC_KEY_FILE, "--authorization-public-key-file", SO_REQ_SEP },
+	{ OPT_KMS_CONN_DISCOVERY_URL_FILE,           "--discover-kms-conn-url-file",            SO_REQ_SEP },
+	{ OPT_KMS_CONNECTOR_TYPE,    "--kms-connector-type",        SO_REQ_SEP },
+	{ OPT_KMS_CONN_VALIDATION_TOKEN_DETAILS,     "--kms-conn-validation-token-details",     SO_REQ_SEP },
+	{ OPT_KMS_CONN_GET_ENCRYPTION_KEYS_ENDPOINT, "--kms-conn-get-encryption-keys-endpoint", SO_REQ_SEP },
+	{ OPT_USE_FUTURE_PROTOCOL_VERSION, 			 "--use-future-protocol-version",			SO_REQ_SEP },
 	TLS_OPTION_FLAGS,
 	SO_END_OF_OPTIONS
 };
@@ -345,17 +357,17 @@ UID getSharedMemoryMachineId() {
 #endif
 }
 
-ACTOR void failAfter(Future<Void> trigger, ISimulator::ProcessInfo* m = g_simulator.getCurrentProcess()) {
+ACTOR void failAfter(Future<Void> trigger, ISimulator::ProcessInfo* m = g_simulator->getCurrentProcess()) {
 	wait(trigger);
 	if (enableFailures) {
 		printf("Killing machine: %s at %f\n", m->address.toString().c_str(), now());
-		g_simulator.killProcess(m, ISimulator::KillInstantly);
+		g_simulator->killProcess(m, ISimulator::KillInstantly);
 	}
 }
 
 void failAfter(Future<Void> trigger, Endpoint e) {
-	if (g_network == &g_simulator)
-		failAfter(trigger, g_simulator.getProcess(e));
+	if (g_network == g_simulator)
+		failAfter(trigger, g_simulator->getProcess(e));
 }
 
 ACTOR Future<Void> histogramReport() {
@@ -382,12 +394,10 @@ void testSerializationSpeed() {
 				CommitTransactionRef& tr = batch[t];
 				tr.read_snapshot = 0;
 				for (int i = 0; i < 2; i++)
-					tr.mutations.push_back_deep(
-					    batchArena,
-					    MutationRef(MutationRef::SetValue, LiteralStringRef("KeyABCDE"), LiteralStringRef("SomeValu")));
-				tr.mutations.push_back_deep(
-				    batchArena,
-				    MutationRef(MutationRef::ClearRange, LiteralStringRef("BeginKey"), LiteralStringRef("EndKeyAB")));
+					tr.mutations.push_back_deep(batchArena,
+					                            MutationRef(MutationRef::SetValue, "KeyABCDE"_sr, "SomeValu"_sr));
+				tr.mutations.push_back_deep(batchArena,
+				                            MutationRef(MutationRef::ClearRange, "BeginKey"_sr, "EndKeyAB"_sr));
 			}
 
 			build += timer() - tstart;
@@ -534,7 +544,7 @@ static void printBuildInformation() {
 static void printVersion() {
 	printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
 	printf("source version %s\n", getSourceVersion());
-	printf("protocol %" PRIx64 "\n", currentProtocolVersion.version());
+	printf("protocol %" PRIx64 "\n", currentProtocolVersion().version());
 }
 
 static void printHelpTeaser(const char* name) {
@@ -727,6 +737,9 @@ static void printUsage(const char* name, bool devhelp) {
 		printOptionUsage("--io-trust-warn-only",
 		                 " Instead of failing when an I/O operation exceeds io_trust_seconds, just"
 		                 " log a warning to the trace log. Has no effect if io_trust_seconds is unspecified.");
+		printOptionUsage("--use-future-protocol-version [true,false]",
+		                 " Run the process with a simulated future protocol version."
+		                 " This option can be used testing purposes only!");
 		printf("\n"
 		       "The 'kvfiledump' role dump all key-values from kvfile to stdout in binary format:\n"
 		       "{key length}{key binary}{value length}{value binary}, length is 4 bytes int\n"
@@ -734,6 +747,18 @@ static void printUsage(const char* name, bool devhelp) {
 		       " - FDB_DUMP_STARTKEY: start key for the dump, default is empty\n"
 		       " - FDB_DUMP_ENDKEY: end key for the dump, default is \"\\xff\\xff\"\n"
 		       " - FDB_DUMP_DEBUG: print key-values to stderr in escaped format\n");
+
+		printf(
+		    "\n"
+		    "The 'changedescription' role replaces the old cluster key in all coordinators' data file to the specified "
+		    "new cluster key,\n"
+		    "which is passed in by '--new-cluster-key'. In particular, cluster key means '[description]:[id]'.\n"
+		    "'--datadir' is supposed to point to the top level directory of FDB's data, where subdirectories are for "
+		    "each process's data.\n"
+		    "The given cluster file passed in by '-C, --cluster-file' is considered to contain the old cluster key.\n"
+		    "It is used before restoring a snapshotted cluster to let the cluster have a different cluster key.\n"
+		    "Please make sure run it on every host in the cluster with the same '--new-cluster-key'.\n");
+
 	} else {
 		printOptionUsage("--dev-help", "Display developer-specific help and exit.");
 	}
@@ -862,7 +887,7 @@ std::pair<NetworkAddressList, NetworkAddressList> buildNetworkAddresses(
 
 	for (int ii = 0; ii < publicAddressStrs.size(); ++ii) {
 		const std::string& publicAddressStr = publicAddressStrs[ii];
-		bool autoPublicAddress = StringRef(publicAddressStr).startsWith(LiteralStringRef("auto:"));
+		bool autoPublicAddress = StringRef(publicAddressStr).startsWith("auto:"_sr);
 		NetworkAddress currentPublicAddress;
 		if (autoPublicAddress) {
 			try {
@@ -979,10 +1004,12 @@ void restoreRoleFilesHelper(std::string dirSrc, std::string dirToMove, std::stri
 
 namespace {
 enum class ServerRole {
+	ChangeClusterKey,
 	ConsistencyCheck,
 	CreateTemplateDatabase,
 	DSLTest,
 	FDBD,
+	FlowProcess,
 	KVFileGenerateIOLogChecksums,
 	KVFileIntegrityCheck,
 	KVFileDump,
@@ -995,13 +1022,12 @@ enum class ServerRole {
 	SkipListTest,
 	Test,
 	VersionedMapTest,
-	UnitTests,
-	FlowProcess
+	UnitTests
 };
 struct CLIOptions {
 	std::string commandLine;
-	std::string fileSystemPath, dataFolder, connFile, seedConnFile, seedConnString, logFolder = ".", metricsConnFile,
-	                                                                                metricsPrefix;
+	std::string fileSystemPath, dataFolder, connFile, seedConnFile, seedConnString,
+	    logFolder = ".", metricsConnFile, metricsPrefix, newClusterKey, authzPublicKeyFile;
 	std::string logGroup = "default";
 	uint64_t rollsize = TRACE_DEFAULT_ROLL_SIZE;
 	uint64_t maxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE;
@@ -1046,7 +1072,7 @@ struct CLIOptions {
 	const char* blobCredsFromENV = nullptr;
 
 	std::string configPath;
-	ConfigDBType configDBType{ ConfigDBType::DISABLED };
+	ConfigDBType configDBType{ ConfigDBType::PAXOS };
 
 	Reference<IClusterConnectionRecord> connectionFile;
 	Standalone<StringRef> machineId;
@@ -1061,6 +1087,7 @@ struct CLIOptions {
 	static CLIOptions parseArgs(int argc, char* argv[]) {
 		CLIOptions opts;
 		opts.parseArgsInternal(argc, argv);
+		opts.parseEnvInternal();
 		return opts;
 	}
 
@@ -1090,6 +1117,25 @@ struct CLIOptions {
 private:
 	CLIOptions() = default;
 
+	void parseEnvInternal() {
+		for (const std::string& knob : getEnvironmentKnobOptions()) {
+			auto pos = knob.find_first_of("=");
+			if (pos == std::string::npos) {
+				fprintf(stderr,
+				        "Error: malformed environment knob option: %s%s\n",
+				        ENVIRONMENT_KNOB_OPTION_PREFIX,
+				        knob.c_str());
+				TraceEvent(SevWarnAlways, "MalformedEnvironmentVariableKnob")
+				    .detail("Key", ENVIRONMENT_KNOB_OPTION_PREFIX + knob);
+			} else {
+				std::string k = knob.substr(0, pos);
+				std::string v = knob.substr(pos + 1, knob.length());
+				knobs.emplace_back(k, v);
+				manualKnobOverrides[k] = v;
+			}
+		}
+	}
+
 	void parseArgsInternal(int argc, char* argv[]) {
 		for (int a = 0; a < argc; a++) {
 			if (a)
@@ -1144,6 +1190,10 @@ private:
 				printUsage(argv[0], true);
 				flushAndExit(FDB_EXIT_SUCCESS);
 				break;
+			case OPT_PRINT_CODE_PROBES:
+				probe::ICodeProbe::printProbesJSON({ std::string(args.OptionArg()) });
+				flushAndExit(FDB_EXIT_SUCCESS);
+				break;
 			case OPT_KNOB: {
 				Optional<std::string> knobName = extractPrefixedArgument("--knob", args.OptionSyntax());
 				if (!knobName.present()) {
@@ -1245,6 +1295,8 @@ private:
 					role = ServerRole::UnitTests;
 				else if (!strcmp(sRole, "flowprocess"))
 					role = ServerRole::FlowProcess;
+				else if (!strcmp(sRole, "changeclusterkey"))
+					role = ServerRole::ChangeClusterKey;
 				else {
 					fprintf(stderr, "ERROR: Unknown role `%s'\n", sRole);
 					printHelpTeaser(argv[0]);
@@ -1578,6 +1630,9 @@ private:
 			case OPT_USE_TEST_CONFIG_DB:
 				configDBType = ConfigDBType::SIMPLE;
 				break;
+			case OPT_NO_CONFIG_DB:
+				configDBType = ConfigDBType::DISABLED;
+				break;
 			case OPT_FLOW_PROCESS_NAME:
 				flowProcessName = args.OptionArg();
 				std::cout << flowProcessName << std::endl;
@@ -1640,6 +1695,10 @@ private:
 				knobs.emplace_back("rest_kms_connector_kms_discovery_url_file", args.OptionArg());
 				break;
 			}
+			case OPT_KMS_CONNECTOR_TYPE: {
+				knobs.emplace_back("kms_connector_type", args.OptionArg());
+				break;
+			}
 			case OPT_KMS_CONN_VALIDATION_TOKEN_DETAILS: {
 				knobs.emplace_back("rest_kms_connector_validation_token_details", args.OptionArg());
 				break;
@@ -1648,6 +1707,29 @@ private:
 				knobs.emplace_back("rest_kms_connector_get_encryption_keys_endpoint", args.OptionArg());
 				break;
 			}
+			case OPT_NEW_CLUSTER_KEY: {
+				newClusterKey = args.OptionArg();
+				try {
+					ClusterConnectionString ccs;
+					// make sure the new cluster key is in valid format
+					ccs.parseKey(newClusterKey);
+				} catch (Error& e) {
+					std::cerr << "Invalid cluster key(description:id) '" << newClusterKey << "' from --new-cluster-key"
+					          << std::endl;
+					flushAndExit(FDB_EXIT_ERROR);
+				}
+				break;
+			}
+			case OPT_AUTHZ_PUBLIC_KEY_FILE: {
+				authzPublicKeyFile = args.OptionArg();
+				break;
+			}
+			case OPT_USE_FUTURE_PROTOCOL_VERSION: {
+				if (!strcmp(args.OptionArg(), "true")) {
+					::useFutureProtocolVersion();
+				}
+				break;
+			}
 			}
 		}
 
@@ -1683,7 +1765,7 @@ private:
 
 		bool autoPublicAddress =
 		    std::any_of(publicAddressStrs.begin(), publicAddressStrs.end(), [](const std::string& addr) {
-			    return StringRef(addr).startsWith(LiteralStringRef("auto:"));
+			    return StringRef(addr).startsWith("auto:"_sr);
 		    });
 		if ((role != ServerRole::Simulation && role != ServerRole::CreateTemplateDatabase &&
 		     role != ServerRole::KVFileIntegrityCheck && role != ServerRole::KVFileGenerateIOLogChecksums &&
@@ -1743,6 +1825,21 @@ private:
 			flushAndExit(FDB_EXIT_ERROR);
 		}
 
+		if (role == ServerRole::ChangeClusterKey) {
+			bool error = false;
+			if (!newClusterKey.size()) {
+				fprintf(stderr, "ERROR: please specify --new-cluster-key\n");
+				error = true;
+			} else if (connectionFile->getConnectionString().clusterKey() == newClusterKey) {
+				fprintf(stderr, "ERROR: the new cluster key is the same as the old one\n");
+				error = true;
+			}
+			if (error) {
+				printHelpTeaser(argv[0]);
+				flushAndExit(FDB_EXIT_ERROR);
+			}
+		}
+
 		// Interpret legacy "maxLogs" option in the most sensible and unsurprising way we can while eliminating its code
 		// path
 		if (maxLogsSet) {
@@ -1775,6 +1872,30 @@ private:
 			localities.set(LocalityData::keyDcId, dcId);
 	}
 };
+
+// Returns true iff validation is successful
+bool validateSimulationDataFiles(std::string const& dataFolder, bool isRestarting) {
+	std::vector<std::string> files = platform::listFiles(dataFolder);
+	if (!isRestarting) {
+		for (const auto& file : files) {
+			if (file != "restartInfo.ini" && file != getTestEncryptionFileName()) {
+				TraceEvent(SevError, "IncompatibleFileFound").detail("DataFolder", dataFolder).detail("FileName", file);
+				fprintf(stderr,
+				        "ERROR: Data folder `%s' is non-empty; please use clean, fdb-only folder\n",
+				        dataFolder.c_str());
+				return false;
+			}
+		}
+	} else if (isRestarting && files.empty()) {
+		TraceEvent(SevWarnAlways, "FileNotFound").detail("DataFolder", dataFolder);
+		printf("ERROR: Data folder `%s' is empty, but restarting option selected. Run Phase 1 test first\n",
+		       dataFolder.c_str());
+		return false;
+	}
+
+	return true;
+}
+
 } // namespace
 
 int main(int argc, char* argv[]) {
@@ -1806,8 +1927,10 @@ int main(int argc, char* argv[]) {
 		auto opts = CLIOptions::parseArgs(argc, argv);
 		const auto role = opts.role;
 
-		if (role == ServerRole::Simulation)
+		if (role == ServerRole::Simulation) {
 			printf("Random seed is %u...\n", opts.randomSeed);
+			bindDeterministicRandomToOpenssl();
+		}
 
 		if (opts.zoneId.present())
 			printf("ZoneId set to %s, dcId to %s\n", printable(opts.zoneId).c_str(), printable(opts.dcId).c_str());
@@ -1917,6 +2040,16 @@ int main(int argc, char* argv[]) {
 			openTraceFile(
 			    opts.publicAddresses.address, opts.rollsize, opts.maxLogsSize, opts.logFolder, "trace", opts.logGroup);
 			g_network->initTLS();
+			if (!opts.authzPublicKeyFile.empty()) {
+				try {
+					FlowTransport::transport().loadPublicKeyFile(opts.authzPublicKeyFile);
+				} catch (Error& e) {
+					TraceEvent("AuthzPublicKeySetLoadError").error(e);
+				}
+				FlowTransport::transport().watchPublicKeyFile(opts.authzPublicKeyFile);
+			} else {
+				TraceEvent(SevInfo, "AuthzPublicKeyFileNotSet");
+			}
 
 			if (expectsPublicAddress) {
 				for (int ii = 0; ii < (opts.publicAddresses.secondaryAddress.present() ? 2 : 1); ++ii) {
@@ -1961,6 +2094,14 @@ int main(int argc, char* argv[]) {
 				throw;
 		}
 
+		std::string environmentKnobOptions;
+		for (const std::string& knobOption : getEnvironmentKnobOptions()) {
+			environmentKnobOptions += knobOption + " ";
+		}
+		if (environmentKnobOptions.length()) {
+			environmentKnobOptions.pop_back();
+		}
+
 		TraceEvent("ProgramStart")
 		    .setMaxEventLength(12000)
 		    .detail("RandomSeed", opts.randomSeed)
@@ -1975,12 +2116,14 @@ int main(int argc, char* argv[]) {
 		            opts.connectionFile ? opts.connectionFile->getConnectionString().toString() : "")
 		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
 		    .setMaxFieldLength(10000)
+		    .detail("EnvironmentKnobOptions", environmentKnobOptions.length() ? environmentKnobOptions : "none")
 		    .detail("CommandLine", opts.commandLine)
 		    .setMaxFieldLength(0)
 		    .detail("BuggifyEnabled", opts.buggifyEnabled)
 		    .detail("FaultInjectionEnabled", opts.faultInjectionEnabled)
 		    .detail("MemoryLimit", opts.memLimit)
 		    .detail("VirtualMemoryLimit", opts.virtualMemLimit)
+		    .detail("ProtocolVersion", currentProtocolVersion())
 		    .trackLatest("ProgramStart");
 
 		Error::init();
@@ -2018,17 +2161,8 @@ int main(int argc, char* argv[]) {
 					flushAndExit(FDB_EXIT_ERROR);
 				}
 			}
-			std::vector<std::string> files = platform::listFiles(dataFolder);
-			if ((files.size() > 1 || (files.size() == 1 && files[0] != "restartInfo.ini")) && !opts.restarting) {
-				TraceEvent(SevError, "IncompatibleFileFound").detail("DataFolder", dataFolder);
-				fprintf(stderr,
-				        "ERROR: Data folder `%s' is non-empty; please use clean, fdb-only folder\n",
-				        dataFolder.c_str());
-				flushAndExit(FDB_EXIT_ERROR);
-			} else if (files.empty() && opts.restarting) {
-				TraceEvent(SevWarnAlways, "FileNotFound").detail("DataFolder", dataFolder);
-				printf("ERROR: Data folder `%s' is empty, but restarting option selected. Run Phase 1 test first\n",
-				       dataFolder.c_str());
+
+			if (!validateSimulationDataFiles(dataFolder, opts.restarting)) {
 				flushAndExit(FDB_EXIT_ERROR);
 			}
 
@@ -2125,13 +2259,15 @@ int main(int argc, char* argv[]) {
 				                KnobValue::create(ini.GetBoolValue("META", "enableEncryption", false)));
 				g_knobs.setKnob("enable_tlog_encryption",
 				                KnobValue::create(ini.GetBoolValue("META", "enableTLogEncryption", false)));
+				g_knobs.setKnob("enable_storage_server_encryption",
+				                KnobValue::create(ini.GetBoolValue("META", "enableStorageServerEncryption", false)));
 				g_knobs.setKnob("enable_blob_granule_encryption",
 				                KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false)));
 				g_knobs.setKnob("enable_blob_granule_compression",
 				                KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false)));
 			}
 			setupAndRun(dataFolder, opts.testFile, opts.restarting, (isRestoring >= 1), opts.whitelistBinPaths);
-			g_simulator.run();
+			g_simulator->run();
 		} else if (role == ServerRole::FDBD) {
 			// Update the global blob credential files list so that both fast
 			// restore workers and backup workers can access blob storage.
@@ -2252,11 +2388,27 @@ int main(int argc, char* argv[]) {
 
 			f = result;
 		} else if (role == ServerRole::FlowProcess) {
-			TraceEvent(SevDebug, "StartingFlowProcess").detail("From", "fdbserver");
-#if defined(__linux__) || defined(__FreeBSD__)
+			std::string traceFormat = getTraceFormatExtension();
+			// close and reopen trace file with the correct process listen address to name the file
+			closeTraceFile();
+			// writer is not shutdown immediately, addref on it
+			disposeTraceFileWriter();
+			// use the same trace format as before
+			selectTraceFormatter(traceFormat);
+			// create the trace file with the correct process address
+			openTraceFile(
+			    g_network->getLocalAddress(), opts.rollsize, opts.maxLogsSize, opts.logFolder, "trace", opts.logGroup);
+			auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
+			TraceEvent(SevDebug, "StartingFlowProcess").detail("FlowProcessName", opts.flowProcessName);
+#if defined(__linux__)
 			prctl(PR_SET_PDEATHSIG, SIGTERM);
 			if (getppid() == 1) /* parent already died before prctl */
 				flushAndExit(FDB_EXIT_SUCCESS);
+#elif defined(__FreeBSD__)
+			const int sig = SIGTERM;
+			procctl(P_PID, 0, PROC_PDEATHSIG_CTL, (void*)&sig);
+			if (getppid() == 1) /* parent already died before procctl */
+				flushAndExit(FDB_EXIT_SUCCESS);
 #endif
 
 			if (opts.flowProcessName == "KeyValueStoreProcess") {
@@ -2267,6 +2419,11 @@ int main(int argc, char* argv[]) {
 		} else if (role == ServerRole::KVFileDump) {
 			f = stopAfter(KVFileDump(opts.kvFile));
 			g_network->run();
+		} else if (role == ServerRole::ChangeClusterKey) {
+			Key newClusterKey(opts.newClusterKey);
+			Key oldClusterKey = opts.connectionFile->getConnectionString().clusterKey();
+			f = stopAfter(coordChangeClusterKey(opts.dataFolder, newClusterKey, oldClusterKey));
+			g_network->run();
 		}
 
 		int rc = FDB_EXIT_SUCCESS;
@@ -2325,7 +2482,7 @@ int main(int argc, char* argv[]) {
 			}
 		}
 
-		// g_simulator.run();
+		// g_simulator->run();
 
 #ifdef ALLOC_INSTRUMENTATION
 		{
@@ -2344,19 +2501,19 @@ int main(int argc, char* argv[]) {
 				char* demangled = abi::__cxa_demangle(i->first, nullptr, nullptr, nullptr);
 				if (demangled) {
 					s = demangled;
-					if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::")))
-						s = s.substr(LiteralStringRef("(anonymous namespace)::").size());
+					if (StringRef(s).startsWith("(anonymous namespace)::"_sr))
+						s = s.substr("(anonymous namespace)::"_sr.size());
 					free(demangled);
 				} else
 					s = i->first;
 #else
 				s = i->first;
-				if (StringRef(s).startsWith(LiteralStringRef("class `anonymous namespace'::")))
-					s = s.substr(LiteralStringRef("class `anonymous namespace'::").size());
-				else if (StringRef(s).startsWith(LiteralStringRef("class ")))
-					s = s.substr(LiteralStringRef("class ").size());
-				else if (StringRef(s).startsWith(LiteralStringRef("struct ")))
-					s = s.substr(LiteralStringRef("struct ").size());
+				if (StringRef(s).startsWith("class `anonymous namespace'::"_sr))
+					s = s.substr("class `anonymous namespace'::"_sr.size());
+				else if (StringRef(s).startsWith("class "_sr))
+					s = s.substr("class "_sr.size());
+				else if (StringRef(s).startsWith("struct "_sr))
+					s = s.substr("struct "_sr.size());
 #endif
 
 				typeNames.emplace_back(s, i->first);
diff --git a/fdbserver/include/fdbserver/ApplyMetadataMutation.h b/fdbserver/include/fdbserver/ApplyMetadataMutation.h
index 015d79b6f7..8977ea45cc 100644
--- a/fdbserver/include/fdbserver/ApplyMetadataMutation.h
+++ b/fdbserver/include/fdbserver/ApplyMetadataMutation.h
@@ -25,6 +25,7 @@
 #include <cstddef>
 
 #include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/MutationList.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/StorageServerInterface.h"
@@ -33,7 +34,6 @@
 #include "fdbserver/LogProtocolMessage.h"
 #include "fdbserver/LogSystem.h"
 #include "fdbserver/ProxyCommitData.actor.h"
-#include "flow/BlobCipher.h"
 #include "flow/FastRef.h"
 
 // Resolver's data for applyMetadataMutations() calls.
@@ -103,7 +103,8 @@ void applyMetadataMutations(SpanContext const& spanContext,
                             const UID& dbgid,
                             Arena& arena,
                             const VectorRef<MutationRef>& mutations,
-                            IKeyValueStore* txnStateStore);
+                            IKeyValueStore* txnStateStore,
+                            Reference<AsyncVar<ServerDBInfo> const> dbInfo);
 
 inline bool isSystemKey(KeyRef key) {
 	return key.size() && key[0] == systemKeys.begin[0];
@@ -130,10 +131,10 @@ inline bool containsMetadataMutation(const VectorRef<MutationRef>& mutations) {
 			    (serverTagKeys.intersects(range)) || (serverTagHistoryKeys.intersects(range)) ||
 			    (range.intersects(applyMutationsEndRange)) || (range.intersects(applyMutationsKeyVersionMapRange)) ||
 			    (range.intersects(logRangesRange)) || (tssMappingKeys.intersects(range)) ||
-			    (tssQuarantineKeys.intersects(range)) || (range.contains(coordinatorsKey)) ||
-			    (range.contains(databaseLockedKey)) || (range.contains(metadataVersionKey)) ||
-			    (range.contains(mustContainSystemMutationsKey)) || (range.contains(writeRecoveryKey)) ||
-			    (range.intersects(testOnlyTxnStateStorePrefixRange))) {
+			    (tssQuarantineKeys.intersects(range)) || (range.contains(previousCoordinatorsKey)) ||
+			    (range.contains(coordinatorsKey)) || (range.contains(databaseLockedKey)) ||
+			    (range.contains(metadataVersionKey)) || (range.contains(mustContainSystemMutationsKey)) ||
+			    (range.contains(writeRecoveryKey)) || (range.intersects(testOnlyTxnStateStorePrefixRange))) {
 				return true;
 			}
 		}
@@ -144,6 +145,7 @@ inline bool containsMetadataMutation(const VectorRef<MutationRef>& mutations) {
 // Resolver's version
 void applyMetadataMutations(SpanContext const& spanContext,
                             ResolverData& resolverData,
-                            const VectorRef<MutationRef>& mutations);
+                            const VectorRef<MutationRef>& mutations,
+                            Reference<AsyncVar<ServerDBInfo> const> dbInfo);
 
 #endif
diff --git a/fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h b/fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h
index 4325b31afb..dbd7195e76 100644
--- a/fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h
+++ b/fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h
@@ -31,20 +31,12 @@
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/Tenant.h"
+
 #include "fdbserver/ServerDBInfo.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbserver/Knobs.h"
 #include "flow/flow.h"
 
-struct GranuleHistory {
-	KeyRange range;
-	Version version;
-	Standalone<BlobGranuleHistoryValue> value;
-
-	GranuleHistory() {}
-
-	GranuleHistory(KeyRange range, Version version, Standalone<BlobGranuleHistoryValue> value)
-	  : range(range), version(version), value(value) {}
-};
+#include "flow/actorcompiler.h" // has to be last include
 
 // Stores info about a file in blob storage
 struct BlobFileIndex {
@@ -84,20 +76,15 @@ struct GranuleFiles {
 	              bool canCollapse,
 	              BlobGranuleChunkRef& chunk,
 	              Arena& replyArena,
-	              int64_t& deltaBytesCounter) const;
+	              int64_t& deltaBytesCounter,
+	              bool summarize) const;
 };
 
 // serialize change feed key as UID bytes, to use 16 bytes on disk
-static Key granuleIDToCFKey(UID granuleID) {
-	BinaryWriter wr(Unversioned());
-	wr << granuleID;
-	return wr.toValue();
-}
+Key granuleIDToCFKey(UID granuleID);
 
 // parse change feed key back to UID, to be human-readable
-static UID cfKeyToGranuleID(Key cfKey) {
-	return BinaryReader::fromStringRef<UID>(cfKey, Unversioned());
-}
+UID cfKeyToGranuleID(Key cfKey);
 
 class Transaction;
 ACTOR Future<Optional<GranuleHistory>> getLatestGranuleHistory(Transaction* tr, KeyRange range);
@@ -105,6 +92,9 @@ ACTOR Future<Void> readGranuleFiles(Transaction* tr, Key* startKey, Key endKey,
 
 ACTOR Future<GranuleFiles> loadHistoryFiles(Database cx, UID granuleID);
 
+enum ForcedPurgeState { NonePurged, SomePurged, AllPurged };
+ACTOR Future<ForcedPurgeState> getForcePurgedState(Transaction* tr, KeyRange keyRange);
+
 // TODO: versioned like SS has?
 struct GranuleTenantData : NonCopyable, ReferenceCounted<GranuleTenantData> {
 	TenantName name;
@@ -145,6 +135,13 @@ private:
 	Future<Void> collection;
 };
 
+ACTOR Future<Void> dumpManifest(Database db, Reference<BlobConnectionProvider> blobConn);
+ACTOR Future<Void> loadManifest(Database db, Reference<BlobConnectionProvider> blobConn);
+ACTOR Future<Void> printRestoreSummary(Database db, Reference<BlobConnectionProvider> blobConn);
+inline bool isFullRestoreMode() {
+	return SERVER_KNOBS->BLOB_FULL_RESTORE_MODE;
+};
+
 #include "flow/unactorcompiler.h"
 
 #endif
diff --git a/fdbserver/include/fdbserver/BlobGranuleValidation.actor.h b/fdbserver/include/fdbserver/BlobGranuleValidation.actor.h
index 4617ce557c..df89b24b96 100644
--- a/fdbserver/include/fdbserver/BlobGranuleValidation.actor.h
+++ b/fdbserver/include/fdbserver/BlobGranuleValidation.actor.h
@@ -51,6 +51,15 @@ bool compareFDBAndBlob(RangeResult fdb,
                        Version v,
                        bool debug);
 
+void printGranuleChunks(const Standalone<VectorRef<BlobGranuleChunkRef>>& chunks);
+
+ACTOR Future<Void> clearAndAwaitMerge(Database cx, KeyRange range);
+
+ACTOR Future<Void> validateGranuleSummaries(Database cx,
+                                            KeyRange range,
+                                            Optional<TenantName> tenantName,
+                                            Promise<Void> testComplete);
+
 #include "flow/unactorcompiler.h"
 
 #endif
diff --git a/fdbserver/include/fdbserver/BlobManagerInterface.h b/fdbserver/include/fdbserver/BlobManagerInterface.h
index 7fb73a220d..8ba341e4fd 100644
--- a/fdbserver/include/fdbserver/BlobManagerInterface.h
+++ b/fdbserver/include/fdbserver/BlobManagerInterface.h
@@ -32,11 +32,13 @@ struct BlobManagerInterface {
 	RequestStream<struct HaltBlobManagerRequest> haltBlobManager;
 	RequestStream<struct HaltBlobGranulesRequest> haltBlobGranules;
 	RequestStream<struct BlobManagerExclusionSafetyCheckRequest> blobManagerExclCheckReq;
+	RequestStream<struct BlobManagerBlockedRequest> blobManagerBlockedReq;
 	struct LocalityData locality;
 	UID myId;
+	int64_t epoch;
 
-	BlobManagerInterface() {}
-	explicit BlobManagerInterface(const struct LocalityData& l, UID id) : locality(l), myId(id) {}
+	BlobManagerInterface() : epoch(0) {}
+	BlobManagerInterface(const struct LocalityData& l, UID id, int64_t epoch) : locality(l), myId(id), epoch(epoch) {}
 
 	void initEndpoints() {}
 	UID id() const { return myId; }
@@ -46,7 +48,15 @@ struct BlobManagerInterface {
 
 	template <class Archive>
 	void serialize(Archive& ar) {
-		serializer(ar, waitFailure, haltBlobManager, haltBlobGranules, blobManagerExclCheckReq, locality, myId);
+		serializer(ar,
+		           waitFailure,
+		           haltBlobManager,
+		           haltBlobGranules,
+		           blobManagerExclCheckReq,
+		           blobManagerBlockedReq,
+		           locality,
+		           myId,
+		           epoch);
 	}
 };
 
@@ -106,4 +116,29 @@ struct BlobManagerExclusionSafetyCheckRequest {
 	}
 };
 
+struct BlobManagerBlockedReply {
+	constexpr static FileIdentifier file_identifier = 8078627;
+	int64_t blockedAssignments;
+
+	BlobManagerBlockedReply() : blockedAssignments(0) {}
+	explicit BlobManagerBlockedReply(int64_t blockedAssignments) : blockedAssignments(blockedAssignments) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, blockedAssignments);
+	}
+};
+
+struct BlobManagerBlockedRequest {
+	constexpr static FileIdentifier file_identifier = 1986387;
+	ReplyPromise<BlobManagerBlockedReply> reply;
+
+	BlobManagerBlockedRequest() {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, reply);
+	}
+};
+
 #endif
diff --git a/fdbserver/include/fdbserver/ClusterController.actor.h b/fdbserver/include/fdbserver/ClusterController.actor.h
index 5dd37c23e9..c0701f1850 100644
--- a/fdbserver/include/fdbserver/ClusterController.actor.h
+++ b/fdbserver/include/fdbserver/ClusterController.actor.h
@@ -51,6 +51,7 @@ struct WorkerInfo : NonCopyable {
 	Future<Void> haltDistributor;
 	Future<Void> haltBlobManager;
 	Future<Void> haltEncryptKeyProxy;
+	Future<Void> haltConsistencyScan;
 	Standalone<VectorRef<StringRef>> issues;
 
 	WorkerInfo()
@@ -73,7 +74,7 @@ struct WorkerInfo : NonCopyable {
 	  : watcher(std::move(r.watcher)), reply(std::move(r.reply)), gen(r.gen), reboots(r.reboots),
 	    initialClass(r.initialClass), priorityInfo(r.priorityInfo), details(std::move(r.details)),
 	    haltRatekeeper(r.haltRatekeeper), haltDistributor(r.haltDistributor), haltBlobManager(r.haltBlobManager),
-	    haltEncryptKeyProxy(r.haltEncryptKeyProxy), issues(r.issues) {}
+	    haltEncryptKeyProxy(r.haltEncryptKeyProxy), haltConsistencyScan(r.haltConsistencyScan), issues(r.issues) {}
 	void operator=(WorkerInfo&& r) noexcept {
 		watcher = std::move(r.watcher);
 		reply = std::move(r.reply);
@@ -139,6 +140,8 @@ public:
 		Future<Void> clientCounter;
 		int clientCount;
 		AsyncVar<bool> blobGranulesEnabled;
+		ClusterType clusterType = ClusterType::STANDALONE;
+		Optional<ClusterName> metaclusterName;
 
 		DBInfo()
 		  : clientInfo(new AsyncVar<ClientDBInfo>()), serverInfo(new AsyncVar<ServerDBInfo>()),
@@ -180,15 +183,30 @@ public:
 
 		void setEncryptKeyProxy(const EncryptKeyProxyInterface& interf) {
 			auto newInfo = serverInfo->get();
+			auto newClientInfo = clientInfo->get();
+			newClientInfo.id = deterministicRandom()->randomUniqueID();
 			newInfo.id = deterministicRandom()->randomUniqueID();
 			newInfo.infoGeneration = ++dbInfoCount;
 			newInfo.encryptKeyProxy = interf;
+			newInfo.client.encryptKeyProxy = interf;
+			newClientInfo.encryptKeyProxy = interf;
+			serverInfo->set(newInfo);
+			clientInfo->set(newClientInfo);
+		}
+
+		void setConsistencyScan(const ConsistencyScanInterface& interf) {
+			auto newInfo = serverInfo->get();
+			newInfo.id = deterministicRandom()->randomUniqueID();
+			newInfo.infoGeneration = ++dbInfoCount;
+			newInfo.consistencyScan = interf;
 			serverInfo->set(newInfo);
 		}
 
 		void clearInterf(ProcessClass::ClassType t) {
 			auto newInfo = serverInfo->get();
+			auto newClientInfo = clientInfo->get();
 			newInfo.id = deterministicRandom()->randomUniqueID();
+			newClientInfo.id = deterministicRandom()->randomUniqueID();
 			newInfo.infoGeneration = ++dbInfoCount;
 			if (t == ProcessClass::DataDistributorClass) {
 				newInfo.distributor = Optional<DataDistributorInterface>();
@@ -198,8 +216,13 @@ public:
 				newInfo.blobManager = Optional<BlobManagerInterface>();
 			} else if (t == ProcessClass::EncryptKeyProxyClass) {
 				newInfo.encryptKeyProxy = Optional<EncryptKeyProxyInterface>();
+				newInfo.client.encryptKeyProxy = Optional<EncryptKeyProxyInterface>();
+				newClientInfo.encryptKeyProxy = Optional<EncryptKeyProxyInterface>();
+			} else if (t == ProcessClass::ConsistencyScanClass) {
+				newInfo.consistencyScan = Optional<ConsistencyScanInterface>();
 			}
 			serverInfo->set(newInfo);
+			clientInfo->set(newClientInfo);
 		}
 
 		ACTOR static Future<Void> countClients(DBInfo* self) {
@@ -292,7 +315,9 @@ public:
 		       (db.serverInfo->get().blobManager.present() &&
 		        db.serverInfo->get().blobManager.get().locality.processId() == processId) ||
 		       (db.serverInfo->get().encryptKeyProxy.present() &&
-		        db.serverInfo->get().encryptKeyProxy.get().locality.processId() == processId);
+		        db.serverInfo->get().encryptKeyProxy.get().locality.processId() == processId) ||
+		       (db.serverInfo->get().consistencyScan.present() &&
+		        db.serverInfo->get().consistencyScan.get().locality.processId() == processId);
 	}
 
 	WorkerDetails getStorageWorker(RecruitStorageRequest const& req) {
@@ -2878,7 +2903,8 @@ public:
 		ASSERT(masterProcessId.present());
 		const auto& pid = worker.interf.locality.processId();
 		if ((role != ProcessClass::DataDistributor && role != ProcessClass::Ratekeeper &&
-		     role != ProcessClass::BlobManager && role != ProcessClass::EncryptKeyProxy) ||
+		     role != ProcessClass::BlobManager && role != ProcessClass::EncryptKeyProxy &&
+		     role != ProcessClass::ConsistencyScan) ||
 		    pid == masterProcessId.get()) {
 			return false;
 		}
@@ -2920,9 +2946,14 @@ public:
 		for (int i = 0; i < req.degradedPeers.size(); ++i) {
 			degradedPeersString += (i == 0 ? "" : " ") + req.degradedPeers[i].toString();
 		}
+		std::string disconnectedPeersString;
+		for (int i = 0; i < req.disconnectedPeers.size(); ++i) {
+			disconnectedPeersString += (i == 0 ? "" : " ") + req.disconnectedPeers[i].toString();
+		}
 		TraceEvent("ClusterControllerUpdateWorkerHealth")
 		    .detail("WorkerAddress", req.address)
-		    .detail("DegradedPeers", degradedPeersString);
+		    .detail("DegradedPeers", degradedPeersString)
+		    .detail("DisconnectedPeers", disconnectedPeersString);
 
 		double currentTime = now();
 
@@ -2934,6 +2965,11 @@ public:
 				workerHealth[req.address].degradedPeers[degradedPeer] = { currentTime, currentTime };
 			}
 
+			// TODO(zhewu): add disconnected peers in worker health.
+			for (const auto& degradedPeer : req.disconnectedPeers) {
+				workerHealth[req.address].degradedPeers[degradedPeer] = { currentTime, currentTime };
+			}
+
 			return;
 		}
 
@@ -2941,14 +2977,23 @@ public:
 
 		auto& health = workerHealth[req.address];
 
-		// Update the worker's degradedPeers.
-		for (const auto& peer : req.degradedPeers) {
-			auto it = health.degradedPeers.find(peer);
+		auto updateDegradedPeer = [&health, currentTime](const NetworkAddress& degradedPeer) {
+			auto it = health.degradedPeers.find(degradedPeer);
 			if (it == health.degradedPeers.end()) {
-				health.degradedPeers[peer] = { currentTime, currentTime };
-				continue;
+				health.degradedPeers[degradedPeer] = { currentTime, currentTime };
+				return;
 			}
 			it->second.lastRefreshTime = currentTime;
+		};
+
+		// Update the worker's degradedPeers.
+		for (const auto& peer : req.degradedPeers) {
+			updateDegradedPeer(peer);
+		}
+
+		// TODO(zhewu): add disconnected peers in worker health.
+		for (const auto& peer : req.disconnectedPeers) {
+			updateDegradedPeer(peer);
 		}
 	}
 
@@ -3261,6 +3306,8 @@ public:
 	Optional<UID> recruitingBlobManagerID;
 	AsyncVar<bool> recruitEncryptKeyProxy;
 	Optional<UID> recruitingEncryptKeyProxyID;
+	AsyncVar<bool> recruitConsistencyScan;
+	Optional<UID> recruitingConsistencyScanID;
 
 	// Stores the health information from a particular worker's perspective.
 	struct WorkerHealth {
@@ -3298,7 +3345,7 @@ public:
 	    goodRecruitmentTime(Never()), goodRemoteRecruitmentTime(Never()), datacenterVersionDifference(0),
 	    versionDifferenceUpdated(false), remoteDCMonitorStarted(false), remoteTransactionSystemDegraded(false),
 	    recruitDistributor(false), recruitRatekeeper(false), recruitBlobManager(false), recruitEncryptKeyProxy(false),
-	    clusterControllerMetrics("ClusterController", id.toString()),
+	    recruitConsistencyScan(false), clusterControllerMetrics("ClusterController", id.toString()),
 	    openDatabaseRequests("OpenDatabaseRequests", clusterControllerMetrics),
 	    registerWorkerRequests("RegisterWorkerRequests", clusterControllerMetrics),
 	    getWorkersRequests("GetWorkersRequests", clusterControllerMetrics),
@@ -3312,6 +3359,7 @@ public:
 		serverInfo.masterLifetime.ccID = id;
 		serverInfo.clusterInterface = ccInterface;
 		serverInfo.myLocality = locality;
+		serverInfo.client.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
 		db.serverInfo->set(serverInfo);
 		cx = openDBOnServer(db.serverInfo, TaskPriority::DefaultEndpoint, LockAware::True);
 
diff --git a/fdbserver/include/fdbserver/ClusterRecovery.actor.h b/fdbserver/include/fdbserver/ClusterRecovery.actor.h
index 810bd35f7a..a7071a8278 100644
--- a/fdbserver/include/fdbserver/ClusterRecovery.actor.h
+++ b/fdbserver/include/fdbserver/ClusterRecovery.actor.h
@@ -93,6 +93,8 @@ public:
 		return previousWrite;
 	}
 
+	ServerCoordinators getCoordinators() const { return coordinators; }
+
 	Future<Void> move(ClusterConnectionString const& nc) { return cstate.move(nc); }
 
 private:
@@ -132,7 +134,7 @@ private:
 			wait(self->cstate.setExclusive(
 			    BinaryWriter::toValue(newState, IncludeVersion(ProtocolVersion::withDBCoreState()))));
 		} catch (Error& e) {
-			TEST(true); // Master displaced during writeMasterState
+			CODE_PROBE(true, "Master displaced during writeMasterState");
 			throw;
 		}
 
@@ -148,7 +150,7 @@ private:
 
 			if (readState != newState) {
 				TraceEvent("RecoveryTerminated", self->dbgid).detail("Reason", "CStateChanged");
-				TEST(true); // Coordinated state changed between writing and reading, recovery restarting
+				CODE_PROBE(true, "Coordinated state changed between writing and reading, recovery restarting");
 				throw worker_removed();
 			}
 			self->switchedState = Promise<Void>();
diff --git a/fdbserver/include/fdbserver/ConfigBroadcastInterface.h b/fdbserver/include/fdbserver/ConfigBroadcastInterface.h
index cb75e1a251..1c7733680b 100644
--- a/fdbserver/include/fdbserver/ConfigBroadcastInterface.h
+++ b/fdbserver/include/fdbserver/ConfigBroadcastInterface.h
@@ -72,16 +72,17 @@ struct ConfigBroadcastSnapshotRequest {
 	static constexpr FileIdentifier file_identifier = 10911925;
 	Version version{ 0 };
 	std::map<ConfigKey, KnobValue> snapshot;
+	double restartDelay{ 0.0 };
 	ReplyPromise<ConfigBroadcastSnapshotReply> reply;
 
 	ConfigBroadcastSnapshotRequest() = default;
 	template <class Snapshot>
-	explicit ConfigBroadcastSnapshotRequest(Version version, Snapshot&& snapshot)
-	  : version(version), snapshot(std::forward<Snapshot>(snapshot)) {}
+	explicit ConfigBroadcastSnapshotRequest(Version version, Snapshot&& snapshot, double restartDelay)
+	  : version(version), snapshot(std::forward<Snapshot>(snapshot)), restartDelay(restartDelay) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, version, snapshot, reply);
+		serializer(ar, version, snapshot, restartDelay, reply);
 	}
 };
 
@@ -98,31 +99,35 @@ struct ConfigBroadcastChangesReply {
 
 struct ConfigBroadcastChangesRequest {
 	static constexpr FileIdentifier file_identifier = 601281;
-	Version mostRecentVersion;
+	Version mostRecentVersion{ 0 };
 	Standalone<VectorRef<VersionedConfigMutationRef>> changes;
+	double restartDelay{ 0.0 };
 	ReplyPromise<ConfigBroadcastChangesReply> reply;
 
 	ConfigBroadcastChangesRequest() = default;
 	explicit ConfigBroadcastChangesRequest(Version mostRecentVersion,
-	                                       Standalone<VectorRef<VersionedConfigMutationRef>> const& changes)
-	  : mostRecentVersion(mostRecentVersion), changes(changes) {}
+	                                       Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+	                                       double restartDelay)
+	  : mostRecentVersion(mostRecentVersion), changes(changes), restartDelay(restartDelay) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, mostRecentVersion, changes, reply);
+		serializer(ar, mostRecentVersion, changes, restartDelay, reply);
 	}
 };
 
 struct ConfigBroadcastRegisteredReply {
 	static constexpr FileIdentifier file_identifier = 12041047;
-	bool registered;
+	bool registered{ false };
+	Version lastSeenVersion{ 0 };
 
 	ConfigBroadcastRegisteredReply() = default;
-	explicit ConfigBroadcastRegisteredReply(bool registered) : registered(registered) {}
+	explicit ConfigBroadcastRegisteredReply(bool registered, Version lastSeenVersion)
+	  : registered(registered), lastSeenVersion(lastSeenVersion) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, registered);
+		serializer(ar, registered, lastSeenVersion);
 	}
 };
 
@@ -151,13 +156,23 @@ struct ConfigBroadcastReadyReply {
 
 struct ConfigBroadcastReadyRequest {
 	static constexpr FileIdentifier file_identifier = 7402862;
+	CoordinatorsHash coordinatorsHash{ 0 };
+	std::map<ConfigKey, KnobValue> snapshot;
+	Version snapshotVersion{ 0 };
+	Version liveVersion{ 0 };
 	ReplyPromise<ConfigBroadcastReadyReply> reply;
 
 	ConfigBroadcastReadyRequest() = default;
+	ConfigBroadcastReadyRequest(CoordinatorsHash coordinatorsHash,
+	                            std::map<ConfigKey, KnobValue> const& snapshot,
+	                            Version snapshotVersion,
+	                            Version liveVersion)
+	  : coordinatorsHash(coordinatorsHash), snapshot(snapshot), snapshotVersion(snapshotVersion),
+	    liveVersion(liveVersion) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, reply);
+		serializer(ar, coordinatorsHash, snapshot, snapshotVersion, liveVersion, reply);
 	}
 };
 
@@ -180,6 +195,7 @@ public:
 	bool operator==(ConfigBroadcastInterface const& rhs) const { return (_id == rhs._id); }
 	bool operator!=(ConfigBroadcastInterface const& rhs) const { return !(*this == rhs); }
 	UID id() const { return _id; }
+	NetworkAddress address() const { return snapshot.getEndpoint().getPrimaryAddress(); }
 
 	template <class Ar>
 	void serialize(Ar& ar) {
diff --git a/fdbserver/include/fdbserver/ConfigBroadcaster.h b/fdbserver/include/fdbserver/ConfigBroadcaster.h
index bddc1f5209..47808e9861 100644
--- a/fdbserver/include/fdbserver/ConfigBroadcaster.h
+++ b/fdbserver/include/fdbserver/ConfigBroadcaster.h
@@ -39,15 +39,16 @@ class ConfigBroadcaster {
 	PImpl<class ConfigBroadcasterImpl> impl;
 
 public:
-	explicit ConfigBroadcaster(ServerCoordinators const&, ConfigDBType);
+	ConfigBroadcaster();
+	explicit ConfigBroadcaster(ServerCoordinators const&, ConfigDBType, Future<Optional<Value>>);
 	ConfigBroadcaster(ConfigBroadcaster&&);
 	ConfigBroadcaster& operator=(ConfigBroadcaster&&);
 	~ConfigBroadcaster();
-	Future<Void> registerNode(WorkerInterface const& w,
+	Future<Void> registerNode(ConfigBroadcastInterface const& broadcastInterface,
 	                          Version lastSeenVersion,
 	                          ConfigClassSet const& configClassSet,
 	                          Future<Void> watcher,
-	                          ConfigBroadcastInterface const& worker);
+	                          bool isCoordinator);
 	void applyChanges(Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
 	                  Version mostRecentVersion,
 	                  Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations,
@@ -57,18 +58,26 @@ public:
 	                             Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
 	                             Version changesVersion,
 	                             Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations,
-	                             std::vector<ConfigFollowerInterface> const& readReplicas);
+	                             std::vector<ConfigFollowerInterface> const& readReplicas,
+	                             Version largestLiveVersion,
+	                             bool fromPreviousCoordinators = false);
 	void applySnapshotAndChanges(std::map<ConfigKey, KnobValue>&& snapshot,
 	                             Version snapshotVersion,
 	                             Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
 	                             Version changesVersion,
 	                             Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations,
-	                             std::vector<ConfigFollowerInterface> const& readReplicas);
+	                             std::vector<ConfigFollowerInterface> const& readReplicas,
+	                             Version largestLiveVersion,
+	                             bool fromPreviousCoordinators = false);
 	Future<Void> getError() const;
 	UID getID() const;
 	JsonBuilderObject getStatus() const;
 	void compact(Version compactionVersion);
 
+	// Locks all ConfigNodes running on the given coordinators, returning when
+	// a quorum have successfully locked.
+	static Future<Void> lockConfigNodes(ServerCoordinators coordinators);
+
 public: // Testing
 	explicit ConfigBroadcaster(ConfigFollowerInterface const&);
 	Future<Void> getClientFailure(UID clientUID) const;
diff --git a/fdbserver/include/fdbserver/ConfigFollowerInterface.h b/fdbserver/include/fdbserver/ConfigFollowerInterface.h
index 67137d86b2..aef438255c 100644
--- a/fdbserver/include/fdbserver/ConfigFollowerInterface.h
+++ b/fdbserver/include/fdbserver/ConfigFollowerInterface.h
@@ -27,7 +27,7 @@
 #include "fdbrpc/fdbrpc.h"
 
 struct VersionedConfigMutationRef {
-	Version version;
+	Version version{ ::invalidVersion };
 	ConfigMutationRef mutation;
 
 	VersionedConfigMutationRef() = default;
@@ -46,7 +46,7 @@ struct VersionedConfigMutationRef {
 using VersionedConfigMutation = Standalone<VersionedConfigMutationRef>;
 
 struct VersionedConfigCommitAnnotationRef {
-	Version version;
+	Version version{ ::invalidVersion };
 	ConfigCommitAnnotationRef annotation;
 
 	VersionedConfigCommitAnnotationRef() = default;
@@ -66,7 +66,7 @@ using VersionedConfigCommitAnnotation = Standalone<VersionedConfigCommitAnnotati
 
 struct ConfigFollowerGetSnapshotAndChangesReply {
 	static constexpr FileIdentifier file_identifier = 1734095;
-	Version snapshotVersion;
+	Version snapshotVersion{ 0 };
 	std::map<ConfigKey, KnobValue> snapshot;
 	// TODO: Share arena
 	Standalone<VectorRef<VersionedConfigMutationRef>> changes;
@@ -84,14 +84,14 @@ struct ConfigFollowerGetSnapshotAndChangesReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, snapshotVersion, snapshot, changes);
+		serializer(ar, snapshotVersion, snapshot, changes, annotations);
 	}
 };
 
 struct ConfigFollowerGetSnapshotAndChangesRequest {
 	static constexpr FileIdentifier file_identifier = 294811;
 	ReplyPromise<ConfigFollowerGetSnapshotAndChangesReply> reply;
-	Version mostRecentVersion;
+	Version mostRecentVersion{ 0 };
 
 	ConfigFollowerGetSnapshotAndChangesRequest() = default;
 	explicit ConfigFollowerGetSnapshotAndChangesRequest(Version mostRecentVersion)
@@ -158,6 +158,7 @@ struct ConfigFollowerRollforwardRequest {
 	Version target{ 0 };
 	Standalone<VectorRef<VersionedConfigMutationRef>> mutations;
 	Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations;
+	bool specialZeroQuorum{ false };
 	ReplyPromise<Void> reply;
 
 	ConfigFollowerRollforwardRequest() = default;
@@ -165,28 +166,34 @@ struct ConfigFollowerRollforwardRequest {
 	                                          Version lastKnownCommitted,
 	                                          Version target,
 	                                          Standalone<VectorRef<VersionedConfigMutationRef>> mutations,
-	                                          Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations)
+	                                          Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations,
+	                                          bool specialZeroQuorum)
 	  : rollback(rollback), lastKnownCommitted(lastKnownCommitted), target(target), mutations(mutations),
-	    annotations(annotations) {}
+	    annotations(annotations), specialZeroQuorum(specialZeroQuorum) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, rollback, lastKnownCommitted, target, mutations, annotations, reply);
+		serializer(ar, rollback, lastKnownCommitted, target, mutations, annotations, specialZeroQuorum, reply);
 	}
 };
 
 struct ConfigFollowerGetCommittedVersionReply {
 	static constexpr FileIdentifier file_identifier = 9214735;
-	Version lastCompacted;
-	Version lastCommitted;
+	bool registered{ false };
+	Version lastCompacted{ 0 };
+	Version lastLive{ 0 };
+	Version lastCommitted{ 0 };
 
 	ConfigFollowerGetCommittedVersionReply() = default;
-	explicit ConfigFollowerGetCommittedVersionReply(Version lastCompacted, Version lastCommitted)
-	  : lastCompacted(lastCompacted), lastCommitted(lastCommitted) {}
+	explicit ConfigFollowerGetCommittedVersionReply(bool registered,
+	                                                Version lastCompacted,
+	                                                Version lastLive,
+	                                                Version lastCommitted)
+	  : registered(registered), lastCompacted(lastCompacted), lastLive(lastLive), lastCommitted(lastCommitted) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, lastCompacted, lastCommitted);
+		serializer(ar, registered, lastCompacted, lastLive, lastCommitted);
 	}
 };
 
@@ -202,9 +209,23 @@ struct ConfigFollowerGetCommittedVersionRequest {
 	}
 };
 
+struct ConfigFollowerLockRequest {
+	static constexpr FileIdentifier file_identifier = 1867800;
+	CoordinatorsHash coordinatorsHash{ 0 };
+	ReplyPromise<Void> reply;
+
+	ConfigFollowerLockRequest() = default;
+	explicit ConfigFollowerLockRequest(CoordinatorsHash coordinatorsHash) : coordinatorsHash(coordinatorsHash) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, coordinatorsHash, reply);
+	}
+};
+
 /*
  * Configuration database nodes serve a ConfigFollowerInterface which contains well known endpoints,
- * used by workers to receive configuration database updates
+ * used by the broadcaster to receive configuration database updates
  */
 class ConfigFollowerInterface {
 	UID _id;
@@ -217,6 +238,7 @@ public:
 	RequestStream<ConfigFollowerRollforwardRequest> rollforward;
 	RequestStream<ConfigFollowerGetCommittedVersionRequest> getCommittedVersion;
 	Optional<Hostname> hostname;
+	RequestStream<ConfigFollowerLockRequest> lock;
 
 	ConfigFollowerInterface();
 	void setupWellKnownEndpoints();
@@ -229,6 +251,7 @@ public:
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, _id, getSnapshotAndChanges, getChanges, compact, rollforward, getCommittedVersion, hostname);
+		serializer(
+		    ar, _id, getSnapshotAndChanges, getChanges, compact, rollforward, getCommittedVersion, hostname, lock);
 	}
 };
diff --git a/fdbserver/include/fdbserver/ConfigNode.h b/fdbserver/include/fdbserver/ConfigNode.h
index 333652893b..5454ac63da 100644
--- a/fdbserver/include/fdbserver/ConfigNode.h
+++ b/fdbserver/include/fdbserver/ConfigNode.h
@@ -38,6 +38,7 @@ public:
 	                   ConfigFollowerInterface const&);
 
 public: // Testing
+	Future<Void> serve(ConfigBroadcastInterface const&);
 	Future<Void> serve(ConfigTransactionInterface const&);
 	Future<Void> serve(ConfigFollowerInterface const&);
 	void close();
diff --git a/fdbserver/include/fdbserver/CoordinatedState.h b/fdbserver/include/fdbserver/CoordinatedState.h
index da88be04eb..186d7a57d7 100644
--- a/fdbserver/include/fdbserver/CoordinatedState.h
+++ b/fdbserver/include/fdbserver/CoordinatedState.h
@@ -23,6 +23,7 @@
 #pragma once
 
 #include "fdbclient/FDBTypes.h"
+#include "fdbclient/PImpl.h"
 
 class CoordinatedState : NonCopyable {
 public:
@@ -53,10 +54,10 @@ public:
 	//   returned from read may or may not ever have been a valid state.  Probably there was a
 	//   call to read() or setExclusive() concurrently with this pair.
 
-	uint64_t getConflict();
+	uint64_t getConflict() const;
 
 private:
-	std::unique_ptr<struct CoordinatedStateImpl> impl;
+	PImpl<struct CoordinatedStateImpl> impl;
 };
 
 class MovableCoordinatedState : NonCopyable {
@@ -78,7 +79,7 @@ public:
 	// (and therefore the caller should die).
 
 private:
-	std::unique_ptr<struct MovableCoordinatedStateImpl> impl;
+	PImpl<struct MovableCoordinatedStateImpl> impl;
 };
 
 #endif
diff --git a/fdbserver/include/fdbserver/CoordinationInterface.h b/fdbserver/include/fdbserver/CoordinationInterface.h
index 826799f7a1..43fff9ed65 100644
--- a/fdbserver/include/fdbserver/CoordinationInterface.h
+++ b/fdbserver/include/fdbserver/CoordinationInterface.h
@@ -224,7 +224,9 @@ class ConfigNode;
 
 class ServerCoordinators : public ClientCoordinators {
 public:
-	explicit ServerCoordinators(Reference<IClusterConnectionRecord> ccr);
+	ServerCoordinators() {}
+	explicit ServerCoordinators(Reference<IClusterConnectionRecord> ccr,
+	                            ConfigDBType configDBType = ConfigDBType::PAXOS);
 
 	std::vector<LeaderElectionRegInterface> leaderElectionServers;
 	std::vector<GenerationRegInterface> stateServers;
@@ -236,4 +238,9 @@ Future<Void> coordinationServer(std::string const& dataFolder,
                                 Reference<ConfigNode> const&,
                                 ConfigBroadcastInterface const&);
 
+// Read a value of MovableValue and if the old cluster key is nested in it, update it to the new key
+Optional<Value> updateCCSInMovableValue(ValueRef movableVal, KeyRef oldClusterKey, KeyRef newClusterKey);
+
+Future<Void> coordChangeClusterKey(std::string dataFolder, KeyRef newClusterKey, KeyRef oldClusterKey);
+
 #endif
diff --git a/fdbserver/include/fdbserver/DDRelocationQueue.h b/fdbserver/include/fdbserver/DDRelocationQueue.h
new file mode 100644
index 0000000000..e4b4509c90
--- /dev/null
+++ b/fdbserver/include/fdbserver/DDRelocationQueue.h
@@ -0,0 +1,35 @@
+/*
+ * DDRelocationQueue.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FOUNDATIONDB_DDRELOCATIONQUEUE_H
+#define FOUNDATIONDB_DDRELOCATIONQUEUE_H
+
+#include "fdbserver/DataDistribution.actor.h"
+// send request/signal to DDRelocationQueue through interface
+// call synchronous method from components outside DDRelocationQueue
+struct IDDRelocationQueue {
+	PromiseStream<RelocateShard> relocationProducer, relocationConsumer; // FIXME(xwang): not used yet
+	// PromiseStream<Promise<int>> getUnhealthyRelocationCount; // FIXME(xwang): change it to a synchronous call
+
+	virtual int getUnhealthyRelocationCount() = 0;
+	virtual ~IDDRelocationQueue() = default;
+	;
+};
+
+#endif // FOUNDATIONDB_DDRELOCATIONQUEUE_H
diff --git a/fdbserver/include/fdbserver/DDShardTracker.h b/fdbserver/include/fdbserver/DDShardTracker.h
new file mode 100644
index 0000000000..62a5409509
--- /dev/null
+++ b/fdbserver/include/fdbserver/DDShardTracker.h
@@ -0,0 +1,40 @@
+/*
+ * DDShardTracker.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FOUNDATIONDB_DDSHARDTRACKER_H
+#define FOUNDATIONDB_DDSHARDTRACKER_H
+#include "fdbserver/DataDistribution.actor.h"
+
+// send request/signal to DDTracker through interface
+// call synchronous method from components outside DDShardTracker
+struct IDDShardTracker {
+	// FIXME: the streams are not used yet
+	Promise<Void> readyToStart;
+	PromiseStream<GetMetricsRequest> getShardMetrics;
+	PromiseStream<GetTopKMetricsRequest> getTopKMetrics;
+	PromiseStream<GetMetricsListRequest> getShardMetricsList;
+	PromiseStream<KeyRange> restartShardTracker;
+
+	// PromiseStream<Promise<int64_t>> averageShardBytes; // FIXME(xwang): change it to a synchronous call
+
+	virtual double getAverageShardBytes() = 0;
+	virtual ~IDDShardTracker() = default;
+};
+
+#endif // FOUNDATIONDB_DDSHARDTRACKER_H
diff --git a/fdbserver/include/fdbserver/DDSharedContext.h b/fdbserver/include/fdbserver/DDSharedContext.h
new file mode 100644
index 0000000000..60cc0a1307
--- /dev/null
+++ b/fdbserver/include/fdbserver/DDSharedContext.h
@@ -0,0 +1,70 @@
+/*
+ * DDSharedContext.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FOUNDATIONDB_DDSHAREDCONTEXT_H
+#define FOUNDATIONDB_DDSHAREDCONTEXT_H
+#include "fdbserver/MoveKeys.actor.h"
+#include "fdbserver/ShardsAffectedByTeamFailure.h"
+#include "fdbserver/DDShardTracker.h"
+#include "fdbserver/DDRelocationQueue.h"
+#include "fdbserver/DDTeamCollection.h"
+
+// The common info shared by all DD components. Normally the DD components should share the reference to the same
+// context.
+// NOTE: We should avoid the shared class become an insanely large class, think twice before add member to it.
+class DDSharedContext : public ReferenceCounted<DDSharedContext> {
+	// FIXME(xwang) mark fields privates
+public:
+	std::unique_ptr<DDEnabledState>
+	    ddEnabledState; // Note: don't operate directly because it's shared with snapshot server
+	IDDShardTracker* shardTracker = nullptr;
+	IDDRelocationQueue* relocationQueue = nullptr;
+	std::vector<IDDTeamCollection*> teamCollections;
+
+	// public:
+	UID ddId;
+	MoveKeysLock lock;
+	bool trackerCancelled = false;
+	DatabaseConfiguration configuration;
+
+	Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure;
+	Reference<AsyncVar<bool>> processingUnhealthy, processingWiggle;
+
+	DDSharedContext() = default;
+
+	DDSharedContext(UID id)
+	  : ddEnabledState(new DDEnabledState), ddId(id), shardsAffectedByTeamFailure(new ShardsAffectedByTeamFailure),
+	    processingUnhealthy(new AsyncVar<bool>(false)), processingWiggle(new AsyncVar<bool>(false)) {}
+
+	UID id() const { return ddId; }
+
+	void markTrackerCancelled() { trackerCancelled = true; }
+
+	bool isTrackerCancelled() const { return trackerCancelled; }
+
+	decltype(auto) usableRegions() const { return configuration.usableRegions; }
+
+	bool isDDEnabled() const { return ddEnabledState->isDDEnabled(); };
+
+	void proposeRelocation(const RelocateShard& rs) const { return relocationQueue->relocationProducer.send(rs); }
+
+	void requestRestartShardTracker(KeyRange keys) const { return shardTracker->restartShardTracker.send(keys); }
+};
+
+#endif // FOUNDATIONDB_DDSHAREDCONTEXT_H
diff --git a/fdbserver/include/fdbserver/DDTeamCollection.h b/fdbserver/include/fdbserver/DDTeamCollection.h
index f719586622..93cf786fea 100644
--- a/fdbserver/include/fdbserver/DDTeamCollection.h
+++ b/fdbserver/include/fdbserver/DDTeamCollection.h
@@ -173,6 +173,13 @@ FDB_DECLARE_BOOLEAN_PARAM(IsRedundantTeam);
 FDB_DECLARE_BOOLEAN_PARAM(IsBadTeam);
 FDB_DECLARE_BOOLEAN_PARAM(WaitWiggle);
 
+// send request/signal to DDTeamCollection through interface
+// call synchronous method from components outside DDTeamCollection
+struct IDDTeamCollection {
+	PromiseStream<GetTeamRequest> getTeam;
+	virtual ~IDDTeamCollection() {}
+};
+
 class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 	friend class DDTeamCollectionImpl;
 	friend class DDTeamCollectionUnitTest;
@@ -402,7 +409,7 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 	// in the next iteration of the loop. Otherwise, you may miss checking some elements in machineTeams
 	bool removeMachineTeam(Reference<TCMachineTeamInfo> targetMT);
 
-	// Adds storage servers held on process of which the Process Id is “id” into excludeServers which prevent
+	// Adds storage servers held on process of which the Process id is “id” into excludeServers which prevent
 	// recruiting the wiggling storage servers and let teamTracker start to move data off the affected teams;
 	// Return a vector of futures wait for all data is moved to other teams.
 	Future<Void> excludeStorageServersForWiggle(const UID& id);
@@ -596,6 +603,7 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 	int addTeamsBestOf(int teamsToBuild, int desiredTeams, int maxTeams);
 
 public:
+	Reference<IDDTxnProcessor> db;
 	Database cx;
 
 	DatabaseConfiguration configuration;
@@ -613,7 +621,7 @@ public:
 	AsyncTrigger printDetailedTeamsInfo;
 	Reference<LocalitySet> storageServerSet;
 
-	DDTeamCollection(Database const& cx,
+	DDTeamCollection(Reference<IDDTxnProcessor>& db,
 	                 UID distributorId,
 	                 MoveKeysLock const& lock,
 	                 PromiseStream<RelocateShard> const& output,
@@ -669,6 +677,12 @@ public:
 
 	UID getDistributorId() const { return distributorId; }
 
+	// Divide TSS evenly in each DC if there are multiple
+	// TODO would it be better to put all of them in primary DC?
+	int32_t getTargetTSSInDC() const;
+
+	bool reachTSSPairTarget() const;
+
 	// Keep track of servers and teams -- serves requests for getRandomTeam
 	static Future<Void> run(Reference<DDTeamCollection> teamCollection,
 	                        Reference<InitialDataDistribution> initData,
diff --git a/fdbserver/include/fdbserver/DDTxnProcessor.h b/fdbserver/include/fdbserver/DDTxnProcessor.h
index 8926a51b10..d2e3158958 100644
--- a/fdbserver/include/fdbserver/DDTxnProcessor.h
+++ b/fdbserver/include/fdbserver/DDTxnProcessor.h
@@ -22,25 +22,35 @@
 #define FOUNDATIONDB_DDTXNPROCESSOR_H
 
 #include "fdbserver/Knobs.h"
+#include "flow/FastRef.h"
 #include "fdbserver/MoveKeys.actor.h"
+#include "fdbserver/MockGlobalState.h"
 
 struct InitialDataDistribution;
+struct DDShardInfo;
+
+struct ServerWorkerInfos {
+	std::vector<std::pair<StorageServerInterface, ProcessClass>> servers;
+	Optional<Version> readVersion; // the read version of the txn reading server lists
+};
 
 /* Testability Contract:
- * a. The DataDistributor has to use this interface to interact with data-plane (aka. run transaction), because the
- * testability benefits from a mock implementation; b. Other control-plane roles should consider providing its own
- * TxnProcessor interface to provide testability, for example, Ratekeeper.
+ * a. The DataDistributor has to use this interface to interact with data-plane (aka. run transaction / use Database),
+ * because the testability benefits from a mock implementation; b. Other control-plane roles should consider providing
+ * its own TxnProcessor interface to provide testability, for example, Ratekeeper.
  * */
-class IDDTxnProcessor {
+class IDDTxnProcessor : public ReferenceCounted<IDDTxnProcessor> {
 public:
 	struct SourceServers {
 		std::vector<UID> srcServers, completeSources; // the same as RelocateData.src, RelocateData.completeSources;
 	};
+	virtual Database context() const = 0;
+	virtual bool isMocked() const = 0;
 	// get the source server list and complete source server list for range
-	virtual Future<SourceServers> getSourceServersForRange(const KeyRangeRef range) = 0;
+	virtual Future<SourceServers> getSourceServersForRange(const KeyRangeRef range) { return SourceServers{}; };
 
-	// get the storage server list and Process class
-	virtual Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses() = 0;
+	// get the storage server list and Process class, only throw transaction non-retryable exceptions
+	virtual Future<ServerWorkerInfos> getServerListAndProcessClasses() = 0;
 
 	virtual Future<Reference<InitialDataDistribution>> getInitialDataDistribution(
 	    const UID& distributorId,
@@ -50,7 +60,7 @@ public:
 
 	virtual ~IDDTxnProcessor() = default;
 
-	[[nodiscard]] virtual Future<MoveKeysLock> takeMoveKeysLock(UID ddId) const { return MoveKeysLock(); }
+	[[nodiscard]] virtual Future<MoveKeysLock> takeMoveKeysLock(const UID& ddId) const { return MoveKeysLock(); }
 
 	virtual Future<DatabaseConfiguration> getDatabaseConfiguration() const { return DatabaseConfiguration(); }
 
@@ -60,7 +70,58 @@ public:
 		return Void();
 	}
 
-	virtual Future<Void> waitForDataDistributionEnabled(const DDEnabledState* ddEnabledState) const = 0;
+	virtual Future<int> tryUpdateReplicasKeyForDc(const Optional<Key>& dcId, const int& storageTeamSize) const {
+		return storageTeamSize;
+	}
+
+	virtual Future<Void> waitForDataDistributionEnabled(const DDEnabledState* ddEnabledState) const { return Void(); };
+
+	virtual Future<bool> isDataDistributionEnabled(const DDEnabledState* ddEnabledState) const {
+		return ddEnabledState->isDDEnabled();
+	};
+
+	virtual Future<Void> pollMoveKeysLock(const MoveKeysLock& lock, const DDEnabledState* ddEnabledState) const {
+		return Never();
+	};
+
+	// Remove the server from shardMapping and set serverKeysFalse to the server's serverKeys list.
+	// Changes to keyServer and serverKey must happen symmetrically in this function.
+	// If serverID is the last source server for a shard, the shard will be erased, and then be assigned
+	// to teamForDroppedRange.
+	// It's used by `exclude failed` command to bypass data movement from failed server.
+	virtual Future<Void> removeKeysFromFailedServer(const UID& serverID,
+	                                                const std::vector<UID>& teamForDroppedRange,
+	                                                const MoveKeysLock& lock,
+	                                                const DDEnabledState* ddEnabledState) const = 0;
+	virtual Future<Void> removeStorageServer(const UID& serverID,
+	                                         const Optional<UID>& tssPairID,
+	                                         const MoveKeysLock& lock,
+	                                         const DDEnabledState* ddEnabledState) const = 0;
+
+	virtual Future<Void> moveKeys(const MoveKeysParams& params) = 0;
+
+	virtual Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(KeyRange const& keys,
+	                                                                            StorageMetrics const& min,
+	                                                                            StorageMetrics const& max,
+	                                                                            StorageMetrics const& permittedError,
+	                                                                            int shardLimit,
+	                                                                            int expectedShardCount) const = 0;
+
+	virtual Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(
+	    KeyRange const& keys,
+	    StorageMetrics const& limit,
+	    StorageMetrics const& estimated,
+	    Optional<int> const& minSplitBytes = {}) const = 0;
+
+	virtual Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(KeyRange const& keys) const = 0;
+
+	virtual Future<HealthMetrics> getHealthMetrics(bool detailed = false) const = 0;
+
+	virtual Future<Optional<Value>> readRebalanceDDIgnoreKey() const { return {}; }
+
+	virtual Future<UID> getClusterId() const { return {}; }
+
+	virtual Future<Void> waitDDTeamInfoPrintSignal() const { return Never(); }
 };
 
 class DDTxnProcessorImpl;
@@ -74,10 +135,13 @@ public:
 	DDTxnProcessor() = default;
 	explicit DDTxnProcessor(Database cx) : cx(cx) {}
 
+	Database context() const override { return cx; };
+	bool isMocked() const override { return false; };
+
 	Future<SourceServers> getSourceServersForRange(const KeyRangeRef range) override;
 
 	// Call NativeAPI implementation directly
-	Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses() override;
+	Future<ServerWorkerInfos> getServerListAndProcessClasses() override;
 
 	Future<Reference<InitialDataDistribution>> getInitialDataDistribution(
 	    const UID& distributorId,
@@ -85,7 +149,7 @@ public:
 	    const std::vector<Optional<Key>>& remoteDcIds,
 	    const DDEnabledState* ddEnabledState) override;
 
-	Future<MoveKeysLock> takeMoveKeysLock(UID ddId) const override;
+	Future<MoveKeysLock> takeMoveKeysLock(UID const& ddId) const override;
 
 	Future<DatabaseConfiguration> getDatabaseConfiguration() const override;
 
@@ -93,12 +157,108 @@ public:
 	                               const std::vector<Optional<Key>>& remoteIds,
 	                               const DatabaseConfiguration& configuration) const override;
 
+	Future<int> tryUpdateReplicasKeyForDc(const Optional<Key>& dcId, const int& storageTeamSize) const override;
+
 	Future<Void> waitForDataDistributionEnabled(const DDEnabledState* ddEnabledState) const override;
+
+	Future<bool> isDataDistributionEnabled(const DDEnabledState* ddEnabledState) const override;
+
+	Future<Void> pollMoveKeysLock(const MoveKeysLock& lock, const DDEnabledState* ddEnabledState) const override;
+
+	Future<Void> removeKeysFromFailedServer(const UID& serverID,
+	                                        const std::vector<UID>& teamForDroppedRange,
+	                                        const MoveKeysLock& lock,
+	                                        const DDEnabledState* ddEnabledState) const override {
+		return ::removeKeysFromFailedServer(cx, serverID, teamForDroppedRange, lock, ddEnabledState);
+	}
+
+	Future<Void> removeStorageServer(const UID& serverID,
+	                                 const Optional<UID>& tssPairID,
+	                                 const MoveKeysLock& lock,
+	                                 const DDEnabledState* ddEnabledState) const override {
+		return ::removeStorageServer(cx, serverID, tssPairID, lock, ddEnabledState);
+	}
+
+	Future<Void> moveKeys(const MoveKeysParams& params) override { return ::moveKeys(cx, params); }
+
+	Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(KeyRange const& keys,
+	                                                                    StorageMetrics const& min,
+	                                                                    StorageMetrics const& max,
+	                                                                    StorageMetrics const& permittedError,
+	                                                                    int shardLimit,
+	                                                                    int expectedShardCount) const override;
+
+	Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(KeyRange const& keys,
+	                                                          StorageMetrics const& limit,
+	                                                          StorageMetrics const& estimated,
+	                                                          Optional<int> const& minSplitBytes = {}) const override;
+
+	Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(KeyRange const& keys) const override;
+
+	Future<HealthMetrics> getHealthMetrics(bool detailed) const override;
+
+	Future<Optional<Value>> readRebalanceDDIgnoreKey() const override;
+
+	Future<UID> getClusterId() const override;
+
+	Future<Void> waitDDTeamInfoPrintSignal() const override;
 };
 
 // A mock transaction implementation for test usage.
 // Contract: every function involving mock transaction should return immediately to mimic the ACI property of real
 // transaction.
-class DDMockTxnProcessor : public IDDTxnProcessor {};
+class DDMockTxnProcessor : public IDDTxnProcessor {
+	std::shared_ptr<MockGlobalState> mgs;
+
+	std::vector<DDShardInfo> getDDShardInfos() const;
+
+public:
+	explicit DDMockTxnProcessor(std::shared_ptr<MockGlobalState> mgs = nullptr) : mgs(std::move(mgs)){};
+
+	Future<ServerWorkerInfos> getServerListAndProcessClasses() override;
+
+	Future<Reference<InitialDataDistribution>> getInitialDataDistribution(
+	    const UID& distributorId,
+	    const MoveKeysLock& moveKeysLock,
+	    const std::vector<Optional<Key>>& remoteDcIds,
+	    const DDEnabledState* ddEnabledState) override;
+
+	Future<Void> removeKeysFromFailedServer(const UID& serverID,
+	                                        const std::vector<UID>& teamForDroppedRange,
+	                                        const MoveKeysLock& lock,
+	                                        const DDEnabledState* ddEnabledState) const override;
+
+	Future<Void> removeStorageServer(const UID& serverID,
+	                                 const Optional<UID>& tssPairID,
+	                                 const MoveKeysLock& lock,
+	                                 const DDEnabledState* ddEnabledState) const override;
+
+	// test only
+	void setupMockGlobalState(Reference<InitialDataDistribution> initData);
+
+	Future<Void> moveKeys(const MoveKeysParams& params) override;
+
+	Database context() const override { UNREACHABLE(); };
+
+	bool isMocked() const override { return true; };
+
+	Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(KeyRange const& keys,
+	                                                                    StorageMetrics const& min,
+	                                                                    StorageMetrics const& max,
+	                                                                    StorageMetrics const& permittedError,
+	                                                                    int shardLimit,
+	                                                                    int expectedShardCount) const override;
+
+	Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(KeyRange const& keys,
+	                                                          StorageMetrics const& limit,
+	                                                          StorageMetrics const& estimated,
+	                                                          Optional<int> const& minSplitBytes = {}) const override;
+
+	Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(KeyRange const& keys) const override {
+		UNREACHABLE();
+	}
+
+	Future<HealthMetrics> getHealthMetrics(bool detailed = false) const override;
+};
 
 #endif // FOUNDATIONDB_DDTXNPROCESSOR_H
diff --git a/fdbserver/include/fdbserver/DataDistribution.actor.h b/fdbserver/include/fdbserver/DataDistribution.actor.h
index 508b7a11ab..e8016101e3 100644
--- a/fdbserver/include/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/include/fdbserver/DataDistribution.actor.h
@@ -25,20 +25,72 @@
 #define FDBSERVER_DATA_DISTRIBUTION_ACTOR_H
 
 #include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/MoveKeys.actor.h"
+#include "fdbserver/TenantCache.h"
+#include "fdbserver/TCInfo.h"
 #include "fdbclient/RunTransaction.actor.h"
 #include "fdbserver/DDTxnProcessor.h"
+#include "fdbserver/ShardsAffectedByTeamFailure.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/LogSystem.h"
 #include "fdbserver/MoveKeys.actor.h"
+#include "fdbserver/ShardsAffectedByTeamFailure.h"
 #include <boost/heap/policies.hpp>
 #include <boost/heap/skew_heap.hpp>
-
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-enum class RelocateReason { INVALID = -1, OTHER, REBALANCE_DISK, REBALANCE_READ };
+/////////////////////////////// Data //////////////////////////////////////
+#ifndef __INTEL_COMPILER
+#pragma region Data
+#endif
+
+// SOMEDAY: whether it's possible to combine RelocateReason and DataMovementReason together?
+// RelocateReason to DataMovementReason is one-to-N mapping
+class RelocateReason {
+public:
+	enum Value : int8_t {
+		OTHER = 0,
+		REBALANCE_DISK,
+		REBALANCE_READ,
+		MERGE_SHARD,
+		SIZE_SPLIT,
+		WRITE_SPLIT,
+		TENANT_SPLIT,
+		__COUNT
+	};
+	RelocateReason(Value v) : value(v) { ASSERT(value != __COUNT); }
+	explicit RelocateReason(int v) : value((Value)v) { ASSERT(value != __COUNT); }
+	std::string toString() const {
+		switch (value) {
+		case OTHER:
+			return "Other";
+		case REBALANCE_DISK:
+			return "RebalanceDisk";
+		case REBALANCE_READ:
+			return "RebalanceRead";
+		case MERGE_SHARD:
+			return "MergeShard";
+		case SIZE_SPLIT:
+			return "SizeSplit";
+		case WRITE_SPLIT:
+			return "WriteSplit";
+		case TENANT_SPLIT:
+			return "TenantSplit";
+		case __COUNT:
+			ASSERT(false);
+		}
+		return "";
+	}
+	operator int() const { return (int)value; }
+	constexpr static int8_t typeCount() { return (int)__COUNT; }
+
+private:
+	Value value;
+};
 
 // One-to-one relationship to the priority knobs
 enum class DataMovementReason {
+	INVALID,
 	RECOVER_MOVE,
 	REBALANCE_UNDERUTILIZED_TEAM,
 	REBALANCE_OVERUTILIZED_TEAM,
@@ -55,8 +107,11 @@ enum class DataMovementReason {
 	TEAM_1_LEFT,
 	TEAM_FAILED,
 	TEAM_0_LEFT,
-	SPLIT_SHARD
+	SPLIT_SHARD,
+	ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD
 };
+extern int dataMovementPriority(DataMovementReason moveReason);
+extern DataMovementReason priorityToDataMovementReason(int priority);
 
 struct DDShardInfo;
 
@@ -89,135 +144,27 @@ struct RelocateShard {
 	std::shared_ptr<DataMove> dataMove; // Not null if this is a restored data move.
 	UID dataMoveId;
 	RelocateReason reason;
-	RelocateShard() : priority(0), cancelled(false), dataMoveId(anonymousShardId), reason(RelocateReason::INVALID) {}
-	RelocateShard(KeyRange const& keys, int priority, RelocateReason reason)
-	  : keys(keys), priority(priority), cancelled(false), dataMoveId(anonymousShardId), reason(reason) {}
+	DataMovementReason moveReason;
+
+	UID traceId; // track the lifetime of this relocate shard
+
+	// Initialization when define is a better practice. We should avoid assignment of member after definition.
+	// static RelocateShard emptyRelocateShard() { return {}; }
+
+	RelocateShard(KeyRange const& keys, DataMovementReason moveReason, RelocateReason reason, UID traceId = UID())
+	  : keys(keys), priority(dataMovementPriority(moveReason)), cancelled(false), dataMoveId(anonymousShardId),
+	    reason(reason), moveReason(moveReason), traceId(traceId) {}
+
+	RelocateShard(KeyRange const& keys, int priority, RelocateReason reason, UID traceId = UID())
+	  : keys(keys), priority(priority), cancelled(false), dataMoveId(anonymousShardId), reason(reason),
+	    moveReason(priorityToDataMovementReason(priority)), traceId(traceId) {}
 
 	bool isRestore() const { return this->dataMove != nullptr; }
-};
-
-struct IDataDistributionTeam {
-	virtual std::vector<StorageServerInterface> getLastKnownServerInterfaces() const = 0;
-	virtual int size() const = 0;
-	virtual std::vector<UID> const& getServerIDs() const = 0;
-	virtual void addDataInFlightToTeam(int64_t delta) = 0;
-	virtual void addReadInFlightToTeam(int64_t delta) = 0;
-	virtual int64_t getDataInFlightToTeam() const = 0;
-	virtual int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const = 0;
-	virtual int64_t getReadInFlightToTeam() const = 0;
-	virtual double getLoadReadBandwidth(bool includeInFlight = true, double inflightPenalty = 1.0) const = 0;
-	virtual int64_t getMinAvailableSpace(bool includeInFlight = true) const = 0;
-	virtual double getMinAvailableSpaceRatio(bool includeInFlight = true) const = 0;
-	virtual bool hasHealthyAvailableSpace(double minRatio) const = 0;
-	virtual Future<Void> updateStorageMetrics() = 0;
-	virtual void addref() const = 0;
-	virtual void delref() const = 0;
-	virtual bool isHealthy() const = 0;
-	virtual void setHealthy(bool) = 0;
-	virtual int getPriority() const = 0;
-	virtual void setPriority(int) = 0;
-	virtual bool isOptimal() const = 0;
-	virtual bool isWrongConfiguration() const = 0;
-	virtual void setWrongConfiguration(bool) = 0;
-	virtual void addServers(const std::vector<UID>& servers) = 0;
-	virtual std::string getTeamID() const = 0;
-
-	std::string getDesc() const {
-		const auto& servers = getLastKnownServerInterfaces();
-		std::string s = format("TeamID %s; ", getTeamID().c_str());
-		s += format("Size %d; ", servers.size());
-		for (int i = 0; i < servers.size(); i++) {
-			if (i)
-				s += ", ";
-			s += servers[i].address().toString() + " " + servers[i].id().shortString();
-		}
-		return s;
-	}
-};
-
-FDB_DECLARE_BOOLEAN_PARAM(WantNewServers);
-FDB_DECLARE_BOOLEAN_PARAM(WantTrueBest);
-FDB_DECLARE_BOOLEAN_PARAM(PreferLowerDiskUtil);
-FDB_DECLARE_BOOLEAN_PARAM(TeamMustHaveShards);
-FDB_DECLARE_BOOLEAN_PARAM(ForReadBalance);
-FDB_DECLARE_BOOLEAN_PARAM(PreferLowerReadUtil);
-FDB_DECLARE_BOOLEAN_PARAM(FindTeamByServers);
-
-struct GetTeamRequest {
-	bool wantsNewServers; // In additional to servers in completeSources, try to find teams with new server
-	bool wantsTrueBest;
-	bool preferLowerDiskUtil; // if true, lower utilized team has higher score
-	bool teamMustHaveShards;
-	bool forReadBalance;
-	bool preferLowerReadUtil; // only make sense when forReadBalance is true
-	double inflightPenalty;
-	bool findTeamByServers;
-	std::vector<UID> completeSources;
-	std::vector<UID> src;
-	Promise<std::pair<Optional<Reference<IDataDistributionTeam>>, bool>> reply;
-
-	typedef Reference<IDataDistributionTeam> TeamRef;
-
-	GetTeamRequest() {}
-	GetTeamRequest(WantNewServers wantsNewServers,
-	               WantTrueBest wantsTrueBest,
-	               PreferLowerDiskUtil preferLowerDiskUtil,
-	               TeamMustHaveShards teamMustHaveShards,
-	               ForReadBalance forReadBalance = ForReadBalance::False,
-	               PreferLowerReadUtil preferLowerReadUtil = PreferLowerReadUtil::False,
-	               double inflightPenalty = 1.0)
-	  : wantsNewServers(wantsNewServers), wantsTrueBest(wantsTrueBest), preferLowerDiskUtil(preferLowerDiskUtil),
-	    teamMustHaveShards(teamMustHaveShards), forReadBalance(forReadBalance),
-	    preferLowerReadUtil(preferLowerReadUtil), inflightPenalty(inflightPenalty),
-	    findTeamByServers(FindTeamByServers::False) {}
-	GetTeamRequest(std::vector<UID> servers)
-	  : wantsNewServers(WantNewServers::False), wantsTrueBest(WantTrueBest::False),
-	    preferLowerDiskUtil(PreferLowerDiskUtil::False), teamMustHaveShards(TeamMustHaveShards::False),
-	    forReadBalance(ForReadBalance::False), preferLowerReadUtil(PreferLowerReadUtil::False), inflightPenalty(1.0),
-	    findTeamByServers(FindTeamByServers::True), src(std::move(servers)) {}
-
-	// return true if a.score < b.score
-	[[nodiscard]] bool lessCompare(TeamRef a, TeamRef b, int64_t aLoadBytes, int64_t bLoadBytes) const {
-		int res = 0;
-		if (forReadBalance) {
-			res = preferLowerReadUtil ? greaterReadLoad(a, b) : lessReadLoad(a, b);
-		}
-		return res == 0 ? lessCompareByLoad(aLoadBytes, bLoadBytes) : res < 0;
-	}
-
-	std::string getDesc() const {
-		std::stringstream ss;
-
-		ss << "WantsNewServers:" << wantsNewServers << " WantsTrueBest:" << wantsTrueBest
-		   << " PreferLowerDiskUtil:" << preferLowerDiskUtil << " teamMustHaveShards:" << teamMustHaveShards
-		   << "forReadBalance" << forReadBalance << " inflightPenalty:" << inflightPenalty
-		   << " findTeamByServers:" << findTeamByServers << ";";
-		ss << "CompleteSources:";
-		for (const auto& cs : completeSources) {
-			ss << cs.toString() << ",";
-		}
-
-		return std::move(ss).str();
-	}
 
 private:
-	// return true if preferHigherUtil && aLoadBytes <= bLoadBytes (higher load bytes has larger score)
-	// or preferLowerUtil && aLoadBytes > bLoadBytes
-	bool lessCompareByLoad(int64_t aLoadBytes, int64_t bLoadBytes) const {
-		bool lessLoad = aLoadBytes <= bLoadBytes;
-		return preferLowerDiskUtil ? !lessLoad : lessLoad;
-	}
-
-	// return -1 if a.readload > b.readload
-	static int greaterReadLoad(TeamRef a, TeamRef b) {
-		auto r1 = a->getLoadReadBandwidth(true), r2 = b->getLoadReadBandwidth(true);
-		return r1 == r2 ? 0 : (r1 > r2 ? -1 : 1);
-	}
-	// return -1 if a.readload < b.readload
-	static int lessReadLoad(TeamRef a, TeamRef b) {
-		auto r1 = a->getLoadReadBandwidth(false), r2 = b->getLoadReadBandwidth(false);
-		return r1 == r2 ? 0 : (r1 < r2 ? -1 : 1);
-	}
+	RelocateShard()
+	  : priority(0), cancelled(false), dataMoveId(anonymousShardId), reason(RelocateReason::OTHER),
+	    moveReason(DataMovementReason::INVALID) {}
 };
 
 struct GetMetricsRequest {
@@ -228,17 +175,23 @@ struct GetMetricsRequest {
 };
 
 struct GetTopKMetricsReply {
-	std::vector<StorageMetrics> metrics;
+	struct KeyRangeStorageMetrics {
+		KeyRange range;
+		StorageMetrics metrics;
+		KeyRangeStorageMetrics() = default;
+		KeyRangeStorageMetrics(const KeyRange& range, const StorageMetrics& s) : range(range), metrics(s) {}
+	};
+	std::vector<KeyRangeStorageMetrics> shardMetrics;
 	double minReadLoad = -1, maxReadLoad = -1;
 	GetTopKMetricsReply() {}
-	GetTopKMetricsReply(std::vector<StorageMetrics> const& m, double minReadLoad, double maxReadLoad)
-	  : metrics(m), minReadLoad(minReadLoad), maxReadLoad(maxReadLoad) {}
+	GetTopKMetricsReply(std::vector<KeyRangeStorageMetrics> const& m, double minReadLoad, double maxReadLoad)
+	  : shardMetrics(m), minReadLoad(minReadLoad), maxReadLoad(maxReadLoad) {}
 };
+
 struct GetTopKMetricsRequest {
-	// whether a > b
-	typedef std::function<bool(const StorageMetrics& a, const StorageMetrics& b)> MetricsComparator;
-	int topK = 1; // default only return the top 1 shard based on the comparator
-	MetricsComparator comparator; // Return true if a.score > b.score, return the largest topK in keys
+private:
+	int topK = 1; // default only return the top 1 shard based on the GetTopKMetricsRequest::compare function
+public:
 	std::vector<KeyRange> keys;
 	Promise<GetTopKMetricsReply> reply; // topK storage metrics
 	double maxBytesReadPerKSecond = 0, minBytesReadPerKSecond = 0; // all returned shards won't exceed this read load
@@ -249,7 +202,25 @@ struct GetTopKMetricsRequest {
 	                      double maxBytesReadPerKSecond = std::numeric_limits<double>::max(),
 	                      double minBytesReadPerKSecond = 0)
 	  : topK(topK), keys(keys), maxBytesReadPerKSecond(maxBytesReadPerKSecond),
-	    minBytesReadPerKSecond(minBytesReadPerKSecond) {}
+	    minBytesReadPerKSecond(minBytesReadPerKSecond) {
+		ASSERT_GE(topK, 1);
+	}
+
+	int getTopK() const { return topK; };
+
+	// Return true if a.score > b.score, return the largest topK in keys
+	static bool compare(const GetTopKMetricsReply::KeyRangeStorageMetrics& a,
+	                    const GetTopKMetricsReply::KeyRangeStorageMetrics& b) {
+		return compareByReadDensity(a, b);
+	}
+
+private:
+	// larger read density means higher score
+	static bool compareByReadDensity(const GetTopKMetricsReply::KeyRangeStorageMetrics& a,
+	                                 const GetTopKMetricsReply::KeyRangeStorageMetrics& b) {
+		return a.metrics.bytesReadPerKSecond / std::max(a.metrics.bytes * 1.0, 1.0) >
+		       b.metrics.bytesReadPerKSecond / std::max(b.metrics.bytes * 1.0, 1.0);
+	}
 };
 
 struct GetMetricsListRequest {
@@ -261,89 +232,192 @@ struct GetMetricsListRequest {
 	GetMetricsListRequest(KeyRange const& keys, const int shardLimit) : keys(keys), shardLimit(shardLimit) {}
 };
 
-struct TeamCollectionInterface {
-	PromiseStream<GetTeamRequest> getTeam;
-};
+// PhysicalShardCollection maintains physical shard concepts in data distribution
+// A physical shard contains one or multiple shards (key range)
+// PhysicalShardCollection is responsible for creation and maintenance of physical shards (including metrics)
+// For multiple DCs, PhysicalShardCollection maintains a pair of primary team and remote team
+// A primary team and a remote team shares a physical shard
+// For each shard (key-range) move, PhysicalShardCollection decides which physical shard and corresponding team(s) to
+// move The current design of PhysicalShardCollection assumes that there exists at most two teamCollections
+// TODO: unit test needed
+FDB_DECLARE_BOOLEAN_PARAM(InAnonymousPhysicalShard);
+FDB_DECLARE_BOOLEAN_PARAM(PhysicalShardHasMoreThanKeyRange);
+FDB_DECLARE_BOOLEAN_PARAM(InOverSizePhysicalShard);
+FDB_DECLARE_BOOLEAN_PARAM(PhysicalShardAvailable);
+FDB_DECLARE_BOOLEAN_PARAM(MoveKeyRangeOutPhysicalShard);
 
-class ShardsAffectedByTeamFailure : public ReferenceCounted<ShardsAffectedByTeamFailure> {
+class PhysicalShardCollection : public ReferenceCounted<PhysicalShardCollection> {
 public:
-	ShardsAffectedByTeamFailure() {}
+	PhysicalShardCollection() : lastTransitionStartTime(now()), requireTransition(false) {}
 
-	struct Team {
-		std::vector<UID> servers; // sorted
-		bool primary;
+	enum class PhysicalShardCreationTime { DDInit, DDRelocator };
 
-		Team() : primary(true) {}
-		Team(std::vector<UID> const& servers, bool primary) : servers(servers), primary(primary) {}
+	struct PhysicalShard {
+		PhysicalShard() : id(UID().first()) {}
 
-		bool operator<(const Team& r) const {
-			if (servers == r.servers)
-				return primary < r.primary;
-			return servers < r.servers;
-		}
-		bool operator>(const Team& r) const { return r < *this; }
-		bool operator<=(const Team& r) const { return !(*this > r); }
-		bool operator>=(const Team& r) const { return !(*this < r); }
-		bool operator==(const Team& r) const { return servers == r.servers && primary == r.primary; }
-		bool operator!=(const Team& r) const { return !(*this == r); }
+		PhysicalShard(uint64_t id,
+		              StorageMetrics const& metrics,
+		              std::vector<ShardsAffectedByTeamFailure::Team> teams,
+		              PhysicalShardCreationTime whenCreated)
+		  : id(id), metrics(metrics), teams(teams), whenCreated(whenCreated) {}
 
-		std::string toString() const { return describe(servers); };
+		std::string toString() const { return fmt::format("{}", std::to_string(id)); }
+
+		uint64_t id; // physical shard id (never changed)
+		StorageMetrics metrics; // current metrics, updated by shardTracker
+		std::vector<ShardsAffectedByTeamFailure::Team> teams; // which team owns this physical shard (never changed)
+		PhysicalShardCreationTime whenCreated; // when this physical shard is created (never changed)
 	};
 
-	// This tracks the data distribution on the data distribution server so that teamTrackers can
-	//   relocate the right shards when a team is degraded.
+	// Two-step team selection
+	// Usage: getting primary dest team and remote dest team in dataDistributionRelocator()
+	// The overall process has two steps:
+	// Step 1: get a physical shard id given the input primary team
+	// Return a new physical shard id if the input primary team is new or the team has no available physical shard
+	// checkPhysicalShardAvailable() defines whether a physical shard is available
+	uint64_t determinePhysicalShardIDGivenPrimaryTeam(ShardsAffectedByTeamFailure::Team primaryTeam,
+	                                                  StorageMetrics const& metrics,
+	                                                  bool forceToUseNewPhysicalShard,
+	                                                  uint64_t debugID);
 
-	// The following are important to make sure that failure responses don't revert splits or merges:
-	//   - The shards boundaries in the two data structures reflect "queued" RelocateShard requests
-	//       (i.e. reflects the desired set of shards being tracked by dataDistributionTracker,
-	//       rather than the status quo).  These boundaries are modified in defineShard and the content
-	//       of what servers correspond to each shard is a copy or union of the shards already there
-	//   - The teams associated with each shard reflect either the sources for non-moving shards
-	//       or the destination team for in-flight shards (the change is atomic with respect to team selection).
-	//       moveShard() changes the servers associated with a shard and will never adjust the shard
-	//       boundaries. If a move is received for a shard that has been redefined (the exact shard is
-	//       no longer in the map), the servers will be set for all contained shards and added to all
-	//       intersecting shards.
+	// Step 2: get a remote team which has the input physical shard
+	// Return empty if no such remote team
+	// May return a problematic remote team, and re-selection is required for this case
+	Optional<ShardsAffectedByTeamFailure::Team> tryGetAvailableRemoteTeamWith(uint64_t inputPhysicalShardID,
+	                                                                          StorageMetrics const& moveInMetrics,
+	                                                                          uint64_t debugID);
+	// Invariant:
+	// (1) If forceToUseNewPhysicalShard is set, use the bestTeams selected by getTeam(), and create a new physical
+	// shard for the teams
+	// (2) If forceToUseNewPhysicalShard is not set, use the primary team selected by getTeam()
+	//     If there exists a remote team which has an available physical shard with the primary team
+	//         Then, use the remote team. Note that the remote team may be unhealthy and the remote team
+	//         may be one who issues the current data relocation.
+	//         In this case, we set forceToUseNewPhysicalShard to use getTeam() to re-select the remote team
+	//     Otherwise, use getTeam() to re-select the remote team
 
-	int getNumberOfShards(UID ssID) const;
-	std::vector<KeyRange> getShardsFor(Team team) const;
-	bool hasShards(Team team) const;
+	// Create a physical shard when initializing PhysicalShardCollection
+	void initPhysicalShardCollection(KeyRange keys,
+	                                 std::vector<ShardsAffectedByTeamFailure::Team> selectedTeams,
+	                                 uint64_t physicalShardID,
+	                                 uint64_t debugID);
 
-	// The first element of the pair is either the source for non-moving shards or the destination team for in-flight
-	// shards The second element of the pair is all previous sources for in-flight shards
-	std::pair<std::vector<Team>, std::vector<Team>> getTeamsFor(KeyRangeRef keys);
+	// Create a physical shard when updating PhysicalShardCollection
+	void updatePhysicalShardCollection(KeyRange keys,
+	                                   bool isRestore,
+	                                   std::vector<ShardsAffectedByTeamFailure::Team> selectedTeams,
+	                                   uint64_t physicalShardID,
+	                                   const StorageMetrics& metrics,
+	                                   uint64_t debugID);
 
-	void defineShard(KeyRangeRef keys);
-	void moveShard(KeyRangeRef keys, std::vector<Team> destinationTeam);
-	void finishMove(KeyRangeRef keys);
-	void check() const;
+	// Update physicalShard metrics and return whether the keyRange needs to move out of its physical shard
+	MoveKeyRangeOutPhysicalShard trackPhysicalShard(KeyRange keyRange,
+	                                                StorageMetrics const& newMetrics,
+	                                                StorageMetrics const& oldMetrics,
+	                                                bool initWithNewMetrics);
 
-	PromiseStream<KeyRange> restartShardTracker;
+	// Clean up empty physicalShard
+	void cleanUpPhysicalShardCollection();
+
+	// Log physicalShard
+	void logPhysicalShardCollection();
 
 private:
-	struct OrderByTeamKey {
-		bool operator()(const std::pair<Team, KeyRange>& lhs, const std::pair<Team, KeyRange>& rhs) const {
-			if (lhs.first < rhs.first)
-				return true;
-			if (lhs.first > rhs.first)
-				return false;
-			return lhs.second.begin < rhs.second.begin;
+	// Track physicalShard metrics by tracking keyRange metrics
+	void updatePhysicalShardMetricsByKeyRange(KeyRange keyRange,
+	                                          StorageMetrics const& newMetrics,
+	                                          StorageMetrics const& oldMetrics,
+	                                          bool initWithNewMetrics);
+
+	// Check the input keyRange is in the anonymous physical shard
+	InAnonymousPhysicalShard isInAnonymousPhysicalShard(KeyRange keyRange);
+
+	// Check the input physicalShard has more keyRanges in addition to the input keyRange
+	PhysicalShardHasMoreThanKeyRange whetherPhysicalShardHasMoreThanKeyRange(uint64_t physicalShardID,
+	                                                                         KeyRange keyRange);
+
+	// Check the input keyRange is in an oversize physical shard
+	// This function returns true to enforce the keyRange to move out the physical shard
+	// Note that if the physical shard only contains the keyRange, always return FALSE
+	InOverSizePhysicalShard isInOverSizePhysicalShard(KeyRange keyRange);
+
+	// Generate a random physical shard ID, which is not UID().first() nor anonymousShardId.first()
+	uint64_t generateNewPhysicalShardID(uint64_t debugID);
+
+	// Check whether the input physical shard is available
+	// A physical shard is available if the current metric + moveInMetrics <= a threshold
+	PhysicalShardAvailable checkPhysicalShardAvailable(uint64_t physicalShardID, StorageMetrics const& moveInMetrics);
+
+	// If the input team has any available physical shard, return an available physical shard of the input team
+	Optional<uint64_t> trySelectAvailablePhysicalShardFor(ShardsAffectedByTeamFailure::Team team,
+	                                                      StorageMetrics const& metrics,
+	                                                      uint64_t debugID);
+
+	// Reduce the metics of input physical shard by the input metrics
+	void reduceMetricsForMoveOut(uint64_t physicalShardID, StorageMetrics const& metrics);
+
+	// Add the input metrics to the metrics of input physical shard
+	void increaseMetricsForMoveIn(uint64_t physicalShardID, StorageMetrics const& metrics);
+
+	// In physicalShardCollection, add a physical shard initialized by the input parameters to the collection
+	void insertPhysicalShardToCollection(uint64_t physicalShardID,
+	                                     StorageMetrics const& metrics,
+	                                     std::vector<ShardsAffectedByTeamFailure::Team> teams,
+	                                     uint64_t debugID,
+	                                     PhysicalShardCreationTime whenCreated);
+
+	// In teamPhysicalShardIDs, add the input physical shard id to the input teams
+	void updateTeamPhysicalShardIDsMap(uint64_t physicalShardID,
+	                                   std::vector<ShardsAffectedByTeamFailure::Team> inputTeams,
+	                                   uint64_t debugID);
+
+	// In keyRangePhysicalShardIDMap, set the input physical shard id to the input key range
+	void updatekeyRangePhysicalShardIDMap(KeyRange keyRange, uint64_t physicalShardID, uint64_t debugID);
+
+	// Return a string concating the input IDs interleaving with " "
+	std::string convertIDsToString(std::set<uint64_t> ids);
+
+	// Reset TransitionStartTime
+	// Consider a system without concept of physicalShard
+	// When restart, the system begins with a state where all keyRanges are in the anonymousShard
+	// Our goal is to make all keyRanges are out of the anonymousShard
+	// A keyRange moves out of the anonymousShard when the keyRange is triggered a data move
+	// It is possible that a keyRange is cold and no data move is triggered on this keyRange for long time
+	// In this case, we need to intentionally trigger data move on that keyRange
+	// The minimal time span between two successive data move for this purpose is TransitionStartTime
+	inline void resetLastTransitionStartTime() { // reset when a keyRange move is triggered for the transition
+		lastTransitionStartTime = now();
+		return;
+	}
+
+	// When DD restarts, it checks whether keyRange has anonymousShard
+	// If yes, setTransitionCheck() is call to trigger the process of removing anonymousShard
+	inline void setTransitionCheck() {
+		if (requireTransition == true) {
+			return;
 		}
-	};
+		requireTransition = true;
+		TraceEvent("PhysicalShardSetTransitionCheck");
+		return;
+	}
 
-	KeyRangeMap<std::pair<std::vector<Team>, std::vector<Team>>>
-	    shard_teams; // A shard can be affected by the failure of multiple teams if it is a queued merge, or when
-	                 // usable_regions > 1
-	std::set<std::pair<Team, KeyRange>, OrderByTeamKey> team_shards;
-	std::map<UID, int> storageServerShards;
+	inline bool requireTransitionCheck() { return requireTransition; }
 
-	void erase(Team team, KeyRange const& range);
-	void insert(Team team, KeyRange const& range);
+	// Core data structures
+	// Physical shard instances indexed by physical shard id
+	std::unordered_map<uint64_t, PhysicalShard> physicalShardInstances;
+	// Indicate a key range belongs to which physical shard
+	KeyRangeMap<uint64_t> keyRangePhysicalShardIDMap;
+	// Indicate what physical shards owned by a team
+	std::map<ShardsAffectedByTeamFailure::Team, std::set<uint64_t>> teamPhysicalShardIDs;
+	bool requireTransition;
+	double lastTransitionStartTime;
 };
 
 // DDShardInfo is so named to avoid link-time name collision with ShardInfo within the StorageServer
 struct DDShardInfo {
 	Key key;
+	// all UID are sorted
 	std::vector<UID> primarySrc;
 	std::vector<UID> remoteSrc;
 	std::vector<UID> primaryDest;
@@ -390,37 +464,6 @@ struct ShardTrackedData {
 	Reference<AsyncVar<Optional<ShardMetrics>>> stats;
 };
 
-ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> initData,
-                                           Database cx,
-                                           PromiseStream<RelocateShard> output,
-                                           Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
-                                           PromiseStream<GetMetricsRequest> getShardMetrics,
-                                           FutureStream<GetTopKMetricsRequest> getTopKMetrics,
-                                           PromiseStream<GetMetricsListRequest> getShardMetricsList,
-                                           FutureStream<Promise<int64_t>> getAverageShardBytes,
-                                           Promise<Void> readyToStart,
-                                           Reference<AsyncVar<bool>> zeroHealthyTeams,
-                                           UID distributorId,
-                                           KeyRangeMap<ShardTrackedData>* shards,
-                                           bool* trackerCancelled);
-
-ACTOR Future<Void> dataDistributionQueue(Database cx,
-                                         PromiseStream<RelocateShard> output,
-                                         FutureStream<RelocateShard> input,
-                                         PromiseStream<GetMetricsRequest> getShardMetrics,
-                                         PromiseStream<GetTopKMetricsRequest> getTopKMetrics,
-                                         Reference<AsyncVar<bool>> processingUnhealthy,
-                                         Reference<AsyncVar<bool>> processingWiggle,
-                                         std::vector<TeamCollectionInterface> teamCollection,
-                                         Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
-                                         MoveKeysLock lock,
-                                         PromiseStream<Promise<int64_t>> getAverageShardBytes,
-                                         FutureStream<Promise<int>> getUnhealthyRelocationCount,
-                                         UID distributorId,
-                                         int teamSize,
-                                         int singleRegionTeamSize,
-                                         const DDEnabledState* ddEnabledState);
-
 // Holds the permitted size and IO Bounds for a shard
 struct ShardSizeBounds {
 	StorageMetrics max;
@@ -438,6 +481,65 @@ ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize);
 // Determines the maximum shard size based on the size of the database
 int64_t getMaxShardSize(double dbSizeEstimate);
 
+struct StorageQuotaInfo {
+	std::map<Key, uint64_t> quotaMap;
+};
+
+#ifndef __INTEL_COMPILER
+#pragma endregion
+#endif
+
+// FIXME(xwang): Delete Old DD Actors once the refactoring is done
+/////////////////////////////// Old DD Actors //////////////////////////////////////
+#ifndef __INTEL_COMPILER
+#pragma region Old DD Actors
+#endif
+
+struct TeamCollectionInterface {
+	PromiseStream<GetTeamRequest> getTeam;
+};
+
+ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> initData,
+                                           Reference<IDDTxnProcessor> db,
+                                           PromiseStream<RelocateShard> output,
+                                           Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
+                                           Reference<PhysicalShardCollection> physicalShardCollection,
+                                           PromiseStream<GetMetricsRequest> getShardMetrics,
+                                           FutureStream<GetTopKMetricsRequest> getTopKMetrics,
+                                           PromiseStream<GetMetricsListRequest> getShardMetricsList,
+                                           FutureStream<Promise<int64_t>> getAverageShardBytes,
+                                           Promise<Void> readyToStart,
+                                           Reference<AsyncVar<bool>> zeroHealthyTeams,
+                                           UID distributorId,
+                                           KeyRangeMap<ShardTrackedData>* shards,
+                                           bool* trackerCancelled,
+                                           Optional<Reference<TenantCache>> ddTenantCache);
+
+ACTOR Future<Void> dataDistributionQueue(Reference<IDDTxnProcessor> db,
+                                         PromiseStream<RelocateShard> output,
+                                         FutureStream<RelocateShard> input,
+                                         PromiseStream<GetMetricsRequest> getShardMetrics,
+                                         PromiseStream<GetTopKMetricsRequest> getTopKMetrics,
+                                         Reference<AsyncVar<bool>> processingUnhealthy,
+                                         Reference<AsyncVar<bool>> processingWiggle,
+                                         std::vector<TeamCollectionInterface> teamCollections,
+                                         Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
+                                         Reference<PhysicalShardCollection> physicalShardCollection,
+                                         MoveKeysLock lock,
+                                         PromiseStream<Promise<int64_t>> getAverageShardBytes,
+                                         FutureStream<Promise<int>> getUnhealthyRelocationCount,
+                                         UID distributorId,
+                                         int teamSize,
+                                         int singleRegionTeamSize,
+                                         const DDEnabledState* ddEnabledState);
+#ifndef __INTEL_COMPILER
+#pragma endregion
+#endif
+
+/////////////////////////////// Perpetual Storage Wiggle //////////////////////////////////////
+#ifndef __INTEL_COMPILER
+#pragma region Perpetual Storage Wiggle
+#endif
 class DDTeamCollection;
 
 struct StorageWiggleMetrics {
@@ -532,8 +634,9 @@ struct StorageWiggleMetrics {
 };
 
 struct StorageWiggler : ReferenceCounted<StorageWiggler> {
+	static constexpr double MIN_ON_CHECK_DELAY_SEC = 5.0;
 	enum State : uint8_t { INVALID = 0, RUN = 1, PAUSE = 2 };
-	AsyncVar<bool> nonEmpty;
+
 	DDTeamCollection const* teamCollection;
 	StorageWiggleMetrics metrics;
 	// data structures
@@ -546,7 +649,7 @@ struct StorageWiggler : ReferenceCounted<StorageWiggler> {
 	State wiggleState = State::INVALID;
 	double lastStateChangeTs = 0.0; // timestamp describes when did the state change
 
-	explicit StorageWiggler(DDTeamCollection* collection) : nonEmpty(false), teamCollection(collection){};
+	explicit StorageWiggler(DDTeamCollection* collection) : teamCollection(collection){};
 	// add server to wiggling queue
 	void addServer(const UID& serverId, const StorageMetadataType& metadata);
 	// remove server from wiggling queue
@@ -555,8 +658,14 @@ struct StorageWiggler : ReferenceCounted<StorageWiggler> {
 	void updateMetadata(const UID& serverId, const StorageMetadataType& metadata);
 	bool contains(const UID& serverId) const { return pq_handles.count(serverId) > 0; }
 	bool empty() const { return wiggle_pq.empty(); }
-	Optional<UID> getNextServerId();
 
+	// It's guarantee that When a.metadata >= b.metadata, if !necessary(a) then !necessary(b)
+	bool necessary(const UID& serverId, const StorageMetadataType& metadata) const;
+
+	// try to return the next storage server that is necessary to wiggle
+	Optional<UID> getNextServerId(bool necessaryOnly = true);
+	// next check time to avoid busy loop
+	Future<Void> onCheck() const;
 	State getWiggleState() const { return wiggleState; }
 	void setWiggleState(State s) {
 		if (wiggleState != s) {
@@ -592,5 +701,9 @@ struct StorageWiggler : ReferenceCounted<StorageWiggler> {
 	}
 };
 
+#ifndef __INTEL_COMPILER
+#pragma endregion
+#endif
+
 #include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbserver/include/fdbserver/DataDistributionTeam.h b/fdbserver/include/fdbserver/DataDistributionTeam.h
new file mode 100644
index 0000000000..0da1f5ac27
--- /dev/null
+++ b/fdbserver/include/fdbserver/DataDistributionTeam.h
@@ -0,0 +1,147 @@
+/*
+ * DataDistributionTeam.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/StorageServerInterface.h"
+
+struct IDataDistributionTeam {
+	virtual std::vector<StorageServerInterface> getLastKnownServerInterfaces() const = 0;
+	virtual int size() const = 0;
+	virtual std::vector<UID> const& getServerIDs() const = 0;
+	virtual void addDataInFlightToTeam(int64_t delta) = 0;
+	virtual void addReadInFlightToTeam(int64_t delta) = 0;
+	virtual int64_t getDataInFlightToTeam() const = 0;
+	virtual int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const = 0;
+	virtual int64_t getReadInFlightToTeam() const = 0;
+	virtual double getLoadReadBandwidth(bool includeInFlight = true, double inflightPenalty = 1.0) const = 0;
+	virtual int64_t getMinAvailableSpace(bool includeInFlight = true) const = 0;
+	virtual double getMinAvailableSpaceRatio(bool includeInFlight = true) const = 0;
+	virtual bool hasHealthyAvailableSpace(double minRatio) const = 0;
+	virtual Future<Void> updateStorageMetrics() = 0;
+	virtual void addref() const = 0;
+	virtual void delref() const = 0;
+	virtual bool isHealthy() const = 0;
+	virtual void setHealthy(bool) = 0;
+	virtual int getPriority() const = 0;
+	virtual void setPriority(int) = 0;
+	virtual bool isOptimal() const = 0;
+	virtual bool isWrongConfiguration() const = 0;
+	virtual void setWrongConfiguration(bool) = 0;
+	virtual void addServers(const std::vector<UID>& servers) = 0;
+	virtual std::string getTeamID() const = 0;
+
+	std::string getDesc() const {
+		const auto& servers = getLastKnownServerInterfaces();
+		std::string s = format("TeamID %s; ", getTeamID().c_str());
+		s += format("Size %d; ", servers.size());
+		for (int i = 0; i < servers.size(); i++) {
+			if (i)
+				s += ", ";
+			s += servers[i].address().toString() + " " + servers[i].id().shortString();
+		}
+		return s;
+	}
+};
+
+FDB_DECLARE_BOOLEAN_PARAM(WantNewServers);
+FDB_DECLARE_BOOLEAN_PARAM(WantTrueBest);
+FDB_DECLARE_BOOLEAN_PARAM(PreferLowerDiskUtil);
+FDB_DECLARE_BOOLEAN_PARAM(TeamMustHaveShards);
+FDB_DECLARE_BOOLEAN_PARAM(ForReadBalance);
+FDB_DECLARE_BOOLEAN_PARAM(PreferLowerReadUtil);
+FDB_DECLARE_BOOLEAN_PARAM(FindTeamByServers);
+
+struct GetTeamRequest {
+	bool wantsNewServers; // In additional to servers in completeSources, try to find teams with new server
+	bool wantsTrueBest;
+	bool preferLowerDiskUtil; // if true, lower utilized team has higher score
+	bool teamMustHaveShards;
+	bool forReadBalance;
+	bool preferLowerReadUtil; // only make sense when forReadBalance is true
+	double inflightPenalty;
+	bool findTeamByServers;
+	std::vector<UID> completeSources;
+	std::vector<UID> src;
+	Promise<std::pair<Optional<Reference<IDataDistributionTeam>>, bool>> reply;
+
+	typedef Reference<IDataDistributionTeam> TeamRef;
+
+	GetTeamRequest() {}
+	GetTeamRequest(WantNewServers wantsNewServers,
+	               WantTrueBest wantsTrueBest,
+	               PreferLowerDiskUtil preferLowerDiskUtil,
+	               TeamMustHaveShards teamMustHaveShards,
+	               ForReadBalance forReadBalance = ForReadBalance::False,
+	               PreferLowerReadUtil preferLowerReadUtil = PreferLowerReadUtil::False,
+	               double inflightPenalty = 1.0)
+	  : wantsNewServers(wantsNewServers), wantsTrueBest(wantsTrueBest), preferLowerDiskUtil(preferLowerDiskUtil),
+	    teamMustHaveShards(teamMustHaveShards), forReadBalance(forReadBalance),
+	    preferLowerReadUtil(preferLowerReadUtil), inflightPenalty(inflightPenalty),
+	    findTeamByServers(FindTeamByServers::False) {}
+	GetTeamRequest(std::vector<UID> servers)
+	  : wantsNewServers(WantNewServers::False), wantsTrueBest(WantTrueBest::False),
+	    preferLowerDiskUtil(PreferLowerDiskUtil::False), teamMustHaveShards(TeamMustHaveShards::False),
+	    forReadBalance(ForReadBalance::False), preferLowerReadUtil(PreferLowerReadUtil::False), inflightPenalty(1.0),
+	    findTeamByServers(FindTeamByServers::True), src(std::move(servers)) {}
+
+	// return true if a.score < b.score
+	[[nodiscard]] bool lessCompare(TeamRef a, TeamRef b, int64_t aLoadBytes, int64_t bLoadBytes) const {
+		int res = 0;
+		if (forReadBalance) {
+			res = preferLowerReadUtil ? greaterReadLoad(a, b) : lessReadLoad(a, b);
+		}
+		return res == 0 ? lessCompareByLoad(aLoadBytes, bLoadBytes) : res < 0;
+	}
+
+	std::string getDesc() const {
+		std::stringstream ss;
+
+		ss << "WantsNewServers:" << wantsNewServers << " WantsTrueBest:" << wantsTrueBest
+		   << " PreferLowerDiskUtil:" << preferLowerDiskUtil << " teamMustHaveShards:" << teamMustHaveShards
+		   << "forReadBalance" << forReadBalance << " inflightPenalty:" << inflightPenalty
+		   << " findTeamByServers:" << findTeamByServers << ";";
+		ss << "CompleteSources:";
+		for (const auto& cs : completeSources) {
+			ss << cs.toString() << ",";
+		}
+
+		return std::move(ss).str();
+	}
+
+private:
+	// return true if preferHigherUtil && aLoadBytes <= bLoadBytes (higher load bytes has larger score)
+	// or preferLowerUtil && aLoadBytes > bLoadBytes
+	bool lessCompareByLoad(int64_t aLoadBytes, int64_t bLoadBytes) const {
+		bool lessLoad = aLoadBytes <= bLoadBytes;
+		return preferLowerDiskUtil ? !lessLoad : lessLoad;
+	}
+
+	// return -1 if a.readload > b.readload
+	static int greaterReadLoad(TeamRef a, TeamRef b) {
+		auto r1 = a->getLoadReadBandwidth(true), r2 = b->getLoadReadBandwidth(true);
+		return r1 == r2 ? 0 : (r1 > r2 ? -1 : 1);
+	}
+	// return -1 if a.readload < b.readload
+	static int lessReadLoad(TeamRef a, TeamRef b) {
+		auto r1 = a->getLoadReadBandwidth(false), r2 = b->getLoadReadBandwidth(false);
+		return r1 == r2 ? 0 : (r1 < r2 ? -1 : 1);
+	}
+};
diff --git a/fdbserver/include/fdbserver/DataDistributorInterface.h b/fdbserver/include/fdbserver/DataDistributorInterface.h
index a7bfcbe1e6..93ed3e358f 100644
--- a/fdbserver/include/fdbserver/DataDistributorInterface.h
+++ b/fdbserver/include/fdbserver/DataDistributorInterface.h
@@ -120,7 +120,7 @@ struct DistributorSnapRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, snapPayload, snapUID, reply, arena, debugID);
+		serializer(ar, snapPayload, snapUID, reply, debugID, arena);
 	}
 };
 
diff --git a/fdbserver/include/fdbserver/DeltaTree.h b/fdbserver/include/fdbserver/DeltaTree.h
index a7c73f0d78..6a87ee529d 100644
--- a/fdbserver/include/fdbserver/DeltaTree.h
+++ b/fdbserver/include/fdbserver/DeltaTree.h
@@ -34,40 +34,6 @@
 #define deltatree_printf(...)
 #endif
 
-typedef uint64_t Word;
-// Get the number of prefix bytes that are the same between a and b, up to their common length of cl
-static inline int commonPrefixLength(uint8_t const* ap, uint8_t const* bp, int cl) {
-	int i = 0;
-	const int wordEnd = cl - sizeof(Word) + 1;
-
-	for (; i < wordEnd; i += sizeof(Word)) {
-		Word a = *(Word*)ap;
-		Word b = *(Word*)bp;
-		if (a != b) {
-			return i + ctzll(a ^ b) / 8;
-		}
-		ap += sizeof(Word);
-		bp += sizeof(Word);
-	}
-
-	for (; i < cl; i++) {
-		if (*ap != *bp) {
-			return i;
-		}
-		++ap;
-		++bp;
-	}
-	return cl;
-}
-
-static inline int commonPrefixLength(const StringRef& a, const StringRef& b) {
-	return commonPrefixLength(a.begin(), b.begin(), std::min(a.size(), b.size()));
-}
-
-static inline int commonPrefixLength(const StringRef& a, const StringRef& b, int skipLen) {
-	return commonPrefixLength(a.begin() + skipLen, b.begin() + skipLen, std::min(a.size(), b.size()) - skipLen);
-}
-
 // This appears to be the fastest version
 static int lessOrEqualPowerOfTwo(int n) {
 	int p;
diff --git a/fdbserver/include/fdbserver/EncryptedMutationMessage.h b/fdbserver/include/fdbserver/EncryptedMutationMessage.h
deleted file mode 100644
index a07643909d..0000000000
--- a/fdbserver/include/fdbserver/EncryptedMutationMessage.h
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * EncryptedMutationMessage.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBSERVER_ENCRYPTEDMUTATIONMESSAGE_H
-#define FDBSERVER_ENCRYPTEDMUTATIONMESSAGE_H
-
-#pragma once
-
-#include "fdbclient/CommitTransaction.h"
-#include "fdbserver/Knobs.h"
-#include "flow/BlobCipher.h"
-
-struct EncryptedMutationMessage {
-
-	BlobCipherEncryptHeader header;
-	StringRef encrypted;
-
-	EncryptedMutationMessage() {}
-
-	std::string toString() const {
-		return format("code: %d, encryption info: %s",
-		              MutationRef::Reserved_For_EncryptedMutationMessage,
-		              header.toString().c_str());
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		uint8_t poly = MutationRef::Reserved_For_EncryptedMutationMessage;
-		serializer(ar, poly, header, encrypted);
-	}
-
-	static bool startsEncryptedMutationMessage(uint8_t byte) {
-		return byte == MutationRef::Reserved_For_EncryptedMutationMessage;
-	}
-	template <class Ar>
-	static bool isNextIn(Ar& ar) {
-		return startsEncryptedMutationMessage(*(const uint8_t*)ar.peekBytes(1));
-	}
-
-	// Encrypt given mutation and return an EncryptedMutationMessage.
-	static EncryptedMutationMessage encrypt(
-	    Arena& arena,
-	    const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys,
-	    const EncryptCipherDomainId& domainId,
-	    const MutationRef& mutation) {
-		ASSERT_NE(domainId, ENCRYPT_INVALID_DOMAIN_ID);
-		auto textCipherItr = cipherKeys.find(domainId);
-		auto headerCipherItr = cipherKeys.find(ENCRYPT_HEADER_DOMAIN_ID);
-		ASSERT(textCipherItr != cipherKeys.end() && textCipherItr->second.isValid());
-		ASSERT(headerCipherItr != cipherKeys.end() && headerCipherItr->second.isValid());
-		uint8_t iv[AES_256_IV_LENGTH];
-		generateRandomData(iv, AES_256_IV_LENGTH);
-		BinaryWriter bw(AssumeVersion(g_network->protocolVersion()));
-		bw << mutation;
-		EncryptedMutationMessage encrypted_mutation;
-		EncryptBlobCipherAes265Ctr cipher(textCipherItr->second,
-		                                  headerCipherItr->second,
-		                                  iv,
-		                                  AES_256_IV_LENGTH,
-		                                  ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
-		encrypted_mutation.encrypted =
-		    cipher
-		        .encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), &encrypted_mutation.header, arena)
-		        ->toStringRef();
-		return encrypted_mutation;
-	}
-
-	// Encrypt system key space mutation and return an EncryptedMutationMessage.
-	static EncryptedMutationMessage encryptMetadata(
-	    Arena& arena,
-	    const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys,
-	    const MutationRef& mutation) {
-		return encrypt(arena, cipherKeys, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, mutation);
-	}
-
-	// Read an EncryptedMutationMessage from given reader, decrypt and return the encrypted mutation.
-	// Also return decrypt buffer through buf, if it is specified.
-	template <class Ar>
-	static MutationRef decrypt(Ar& ar,
-	                           Arena& arena,
-	                           const std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>& cipherKeys,
-	                           StringRef* buf = nullptr) {
-		ASSERT(SERVER_KNOBS->ENABLE_ENCRYPTION);
-		EncryptedMutationMessage msg;
-		ar >> msg;
-		auto textCipherItr = cipherKeys.find(msg.header.cipherTextDetails);
-		auto headerCipherItr = cipherKeys.find(msg.header.cipherHeaderDetails);
-		ASSERT(textCipherItr != cipherKeys.end() && textCipherItr->second.isValid());
-		ASSERT(headerCipherItr != cipherKeys.end() && headerCipherItr->second.isValid());
-		DecryptBlobCipherAes256Ctr cipher(textCipherItr->second, headerCipherItr->second, msg.header.iv);
-		StringRef plaintext =
-		    cipher.decrypt(msg.encrypted.begin(), msg.encrypted.size(), msg.header, arena)->toStringRef();
-		if (buf != nullptr) {
-			*buf = plaintext;
-		}
-		ArenaReader reader(arena, plaintext, AssumeVersion(g_network->protocolVersion()));
-		MutationRef mutation;
-		reader >> mutation;
-		return mutation;
-	}
-};
-#endif
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/EncryptionOpsUtils.h b/fdbserver/include/fdbserver/EncryptionOpsUtils.h
new file mode 100644
index 0000000000..157633fe10
--- /dev/null
+++ b/fdbserver/include/fdbserver/EncryptionOpsUtils.h
@@ -0,0 +1,48 @@
+/*
+ * EncryptionOpUtils.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_ENCRYPTION_OPS_UTIL_H
+#define FDBSERVER_ENCRYPTION_OPS_UTIL_H
+#pragma once
+
+#include "fdbserver/Knobs.h"
+#include "fdbclient/CommitProxyInterface.h"
+
+typedef enum { TLOG_ENCRYPTION = 0, STORAGE_SERVER_ENCRYPTION = 1, BLOB_GRANULE_ENCRYPTION = 2 } EncryptOperationType;
+
+inline bool isEncryptionOpSupported(EncryptOperationType operation_type, const ClientDBInfo& dbInfo) {
+	if (!dbInfo.isEncryptionEnabled) {
+		return false;
+	}
+
+	if (operation_type == TLOG_ENCRYPTION) {
+		return SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION;
+	} else if (operation_type == STORAGE_SERVER_ENCRYPTION) {
+		return SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION;
+	} else if (operation_type == BLOB_GRANULE_ENCRYPTION) {
+		bool supported = SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION && SERVER_KNOBS->BG_METADATA_SOURCE == "tenant";
+		ASSERT((supported && SERVER_KNOBS->ENABLE_ENCRYPTION) || !supported);
+		return supported;
+	} else {
+		return false;
+	}
+}
+
+#endif // FDBSERVER_ENCRYPTION_OPS_UTIL_H
diff --git a/fdbserver/include/fdbserver/GetEncryptCipherKeys.actor.h b/fdbserver/include/fdbserver/GetEncryptCipherKeys.actor.h
deleted file mode 100644
index ca1f880afe..0000000000
--- a/fdbserver/include/fdbserver/GetEncryptCipherKeys.actor.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * GetEncryptCipherKeys.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#pragma once
-
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_GETCIPHERKEYS_ACTOR_G_H)
-#define FDBSERVER_GETCIPHERKEYS_ACTOR_G_H
-#include "fdbserver/GetEncryptCipherKeys.actor.g.h"
-#elif !defined(FDBSERVER_GETCIPHERKEYS_ACTOR_H)
-#define FDBSERVER_GETCIPHERKEYS_ACTOR_H
-
-#include "fdbserver/ServerDBInfo.h"
-#include "flow/BlobCipher.h"
-
-#include <unordered_map>
-#include <unordered_set>
-
-#include "flow/actorcompiler.h"
-
-// Get latest cipher keys for given encryption domains. It tries to get the cipher keys from local cache.
-// In case of cache miss, it fetches the cipher keys from EncryptKeyProxy and put the result in the local cache
-// before return.
-ACTOR Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getLatestEncryptCipherKeys(
-    Reference<AsyncVar<ServerDBInfo> const> db,
-    std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains);
-
-// Get cipher keys specified by the list of cipher details. It tries to get the cipher keys from local cache.
-// In case of cache miss, it fetches the cipher keys from EncryptKeyProxy and put the result in the local cache
-// before return.
-ACTOR Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncryptCipherKeys(
-    Reference<AsyncVar<ServerDBInfo> const> db,
-    std::unordered_set<BlobCipherDetails> cipherDetails);
-
-struct TextAndHeaderCipherKeys {
-	Reference<BlobCipherKey> cipherTextKey;
-	Reference<BlobCipherKey> cipherHeaderKey;
-};
-
-// Helper method to get latest cipher text key and cipher header key for system domain,
-// used for encrypting system data.
-ACTOR Future<TextAndHeaderCipherKeys> getLatestSystemEncryptCipherKeys(Reference<AsyncVar<ServerDBInfo> const> db);
-
-// Helper method to get both text cipher key and header cipher key for the given encryption header,
-// used for decrypting given encrypted data with encryption header.
-ACTOR Future<TextAndHeaderCipherKeys> getEncryptCipherKeys(Reference<AsyncVar<ServerDBInfo> const> db,
-                                                           BlobCipherEncryptHeader header);
-
-#include "flow/unactorcompiler.h"
-#endif
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/IClosable.h b/fdbserver/include/fdbserver/IClosable.h
new file mode 100644
index 0000000000..131dc84df0
--- /dev/null
+++ b/fdbserver/include/fdbserver/IClosable.h
@@ -0,0 +1,40 @@
+/*
+ * IClosable.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FDBSERVER_ICLOSABLE_H
+#define FDBSERVER_ICLOSABLE_H
+#pragma once
+
+class IClosable {
+public:
+	// IClosable is a base interface for any disk-backed data structure that needs to support asynchronous errors,
+	// shutdown and deletion
+
+	virtual Future<Void> getError()
+	    const = 0; // asynchronously throws an error if there is an internal error. Never set
+	               // inside (on the stack of) a call to another API function on this object.
+	virtual Future<Void> onClosed()
+	    const = 0; // the future is set to Void when this is totally shut down after dispose() or
+	               // close().  But this function cannot be called after dispose or close!
+	virtual void dispose() = 0; // permanently delete the data AND invalidate this interface
+	virtual void close() = 0; // invalidate this interface, but do not delete the data.  Outstanding operations may or
+	                          // may not take effect in the background.
+};
+
+#endif
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/IConfigConsumer.h b/fdbserver/include/fdbserver/IConfigConsumer.h
index f485d9f698..83cf0818b4 100644
--- a/fdbserver/include/fdbserver/IConfigConsumer.h
+++ b/fdbserver/include/fdbserver/IConfigConsumer.h
@@ -35,7 +35,9 @@
 class IConfigConsumer {
 public:
 	virtual ~IConfigConsumer() = default;
+	virtual Future<Void> readSnapshot(ConfigBroadcaster& broadcaster) = 0;
 	virtual Future<Void> consume(ConfigBroadcaster& broadcaster) = 0;
+	virtual void allowSpecialCaseRollforward() = 0;
 	virtual UID getID() const = 0;
 
 	static std::unique_ptr<IConfigConsumer> createTestSimple(ConfigFollowerInterface const& cfi,
@@ -46,5 +48,6 @@ public:
 	                                                     Optional<double> compactionInterval);
 	static std::unique_ptr<IConfigConsumer> createPaxos(ServerCoordinators const& coordinators,
 	                                                    double pollingInterval,
-	                                                    Optional<double> compactionInterval);
+	                                                    Optional<double> compactionInterval,
+	                                                    bool readPreviousCoordinators = false);
 };
diff --git a/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h b/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h
new file mode 100644
index 0000000000..e9fd08afe5
--- /dev/null
+++ b/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h
@@ -0,0 +1,299 @@
+/*
+ * IEncryptionKeyProvider.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/BlobCipher.h"
+#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_IENCRYPTIONKEYPROVIDER_ACTOR_G_H)
+#define FDBSERVER_IENCRYPTIONKEYPROVIDER_ACTOR_G_H
+#include "fdbserver/IEncryptionKeyProvider.actor.g.h"
+#elif !defined(FDBSERVER_IENCRYPTIONKEYPROVIDER_ACTOR_H)
+#define FDBSERVER_IENCRYPTIONKEYPROVIDER_ACTOR_H
+
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/Tenant.h"
+
+#include "fdbserver/EncryptionOpsUtils.h"
+#include "fdbserver/ServerDBInfo.h"
+
+#include "flow/Arena.h"
+#include "flow/EncryptUtils.h"
+#define XXH_INLINE_ALL
+#include "flow/xxhash.h"
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+typedef uint64_t XOREncryptionKeyID;
+
+// EncryptionKeyRef is somewhat multi-variant, it will contain members representing the union
+// of all fields relevant to any implemented encryption scheme.  They are generally of
+// the form
+//   Page Fields - fields which come from or are stored in the Page
+//   Secret Fields - fields which are only known by the Key Provider
+// but it is up to each encoding and provider which fields are which and which ones are used
+//
+// TODO(yiwu): Rename and/or refactor this struct. It doesn't sound like an encryption key should
+// contain page fields like encryption header.
+struct EncryptionKeyRef {
+
+	EncryptionKeyRef(){};
+	EncryptionKeyRef(Arena& arena, const EncryptionKeyRef& toCopy)
+	  : cipherKeys(toCopy.cipherKeys), secret(arena, toCopy.secret), id(toCopy.id) {}
+	int expectedSize() const { return secret.size(); }
+
+	// Fields for AESEncryptionV1
+	TextAndHeaderCipherKeys cipherKeys;
+	Optional<BlobCipherEncryptHeader> cipherHeader;
+	// Fields for XOREncryption_TestOnly
+	StringRef secret;
+	Optional<XOREncryptionKeyID> id;
+};
+typedef Standalone<EncryptionKeyRef> EncryptionKey;
+
+// Interface used by pager to get encryption keys reading pages from disk
+// and by the BTree to get encryption keys to use for new pages
+class IEncryptionKeyProvider : public ReferenceCounted<IEncryptionKeyProvider> {
+public:
+	virtual ~IEncryptionKeyProvider() {}
+
+	// Get an EncryptionKey with Secret Fields populated based on the given Page Fields.
+	// It is up to the implementation which fields those are.
+	// The output Page Fields must match the input Page Fields.
+	virtual Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) = 0;
+
+	// Get encryption key that should be used for a given user Key-Value range
+	virtual Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) = 0;
+
+	// Setting tenant prefix to tenant name map.
+	virtual void setTenantPrefixIndex(Reference<TenantPrefixIndex> tenantPrefixIndex) {}
+
+	virtual bool shouldEnableEncryption() const = 0;
+};
+
+// The null key provider is useful to simplify page decoding.
+// It throws an error for any key info requested.
+class NullKeyProvider : public IEncryptionKeyProvider {
+public:
+	virtual ~NullKeyProvider() {}
+	bool shouldEnableEncryption() const override { return true; }
+	Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override { throw encryption_key_not_found(); }
+	Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
+		throw encryption_key_not_found();
+	}
+};
+
+// Key provider for dummy XOR encryption scheme
+class XOREncryptionKeyProvider_TestOnly : public IEncryptionKeyProvider {
+public:
+	XOREncryptionKeyProvider_TestOnly(std::string filename) {
+		ASSERT(g_network->isSimulated());
+
+		// Choose a deterministic random filename (without path) byte for secret generation
+		// Remove any leading directory names
+		size_t lastSlash = filename.find_last_of("\\/");
+		if (lastSlash != filename.npos) {
+			filename.erase(0, lastSlash);
+		}
+		xorWith = filename.empty() ? 0x5e
+		                           : (uint8_t)filename[XXH3_64bits(filename.data(), filename.size()) % filename.size()];
+	}
+
+	virtual ~XOREncryptionKeyProvider_TestOnly() {}
+
+	bool shouldEnableEncryption() const override { return true; }
+
+	Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override {
+		if (!key.id.present()) {
+			throw encryption_key_not_found();
+		}
+		EncryptionKey s = key;
+		uint8_t secret = ~(uint8_t)key.id.get() ^ xorWith;
+		s.secret = StringRef(s.arena(), &secret, 1);
+		return s;
+	}
+
+	Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
+		EncryptionKeyRef k;
+		k.id = end.empty() ? 0 : *(end.end() - 1);
+		return getSecrets(k);
+	}
+
+	uint8_t xorWith;
+};
+
+// Key provider to provider cipher keys randomly from a pre-generated pool. Use for testing.
+class RandomEncryptionKeyProvider : public IEncryptionKeyProvider {
+public:
+	RandomEncryptionKeyProvider() {
+		for (unsigned i = 0; i < NUM_CIPHER; i++) {
+			BlobCipherDetails cipherDetails;
+			cipherDetails.encryptDomainId = i;
+			cipherDetails.baseCipherId = deterministicRandom()->randomUInt64();
+			cipherDetails.salt = deterministicRandom()->randomUInt64();
+			cipherKeys[i] = generateCipherKey(cipherDetails);
+		}
+	}
+	virtual ~RandomEncryptionKeyProvider() = default;
+
+	bool shouldEnableEncryption() const override { return true; }
+
+	Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override {
+		ASSERT(key.cipherHeader.present());
+		EncryptionKey s = key;
+		s.cipherKeys.cipherTextKey = cipherKeys[key.cipherHeader.get().cipherTextDetails.encryptDomainId];
+		s.cipherKeys.cipherHeaderKey = cipherKeys[key.cipherHeader.get().cipherHeaderDetails.encryptDomainId];
+		return s;
+	}
+
+	Future<EncryptionKey> getByRange(const KeyRef& /*begin*/, const KeyRef& /*end*/) override {
+		EncryptionKey s;
+		s.cipherKeys.cipherTextKey = getRandomCipherKey();
+		s.cipherKeys.cipherHeaderKey = getRandomCipherKey();
+		return s;
+	}
+
+private:
+	Reference<BlobCipherKey> generateCipherKey(const BlobCipherDetails& cipherDetails) {
+		static unsigned char SHA_KEY[] = "3ab9570b44b8315fdb261da6b1b6c13b";
+		uint8_t digest[AUTH_TOKEN_SIZE];
+		computeAuthToken(reinterpret_cast<const unsigned char*>(&cipherDetails.baseCipherId),
+		                 sizeof(EncryptCipherBaseKeyId),
+		                 SHA_KEY,
+		                 AES_256_KEY_LENGTH,
+		                 &digest[0],
+		                 AUTH_TOKEN_SIZE);
+		return makeReference<BlobCipherKey>(cipherDetails.encryptDomainId,
+		                                    cipherDetails.baseCipherId,
+		                                    &digest[0],
+		                                    AES_256_KEY_LENGTH,
+		                                    cipherDetails.salt,
+		                                    std::numeric_limits<int64_t>::max() /* refreshAt */,
+		                                    std::numeric_limits<int64_t>::max() /* expireAt */);
+	}
+
+	Reference<BlobCipherKey> getRandomCipherKey() {
+		return cipherKeys[deterministicRandom()->randomInt(0, NUM_CIPHER)];
+	}
+
+	static constexpr int NUM_CIPHER = 1000;
+	Reference<BlobCipherKey> cipherKeys[NUM_CIPHER];
+};
+
+// Key provider which extract tenant id from range key prefixes, and fetch tenant specific encryption keys from
+// EncryptKeyProxy.
+class TenantAwareEncryptionKeyProvider : public IEncryptionKeyProvider {
+public:
+	TenantAwareEncryptionKeyProvider(Reference<AsyncVar<ServerDBInfo> const> db) : db(db) {}
+
+	virtual ~TenantAwareEncryptionKeyProvider() = default;
+
+	bool shouldEnableEncryption() const override {
+		return isEncryptionOpSupported(EncryptOperationType::STORAGE_SERVER_ENCRYPTION, db->get().client);
+	}
+
+	ACTOR static Future<EncryptionKey> getSecrets(TenantAwareEncryptionKeyProvider* self, EncryptionKeyRef key) {
+		if (!key.cipherHeader.present()) {
+			TraceEvent("TenantAwareEncryptionKeyProvider_CipherHeaderMissing");
+			throw encrypt_ops_error();
+		}
+		TextAndHeaderCipherKeys cipherKeys =
+		    wait(getEncryptCipherKeys(self->db, key.cipherHeader.get(), BlobCipherMetrics::KV_REDWOOD));
+		EncryptionKey s = key;
+		s.cipherKeys = cipherKeys;
+		return s;
+	}
+
+	Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override { return getSecrets(this, key); }
+
+	ACTOR static Future<EncryptionKey> getByRange(TenantAwareEncryptionKeyProvider* self, KeyRef begin, KeyRef end) {
+		EncryptCipherDomainNameRef domainName;
+		EncryptCipherDomainId domainId = self->getEncryptionDomainId(begin, end, &domainName);
+		TextAndHeaderCipherKeys cipherKeys =
+		    wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, domainName, BlobCipherMetrics::KV_REDWOOD));
+		EncryptionKey s;
+		s.cipherKeys = cipherKeys;
+		return s;
+	}
+
+	Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
+		return getByRange(this, begin, end);
+	}
+
+	void setTenantPrefixIndex(Reference<TenantPrefixIndex> tenantPrefixIndex) override {
+		ASSERT(tenantPrefixIndex.isValid());
+		this->tenantPrefixIndex = tenantPrefixIndex;
+	}
+
+private:
+	EncryptCipherDomainId getEncryptionDomainId(const KeyRef& begin,
+	                                            const KeyRef& end,
+	                                            EncryptCipherDomainNameRef* domainName) {
+		int64_t domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
+		int64_t beginTenantId = getTenantId(begin, true /*inclusive*/);
+		int64_t endTenantId = getTenantId(end, false /*inclusive*/);
+		if (beginTenantId == endTenantId && beginTenantId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
+			ASSERT(tenantPrefixIndex.isValid());
+			Key tenantPrefix = TenantMapEntry::idToPrefix(beginTenantId);
+			auto view = tenantPrefixIndex->atLatest();
+			auto itr = view.find(tenantPrefix);
+			if (itr != view.end()) {
+				*domainName = *itr;
+				domainId = beginTenantId;
+			} else {
+				// No tenant with the same tenant id. We could be in optional or disabled tenant mode.
+			}
+		}
+		if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
+			*domainName = FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
+		}
+		return domainId;
+	}
+
+	int64_t getTenantId(const KeyRef& key, bool inclusive) {
+		// A valid tenant id is always a valid encrypt domain id.
+		static_assert(INVALID_ENCRYPT_DOMAIN_ID == -1);
+
+		if (key.size() && key >= systemKeys.begin) {
+			return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
+		}
+
+		if (key.size() < TENANT_PREFIX_SIZE) {
+			// Encryption domain information not available, leverage 'default encryption domain'
+			return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
+		}
+
+		StringRef prefix = key.substr(0, TENANT_PREFIX_SIZE);
+		int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False);
+		if (tenantId == TenantInfo::INVALID_TENANT) {
+			// Encryption domain information not available, leverage 'default encryption domain'
+			return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
+		}
+
+		if (!inclusive && key.size() == TENANT_PREFIX_SIZE) {
+			tenantId = tenantId - 1;
+		}
+		ASSERT(tenantId >= 0);
+		return tenantId;
+	}
+
+	Reference<AsyncVar<ServerDBInfo> const> db;
+	Reference<TenantPrefixIndex> tenantPrefixIndex;
+};
+
+#include "flow/unactorcompiler.h"
+#endif
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/IKeyValueStore.h b/fdbserver/include/fdbserver/IKeyValueStore.h
index 942ff37539..b9679d9f92 100644
--- a/fdbserver/include/fdbserver/IKeyValueStore.h
+++ b/fdbserver/include/fdbserver/IKeyValueStore.h
@@ -23,9 +23,12 @@
 #pragma once
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/ServerDBInfo.h"
 #include "fdbclient/StorageCheckpoint.h"
+#include "fdbclient/Tenant.h"
+#include "fdbserver/Knobs.h"
+#include "fdbserver/IClosable.h"
+#include "fdbserver/IPageEncryptionKeyProvider.actor.h"
+#include "fdbserver/ServerDBInfo.h"
 
 struct CheckpointRequest {
 	const Version version; // The FDB version at which the checkpoint is created.
@@ -42,55 +45,31 @@ struct CheckpointRequest {
 	  : version(version), range(range), format(format), checkpointID(id), checkpointDir(checkpointDir) {}
 };
 
-class IClosable {
-public:
-	// IClosable is a base interface for any disk-backed data structure that needs to support asynchronous errors,
-	// shutdown and deletion
-
-	virtual Future<Void> getError()
-	    const = 0; // asynchronously throws an error if there is an internal error. Never set
-	               // inside (on the stack of) a call to another API function on this object.
-	virtual Future<Void> onClosed()
-	    const = 0; // the future is set to Void when this is totally shut down after dispose() or
-	               // close().  But this function cannot be called after dispose or close!
-	virtual void dispose() = 0; // permanently delete the data AND invalidate this interface
-	virtual void close() = 0; // invalidate this interface, but do not delete the data.  Outstanding operations may or
-	                          // may not take effect in the background.
-};
-
 class IKeyValueStore : public IClosable {
 public:
 	virtual KeyValueStoreType getType() const = 0;
+	// Returns true if the KV store supports shards, i.e., implements addRange(), removeRange(), and
+	// persistRangeMapping().
+	virtual bool shardAware() const { return false; }
 	virtual void set(KeyValueRef keyValue, const Arena* arena = nullptr) = 0;
 	virtual void clear(KeyRangeRef range, const Arena* arena = nullptr) = 0;
 	virtual Future<Void> canCommit() { return Void(); }
 	virtual Future<Void> commit(
 	    bool sequential = false) = 0; // returns when prior sets and clears are (atomically) durable
 
-	enum class ReadType {
-		EAGER,
-		FETCH,
-		LOW,
-		NORMAL,
-		HIGH,
-	};
-
-	virtual Future<Optional<Value>> readValue(KeyRef key,
-	                                          ReadType type = ReadType::NORMAL,
-	                                          Optional<UID> debugID = Optional<UID>()) = 0;
+	virtual Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options = Optional<ReadOptions>()) = 0;
 
 	// Like readValue(), but returns only the first maxLength bytes of the value if it is longer
 	virtual Future<Optional<Value>> readValuePrefix(KeyRef key,
 	                                                int maxLength,
-	                                                ReadType type = ReadType::NORMAL,
-	                                                Optional<UID> debugID = Optional<UID>()) = 0;
+	                                                Optional<ReadOptions> options = Optional<ReadOptions>()) = 0;
 
 	// If rowLimit>=0, reads first rows sorted ascending, otherwise reads last rows sorted descending
 	// The total size of the returned value (less the last entry) will be less than byteLimit
 	virtual Future<RangeResult> readRange(KeyRangeRef keys,
 	                                      int rowLimit = 1 << 30,
 	                                      int byteLimit = 1 << 30,
-	                                      ReadType type = ReadType::NORMAL) = 0;
+	                                      Optional<ReadOptions> options = Optional<ReadOptions>()) = 0;
 
 	// Shard management APIs.
 	// Adds key range to a physical shard.
@@ -155,7 +134,9 @@ extern IKeyValueStore* keyValueStoreSQLite(std::string const& filename,
                                            KeyValueStoreType storeType,
                                            bool checkChecksums = false,
                                            bool checkIntegrity = false);
-extern IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename, UID logID);
+extern IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename,
+                                              UID logID,
+                                              Reference<IPageEncryptionKeyProvider> encryptionKeyProvider = {});
 extern IKeyValueStore* keyValueStoreRocksDB(std::string const& path,
                                             UID logID,
                                             KeyValueStoreType storeType,
@@ -193,7 +174,8 @@ inline IKeyValueStore* openKVStore(KeyValueStoreType storeType,
                                    int64_t memoryLimit,
                                    bool checkChecksums = false,
                                    bool checkIntegrity = false,
-                                   bool openRemotely = false) {
+                                   bool openRemotely = false,
+                                   Reference<IPageEncryptionKeyProvider> encryptionKeyProvider = {}) {
 	if (openRemotely) {
 		return openRemoteKVStore(storeType, filename, logID, memoryLimit, checkChecksums, checkIntegrity);
 	}
@@ -205,7 +187,7 @@ inline IKeyValueStore* openKVStore(KeyValueStoreType storeType,
 	case KeyValueStoreType::MEMORY:
 		return keyValueStoreMemory(filename, logID, memoryLimit);
 	case KeyValueStoreType::SSD_REDWOOD_V1:
-		return keyValueStoreRedwoodV1(filename, logID);
+		return keyValueStoreRedwoodV1(filename, logID, encryptionKeyProvider);
 	case KeyValueStoreType::SSD_ROCKSDB_V1:
 		return keyValueStoreRocksDB(filename, logID, storeType);
 	case KeyValueStoreType::SSD_SHARDED_ROCKSDB:
diff --git a/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h b/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h
new file mode 100644
index 0000000000..605f62572a
--- /dev/null
+++ b/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h
@@ -0,0 +1,403 @@
+/*
+ * IPageEncryptionKeyProvider.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_IPAGEENCRYPTIONKEYPROVIDER_ACTOR_G_H)
+#define FDBSERVER_IPAGEENCRYPTIONKEYPROVIDER_ACTOR_G_H
+#include "fdbserver/IPageEncryptionKeyProvider.actor.g.h"
+#elif !defined(FDBSERVER_IPAGEENCRYPTIONKEYPROVIDER_ACTOR_H)
+#define FDBSERVER_IPAGEENCRYPTIONKEYPROVIDER_ACTOR_H
+
+#include "fdbclient/BlobCipher.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/Tenant.h"
+
+#include "fdbserver/EncryptionOpsUtils.h"
+#include "fdbserver/IPager.h"
+#include "fdbserver/ServerDBInfo.h"
+
+#include "flow/Arena.h"
+#include "flow/EncryptUtils.h"
+#define XXH_INLINE_ALL
+#include "flow/xxhash.h"
+
+#include <functional>
+#include <tuple>
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+// Interface used by pager to get encryption keys reading pages from disk
+// and by the BTree to get encryption keys to use for new pages.
+//
+// Cipher key rotation:
+// The key provider can rotate encryption keys, potentially per encryption domain (see below). Each of the new pages
+// are encrypted using the latest encryption keys.
+//
+// Encryption domains:
+// The key provider can specify how the page split the full key range into encryption domains by key prefixes.
+// Encryption domains are expected to have their own set of encryption keys, which is managed by the key provider.
+// The pager will ensure that data from different encryption domain won't fall in the same page, to make
+// sure it can use one single encryption key to encrypt the whole page.
+// The key provider needs to provide a default encryption domain, which is used to encrypt pages contain only
+// full or partial encryption domain prefixes.
+class IPageEncryptionKeyProvider : public ReferenceCounted<IPageEncryptionKeyProvider> {
+public:
+	using EncryptionKey = ArenaPage::EncryptionKey;
+
+	virtual ~IPageEncryptionKeyProvider() = default;
+
+	// Expected encoding type being used with the encryption key provider.
+	virtual EncodingType expectedEncodingType() const = 0;
+
+	// Checks whether encryption should be enabled. If not, the encryption key provider will not be used by
+	// the pager, and instead the default non-encrypted encoding type (XXHash64) is used.
+	virtual bool enableEncryption() const = 0;
+
+	// Whether encryption domain is enabled.
+	virtual bool enableEncryptionDomain() const { return false; }
+
+	// Get an encryption key from given encoding header.
+	virtual Future<EncryptionKey> getEncryptionKey(const void* encodingHeader) { throw not_implemented(); }
+
+	// Get latest encryption key. If encryption domain is enabled, get encryption key for the default domain.
+	virtual Future<EncryptionKey> getLatestDefaultEncryptionKey() {
+		return getLatestEncryptionKey(getDefaultEncryptionDomainId());
+	}
+
+	// Get latest encryption key for data in given encryption domain.
+	virtual Future<EncryptionKey> getLatestEncryptionKey(int64_t domainId) { throw not_implemented(); }
+
+	// Return the default encryption domain.
+	virtual int64_t getDefaultEncryptionDomainId() const { throw not_implemented(); }
+
+	// Get encryption domain from a key. Return the domain id, and the size of the encryption domain prefix.
+	// It is assumed that all keys with the same encryption domain prefix as the given key falls in the same encryption
+	// domain. If possibleDomainId is given, it is a valid domain id previously returned by the key provider,
+	// potentially for a different key. The possibleDomainId parm is used by TenantAwareEncryptionKeyProvider to speed
+	// up encryption domain lookup.
+	virtual std::tuple<int64_t, size_t> getEncryptionDomain(const KeyRef& key,
+	                                                        Optional<int64_t> possibleDomainId = Optional<int64_t>()) {
+		throw not_implemented();
+	}
+
+	// Get encryption domain of a page given encoding header.
+	virtual int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) { throw not_implemented(); }
+
+	// Setting tenant prefix to tenant name map. Used by TenantAwareEncryptionKeyProvider.
+	virtual void setTenantPrefixIndex(Reference<TenantPrefixIndex> tenantPrefixIndex) {}
+
+	// Helper methods.
+
+	// Check if a key fits in an encryption domain.
+	bool keyFitsInDomain(int64_t domainId, const KeyRef& key, bool canUseDefaultDomain) {
+		ASSERT(enableEncryptionDomain());
+		int64_t keyDomainId;
+		size_t prefixLength;
+		std::tie(keyDomainId, prefixLength) = getEncryptionDomain(key);
+		return keyDomainId == domainId ||
+		       (canUseDefaultDomain && (domainId == getDefaultEncryptionDomainId() && key.size() == prefixLength));
+	}
+};
+
+// The null key provider is useful to simplify page decoding.
+// It throws an error for any key info requested.
+class NullKeyProvider : public IPageEncryptionKeyProvider {
+public:
+	virtual ~NullKeyProvider() {}
+	EncodingType expectedEncodingType() const override { return EncodingType::XXHash64; }
+	bool enableEncryption() const override { return false; }
+};
+
+// Key provider for dummy XOR encryption scheme
+class XOREncryptionKeyProvider_TestOnly : public IPageEncryptionKeyProvider {
+public:
+	using EncodingHeader = ArenaPage::XOREncryptionEncoder::Header;
+
+	XOREncryptionKeyProvider_TestOnly(std::string filename) {
+		ASSERT(g_network->isSimulated());
+
+		// Choose a deterministic random filename (without path) byte for secret generation
+		// Remove any leading directory names
+		size_t lastSlash = filename.find_last_of("\\/");
+		if (lastSlash != filename.npos) {
+			filename.erase(0, lastSlash);
+		}
+		xorWith = filename.empty() ? 0x5e
+		                           : (uint8_t)filename[XXH3_64bits(filename.data(), filename.size()) % filename.size()];
+	}
+
+	virtual ~XOREncryptionKeyProvider_TestOnly() {}
+
+	EncodingType expectedEncodingType() const override { return EncodingType::XOREncryption_TestOnly; }
+
+	bool enableEncryption() const override { return true; }
+
+	Future<EncryptionKey> getEncryptionKey(const void* encodingHeader) override {
+
+		const EncodingHeader* h = reinterpret_cast<const EncodingHeader*>(encodingHeader);
+		EncryptionKey s;
+		s.xorKey = h->xorKey;
+		return s;
+	}
+
+	Future<EncryptionKey> getLatestDefaultEncryptionKey() override {
+		EncryptionKey s;
+		s.xorKey = xorWith;
+		return s;
+	}
+
+	uint8_t xorWith;
+};
+
+// Key provider to provider cipher keys randomly from a pre-generated pool. It does not maintain encryption domains.
+// Use for testing.
+class RandomEncryptionKeyProvider : public IPageEncryptionKeyProvider {
+public:
+	enum EncryptionDomainMode : unsigned int {
+		DISABLED = 0, // disable encryption domain
+		RANDOM, // for each key prefix, deterministic randomly decide if there's an encryption domain for it.
+		ALL, // all key prefixes has an encryption domain assigned to it.
+		MAX,
+	};
+
+	explicit RandomEncryptionKeyProvider(EncryptionDomainMode mode) : mode(mode) {
+		ASSERT(mode < EncryptionDomainMode::MAX);
+		for (unsigned i = 0; i < NUM_CIPHER; i++) {
+			BlobCipherDetails cipherDetails;
+			cipherDetails.encryptDomainId = 0;
+			cipherDetails.baseCipherId = i;
+			cipherDetails.salt = deterministicRandom()->randomUInt64();
+			cipherKeys[i] = generateCipherKey(cipherDetails);
+		}
+	}
+	virtual ~RandomEncryptionKeyProvider() = default;
+
+	EncodingType expectedEncodingType() const override { return EncodingType::AESEncryptionV1; }
+
+	bool enableEncryption() const override { return true; }
+
+	bool enableEncryptionDomain() const override { return mode > 1; }
+
+	Future<EncryptionKey> getEncryptionKey(const void* encodingHeader) override {
+		using Header = ArenaPage::AESEncryptionV1Encoder::Header;
+		const Header* h = reinterpret_cast<const Header*>(encodingHeader);
+		EncryptionKey s;
+		s.aesKey.cipherTextKey = getCipherKey(h->cipherTextDetails.encryptDomainId, h->cipherTextDetails.baseCipherId);
+		s.aesKey.cipherHeaderKey =
+		    getCipherKey(h->cipherHeaderDetails.encryptDomainId, h->cipherHeaderDetails.baseCipherId);
+		return s;
+	}
+
+	Future<EncryptionKey> getLatestEncryptionKey(int64_t domainId) override {
+		domainId = checkDomainId(domainId);
+		EncryptionKey s;
+		s.aesKey.cipherTextKey = getCipherKey(domainId, deterministicRandom()->randomInt(0, NUM_CIPHER));
+		s.aesKey.cipherHeaderKey =
+		    getCipherKey(ENCRYPT_HEADER_DOMAIN_ID, deterministicRandom()->randomInt(0, NUM_CIPHER));
+		return s;
+	}
+
+	int64_t getDefaultEncryptionDomainId() const override { return FDB_DEFAULT_ENCRYPT_DOMAIN_ID; }
+
+	std::tuple<int64_t, size_t> getEncryptionDomain(const KeyRef& key, Optional<int64_t>) override {
+		int64_t domainId;
+		if (key.size() < PREFIX_LENGTH) {
+			domainId = getDefaultEncryptionDomainId();
+		} else {
+			// Use first 4 bytes as a 32-bit int for the domain id.
+			domainId = checkDomainId(static_cast<int64_t>(*reinterpret_cast<const int32_t*>(key.begin())));
+		}
+		return { domainId, (domainId == getDefaultEncryptionDomainId() ? 0 : PREFIX_LENGTH) };
+	}
+
+	int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override {
+		ASSERT(encodingHeader != nullptr);
+		using Header = ArenaPage::AESEncryptionV1Encoder::Header;
+		const Header* h = reinterpret_cast<const Header*>(encodingHeader);
+		return h->cipherTextDetails.encryptDomainId;
+	}
+
+private:
+	Reference<BlobCipherKey> generateCipherKey(const BlobCipherDetails& cipherDetails) {
+		static unsigned char SHA_KEY[] = "3ab9570b44b8315fdb261da6b1b6c13b";
+		Arena arena;
+		uint8_t digest[AUTH_TOKEN_HMAC_SHA_SIZE];
+		computeAuthToken(
+		    { { reinterpret_cast<const uint8_t*>(&cipherDetails.baseCipherId), sizeof(EncryptCipherBaseKeyId) } },
+		    SHA_KEY,
+		    AES_256_KEY_LENGTH,
+		    &digest[0],
+		    EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
+		    AUTH_TOKEN_HMAC_SHA_SIZE);
+		ASSERT_EQ(AUTH_TOKEN_HMAC_SHA_SIZE, AES_256_KEY_LENGTH);
+		return makeReference<BlobCipherKey>(cipherDetails.encryptDomainId,
+		                                    cipherDetails.baseCipherId,
+		                                    &digest[0],
+		                                    AES_256_KEY_LENGTH,
+		                                    cipherDetails.salt,
+		                                    std::numeric_limits<int64_t>::max() /* refreshAt */,
+		                                    std::numeric_limits<int64_t>::max() /* expireAt */);
+	}
+
+	int64_t checkDomainId(int64_t domainId) {
+		std::hash<int64_t> hasher;
+		if (mode == DISABLED || (mode == RANDOM && hasher(domainId) % 2 == 0)) {
+			return getDefaultEncryptionDomainId();
+		}
+		return domainId;
+	}
+
+	Reference<BlobCipherKey> getCipherKey(EncryptCipherDomainId domainId, EncryptCipherBaseKeyId cipherId) {
+		// Create a new cipher key by replacing the domain id.
+		return makeReference<BlobCipherKey>(domainId,
+		                                    cipherId,
+		                                    cipherKeys[cipherId]->rawBaseCipher(),
+		                                    AES_256_KEY_LENGTH,
+		                                    cipherKeys[cipherId]->getSalt(),
+		                                    std::numeric_limits<int64_t>::max() /* refreshAt */,
+		                                    std::numeric_limits<int64_t>::max() /* expireAt */);
+	}
+
+	static constexpr int NUM_CIPHER = 1000;
+	static constexpr size_t PREFIX_LENGTH = 4;
+	EncryptionDomainMode mode;
+	Reference<BlobCipherKey> cipherKeys[NUM_CIPHER];
+};
+
+// Key provider which extract tenant id from range key prefixes, and fetch tenant specific encryption keys from
+// EncryptKeyProxy.
+class TenantAwareEncryptionKeyProvider : public IPageEncryptionKeyProvider {
+public:
+	using EncodingHeader = ArenaPage::AESEncryptionV1Encoder::Header;
+
+	TenantAwareEncryptionKeyProvider(Reference<AsyncVar<ServerDBInfo> const> db) : db(db) {}
+
+	virtual ~TenantAwareEncryptionKeyProvider() = default;
+
+	EncodingType expectedEncodingType() const override { return EncodingType::AESEncryptionV1; }
+
+	bool enableEncryption() const override {
+		return isEncryptionOpSupported(EncryptOperationType::STORAGE_SERVER_ENCRYPTION, db->get().client);
+	}
+
+	bool enableEncryptionDomain() const override { return true; }
+
+	ACTOR static Future<EncryptionKey> getEncryptionKey(TenantAwareEncryptionKeyProvider* self,
+	                                                    const void* encodingHeader) {
+		const BlobCipherEncryptHeader* header = reinterpret_cast<const EncodingHeader*>(encodingHeader);
+		TextAndHeaderCipherKeys cipherKeys =
+		    wait(getEncryptCipherKeys(self->db, *header, BlobCipherMetrics::KV_REDWOOD));
+		EncryptionKey encryptionKey;
+		encryptionKey.aesKey = cipherKeys;
+		return encryptionKey;
+	}
+
+	Future<EncryptionKey> getEncryptionKey(const void* encodingHeader) override {
+		return getEncryptionKey(this, encodingHeader);
+	}
+
+	Future<EncryptionKey> getLatestDefaultEncryptionKey() override {
+		return getLatestEncryptionKey(getDefaultEncryptionDomainId());
+	}
+
+	ACTOR static Future<EncryptionKey> getLatestEncryptionKey(TenantAwareEncryptionKeyProvider* self,
+	                                                          int64_t domainId) {
+
+		EncryptCipherDomainNameRef domainName = self->getDomainName(domainId);
+		TextAndHeaderCipherKeys cipherKeys =
+		    wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, domainName, BlobCipherMetrics::KV_REDWOOD));
+		EncryptionKey encryptionKey;
+		encryptionKey.aesKey = cipherKeys;
+		return encryptionKey;
+	}
+
+	Future<EncryptionKey> getLatestEncryptionKey(int64_t domainId) override {
+		return getLatestEncryptionKey(this, domainId);
+	}
+
+	int64_t getDefaultEncryptionDomainId() const override { return FDB_DEFAULT_ENCRYPT_DOMAIN_ID; }
+
+	std::tuple<int64_t, size_t> getEncryptionDomain(const KeyRef& key, Optional<int64_t> possibleDomainId) override {
+		// System key.
+		if (key.startsWith("\xff\xff"_sr)) {
+			return { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, 2 };
+		}
+		// Key smaller than tenant prefix in size belongs to the default domain.
+		if (key.size() < TENANT_PREFIX_SIZE) {
+			return { FDB_DEFAULT_ENCRYPT_DOMAIN_ID, 0 };
+		}
+		StringRef prefix = key.substr(0, TENANT_PREFIX_SIZE);
+		int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False);
+		// Tenant id must be non-negative.
+		if (tenantId < 0) {
+			return { FDB_DEFAULT_ENCRYPT_DOMAIN_ID, 0 };
+		}
+		// Optimization: Caller guarantee possibleDomainId is a valid domain id that we previously returned.
+		// We can return immediately without checking with tenant map.
+		if (possibleDomainId.present() && possibleDomainId.get() == tenantId) {
+			return { tenantId, TENANT_PREFIX_SIZE };
+		}
+		if (tenantPrefixIndex.isValid()) {
+			auto view = tenantPrefixIndex->atLatest();
+			auto itr = view.find(prefix);
+			if (itr != view.end()) {
+				// Tenant not found. Tenant must be disabled, or in optional mode.
+				return { tenantId, TENANT_PREFIX_SIZE };
+			}
+		}
+		// The prefix does not belong to any tenant. The key belongs to the default domain.
+		return { FDB_DEFAULT_ENCRYPT_DOMAIN_ID, 0 };
+	}
+
+	int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override {
+		ASSERT(encodingHeader != nullptr);
+		const BlobCipherEncryptHeader* header = reinterpret_cast<const EncodingHeader*>(encodingHeader);
+		return header->cipherTextDetails.encryptDomainId;
+	}
+
+	void setTenantPrefixIndex(Reference<TenantPrefixIndex> tenantPrefixIndex) override {
+		this->tenantPrefixIndex = tenantPrefixIndex;
+	}
+
+private:
+	EncryptCipherDomainNameRef getDomainName(int64_t domainId) {
+		if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
+			return FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
+		}
+		if (domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
+			return FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
+		}
+		if (tenantPrefixIndex.isValid()) {
+			StringRef prefix = TenantMapEntry::idToPrefix(domainId);
+			auto view = tenantPrefixIndex->atLatest();
+			auto itr = view.find(prefix);
+			if (itr != view.end()) {
+				return *itr;
+			}
+		}
+		TraceEvent(SevWarn, "TenantAwareEncryptionKeyProvider_TenantNotFoundForDomain").detail("DomainId", domainId);
+		throw tenant_not_found();
+	}
+
+	Reference<AsyncVar<ServerDBInfo> const> db;
+	Reference<TenantPrefixIndex> tenantPrefixIndex;
+};
+
+#include "flow/unactorcompiler.h"
+#endif
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/IPager.h b/fdbserver/include/fdbserver/IPager.h
index fdfc3c364d..2ec149cb5b 100644
--- a/fdbserver/include/fdbserver/IPager.h
+++ b/fdbserver/include/fdbserver/IPager.h
@@ -17,20 +17,25 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#pragma once
 
 #ifndef FDBSERVER_IPAGER_H
 #define FDBSERVER_IPAGER_H
-#include "flow/Error.h"
-#include "flow/FastAlloc.h"
-#include "flow/ProtocolVersion.h"
+
 #include <cstddef>
 #include <stdint.h>
-#pragma once
-
-#include "fdbserver/IKeyValueStore.h"
-
-#include "flow/flow.h"
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/FDBTypes.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/Tenant.h"
+#include "fdbserver/IClosable.h"
+#include "flow/EncryptUtils.h"
+#include "flow/Error.h"
+#include "flow/FastAlloc.h"
+#include "flow/Knobs.h"
+#include "flow/flow.h"
+#include "flow/ProtocolVersion.h"
+
 #define XXH_INLINE_ALL
 #include "flow/xxhash.h"
 
@@ -46,10 +51,18 @@ typedef uint32_t QueueID;
 enum class PagerEvents { CacheLookup = 0, CacheHit, CacheMiss, PageWrite, MAXEVENTS };
 static const char* const PagerEventsStrings[] = { "Lookup", "Hit", "Miss", "Write", "Unknown" };
 // Reasons for page level events.
-enum class PagerEventReasons { PointRead = 0, RangeRead, RangePrefetch, Commit, LazyClear, MetaData, MAXEVENTREASONS };
-static const char* const PagerEventReasonsStrings[] = {
-	"Get", "GetR", "GetRPF", "Commit", "LazyClr", "Meta", "Unknown"
+enum class PagerEventReasons {
+	PointRead = 0,
+	FetchRange,
+	RangeRead,
+	RangePrefetch,
+	Commit,
+	LazyClear,
+	MetaData,
+	MAXEVENTREASONS
 };
+static const char* const PagerEventReasonsStrings[] = { "Get",    "FetchR",  "GetR", "GetRPF",
+	                                                    "Commit", "LazyClr", "Meta", "Unknown" };
 
 static const unsigned int nonBtreeLevel = 0;
 static const std::vector<std::pair<PagerEvents, PagerEventReasons>> possibleEventReasonPairs = {
@@ -57,14 +70,17 @@ static const std::vector<std::pair<PagerEvents, PagerEventReasons>> possibleEven
 	{ PagerEvents::CacheLookup, PagerEventReasons::LazyClear },
 	{ PagerEvents::CacheLookup, PagerEventReasons::PointRead },
 	{ PagerEvents::CacheLookup, PagerEventReasons::RangeRead },
+	{ PagerEvents::CacheLookup, PagerEventReasons::FetchRange },
 	{ PagerEvents::CacheHit, PagerEventReasons::Commit },
 	{ PagerEvents::CacheHit, PagerEventReasons::LazyClear },
 	{ PagerEvents::CacheHit, PagerEventReasons::PointRead },
 	{ PagerEvents::CacheHit, PagerEventReasons::RangeRead },
+	{ PagerEvents::CacheHit, PagerEventReasons::FetchRange },
 	{ PagerEvents::CacheMiss, PagerEventReasons::Commit },
 	{ PagerEvents::CacheMiss, PagerEventReasons::LazyClear },
 	{ PagerEvents::CacheMiss, PagerEventReasons::PointRead },
 	{ PagerEvents::CacheMiss, PagerEventReasons::RangeRead },
+	{ PagerEvents::CacheMiss, PagerEventReasons::FetchRange },
 	{ PagerEvents::PageWrite, PagerEventReasons::Commit },
 	{ PagerEvents::PageWrite, PagerEventReasons::LazyClear },
 };
@@ -78,11 +94,7 @@ static const std::vector<std::pair<PagerEvents, PagerEventReasons>> L0PossibleEv
 	{ PagerEvents::PageWrite, PagerEventReasons::MetaData },
 };
 
-enum EncodingType : uint8_t {
-	XXHash64 = 0,
-	// For testing purposes
-	XOREncryption = 1
-};
+enum EncodingType : uint8_t { XXHash64 = 0, XOREncryption_TestOnly = 1, AESEncryptionV1 = 2, MAX_ENCODING_TYPE = 3 };
 
 enum PageType : uint8_t {
 	HeaderPage = 0,
@@ -93,41 +105,6 @@ enum PageType : uint8_t {
 	QueuePageInExtent = 5
 };
 
-// Encryption key ID
-typedef uint64_t KeyID;
-
-// EncryptionKeyRef is somewhat multi-variant, it will contain members representing the union
-// of all fields relevant to any implemented encryption scheme.  They are generally of
-// the form
-//   Page Fields - fields which come from or are stored in the Page
-//   Secret Fields - fields which are only known by the Key Provider
-// but it is up to each encoding and provider which fields are which and which ones are used
-struct EncryptionKeyRef {
-
-	EncryptionKeyRef(){};
-	EncryptionKeyRef(Arena& arena, const EncryptionKeyRef& toCopy) : secret(arena, toCopy.secret), id(toCopy.id) {}
-	int expectedSize() const { return secret.size(); }
-
-	StringRef secret;
-	Optional<KeyID> id;
-};
-typedef Standalone<EncryptionKeyRef> EncryptionKey;
-
-// Interface used by pager to get encryption keys by ID when reading pages from disk
-// and by the BTree to get encryption keys to use for new pages
-class IEncryptionKeyProvider {
-public:
-	virtual ~IEncryptionKeyProvider() {}
-
-	// Get an EncryptionKey with Secret Fields populated based on the given Page Fields.
-	// It is up to the implementation which fields those are.
-	// The output Page Fields must match the input Page Fields.
-	virtual Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) = 0;
-
-	// Get encryption key that should be used for a given user Key-Value range
-	virtual Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) = 0;
-};
-
 // This is a hacky way to attach an additional object of an arbitrary type at runtime to another object.
 // It stores an arbitrary void pointer and a void pointer function to call when the ArbitraryObject
 // is destroyed.
@@ -246,6 +223,14 @@ public:
 	uint8_t* rawData() { return buffer; }
 	int rawSize() const { return bufferSize; }
 
+	// Encryption key used to encrypt a page. Different encoding types may use different structs to represent
+	// an encryption key, and EncryptionKeyRef is a union of these structs.
+	struct EncryptionKeyRef {
+		TextAndHeaderCipherKeys aesKey; // For AESEncryptionV1
+		uint8_t xorKey; // For XOREncryption_TestOnly
+	};
+	using EncryptionKey = Standalone<EncryptionKeyRef>;
+
 #pragma pack(push, 1)
 
 	// The next few structs describe the byte-packed physical structure.  The fields of Page
@@ -270,10 +255,7 @@ public:
 		}
 
 		// Get encoding header pointer, casting to its type
-		template <typename T>
-		T* getEncodingHeader() const {
-			return (T*)((uint8_t*)this + encodingHeaderOffset);
-		}
+		void* getEncodingHeader() const { return (uint8_t*)this + encodingHeaderOffset; }
 
 		// Get payload pointer
 		uint8_t* getPayload() const { return (uint8_t*)this + payloadOffset; }
@@ -328,13 +310,19 @@ public:
 	};
 
 	// An encoding that validates the payload with an XXHash checksum
-	struct XXHashEncodingHeader {
-		XXH64_hash_t checksum;
-		void encode(uint8_t* payload, int len, PhysicalPageID seed) {
-			checksum = XXH3_64bits_withSeed(payload, len, seed);
+	struct XXHashEncoder {
+		struct Header {
+			XXH64_hash_t checksum;
+		};
+
+		static void encode(void* header, uint8_t* payload, int len, PhysicalPageID seed) {
+			Header* h = reinterpret_cast<Header*>(header);
+			h->checksum = XXH3_64bits_withSeed(payload, len, seed);
 		}
-		void decode(uint8_t* payload, int len, PhysicalPageID seed) {
-			if (checksum != XXH3_64bits_withSeed(payload, len, seed)) {
+
+		static void decode(void* header, uint8_t* payload, int len, PhysicalPageID seed) {
+			Header* h = reinterpret_cast<Header*>(header);
+			if (h->checksum != XXH3_64bits_withSeed(payload, len, seed)) {
 				throw page_decoding_failed();
 			}
 		}
@@ -342,26 +330,66 @@ public:
 
 	// A dummy "encrypting" encoding which uses XOR with a 1 byte secret key on
 	// the payload to obfuscate it and protects the payload with an XXHash checksum.
-	struct XOREncryptionEncodingHeader {
-		// Checksum is on unencrypted payload
-		XXH64_hash_t checksum;
-		uint8_t keyID;
+	struct XOREncryptionEncoder {
+		struct Header {
+			// Checksum is on unencrypted payload
+			XXH64_hash_t checksum;
+			uint8_t xorKey;
+		};
 
-		void encode(uint8_t secret, uint8_t* payload, int len, PhysicalPageID seed) {
-			checksum = XXH3_64bits_withSeed(payload, len, seed);
+		static void encode(void* header,
+		                   const EncryptionKey& encryptionKey,
+		                   uint8_t* payload,
+		                   int len,
+		                   PhysicalPageID seed) {
+			Header* h = reinterpret_cast<Header*>(header);
+			h->checksum = XXH3_64bits_withSeed(payload, len, seed);
+			h->xorKey = encryptionKey.xorKey;
 			for (int i = 0; i < len; ++i) {
-				payload[i] ^= secret;
+				payload[i] ^= h->xorKey;
 			}
 		}
-		void decode(uint8_t secret, uint8_t* payload, int len, PhysicalPageID seed) {
+
+		static void decode(void* header,
+		                   const EncryptionKey& encryptionKey,
+		                   uint8_t* payload,
+		                   int len,
+		                   PhysicalPageID seed) {
+			Header* h = reinterpret_cast<Header*>(header);
 			for (int i = 0; i < len; ++i) {
-				payload[i] ^= secret;
+				payload[i] ^= h->xorKey;
 			}
-			if (checksum != XXH3_64bits_withSeed(payload, len, seed)) {
+			if (h->checksum != XXH3_64bits_withSeed(payload, len, seed)) {
 				throw page_decoding_failed();
 			}
 		}
 	};
+
+	struct AESEncryptionV1Encoder {
+		using Header = BlobCipherEncryptHeader;
+
+		static void encode(void* header, const TextAndHeaderCipherKeys& cipherKeys, uint8_t* payload, int len) {
+			Header* h = reinterpret_cast<Header*>(header);
+			EncryptBlobCipherAes265Ctr cipher(cipherKeys.cipherTextKey,
+			                                  cipherKeys.cipherHeaderKey,
+			                                  getEncryptAuthTokenMode(ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
+			                                  BlobCipherMetrics::KV_REDWOOD);
+			Arena arena;
+			StringRef ciphertext = cipher.encrypt(payload, len, h, arena)->toStringRef();
+			ASSERT_EQ(len, ciphertext.size());
+			memcpy(payload, ciphertext.begin(), len);
+		}
+
+		static void decode(void* header, const TextAndHeaderCipherKeys& cipherKeys, uint8_t* payload, int len) {
+			Header* h = reinterpret_cast<Header*>(header);
+			DecryptBlobCipherAes256Ctr cipher(
+			    cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, h->iv, BlobCipherMetrics::KV_REDWOOD);
+			Arena arena;
+			StringRef plaintext = cipher.decrypt(payload, len, *h, arena)->toStringRef();
+			ASSERT_EQ(len, plaintext.size());
+			memcpy(payload, plaintext.begin(), len);
+		}
+	};
 #pragma pack(pop)
 
 	// Get the size of the encoding header based on type
@@ -369,9 +397,11 @@ public:
 	// existing pages, the payload offset is stored in the page.
 	static int encodingHeaderSize(EncodingType t) {
 		if (t == EncodingType::XXHash64) {
-			return sizeof(XXHashEncodingHeader);
-		} else if (t == EncodingType::XOREncryption) {
-			return sizeof(XOREncryptionEncodingHeader);
+			return sizeof(XXHashEncoder::Header);
+		} else if (t == EncodingType::XOREncryption_TestOnly) {
+			return sizeof(XOREncryptionEncoder::Header);
+		} else if (t == EncodingType::AESEncryptionV1) {
+			return sizeof(AESEncryptionV1Encoder::Header);
 		} else {
 			throw page_encoding_not_supported();
 		}
@@ -468,19 +498,18 @@ public:
 	//        Secret is set if needed
 	// Post:  Main and Encoding subheaders are updated
 	//        Payload is possibly encrypted
-	void preWrite(PhysicalPageID pageID) const {
+	void preWrite(PhysicalPageID pageID) {
 		// Explicitly check payload definedness to make the source of valgrind errors more clear.
 		// Without this check, calculating a checksum on a payload with undefined bytes does not
 		// cause a valgrind error but the resulting checksum is undefined which causes errors later.
 		ASSERT(VALGRIND_CHECK_MEM_IS_DEFINED(pPayload, payloadSize) == 0);
 
 		if (page->encodingType == EncodingType::XXHash64) {
-			page->getEncodingHeader<XXHashEncodingHeader>()->encode(pPayload, payloadSize, pageID);
-		} else if (page->encodingType == EncodingType::XOREncryption) {
-			ASSERT(encryptionKey.secret.size() == 1);
-			XOREncryptionEncodingHeader* xh = page->getEncodingHeader<XOREncryptionEncodingHeader>();
-			xh->keyID = encryptionKey.id.orDefault(0);
-			xh->encode(encryptionKey.secret[0], pPayload, payloadSize, pageID);
+			XXHashEncoder::encode(page->getEncodingHeader(), pPayload, payloadSize, pageID);
+		} else if (page->encodingType == EncodingType::XOREncryption_TestOnly) {
+			XOREncryptionEncoder::encode(page->getEncodingHeader(), encryptionKey, pPayload, payloadSize, pageID);
+		} else if (page->encodingType == EncodingType::AESEncryptionV1) {
+			AESEncryptionV1Encoder::encode(page->getEncodingHeader(), encryptionKey.aesKey, pPayload, payloadSize);
 		} else {
 			throw page_encoding_not_supported();
 		}
@@ -490,6 +519,7 @@ public:
 		} else {
 			throw page_header_version_not_supported();
 		}
+		encodingHeaderAvailable = true;
 	}
 
 	// Must be called after reading from disk to verify all non-payload bytes
@@ -502,11 +532,7 @@ public:
 	void postReadHeader(PhysicalPageID pageID, bool verify = true) {
 		pPayload = page->getPayload();
 		payloadSize = logicalSize - (pPayload - buffer);
-
-		// Populate encryption key with relevant fields from page
-		if (page->encodingType == EncodingType::XOREncryption) {
-			encryptionKey.id = page->getEncodingHeader<XOREncryptionEncodingHeader>()->keyID;
-		}
+		encodingHeaderAvailable = true;
 
 		if (page->headerVersion == 1) {
 			if (verify) {
@@ -525,11 +551,11 @@ public:
 	// Post:  Payload has been verified and decrypted if necessary
 	void postReadPayload(PhysicalPageID pageID) {
 		if (page->encodingType == EncodingType::XXHash64) {
-			page->getEncodingHeader<XXHashEncodingHeader>()->decode(pPayload, payloadSize, pageID);
-		} else if (page->encodingType == EncodingType::XOREncryption) {
-			ASSERT(encryptionKey.secret.size() == 1);
-			page->getEncodingHeader<XOREncryptionEncodingHeader>()->decode(
-			    encryptionKey.secret[0], pPayload, payloadSize, pageID);
+			XXHashEncoder::decode(page->getEncodingHeader(), pPayload, payloadSize, pageID);
+		} else if (page->encodingType == EncodingType::XOREncryption_TestOnly) {
+			XOREncryptionEncoder::decode(page->getEncodingHeader(), encryptionKey, pPayload, payloadSize, pageID);
+		} else if (page->encodingType == EncodingType::AESEncryptionV1) {
+			AESEncryptionV1Encoder::decode(page->getEncodingHeader(), encryptionKey.aesKey, pPayload, payloadSize);
 		} else {
 			throw page_encoding_not_supported();
 		}
@@ -537,11 +563,26 @@ public:
 
 	const Arena& getArena() const { return arena; }
 
-	static bool isEncodingTypeEncrypted(EncodingType t) { return t == EncodingType::XOREncryption; }
+	static bool isEncodingTypeEncrypted(EncodingType t) {
+		return t == EncodingType::AESEncryptionV1 || t == EncodingType::XOREncryption_TestOnly;
+	}
 
 	// Returns true if the page's encoding type employs encryption
 	bool isEncrypted() const { return isEncodingTypeEncrypted(getEncodingType()); }
 
+	// Return encryption domain id used. This method only use information from the encryptionKey.
+	// Caller should make sure encryption domain is in use.
+	int64_t getEncryptionDomainId() const {
+		// encryption domain is only supported by AESEncryptionV1.
+		ASSERT(getEncodingType() == EncodingType::AESEncryptionV1);
+		const Reference<BlobCipherKey>& cipherKey = encryptionKey.aesKey.cipherTextKey;
+		ASSERT(cipherKey.isValid());
+		return cipherKey->getDomainId();
+	}
+
+	// Return pointer to encoding header.
+	const void* getEncodingHeader() const { return encodingHeaderAvailable ? page->getEncodingHeader() : nullptr; }
+
 private:
 	Arena arena;
 
@@ -580,6 +621,9 @@ public:
 	// Used by encodings that do encryption
 	EncryptionKey encryptionKey;
 
+	// Whether encoding header is set
+	bool encodingHeaderAvailable = false;
+
 	mutable ArbitraryObject extra;
 };
 
@@ -739,52 +783,4 @@ protected:
 	~IPager2() {} // Destruction should be done using close()/dispose() from the IClosable interface
 };
 
-// The null key provider is useful to simplify page decoding.
-// It throws an error for any key info requested.
-class NullKeyProvider : public IEncryptionKeyProvider {
-public:
-	virtual ~NullKeyProvider() {}
-	Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override { throw encryption_key_not_found(); }
-	Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
-		throw encryption_key_not_found();
-	}
-};
-
-// Key provider for dummy XOR encryption scheme
-class XOREncryptionKeyProvider : public IEncryptionKeyProvider {
-public:
-	XOREncryptionKeyProvider(std::string filename) {
-		ASSERT(g_network->isSimulated());
-
-		// Choose a deterministic random filename (without path) byte for secret generation
-		// Remove any leading directory names
-		size_t lastSlash = filename.find_last_of("\\/");
-		if (lastSlash != filename.npos) {
-			filename.erase(0, lastSlash);
-		}
-		xorWith = filename.empty() ? 0x5e
-		                           : (uint8_t)filename[XXH3_64bits(filename.data(), filename.size()) % filename.size()];
-	}
-
-	virtual ~XOREncryptionKeyProvider() {}
-
-	virtual Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override {
-		if (!key.id.present()) {
-			throw encryption_key_not_found();
-		}
-		EncryptionKey s = key;
-		uint8_t secret = ~(uint8_t)key.id.get() ^ xorWith;
-		s.secret = StringRef(s.arena(), &secret, 1);
-		return s;
-	}
-
-	virtual Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
-		EncryptionKeyRef k;
-		k.id = end.empty() ? 0 : *(end.end() - 1);
-		return getSecrets(k);
-	}
-
-	uint8_t xorWith;
-};
-
 #endif
diff --git a/fdbserver/include/fdbserver/KmsConnectorInterface.h b/fdbserver/include/fdbserver/KmsConnectorInterface.h
index 3e91fd38f6..ac31bc5d37 100644
--- a/fdbserver/include/fdbserver/KmsConnectorInterface.h
+++ b/fdbserver/include/fdbserver/KmsConnectorInterface.h
@@ -67,7 +67,7 @@ struct KmsConnectorInterface {
 	}
 };
 
-struct EncryptCipherKeyDetails {
+struct EncryptCipherKeyDetailsRef {
 	constexpr static FileIdentifier file_identifier = 1227025;
 	EncryptCipherDomainId encryptDomainId;
 	EncryptCipherBaseKeyId encryptKeyId;
@@ -75,33 +75,33 @@ struct EncryptCipherKeyDetails {
 	Optional<int64_t> refreshAfterSec;
 	Optional<int64_t> expireAfterSec;
 
-	EncryptCipherKeyDetails() {}
-	explicit EncryptCipherKeyDetails(Arena& arena,
-	                                 EncryptCipherDomainId dId,
-	                                 EncryptCipherBaseKeyId keyId,
-	                                 StringRef key)
+	EncryptCipherKeyDetailsRef() {}
+	explicit EncryptCipherKeyDetailsRef(Arena& arena,
+	                                    EncryptCipherDomainId dId,
+	                                    EncryptCipherBaseKeyId keyId,
+	                                    StringRef key)
 	  : encryptDomainId(dId), encryptKeyId(keyId), encryptKey(StringRef(arena, key)),
 	    refreshAfterSec(Optional<int64_t>()), expireAfterSec(Optional<int64_t>()) {}
-	explicit EncryptCipherKeyDetails(EncryptCipherDomainId dId, EncryptCipherBaseKeyId keyId, StringRef key)
+	explicit EncryptCipherKeyDetailsRef(EncryptCipherDomainId dId, EncryptCipherBaseKeyId keyId, StringRef key)
 	  : encryptDomainId(dId), encryptKeyId(keyId), encryptKey(key), refreshAfterSec(Optional<int64_t>()),
 	    expireAfterSec(Optional<int64_t>()) {}
-	explicit EncryptCipherKeyDetails(Arena& arena,
-	                                 EncryptCipherDomainId dId,
-	                                 EncryptCipherBaseKeyId keyId,
-	                                 StringRef key,
-	                                 Optional<int64_t> refAfterSec,
-	                                 Optional<int64_t> expAfterSec)
+	explicit EncryptCipherKeyDetailsRef(Arena& arena,
+	                                    EncryptCipherDomainId dId,
+	                                    EncryptCipherBaseKeyId keyId,
+	                                    StringRef key,
+	                                    Optional<int64_t> refAfterSec,
+	                                    Optional<int64_t> expAfterSec)
 	  : encryptDomainId(dId), encryptKeyId(keyId), encryptKey(StringRef(arena, key)), refreshAfterSec(refAfterSec),
 	    expireAfterSec(expAfterSec) {}
-	explicit EncryptCipherKeyDetails(EncryptCipherDomainId dId,
-	                                 EncryptCipherBaseKeyId keyId,
-	                                 StringRef key,
-	                                 Optional<int64_t> refAfterSec,
-	                                 Optional<int64_t> expAfterSec)
+	explicit EncryptCipherKeyDetailsRef(EncryptCipherDomainId dId,
+	                                    EncryptCipherBaseKeyId keyId,
+	                                    StringRef key,
+	                                    Optional<int64_t> refAfterSec,
+	                                    Optional<int64_t> expAfterSec)
 	  : encryptDomainId(dId), encryptKeyId(keyId), encryptKey(key), refreshAfterSec(refAfterSec),
 	    expireAfterSec(expAfterSec) {}
 
-	bool operator==(const EncryptCipherKeyDetails& toCompare) {
+	bool operator==(const EncryptCipherKeyDetailsRef& toCompare) {
 		return encryptDomainId == toCompare.encryptDomainId && encryptKeyId == toCompare.encryptKeyId &&
 		       encryptKey.compare(toCompare.encryptKey) == 0;
 	}
@@ -115,30 +115,31 @@ struct EncryptCipherKeyDetails {
 struct KmsConnLookupEKsByKeyIdsRep {
 	constexpr static FileIdentifier file_identifier = 2313778;
 	Arena arena;
-	VectorRef<EncryptCipherKeyDetails> cipherKeyDetails;
+	VectorRef<EncryptCipherKeyDetailsRef> cipherKeyDetails;
 
 	KmsConnLookupEKsByKeyIdsRep() {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, cipherKeyDetails);
+		serializer(ar, cipherKeyDetails, arena);
 	}
 };
 
-struct KmsConnLookupKeyIdsReqInfo {
+struct KmsConnLookupKeyIdsReqInfoRef {
 	constexpr static FileIdentifier file_identifier = 3092256;
 	EncryptCipherDomainId domainId;
 	EncryptCipherBaseKeyId baseCipherId;
-	EncryptCipherDomainName domainName;
+	EncryptCipherDomainNameRef domainName;
 
-	KmsConnLookupKeyIdsReqInfo() : domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {}
-	explicit KmsConnLookupKeyIdsReqInfo(Arena& arena,
-	                                    const EncryptCipherDomainId dId,
-	                                    const EncryptCipherBaseKeyId bCId,
-	                                    StringRef name)
+	KmsConnLookupKeyIdsReqInfoRef()
+	  : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
+	explicit KmsConnLookupKeyIdsReqInfoRef(Arena& arena,
+	                                       const EncryptCipherDomainId dId,
+	                                       const EncryptCipherBaseKeyId bCId,
+	                                       StringRef name)
 	  : domainId(dId), baseCipherId(bCId), domainName(StringRef(arena, name)) {}
 
-	bool operator==(const KmsConnLookupKeyIdsReqInfo& info) const {
+	bool operator==(const KmsConnLookupKeyIdsReqInfoRef& info) const {
 		return domainId == info.domainId && baseCipherId == info.baseCipherId &&
 		       (domainName.compare(info.domainName) == 0);
 	}
@@ -152,45 +153,45 @@ struct KmsConnLookupKeyIdsReqInfo {
 struct KmsConnLookupEKsByKeyIdsReq {
 	constexpr static FileIdentifier file_identifier = 6913396;
 	Arena arena;
-	VectorRef<KmsConnLookupKeyIdsReqInfo> encryptKeyInfos;
+	VectorRef<KmsConnLookupKeyIdsReqInfoRef> encryptKeyInfos;
 	Optional<UID> debugId;
 	ReplyPromise<KmsConnLookupEKsByKeyIdsRep> reply;
 
 	KmsConnLookupEKsByKeyIdsReq() {}
-	explicit KmsConnLookupEKsByKeyIdsReq(VectorRef<KmsConnLookupKeyIdsReqInfo> keyInfos, Optional<UID> dbgId)
+	explicit KmsConnLookupEKsByKeyIdsReq(VectorRef<KmsConnLookupKeyIdsReqInfoRef> keyInfos, Optional<UID> dbgId)
 	  : encryptKeyInfos(keyInfos), debugId(dbgId) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, encryptKeyInfos, debugId, reply);
+		serializer(ar, encryptKeyInfos, debugId, reply, arena);
 	}
 };
 
 struct KmsConnLookupEKsByDomainIdsRep {
 	constexpr static FileIdentifier file_identifier = 3009025;
 	Arena arena;
-	VectorRef<EncryptCipherKeyDetails> cipherKeyDetails;
+	VectorRef<EncryptCipherKeyDetailsRef> cipherKeyDetails;
 
 	KmsConnLookupEKsByDomainIdsRep() {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, cipherKeyDetails);
+		serializer(ar, cipherKeyDetails, arena);
 	}
 };
 
-struct KmsConnLookupDomainIdsReqInfo {
+struct KmsConnLookupDomainIdsReqInfoRef {
 	constexpr static FileIdentifier file_identifier = 8980149;
 	EncryptCipherDomainId domainId;
-	EncryptCipherDomainName domainName;
+	EncryptCipherDomainNameRef domainName;
 
-	KmsConnLookupDomainIdsReqInfo() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {}
-	explicit KmsConnLookupDomainIdsReqInfo(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
+	KmsConnLookupDomainIdsReqInfoRef() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
+	explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
 	  : domainId(dId), domainName(StringRef(arena, name)) {}
-	explicit KmsConnLookupDomainIdsReqInfo(const EncryptCipherDomainId dId, StringRef name)
+	explicit KmsConnLookupDomainIdsReqInfoRef(const EncryptCipherDomainId dId, StringRef name)
 	  : domainId(dId), domainName(name) {}
 
-	bool operator==(const KmsConnLookupDomainIdsReqInfo& info) const {
+	bool operator==(const KmsConnLookupDomainIdsReqInfoRef& info) const {
 		return domainId == info.domainId && (domainName.compare(info.domainName) == 0);
 	}
 
@@ -203,17 +204,17 @@ struct KmsConnLookupDomainIdsReqInfo {
 struct KmsConnLookupEKsByDomainIdsReq {
 	constexpr static FileIdentifier file_identifier = 9918682;
 	Arena arena;
-	VectorRef<KmsConnLookupDomainIdsReqInfo> encryptDomainInfos;
+	VectorRef<KmsConnLookupDomainIdsReqInfoRef> encryptDomainInfos;
 	Optional<UID> debugId;
 	ReplyPromise<KmsConnLookupEKsByDomainIdsRep> reply;
 
 	KmsConnLookupEKsByDomainIdsReq() {}
-	explicit KmsConnLookupEKsByDomainIdsReq(VectorRef<KmsConnLookupDomainIdsReqInfo>& infos, Optional<UID> dbgId)
+	explicit KmsConnLookupEKsByDomainIdsReq(VectorRef<KmsConnLookupDomainIdsReqInfoRef>& infos, Optional<UID> dbgId)
 	  : encryptDomainInfos(infos), debugId(dbgId) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, encryptDomainInfos, debugId, reply);
+		serializer(ar, encryptDomainInfos, debugId, reply, arena);
 	}
 };
 
diff --git a/fdbserver/include/fdbserver/LocalConfiguration.h b/fdbserver/include/fdbserver/LocalConfiguration.h
index 5458e9b928..f748408775 100644
--- a/fdbserver/include/fdbserver/LocalConfiguration.h
+++ b/fdbserver/include/fdbserver/LocalConfiguration.h
@@ -64,7 +64,8 @@ public:
 
 public: // Testing
 	Future<Void> addChanges(Standalone<VectorRef<VersionedConfigMutationRef>> versionedMutations,
-	                        Version mostRecentVersion);
+	                        Version mostRecentVersion,
+	                        double restartDelay);
 	void close();
 	Future<Void> onClosed();
 };
diff --git a/fdbserver/include/fdbserver/LogSystem.h b/fdbserver/include/fdbserver/LogSystem.h
index eb1e2d52cd..fd3f31ab6a 100644
--- a/fdbserver/include/fdbserver/LogSystem.h
+++ b/fdbserver/include/fdbserver/LogSystem.h
@@ -865,7 +865,7 @@ void LogPushData::writeTypedMessage(T const& item, bool metadataMessage, bool al
 		// subsequence wasn't actually used and can be decremented.
 		if (!updatedLocation) {
 			this->subsequence--;
-			TEST(true); // No new SpanContextMessage written to transaction logs
+			CODE_PROBE(true, "No new SpanContextMessage written to transaction logs");
 			ASSERT(this->subsequence > 0);
 		}
 	} else {
diff --git a/fdbserver/include/fdbserver/MasterInterface.h b/fdbserver/include/fdbserver/MasterInterface.h
index 4cf4865a42..5c45ae7323 100644
--- a/fdbserver/include/fdbserver/MasterInterface.h
+++ b/fdbserver/include/fdbserver/MasterInterface.h
@@ -83,13 +83,15 @@ struct ChangeCoordinatorsRequest {
 	constexpr static FileIdentifier file_identifier = 13605416;
 	Standalone<StringRef> newConnectionString;
 	ReplyPromise<Void> reply; // normally throws even on success!
+	UID masterId;
 
 	ChangeCoordinatorsRequest() {}
-	ChangeCoordinatorsRequest(Standalone<StringRef> newConnectionString) : newConnectionString(newConnectionString) {}
+	ChangeCoordinatorsRequest(Standalone<StringRef> newConnectionString, UID masterId)
+	  : newConnectionString(newConnectionString), masterId(masterId) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, newConnectionString, reply);
+		serializer(ar, newConnectionString, reply, masterId);
 	}
 };
 
diff --git a/fdbserver/include/fdbserver/MockGlobalState.h b/fdbserver/include/fdbserver/MockGlobalState.h
new file mode 100644
index 0000000000..bc3227b14e
--- /dev/null
+++ b/fdbserver/include/fdbserver/MockGlobalState.h
@@ -0,0 +1,128 @@
+/*
+ * MockGlobalState.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOUNDATIONDB_MOCKGLOBALSTATE_H
+#define FOUNDATIONDB_MOCKGLOBALSTATE_H
+
+#include "StorageMetrics.h"
+#include "fdbclient/KeyRangeMap.h"
+#include "fdbclient/StorageServerInterface.h"
+#include "fdbclient/DatabaseConfiguration.h"
+#include "SimulatedCluster.h"
+#include "ShardsAffectedByTeamFailure.h"
+
+template <class Metric>
+struct ShardSizeMetric {
+	template <typename pair_type>
+	Metric operator()(pair_type const& p) const {
+		return Metric(p.value.shardSize);
+	}
+};
+
+enum class MockShardStatus { EMPTY = 0, COMPLETED, INFLIGHT };
+
+class MockStorageServer {
+public:
+	struct ShardInfo {
+		MockShardStatus status;
+		uint64_t shardSize;
+	};
+
+	static constexpr uint64_t DEFAULT_DISK_SPACE = 1000LL * 1024 * 1024 * 1024;
+
+	// control plane statistics associated with a real storage server
+	uint64_t usedDiskSpace = 0, availableDiskSpace = DEFAULT_DISK_SPACE;
+
+	// In-memory counterpart of the `serverKeys` in system keyspace
+	// the value ShardStatus is [InFlight, Completed, Empty] and metrics uint64_t is the shard size
+	KeyRangeMap<ShardInfo, uint64_t, ShardSizeMetric<uint64_t>> serverKeys;
+
+	// sampled metrics
+	StorageServerMetrics metrics;
+	CoalescedKeyRangeMap<bool, int64_t, KeyBytesMetric<int64_t>> byteSampleClears;
+
+	StorageServerInterface ssi; // serve RPC requests
+	UID id;
+	bool primary = true; // Only support single region MGS for now
+
+	MockStorageServer() = default;
+
+	MockStorageServer(StorageServerInterface ssi, uint64_t availableDiskSpace, uint64_t usedDiskSpace = 0)
+	  : usedDiskSpace(usedDiskSpace), availableDiskSpace(availableDiskSpace), ssi(ssi), id(ssi.id()) {}
+
+	MockStorageServer(const UID& id, uint64_t availableDiskSpace, uint64_t usedDiskSpace = 0)
+	  : MockStorageServer(StorageServerInterface(id), availableDiskSpace, usedDiskSpace) {}
+
+	decltype(serverKeys)::Ranges getAllRanges() { return serverKeys.ranges(); }
+
+	bool allShardStatusEqual(KeyRangeRef range, MockShardStatus status);
+};
+
+class MockGlobalState {
+public:
+	typedef ShardsAffectedByTeamFailure::Team Team;
+	// In-memory counterpart of the `keyServers` in system keyspace
+	Reference<ShardsAffectedByTeamFailure> shardMapping;
+	// In-memory counterpart of the `serverListKeys` in system keyspace
+	std::map<UID, MockStorageServer> allServers;
+	DatabaseConfiguration configuration;
+
+	// user defined parameters for mock workload purpose
+	double emptyProb; // probability of doing an empty read
+	uint32_t minByteSize, maxByteSize; // the size band of a point data operation
+
+	MockGlobalState() : shardMapping(new ShardsAffectedByTeamFailure) {}
+
+	static UID indexToUID(uint64_t a) { return UID(a, a); }
+	void initializeAsEmptyDatabaseMGS(const DatabaseConfiguration& conf,
+	                                  uint64_t defaultDiskSpace = MockStorageServer::DEFAULT_DISK_SPACE);
+
+	void addStorageServer(StorageServerInterface server, uint64_t diskSpace = MockStorageServer::DEFAULT_DISK_SPACE);
+
+	// check methods
+	/* Shard status contract:
+	 * Shard is static.
+	 * * In mgs.shardMapping, the destination teams is empty for the given shard;
+	 * * For each MSS belonging to the source teams, mss.serverKeys[shard] = Completed
+	 * Shard is in-flight.
+	 * * In mgs.shardMapping,the destination teams is non-empty for a given shard;
+	 * * For each MSS belonging to the source teams, mss.serverKeys[shard] = Completed
+	 * * For each MSS belonging to the destination teams, mss.serverKeys[shard] = InFlight
+	 * Shard is lost.
+	 * * In mgs.shardMapping,  the destination teams is empty for the given shard;
+	 * * For each MSS belonging to the source teams, mss.serverKeys[shard] = Empty
+	 */
+	bool serverIsSourceForShard(const UID& serverId, KeyRangeRef shard, bool inFlightShard = false);
+	bool serverIsDestForShard(const UID& serverId, KeyRangeRef shard);
+
+	/* Server status contract:
+	 * Server X  is removed
+	 * * mgs.shardMapping doesn’t have any information about X
+	 * * mgs.allServer doesn’t contain X
+	 * Server X is healthy
+	 * * mgs.allServer[X] is existed
+	 * Server X is failed but haven’t been removed (a temporary status between healthy and removed)
+	 * * mgs.shardMapping doesn’t have any information about X
+	 * * mgs.allServer[X] is existed
+	 */
+	bool allShardRemovedFromServer(const UID& serverId);
+};
+
+#endif // FOUNDATIONDB_MOCKGLOBALSTATE_H
diff --git a/fdbserver/include/fdbserver/MoveKeys.actor.h b/fdbserver/include/fdbserver/MoveKeys.actor.h
index b809b02521..63d3ae08b2 100644
--- a/fdbserver/include/fdbserver/MoveKeys.actor.h
+++ b/fdbserver/include/fdbserver/MoveKeys.actor.h
@@ -56,6 +56,23 @@ public:
 	bool setDDEnabled(bool status, UID snapUID);
 };
 
+struct MoveKeysParams {
+	UID dataMoveId;
+	KeyRange keys;
+	std::vector<UID> destinationTeam, healthyDestinations;
+	MoveKeysLock lock;
+	Promise<Void> dataMovementComplete;
+	FlowLock* startMoveKeysParallelismLock = nullptr;
+	FlowLock* finishMoveKeysParallelismLock = nullptr;
+	bool hasRemote;
+	UID relocationIntervalId;
+	const DDEnabledState* ddEnabledState = nullptr;
+	CancelConflictingDataMoves cancelConflictingDataMoves = CancelConflictingDataMoves::False;
+};
+
+// read the lock value in system keyspace but do not change anything
+ACTOR Future<MoveKeysLock> readMoveKeysLock(Database cx);
+
 // Calling moveKeys, etc with the return value of this actor ensures that no movekeys, etc
 // has been executed by a different locker since takeMoveKeysLock(), as calling
 // takeMoveKeysLock() updates "moveKeysLockOwnerKey" to a random UID.
@@ -74,19 +91,7 @@ void seedShardServers(Arena& trArena, CommitTransactionRef& tr, std::vector<Stor
 // for restarting the remainder, and for not otherwise cancelling it before
 // it returns (since it needs to execute the finishMoveKeys transaction).
 // When dataMoveId.isValid(), the keyrange will be moved to a shard designated as dataMoveId.
-ACTOR Future<Void> moveKeys(Database occ,
-                            UID dataMoveId,
-                            KeyRange keys,
-                            std::vector<UID> destinationTeam,
-                            std::vector<UID> healthyDestinations,
-                            MoveKeysLock lock,
-                            Promise<Void> dataMovementComplete,
-                            FlowLock* startMoveKeysParallelismLock,
-                            FlowLock* finishMoveKeysParallelismLock,
-                            bool hasRemote,
-                            UID relocationIntervalId, // for logging only
-                            const DDEnabledState* ddEnabledState,
-                            CancelConflictingDataMoves cancelConflictingDataMoves = CancelConflictingDataMoves::False);
+ACTOR Future<Void> moveKeys(Database occ, MoveKeysParams params);
 
 // Cancels a data move designated by dataMoveId.
 ACTOR Future<Void> cleanUpDataMove(Database occ,
diff --git a/fdbserver/include/fdbserver/PaxosConfigConsumer.h b/fdbserver/include/fdbserver/PaxosConfigConsumer.h
index fc9e424d23..b3e01bc761 100644
--- a/fdbserver/include/fdbserver/PaxosConfigConsumer.h
+++ b/fdbserver/include/fdbserver/PaxosConfigConsumer.h
@@ -32,13 +32,17 @@ class PaxosConfigConsumer : public IConfigConsumer {
 public:
 	PaxosConfigConsumer(ServerCoordinators const& coordinators,
 	                    double pollingInterval,
-	                    Optional<double> compactionInterval);
+	                    Optional<double> compactionInterval,
+	                    bool readPreviousCoordinators);
 	~PaxosConfigConsumer();
+	Future<Void> readSnapshot(ConfigBroadcaster& broadcaster) override;
 	Future<Void> consume(ConfigBroadcaster& broadcaster) override;
+	void allowSpecialCaseRollforward() override;
 	UID getID() const override;
 
 public: // Testing
 	PaxosConfigConsumer(std::vector<ConfigFollowerInterface> const& cfis,
 	                    double pollingInterval,
-	                    Optional<double> compactionInterval);
+	                    Optional<double> compactionInterval,
+	                    bool readPreviousCoordinators);
 };
diff --git a/fdbserver/include/fdbserver/ProxyCommitData.actor.h b/fdbserver/include/fdbserver/ProxyCommitData.actor.h
index b6a75f7ea7..a6cd7c1575 100644
--- a/fdbserver/include/fdbserver/ProxyCommitData.actor.h
+++ b/fdbserver/include/fdbserver/ProxyCommitData.actor.h
@@ -19,6 +19,8 @@
  */
 
 #pragma once
+#include "fdbserver/EncryptionOpsUtils.h"
+#include <unordered_map>
 #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H)
 #define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H
 #include "fdbserver/ProxyCommitData.actor.g.h"
@@ -30,9 +32,9 @@
 #include "fdbrpc/Stats.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/LogSystem.h"
+#include "fdbserver/LogSystemDiskQueueAdapter.h"
 #include "fdbserver/MasterInterface.h"
 #include "fdbserver/ResolverInterface.h"
-#include "fdbserver/LogSystemDiskQueueAdapter.h"
 #include "flow/IRandom.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -134,27 +136,17 @@ struct ProxyStats {
 	                   SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
 	                   SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
 	    maxComputeNS(0), minComputeNS(1e12),
-	    commitBatchQueuingDist(Histogram::getHistogram(LiteralStringRef("CommitProxy"),
-	                                                   LiteralStringRef("CommitBatchQueuing"),
-	                                                   Histogram::Unit::microseconds)),
-	    getCommitVersionDist(Histogram::getHistogram(LiteralStringRef("CommitProxy"),
-	                                                 LiteralStringRef("GetCommitVersion"),
-	                                                 Histogram::Unit::microseconds)),
-	    resolutionDist(Histogram::getHistogram(LiteralStringRef("CommitProxy"),
-	                                           LiteralStringRef("Resolution"),
-	                                           Histogram::Unit::microseconds)),
-	    postResolutionDist(Histogram::getHistogram(LiteralStringRef("CommitProxy"),
-	                                               LiteralStringRef("PostResolutionQueuing"),
-	                                               Histogram::Unit::microseconds)),
-	    processingMutationDist(Histogram::getHistogram(LiteralStringRef("CommitProxy"),
-	                                                   LiteralStringRef("ProcessingMutation"),
-	                                                   Histogram::Unit::microseconds)),
-	    tlogLoggingDist(Histogram::getHistogram(LiteralStringRef("CommitProxy"),
-	                                            LiteralStringRef("TlogLogging"),
-	                                            Histogram::Unit::microseconds)),
-	    replyCommitDist(Histogram::getHistogram(LiteralStringRef("CommitProxy"),
-	                                            LiteralStringRef("ReplyCommit"),
-	                                            Histogram::Unit::microseconds)) {
+	    commitBatchQueuingDist(
+	        Histogram::getHistogram("CommitProxy"_sr, "CommitBatchQueuing"_sr, Histogram::Unit::microseconds)),
+	    getCommitVersionDist(
+	        Histogram::getHistogram("CommitProxy"_sr, "GetCommitVersion"_sr, Histogram::Unit::microseconds)),
+	    resolutionDist(Histogram::getHistogram("CommitProxy"_sr, "Resolution"_sr, Histogram::Unit::microseconds)),
+	    postResolutionDist(
+	        Histogram::getHistogram("CommitProxy"_sr, "PostResolutionQueuing"_sr, Histogram::Unit::microseconds)),
+	    processingMutationDist(
+	        Histogram::getHistogram("CommitProxy"_sr, "ProcessingMutation"_sr, Histogram::Unit::microseconds)),
+	    tlogLoggingDist(Histogram::getHistogram("CommitProxy"_sr, "TlogLogging"_sr, Histogram::Unit::microseconds)),
+	    replyCommitDist(Histogram::getHistogram("CommitProxy"_sr, "ReplyCommit"_sr, Histogram::Unit::microseconds)) {
 		specialCounter(cc, "LastAssignedCommitVersion", [this]() { return this->lastCommitVersionAssigned; });
 		specialCounter(cc, "Version", [pVersion]() { return pVersion->get(); });
 		specialCounter(cc, "CommittedVersion", [pCommittedVersion]() { return pCommittedVersion->get(); });
@@ -172,6 +164,7 @@ struct ProxyCommitData {
 	UID dbgid;
 	int64_t commitBatchesMemBytesCount;
 	std::map<TenantName, TenantMapEntry> tenantMap;
+	std::unordered_map<int64_t, TenantName> tenantIdIndex;
 	ProxyStats stats;
 	MasterInterface master;
 	std::vector<ResolverInterface> resolvers;
@@ -231,6 +224,8 @@ struct ProxyCommitData {
 	double lastResolverReset;
 	int localTLogCount = -1;
 
+	bool isEncryptionEnabled = false;
+
 	// The tag related to a storage server rarely change, so we keep a vector of tags for each key range to be slightly
 	// more CPU efficient. When a tag related to a storage server does change, we empty out all of these vectors to
 	// signify they must be repopulated. We do not repopulate them immediately to avoid a slow task.
@@ -297,9 +292,10 @@ struct ProxyCommitData {
 	    commitBatchInterval(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_MIN), localCommitBatchesStarted(0),
 	    getConsistentReadVersion(getConsistentReadVersion), commit(commit),
 	    cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True)), db(db),
-	    singleKeyMutationEvent(LiteralStringRef("SingleKeyMutation")), lastTxsPop(0), popRemoteTxs(false),
-	    lastStartCommit(0), lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), lastCommitTime(0),
-	    lastMasterReset(now()), lastResolverReset(now()) {
+	    singleKeyMutationEvent("SingleKeyMutation"_sr), lastTxsPop(0), popRemoteTxs(false), lastStartCommit(0),
+	    lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), lastCommitTime(0), lastMasterReset(now()),
+	    lastResolverReset(now()),
+	    isEncryptionEnabled(isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION, db->get().client)) {
 		commitComputePerOperation.resize(SERVER_KNOBS->PROXY_COMPUTE_BUCKETS, 0.0);
 	}
 };
diff --git a/fdbserver/include/fdbserver/QuietDatabase.h b/fdbserver/include/fdbserver/QuietDatabase.h
index f2f23b5e6a..2dd814c20e 100644
--- a/fdbserver/include/fdbserver/QuietDatabase.h
+++ b/fdbserver/include/fdbserver/QuietDatabase.h
@@ -41,13 +41,19 @@ Future<bool> getTeamCollectionValid(Database const& cx, WorkerInterface const&);
 Future<bool> getTeamCollectionValid(Database const& cx, Reference<AsyncVar<struct ServerDBInfo> const> const&);
 Future<std::vector<StorageServerInterface>> getStorageServers(Database const& cx,
                                                               bool const& use_system_priority = false);
-Future<std::vector<BlobWorkerInterface>> getBlobWorkers(Database const& cx, bool const& use_system_priority = false);
+Future<std::vector<BlobWorkerInterface>> getBlobWorkers(Database const& cx,
+                                                        bool const& use_system_priority = false,
+                                                        Version* const& grv = nullptr);
 Future<std::vector<WorkerDetails>> getWorkers(Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
                                               int const& flags = 0);
 Future<WorkerInterface> getMasterWorker(Database const& cx, Reference<AsyncVar<ServerDBInfo> const> const& dbInfo);
 Future<Void> repairDeadDatacenter(Database const& cx,
                                   Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
                                   std::string const& context);
+Future<Void> reconfigureAfter(Database const& cx,
+                              double const& time,
+                              Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
+                              std::string const& context);
 
 // Returns list of worker interfaces for available storage servers and the number of unavailable
 // storage servers
diff --git a/fdbserver/include/fdbserver/Ratekeeper.h b/fdbserver/include/fdbserver/Ratekeeper.h
index 948cca851b..02d9d00c54 100644
--- a/fdbserver/include/fdbserver/Ratekeeper.h
+++ b/fdbserver/include/fdbserver/Ratekeeper.h
@@ -18,6 +18,9 @@
  * limitations under the License.
  */
 
+#ifndef FDBSERVER_RATEKEEPER_H
+#define FDBSERVER_RATEKEEPER_H
+
 #pragma once
 
 #include "fdbclient/DatabaseConfiguration.h"
@@ -43,19 +46,21 @@ enum limitReason_t {
 	log_server_min_free_space_ratio,
 	storage_server_durability_lag, // 10
 	storage_server_list_fetch_failed,
+	blob_worker_lag,
+	blob_worker_missing,
 	limitReason_t_end
 };
 
 class StorageQueueInfo {
 	uint64_t totalWriteCosts{ 0 };
 	int totalWriteOps{ 0 };
-	Reference<EventCacheHolder> busiestWriteTagEventHolder;
 
 	// refresh periodically
 	TransactionTagMap<TransactionCommitCostEstimation> tagCostEst;
 
 public:
 	bool valid;
+	UID ratekeeperID;
 	UID id;
 	LocalityData locality;
 	StorageQueuingMetricsReply lastReply;
@@ -67,12 +72,17 @@ public:
 	limitReason_t limitReason;
 	std::vector<StorageQueuingMetricsReply::TagInfo> busiestReadTags, busiestWriteTags;
 
-	StorageQueueInfo(UID id, LocalityData locality);
-	void refreshCommitCost(double elapsed);
+	StorageQueueInfo(const UID& id, const LocalityData& locality);
+	StorageQueueInfo(const UID& rateKeeperID, const UID& id, const LocalityData& locality);
+	// Summarizes up the commit cost per storage server. Returns the UpdateCommitCostRequest for corresponding SS.
+	UpdateCommitCostRequest refreshCommitCost(double elapsed);
 	int64_t getStorageQueueBytes() const { return lastReply.bytesInput - smoothDurableBytes.smoothTotal(); }
 	int64_t getDurabilityLag() const { return smoothLatestVersion.smoothTotal() - smoothDurableVersion.smoothTotal(); }
 	void update(StorageQueuingMetricsReply const&, Smoother& smoothTotalDurableBytes);
 	void addCommitCost(TransactionTagRef tagName, TransactionCommitCostEstimation const& cost);
+
+	// Determine the ratio (limit / current throughput) for throttling based on write queue size
+	Optional<double> getThrottlingRatio(int64_t storageTargetBytes, int64_t storageSpringBytes) const;
 };
 
 struct TLogQueueInfo {
@@ -103,6 +113,8 @@ struct RatekeeperLimits {
 	int64_t lastDurabilityLag;
 	double durabilityLagLimit;
 
+	double bwLagTarget;
+
 	TransactionPriority priority;
 	std::string context;
 
@@ -115,7 +127,8 @@ struct RatekeeperLimits {
 	                 int64_t logTargetBytes,
 	                 int64_t logSpringBytes,
 	                 double maxVersionDifference,
-	                 int64_t durabilityLagTargetVersions);
+	                 int64_t durabilityLagTargetVersions,
+	                 double bwLagTarget);
 };
 
 class Ratekeeper {
@@ -129,6 +142,18 @@ class Ratekeeper {
 
 		double lastUpdateTime{ 0.0 };
 		double lastTagPushTime{ 0.0 };
+		Version version{ 0 };
+	};
+
+	struct VersionInfo {
+		int64_t totalTransactions;
+		int64_t batchTransactions;
+		double created;
+
+		VersionInfo(int64_t totalTransactions, int64_t batchTransactions, double created)
+		  : totalTransactions(totalTransactions), batchTransactions(batchTransactions), created(created) {}
+
+		VersionInfo() : totalTransactions(0), batchTransactions(0), created(0.0) {}
 	};
 
 	UID id;
@@ -150,12 +175,35 @@ class Ratekeeper {
 
 	std::unique_ptr<class ITagThrottler> tagThrottler;
 
+	// Maps storage server ID to storage server interface
+	std::unordered_map<UID, StorageServerInterface> storageServerInterfaces;
+
 	RatekeeperLimits normalLimits;
 	RatekeeperLimits batchLimits;
 
 	Deque<double> actualTpsHistory;
+	Version maxVersion;
+	double blobWorkerTime;
+	double unblockedAssignmentTime;
+	std::map<Version, Ratekeeper::VersionInfo> version_transactions;
+	std::map<Version, std::pair<double, Optional<double>>> version_recovery;
+	Deque<std::pair<double, Version>> blobWorkerVersionHistory;
 	Optional<Key> remoteDC;
 
+	double getRecoveryDuration(Version ver) {
+		auto it = version_recovery.lower_bound(ver);
+		double recoveryDuration = 0;
+		while (it != version_recovery.end()) {
+			if (it->second.second.present()) {
+				recoveryDuration += it->second.second.get() - it->second.first;
+			} else {
+				recoveryDuration += now() - it->second.first;
+			}
+			++it;
+		}
+		return recoveryDuration;
+	}
+
 	Ratekeeper(UID id, Database db);
 
 	Future<Void> configurationMonitor();
@@ -163,7 +211,6 @@ class Ratekeeper {
 	void updateRate(RatekeeperLimits* limits);
 	Future<Void> refreshStorageServerCommitCosts();
 	Future<Void> monitorServerListChange(PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges);
-	Future<Void> trackEachStorageServer(FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges);
 
 	// SOMEDAY: template trackStorageServerQueueInfo and trackTLogQueueInfo into one function
 	Future<Void> trackStorageServerQueueInfo(StorageServerInterface);
@@ -172,7 +219,10 @@ class Ratekeeper {
 	void tryAutoThrottleTag(TransactionTag, double rate, double busyness, TagThrottledReason);
 	void tryAutoThrottleTag(StorageQueueInfo&, int64_t storageQueue, int64_t storageDurabilityLag);
 	Future<Void> monitorThrottlingChanges();
+	Future<Void> monitorBlobWorkers(Reference<AsyncVar<ServerDBInfo> const> dbInfo);
 
 public:
 	static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo);
 };
+
+#endif // FDBSERVER_RATEKEEPER_H
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/RatekeeperInterface.h b/fdbserver/include/fdbserver/RatekeeperInterface.h
index 9d90a980ef..2ed6775736 100644
--- a/fdbserver/include/fdbserver/RatekeeperInterface.h
+++ b/fdbserver/include/fdbserver/RatekeeperInterface.h
@@ -51,6 +51,8 @@ struct RatekeeperInterface {
 };
 
 struct TransactionCommitCostEstimation {
+	// NOTE: If class variables are changed, counterparts in StorageServerInterface.h:UpdateCommitCostRequest should be
+	// updated too.
 	int opsSum = 0;
 	uint64_t costSum = 0;
 
@@ -76,11 +78,20 @@ struct GetRateInfoReply {
 	double leaseDuration;
 	HealthMetrics healthMetrics;
 
-	Optional<PrioritizedTransactionTagMap<ClientTagThrottleLimits>> throttledTags;
+	// Depending on the value of SERVER_KNOBS->ENFORCE_TAG_THROTTLING_ON_PROXIES,
+	// one of these fields may be populated
+	Optional<PrioritizedTransactionTagMap<ClientTagThrottleLimits>> clientThrottledTags;
+	Optional<PrioritizedTransactionTagMap<double>> proxyThrottledTags;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, transactionRate, batchTransactionRate, leaseDuration, healthMetrics, throttledTags);
+		serializer(ar,
+		           transactionRate,
+		           batchTransactionRate,
+		           leaseDuration,
+		           healthMetrics,
+		           clientThrottledTags,
+		           proxyThrottledTags);
 	}
 };
 
@@ -89,6 +100,7 @@ struct GetRateInfoRequest {
 	UID requesterID;
 	int64_t totalReleasedTransactions;
 	int64_t batchReleasedTransactions;
+	Version version;
 
 	TransactionTagMap<uint64_t> throttledTagCounts;
 	bool detailed;
@@ -98,16 +110,23 @@ struct GetRateInfoRequest {
 	GetRateInfoRequest(UID const& requesterID,
 	                   int64_t totalReleasedTransactions,
 	                   int64_t batchReleasedTransactions,
+	                   Version version,
 	                   TransactionTagMap<uint64_t> throttledTagCounts,
 	                   bool detailed)
 	  : requesterID(requesterID), totalReleasedTransactions(totalReleasedTransactions),
-	    batchReleasedTransactions(batchReleasedTransactions), throttledTagCounts(throttledTagCounts),
+	    batchReleasedTransactions(batchReleasedTransactions), version(version), throttledTagCounts(throttledTagCounts),
 	    detailed(detailed) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(
-		    ar, requesterID, totalReleasedTransactions, batchReleasedTransactions, throttledTagCounts, detailed, reply);
+		serializer(ar,
+		           requesterID,
+		           totalReleasedTransactions,
+		           batchReleasedTransactions,
+		           version,
+		           throttledTagCounts,
+		           detailed,
+		           reply);
 	}
 };
 
diff --git a/fdbserver/include/fdbserver/RemoteIKeyValueStore.actor.h b/fdbserver/include/fdbserver/RemoteIKeyValueStore.actor.h
index 0465e4afbb..e110c480c8 100644
--- a/fdbserver/include/fdbserver/RemoteIKeyValueStore.actor.h
+++ b/fdbserver/include/fdbserver/RemoteIKeyValueStore.actor.h
@@ -155,13 +155,12 @@ struct OpenKVStoreRequest {
 struct IKVSGetValueRequest {
 	constexpr static FileIdentifier file_identifier = 1029439;
 	KeyRef key;
-	IKeyValueStore::ReadType type;
-	Optional<UID> debugID = Optional<UID>();
+	Optional<ReadOptions> options;
 	ReplyPromise<Optional<Value>> reply;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, key, type, debugID, reply);
+		serializer(ar, key, options, reply);
 	}
 };
 
@@ -202,13 +201,12 @@ struct IKVSReadValuePrefixRequest {
 	constexpr static FileIdentifier file_identifier = 1928374;
 	KeyRef key;
 	int maxLength;
-	IKeyValueStore::ReadType type;
-	Optional<UID> debugID = Optional<UID>();
+	Optional<ReadOptions> options;
 	ReplyPromise<Optional<Value>> reply;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, key, maxLength, type, debugID, reply);
+		serializer(ar, key, maxLength, options, reply);
 	}
 };
 
@@ -246,12 +244,12 @@ struct IKVSReadRangeRequest {
 	KeyRangeRef keys;
 	int rowLimit;
 	int byteLimit;
-	IKeyValueStore::ReadType type;
+	Optional<ReadOptions> options;
 	ReplyPromise<IKVSReadRangeReply> reply;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, keys, rowLimit, byteLimit, type, reply);
+		serializer(ar, keys, rowLimit, byteLimit, options, reply);
 	}
 };
 
@@ -402,25 +400,22 @@ struct RemoteIKeyValueStore : public IKeyValueStore {
 		return commitAndGetStorageBytes(this, commitReply);
 	}
 
-	Future<Optional<Value>> readValue(KeyRef key,
-	                                  ReadType type = ReadType::NORMAL,
-	                                  Optional<UID> debugID = Optional<UID>()) override {
-		return readValueImpl(this, IKVSGetValueRequest{ key, type, debugID, ReplyPromise<Optional<Value>>() });
+	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options = Optional<ReadOptions>()) override {
+		return readValueImpl(this, IKVSGetValueRequest{ key, options, ReplyPromise<Optional<Value>>() });
 	}
 
 	Future<Optional<Value>> readValuePrefix(KeyRef key,
 	                                        int maxLength,
-	                                        ReadType type = ReadType::NORMAL,
-	                                        Optional<UID> debugID = Optional<UID>()) override {
+	                                        Optional<ReadOptions> options = Optional<ReadOptions>()) override {
 		return interf.readValuePrefix.getReply(
-		    IKVSReadValuePrefixRequest{ key, maxLength, type, debugID, ReplyPromise<Optional<Value>>() });
+		    IKVSReadValuePrefixRequest{ key, maxLength, options, ReplyPromise<Optional<Value>>() });
 	}
 
 	Future<RangeResult> readRange(KeyRangeRef keys,
 	                              int rowLimit = 1 << 30,
 	                              int byteLimit = 1 << 30,
-	                              ReadType type = ReadType::NORMAL) override {
-		IKVSReadRangeRequest req{ keys, rowLimit, byteLimit, type, ReplyPromise<IKVSReadRangeReply>() };
+	                              Optional<ReadOptions> options = Optional<ReadOptions>()) override {
+		IKVSReadRangeRequest req{ keys, rowLimit, byteLimit, options, ReplyPromise<IKVSReadRangeReply>() };
 		return fmap([](const IKVSReadRangeReply& reply) { return reply.toRangeResult(); },
 		            interf.readRange.getReply(req));
 	}
@@ -449,7 +444,7 @@ struct RemoteIKeyValueStore : public IKeyValueStore {
 			when(wait(delay(SERVER_KNOBS->REMOTE_KV_STORE_MAX_INIT_DURATION))) {
 				TraceEvent(SevError, "RemoteIKVSInitTooLong")
 				    .detail("TimeLimit", SERVER_KNOBS->REMOTE_KV_STORE_MAX_INIT_DURATION);
-				throw please_reboot_remote_kv_store();
+				throw please_reboot_kv_store(); // this will reboot the kv store
 			}
 		}
 		state Future<Void> connectionCheckingDelay = delay(FLOW_KNOBS->FAILURE_DETECTION_DELAY);
@@ -463,21 +458,21 @@ struct RemoteIKeyValueStore : public IKeyValueStore {
 				if (e.isError())
 					throw e.getError();
 				else
-					return e.get();
+					return Never();
 			}
 			when(int res = wait(returnCode)) {
 				TraceEvent(res != 0 ? SevError : SevInfo, "SpawnedProcessDied").detail("Res", res);
 				if (res)
-					throw please_reboot_remote_kv_store(); // this will reboot the worker
+					throw please_reboot_kv_store(); // this will reboot the kv store
 				else
-					return Void();
+					return Never();
 			}
 			when(wait(connectionCheckingDelay)) {
 				// for the corner case where the child process stuck and waitpid also does not give update on it
 				// In this scenario, we need to manually reboot the storage engine process
 				if (IFailureMonitor::failureMonitor().getState(childAddr).isFailed()) {
 					TraceEvent(SevError, "RemoteKVStoreConnectionStuck").log();
-					throw please_reboot_remote_kv_store(); // this will reboot the worker
+					throw please_reboot_kv_store(); // this will reboot the kv store
 				}
 				connectionCheckingDelay = delay(FLOW_KNOBS->FAILURE_DETECTION_DELAY);
 			}
diff --git a/fdbserver/include/fdbserver/ResolverInterface.h b/fdbserver/include/fdbserver/ResolverInterface.h
index 51110e5c01..b746aa8fe6 100644
--- a/fdbserver/include/fdbserver/ResolverInterface.h
+++ b/fdbserver/include/fdbserver/ResolverInterface.h
@@ -139,10 +139,10 @@ struct ResolveTransactionBatchRequest {
 		           transactions,
 		           txnStateTransactions,
 		           reply,
-		           arena,
 		           debugID,
 		           writtenTags,
-		           spanContext);
+		           spanContext,
+		           arena);
 	}
 };
 
diff --git a/fdbserver/include/fdbserver/RestoreApplier.actor.h b/fdbserver/include/fdbserver/RestoreApplier.actor.h
index 464b8fea8a..da370bc62c 100644
--- a/fdbserver/include/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/include/fdbserver/RestoreApplier.actor.h
@@ -371,7 +371,7 @@ struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreAppl
 		nodeIndex = assignedIndex;
 
 		// Q: Why do we need to initMetric?
-		// version.initMetric(LiteralStringRef("RestoreApplier.Version"), cc.id);
+		// version.initMetric("RestoreApplier.Version"_sr, cc.id);
 
 		role = RestoreRole::Applier;
 	}
diff --git a/fdbserver/include/fdbserver/RestoreCommon.actor.h b/fdbserver/include/fdbserver/RestoreCommon.actor.h
index b647bb5326..4b4dc6f0f9 100644
--- a/fdbserver/include/fdbserver/RestoreCommon.actor.h
+++ b/fdbserver/include/fdbserver/RestoreCommon.actor.h
@@ -112,13 +112,7 @@ public:
 
 		Tuple pack() const {
 			// fprintf(stderr, "Filename:%s\n", fileName.c_str());
-			return Tuple()
-			    .append(version)
-			    .append(StringRef(fileName))
-			    .append(isRange)
-			    .append(fileSize)
-			    .append(blockSize)
-			    .append(endVersion);
+			return Tuple::makeTuple(version, fileName, (int)isRange, fileSize, blockSize, endVersion);
 		}
 		static RestoreFile unpack(Tuple const& t) {
 			RestoreFile r;
@@ -190,17 +184,16 @@ struct RestoreFileFR {
 	int partitionId = -1; // Partition ID (Log Router Tag ID) for mutation files.
 
 	Tuple pack() const {
-		return Tuple()
-		    .append(version)
-		    .append(StringRef(fileName))
-		    .append(isRange)
-		    .append(fileSize)
-		    .append(blockSize)
-		    .append(endVersion)
-		    .append(beginVersion)
-		    .append(cursor)
-		    .append(fileIndex)
-		    .append(partitionId);
+		return Tuple::makeTuple(version,
+		                        fileName,
+		                        (int)isRange,
+		                        fileSize,
+		                        blockSize,
+		                        endVersion,
+		                        beginVersion,
+		                        cursor,
+		                        fileIndex,
+		                        partitionId);
 	}
 	static RestoreFileFR unpack(Tuple const& t) {
 		RestoreFileFR r;
diff --git a/fdbserver/include/fdbserver/ServerDBInfo.actor.h b/fdbserver/include/fdbserver/ServerDBInfo.actor.h
index 1a7a9a4211..8f3b598e71 100644
--- a/fdbserver/include/fdbserver/ServerDBInfo.actor.h
+++ b/fdbserver/include/fdbserver/ServerDBInfo.actor.h
@@ -31,6 +31,7 @@
 #include "fdbserver/LogSystemConfig.h"
 #include "fdbserver/RatekeeperInterface.h"
 #include "fdbserver/BlobManagerInterface.h"
+#include "fdbclient/ConsistencyScanInterface.actor.h"
 #include "fdbserver/RecoveryState.h"
 #include "fdbserver/LatencyBandConfig.h"
 #include "fdbserver/WorkerInterface.actor.h"
@@ -50,6 +51,7 @@ struct ServerDBInfo {
 	Optional<RatekeeperInterface> ratekeeper;
 	Optional<BlobManagerInterface> blobManager;
 	Optional<EncryptKeyProxyInterface> encryptKeyProxy;
+	Optional<ConsistencyScanInterface> consistencyScan;
 	std::vector<ResolverInterface> resolvers;
 	DBRecoveryCount
 	    recoveryCount; // A recovery count from DBCoreState.  A successful cluster recovery increments it twice;
@@ -83,6 +85,7 @@ struct ServerDBInfo {
 		           ratekeeper,
 		           blobManager,
 		           encryptKeyProxy,
+		           consistencyScan,
 		           resolvers,
 		           recoveryCount,
 		           recoveryState,
diff --git a/fdbserver/include/fdbserver/ShardsAffectedByTeamFailure.h b/fdbserver/include/fdbserver/ShardsAffectedByTeamFailure.h
new file mode 100644
index 0000000000..3f85fefb7a
--- /dev/null
+++ b/fdbserver/include/fdbserver/ShardsAffectedByTeamFailure.h
@@ -0,0 +1,132 @@
+/*
+ * ShardsAffectedByTeamFailure.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FOUNDATIONDB_SHARDSAFFECTEDBYTEAMFAILURE_H
+#define FOUNDATIONDB_SHARDSAFFECTEDBYTEAMFAILURE_H
+
+#include "flow/FastRef.h"
+#include "flow/IRandom.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/KeyRangeMap.h"
+
+class ShardsAffectedByTeamFailure : public ReferenceCounted<ShardsAffectedByTeamFailure> {
+public:
+	ShardsAffectedByTeamFailure() {}
+
+	enum class CheckMode { Normal = 0, ForceCheck, ForceNoCheck };
+
+	struct Team {
+		std::vector<UID> servers; // sorted
+		bool primary;
+
+		Team() : primary(true) {}
+		Team(std::vector<UID> const& servers, bool primary) : servers(servers), primary(primary) {}
+
+		bool operator<(const Team& r) const {
+			if (servers == r.servers)
+				return primary < r.primary;
+			return servers < r.servers;
+		}
+		bool operator>(const Team& r) const { return r < *this; }
+		bool operator<=(const Team& r) const { return !(*this > r); }
+		bool operator>=(const Team& r) const { return !(*this < r); }
+		bool operator==(const Team& r) const { return servers == r.servers && primary == r.primary; }
+		bool operator!=(const Team& r) const { return !(*this == r); }
+
+		bool hasServer(const UID& id) const { return std::find(servers.begin(), servers.end(), id) != servers.end(); }
+
+		bool removeServer(const UID& id) {
+			auto oldSize = servers.size();
+			servers.erase(std::remove(servers.begin(), servers.end(), id), servers.end());
+			return oldSize != servers.size();
+		}
+
+		std::string toString() const { return describe(servers); };
+	};
+
+	// This tracks the data distribution on the data distribution server so that teamTrackers can
+	//   relocate the right shards when a team is degraded.
+
+	// The following are important to make sure that failure responses don't revert splits or merges:
+	//   - The shards boundaries in the two data structures reflect "queued" RelocateShard requests
+	//       (i.e. reflects the desired set of shards being tracked by dataDistributionTracker,
+	//       rather than the status quo).  These boundaries are modified in defineShard and the content
+	//       of what servers correspond to each shard is a copy or union of the shards already there
+	//   - The teams associated with each shard reflect either the sources for non-moving shards
+	//       or the destination team for in-flight shards (the change is atomic with respect to team selection).
+	//       moveShard() changes the servers associated with a shard and will never adjust the shard
+	//       boundaries. If a move is received for a shard that has been redefined (the exact shard is
+	//       no longer in the map), the servers will be set for all contained shards and added to all
+	//       intersecting shards.
+
+	int getNumberOfShards(UID ssID) const;
+	std::vector<KeyRange> getShardsFor(Team team) const;
+	bool hasShards(Team team) const;
+
+	// The first element of the pair is either the source for non-moving shards or the destination team for in-flight
+	// shards The second element of the pair is all previous sources for in-flight shards
+	std::pair<std::vector<Team>, std::vector<Team>> getTeamsFor(KeyRangeRef keys);
+	// Shard boundaries are modified in defineShard and the content of what servers correspond to each shard is a copy
+	// or union of the shards already there
+	void defineShard(KeyRangeRef keys);
+	// moveShard never change the shard boundary but just change the team value
+	void moveShard(KeyRangeRef keys, std::vector<Team> destinationTeam);
+	// finishMove never change the shard boundary but just clear the old source team value
+	void finishMove(KeyRangeRef keys);
+	// a convenient function for (defineShard, moveShard, finishMove) pipeline
+	void assignRangeToTeams(KeyRangeRef keys, const std::vector<Team>& destinationTeam);
+	void check() const;
+	void setCheckMode(CheckMode);
+
+	PromiseStream<KeyRange> restartShardTracker;
+
+private:
+	struct OrderByTeamKey {
+		bool operator()(const std::pair<Team, KeyRange>& lhs, const std::pair<Team, KeyRange>& rhs) const {
+			if (lhs.first < rhs.first)
+				return true;
+			if (lhs.first > rhs.first)
+				return false;
+			return lhs.second.begin < rhs.second.begin;
+		}
+	};
+
+	CheckMode checkMode = CheckMode::Normal;
+	KeyRangeMap<std::pair<std::vector<Team>, std::vector<Team>>>
+	    shard_teams; // A shard can be affected by the failure of multiple teams if it is a queued merge, or when
+	                 // usable_regions > 1
+	std::set<std::pair<Team, KeyRange>, OrderByTeamKey> team_shards;
+	std::map<UID, int> storageServerShards;
+
+	// only erase from team_shards
+	void erase(Team team, KeyRange const& range);
+	// only insert into team_shards
+	void insert(Team team, KeyRange const& range);
+
+	bool removeFailedServerForSingleRange(ShardsAffectedByTeamFailure::Team& team, const UID& id, KeyRangeRef keys);
+
+public:
+	// return the iterator that traversing all ranges
+	auto getAllRanges() const -> decltype(shard_teams)::ConstRanges;
+	// get total shards count
+	size_t getNumberOfShards() const;
+	void removeFailedServerForRange(KeyRangeRef keys, const UID& serverID);
+};
+
+#endif // FOUNDATIONDB_SHARDSAFFECTEDBYTEAMFAILURE_H
diff --git a/fdbserver/include/fdbserver/SimEncryptKmsProxy.actor.h b/fdbserver/include/fdbserver/SimEncryptKmsProxy.actor.h
index 588f6d4cfb..dcbaa4969c 100644
--- a/fdbserver/include/fdbserver/SimEncryptKmsProxy.actor.h
+++ b/fdbserver/include/fdbserver/SimEncryptKmsProxy.actor.h
@@ -98,7 +98,7 @@ struct SimGetEncryptKeysByKeyIdsReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, encryptKeyDetails);
+		serializer(ar, encryptKeyDetails, arena);
 	}
 };
 
@@ -127,7 +127,7 @@ struct SimGetEncryptKeyByDomainIdReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, encryptKeyDetails);
+		serializer(ar, encryptKeyDetails, arena);
 	}
 };
 
diff --git a/fdbserver/include/fdbserver/SimKmsConnector.h b/fdbserver/include/fdbserver/SimKmsConnector.h
index c26ba808b4..51c39ebf8f 100644
--- a/fdbserver/include/fdbserver/SimKmsConnector.h
+++ b/fdbserver/include/fdbserver/SimKmsConnector.h
@@ -22,8 +22,8 @@
 #define SIM_KMS_CONNECTOR_H
 #pragma once
 
+#include "fdbclient/BlobCipher.h"
 #include "fdbserver/KmsConnector.h"
-#include "flow/BlobCipher.h"
 
 class SimKmsConnector : public KmsConnector {
 public:
diff --git a/fdbserver/include/fdbserver/SimpleConfigConsumer.h b/fdbserver/include/fdbserver/SimpleConfigConsumer.h
index 8701f51d60..a52d40f29d 100644
--- a/fdbserver/include/fdbserver/SimpleConfigConsumer.h
+++ b/fdbserver/include/fdbserver/SimpleConfigConsumer.h
@@ -37,7 +37,9 @@ public:
 	                     double pollingInterval,
 	                     Optional<double> compactionInterval);
 	~SimpleConfigConsumer();
+	Future<Void> readSnapshot(ConfigBroadcaster& broadcaster) override;
 	Future<Void> consume(ConfigBroadcaster& broadcaster) override;
+	void allowSpecialCaseRollforward() override { ASSERT(false); }
 	UID getID() const override;
 
 public: // Testing
diff --git a/fdbserver/include/fdbserver/SimulatedCluster.h b/fdbserver/include/fdbserver/SimulatedCluster.h
index d92d3dbb98..2637b52a25 100644
--- a/fdbserver/include/fdbserver/SimulatedCluster.h
+++ b/fdbserver/include/fdbserver/SimulatedCluster.h
@@ -28,4 +28,16 @@ void setupAndRun(std::string const& dataFolder,
                  bool const& restoring,
                  std::string const& whitelistBinPath);
 
+class BasicTestConfig {
+public:
+	int minimumReplication = 0;
+	int logAntiQuorum = -1;
+	// Set true to simplify simulation configs for easier debugging
+	bool simpleConfig = false;
+	Optional<int> desiredTLogCount, commitProxyCount, grvProxyCount, resolverCount, storageEngineType, machineCount,
+	    coordinators;
+};
+
+DatabaseConfiguration generateNormalDatabaseConfiguration(const BasicTestConfig& testConfig);
+
 #endif
diff --git a/fdbserver/include/fdbserver/StorageMetrics.h b/fdbserver/include/fdbserver/StorageMetrics.h
index 1ecf102e3e..f0f7008cc4 100644
--- a/fdbserver/include/fdbserver/StorageMetrics.h
+++ b/fdbserver/include/fdbserver/StorageMetrics.h
@@ -46,51 +46,8 @@ struct StorageMetricSample {
 
 	explicit StorageMetricSample(int64_t metricUnitsPerSample) : metricUnitsPerSample(metricUnitsPerSample) {}
 
-	int64_t getEstimate(KeyRangeRef keys) const { return sample.sumRange(keys.begin, keys.end); }
-	KeyRef splitEstimate(KeyRangeRef range, int64_t offset, bool front = true) const {
-		auto fwd_split = sample.index(front ? sample.sumTo(sample.lower_bound(range.begin)) + offset
-		                                    : sample.sumTo(sample.lower_bound(range.end)) - offset);
-
-		if (fwd_split == sample.end() || *fwd_split >= range.end)
-			return range.end;
-
-		if (!front && *fwd_split <= range.begin)
-			return range.begin;
-
-		auto bck_split = fwd_split;
-
-		// Butterfly search - start at midpoint then go in both directions.
-		while ((fwd_split != sample.end() && *fwd_split < range.end) ||
-		       (bck_split != sample.begin() && *bck_split > range.begin)) {
-			if (bck_split != sample.begin() && *bck_split > range.begin) {
-				auto it = bck_split;
-				bck_split.decrementNonEnd();
-
-				KeyRef split = keyBetween(KeyRangeRef(
-				    bck_split != sample.begin() ? std::max<KeyRef>(*bck_split, range.begin) : range.begin, *it));
-				if (!front || (getEstimate(KeyRangeRef(range.begin, split)) > 0 &&
-				               split.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT))
-					return split;
-			}
-
-			if (fwd_split != sample.end() && *fwd_split < range.end) {
-				auto it = fwd_split;
-				++it;
-
-				KeyRef split = keyBetween(
-				    KeyRangeRef(*fwd_split, it != sample.end() ? std::min<KeyRef>(*it, range.end) : range.end));
-				if (front || (getEstimate(KeyRangeRef(split, range.end)) > 0 &&
-				              split.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT))
-					return split;
-
-				fwd_split = it;
-			}
-		}
-
-		// If we didn't return above, we didn't find anything.
-		TraceEvent(SevWarn, "CannotSplitLastSampleKey").detail("Range", range).detail("Offset", offset);
-		return front ? range.end : range.begin;
-	}
+	int64_t getEstimate(KeyRangeRef keys) const;
+	KeyRef splitEstimate(KeyRangeRef range, int64_t offset, bool front = true) const;
 };
 
 struct TransientStorageMetricSample : StorageMetricSample {
@@ -98,83 +55,18 @@ struct TransientStorageMetricSample : StorageMetricSample {
 
 	explicit TransientStorageMetricSample(int64_t metricUnitsPerSample) : StorageMetricSample(metricUnitsPerSample) {}
 
-	// Returns the sampled metric value (possibly 0, possibly increased by the sampling factor)
-	int64_t addAndExpire(KeyRef key, int64_t metric, double expiration) {
-		int64_t x = add(key, metric);
-		if (x)
-			queue.emplace_back(expiration, std::make_pair(*sample.find(key), -x));
-		return x;
-	}
+	int64_t addAndExpire(KeyRef key, int64_t metric, double expiration);
 
-	// FIXME: both versions of erase are broken, because they do not remove items in the queue with will subtract a
-	// metric from the value sometime in the future
-	int64_t erase(KeyRef key) {
-		auto it = sample.find(key);
-		if (it == sample.end())
-			return 0;
-		int64_t x = sample.getMetric(it);
-		sample.erase(it);
-		return x;
-	}
-	void erase(KeyRangeRef keys) { sample.erase(keys.begin, keys.end); }
+	int64_t erase(KeyRef key);
+	void erase(KeyRangeRef keys);
 
-	void poll(KeyRangeMap<std::vector<PromiseStream<StorageMetrics>>>& waitMap, StorageMetrics m) {
-		double now = ::now();
-		while (queue.size() && queue.front().first <= now) {
-			KeyRef key = queue.front().second.first;
-			int64_t delta = queue.front().second.second;
-			ASSERT(delta != 0);
+	void poll(KeyRangeMap<std::vector<PromiseStream<StorageMetrics>>>& waitMap, StorageMetrics m);
 
-			if (sample.addMetric(key, delta) == 0)
-				sample.erase(key);
-
-			StorageMetrics deltaM = m * delta;
-			auto v = waitMap[key];
-			for (int i = 0; i < v.size(); i++) {
-				TEST(true); // TransientStorageMetricSample poll update
-				v[i].send(deltaM);
-			}
-
-			queue.pop_front();
-		}
-	}
-
-	void poll() {
-		double now = ::now();
-		while (queue.size() && queue.front().first <= now) {
-			KeyRef key = queue.front().second.first;
-			int64_t delta = queue.front().second.second;
-			ASSERT(delta != 0);
-
-			if (sample.addMetric(key, delta) == 0)
-				sample.erase(key);
-
-			queue.pop_front();
-		}
-	}
+	void poll();
 
 private:
-	bool roll(KeyRef key, int64_t metric) const {
-		return deterministicRandom()->random01() <
-		       (double)metric / metricUnitsPerSample; //< SOMEDAY: Better randomInt64?
-	}
-
-	int64_t add(KeyRef key, int64_t metric) {
-		if (!metric)
-			return 0;
-		int64_t mag = metric < 0 ? -metric : metric;
-
-		if (mag < metricUnitsPerSample) {
-			if (!roll(key, mag))
-				return 0;
-			metric = metric < 0 ? -metricUnitsPerSample : metricUnitsPerSample;
-		}
-
-		if (sample.addMetric(key, metric) == 0)
-			sample.erase(key);
-
-		return metric;
-	}
+	bool roll(KeyRef key, int64_t metric) const;
+	int64_t add(KeyRef key, int64_t metric);
 };
 
 struct StorageServerMetrics {
@@ -190,131 +82,23 @@ struct StorageServerMetrics {
 	    bandwidthSample(SERVER_KNOBS->BANDWIDTH_UNITS_PER_SAMPLE),
 	    bytesReadSample(SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE) {}
 
-	// Get the current estimated metrics for the given keys
-	StorageMetrics getMetrics(KeyRangeRef const& keys) const {
-		StorageMetrics result;
-		result.bytes = byteSample.getEstimate(keys);
-		result.bytesPerKSecond =
-		    bandwidthSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-		result.iosPerKSecond =
-		    iopsSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-		result.bytesReadPerKSecond =
-		    bytesReadSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-		return result;
-	}
+	StorageMetrics getMetrics(KeyRangeRef const& keys) const;
 
-	// Called when metrics should change (IO for a given key)
-	// Notifies waiting WaitMetricsRequests through waitMetricsMap, and updates metricsAverageQueue and metricsSampleMap
-	void notify(KeyRef key, StorageMetrics& metrics) {
-		ASSERT(metrics.bytes == 0); // ShardNotifyMetrics
-		if (g_network->isSimulated()) {
-			TEST(metrics.bytesPerKSecond != 0); // ShardNotifyMetrics bytes
-			TEST(metrics.iosPerKSecond != 0); // ShardNotifyMetrics ios
-			TEST(metrics.bytesReadPerKSecond != 0); // ShardNotifyMetrics bytesRead
-		}
+	void notify(KeyRef key, StorageMetrics& metrics);
 
-		double expire = now() + SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL;
+	void notifyBytesReadPerKSecond(KeyRef key, int64_t in);
 
-		StorageMetrics notifyMetrics;
-
-		if (metrics.bytesPerKSecond)
-			notifyMetrics.bytesPerKSecond = bandwidthSample.addAndExpire(key, metrics.bytesPerKSecond, expire) *
-			                                SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-		if (metrics.iosPerKSecond)
-			notifyMetrics.iosPerKSecond = iopsSample.addAndExpire(key, metrics.iosPerKSecond, expire) *
-			                              SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-		if (metrics.bytesReadPerKSecond)
-			notifyMetrics.bytesReadPerKSecond = bytesReadSample.addAndExpire(key, metrics.bytesReadPerKSecond, expire) *
-			                                    SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-		if (!notifyMetrics.allZero()) {
-			auto& v = waitMetricsMap[key];
-			for (int i = 0; i < v.size(); i++) {
-				if (g_network->isSimulated()) {
-					TEST(true); // shard notify metrics
-				}
-				// ShardNotifyMetrics
-				v[i].send(notifyMetrics);
-			}
-		}
-	}
-
-	// Due to the fact that read sampling will be called on all reads, use this specialized function to avoid overhead
-	// around branch misses and unnecessary stack allocation which eventually addes up under heavy load.
-	void notifyBytesReadPerKSecond(KeyRef key, int64_t in) {
-		double expire = now() + SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL;
-		int64_t bytesReadPerKSecond =
-		    bytesReadSample.addAndExpire(key, in, expire) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-		if (bytesReadPerKSecond > 0) {
-			StorageMetrics notifyMetrics;
-			notifyMetrics.bytesReadPerKSecond = bytesReadPerKSecond;
-			auto& v = waitMetricsMap[key];
-			for (int i = 0; i < v.size(); i++) {
-				TEST(true); // ShardNotifyMetrics
-				v[i].send(notifyMetrics);
-			}
-		}
-	}
-
-	// Called by StorageServerDisk when the size of a key in byteSample changes, to notify WaitMetricsRequest
-	// Should not be called for keys past allKeys.end
 	void notifyBytes(RangeMap<Key, std::vector<PromiseStream<StorageMetrics>>, KeyRangeRef>::iterator shard,
-	                 int64_t bytes) {
-		ASSERT(shard.end() <= allKeys.end);
+	                 int64_t bytes);
 
-		StorageMetrics notifyMetrics;
-		notifyMetrics.bytes = bytes;
-		for (int i = 0; i < shard.value().size(); i++) {
-			TEST(true); // notifyBytes
-			shard.value()[i].send(notifyMetrics);
-		}
-	}
+	void notifyBytes(KeyRef key, int64_t bytes);
 
-	// Called by StorageServerDisk when the size of a key in byteSample changes, to notify WaitMetricsRequest
-	void notifyBytes(KeyRef key, int64_t bytes) {
-		if (key >= allKeys.end) // Do not notify on changes to internal storage server state
-			return;
+	void notifyNotReadable(KeyRangeRef keys);
 
-		notifyBytes(waitMetricsMap.rangeContaining(key), bytes);
-	}
-
-	// Called when a range of keys becomes unassigned (and therefore not readable), to notify waiting
-	// WaitMetricsRequests (also other types of wait
-	//   requests in the future?)
-	void notifyNotReadable(KeyRangeRef keys) {
-		auto rs = waitMetricsMap.intersectingRanges(keys);
-		for (auto r = rs.begin(); r != rs.end(); ++r) {
-			auto& v = r->value();
-			TEST(v.size()); // notifyNotReadable() sending errors to intersecting ranges
-			for (int n = 0; n < v.size(); n++)
-				v[n].sendError(wrong_shard_server());
-		}
-	}
-
-	// Called periodically (~1 sec intervals) to remove older IOs from the averages
-	// Removes old entries from metricsAverageQueue, updates metricsSampleMap accordingly, and notifies
-	//   WaitMetricsRequests through waitMetricsMap.
-	void poll() {
-		{
-			StorageMetrics m;
-			m.bytesPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-			bandwidthSample.poll(waitMetricsMap, m);
-		}
-		{
-			StorageMetrics m;
-			m.iosPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-			iopsSample.poll(waitMetricsMap, m);
-		}
-		{
-			StorageMetrics m;
-			m.bytesReadPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-			bytesReadSample.poll(waitMetricsMap, m);
-		}
-		// bytesSample doesn't need polling because we never call addExpire() on it
-	}
+	void poll();
 
 	// static void waitMetrics( StorageServerMetrics* const& self, WaitMetricsRequest const& req );
 
-	// This function can run on untrusted user data.  We must validate all divisions carefully.
 	KeyRef getSplitKey(int64_t remaining,
 	                   int64_t estimated,
 	                   int64_t limits,
@@ -325,276 +109,32 @@ struct StorageServerMetrics {
 	                   double divisor,
 	                   KeyRef const& lastKey,
 	                   KeyRef const& key,
-	                   bool hasUsed) const {
-		ASSERT(remaining >= 0);
-		ASSERT(limits > 0);
-		ASSERT(divisor > 0);
+	                   bool hasUsed) const;
 
-		if (limits < infinity / 2) {
-			int64_t expectedSize;
-			if (isLastShard || remaining > estimated) {
-				double remaining_divisor = (double(remaining) / limits) + 0.5;
-				expectedSize = remaining / remaining_divisor;
-			} else {
-				// If we are here, then estimated >= remaining >= 0
-				double estimated_divisor = (double(estimated) / limits) + 0.5;
-				expectedSize = remaining / estimated_divisor;
-			}
-
-			if (remaining > expectedSize) {
-				// This does the conversion from native units to bytes using the divisor.
-				double offset = (expectedSize - used) / divisor;
-				if (offset <= 0)
-					return hasUsed ? lastKey : key;
-				return sample.splitEstimate(
-				    KeyRangeRef(lastKey, key),
-				    offset * ((1.0 - SERVER_KNOBS->SPLIT_JITTER_AMOUNT) +
-				              2 * deterministicRandom()->random01() * SERVER_KNOBS->SPLIT_JITTER_AMOUNT));
-			}
-		}
-
-		return key;
-	}
-
-	void splitMetrics(SplitMetricsRequest req) const {
-		int minSplitBytes = req.minSplitBytes.present() ? req.minSplitBytes.get() : SERVER_KNOBS->MIN_SHARD_BYTES;
-		try {
-			SplitMetricsReply reply;
-			KeyRef lastKey = req.keys.begin;
-			StorageMetrics used = req.used;
-			StorageMetrics estimated = req.estimated;
-			StorageMetrics remaining = getMetrics(req.keys) + used;
-
-			//TraceEvent("SplitMetrics").detail("Begin", req.keys.begin).detail("End", req.keys.end).detail("Remaining", remaining.bytes).detail("Used", used.bytes).detail("MinSplitBytes", minSplitBytes);
-
-			while (true) {
-				if (remaining.bytes < 2 * minSplitBytes)
-					break;
-				KeyRef key = req.keys.end;
-				bool hasUsed = used.bytes != 0 || used.bytesPerKSecond != 0 || used.iosPerKSecond != 0;
-				key = getSplitKey(remaining.bytes,
-				                  estimated.bytes,
-				                  req.limits.bytes,
-				                  used.bytes,
-				                  req.limits.infinity,
-				                  req.isLastShard,
-				                  byteSample,
-				                  1,
-				                  lastKey,
-				                  key,
-				                  hasUsed);
-				if (used.bytes < minSplitBytes)
-					key = std::max(
-					    key, byteSample.splitEstimate(KeyRangeRef(lastKey, req.keys.end), minSplitBytes - used.bytes));
-				key = getSplitKey(remaining.iosPerKSecond,
-				                  estimated.iosPerKSecond,
-				                  req.limits.iosPerKSecond,
-				                  used.iosPerKSecond,
-				                  req.limits.infinity,
-				                  req.isLastShard,
-				                  iopsSample,
-				                  SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS,
-				                  lastKey,
-				                  key,
-				                  hasUsed);
-				key = getSplitKey(remaining.bytesPerKSecond,
-				                  estimated.bytesPerKSecond,
-				                  req.limits.bytesPerKSecond,
-				                  used.bytesPerKSecond,
-				                  req.limits.infinity,
-				                  req.isLastShard,
-				                  bandwidthSample,
-				                  SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS,
-				                  lastKey,
-				                  key,
-				                  hasUsed);
-				ASSERT(key != lastKey || hasUsed);
-				if (key == req.keys.end)
-					break;
-				reply.splits.push_back_deep(reply.splits.arena(), key);
-
-				StorageMetrics diff = (getMetrics(KeyRangeRef(lastKey, key)) + used);
-				remaining -= diff;
-				estimated -= diff;
-
-				used = StorageMetrics();
-				lastKey = key;
-			}
-
-			reply.used = getMetrics(KeyRangeRef(lastKey, req.keys.end)) + used;
-			req.reply.send(reply);
-		} catch (Error& e) {
-			req.reply.sendError(e);
-		}
-	}
+	void splitMetrics(SplitMetricsRequest req) const;
 
 	void getStorageMetrics(GetStorageMetricsRequest req,
 	                       StorageBytes sb,
 	                       double bytesInputRate,
 	                       int64_t versionLag,
-	                       double lastUpdate) const {
-		GetStorageMetricsReply rep;
-
-		// SOMEDAY: make bytes dynamic with hard disk space
-		rep.load = getMetrics(allKeys);
-
-		if (sb.free < 1e9) {
-			TraceEvent(SevWarn, "PhysicalDiskMetrics")
-			    .suppressFor(60.0)
-			    .detail("Free", sb.free)
-			    .detail("Total", sb.total)
-			    .detail("Available", sb.available)
-			    .detail("Load", rep.load.bytes);
-		}
-
-		rep.available.bytes = sb.available;
-		rep.available.iosPerKSecond = 10e6;
-		rep.available.bytesPerKSecond = 100e9;
-		rep.available.bytesReadPerKSecond = 100e9;
-
-		rep.capacity.bytes = sb.total;
-		rep.capacity.iosPerKSecond = 10e6;
-		rep.capacity.bytesPerKSecond = 100e9;
-		rep.capacity.bytesReadPerKSecond = 100e9;
-
-		rep.bytesInputRate = bytesInputRate;
-
-		rep.versionLag = versionLag;
-		rep.lastUpdate = lastUpdate;
-
-		req.reply.send(rep);
-	}
+	                       double lastUpdate) const;
 
 	Future<Void> waitMetrics(WaitMetricsRequest req, Future<Void> delay);
 
-	// Given a read hot shard, this function will divide the shard into chunks and find those chunks whose
-	// readBytes/sizeBytes exceeds the `readDensityRatio`. Please make sure to run unit tests
-	// `StorageMetricsSampleTests.txt` after change made.
 	std::vector<ReadHotRangeWithMetrics> getReadHotRanges(KeyRangeRef shard,
 	                                                      double readDensityRatio,
 	                                                      int64_t baseChunkSize,
-	                                                      int64_t minShardReadBandwidthPerKSeconds) const {
-		std::vector<ReadHotRangeWithMetrics> toReturn;
+	                                                      int64_t minShardReadBandwidthPerKSeconds) const;
 
-		double shardSize = (double)byteSample.getEstimate(shard);
-		int64_t shardReadBandwidth = bytesReadSample.getEstimate(shard);
-		if (shardReadBandwidth * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS <=
-		    minShardReadBandwidthPerKSeconds) {
-			return toReturn;
-		}
-		if (shardSize <= baseChunkSize) {
-			// Shard is small, use it as is
-			if (bytesReadSample.getEstimate(shard) > (readDensityRatio * shardSize)) {
-				toReturn.emplace_back(shard,
-				                      bytesReadSample.getEstimate(shard) / shardSize,
-				                      bytesReadSample.getEstimate(shard) /
-				                          SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL);
-			}
-			return toReturn;
-		}
-		KeyRef beginKey = shard.begin;
-		auto endKey =
-		    byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + baseChunkSize);
-		while (endKey != byteSample.sample.end()) {
-			if (*endKey > shard.end) {
-				endKey = byteSample.sample.lower_bound(shard.end);
-				if (*endKey == beginKey) {
-					// No need to increment endKey since otherwise it would stuck here forever.
-					break;
-				}
-			}
-			if (*endKey == beginKey) {
-				++endKey;
-				continue;
-			}
-			if (bytesReadSample.getEstimate(KeyRangeRef(beginKey, *endKey)) >
-			    (readDensityRatio * std::max(baseChunkSize, byteSample.getEstimate(KeyRangeRef(beginKey, *endKey))))) {
-				auto range = KeyRangeRef(beginKey, *endKey);
-				if (!toReturn.empty() && toReturn.back().keys.end == range.begin) {
-					// in case two consecutive chunks both are over the ratio, merge them.
-					range = KeyRangeRef(toReturn.back().keys.begin, *endKey);
-					toReturn.pop_back();
-				}
-				toReturn.emplace_back(
-				    range,
-				    (double)bytesReadSample.getEstimate(range) / std::max(baseChunkSize, byteSample.getEstimate(range)),
-				    bytesReadSample.getEstimate(range) / SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL);
-			}
-			beginKey = *endKey;
-			endKey = byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) +
-			                                 baseChunkSize);
-		}
-		return toReturn;
-	}
+	void getReadHotRanges(ReadHotSubRangeRequest req) const;
 
-	void getReadHotRanges(ReadHotSubRangeRequest req) const {
-		ReadHotSubRangeReply reply;
-		auto _ranges = getReadHotRanges(req.keys,
-		                                SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO,
-		                                SERVER_KNOBS->READ_HOT_SUB_RANGE_CHUNK_SIZE,
-		                                SERVER_KNOBS->SHARD_READ_HOT_BANDWIDTH_MIN_PER_KSECONDS);
-		reply.readHotRanges = VectorRef(_ranges.data(), _ranges.size());
-		req.reply.send(reply);
-	}
+	std::vector<KeyRef> getSplitPoints(KeyRangeRef range, int64_t chunkSize, Optional<Key> prefixToRemove) const;
 
-	std::vector<KeyRef> getSplitPoints(KeyRangeRef range, int64_t chunkSize, Optional<Key> prefixToRemove) const {
-		std::vector<KeyRef> toReturn;
-		KeyRef beginKey = range.begin;
-		IndexedSet<Key, int64_t>::const_iterator endKey =
-		    byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + chunkSize);
-		while (endKey != byteSample.sample.end()) {
-			if (*endKey > range.end) {
-				break;
-			}
-			if (*endKey == beginKey) {
-				++endKey;
-				continue;
-			}
-			KeyRef splitPoint = *endKey;
-			if (prefixToRemove.present()) {
-				splitPoint = splitPoint.removePrefix(prefixToRemove.get());
-			}
-			toReturn.push_back(splitPoint);
-			beginKey = *endKey;
-			endKey =
-			    byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + chunkSize);
-		}
-		return toReturn;
-	}
-
-	void getSplitPoints(SplitRangeRequest req, Optional<Key> prefix) const {
-		SplitRangeReply reply;
-		KeyRangeRef range = req.keys;
-		if (prefix.present()) {
-			range = range.withPrefix(prefix.get(), req.arena);
-		}
-		std::vector<KeyRef> points = getSplitPoints(range, req.chunkSize, prefix);
-
-		reply.splitPoints.append_deep(reply.splitPoints.arena(), points.data(), points.size());
-		req.reply.send(reply);
-	}
+	void getSplitPoints(SplitRangeRequest req, Optional<Key> prefix) const;
 
 private:
-	static void collapse(KeyRangeMap<int>& map, KeyRef const& key) {
-		auto range = map.rangeContaining(key);
-		if (range == map.ranges().begin() || range == map.ranges().end())
-			return;
-		int value = range->value();
-		auto prev = range;
-		--prev;
-		if (prev->value() != value)
-			return;
-		KeyRange keys = KeyRangeRef(prev->begin(), range->end());
-		map.insert(keys, value);
-	}
-
-	static void add(KeyRangeMap<int>& map, KeyRangeRef const& keys, int delta) {
-		auto rs = map.modify(keys);
-		for (auto r = rs.begin(); r != rs.end(); ++r)
-			r->value() += delta;
-		collapse(map, keys.begin);
-		collapse(map, keys.end);
-	}
+	static void collapse(KeyRangeMap<int>& map, KeyRef const& key);
+	static void add(KeyRangeMap<int>& map, KeyRangeRef const& keys, int delta);
 };
 
 // Contains information about whether or not a key-value pair should be included in a byte sample
diff --git a/fdbserver/include/fdbserver/TCInfo.h b/fdbserver/include/fdbserver/TCInfo.h
index 1abc393c0b..4911658bb2 100644
--- a/fdbserver/include/fdbserver/TCInfo.h
+++ b/fdbserver/include/fdbserver/TCInfo.h
@@ -20,8 +20,11 @@
 
 #pragma once
 
+#include "fdbclient/SystemData.h"
 #include "fdbclient/Tenant.h"
-#include "fdbserver/DDTeamCollection.h"
+#include "fdbrpc/ReplicationTypes.h"
+#include "fdbserver/DataDistributionTeam.h"
+
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 
@@ -29,6 +32,7 @@ class TCTeamInfo;
 class TCTenantInfo;
 class TCMachineInfo;
 class TCMachineTeamInfo;
+class DDTeamCollection;
 
 class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	friend class TCServerInfoImpl;
@@ -257,8 +261,8 @@ public:
 	TCTenantInfo(TenantInfo tinfo, Key prefix) : m_tenantInfo(tinfo), m_prefix(prefix) {}
 	std::vector<Reference<TCTeamInfo>>& teams() { return m_tenantTeams; }
 
-	TenantName name() { return m_tenantInfo.name.get(); }
-	std::string prefixDesc() { return m_prefix.printable(); }
+	TenantName name() const { return m_tenantInfo.name.get(); }
+	std::string prefixDesc() const { return m_prefix.printable(); }
 
 	void addTeam(TCTeamInfo team);
 	void removeTeam(TCTeamInfo team);
diff --git a/fdbserver/include/fdbserver/TLogInterface.h b/fdbserver/include/fdbserver/TLogInterface.h
index 9da4ecedd4..cc376fc595 100644
--- a/fdbserver/include/fdbserver/TLogInterface.h
+++ b/fdbserver/include/fdbserver/TLogInterface.h
@@ -189,7 +189,7 @@ struct TLogPeekReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, arena, messages, end, popped, maxKnownVersion, minKnownCommittedVersion, begin, onlySpilled);
+		serializer(ar, messages, end, popped, maxKnownVersion, minKnownCommittedVersion, begin, onlySpilled, arena);
 	}
 };
 
@@ -328,10 +328,10 @@ struct TLogCommitRequest {
 		           minKnownCommittedVersion,
 		           messages,
 		           reply,
-		           arena,
 		           debugID,
 		           tLogCount,
-		           spanContext);
+		           spanContext,
+		           arena);
 	}
 };
 
diff --git a/fdbserver/include/fdbserver/TSSMappingUtil.actor.h b/fdbserver/include/fdbserver/TSSMappingUtil.actor.h
index d9d0743147..6f764d10f9 100644
--- a/fdbserver/include/fdbserver/TSSMappingUtil.actor.h
+++ b/fdbserver/include/fdbserver/TSSMappingUtil.actor.h
@@ -28,6 +28,7 @@
 #elif !defined(TSS_MAPPING_UTIL_SERVER_H)
 #define TSS_MAPPING_UTIL_SERVER_H
 
+#include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
diff --git a/fdbserver/include/fdbserver/TagThrottler.h b/fdbserver/include/fdbserver/TagThrottler.h
index 830c2fea06..5e00e7be46 100644
--- a/fdbserver/include/fdbserver/TagThrottler.h
+++ b/fdbserver/include/fdbserver/TagThrottler.h
@@ -40,6 +40,10 @@ public:
 	// Also, erase expired tags
 	virtual PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() = 0;
 
+	// For each tag and priority combination, return the throughput limit for the cluster
+	// (to be shared across all GRV proxies)
+	virtual PrioritizedTransactionTagMap<double> getProxyRates(int numProxies) = 0;
+
 	virtual int64_t autoThrottleCount() const = 0;
 	virtual uint32_t busyReadTagCount() const = 0;
 	virtual uint32_t busyWriteTagCount() const = 0;
@@ -62,6 +66,7 @@ public:
 	void addRequests(TransactionTag tag, int count) override;
 	uint64_t getThrottledTagChangeId() const override;
 	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() override;
+	PrioritizedTransactionTagMap<double> getProxyRates(int numProxies) override { throw not_implemented(); }
 	int64_t autoThrottleCount() const override;
 	uint32_t busyReadTagCount() const override;
 	uint32_t busyWriteTagCount() const override;
@@ -80,15 +85,19 @@ public:
 	Future<Void> monitorThrottlingChanges() override;
 	void addRequests(TransactionTag tag, int count) override;
 	uint64_t getThrottledTagChangeId() const override;
-	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() override;
+
 	int64_t autoThrottleCount() const override;
 	uint32_t busyReadTagCount() const override;
 	uint32_t busyWriteTagCount() const override;
 	int64_t manualThrottleCount() const override;
 	bool isAutoThrottlingEnabled() const override;
-	Future<Void> tryUpdateAutoThrottling(StorageQueueInfo const&) override;
 
-	// testing only
+	Future<Void> tryUpdateAutoThrottling(StorageQueueInfo const&) override;
+	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() override;
+	PrioritizedTransactionTagMap<double> getProxyRates(int numProxies) override;
+
+	// Testing only:
 public:
 	void setQuota(TransactionTagRef, ThrottleApi::TagQuotaValue const&);
+	void removeQuota(TransactionTagRef);
 };
diff --git a/fdbserver/include/fdbserver/TenantCache.h b/fdbserver/include/fdbserver/TenantCache.h
index 90c9ee2675..b15bdcbf7a 100644
--- a/fdbserver/include/fdbserver/TenantCache.h
+++ b/fdbserver/include/fdbserver/TenantCache.h
@@ -18,17 +18,26 @@
  * limitations under the License.
  */
 
+#pragma once
+
 #include "fdbclient/FDBTypes.h"
+#include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/Tenant.h"
-#include "fdbserver/DDTeamCollection.h"
 #include "fdbserver/TCInfo.h"
 #include "flow/IRandom.h"
 #include "flow/IndexedSet.h"
+#include "flow/flow.h"
 #include <limits>
 #include <string>
 
 typedef Map<KeyRef, Reference<TCTenantInfo>> TenantMapByPrefix;
 
+struct TenantCacheTenantCreated {
+	KeyRange keys;
+	Promise<bool> reply;
+	TenantCacheTenantCreated(Key prefix) { keys = prefixRange(prefix); }
+};
+
 class TenantCache : public ReferenceCounted<TenantCache> {
 	friend class TenantCacheImpl;
 	friend class TenantCacheUnitTest;
@@ -62,11 +71,15 @@ public:
 		generation = deterministicRandom()->randomUInt32();
 	}
 
-	Future<Void> build(Database cx);
+	PromiseStream<TenantCacheTenantCreated> tenantCreationSignal;
+
+	Future<Void> build();
 
 	Future<Void> monitorTenantMap();
 
 	std::string desc() const;
 
 	bool isTenantKey(KeyRef key) const;
+
+	Optional<Reference<TCTenantInfo>> tenantOwning(KeyRef key) const;
 };
diff --git a/fdbserver/include/fdbserver/TesterInterface.actor.h b/fdbserver/include/fdbserver/TesterInterface.actor.h
index 7d049eac3d..b48bd66415 100644
--- a/fdbserver/include/fdbserver/TesterInterface.actor.h
+++ b/fdbserver/include/fdbserver/TesterInterface.actor.h
@@ -66,6 +66,7 @@ struct WorkloadRequest {
 	double databasePingDelay;
 	int64_t sharedRandomNumber;
 	bool useDatabase;
+	bool runFailureWorkloads = true;
 	Optional<TenantNameRef> defaultTenant;
 
 	// The vector of option lists are to construct compound workloads.  If there
@@ -98,6 +99,7 @@ struct WorkloadRequest {
 		           clientCount,
 		           reply,
 		           defaultTenant,
+		           runFailureWorkloads,
 		           arena);
 	}
 };
@@ -132,7 +134,8 @@ ACTOR Future<Void> runTests(
     LocalityData locality = LocalityData(),
     UnitTestParameters testOptions = UnitTestParameters(),
     Optional<TenantName> defaultTenant = Optional<TenantName>(),
-    Standalone<VectorRef<TenantNameRef>> tenantsToCreate = Standalone<VectorRef<TenantNameRef>>());
+    Standalone<VectorRef<TenantNameRef>> tenantsToCreate = Standalone<VectorRef<TenantNameRef>>(),
+    bool restartingTest = false);
 
 #include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbserver/include/fdbserver/VFSAsync.h b/fdbserver/include/fdbserver/VFSAsync.h
index d5568b41fe..2a0ac8caeb 100644
--- a/fdbserver/include/fdbserver/VFSAsync.h
+++ b/fdbserver/include/fdbserver/VFSAsync.h
@@ -68,7 +68,7 @@ struct VFSAsyncFile {
 		// Error code is only checked for non-zero because the SQLite API error code after an injected error
 		// may not match the error code returned by VFSAsyncFile when the inject error occurred.
 		bool e = g_network->global(INetwork::enSQLiteInjectedError) != (flowGlobalType)0;
-		bool f = g_simulator.checkInjectedCorruption();
+		bool f = g_simulator->checkInjectedCorruption();
 		TraceEvent("VFSCheckInjectedError")
 		    .detail("InjectedIOError", e)
 		    .detail("InjectedCorruption", f)
diff --git a/fdbserver/include/fdbserver/WorkerInterface.actor.h b/fdbserver/include/fdbserver/WorkerInterface.actor.h
index dd73704f5f..83dc7c16dd 100644
--- a/fdbserver/include/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/include/fdbserver/WorkerInterface.actor.h
@@ -27,10 +27,11 @@
 
 #include "fdbserver/BackupInterface.h"
 #include "fdbserver/DataDistributorInterface.h"
-#include "fdbserver/EncryptKeyProxyInterface.h"
+#include "fdbclient/EncryptKeyProxyInterface.h"
 #include "fdbserver/MasterInterface.h"
 #include "fdbserver/TLogInterface.h"
 #include "fdbserver/RatekeeperInterface.h"
+#include "fdbclient/ConsistencyScanInterface.actor.h"
 #include "fdbserver/BlobManagerInterface.h"
 #include "fdbserver/ResolverInterface.h"
 #include "fdbclient/BlobWorkerInterface.h"
@@ -57,6 +58,7 @@ struct WorkerInterface {
 	RequestStream<struct InitializeRatekeeperRequest> ratekeeper;
 	RequestStream<struct InitializeBlobManagerRequest> blobManager;
 	RequestStream<struct InitializeBlobWorkerRequest> blobWorker;
+	RequestStream<struct InitializeConsistencyScanRequest> consistencyScan;
 	RequestStream<struct InitializeResolverRequest> resolver;
 	RequestStream<struct InitializeStorageRequest> storage;
 	RequestStream<struct InitializeLogRouterRequest> logRouter;
@@ -74,7 +76,6 @@ struct WorkerInterface {
 	RequestStream<struct WorkerSnapRequest> workerSnapReq;
 	RequestStream<struct UpdateServerDBInfoRequest> updateServerDBInfo;
 
-	ConfigBroadcastInterface configBroadcastInterface;
 	TesterInterface testerInterface;
 
 	UID id() const { return tLog.getEndpoint().token; }
@@ -113,6 +114,7 @@ struct WorkerInterface {
 		           ratekeeper,
 		           blobManager,
 		           blobWorker,
+		           consistencyScan,
 		           resolver,
 		           storage,
 		           logRouter,
@@ -128,8 +130,7 @@ struct WorkerInterface {
 		           workerSnapReq,
 		           backup,
 		           encryptKeyProxy,
-		           updateServerDBInfo,
-		           configBroadcastInterface);
+		           updateServerDBInfo);
 	}
 };
 
@@ -140,7 +141,7 @@ struct WorkerDetails {
 	bool degraded;
 	bool recoveredDiskFiles;
 
-	WorkerDetails() : degraded(false) {}
+	WorkerDetails() : degraded(false), recoveredDiskFiles(false) {}
 	WorkerDetails(const WorkerInterface& interf, ProcessClass processClass, bool degraded, bool recoveredDiskFiles)
 	  : interf(interf), processClass(processClass), degraded(degraded), recoveredDiskFiles(recoveredDiskFiles) {}
 
@@ -148,7 +149,7 @@ struct WorkerDetails {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, interf, processClass, degraded);
+		serializer(ar, interf, processClass, degraded, recoveredDiskFiles);
 	}
 };
 
@@ -430,14 +431,16 @@ struct RegisterWorkerRequest {
 	Optional<RatekeeperInterface> ratekeeperInterf;
 	Optional<BlobManagerInterface> blobManagerInterf;
 	Optional<EncryptKeyProxyInterface> encryptKeyProxyInterf;
+	Optional<ConsistencyScanInterface> consistencyScanInterf;
 	Standalone<VectorRef<StringRef>> issues;
 	std::vector<NetworkAddress> incompatiblePeers;
 	ReplyPromise<RegisterWorkerReply> reply;
 	bool degraded;
-	Version lastSeenKnobVersion;
-	ConfigClassSet knobConfigClassSet;
+	Optional<Version> lastSeenKnobVersion;
+	Optional<ConfigClassSet> knobConfigClassSet;
 	bool requestDbInfo;
 	bool recoveredDiskFiles;
+	ConfigBroadcastInterface configBroadcastInterface;
 
 	RegisterWorkerRequest()
 	  : priorityInfo(ProcessClass::UnsetFit, false, ClusterControllerPriorityInfo::FitnessUnknown), degraded(false) {}
@@ -450,14 +453,17 @@ struct RegisterWorkerRequest {
 	                      Optional<RatekeeperInterface> rkInterf,
 	                      Optional<BlobManagerInterface> bmInterf,
 	                      Optional<EncryptKeyProxyInterface> ekpInterf,
+	                      Optional<ConsistencyScanInterface> csInterf,
 	                      bool degraded,
-	                      Version lastSeenKnobVersion,
-	                      ConfigClassSet knobConfigClassSet,
-	                      bool recoveredDiskFiles)
+	                      Optional<Version> lastSeenKnobVersion,
+	                      Optional<ConfigClassSet> knobConfigClassSet,
+	                      bool recoveredDiskFiles,
+	                      ConfigBroadcastInterface configBroadcastInterface)
 	  : wi(wi), initialClass(initialClass), processClass(processClass), priorityInfo(priorityInfo),
 	    generation(generation), distributorInterf(ddInterf), ratekeeperInterf(rkInterf), blobManagerInterf(bmInterf),
-	    encryptKeyProxyInterf(ekpInterf), degraded(degraded), lastSeenKnobVersion(lastSeenKnobVersion),
-	    knobConfigClassSet(knobConfigClassSet), requestDbInfo(false), recoveredDiskFiles(recoveredDiskFiles) {}
+	    encryptKeyProxyInterf(ekpInterf), consistencyScanInterf(csInterf), degraded(degraded),
+	    lastSeenKnobVersion(lastSeenKnobVersion), knobConfigClassSet(knobConfigClassSet), requestDbInfo(false),
+	    recoveredDiskFiles(recoveredDiskFiles), configBroadcastInterface(configBroadcastInterface) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -471,6 +477,7 @@ struct RegisterWorkerRequest {
 		           ratekeeperInterf,
 		           blobManagerInterf,
 		           encryptKeyProxyInterf,
+		           consistencyScanInterf,
 		           issues,
 		           incompatiblePeers,
 		           reply,
@@ -478,7 +485,8 @@ struct RegisterWorkerRequest {
 		           lastSeenKnobVersion,
 		           knobConfigClassSet,
 		           requestDbInfo,
-		           recoveredDiskFiles);
+		           recoveredDiskFiles,
+		           configBroadcastInterface);
 	}
 };
 
@@ -502,13 +510,14 @@ struct UpdateWorkerHealthRequest {
 	constexpr static FileIdentifier file_identifier = 5789927;
 	NetworkAddress address;
 	std::vector<NetworkAddress> degradedPeers;
+	std::vector<NetworkAddress> disconnectedPeers;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
 		if constexpr (!is_fb_function<Ar>) {
 			ASSERT(ar.protocolVersion().isValid());
 		}
-		serializer(ar, address, degradedPeers);
+		serializer(ar, address, degradedPeers, disconnectedPeers);
 	}
 };
 
@@ -726,6 +735,19 @@ struct InitializeRatekeeperRequest {
 	}
 };
 
+struct InitializeConsistencyScanRequest {
+	constexpr static FileIdentifier file_identifier = 3104275;
+	UID reqId;
+	ReplyPromise<ConsistencyScanInterface> reply;
+
+	InitializeConsistencyScanRequest() {}
+	explicit InitializeConsistencyScanRequest(UID uid) : reqId(uid) {}
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, reqId, reply);
+	}
+};
+
 struct InitializeBlobManagerRequest {
 	constexpr static FileIdentifier file_identifier = 2567474;
 	UID reqId;
@@ -988,6 +1010,7 @@ struct Role {
 	static const Role COORDINATOR;
 	static const Role BACKUP;
 	static const Role ENCRYPT_KEY_PROXY;
+	static const Role CONSISTENCYSCAN;
 
 	std::string roleName;
 	std::string abbreviation;
@@ -1025,6 +1048,8 @@ struct Role {
 			return BACKUP;
 		case ProcessClass::EncryptKeyProxy:
 			return ENCRYPT_KEY_PROXY;
+		case ProcessClass::ConsistencyScan:
+			return CONSISTENCYSCAN;
 		case ProcessClass::Worker:
 			return WORKER;
 		case ProcessClass::NoRole:
@@ -1092,6 +1117,7 @@ ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ei, Reference<
 class IKeyValueStore;
 class ServerCoordinators;
 class IDiskQueue;
+class IPageEncryptionKeyProvider;
 ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
                                  StorageServerInterface ssi,
                                  Tag seedTag,
@@ -1100,7 +1126,8 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
                                  Version tssSeedVersion,
                                  ReplyPromise<InitializeStorageReply> recruitReply,
                                  Reference<AsyncVar<ServerDBInfo> const> db,
-                                 std::string folder);
+                                 std::string folder,
+                                 Reference<IPageEncryptionKeyProvider> encryptionKeyProvider);
 ACTOR Future<Void> storageServer(
     IKeyValueStore* persistentData,
     StorageServerInterface ssi,
@@ -1108,7 +1135,8 @@ ACTOR Future<Void> storageServer(
     std::string folder,
     Promise<Void> recovered,
     Reference<IClusterConnectionRecord>
-        connRecord); // changes pssi->id() to be the recovered ID); // changes pssi->id() to be the recovered ID
+        connRecord, // changes pssi->id() to be the recovered ID); // changes pssi->id() to be the recovered ID
+    Reference<IPageEncryptionKeyProvider> encryptionKeyProvider);
 ACTOR Future<Void> masterServer(MasterInterface mi,
                                 Reference<AsyncVar<ServerDBInfo> const> db,
                                 Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
@@ -1143,6 +1171,7 @@ ACTOR Future<Void> logRouter(TLogInterface interf,
                              Reference<AsyncVar<ServerDBInfo> const> db);
 ACTOR Future<Void> dataDistributor(DataDistributorInterface ddi, Reference<AsyncVar<ServerDBInfo> const> db);
 ACTOR Future<Void> ratekeeper(RatekeeperInterface rki, Reference<AsyncVar<ServerDBInfo> const> db);
+ACTOR Future<Void> consistencyScan(ConsistencyScanInterface csInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo);
 ACTOR Future<Void> blobManager(BlobManagerInterface bmi, Reference<AsyncVar<ServerDBInfo> const> db, int64_t epoch);
 ACTOR Future<Void> storageCacheServer(StorageServerInterface interf,
                                       uint16_t id,
@@ -1207,15 +1236,17 @@ ACTOR template <class T>
 Future<T> ioTimeoutError(Future<T> what, double time) {
 	// Before simulation is sped up, IO operations can take a very long time so limit timeouts
 	// to not end until at least time after simulation is sped up.
-	if (g_network->isSimulated() && !g_simulator.speedUpSimulation) {
+	if (g_network->isSimulated() && !g_simulator->speedUpSimulation) {
 		time += std::max(0.0, FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS - now());
 	}
 	Future<Void> end = lowPriorityDelay(time);
 	choose {
-		when(T t = wait(what)) { return t; }
+		when(T t = wait(what)) {
+			return t;
+		}
 		when(wait(end)) {
 			Error err = io_timeout();
-			if (g_network->isSimulated() && !g_simulator.getCurrentProcess()->isReliable()) {
+			if (g_network->isSimulated() && !g_simulator->getCurrentProcess()->isReliable()) {
 				err = err.asInjectedFault();
 			}
 			TraceEvent(SevError, "IoTimeoutError").error(err);
@@ -1231,7 +1262,7 @@ Future<T> ioDegradedOrTimeoutError(Future<T> what,
                                    double degradedTime) {
 	// Before simulation is sped up, IO operations can take a very long time so limit timeouts
 	// to not end until at least time after simulation is sped up.
-	if (g_network->isSimulated() && !g_simulator.speedUpSimulation) {
+	if (g_network->isSimulated() && !g_simulator->speedUpSimulation) {
 		double timeShift = std::max(0.0, FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS - now());
 		errTime += timeShift;
 		degradedTime += timeShift;
@@ -1240,9 +1271,11 @@ Future<T> ioDegradedOrTimeoutError(Future<T> what,
 	if (degradedTime < errTime) {
 		Future<Void> degradedEnd = lowPriorityDelay(degradedTime);
 		choose {
-			when(T t = wait(what)) { return t; }
+			when(T t = wait(what)) {
+				return t;
+			}
 			when(wait(degradedEnd)) {
-				TEST(true); // TLog degraded
+				CODE_PROBE(true, "TLog degraded", probe::func::deduplicate);
 				TraceEvent(SevWarnAlways, "IoDegraded").log();
 				degraded->set(true);
 			}
@@ -1251,10 +1284,12 @@ Future<T> ioDegradedOrTimeoutError(Future<T> what,
 
 	Future<Void> end = lowPriorityDelay(errTime - degradedTime);
 	choose {
-		when(T t = wait(what)) { return t; }
+		when(T t = wait(what)) {
+			return t;
+		}
 		when(wait(end)) {
 			Error err = io_timeout();
-			if (g_network->isSimulated() && !g_simulator.getCurrentProcess()->isReliable()) {
+			if (g_network->isSimulated() && !g_simulator->getCurrentProcess()->isReliable()) {
 				err = err.asInjectedFault();
 			}
 			TraceEvent(SevError, "IoTimeoutError").error(err);
diff --git a/fdbserver/include/fdbserver/workloads/ApiWorkload.h b/fdbserver/include/fdbserver/workloads/ApiWorkload.h
index 4972013f5c..0ab1b17e9b 100644
--- a/fdbserver/include/fdbserver/workloads/ApiWorkload.h
+++ b/fdbserver/include/fdbserver/workloads/ApiWorkload.h
@@ -273,23 +273,23 @@ struct ApiWorkload : TestWorkload {
 
 	ApiWorkload(WorkloadContext const& wcx, int maxClients = -1)
 	  : TestWorkload(wcx), maxClients(maxClients), success(true), transactionFactory(nullptr) {
-		clientPrefixInt = getOption(options, LiteralStringRef("clientId"), clientId);
+		clientPrefixInt = getOption(options, "clientId"_sr, clientId);
 		clientPrefix = format("%010d", clientPrefixInt);
 
-		numKeys = getOption(options, LiteralStringRef("numKeys"), 5000);
-		onlyLowerCase = getOption(options, LiteralStringRef("onlyLowerCase"), false);
-		shortKeysRatio = getOption(options, LiteralStringRef("shortKeysRatio"), 0.5);
-		minShortKeyLength = getOption(options, LiteralStringRef("minShortKeyLength"), 1);
-		maxShortKeyLength = getOption(options, LiteralStringRef("maxShortKeyLength"), 3);
-		minLongKeyLength = getOption(options, LiteralStringRef("minLongKeyLength"), 1);
-		maxLongKeyLength = getOption(options, LiteralStringRef("maxLongKeyLength"), 128);
-		minValueLength = getOption(options, LiteralStringRef("minValueLength"), 1);
-		maxValueLength = getOption(options, LiteralStringRef("maxValueLength"), 10000);
+		numKeys = getOption(options, "numKeys"_sr, 5000);
+		onlyLowerCase = getOption(options, "onlyLowerCase"_sr, false);
+		shortKeysRatio = getOption(options, "shortKeysRatio"_sr, 0.5);
+		minShortKeyLength = getOption(options, "minShortKeyLength"_sr, 1);
+		maxShortKeyLength = getOption(options, "maxShortKeyLength"_sr, 3);
+		minLongKeyLength = getOption(options, "minLongKeyLength"_sr, 1);
+		maxLongKeyLength = getOption(options, "maxLongKeyLength"_sr, 128);
+		minValueLength = getOption(options, "minValueLength"_sr, 1);
+		maxValueLength = getOption(options, "maxValueLength"_sr, 10000);
 
-		useExtraDB = g_network->isSimulated() && g_simulator.extraDB != nullptr;
+		useExtraDB = g_network->isSimulated() && !g_simulator->extraDatabases.empty();
 		if (useExtraDB) {
-			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-			extraDB = Database::createDatabase(extraFile, -1);
+			ASSERT(g_simulator->extraDatabases.size() == 1);
+			extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], wcx.defaultTenant);
 		}
 	}
 
diff --git a/fdbserver/include/fdbserver/workloads/BulkSetup.actor.h b/fdbserver/include/fdbserver/workloads/BulkSetup.actor.h
index 9b4dfca577..390883f46d 100644
--- a/fdbserver/include/fdbserver/workloads/BulkSetup.actor.h
+++ b/fdbserver/include/fdbserver/workloads/BulkSetup.actor.h
@@ -39,10 +39,29 @@
 #include "fdbrpc/simulator.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+template <class T>
+struct sfinae_true : std::true_type {};
+
+template <class T>
+auto testAuthToken(int) -> sfinae_true<decltype(std::declval<T>().setAuthToken(std::declval<Transaction&>()))>;
+template <class>
+auto testAuthToken(long) -> std::false_type;
+
+template <class T>
+struct hasAuthToken : decltype(testAuthToken<T>(0)) {};
+
+template <class T>
+void setAuthToken(T const& self, Transaction& tr) {
+	if constexpr (hasAuthToken<T>::value) {
+		self.setAuthToken(tr);
+	}
+}
+
 ACTOR template <class T>
 Future<bool> checkRangeSimpleValueSize(Database cx, T* workload, uint64_t begin, uint64_t end) {
 	loop {
 		state Transaction tr(cx);
+		setAuthToken(*workload, tr);
 		try {
 			state Standalone<KeyValueRef> firstKV = (*workload)(begin);
 			state Standalone<KeyValueRef> lastKV = (*workload)(end - 1);
@@ -59,10 +78,15 @@ Future<bool> checkRangeSimpleValueSize(Database cx, T* workload, uint64_t begin,
 
 // Returns true if the range was added
 ACTOR template <class T>
-Future<uint64_t> setupRange(Database cx, T* workload, uint64_t begin, uint64_t end) {
+Future<uint64_t> setupRange(Database cx, T* workload, uint64_t begin, uint64_t end, std::vector<TenantName> tenants) {
 	state uint64_t bytesInserted = 0;
 	loop {
-		state Transaction tr(cx);
+		Optional<TenantName> tenant;
+		if (tenants.size() > 0) {
+			tenant = tenants.at(deterministicRandom()->randomInt(0, tenants.size()));
+		}
+		state Transaction tr(cx, tenant);
+		setAuthToken(*workload, tr);
 		try {
 			// if( deterministicRandom()->random01() < 0.001 )
 			//	tr.debugTransaction( deterministicRandom()->randomUniqueID() );
@@ -108,7 +132,8 @@ Future<uint64_t> setupRangeWorker(Database cx,
                                   std::vector<std::pair<uint64_t, uint64_t>>* jobs,
                                   double maxKeyInsertRate,
                                   int keySaveIncrement,
-                                  int actorId) {
+                                  int actorId,
+                                  std::vector<TenantName> tenants) {
 	state double nextStart;
 	state uint64_t loadedRanges = 0;
 	state int lastStoredKeysLoaded = 0;
@@ -118,7 +143,7 @@ Future<uint64_t> setupRangeWorker(Database cx,
 		state std::pair<uint64_t, uint64_t> job = jobs->back();
 		jobs->pop_back();
 		nextStart = now() + (job.second - job.first) / maxKeyInsertRate;
-		uint64_t numBytes = wait(setupRange(cx, workload, job.first, job.second));
+		uint64_t numBytes = wait(setupRange(cx, workload, job.first, job.second, tenants));
 		if (numBytes > 0)
 			loadedRanges++;
 
@@ -128,6 +153,7 @@ Future<uint64_t> setupRangeWorker(Database cx,
 
 			if (keysLoaded - lastStoredKeysLoaded >= keySaveIncrement || jobs->size() == 0) {
 				state Transaction tr(cx);
+				setAuthToken(*workload, tr);
 				try {
 					std::string countKey = format("keycount|%d|%d", workload->clientId, actorId);
 					std::string bytesKey = format("bytesstored|%d|%d", workload->clientId, actorId);
@@ -160,15 +186,31 @@ ACTOR Future<std::vector<std::pair<uint64_t, double>>> trackInsertionCount(Datab
 
 ACTOR template <class T>
 Future<Void> waitForLowInFlight(Database cx, T* workload) {
+	state Future<Void> timeout = delay(600.0);
 	loop {
-		int64_t inFlight = wait(getDataInFlight(cx, workload->dbInfo));
-		TraceEvent("DynamicWarming").detail("InFlight", inFlight);
-		if (inFlight > 1e6) { // Wait for just 1 MB to be in flight
-			wait(delay(1.0));
-		} else {
-			wait(delay(1.0));
-			TraceEvent("DynamicWarmingDone").log();
-			break;
+		try {
+			if (timeout.isReady()) {
+				throw timed_out();
+			}
+
+			int64_t inFlight = wait(getDataInFlight(cx, workload->dbInfo));
+			TraceEvent("DynamicWarming").detail("InFlight", inFlight);
+			if (inFlight > 1e6) { // Wait for just 1 MB to be in flight
+				wait(delay(1.0));
+			} else {
+				wait(delay(1.0));
+				TraceEvent("DynamicWarmingDone").log();
+				break;
+			}
+		} catch (Error& e) {
+			if (e.code() == error_code_attribute_not_found ||
+			    (e.code() == error_code_timed_out && !timeout.isReady())) {
+				// DD may not be initialized yet and attribute "DataInFlight" can be missing
+				wait(delay(1.0));
+			} else {
+				TraceEvent(SevWarn, "WaitForLowInFlightError").error(e);
+				throw;
+			}
 		}
 	}
 	return Void();
@@ -188,7 +230,8 @@ Future<Void> bulkSetup(Database cx,
                        int keySaveIncrement = 0,
                        double keyCheckInterval = 0.1,
                        uint64_t startNodeIdx = 0,
-                       uint64_t endNodeIdx = 0) {
+                       uint64_t endNodeIdx = 0,
+                       std::vector<TenantName> tenants = std::vector<TenantName>()) {
 
 	state std::vector<std::pair<uint64_t, uint64_t>> jobs;
 	state uint64_t startNode = startNodeIdx ? startNodeIdx : (nodeCount * workload->clientId) / workload->clientCount;
@@ -268,7 +311,7 @@ Future<Void> bulkSetup(Database cx,
 		keySaveIncrement = 0;
 
 	for (int j = 0; j < BULK_SETUP_WORKERS; j++)
-		fs.push_back(setupRangeWorker(cx, workload, &jobs, maxWorkerInsertRate, keySaveIncrement, j));
+		fs.push_back(setupRangeWorker(cx, workload, &jobs, maxWorkerInsertRate, keySaveIncrement, j, tenants));
 	try {
 		wait(success(insertionTimes) && waitForAll(fs));
 	} catch (Error& e) {
@@ -294,8 +337,8 @@ Future<Void> bulkSetup(Database cx,
 	// Here we wait for data in flight to go to 0 (this will not work on a database with other users)
 	if (postSetupWarming != 0) {
 		try {
-			wait(delay(5.0) >>
-			     waitForLowInFlight(cx, workload)); // Wait for the data distribution in a small test to start
+			wait(delay(5.0));
+			wait(waitForLowInFlight(cx, workload)); // Wait for the data distribution in a small test to start
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
 				throw;
diff --git a/fdbserver/include/fdbserver/workloads/MetaclusterConsistency.actor.h b/fdbserver/include/fdbserver/workloads/MetaclusterConsistency.actor.h
new file mode 100644
index 0000000000..7b8275d2b7
--- /dev/null
+++ b/fdbserver/include/fdbserver/workloads/MetaclusterConsistency.actor.h
@@ -0,0 +1,363 @@
+
+/*
+ * MetaclusterConsistency.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
+// version.
+#include "fdbclient/FDBOptions.g.h"
+#include "flow/BooleanParam.h"
+#if defined(NO_INTELLISENSE) && !defined(WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_G_H)
+#define WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_G_H
+#include "fdbserver/workloads/MetaclusterConsistency.actor.g.h"
+#elif !defined(WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_H)
+#define WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_H
+
+#include "fdbclient/Metacluster.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
+#include "fdbserver/workloads/TenantConsistency.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+FDB_DECLARE_BOOLEAN_PARAM(AllowPartialMetaclusterOperations);
+
+template <class DB>
+class MetaclusterConsistencyCheck {
+private:
+	Reference<DB> managementDb;
+	AllowPartialMetaclusterOperations allowPartialMetaclusterOperations = AllowPartialMetaclusterOperations::True;
+
+	struct ManagementClusterData {
+		Optional<MetaclusterRegistrationEntry> metaclusterRegistration;
+		std::map<ClusterName, DataClusterMetadata> dataClusters;
+		KeyBackedRangeResult<Tuple> clusterCapacityTuples;
+		KeyBackedRangeResult<std::pair<ClusterName, int64_t>> clusterTenantCounts;
+		KeyBackedRangeResult<Tuple> clusterTenantTuples;
+		KeyBackedRangeResult<Tuple> clusterTenantGroupTuples;
+
+		std::map<TenantName, TenantMapEntry> tenantMap;
+		KeyBackedRangeResult<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups;
+
+		std::map<ClusterName, std::set<TenantName>> clusterTenantMap;
+		std::map<ClusterName, std::set<TenantGroupName>> clusterTenantGroupMap;
+
+		int64_t tenantCount;
+		RangeResult systemTenantSubspaceKeys;
+	};
+
+	ManagementClusterData managementMetadata;
+
+	// Note: this check can only be run on metaclusters with a reasonable number of tenants, as should be
+	// the case with the current metacluster simulation workloads
+	static inline const int metaclusterMaxTenants = 10e6;
+
+	ACTOR static Future<Void> loadManagementClusterMetadata(MetaclusterConsistencyCheck* self) {
+		state Reference<typename DB::TransactionT> managementTr = self->managementDb->createTransaction();
+		state std::vector<std::pair<TenantName, TenantMapEntry>> tenantList;
+
+		loop {
+			try {
+				managementTr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				state typename transaction_future_type<typename DB::TransactionT, RangeResult>::type
+				    systemTenantSubspaceKeysFuture = managementTr->getRange(prefixRange(TenantMetadata::subspace()), 1);
+
+				wait(store(self->managementMetadata.metaclusterRegistration,
+				           MetaclusterMetadata::metaclusterRegistration().get(managementTr)) &&
+				     store(self->managementMetadata.dataClusters,
+				           MetaclusterAPI::listClustersTransaction(
+				               managementTr, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS + 1)) &&
+				     store(self->managementMetadata.clusterCapacityTuples,
+				           MetaclusterAPI::ManagementClusterMetadata::clusterCapacityIndex.getRange(
+				               managementTr, {}, {}, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
+				     store(self->managementMetadata.clusterTenantCounts,
+				           MetaclusterAPI::ManagementClusterMetadata::clusterTenantCount.getRange(
+				               managementTr, {}, {}, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
+				     store(self->managementMetadata.clusterTenantTuples,
+				           MetaclusterAPI::ManagementClusterMetadata::clusterTenantIndex.getRange(
+				               managementTr, {}, {}, metaclusterMaxTenants)) &&
+				     store(self->managementMetadata.clusterTenantGroupTuples,
+				           MetaclusterAPI::ManagementClusterMetadata::clusterTenantGroupIndex.getRange(
+				               managementTr, {}, {}, metaclusterMaxTenants)) &&
+				     store(self->managementMetadata.tenantCount,
+				           MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantCount.getD(
+				               managementTr, Snapshot::False, 0)) &&
+				     store(tenantList,
+				           MetaclusterAPI::listTenantsTransaction(
+				               managementTr, ""_sr, "\xff\xff"_sr, metaclusterMaxTenants)) &&
+				     store(self->managementMetadata.tenantGroups,
+				           MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantGroupMap.getRange(
+				               managementTr, {}, {}, metaclusterMaxTenants)) &&
+				     store(self->managementMetadata.systemTenantSubspaceKeys,
+				           safeThreadFutureToFuture(systemTenantSubspaceKeysFuture)));
+
+				break;
+			} catch (Error& e) {
+				wait(safeThreadFutureToFuture(managementTr->onError(e)));
+			}
+		}
+
+		self->managementMetadata.tenantMap = std::map<TenantName, TenantMapEntry>(tenantList.begin(), tenantList.end());
+
+		for (auto t : self->managementMetadata.clusterTenantTuples.results) {
+			ASSERT(t.size() == 3);
+			TenantName tenantName = t.getString(1);
+			int64_t tenantId = t.getInt(2);
+			ASSERT(tenantId == self->managementMetadata.tenantMap[tenantName].id);
+			self->managementMetadata.clusterTenantMap[t.getString(0)].insert(tenantName);
+		}
+
+		for (auto t : self->managementMetadata.clusterTenantGroupTuples.results) {
+			ASSERT(t.size() == 2);
+			TenantGroupName tenantGroupName = t.getString(1);
+			self->managementMetadata.clusterTenantGroupMap[t.getString(0)].insert(tenantGroupName);
+		}
+
+		return Void();
+	}
+
+	void validateManagementCluster() {
+		ASSERT(managementMetadata.metaclusterRegistration.present());
+		ASSERT(managementMetadata.metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_MANAGEMENT);
+		ASSERT(managementMetadata.metaclusterRegistration.get().id ==
+		           managementMetadata.metaclusterRegistration.get().metaclusterId &&
+		       managementMetadata.metaclusterRegistration.get().name ==
+		           managementMetadata.metaclusterRegistration.get().metaclusterName);
+		ASSERT(managementMetadata.dataClusters.size() <= CLIENT_KNOBS->MAX_DATA_CLUSTERS);
+		ASSERT(managementMetadata.tenantCount <= metaclusterMaxTenants);
+		ASSERT(managementMetadata.clusterCapacityTuples.results.size() <= managementMetadata.dataClusters.size() &&
+		       !managementMetadata.clusterCapacityTuples.more);
+		ASSERT(managementMetadata.clusterTenantCounts.results.size() <= managementMetadata.dataClusters.size() &&
+		       !managementMetadata.clusterTenantCounts.more);
+		ASSERT(managementMetadata.clusterTenantTuples.results.size() == managementMetadata.tenantCount &&
+		       !managementMetadata.clusterTenantTuples.more);
+		ASSERT(managementMetadata.clusterTenantGroupTuples.results.size() <= managementMetadata.tenantCount &&
+		       !managementMetadata.clusterTenantGroupTuples.more);
+		ASSERT(managementMetadata.tenantMap.size() == managementMetadata.tenantCount);
+		ASSERT(managementMetadata.tenantGroups.results.size() <= managementMetadata.tenantCount &&
+		       !managementMetadata.tenantGroups.more);
+		ASSERT(managementMetadata.clusterTenantGroupTuples.results.size() ==
+		       managementMetadata.tenantGroups.results.size());
+
+		// Parse the cluster capacity index. Check that no cluster is represented in the index more than once.
+		std::map<ClusterName, int64_t> clusterAllocatedMap;
+		for (auto t : managementMetadata.clusterCapacityTuples.results) {
+			ASSERT(t.size() == 2);
+			auto result = clusterAllocatedMap.emplace(t.getString(1), t.getInt(0));
+			ASSERT(result.second);
+		}
+
+		// Validate various properties for each data cluster
+		int numFoundInAllocatedMap = 0;
+		int numFoundInTenantGroupMap = 0;
+		for (auto [clusterName, clusterMetadata] : managementMetadata.dataClusters) {
+			// If the cluster has capacity, it should be in the capacity index and have the correct count of
+			// allocated tenants stored there
+			auto allocatedItr = clusterAllocatedMap.find(clusterName);
+			if (!clusterMetadata.entry.hasCapacity()) {
+				ASSERT(allocatedItr == clusterAllocatedMap.end());
+			} else {
+				ASSERT(allocatedItr->second == clusterMetadata.entry.allocated.numTenantGroups);
+				++numFoundInAllocatedMap;
+			}
+
+			// Check that the number of tenant groups in the cluster is smaller than the allocated number of tenant
+			// groups.
+			auto tenantGroupItr = managementMetadata.clusterTenantGroupMap.find(clusterName);
+			if (tenantGroupItr != managementMetadata.clusterTenantGroupMap.end()) {
+				ASSERT(tenantGroupItr->second.size() <= clusterMetadata.entry.allocated.numTenantGroups);
+				++numFoundInTenantGroupMap;
+			}
+		}
+		// Check that we exhausted the cluster capacity index and the cluster tenant group index
+		ASSERT(numFoundInAllocatedMap == clusterAllocatedMap.size());
+		ASSERT(numFoundInTenantGroupMap == managementMetadata.clusterTenantGroupMap.size());
+
+		// Check that our cluster tenant counters match the number of tenants in the cluster index
+		std::map<ClusterName, int64_t> countsMap(managementMetadata.clusterTenantCounts.results.begin(),
+		                                         managementMetadata.clusterTenantCounts.results.end());
+		for (auto [cluster, clusterTenants] : managementMetadata.clusterTenantMap) {
+			auto itr = countsMap.find(cluster);
+			ASSERT((clusterTenants.empty() && itr == countsMap.end()) || itr->second == clusterTenants.size());
+		}
+
+		// Iterate through all tenants and verify related metadata
+		std::map<ClusterName, int> clusterAllocated;
+		std::set<TenantGroupName> processedTenantGroups;
+		for (auto [name, entry] : managementMetadata.tenantMap) {
+			ASSERT(entry.assignedCluster.present());
+
+			// Each tenant should be assigned to the same cluster where it is stored in the cluster tenant index
+			auto clusterItr = managementMetadata.clusterTenantMap.find(entry.assignedCluster.get());
+			ASSERT(clusterItr != managementMetadata.clusterTenantMap.end());
+			ASSERT(clusterItr->second.count(name));
+
+			if (entry.tenantGroup.present()) {
+				// Count the number of tenant groups allocated in each cluster
+				if (processedTenantGroups.insert(entry.tenantGroup.get()).second) {
+					++clusterAllocated[entry.assignedCluster.get()];
+				}
+				// The tenant group should be stored in the same cluster where it is stored in the cluster tenant
+				// group index
+				auto clusterTenantGroupItr = managementMetadata.clusterTenantGroupMap.find(entry.assignedCluster.get());
+				ASSERT(clusterTenantGroupItr != managementMetadata.clusterTenantMap.end());
+				ASSERT(clusterTenantGroupItr->second.count(entry.tenantGroup.get()));
+			} else {
+				// Track the actual tenant group allocation per cluster (a tenant with no group counts against the
+				// allocation)
+				++clusterAllocated[entry.assignedCluster.get()];
+			}
+		}
+
+		// The actual allocation for each cluster should match what is stored in the cluster metadata
+		for (auto [name, allocated] : clusterAllocated) {
+			auto itr = managementMetadata.dataClusters.find(name);
+			ASSERT(itr != managementMetadata.dataClusters.end());
+			ASSERT(allocated == itr->second.entry.allocated.numTenantGroups);
+		}
+
+		// Each tenant group in the tenant group map should be present in the cluster tenant group map
+		// and have the correct cluster assigned to it.
+		for (auto [name, entry] : managementMetadata.tenantGroups.results) {
+			ASSERT(entry.assignedCluster.present());
+			auto clusterItr = managementMetadata.clusterTenantGroupMap.find(entry.assignedCluster.get());
+			ASSERT(clusterItr->second.count(name));
+		}
+
+		// We should not be storing any data in the `\xff` tenant subspace.
+		ASSERT(managementMetadata.systemTenantSubspaceKeys.empty());
+	}
+
+	ACTOR static Future<Void> validateDataCluster(MetaclusterConsistencyCheck* self,
+	                                              ClusterName clusterName,
+	                                              DataClusterMetadata clusterMetadata) {
+		state Reference<IDatabase> dataDb = wait(MetaclusterAPI::openDatabase(clusterMetadata.connectionString));
+		state Reference<ITransaction> dataTr = dataDb->createTransaction();
+
+		state Optional<MetaclusterRegistrationEntry> dataClusterRegistration;
+		state std::vector<std::pair<TenantName, TenantMapEntry>> dataClusterTenantList;
+		state KeyBackedRangeResult<std::pair<TenantGroupName, TenantGroupEntry>> dataClusterTenantGroups;
+
+		state TenantConsistencyCheck<IDatabase> tenantConsistencyCheck(dataDb);
+		wait(tenantConsistencyCheck.run());
+
+		loop {
+			try {
+				dataTr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				wait(store(dataClusterRegistration, MetaclusterMetadata::metaclusterRegistration().get(dataTr)) &&
+				     store(dataClusterTenantList,
+				           TenantAPI::listTenantsTransaction(
+				               dataTr, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)) &&
+				     store(dataClusterTenantGroups,
+				           TenantMetadata::tenantGroupMap().getRange(
+				               dataTr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)));
+
+				break;
+			} catch (Error& e) {
+				wait(safeThreadFutureToFuture(dataTr->onError(e)));
+			}
+		}
+
+		state std::map<TenantName, TenantMapEntry> dataClusterTenantMap(dataClusterTenantList.begin(),
+		                                                                dataClusterTenantList.end());
+
+		ASSERT(dataClusterRegistration.present());
+		ASSERT(dataClusterRegistration.get().clusterType == ClusterType::METACLUSTER_DATA);
+		ASSERT(dataClusterRegistration.get().matches(self->managementMetadata.metaclusterRegistration.get()));
+		ASSERT(dataClusterRegistration.get().name == clusterName);
+		ASSERT(dataClusterRegistration.get().id == clusterMetadata.entry.id);
+
+		auto& expectedTenants = self->managementMetadata.clusterTenantMap[clusterName];
+
+		if (!self->allowPartialMetaclusterOperations) {
+			ASSERT(dataClusterTenantMap.size() == expectedTenants.size());
+		} else {
+			ASSERT(dataClusterTenantMap.size() <= expectedTenants.size());
+			for (auto tenantName : expectedTenants) {
+				if (!dataClusterTenantMap.count(tenantName)) {
+					TenantMapEntry const& metaclusterEntry = self->managementMetadata.tenantMap[tenantName];
+					if (metaclusterEntry.renamePair.present() &&
+					    (metaclusterEntry.tenantState == TenantState::RENAMING_FROM ||
+					     metaclusterEntry.tenantState == TenantState::RENAMING_TO)) {
+						ASSERT(dataClusterTenantMap.count(metaclusterEntry.renamePair.get()));
+					} else {
+						ASSERT(metaclusterEntry.tenantState == TenantState::REGISTERING ||
+						       metaclusterEntry.tenantState == TenantState::REMOVING);
+					}
+				}
+			}
+		}
+
+		for (auto [name, entry] : dataClusterTenantMap) {
+			ASSERT(expectedTenants.count(name));
+			TenantMapEntry const& metaclusterEntry = self->managementMetadata.tenantMap[name];
+			ASSERT(!entry.assignedCluster.present());
+			ASSERT(entry.id == metaclusterEntry.id);
+			ASSERT(entry.encrypted == metaclusterEntry.encrypted);
+
+			ASSERT(entry.tenantState == TenantState::READY);
+			ASSERT(self->allowPartialMetaclusterOperations || metaclusterEntry.tenantState == TenantState::READY);
+			if (metaclusterEntry.tenantState != TenantState::UPDATING_CONFIGURATION &&
+			    metaclusterEntry.tenantState != TenantState::REMOVING) {
+				ASSERT(entry.configurationSequenceNum == metaclusterEntry.configurationSequenceNum);
+			} else {
+				ASSERT(entry.configurationSequenceNum <= metaclusterEntry.configurationSequenceNum);
+			}
+
+			if (entry.configurationSequenceNum == metaclusterEntry.configurationSequenceNum) {
+				ASSERT(entry.tenantGroup == metaclusterEntry.tenantGroup);
+			}
+		}
+
+		auto& expectedTenantGroups = self->managementMetadata.clusterTenantGroupMap[clusterName];
+		ASSERT(dataClusterTenantGroups.results.size() == expectedTenantGroups.size());
+		for (auto [name, entry] : dataClusterTenantGroups.results) {
+			ASSERT(expectedTenantGroups.count(name));
+			ASSERT(!entry.assignedCluster.present());
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> run(MetaclusterConsistencyCheck* self) {
+		state TenantConsistencyCheck<DB> managementTenantConsistencyCheck(self->managementDb);
+		wait(managementTenantConsistencyCheck.run());
+		wait(loadManagementClusterMetadata(self));
+		self->validateManagementCluster();
+
+		state std::vector<Future<Void>> dataClusterChecks;
+		state std::map<ClusterName, DataClusterMetadata>::iterator dataClusterItr;
+		for (auto [clusterName, clusterMetadata] : self->managementMetadata.dataClusters) {
+			dataClusterChecks.push_back(validateDataCluster(self, clusterName, clusterMetadata));
+		}
+		wait(waitForAll(dataClusterChecks));
+
+		return Void();
+	}
+
+public:
+	MetaclusterConsistencyCheck() {}
+	MetaclusterConsistencyCheck(Reference<DB> managementDb,
+	                            AllowPartialMetaclusterOperations allowPartialMetaclusterOperations)
+	  : managementDb(managementDb), allowPartialMetaclusterOperations(allowPartialMetaclusterOperations) {}
+
+	Future<Void> run() { return run(this); }
+};
+
+#endif
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/workloads/ReadWriteWorkload.actor.h b/fdbserver/include/fdbserver/workloads/ReadWriteWorkload.actor.h
index fe33be0213..89bc36f393 100644
--- a/fdbserver/include/fdbserver/workloads/ReadWriteWorkload.actor.h
+++ b/fdbserver/include/fdbserver/workloads/ReadWriteWorkload.actor.h
@@ -86,51 +86,50 @@ struct ReadWriteCommon : KVWorkload {
 	double loadTime, clientBegin;
 
 	explicit ReadWriteCommon(WorkloadContext const& wcx)
-	  : KVWorkload(wcx), totalReadsMetric(LiteralStringRef("ReadWrite.TotalReads")),
-	    totalRetriesMetric(LiteralStringRef("ReadWrite.TotalRetries")), aTransactions("A Transactions"),
-	    bTransactions("B Transactions"), retries("Retries"), latencies(sampleSize), readLatencies(sampleSize),
-	    commitLatencies(sampleSize), GRVLatencies(sampleSize), fullReadLatencies(sampleSize), readLatencyTotal(0),
-	    readLatencyCount(0), loadTime(0.0), clientBegin(0) {
+	  : KVWorkload(wcx), totalReadsMetric("ReadWrite.TotalReads"_sr), totalRetriesMetric("ReadWrite.TotalRetries"_sr),
+	    aTransactions("A Transactions"), bTransactions("B Transactions"), retries("Retries"), latencies(sampleSize),
+	    readLatencies(sampleSize), commitLatencies(sampleSize), GRVLatencies(sampleSize), fullReadLatencies(sampleSize),
+	    readLatencyTotal(0), readLatencyCount(0), loadTime(0.0), clientBegin(0) {
 
-		transactionSuccessMetric.init(LiteralStringRef("ReadWrite.SuccessfulTransaction"));
-		transactionFailureMetric.init(LiteralStringRef("ReadWrite.FailedTransaction"));
-		readMetric.init(LiteralStringRef("ReadWrite.Read"));
+		transactionSuccessMetric.init("ReadWrite.SuccessfulTransaction"_sr);
+		transactionFailureMetric.init("ReadWrite.FailedTransaction"_sr);
+		readMetric.init("ReadWrite.Read"_sr);
 
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 5000.0) / clientCount;
-		double allowedLatency = getOption(options, LiteralStringRef("allowedLatency"), 0.250);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0) / clientCount;
+		double allowedLatency = getOption(options, "allowedLatency"_sr, 0.250);
 		actorCount = ceil(transactionsPerSecond * allowedLatency);
-		actorCount = getOption(options, LiteralStringRef("actorCountPerTester"), actorCount);
+		actorCount = getOption(options, "actorCountPerTester"_sr, actorCount);
 
-		readsPerTransactionA = getOption(options, LiteralStringRef("readsPerTransactionA"), 10);
-		writesPerTransactionA = getOption(options, LiteralStringRef("writesPerTransactionA"), 0);
-		readsPerTransactionB = getOption(options, LiteralStringRef("readsPerTransactionB"), 1);
-		writesPerTransactionB = getOption(options, LiteralStringRef("writesPerTransactionB"), 9);
-		alpha = getOption(options, LiteralStringRef("alpha"), 0.1);
+		readsPerTransactionA = getOption(options, "readsPerTransactionA"_sr, 10);
+		writesPerTransactionA = getOption(options, "writesPerTransactionA"_sr, 0);
+		readsPerTransactionB = getOption(options, "readsPerTransactionB"_sr, 1);
+		writesPerTransactionB = getOption(options, "writesPerTransactionB"_sr, 9);
+		alpha = getOption(options, "alpha"_sr, 0.1);
 
 		valueString = std::string(maxValueBytes, '.');
 		if (nodePrefix > 0) {
 			keyBytes += 16;
 		}
 
-		metricsStart = getOption(options, LiteralStringRef("metricsStart"), 0.0);
-		metricsDuration = getOption(options, LiteralStringRef("metricsDuration"), testDuration);
-		if (getOption(options, LiteralStringRef("discardEdgeMeasurements"), true)) {
+		metricsStart = getOption(options, "metricsStart"_sr, 0.0);
+		metricsDuration = getOption(options, "metricsDuration"_sr, testDuration);
+		if (getOption(options, "discardEdgeMeasurements"_sr, true)) {
 			// discardEdgeMeasurements keeps the metrics from the middle 3/4 of the test
 			metricsStart += testDuration * 0.125;
 			metricsDuration *= 0.75;
 		}
 
-		warmingDelay = getOption(options, LiteralStringRef("warmingDelay"), 0.0);
-		maxInsertRate = getOption(options, LiteralStringRef("maxInsertRate"), 1e12);
-		debugInterval = getOption(options, LiteralStringRef("debugInterval"), 0.0);
-		debugTime = getOption(options, LiteralStringRef("debugTime"), 0.0);
-		enableReadLatencyLogging = getOption(options, LiteralStringRef("enableReadLatencyLogging"), false);
-		periodicLoggingInterval = getOption(options, LiteralStringRef("periodicLoggingInterval"), 5.0);
-		cancelWorkersAtDuration = getOption(options, LiteralStringRef("cancelWorkersAtDuration"), true);
+		warmingDelay = getOption(options, "warmingDelay"_sr, 0.0);
+		maxInsertRate = getOption(options, "maxInsertRate"_sr, 1e12);
+		debugInterval = getOption(options, "debugInterval"_sr, 0.0);
+		debugTime = getOption(options, "debugTime"_sr, 0.0);
+		enableReadLatencyLogging = getOption(options, "enableReadLatencyLogging"_sr, false);
+		periodicLoggingInterval = getOption(options, "periodicLoggingInterval"_sr, 5.0);
+		cancelWorkersAtDuration = getOption(options, "cancelWorkersAtDuration"_sr, true);
 
-		useRYW = getOption(options, LiteralStringRef("useRYW"), false);
-		doSetup = getOption(options, LiteralStringRef("setup"), true);
+		useRYW = getOption(options, "useRYW"_sr, false);
+		doSetup = getOption(options, "setup"_sr, true);
 
 		// Validate that keyForIndex() is monotonic
 		for (int i = 0; i < 30; i++) {
@@ -144,7 +143,7 @@ struct ReadWriteCommon : KVWorkload {
 		}
 
 		std::vector<std::string> insertionCountsToMeasureString =
-		    getOption(options, LiteralStringRef("insertionCountsToMeasure"), std::vector<std::string>());
+		    getOption(options, "insertionCountsToMeasure"_sr, std::vector<std::string>());
 		for (int i = 0; i < insertionCountsToMeasureString.size(); i++) {
 			try {
 				uint64_t count = boost::lexical_cast<uint64_t>(insertionCountsToMeasureString[i]);
diff --git a/fdbserver/include/fdbserver/workloads/TenantConsistency.actor.h b/fdbserver/include/fdbserver/workloads/TenantConsistency.actor.h
new file mode 100644
index 0000000000..86fe3e9c09
--- /dev/null
+++ b/fdbserver/include/fdbserver/workloads/TenantConsistency.actor.h
@@ -0,0 +1,229 @@
+
+/*
+ * TenantConsistency.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
+// version.
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "flow/BooleanParam.h"
+#if defined(NO_INTELLISENSE) && !defined(WORKLOADS_TENANT_CONSISTENCY_ACTOR_G_H)
+#define WORKLOADS_TENANT_CONSISTENCY_ACTOR_G_H
+#include "fdbserver/workloads/TenantConsistency.actor.g.h"
+#elif !defined(WORKLOADS_TENANT_CONSISTENCY_ACTOR_H)
+#define WORKLOADS_TENANT_CONSISTENCY_ACTOR_H
+
+#include "fdbclient/Metacluster.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
+#include "fdbclient/Tenant.h"
+#include "fdbclient/TenantManagement.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+template <class DB>
+class TenantConsistencyCheck {
+private:
+	Reference<DB> db;
+
+	struct TenantData {
+		Optional<MetaclusterRegistrationEntry> metaclusterRegistration;
+		std::map<TenantName, TenantMapEntry> tenantMap;
+		std::map<int64_t, TenantName> tenantIdIndex;
+		int64_t lastTenantId;
+		int64_t tenantCount;
+		std::set<int64_t> tenantTombstones;
+		Optional<TenantTombstoneCleanupData> tombstoneCleanupData;
+		std::map<TenantGroupName, TenantGroupEntry> tenantGroupMap;
+		std::map<TenantGroupName, std::set<TenantName>> tenantGroupIndex;
+
+		std::set<TenantName> tenantsInTenantGroupIndex;
+
+		ClusterType clusterType;
+	};
+
+	TenantData metadata;
+
+	// Note: this check can only be run on metaclusters with a reasonable number of tenants, as should be
+	// the case with the current metacluster simulation workloads
+	static inline const int metaclusterMaxTenants = 10e6;
+
+	ACTOR static Future<Void> loadTenantMetadata(TenantConsistencyCheck* self) {
+		state Reference<typename DB::TransactionT> tr = self->db->createTransaction();
+		state KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenantList;
+		state KeyBackedRangeResult<std::pair<int64_t, TenantName>> tenantIdIndexList;
+		state KeyBackedRangeResult<int64_t> tenantTombstoneList;
+		state KeyBackedRangeResult<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroupList;
+		state KeyBackedRangeResult<Tuple> tenantGroupTenantTuples;
+		state TenantMetadataSpecification* tenantMetadata;
+
+		loop {
+			try {
+				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				wait(store(self->metadata.metaclusterRegistration,
+				           MetaclusterMetadata::metaclusterRegistration().get(tr)));
+
+				self->metadata.clusterType = self->metadata.metaclusterRegistration.present()
+				                                 ? self->metadata.metaclusterRegistration.get().clusterType
+				                                 : ClusterType::STANDALONE;
+
+				if (self->metadata.clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+					tenantMetadata = &MetaclusterAPI::ManagementClusterMetadata::tenantMetadata();
+				} else {
+					tenantMetadata = &TenantMetadata::instance();
+				}
+
+				wait(
+				    store(tenantList, tenantMetadata->tenantMap.getRange(tr, {}, {}, metaclusterMaxTenants)) &&
+				    store(tenantIdIndexList,
+				          tenantMetadata->tenantIdIndex.getRange(tr, {}, {}, metaclusterMaxTenants)) &&
+				    store(self->metadata.lastTenantId, tenantMetadata->lastTenantId.getD(tr, Snapshot::False, -1)) &&
+				    store(self->metadata.tenantCount, tenantMetadata->tenantCount.getD(tr, Snapshot::False, 0)) &&
+				    store(tenantTombstoneList,
+				          tenantMetadata->tenantTombstones.getRange(tr, {}, {}, metaclusterMaxTenants)) &&
+				    store(self->metadata.tombstoneCleanupData, tenantMetadata->tombstoneCleanupData.get(tr)) &&
+				    store(tenantGroupTenantTuples,
+				          tenantMetadata->tenantGroupTenantIndex.getRange(tr, {}, {}, metaclusterMaxTenants)) &&
+				    store(tenantGroupList, tenantMetadata->tenantGroupMap.getRange(tr, {}, {}, metaclusterMaxTenants)));
+
+				break;
+			} catch (Error& e) {
+				wait(safeThreadFutureToFuture(tr->onError(e)));
+			}
+		}
+
+		ASSERT(!tenantList.more);
+		self->metadata.tenantMap =
+		    std::map<TenantName, TenantMapEntry>(tenantList.results.begin(), tenantList.results.end());
+
+		ASSERT(!tenantIdIndexList.more);
+		self->metadata.tenantIdIndex =
+		    std::map<int64_t, TenantName>(tenantIdIndexList.results.begin(), tenantIdIndexList.results.end());
+
+		ASSERT(!tenantTombstoneList.more);
+		self->metadata.tenantTombstones =
+		    std::set<int64_t>(tenantTombstoneList.results.begin(), tenantTombstoneList.results.end());
+
+		ASSERT(!tenantGroupList.more);
+		self->metadata.tenantGroupMap =
+		    std::map<TenantGroupName, TenantGroupEntry>(tenantGroupList.results.begin(), tenantGroupList.results.end());
+
+		for (auto t : tenantGroupTenantTuples.results) {
+			ASSERT(t.size() == 2);
+			TenantGroupName tenantGroupName = t.getString(0);
+			TenantName tenantName = t.getString(1);
+			ASSERT(self->metadata.tenantGroupMap.count(tenantGroupName));
+			ASSERT(self->metadata.tenantMap.count(tenantName));
+			self->metadata.tenantGroupIndex[tenantGroupName].insert(tenantName);
+			ASSERT(self->metadata.tenantsInTenantGroupIndex.insert(tenantName).second);
+		}
+		ASSERT(self->metadata.tenantGroupIndex.size() == self->metadata.tenantGroupMap.size());
+
+		return Void();
+	}
+
+	void validateTenantMetadata() {
+		if (metadata.clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+			ASSERT(metadata.tenantMap.size() <= metaclusterMaxTenants);
+		} else {
+			ASSERT(metadata.tenantMap.size() <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
+		}
+
+		ASSERT(metadata.tenantMap.size() == metadata.tenantCount);
+		ASSERT(metadata.tenantIdIndex.size() == metadata.tenantCount);
+
+		for (auto [tenantName, tenantMapEntry] : metadata.tenantMap) {
+			if (metadata.clusterType != ClusterType::METACLUSTER_DATA) {
+				ASSERT(tenantMapEntry.id <= metadata.lastTenantId);
+			}
+			ASSERT(metadata.tenantIdIndex[tenantMapEntry.id] == tenantName);
+
+			if (tenantMapEntry.tenantGroup.present()) {
+				auto tenantGroupMapItr = metadata.tenantGroupMap.find(tenantMapEntry.tenantGroup.get());
+				ASSERT(tenantGroupMapItr != metadata.tenantGroupMap.end());
+				ASSERT(tenantMapEntry.assignedCluster == tenantGroupMapItr->second.assignedCluster);
+				ASSERT(metadata.tenantGroupIndex[tenantMapEntry.tenantGroup.get()].count(tenantName));
+			} else {
+				ASSERT(!metadata.tenantsInTenantGroupIndex.count(tenantName));
+			}
+
+			if (metadata.clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
+				ASSERT(tenantMapEntry.assignedCluster.present());
+				// If the rename pair is present, it should be in the map and match our current entry
+				if (tenantMapEntry.renamePair.present()) {
+					auto pairMapEntry = metadata.tenantMap[tenantMapEntry.renamePair.get()];
+					ASSERT(pairMapEntry.id == tenantMapEntry.id);
+					ASSERT(pairMapEntry.prefix == tenantMapEntry.prefix);
+					ASSERT(pairMapEntry.encrypted == tenantMapEntry.encrypted);
+					ASSERT(pairMapEntry.configurationSequenceNum == tenantMapEntry.configurationSequenceNum);
+					ASSERT(pairMapEntry.assignedCluster.present());
+					ASSERT(pairMapEntry.assignedCluster.get() == tenantMapEntry.assignedCluster.get());
+					ASSERT(pairMapEntry.renamePair.present());
+					ASSERT(pairMapEntry.renamePair.get() == tenantName);
+					if (tenantMapEntry.tenantState == TenantState::RENAMING_FROM) {
+						ASSERT(pairMapEntry.tenantState == TenantState::RENAMING_TO);
+					} else if (tenantMapEntry.tenantState == TenantState::RENAMING_TO) {
+						ASSERT(pairMapEntry.tenantState == TenantState::RENAMING_FROM);
+					} else if (tenantMapEntry.tenantState == TenantState::REMOVING) {
+						ASSERT(pairMapEntry.tenantState == TenantState::REMOVING);
+					} else {
+						ASSERT(false); // Entry in an invalid state if we have a rename pair
+					}
+				}
+			} else {
+				ASSERT(tenantMapEntry.tenantState == TenantState::READY);
+				ASSERT(!tenantMapEntry.assignedCluster.present());
+				ASSERT(!tenantMapEntry.renamePair.present());
+			}
+		}
+	}
+
+	// Check that the tenant tombstones are properly cleaned up and only present on a metacluster data cluster
+	void checkTenantTombstones() {
+		if (metadata.clusterType == ClusterType::METACLUSTER_DATA) {
+			if (!metadata.tombstoneCleanupData.present()) {
+				ASSERT(metadata.tenantTombstones.empty());
+			}
+
+			if (!metadata.tenantTombstones.empty()) {
+				ASSERT(*metadata.tenantTombstones.begin() >
+				       metadata.tombstoneCleanupData.get().tombstonesErasedThrough);
+			}
+		} else {
+			ASSERT(metadata.tenantTombstones.empty() && !metadata.tombstoneCleanupData.present());
+		}
+	}
+
+	ACTOR static Future<Void> run(TenantConsistencyCheck* self) {
+		wait(loadTenantMetadata(self));
+		self->validateTenantMetadata();
+		self->checkTenantTombstones();
+
+		return Void();
+	}
+
+public:
+	TenantConsistencyCheck() {}
+	TenantConsistencyCheck(Reference<DB> db) : db(db) {}
+
+	Future<Void> run() { return run(this); }
+};
+
+#endif
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/workloads/workloads.actor.h b/fdbserver/include/fdbserver/workloads/workloads.actor.h
index df08911fc7..27770a10c1 100644
--- a/fdbserver/include/fdbserver/workloads/workloads.actor.h
+++ b/fdbserver/include/fdbserver/workloads/workloads.actor.h
@@ -55,6 +55,7 @@ struct WorkloadContext {
 	int64_t sharedRandomNumber;
 	Reference<AsyncVar<struct ServerDBInfo> const> dbInfo;
 	Reference<IClusterConnectionRecord> ccr;
+	Optional<TenantName> defaultTenant;
 
 	WorkloadContext();
 	WorkloadContext(const WorkloadContext&);
@@ -67,7 +68,7 @@ struct TestWorkload : NonCopyable, WorkloadContext, ReferenceCounted<TestWorkloa
 
 	// Subclasses are expected to also have a constructor with this signature (to work with WorkloadFactory<>):
 	explicit TestWorkload(WorkloadContext const& wcx) : WorkloadContext(wcx) {
-		bool runSetup = getOption(options, LiteralStringRef("runSetup"), true);
+		bool runSetup = getOption(options, "runSetup"_sr, true);
 		phases = TestWorkload::EXECUTION | TestWorkload::CHECK | TestWorkload::METRICS;
 		if (runSetup)
 			phases |= TestWorkload::SETUP;
@@ -75,6 +76,7 @@ struct TestWorkload : NonCopyable, WorkloadContext, ReferenceCounted<TestWorkloa
 	virtual ~TestWorkload(){};
 	virtual Future<Void> initialized() { return Void(); }
 	virtual std::string description() const = 0;
+	virtual void disableFailureInjectionWorkloads(std::set<std::string>& out) const;
 	virtual Future<Void> setup(Database const& cx) { return Void(); }
 	virtual Future<Void> start(Database const& cx) = 0;
 	virtual Future<bool> check(Database const& cx) = 0;
@@ -92,6 +94,66 @@ private:
 	virtual void getMetrics(std::vector<PerfMetric>& m) = 0;
 };
 
+struct CompoundWorkload;
+class DeterministicRandom;
+
+struct NoOptions {};
+
+struct FailureInjectionWorkload : TestWorkload {
+	FailureInjectionWorkload(WorkloadContext const&);
+	virtual ~FailureInjectionWorkload() {}
+	virtual void initFailureInjectionMode(DeterministicRandom& random);
+	virtual bool shouldInject(DeterministicRandom& random, const WorkloadRequest& work, const unsigned count) const;
+
+	Future<Void> setupInjectionWorkload(Database const& cx, Future<Void> done);
+	Future<Void> startInjectionWorkload(Database const& cx, Future<Void> done);
+	Future<bool> checkInjectionWorkload(Database const& cx, Future<bool> done);
+};
+
+struct IFailureInjectorFactory : ReferenceCounted<IFailureInjectorFactory> {
+	virtual ~IFailureInjectorFactory() = default;
+	static std::vector<Reference<IFailureInjectorFactory>>& factories() {
+		static std::vector<Reference<IFailureInjectorFactory>> _factories;
+		return _factories;
+	}
+	virtual Reference<FailureInjectionWorkload> create(WorkloadContext const& wcx) = 0;
+};
+
+template <class W>
+struct FailureInjectorFactory : IFailureInjectorFactory {
+	static_assert(std::is_base_of<FailureInjectionWorkload, W>::value);
+	FailureInjectorFactory() {
+		IFailureInjectorFactory::factories().push_back(Reference<IFailureInjectorFactory>::addRef(this));
+	}
+	Reference<FailureInjectionWorkload> create(WorkloadContext const& wcx) override {
+		return makeReference<W>(wcx, NoOptions());
+	}
+};
+
+struct CompoundWorkload : TestWorkload {
+	bool runFailureWorkloads = true;
+	std::vector<Reference<TestWorkload>> workloads;
+	std::vector<Reference<FailureInjectionWorkload>> failureInjection;
+
+	CompoundWorkload(WorkloadContext& wcx);
+	CompoundWorkload* add(Reference<TestWorkload>&& w);
+	void addFailureInjection(WorkloadRequest& work);
+	bool shouldInjectFailure(DeterministicRandom& random,
+	                         const WorkloadRequest& work,
+	                         Reference<FailureInjectionWorkload> failureInjection) const;
+
+	std::string description() const override;
+
+	Future<Void> setup(Database const& cx) override;
+	Future<Void> start(Database const& cx) override;
+	Future<bool> check(Database const& cx) override;
+
+	Future<std::vector<PerfMetric>> getMetrics() override;
+	double getCheckTimeout() const override;
+
+	void getMetrics(std::vector<PerfMetric>&) override;
+};
+
 struct WorkloadProcess;
 struct ClientWorkload : TestWorkload {
 	WorkloadProcess* impl;
@@ -116,15 +178,15 @@ struct KVWorkload : TestWorkload {
 	double absentFrac;
 
 	explicit KVWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		nodeCount = getOption(options, LiteralStringRef("nodeCount"), (uint64_t)100000);
-		nodePrefix = getOption(options, LiteralStringRef("nodePrefix"), (int64_t)-1);
-		actorCount = getOption(options, LiteralStringRef("actorCount"), 50);
-		keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 4);
-		maxValueBytes = getOption(options, LiteralStringRef("valueBytes"), 96);
-		minValueBytes = getOption(options, LiteralStringRef("minValueBytes"), maxValueBytes);
+		nodeCount = getOption(options, "nodeCount"_sr, (uint64_t)100000);
+		nodePrefix = getOption(options, "nodePrefix"_sr, (int64_t)-1);
+		actorCount = getOption(options, "actorCount"_sr, 50);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 4);
+		maxValueBytes = getOption(options, "valueBytes"_sr, 96);
+		minValueBytes = getOption(options, "minValueBytes"_sr, maxValueBytes);
 		ASSERT(minValueBytes <= maxValueBytes);
 
-		absentFrac = getOption(options, LiteralStringRef("absentFrac"), 0.0);
+		absentFrac = getOption(options, "absentFrac"_sr, 0.0);
 	}
 	Key getRandomKey() const;
 	Key getRandomKey(double absentFrac) const;
@@ -223,6 +285,7 @@ public:
 	bool dumpAfterTest;
 	bool clearAfterTest;
 	bool useDB;
+	bool runFailureWorkloads = true;
 	double startDelay;
 	int phases;
 	Standalone<VectorRef<VectorRef<KeyValueRef>>> options;
@@ -296,6 +359,8 @@ Future<Void> testExpectedError(Future<Void> test,
                                Optional<Error> throwOnError = Optional<Error>(),
                                UID id = UID());
 
+std::string getTestEncryptionFileName();
+
 #include "flow/unactorcompiler.h"
 
 #endif
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index b18ba6a047..9b793cce56 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -160,15 +160,16 @@ ACTOR Future<Void> getVersion(Reference<MasterData> self, GetCommitVersionReques
 		return Void();
 	}
 
-	TEST(proxyItr->second.latestRequestNum.get() < req.requestNum - 1); // Commit version request queued up
+	CODE_PROBE(proxyItr->second.latestRequestNum.get() < req.requestNum - 1, "Commit version request queued up");
 	wait(proxyItr->second.latestRequestNum.whenAtLeast(req.requestNum - 1));
 
 	auto itr = proxyItr->second.replies.find(req.requestNum);
 	if (itr != proxyItr->second.replies.end()) {
-		TEST(true); // Duplicate request for sequence
+		CODE_PROBE(true, "Duplicate request for sequence");
 		req.reply.send(itr->second);
 	} else if (req.requestNum <= proxyItr->second.latestRequestNum.get()) {
-		TEST(true); // Old request for previously acknowledged sequence - may be impossible with current FlowTransport
+		CODE_PROBE(true,
+		           "Old request for previously acknowledged sequence - may be impossible with current FlowTransport");
 		ASSERT(req.requestNum <
 		       proxyItr->second.latestRequestNum.get()); // The latest request can never be acknowledged
 		req.reply.send(Never());
@@ -204,10 +205,10 @@ ACTOR Future<Void> getVersion(Reference<MasterData> self, GetCommitVersionReques
 				self->version = self->version + toAdd;
 			}
 
-			TEST(self->version - rep.prevVersion == 1); // Minimum possible version gap
+			CODE_PROBE(self->version - rep.prevVersion == 1, "Minimum possible version gap");
 
 			bool maxVersionGap = self->version - rep.prevVersion == SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS;
-			TEST(maxVersionGap); // Maximum possible version gap
+			CODE_PROBE(maxVersionGap, "Maximum possible version gap");
 			self->lastVersionTime = t1;
 
 			self->resolutionBalancer.setChangesInReply(req.requestingProxy, rep);
@@ -399,8 +400,8 @@ ACTOR Future<Void> masterServer(MasterInterface mi,
 
 	state Future<Void> onDBChange = Void();
 	state PromiseStream<Future<Void>> addActor;
-	state Reference<MasterData> self(new MasterData(
-	    db, mi, coordinators, db->get().clusterInterface, LiteralStringRef(""), addActor, forceRecovery));
+	state Reference<MasterData> self(
+	    new MasterData(db, mi, coordinators, db->get().clusterInterface, ""_sr, addActor, forceRecovery));
 	state Future<Void> collection = actorCollection(addActor.getFuture());
 
 	addActor.send(traceRole(Role::MASTER, mi.id()));
@@ -408,7 +409,8 @@ ACTOR Future<Void> masterServer(MasterInterface mi,
 	addActor.send(serveLiveCommittedVersion(self));
 	addActor.send(updateRecoveryData(self));
 
-	TEST(!lifetime.isStillValid(db->get().masterLifetime, mi.id() == db->get().master.id())); // Master born doomed
+	CODE_PROBE(!lifetime.isStillValid(db->get().masterLifetime, mi.id() == db->get().master.id()),
+	           "Master born doomed");
 	TraceEvent("MasterLifetime", self->dbgid).detail("LifetimeToken", lifetime.toString());
 
 	try {
@@ -420,7 +422,7 @@ ACTOR Future<Void> masterServer(MasterInterface mi,
 					    .detail("Reason", "LifetimeToken")
 					    .detail("MyToken", lifetime.toString())
 					    .detail("CurrentToken", db->get().masterLifetime.toString());
-					TEST(true); // Master replaced, dying
+					CODE_PROBE(true, "Master replaced, dying");
 					if (BUGGIFY)
 						wait(delay(5));
 					throw worker_removed();
@@ -440,11 +442,11 @@ ACTOR Future<Void> masterServer(MasterInterface mi,
 			addActor.getFuture().pop();
 		}
 
-		TEST(err.code() == error_code_tlog_failed); // Master: terminated due to tLog failure
-		TEST(err.code() == error_code_commit_proxy_failed); // Master: terminated due to commit proxy failure
-		TEST(err.code() == error_code_grv_proxy_failed); // Master: terminated due to GRV proxy failure
-		TEST(err.code() == error_code_resolver_failed); // Master: terminated due to resolver failure
-		TEST(err.code() == error_code_backup_worker_failed); // Master: terminated due to backup worker failure
+		CODE_PROBE(err.code() == error_code_tlog_failed, "Master: terminated due to tLog failure");
+		CODE_PROBE(err.code() == error_code_commit_proxy_failed, "Master: terminated due to commit proxy failure");
+		CODE_PROBE(err.code() == error_code_grv_proxy_failed, "Master: terminated due to GRV proxy failure");
+		CODE_PROBE(err.code() == error_code_resolver_failed, "Master: terminated due to resolver failure");
+		CODE_PROBE(err.code() == error_code_backup_worker_failed, "Master: terminated due to backup worker failure");
 
 		if (normalMasterErrors().count(err.code())) {
 			TraceEvent("MasterTerminated", mi.id()).error(err);
@@ -487,10 +489,3 @@ TEST_CASE("/fdbserver/MasterServer/FigureVersion/NegativeReferenceVersion") {
 	ASSERT_EQ(figureVersion(0, 2.0, -1e6, 5e5, 0.1, 1e6), 550000);
 	return Void();
 }
-
-TEST_CASE("/fdbserver/MasterServer/FigureVersion/Overflow") {
-	// The upper range used in std::clamp should overflow.
-	ASSERT_EQ(figureVersion(std::numeric_limits<Version>::max() - static_cast<Version>(1e6), 1.0, 0, 1e6, 0.1, 1e6),
-	          std::numeric_limits<Version>::max() - static_cast<Version>(1e6 * 0.1));
-	return Void();
-}
diff --git a/fdbserver/networktest.actor.cpp b/fdbserver/networktest.actor.cpp
index 270329b5c6..6a537e3d64 100644
--- a/fdbserver/networktest.actor.cpp
+++ b/fdbserver/networktest.actor.cpp
@@ -193,7 +193,8 @@ ACTOR Future<Void> testClientStream(std::vector<NetworkTestInterface> interfs,
 				ASSERT(rep.index == j++);
 			}
 		} catch (Error& e) {
-			ASSERT(e.code() == error_code_end_of_stream || e.code() == error_code_connection_failed);
+			ASSERT(e.code() == error_code_end_of_stream || e.code() == error_code_connection_failed ||
+			       e.code() == error_code_request_maybe_delivered);
 		}
 		latency->tock(sample);
 		(*completed)++;
diff --git a/fdbserver/pubsub.actor.cpp b/fdbserver/pubsub.actor.cpp
index 49a52b49ae..8338c22b62 100644
--- a/fdbserver/pubsub.actor.cpp
+++ b/fdbserver/pubsub.actor.cpp
@@ -89,7 +89,7 @@ Key keyForFeedWatcher(uint64_t feed, uint64_t inbox) {
 	return StringRef(format("f/%016llx/watchers/%016llx", feed, inbox));
 }
 
-Standalone<StringRef> messagePrefix(LiteralStringRef("m/"));
+Standalone<StringRef> messagePrefix("m/"_sr);
 
 Key keyForMessage(uint64_t message) {
 	return StringRef(format("m/%016llx", message));
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 1202becf67..def3bf9078 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -23,6 +23,9 @@
 #include <type_traits>
 #include <unordered_map>
 
+#include "fdbclient/BlobCipher.h"
+#include "fdbclient/BlobGranuleCommon.h"
+#include "flow/ApiVersion.h"
 #include "fmt/format.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
@@ -41,42 +44,57 @@
 #include "fdbclient/Tracing.h"
 #include "flow/Util.h"
 #include "fdbclient/Atomic.h"
-#include "fdbclient/DatabaseContext.h"
-#include "fdbclient/KeyRangeMap.h"
+#include "fdbclient/BlobConnectionProvider.h"
+#include "fdbclient/BlobGranuleReader.actor.h"
 #include "fdbclient/CommitProxyInterface.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/FDBTypes.h"
 #include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/StatusClient.h"
-#include "fdbclient/Tenant.h"
-#include "fdbclient/Tuple.h"
+#include "fdbclient/StorageServerShard.h"
 #include "fdbclient/SystemData.h"
+#include "fdbclient/Tenant.h"
 #include "fdbclient/TransactionLineage.h"
+#include "fdbclient/Tuple.h"
 #include "fdbclient/VersionedMap.h"
-#include "fdbserver/EncryptedMutationMessage.h"
+#include "fdbrpc/sim_validation.h"
+#include "fdbrpc/Smoother.h"
+#include "fdbrpc/Stats.h"
 #include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/GetEncryptCipherKeys.actor.h"
+#include "fdbclient/GetEncryptCipherKeys.actor.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/LatencyBandConfig.h"
 #include "fdbserver/LogProtocolMessage.h"
-#include "fdbserver/SpanContextMessage.h"
 #include "fdbserver/LogSystem.h"
 #include "fdbserver/MoveKeys.actor.h"
 #include "fdbserver/MutationTracking.h"
+#include "fdbserver/OTELSpanContextMessage.h"
+#include "fdbserver/Ratekeeper.h"
 #include "fdbserver/RecoveryState.h"
 #include "fdbserver/RocksDBCheckpointUtils.actor.h"
-#include "fdbserver/StorageMetrics.h"
 #include "fdbserver/ServerCheckpoint.actor.h"
 #include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/SpanContextMessage.h"
+#include "fdbserver/StorageMetrics.h"
 #include "fdbserver/TLogInterface.h"
 #include "fdbserver/TransactionTagCounter.h"
 #include "fdbserver/WaitFailure.h"
 #include "fdbserver/WorkerInterface.actor.h"
-#include "fdbrpc/sim_validation.h"
-#include "fdbrpc/Smoother.h"
-#include "fdbrpc/Stats.h"
+#include "flow/ActorCollection.h"
+#include "flow/Arena.h"
+#include "flow/Error.h"
+#include "flow/Hash3.h"
+#include "flow/Histogram.h"
+#include "flow/IRandom.h"
+#include "flow/IndexedSet.h"
+#include "flow/SystemMonitor.h"
 #include "flow/TDMetric.actor.h"
+#include "flow/Trace.h"
+#include "flow/Util.h"
 #include "flow/genericactors.actor.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -88,6 +106,22 @@
 #define SHORT_CIRCUT_ACTUAL_STORAGE 0
 
 namespace {
+enum ChangeServerKeysContext { CSK_UPDATE, CSK_RESTORE, CSK_ASSIGN_EMPTY };
+
+std::string changeServerKeysContextName(const ChangeServerKeysContext& context) {
+	switch (context) {
+	case CSK_UPDATE:
+		return "Update";
+	case CSK_RESTORE:
+		return "Restore";
+	case CSK_ASSIGN_EMPTY:
+		return "AssignEmpty";
+	default:
+		ASSERT(false);
+	}
+	return "UnknownContext";
+}
+
 bool canReplyWith(Error e) {
 	switch (e.code()) {
 	case error_code_transaction_too_old:
@@ -121,42 +155,43 @@ bool canReplyWith(Error e) {
 
 #define PERSIST_PREFIX "\xff\xff"
 
+FDB_DECLARE_BOOLEAN_PARAM(UnlimitedCommitBytes);
+FDB_DEFINE_BOOLEAN_PARAM(UnlimitedCommitBytes);
+
 // Immutable
-static const KeyValueRef persistFormat(LiteralStringRef(PERSIST_PREFIX "Format"),
-                                       LiteralStringRef("FoundationDB/StorageServer/1/4"));
-static const KeyRangeRef persistFormatReadableRange(LiteralStringRef("FoundationDB/StorageServer/1/2"),
-                                                    LiteralStringRef("FoundationDB/StorageServer/1/5"));
-static const KeyRef persistID = LiteralStringRef(PERSIST_PREFIX "ID");
-static const KeyRef persistTssPairID = LiteralStringRef(PERSIST_PREFIX "tssPairID");
-static const KeyRef persistSSPairID = LiteralStringRef(PERSIST_PREFIX "ssWithTSSPairID");
-static const KeyRef persistTssQuarantine = LiteralStringRef(PERSIST_PREFIX "tssQ");
-static const KeyRef persistClusterIdKey = LiteralStringRef(PERSIST_PREFIX "clusterId");
+static const KeyValueRef persistFormat(PERSIST_PREFIX "Format"_sr, "FoundationDB/StorageServer/1/4"_sr);
+static const KeyValueRef persistShardAwareFormat(PERSIST_PREFIX "Format"_sr, "FoundationDB/StorageServer/1/5"_sr);
+static const KeyRangeRef persistFormatReadableRange("FoundationDB/StorageServer/1/2"_sr,
+                                                    "FoundationDB/StorageServer/1/6"_sr);
+static const KeyRef persistID = PERSIST_PREFIX "ID"_sr;
+static const KeyRef persistTssPairID = PERSIST_PREFIX "tssPairID"_sr;
+static const KeyRef persistSSPairID = PERSIST_PREFIX "ssWithTSSPairID"_sr;
+static const KeyRef persistTssQuarantine = PERSIST_PREFIX "tssQ"_sr;
+static const KeyRef persistClusterIdKey = PERSIST_PREFIX "clusterId"_sr;
 
 // (Potentially) change with the durable version or when fetchKeys completes
-static const KeyRef persistVersion = LiteralStringRef(PERSIST_PREFIX "Version");
+static const KeyRef persistVersion = PERSIST_PREFIX "Version"_sr;
 static const KeyRangeRef persistShardAssignedKeys =
-    KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "ShardAssigned/"), LiteralStringRef(PERSIST_PREFIX "ShardAssigned0"));
+    KeyRangeRef(PERSIST_PREFIX "ShardAssigned/"_sr, PERSIST_PREFIX "ShardAssigned0"_sr);
 static const KeyRangeRef persistShardAvailableKeys =
-    KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "ShardAvailable/"), LiteralStringRef(PERSIST_PREFIX "ShardAvailable0"));
-static const KeyRangeRef persistByteSampleKeys =
-    KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "BS/"), LiteralStringRef(PERSIST_PREFIX "BS0"));
+    KeyRangeRef(PERSIST_PREFIX "ShardAvailable/"_sr, PERSIST_PREFIX "ShardAvailable0"_sr);
+static const KeyRangeRef persistByteSampleKeys = KeyRangeRef(PERSIST_PREFIX "BS/"_sr, PERSIST_PREFIX "BS0"_sr);
 static const KeyRangeRef persistByteSampleSampleKeys =
-    KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "BS/" PERSIST_PREFIX "BS/"),
-                LiteralStringRef(PERSIST_PREFIX "BS/" PERSIST_PREFIX "BS0"));
-static const KeyRef persistLogProtocol = LiteralStringRef(PERSIST_PREFIX "LogProtocol");
-static const KeyRef persistPrimaryLocality = LiteralStringRef(PERSIST_PREFIX "PrimaryLocality");
-static const KeyRangeRef persistChangeFeedKeys =
-    KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "CF/"), LiteralStringRef(PERSIST_PREFIX "CF0"));
-static const KeyRangeRef persistTenantMapKeys =
-    KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "TM/"), LiteralStringRef(PERSIST_PREFIX "TM0"));
+    KeyRangeRef(PERSIST_PREFIX "BS/"_sr PERSIST_PREFIX "BS/"_sr, PERSIST_PREFIX "BS/"_sr PERSIST_PREFIX "BS0"_sr);
+static const KeyRef persistLogProtocol = PERSIST_PREFIX "LogProtocol"_sr;
+static const KeyRef persistPrimaryLocality = PERSIST_PREFIX "PrimaryLocality"_sr;
+static const KeyRangeRef persistChangeFeedKeys = KeyRangeRef(PERSIST_PREFIX "CF/"_sr, PERSIST_PREFIX "CF0"_sr);
+static const KeyRangeRef persistTenantMapKeys = KeyRangeRef(PERSIST_PREFIX "TM/"_sr, PERSIST_PREFIX "TM0"_sr);
 // data keys are unmangled (but never start with PERSIST_PREFIX because they are always in allKeys)
 
+static const KeyRangeRef persistStorageServerShardKeys =
+    KeyRangeRef(PERSIST_PREFIX "StorageServerShard/"_sr, PERSIST_PREFIX "StorageServerShard0"_sr);
+
 // Checkpoint related prefixes.
 static const KeyRangeRef persistCheckpointKeys =
-    KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "Checkpoint/"), LiteralStringRef(PERSIST_PREFIX "Checkpoint0"));
+    KeyRangeRef(PERSIST_PREFIX "Checkpoint/"_sr, PERSIST_PREFIX "Checkpoint0"_sr);
 static const KeyRangeRef persistPendingCheckpointKeys =
-    KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "PendingCheckpoint/"),
-                LiteralStringRef(PERSIST_PREFIX "PendingCheckpoint0"));
+    KeyRangeRef(PERSIST_PREFIX "PendingCheckpoint/"_sr, PERSIST_PREFIX "PendingCheckpoint0"_sr);
 static const std::string rocksdbCheckpointDirPrefix = "/rockscheckpoints_";
 
 struct AddingShard : NonCopyable {
@@ -212,7 +247,7 @@ struct AddingShard : NonCopyable {
 
 class ShardInfo : public ReferenceCounted<ShardInfo>, NonCopyable {
 	ShardInfo(KeyRange keys, std::unique_ptr<AddingShard>&& adding, StorageServer* readWrite)
-	  : adding(std::move(adding)), readWrite(readWrite), keys(keys) {}
+	  : adding(std::move(adding)), readWrite(readWrite), keys(keys), version(0) {}
 
 public:
 	// A shard has 3 mutual exclusive states: adding, readWrite and notAssigned.
@@ -220,6 +255,9 @@ public:
 	struct StorageServer* readWrite;
 	KeyRange keys;
 	uint64_t changeCounter;
+	uint64_t shardId;
+	uint64_t desiredShardId;
+	Version version;
 
 	static ShardInfo* newNotAssigned(KeyRange keys) { return new ShardInfo(keys, nullptr, nullptr); }
 	static ShardInfo* newReadWrite(KeyRange keys, StorageServer* data) { return new ShardInfo(keys, nullptr, data); }
@@ -230,6 +268,72 @@ public:
 		return new ShardInfo(keys, std::make_unique<AddingShard>(oldShard, keys), nullptr);
 	}
 
+	static ShardInfo* newShard(StorageServer* data, const StorageServerShard& shard) {
+		ShardInfo* res = nullptr;
+		switch (shard.getShardState()) {
+		case StorageServerShard::NotAssigned:
+			res = newNotAssigned(shard.range);
+			break;
+		case StorageServerShard::MovingIn:
+		case StorageServerShard::ReadWritePending:
+			res = newAdding(data, shard.range);
+			break;
+		case StorageServerShard::ReadWrite:
+			res = newReadWrite(shard.range, data);
+			break;
+		default:
+			TraceEvent(SevError, "UnknownShardState").detail("State", shard.shardState);
+		}
+		res->populateShard(shard);
+		return res;
+	}
+
+	static bool canMerge(const ShardInfo* l, const ShardInfo* r) {
+		if (l == nullptr || r == nullptr || l->keys.end != r->keys.begin || l->version == invalidVersion ||
+		    r->version == invalidVersion) {
+			return false;
+		}
+		if (l->shardId != r->shardId || l->desiredShardId != r->desiredShardId) {
+			return false;
+		}
+		return (l->isReadable() && r->isReadable()) || (!l->assigned() && !r->assigned());
+	}
+
+	StorageServerShard toStorageServerShard() const {
+		StorageServerShard::ShardState st = StorageServerShard::NotAssigned;
+		if (this->isReadable()) {
+			st = StorageServerShard::ReadWrite;
+		} else if (!this->assigned()) {
+			st = StorageServerShard::NotAssigned;
+		} else {
+			ASSERT(this->adding);
+			st = this->adding->phase == AddingShard::Waiting ? StorageServerShard::ReadWritePending
+			                                                 : StorageServerShard::MovingIn;
+		}
+		return StorageServerShard(this->keys, this->version, this->shardId, this->desiredShardId, st);
+	}
+
+	// Copies necessary information from `shard`.
+	void populateShard(const StorageServerShard& shard) {
+		this->version = shard.version;
+		this->shardId = shard.id;
+		this->desiredShardId = shard.desiredId;
+	}
+
+	// Returns true if the current shard is merged with `other`.
+	bool mergeWith(const ShardInfo* other) {
+		if (!canMerge(this, other)) {
+			return false;
+		}
+		this->keys = KeyRangeRef(this->keys.begin, other->keys.end);
+		this->version = std::max(this->version, other->version);
+		return true;
+	}
+
+	void validate() const {
+		// TODO: Complete this.
+	}
+
 	bool isReadable() const { return readWrite != nullptr; }
 	bool notAssigned() const { return !readWrite && !adding; }
 	bool assigned() const { return readWrite || adding; }
@@ -255,8 +359,11 @@ public:
 struct StorageServerDisk {
 	explicit StorageServerDisk(struct StorageServer* data, IKeyValueStore* storage) : data(data), storage(storage) {}
 
-	void makeNewStorageServerDurable();
-	bool makeVersionMutationsDurable(Version& prevStorageVersion, Version newStorageVersion, int64_t& bytesLeft);
+	void makeNewStorageServerDurable(const bool shardAware);
+	bool makeVersionMutationsDurable(Version& prevStorageVersion,
+	                                 Version newStorageVersion,
+	                                 int64_t& bytesLeft,
+	                                 UnlimitedCommitBytes unlimitedCommitBytes);
 	void makeVersionDurable(Version version);
 	void makeTssQuarantineDurable();
 	Future<bool> restoreDurableState();
@@ -267,6 +374,16 @@ struct StorageServerDisk {
 	void writeKeyValue(KeyValueRef kv);
 	void clearRange(KeyRangeRef keys);
 
+	Future<Void> addRange(KeyRangeRef range, std::string id) { return storage->addRange(range, id); }
+
+	std::vector<std::string> removeRange(KeyRangeRef range) { return storage->removeRange(range); }
+
+	void persistRangeMapping(KeyRangeRef range, bool isAdd) { storage->persistRangeMapping(range, isAdd); }
+
+	Future<Void> cleanUpShardsIfNeeded(const std::vector<std::string>& shardIds) {
+		return storage->cleanUpShardsIfNeeded(shardIds);
+	};
+
 	Future<Void> getError() { return storage->getError(); }
 	Future<Void> init() { return storage->init(); }
 	Future<Void> canCommit() { return storage->canCommit(); }
@@ -278,29 +395,26 @@ struct StorageServerDisk {
 	//  - "a", if key "a" exist
 	//  - "b", if key "a" doesn't exist, and "b" is the next existing key in total order
 	//  - allKeys.end, if keyrange [a, allKeys.end) is empty
-	Future<Key> readNextKeyInclusive(KeyRef key, IKeyValueStore::ReadType type = IKeyValueStore::ReadType::NORMAL) {
+	Future<Key> readNextKeyInclusive(KeyRef key, Optional<ReadOptions> options = Optional<ReadOptions>()) {
 		++(*kvScans);
-		return readFirstKey(storage, KeyRangeRef(key, allKeys.end), type);
+		return readFirstKey(storage, KeyRangeRef(key, allKeys.end), options);
 	}
-	Future<Optional<Value>> readValue(KeyRef key,
-	                                  IKeyValueStore::ReadType type = IKeyValueStore::ReadType::NORMAL,
-	                                  Optional<UID> debugID = Optional<UID>()) {
+	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options = Optional<ReadOptions>()) {
 		++(*kvGets);
-		return storage->readValue(key, type, debugID);
+		return storage->readValue(key, options);
 	}
 	Future<Optional<Value>> readValuePrefix(KeyRef key,
 	                                        int maxLength,
-	                                        IKeyValueStore::ReadType type = IKeyValueStore::ReadType::NORMAL,
-	                                        Optional<UID> debugID = Optional<UID>()) {
+	                                        Optional<ReadOptions> options = Optional<ReadOptions>()) {
 		++(*kvGets);
-		return storage->readValuePrefix(key, maxLength, type, debugID);
+		return storage->readValuePrefix(key, maxLength, options);
 	}
 	Future<RangeResult> readRange(KeyRangeRef keys,
 	                              int rowLimit = 1 << 30,
 	                              int byteLimit = 1 << 30,
-	                              IKeyValueStore::ReadType type = IKeyValueStore::ReadType::NORMAL) {
+	                              Optional<ReadOptions> options = Optional<ReadOptions>()) {
 		++(*kvScans);
-		return storage->readRange(keys, rowLimit, byteLimit, type);
+		return storage->readRange(keys, rowLimit, byteLimit, options);
 	}
 
 	Future<CheckpointMetaData> checkpoint(const CheckpointRequest& request) { return storage->checkpoint(request); }
@@ -327,8 +441,8 @@ private:
 	IKeyValueStore* storage;
 	void writeMutations(const VectorRef<MutationRef>& mutations, Version debugVersion, const char* debugContext);
 
-	ACTOR static Future<Key> readFirstKey(IKeyValueStore* storage, KeyRangeRef range, IKeyValueStore::ReadType type) {
-		RangeResult r = wait(storage->readRange(range, 1, 1 << 30, type));
+	ACTOR static Future<Key> readFirstKey(IKeyValueStore* storage, KeyRangeRef range, Optional<ReadOptions> options) {
+		RangeResult r = wait(storage->readRange(range, 1, 1 << 30, options));
 		if (r.size())
 			return r[0].key;
 		else
@@ -432,6 +546,8 @@ struct ChangeFeedInfo : ReferenceCounted<ChangeFeedInfo> {
 	Version storageVersion = invalidVersion; // The version between the storage version and the durable version are
 	                                         // being written to disk as part of the current commit in updateStorage.
 	Version durableVersion = invalidVersion; // All versions before the durable version are durable on disk
+	Version metadataVersion = invalidVersion; // Last update to the change feed metadata. Used for reasoning about
+	                                          // fetched metadata vs local metadata
 	Version emptyVersion = 0; // The change feed does not have any mutations before emptyVersion
 	KeyRange range;
 	Key id;
@@ -447,8 +563,6 @@ struct ChangeFeedInfo : ReferenceCounted<ChangeFeedInfo> {
 
 	bool removing = false;
 	bool destroyed = false;
-	bool possiblyDestroyed = false;
-	bool refreshInProgress = false;
 
 	KeyRangeMap<std::unordered_map<UID, Promise<Void>>> moveTriggers;
 
@@ -483,12 +597,21 @@ struct ChangeFeedInfo : ReferenceCounted<ChangeFeedInfo> {
 	}
 
 	void destroy(Version destroyVersion) {
+		updateMetadataVersion(destroyVersion);
 		removing = true;
 		destroyed = true;
-		refreshInProgress = false;
 		moved(range);
 		newMutations.trigger();
 	}
+
+	bool updateMetadataVersion(Version version) {
+		// don't update metadata version if removing, so that metadata version remains the moved away version
+		if (!removing && version > metadataVersion) {
+			metadataVersion = version;
+			return true;
+		}
+		return false;
+	}
 };
 
 class ServerWatchMetadata : public ReferenceCounted<ServerWatchMetadata> {
@@ -505,6 +628,17 @@ public:
 	  : key(key), value(value), version(version), tags(tags), debugID(debugID) {}
 };
 
+struct BusiestWriteTagContext {
+	const std::string busiestWriteTagTrackingKey;
+	UID ratekeeperID;
+	Reference<EventCacheHolder> busiestWriteTagEventHolder;
+	double lastUpdateTime;
+
+	BusiestWriteTagContext(const UID& thisServerID)
+	  : busiestWriteTagTrackingKey(thisServerID.toString() + "/BusiestWriteTag"), ratekeeperID(UID()),
+	    busiestWriteTagEventHolder(makeReference<EventCacheHolder>(busiestWriteTagTrackingKey)), lastUpdateTime(-1) {}
+};
+
 struct StorageServer {
 	typedef VersionedMap<KeyRef, ValueOrClearToRef> VersionedData;
 
@@ -541,10 +675,31 @@ private:
 	std::unordered_map<KeyRef, Reference<ServerWatchMetadata>> watchMap; // keep track of server watches
 
 public:
+	struct PendingNewShard {
+		PendingNewShard(uint64_t shardId, KeyRangeRef range) : shardId(format("%016llx", shardId)), range(range) {}
+
+		std::string toString() const {
+			return fmt::format("PendingNewShard: [ShardID]: {} [Range]: {}",
+			                   this->shardId,
+			                   Traceable<KeyRangeRef>::toString(this->range));
+		}
+
+		std::string shardId;
+		KeyRange range;
+	};
+
 	std::map<Version, std::vector<CheckpointMetaData>> pendingCheckpoints; // Pending checkpoint requests
 	std::unordered_map<UID, CheckpointMetaData> checkpoints; // Existing and deleting checkpoints
 	TenantMap tenantMap;
-	TenantPrefixIndex tenantPrefixIndex;
+	Reference<TenantPrefixIndex> tenantPrefixIndex;
+	std::map<Version, std::vector<PendingNewShard>>
+	    pendingAddRanges; // Pending requests to add ranges to physical shards
+	std::map<Version, std::vector<KeyRange>>
+	    pendingRemoveRanges; // Pending requests to remove ranges from physical shards
+
+	Reference<IPageEncryptionKeyProvider> encryptionKeyProvider;
+
+	bool shardAware; // True if the storage server is aware of the physical shards.
 
 	// Histograms
 	struct FetchKeysHistograms {
@@ -580,12 +735,15 @@ public:
 	void clearWatchMetadata();
 
 	// tenant map operations
-	void insertTenant(TenantNameRef tenantName, ValueRef value, Version version, bool insertIntoMutationLog);
+	bool insertTenant(TenantNameRef tenantName, TenantMapEntry tenantEntry, Version version);
+	void insertTenant(TenantNameRef tenantName, ValueRef value, Version version);
 	void clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version);
 
 	Optional<TenantMapEntry> getTenantEntry(Version version, TenantInfo tenant);
 	KeyRangeRef clampRangeToTenant(KeyRangeRef range, Optional<TenantMapEntry> tenantEntry, Arena& arena);
 
+	std::vector<StorageServerShard> getStorageServerShards(KeyRangeRef range);
+
 	class CurrentRunningFetchKeys {
 		std::unordered_map<UID, double> startTimeMap;
 		std::unordered_map<UID, KeyRange> keyRangeMap;
@@ -734,11 +892,11 @@ public:
 		// Set up tss fault injection here, only if we are in simulated mode and with fault injection.
 		// With fault injection enabled, the tss will start acting normal for a bit, then after the specified delay
 		// start behaving incorrectly.
-		if (g_network->isSimulated() && !g_simulator.speedUpSimulation &&
-		    g_simulator.tssMode >= ISimulator::TSSMode::EnabledAddDelay) {
+		if (g_network->isSimulated() && !g_simulator->speedUpSimulation &&
+		    g_simulator->tssMode >= ISimulator::TSSMode::EnabledAddDelay) {
 			tssFaultInjectTime = now() + deterministicRandom()->randomInt(60, 300);
 			TraceEvent(SevWarnAlways, "TSSInjectFaultEnabled", thisServerID)
-			    .detail("Mode", g_simulator.tssMode)
+			    .detail("Mode", g_simulator->tssMode)
 			    .detail("At", tssFaultInjectTime.get());
 		}
 	}
@@ -769,12 +927,14 @@ public:
 	KeyRangeMap<std::vector<Reference<ChangeFeedInfo>>> keyChangeFeed;
 	std::map<Key, Reference<ChangeFeedInfo>> uidChangeFeed;
 	Deque<std::pair<std::vector<Key>, Version>> changeFeedVersions;
-	std::map<UID, PromiseStream<Key>> changeFeedRemovals;
+	std::map<UID, PromiseStream<Key>> changeFeedDestroys;
 	std::set<Key> currentChangeFeeds;
 	std::set<Key> fetchingChangeFeeds;
 	std::unordered_map<NetworkAddress, std::map<UID, Version>> changeFeedClientVersions;
 	std::unordered_map<Key, Version> changeFeedCleanupDurable;
 	int64_t activeFeedQueries = 0;
+	int64_t changeFeedMemoryBytes = 0;
+	std::deque<std::pair<Version, int64_t>> feedMemoryBytesByVersion;
 
 	// newestAvailableVersion[k]
 	//   == invalidVersion -> k is unavailable at all versions
@@ -838,8 +998,6 @@ public:
 	AsyncVar<bool> noRecentUpdates;
 	double lastUpdate;
 
-	Int64MetricHandle readQueueSizeMetric;
-
 	std::string folder;
 
 	// defined only during splitMutations()/addMutation()
@@ -847,7 +1005,10 @@ public:
 
 	FlowLock durableVersionLock;
 	FlowLock fetchKeysParallelismLock;
-	FlowLock fetchChangeFeedParallelismLock;
+	// Extra lock that prevents too much post-initial-fetch work from building up, such as mutation applying and change
+	// feed tail fetching
+	FlowLock fetchKeysParallelismFullLock;
+	FlowLock changeFeedDiskReadsLock;
 	int64_t fetchKeysBytesBudget;
 	AsyncVar<bool> fetchKeysBudgetUsed;
 	std::vector<Promise<FetchInjectionInfo*>> readyFetchKeys;
@@ -880,6 +1041,7 @@ public:
 	}
 
 	TransactionTagCounter transactionTagCounter;
+	BusiestWriteTagContext busiestWriteTagContext;
 
 	Optional<LatencyBandConfig> latencyBandConfig;
 
@@ -887,7 +1049,8 @@ public:
 		CounterCollection cc;
 		Counter allQueries, getKeyQueries, getValueQueries, getRangeQueries, getMappedRangeQueries,
 		    getRangeStreamQueries, finishedQueries, lowPriorityQueries, rowsQueried, bytesQueried, watchQueries,
-		    emptyQueries, feedRowsQueried, feedBytesQueried, feedStreamQueries, feedVersionQueries;
+		    emptyQueries, feedRowsQueried, feedBytesQueried, feedStreamQueries, rejectedFeedStreamQueries,
+		    feedVersionQueries;
 
 		// Bytes of the mutations that have been added to the memory of the storage server. When the data is durable
 		// and cleared from the memory, we do not subtract it but add it to bytesDurable.
@@ -922,7 +1085,8 @@ public:
 		Counter sampledBytesCleared;
 		// The number of key-value pairs fetched by fetchKeys()
 		Counter kvFetched;
-		Counter mutations, setMutations, clearRangeMutations, atomicMutations;
+		Counter mutations, setMutations, clearRangeMutations, atomicMutations, changeFeedMutations,
+		    changeFeedMutationsDurable;
 		Counter updateBatches, updateVersions;
 		Counter loops;
 		Counter fetchWaitingMS, fetchWaitingCount, fetchExecutingMS, fetchExecutingCount;
@@ -947,9 +1111,20 @@ public:
 		Counter kvScans;
 		// The count of commit operation to the storage engine.
 		Counter kvCommits;
+		// The count of change feed reads that hit disk
+		Counter changeFeedDiskReads;
 
 		LatencySample readLatencySample;
+		LatencySample readKeyLatencySample;
+		LatencySample readValueLatencySample;
+		LatencySample readRangeLatencySample;
+		LatencySample readVersionWaitSample;
+		LatencySample readQueueWaitSample;
+
 		LatencyBands readLatencyBands;
+		LatencySample mappedRangeSample; // Samples getMappedRange latency
+		LatencySample mappedRangeRemoteSample; // Samples getMappedRange remote subquery latency
+		LatencySample mappedRangeLocalSample; // Samples getMappedRange local subquery latency
 
 		Counters(StorageServer* self)
 		  : cc("StorageServer", self->thisServerID.toString()), allQueries("QueryQueue", cc),
@@ -959,29 +1134,63 @@ public:
 		    lowPriorityQueries("LowPriorityQueries", cc), rowsQueried("RowsQueried", cc),
 		    bytesQueried("BytesQueried", cc), watchQueries("WatchQueries", cc), emptyQueries("EmptyQueries", cc),
 		    feedRowsQueried("FeedRowsQueried", cc), feedBytesQueried("FeedBytesQueried", cc),
-		    feedStreamQueries("FeedStreamQueries", cc), feedVersionQueries("FeedVersionQueries", cc),
-		    bytesInput("BytesInput", cc), logicalBytesInput("LogicalBytesInput", cc),
-		    logicalBytesMoveInOverhead("LogicalBytesMoveInOverhead", cc),
+		    feedStreamQueries("FeedStreamQueries", cc), rejectedFeedStreamQueries("RejectedFeedStreamQueries", cc),
+		    feedVersionQueries("FeedVersionQueries", cc), bytesInput("BytesInput", cc),
+		    logicalBytesInput("LogicalBytesInput", cc), logicalBytesMoveInOverhead("LogicalBytesMoveInOverhead", cc),
 		    kvCommitLogicalBytes("KVCommitLogicalBytes", cc), kvClearRanges("KVClearRanges", cc),
 		    kvSystemClearRanges("KVSystemClearRanges", cc), bytesDurable("BytesDurable", cc),
 		    bytesFetched("BytesFetched", cc), mutationBytes("MutationBytes", cc),
 		    feedBytesFetched("FeedBytesFetched", cc), sampledBytesCleared("SampledBytesCleared", cc),
 		    kvFetched("KVFetched", cc), mutations("Mutations", cc), setMutations("SetMutations", cc),
 		    clearRangeMutations("ClearRangeMutations", cc), atomicMutations("AtomicMutations", cc),
-		    updateBatches("UpdateBatches", cc), updateVersions("UpdateVersions", cc), loops("Loops", cc),
-		    fetchWaitingMS("FetchWaitingMS", cc), fetchWaitingCount("FetchWaitingCount", cc),
-		    fetchExecutingMS("FetchExecutingMS", cc), fetchExecutingCount("FetchExecutingCount", cc),
-		    readsRejected("ReadsRejected", cc), wrongShardServer("WrongShardServer", cc),
-		    fetchedVersions("FetchedVersions", cc), fetchesFromLogs("FetchesFromLogs", cc),
-		    quickGetValueHit("QuickGetValueHit", cc), quickGetValueMiss("QuickGetValueMiss", cc),
-		    quickGetKeyValuesHit("QuickGetKeyValuesHit", cc), quickGetKeyValuesMiss("QuickGetKeyValuesMiss", cc),
-		    kvScanBytes("KVScanBytes", cc), kvGetBytes("KVGetBytes", cc), eagerReadsKeys("EagerReadsKeys", cc),
-		    kvGets("KVGets", cc), kvScans("KVScans", cc), kvCommits("KVCommits", cc),
+		    changeFeedMutations("ChangeFeedMutations", cc),
+		    changeFeedMutationsDurable("ChangeFeedMutationsDurable", cc), updateBatches("UpdateBatches", cc),
+		    updateVersions("UpdateVersions", cc), loops("Loops", cc), fetchWaitingMS("FetchWaitingMS", cc),
+		    fetchWaitingCount("FetchWaitingCount", cc), fetchExecutingMS("FetchExecutingMS", cc),
+		    fetchExecutingCount("FetchExecutingCount", cc), readsRejected("ReadsRejected", cc),
+		    wrongShardServer("WrongShardServer", cc), fetchedVersions("FetchedVersions", cc),
+		    fetchesFromLogs("FetchesFromLogs", cc), quickGetValueHit("QuickGetValueHit", cc),
+		    quickGetValueMiss("QuickGetValueMiss", cc), quickGetKeyValuesHit("QuickGetKeyValuesHit", cc),
+		    quickGetKeyValuesMiss("QuickGetKeyValuesMiss", cc), kvScanBytes("KVScanBytes", cc),
+		    kvGetBytes("KVGetBytes", cc), eagerReadsKeys("EagerReadsKeys", cc), kvGets("KVGets", cc),
+		    kvScans("KVScans", cc), kvCommits("KVCommits", cc), changeFeedDiskReads("ChangeFeedDiskReads", cc),
 		    readLatencySample("ReadLatencyMetrics",
 		                      self->thisServerID,
 		                      SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
 		                      SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
-		    readLatencyBands("ReadLatencyBands", self->thisServerID, SERVER_KNOBS->STORAGE_LOGGING_DELAY) {
+		    readKeyLatencySample("GetKeyMetrics",
+		                         self->thisServerID,
+		                         SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+		                         SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+		    readValueLatencySample("GetValueMetrics",
+		                           self->thisServerID,
+		                           SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+		                           SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+		    readRangeLatencySample("GetRangeMetrics",
+		                           self->thisServerID,
+		                           SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+		                           SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+		    readVersionWaitSample("ReadVersionWaitMetrics",
+		                          self->thisServerID,
+		                          SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+		                          SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+		    readQueueWaitSample("ReadQueueWaitMetrics",
+		                        self->thisServerID,
+		                        SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+		                        SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+		    readLatencyBands("ReadLatencyBands", self->thisServerID, SERVER_KNOBS->STORAGE_LOGGING_DELAY),
+		    mappedRangeSample("GetMappedRangeMetrics",
+		                      self->thisServerID,
+		                      SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+		                      SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+		    mappedRangeRemoteSample("GetMappedRangeRemoteMetrics",
+		                            self->thisServerID,
+		                            SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+		                            SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
+		    mappedRangeLocalSample("GetMappedRangeLocalMetrics",
+		                           self->thisServerID,
+		                           SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+		                           SERVER_KNOBS->LATENCY_SAMPLE_SIZE) {
 			specialCounter(cc, "LastTLogVersion", [self]() { return self->lastTLogVersion; });
 			specialCounter(cc, "Version", [self]() { return self->version.get(); });
 			specialCounter(cc, "StorageVersion", [self]() { return self->storageVersion(); });
@@ -994,17 +1203,21 @@ public:
 			specialCounter(
 			    cc, "FetchKeysFetchActive", [self]() { return self->fetchKeysParallelismLock.activePermits(); });
 			specialCounter(cc, "FetchKeysWaiting", [self]() { return self->fetchKeysParallelismLock.waiters(); });
-			specialCounter(cc, "FetchChangeFeedFetchActive", [self]() {
-				return self->fetchChangeFeedParallelismLock.activePermits();
+			specialCounter(cc, "FetchKeysFullFetchActive", [self]() {
+				return self->fetchKeysParallelismFullLock.activePermits();
 			});
 			specialCounter(
-			    cc, "FetchChangeFeedWaiting", [self]() { return self->fetchChangeFeedParallelismLock.waiters(); });
+			    cc, "FetchKeysFullFetchWaiting", [self]() { return self->fetchKeysParallelismFullLock.waiters(); });
 			specialCounter(cc, "ServeFetchCheckpointActive", [self]() {
 				return self->serveFetchCheckpointParallelismLock.activePermits();
 			});
 			specialCounter(cc, "ServeFetchCheckpointWaiting", [self]() {
 				return self->serveFetchCheckpointParallelismLock.waiters();
 			});
+			specialCounter(
+			    cc, "ChangeFeedDiskReadsActive", [self]() { return self->changeFeedDiskReadsLock.activePermits(); });
+			specialCounter(
+			    cc, "ChangeFeedDiskReadsWaiting", [self]() { return self->changeFeedDiskReadsLock.waiters(); });
 			specialCounter(cc, "QueryQueueMax", [self]() { return self->getAndResetMaxQueryQueueSize(); });
 			specialCounter(cc, "BytesStored", [self]() { return self->metrics.byteSample.getEstimate(allKeys); });
 			specialCounter(cc, "ActiveWatches", [self]() { return self->numWatches; });
@@ -1014,6 +1227,7 @@ public:
 			specialCounter(cc, "KvstoreInlineKey", [self]() { return std::get<2>(self->storage.getSize()); });
 			specialCounter(cc, "ActiveChangeFeeds", [self]() { return self->uidChangeFeed.size(); });
 			specialCounter(cc, "ActiveChangeFeedQueries", [self]() { return self->activeFeedQueries; });
+			specialCounter(cc, "ChangeFeedMemoryBytes", [self]() { return self->changeFeedMemoryBytes; });
 		}
 	} counters;
 
@@ -1022,12 +1236,17 @@ public:
 
 	Reference<EventCacheHolder> storageServerSourceTLogIDEventHolder;
 
+	// Connection to blob store for fetchKeys()
+	Reference<BlobConnectionProvider> blobConn;
+
 	StorageServer(IKeyValueStore* storage,
 	              Reference<AsyncVar<ServerDBInfo> const> const& db,
-	              StorageServerInterface const& ssi)
-	  : tlogCursorReadsLatencyHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
-	                                                            TLOG_CURSOR_READS_LATENCY_HISTOGRAM,
-	                                                            Histogram::Unit::microseconds)),
+	              StorageServerInterface const& ssi,
+	              Reference<IPageEncryptionKeyProvider> encryptionKeyProvider)
+	  : tenantPrefixIndex(makeReference<TenantPrefixIndex>()), encryptionKeyProvider(encryptionKeyProvider),
+	    shardAware(false), tlogCursorReadsLatencyHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
+	                                                                               TLOG_CURSOR_READS_LATENCY_HISTOGRAM,
+	                                                                               Histogram::Unit::microseconds)),
 	    ssVersionLockLatencyHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
 	                                                          SS_VERSION_LOCK_LATENCY_HISTOGRAM,
 	                                                          Histogram::Unit::microseconds)),
@@ -1054,26 +1273,32 @@ public:
 	    prevVersion(0), rebootAfterDurableVersion(std::numeric_limits<Version>::max()),
 	    primaryLocality(tagLocalityInvalid), knownCommittedVersion(0), versionLag(0), logProtocol(0),
 	    thisServerID(ssi.id()), tssInQuarantine(false), db(db), actors(false),
-	    byteSampleClears(false, LiteralStringRef("\xff\xff\xff")), durableInProgress(Void()), watchBytes(0),
-	    numWatches(0), noRecentUpdates(false), lastUpdate(now()),
-	    readQueueSizeMetric(LiteralStringRef("StorageServer.ReadQueueSize")), updateEagerReads(nullptr),
+	    byteSampleClears(false, "\xff\xff\xff"_sr), durableInProgress(Void()), watchBytes(0), numWatches(0),
+	    noRecentUpdates(false), lastUpdate(now()), updateEagerReads(nullptr),
 	    fetchKeysParallelismLock(SERVER_KNOBS->FETCH_KEYS_PARALLELISM),
-	    fetchChangeFeedParallelismLock(SERVER_KNOBS->FETCH_KEYS_PARALLELISM),
+	    fetchKeysParallelismFullLock(SERVER_KNOBS->FETCH_KEYS_PARALLELISM_FULL),
+	    changeFeedDiskReadsLock(SERVER_KNOBS->CHANGE_FEED_DISK_READS_PARALLELISM),
 	    fetchKeysBytesBudget(SERVER_KNOBS->STORAGE_FETCH_BYTES), fetchKeysBudgetUsed(false),
 	    serveFetchCheckpointParallelismLock(SERVER_KNOBS->SERVE_FETCH_CHECKPOINT_PARALLELISM),
 	    instanceID(deterministicRandom()->randomUniqueID().first()), shuttingDown(false), behind(false),
 	    versionBehind(false), debug_inApplyUpdate(false), debug_lastValidateTime(0), lastBytesInputEBrake(0),
-	    lastDurableVersionEBrake(0), maxQueryQueue(0), transactionTagCounter(ssi.id()), counters(this),
+	    lastDurableVersionEBrake(0), maxQueryQueue(0), transactionTagCounter(ssi.id()),
+	    busiestWriteTagContext(ssi.id()), counters(this),
 	    storageServerSourceTLogIDEventHolder(
 	        makeReference<EventCacheHolder>(ssi.id().toString() + "/StorageServerSourceTLogID")) {
-		version.initMetric(LiteralStringRef("StorageServer.Version"), counters.cc.id);
-		oldestVersion.initMetric(LiteralStringRef("StorageServer.OldestVersion"), counters.cc.id);
-		durableVersion.initMetric(LiteralStringRef("StorageServer.DurableVersion"), counters.cc.id);
-		desiredOldestVersion.initMetric(LiteralStringRef("StorageServer.DesiredOldestVersion"), counters.cc.id);
+
+		version.initMetric("StorageServer.Version"_sr, counters.cc.id);
+		oldestVersion.initMetric("StorageServer.OldestVersion"_sr, counters.cc.id);
+		durableVersion.initMetric("StorageServer.DurableVersion"_sr, counters.cc.id);
+		desiredOldestVersion.initMetric("StorageServer.DesiredOldestVersion"_sr, counters.cc.id);
 
 		newestAvailableVersion.insert(allKeys, invalidVersion);
 		newestDirtyVersion.insert(allKeys, invalidVersion);
-		addShard(ShardInfo::newNotAssigned(allKeys));
+		if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && storage->shardAware()) {
+			addShard(ShardInfo::newShard(this, StorageServerShard::notAssigned(allKeys)));
+		} else {
+			addShard(ShardInfo::newNotAssigned(allKeys));
+		}
 
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True);
 
@@ -1082,6 +1307,14 @@ public:
 		this->storage.kvGets = &counters.kvGets;
 		this->storage.kvScans = &counters.kvScans;
 		this->storage.kvCommits = &counters.kvCommits;
+
+		if (SERVER_KNOBS->BG_METADATA_SOURCE != "tenant") {
+			try {
+				blobConn = BlobConnectionProvider::newBlobConnectionProvider(SERVER_KNOBS->BG_URL);
+			} catch (Error& e) {
+				// Skip any error when establishing blob connection
+			}
+		}
 	}
 
 	//~StorageServer() { fclose(log); }
@@ -1136,7 +1369,7 @@ public:
 
 	void checkChangeCounter(uint64_t oldShardChangeCounter, KeyRef const& key) {
 		if (oldShardChangeCounter != shardChangeCounter && shards[key]->changeCounter > oldShardChangeCounter) {
-			TEST(true); // shard change during getValueQ
+			CODE_PROBE(true, "shard change during getValueQ");
 			throw wrong_shard_server();
 		}
 	}
@@ -1146,7 +1379,7 @@ public:
 			auto sh = shards.intersectingRanges(keys);
 			for (auto i = sh.begin(); i != sh.end(); ++i)
 				if (i->value()->changeCounter > oldShardChangeCounter) {
-					TEST(true); // shard change during range operation
+					CODE_PROBE(true, "shard change during range operation");
 					throw wrong_shard_server();
 				}
 		}
@@ -1235,6 +1468,20 @@ public:
 		return minVersion;
 	}
 
+	// count in-memory change feed bytes towards storage queue size, for the purposes of memory management and
+	// throttling
+	void addFeedBytesAtVersion(int64_t bytes, Version version) {
+		if (feedMemoryBytesByVersion.empty() || version != feedMemoryBytesByVersion.back().first) {
+			ASSERT(feedMemoryBytesByVersion.empty() || version >= feedMemoryBytesByVersion.back().first);
+			feedMemoryBytesByVersion.push_back({ version, 0 });
+		}
+		feedMemoryBytesByVersion.back().second += bytes;
+		changeFeedMemoryBytes += bytes;
+		if (SERVER_KNOBS->STORAGE_INCLUDE_FEED_STORAGE_QUEUE) {
+			counters.bytesInput += bytes;
+		}
+	}
+
 	void getSplitPoints(SplitRangeRequest const& req) {
 		try {
 			Optional<TenantMapEntry> entry = getTenantEntry(version.get(), req.tenantInfo);
@@ -1243,9 +1490,31 @@ public:
 			req.reply.sendError(e);
 		}
 	}
+
+	void maybeInjectTargetedRestart(Version v) {
+		// inject an SS restart at most once per test
+		if (g_network->isSimulated() && !g_simulator->speedUpSimulation &&
+		    now() > g_simulator->injectTargetedSSRestartTime &&
+		    rebootAfterDurableVersion == std::numeric_limits<Version>::max()) {
+			CODE_PROBE(true, "Injecting SS targeted restart");
+			TraceEvent("SimSSInjectTargetedRestart", thisServerID).detail("Version", v);
+			rebootAfterDurableVersion = v;
+			g_simulator->injectTargetedSSRestartTime = std::numeric_limits<double>::max();
+		}
+	}
+
+	bool maybeInjectDelay() {
+		if (g_network->isSimulated() && !g_simulator->speedUpSimulation && now() > g_simulator->injectSSDelayTime) {
+			CODE_PROBE(true, "Injecting SS targeted delay");
+			TraceEvent("SimSSInjectDelay", thisServerID).log();
+			g_simulator->injectSSDelayTime = std::numeric_limits<double>::max();
+			return true;
+		}
+		return false;
+	}
 };
 
-const StringRef StorageServer::CurrentRunningFetchKeys::emptyString = LiteralStringRef("");
+const StringRef StorageServer::CurrentRunningFetchKeys::emptyString = ""_sr;
 const KeyRangeRef StorageServer::CurrentRunningFetchKeys::emptyKeyRange =
     KeyRangeRef(StorageServer::CurrentRunningFetchKeys::emptyString,
                 StorageServer::CurrentRunningFetchKeys::emptyString);
@@ -1336,16 +1605,33 @@ void validate(StorageServer* data, bool force = false) {
 			data->newestDirtyVersion.validateCoalesced();
 
 			for (auto s = data->shards.ranges().begin(); s != data->shards.ranges().end(); ++s) {
+				TraceEvent(SevVerbose, "ValidateShard", data->thisServerID)
+				    .detail("Range", s->range())
+				    .detail("ShardRange", s->value()->keys)
+				    .detail("ShardState", s->value()->debugDescribeState())
+				    .log();
 				ASSERT(s->value()->keys == s->range());
 				ASSERT(!s->value()->keys.empty());
+				if (data->shardAware) {
+					s->value()->validate();
+				}
 			}
 
-			for (auto s = data->shards.ranges().begin(); s != data->shards.ranges().end(); ++s)
+			for (auto s = data->shards.ranges().begin(); s != data->shards.ranges().end(); ++s) {
 				if (s->value()->isReadable()) {
 					auto ar = data->newestAvailableVersion.intersectingRanges(s->range());
-					for (auto a = ar.begin(); a != ar.end(); ++a)
+					for (auto a = ar.begin(); a != ar.end(); ++a) {
+						TraceEvent(SevVerbose, "ValidateShardReadable", data->thisServerID)
+						    .detail("Range", s->range())
+						    .detail("ShardRange", s->value()->keys)
+						    .detail("ShardState", s->value()->debugDescribeState())
+						    .detail("AvailableRange", a->range())
+						    .detail("AvailableVersion", a->value())
+						    .log();
 						ASSERT(a->value() == latestVersion);
+					}
 				}
+			}
 
 			// * versionedData contains versions [storageVersion(), version.get()].  It might also contain version
 			// (version.get()+1), in which changeDurableVersion may be deleting ghosts, and/or it might
@@ -1547,6 +1833,14 @@ Optional<TenantMapEntry> StorageServer::getTenantEntry(Version version, TenantIn
 	return Optional<TenantMapEntry>();
 }
 
+std::vector<StorageServerShard> StorageServer::getStorageServerShards(KeyRangeRef range) {
+	std::vector<StorageServerShard> res;
+	for (auto t : this->shards.intersectingRanges(range)) {
+		res.push_back(t.value()->toStorageServerShard());
+	}
+	return res;
+}
+
 ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 	state int64_t resultSize = 0;
 	Span span("SS:getValue"_loc, req.spanContext);
@@ -1560,7 +1854,6 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 	try {
 		++data->counters.getValueQueries;
 		++data->counters.allQueries;
-		++data->readQueueSizeMetric;
 		data->maxQueryQueue = std::max<int>(
 		    data->maxQueryQueue, data->counters.allQueries.getValue() - data->counters.finishedQueries.getValue());
 
@@ -1568,17 +1861,23 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 		// so we need to downgrade here
 		wait(data->getQueryDelay());
 
-		if (req.debugID.present())
+		// Track time from requestTime through now as read queueing wait time
+		state double queueWaitEnd = g_network->timer();
+		data->counters.readQueueWaitSample.addMeasurement(queueWaitEnd - req.requestTime());
+
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("GetValueDebug",
-			                      req.debugID.get().first(),
+			                      req.options.get().debugID.get().first(),
 			                      "getValueQ.DoRead"); //.detail("TaskID", g_network->getCurrentTask());
 
 		state Optional<Value> v;
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
 		state Version version = wait(waitForVersion(data, commitVersion, req.version, req.spanContext));
-		if (req.debugID.present())
+		data->counters.readVersionWaitSample.addMeasurement(g_network->timer() - queueWaitEnd);
+
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("GetValueDebug",
-			                      req.debugID.get().first(),
+			                      req.options.get().debugID.get().first(),
 			                      "getValueQ.AfterVersion"); //.detail("TaskID", g_network->getCurrentTask());
 
 		Optional<TenantMapEntry> entry = data->getTenantEntry(version, req.tenantInfo);
@@ -1599,11 +1898,11 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 			path = 1;
 		} else if (!i || !i->isClearTo() || i->getEndKey() <= req.key) {
 			path = 2;
-			Optional<Value> vv = wait(data->storage.readValue(req.key, IKeyValueStore::ReadType::NORMAL, req.debugID));
+			Optional<Value> vv = wait(data->storage.readValue(req.key, req.options));
 			data->counters.kvGetBytes += vv.expectedSize();
 			// Validate that while we were reading the data we didn't lose the version or shard
 			if (version < data->storageVersion()) {
-				TEST(true); // transaction_too_old after readValue
+				CODE_PROBE(true, "transaction_too_old after readValue");
 				throw transaction_too_old();
 			}
 			data->checkChangeCounter(changeCounter, req.key);
@@ -1612,15 +1911,15 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 
 		DEBUG_MUTATION("ShardGetValue",
 		               version,
-		               MutationRef(MutationRef::DebugKey, req.key, v.present() ? v.get() : LiteralStringRef("<null>")),
+		               MutationRef(MutationRef::DebugKey, req.key, v.present() ? v.get() : "<null>"_sr),
 		               data->thisServerID);
 		DEBUG_MUTATION("ShardGetPath",
 		               version,
 		               MutationRef(MutationRef::DebugKey,
 		                           req.key,
-		                           path == 0   ? LiteralStringRef("0")
-		                           : path == 1 ? LiteralStringRef("1")
-		                                       : LiteralStringRef("2")),
+		                           path == 0   ? "0"_sr
+		                           : path == 1 ? "1"_sr
+		                                       : "2"_sr),
 		               data->thisServerID);
 
 		/*
@@ -1646,9 +1945,9 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 			data->metrics.notifyBytesReadPerKSecond(req.key, bytesReadPerKSecond);
 		}
 
-		if (req.debugID.present())
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("GetValueDebug",
-			                      req.debugID.get().first(),
+			                      req.options.get().debugID.get().first(),
 			                      "getValueQ.AfterRead"); //.detail("TaskID", g_network->getCurrentTask());
 
 		// Check if the desired key might be cached
@@ -1668,10 +1967,10 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 	data->transactionTagCounter.addRequest(req.tags, resultSize);
 
 	++data->counters.finishedQueries;
-	--data->readQueueSizeMetric;
 
 	double duration = g_network->timer() - req.requestTime();
 	data->counters.readLatencySample.addMeasurement(duration);
+	data->counters.readValueLatencySample.addMeasurement(duration);
 	if (data->latencyBandConfig.present()) {
 		int maxReadBytes =
 		    data->latencyBandConfig.get().readConfig.maxReadBytes.orDefault(std::numeric_limits<int>::max());
@@ -1704,14 +2003,17 @@ ACTOR Future<Version> watchWaitForValueChange(StorageServer* data, SpanContext p
 
 	state Version minVersion = data->data().latestVersion;
 	state Future<Void> watchFuture = data->watches.onChange(metadata->key);
+	state ReadOptions options;
 	loop {
 		try {
 			metadata = data->getWatchMetadata(key);
 			state Version latest = data->version.get();
-			TEST(latest >= minVersion &&
-			     latest < data->data().latestVersion); // Starting watch loop with latestVersion > data->version
+			options.debugID = metadata->debugID;
+
+			CODE_PROBE(latest >= minVersion && latest < data->data().latestVersion,
+			           "Starting watch loop with latestVersion > data->version");
 			GetValueRequest getReq(
-			    span.context, TenantInfo(), metadata->key, latest, metadata->tags, metadata->debugID, VersionVector());
+			    span.context, TenantInfo(), metadata->key, latest, metadata->tags, options, VersionVector());
 			state Future<Void> getValue = getValueQ(
 			    data, getReq); // we are relying on the delay zero at the top of getValueQ, if removed we need one here
 			GetValueReply reply = wait(getReq.reply.getFuture());
@@ -1725,13 +2027,12 @@ ACTOR Future<Version> watchWaitForValueChange(StorageServer* data, SpanContext p
 				throw transaction_too_old();
 			}
 
-			DEBUG_MUTATION(
-			    "ShardWatchValue",
-			    latest,
-			    MutationRef(MutationRef::DebugKey,
-			                metadata->key,
-			                reply.value.present() ? StringRef(reply.value.get()) : LiteralStringRef("<null>")),
-			    data->thisServerID);
+			DEBUG_MUTATION("ShardWatchValue",
+			               latest,
+			               MutationRef(MutationRef::DebugKey,
+			                           metadata->key,
+			                           reply.value.present() ? StringRef(reply.value.get()) : "<null>"_sr),
+			               data->thisServerID);
 
 			if (metadata->debugID.present())
 				g_traceBatch.addEvent(
@@ -1744,7 +2045,7 @@ ACTOR Future<Version> watchWaitForValueChange(StorageServer* data, SpanContext p
 			}
 
 			if (data->watchBytes > SERVER_KNOBS->MAX_STORAGE_SERVER_WATCH_BYTES) {
-				TEST(true); // Too many watches, reverting to polling
+				CODE_PROBE(true, "Too many watches, reverting to polling");
 				throw watch_cancelled();
 			}
 
@@ -1775,7 +2076,7 @@ ACTOR Future<Version> watchWaitForValueChange(StorageServer* data, SpanContext p
 				throw e;
 			}
 
-			TEST(true); // Reading a watched key failed with transaction_too_old
+			CODE_PROBE(true, "Reading a watched key failed with transaction_too_old");
 		}
 
 		watchFuture = data->watches.onChange(metadata->key);
@@ -1957,7 +2258,6 @@ ACTOR Future<Void> fetchCheckpointQ(StorageServer* self, FetchCheckpointRequest
 
 // Serves FetchCheckpointKeyValuesRequest, reads local checkpoint and sends it to the client over wire.
 ACTOR Future<Void> fetchCheckpointKeyValuesQ(StorageServer* self, FetchCheckpointKeyValuesRequest req) {
-	state ICheckpointReader* reader = nullptr;
 	wait(self->serveFetchCheckpointParallelismLock.take(TaskPriority::DefaultYield));
 	state FlowLock::Releaser holder(self->serveFetchCheckpointParallelismLock);
 
@@ -1975,6 +2275,7 @@ ACTOR Future<Void> fetchCheckpointKeyValuesQ(StorageServer* self, FetchCheckpoin
 		return Void();
 	}
 
+	state ICheckpointReader* reader = nullptr;
 	try {
 		reader = newCheckpointReader(it->second, self->thisServerID);
 		wait(reader->init(BinaryWriter::toValue(req.range, IncludeVersion())));
@@ -2026,53 +2327,73 @@ ACTOR Future<Void> fetchCheckpointKeyValuesQ(StorageServer* self, FetchCheckpoin
 
 ACTOR Future<Void> overlappingChangeFeedsQ(StorageServer* data, OverlappingChangeFeedsRequest req) {
 	wait(delay(0));
-	wait(data->version.whenAtLeast(req.minVersion));
+	try {
+		wait(success(waitForVersionNoTooOld(data, req.minVersion)));
+	} catch (Error& e) {
+		if (!canReplyWith(e))
+			throw;
+		req.reply.sendError(e);
+		return Void();
+	}
 
 	if (!data->isReadable(req.range)) {
 		req.reply.sendError(wrong_shard_server());
 		return Void();
 	}
 
-	Version metadataVersion = invalidVersion;
+	Version metadataWaitVersion = invalidVersion;
 
 	auto ranges = data->keyChangeFeed.intersectingRanges(req.range);
-	std::map<Key, std::tuple<KeyRange, Version, Version>> rangeIds;
+	std::map<Key, std::tuple<KeyRange, Version, Version, Version>> rangeIds;
 	for (auto r : ranges) {
 		for (auto& it : r.value()) {
 			if (!it->removing) {
 				// Can't tell other SS about a change feed create or stopVersion that may get rolled back, and we only
 				// need to tell it about the metadata if req.minVersion > metadataVersion, since it will get the
 				// information from its own private mutations if it hasn't processed up that version yet
-				metadataVersion = std::max(metadataVersion, it->metadataCreateVersion);
+				metadataWaitVersion = std::max(metadataWaitVersion, it->metadataCreateVersion);
 
+				// don't wait for all it->metadataVersion updates, if metadata was fetched from elsewhere it's already
+				// durable, and some updates are unecessary to wait for
 				Version stopVersion;
 				if (it->stopVersion != MAX_VERSION && req.minVersion > it->stopVersion) {
 					stopVersion = it->stopVersion;
-					metadataVersion = std::max(metadataVersion, stopVersion);
+					metadataWaitVersion = std::max(metadataWaitVersion, stopVersion);
 				} else {
 					stopVersion = MAX_VERSION;
 				}
 
-				rangeIds[it->id] = std::tuple(it->range, it->emptyVersion, stopVersion);
+				rangeIds[it->id] = std::tuple(it->range, it->emptyVersion, stopVersion, it->metadataVersion);
+			} else if (it->destroyed && it->metadataVersion > metadataWaitVersion) {
+				// if we communicate the lack of a change feed because it's destroying, ensure the feed destroy isn't
+				// rolled back first
+				CODE_PROBE(true, "Overlapping Change Feeds ensuring destroy isn't rolled back");
+				metadataWaitVersion = it->metadataVersion;
 			}
 		}
 	}
 	state OverlappingChangeFeedsReply reply;
+	reply.feedMetadataVersion = data->version.get();
 	for (auto& it : rangeIds) {
-		reply.rangeIds.push_back(OverlappingChangeFeedEntry(
-		    it.first, std::get<0>(it.second), std::get<1>(it.second), std::get<2>(it.second)));
+		reply.feeds.push_back_deep(reply.arena,
+		                           OverlappingChangeFeedEntry(it.first,
+		                                                      std::get<0>(it.second),
+		                                                      std::get<1>(it.second),
+		                                                      std::get<2>(it.second),
+		                                                      std::get<3>(it.second)));
 		TraceEvent(SevDebug, "OverlappingChangeFeedEntry", data->thisServerID)
 		    .detail("MinVersion", req.minVersion)
 		    .detail("FeedID", it.first)
 		    .detail("Range", std::get<0>(it.second))
 		    .detail("EmptyVersion", std::get<1>(it.second))
-		    .detail("StopVersion", std::get<2>(it.second));
+		    .detail("StopVersion", std::get<2>(it.second))
+		    .detail("FeedMetadataVersion", std::get<3>(it.second));
 	}
 
 	// Make sure all of the metadata we are sending won't get rolled back
-	if (metadataVersion != invalidVersion && metadataVersion > data->knownCommittedVersion.get()) {
-		TEST(true); // overlapping change feeds waiting for metadata version to be committed
-		wait(data->desiredOldestVersion.whenAtLeast(metadataVersion));
+	if (metadataWaitVersion != invalidVersion && metadataWaitVersion > data->desiredOldestVersion.get()) {
+		CODE_PROBE(true, "overlapping change feeds waiting for metadata version to be safe from rollback");
+		wait(data->desiredOldestVersion.whenAtLeast(metadataWaitVersion));
 	}
 	req.reply.send(reply);
 	return Void();
@@ -2201,12 +2522,10 @@ static std::deque<Standalone<MutationsAndVersionRef>>::const_iterator searchChan
 				break;
 			}
 			lastEnd = currentEnd + 1;
+			jump = std::min((int)(currentEnd - mutations.begin()), jump);
 			currentEnd -= jump;
 			jump <<= 1;
 		}
-		if (currentEnd < mutations.begin()) {
-			currentEnd = mutations.begin();
-		}
 		auto ret = std::lower_bound(currentEnd, lastEnd, searchKey, MutationsAndVersionRef::OrderByVersion());
 		// TODO REMOVE: for validation
 		if (ret != mutations.end()) {
@@ -2239,6 +2558,9 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 	state int remainingLimitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
 	state int remainingDurableBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
 	state Version startVersion = data->version.get();
+	// TODO: Change feed reads should probably at least set cacheResult to false, possibly set a different ReadType as
+	// well, perhaps high priority?
+	state ReadOptions options;
 
 	if (DEBUG_CF_TRACE) {
 		TraceEvent(SevDebug, "TraceChangeFeedMutationsBegin", data->thisServerID)
@@ -2246,7 +2568,8 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 		    .detail("StreamUID", streamUID)
 		    .detail("Range", req.range)
 		    .detail("Begin", req.begin)
-		    .detail("End", req.end);
+		    .detail("End", req.end)
+		    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 	}
 
 	if (data->version.get() < req.begin) {
@@ -2274,6 +2597,7 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 	state Version dequeVersion = data->version.get();
 	state Version dequeKnownCommit = data->knownCommittedVersion.get();
 	state Version emptyVersion = feedInfo->emptyVersion;
+	state Version durableValidationVersion = std::min(data->durableVersion.get(), feedInfo->durableFetchVersion.get());
 	Version fetchStorageVersion = std::max(feedInfo->fetchVersion, feedInfo->durableFetchVersion.get());
 
 	if (DEBUG_CF_TRACE) {
@@ -2290,7 +2614,9 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 		    .detail("DurableVersion", feedInfo->durableVersion)
 		    .detail("FetchStorageVersion", fetchStorageVersion)
 		    .detail("FetchVersion", feedInfo->fetchVersion)
-		    .detail("DurableFetchVersion", feedInfo->durableFetchVersion.get());
+		    .detail("DurableFetchVersion", feedInfo->durableFetchVersion.get())
+		    .detail("DurableValidationVersion", durableValidationVersion)
+		    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 	}
 
 	if (req.end > emptyVersion + 1) {
@@ -2319,20 +2645,26 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 			// To not block fetchKeys making normal SS data readable on making change feed data written to storage, we
 			// wait in here instead for all fetched data to become readable from the storage engine.
 			ASSERT(req.begin <= feedInfo->fetchVersion);
-			TEST(true); // getChangeFeedMutations before fetched data durable
+			CODE_PROBE(true, "getChangeFeedMutations before fetched data durable");
 
 			// Wait for next commit to write pending feed data to storage
 			wait(feedInfo->durableFetchVersion.whenAtLeast(feedInfo->fetchVersion));
 			// To let update storage finish
 			wait(delay(0));
 		}
+
+		wait(data->changeFeedDiskReadsLock.take(TaskPriority::DefaultYield));
+		state FlowLock::Releaser holdingDiskReadsLock(data->changeFeedDiskReadsLock);
 		RangeResult res = wait(
 		    data->storage.readRange(KeyRangeRef(changeFeedDurableKey(req.rangeID, std::max(req.begin, emptyVersion)),
 		                                        changeFeedDurableKey(req.rangeID, req.end)),
 		                            1 << 30,
-		                            remainingDurableBytes));
+		                            remainingDurableBytes,
+		                            options));
+		holdingDiskReadsLock.release();
 
 		data->counters.kvScanBytes += res.logicalSize();
+		++data->counters.changeFeedDiskReads;
 
 		if (!inverted && !req.range.empty()) {
 			data->checkChangeCounter(changeCounter, req.range);
@@ -2404,21 +2736,38 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 				}
 			} else if (memoryVerifyIdx < memoryReply.mutations.size() &&
 			           version == memoryReply.mutations[memoryVerifyIdx].version) {
-				fmt::print("ERROR: SS {0} CF {1} SQ {2} has mutation at {3} in memory but all filtered out on disk!\n",
-				           data->thisServerID.toString().substr(0, 4),
-				           req.rangeID.printable().substr(0, 6),
-				           streamUID.toString().substr(0, 8),
-				           version);
+				if (version > durableValidationVersion) {
+					// Another validation case - feed was popped, data was fetched, fetched data was persisted but pop
+					// wasn't yet, then SS restarted. Now SS has the data without the popped version. This looks wrong
+					// here but is fine.
+					memoryVerifyIdx++;
+				} else {
+					fmt::print("ERROR: SS {0} CF {1} SQ {2} has mutation at {3} in memory but all filtered out on "
+					           "disk! (durable validation = {4})\n",
+					           data->thisServerID.toString().substr(0, 4),
+					           req.rangeID.printable().substr(0, 6),
+					           streamUID.toString().substr(0, 8),
+					           version,
+					           durableValidationVersion);
 
-				fmt::print("  Memory: ({})\n", memoryReply.mutations[memoryVerifyIdx].mutations.size());
-				for (auto& it : memoryReply.mutations[memoryVerifyIdx].mutations) {
-					if (it.type == MutationRef::SetValue) {
-						fmt::print("    {}=\n", it.param1.printable().c_str());
-					} else {
-						fmt::print("    {} - {}\n", it.param1.printable().c_str(), it.param2.printable().c_str());
+					fmt::print("  Memory: ({})\n", memoryReply.mutations[memoryVerifyIdx].mutations.size());
+					for (auto& it : memoryReply.mutations[memoryVerifyIdx].mutations) {
+						if (it.type == MutationRef::SetValue) {
+							fmt::print("    {}=\n", it.param1.printable().c_str());
+						} else {
+							fmt::print("    {} - {}\n", it.param1.printable().c_str(), it.param2.printable().c_str());
+						}
 					}
+					fmt::print("  Disk(pre-filter): ({})\n", mutations.size());
+					for (auto& it : mutations) {
+						if (it.type == MutationRef::SetValue) {
+							fmt::print("    {}=\n", it.param1.printable().c_str());
+						} else {
+							fmt::print("    {} - {}\n", it.param1.printable().c_str(), it.param2.printable().c_str());
+						}
+					}
+					ASSERT_WE_THINK(false);
 				}
-				ASSERT(false);
 			}
 			remainingDurableBytes -=
 			    sizeof(KeyValueRef) +
@@ -2439,11 +2788,11 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 			// If still empty, that means disk results were filtered out, but skipped all memory results. Add an empty,
 			// either the last version from disk
 			if (reply.mutations.empty() && res.size()) {
-				TEST(true); // Change feed adding empty version after disk + memory filtered
+				CODE_PROBE(true, "Change feed adding empty version after disk + memory filtered");
 				reply.mutations.push_back(reply.arena, MutationsAndVersionRef(lastVersion, lastKnownCommitted));
 			}
 		} else if (reply.mutations.empty() || reply.mutations.back().version < lastVersion) {
-			TEST(true); // Change feed adding empty version after disk filtered
+			CODE_PROBE(true, "Change feed adding empty version after disk filtered");
 			reply.mutations.push_back(reply.arena, MutationsAndVersionRef(lastVersion, lastKnownCommitted));
 		}
 	} else {
@@ -2454,7 +2803,7 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 	Version finalVersion = std::min(req.end - 1, dequeVersion);
 	if ((reply.mutations.empty() || reply.mutations.back().version < finalVersion) && remainingLimitBytes > 0 &&
 	    remainingDurableBytes > 0) {
-		TEST(true); // Change feed adding empty version after empty results
+		CODE_PROBE(true, "Change feed adding empty version after empty results");
 		reply.mutations.push_back(
 		    reply.arena, MutationsAndVersionRef(finalVersion, finalVersion == dequeVersion ? dequeKnownCommit : 0));
 		// if we add empty mutation after the last thing in memory, and didn't read from disk, gotAll is true
@@ -2473,8 +2822,8 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 		// have happened between req.begin and minVersion.
 		Version minVersion = data->minFeedVersionForAddress(req.reply.getEndpoint().getPrimaryAddress());
 		bool ok = atLatest && minVersion > feedInfo->emptyVersion;
-		TEST(ok); // feed popped while valid read waiting
-		TEST(!ok); // feed popped while invalid read waiting
+		CODE_PROBE(ok, "feed popped while valid read waiting");
+		CODE_PROBE(!ok, "feed popped while invalid read waiting");
 		if (!ok) {
 			TraceEvent("ChangeFeedMutationsPopped", data->thisServerID)
 			    .detail("FeedID", req.rangeID)
@@ -2513,7 +2862,8 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 		    .detail("FirstVersion", reply.mutations.empty() ? invalidVersion : reply.mutations.front().version)
 		    .detail("LastVersion", reply.mutations.empty() ? invalidVersion : reply.mutations.back().version)
 		    .detail("Count", reply.mutations.size())
-		    .detail("GotAll", gotAll);
+		    .detail("GotAll", gotAll)
+		    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 	}
 
 	if (DEBUG_CF_MISSING(req.rangeID, req.range, req.begin, reply.mutations.back().version) && !req.canReadPopped) {
@@ -2572,55 +2922,8 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
 	return std::make_pair(reply, gotAll);
 }
 
-ACTOR Future<Void> localChangeFeedStream(StorageServer* data,
-                                         PromiseStream<Standalone<MutationsAndVersionRef>> results,
-                                         Key rangeID,
-                                         Version begin,
-                                         Version end,
-                                         KeyRange range) {
-	try {
-		loop {
-			state ChangeFeedStreamRequest feedRequest;
-			feedRequest.rangeID = rangeID;
-			feedRequest.begin = begin;
-			feedRequest.end = end;
-			feedRequest.range = range;
-			state std::pair<ChangeFeedStreamReply, bool> feedReply =
-			    wait(getChangeFeedMutations(data, feedRequest, true, false, UID()));
-			begin = feedReply.first.mutations.back().version + 1;
-			state int resultLoc = 0;
-			while (resultLoc < feedReply.first.mutations.size()) {
-				if (feedReply.first.mutations[resultLoc].mutations.size() ||
-				    feedReply.first.mutations[resultLoc].version == end - 1) {
-					wait(results.onEmpty());
-					results.send(feedReply.first.mutations[resultLoc]);
-				}
-				resultLoc++;
-			}
-
-			if (begin == end) {
-				return Void();
-			}
-		}
-	} catch (Error& e) {
-		if (e.code() == error_code_unknown_change_feed) {
-			TEST(true); // CF was moved away, no more local data to merge with
-			// Send endVersion so local stream is effectively done. We couldn't have send that already, because that
-			// would mean the stream would have finished without error
-			results.send(MutationsAndVersionRef(end, invalidVersion));
-		} else {
-			TraceEvent(SevError, "LocalChangeFeedError", data->thisServerID)
-			    .error(e)
-			    .detail("CFID", rangeID.printable());
-		}
-		throw;
-	}
-}
-
 // Change feed stream must be sent an error as soon as it is moved away, or change feed can get incorrect results
 ACTOR Future<Void> stopChangeFeedOnMove(StorageServer* data, ChangeFeedStreamRequest req, UID streamUID) {
-	wait(delay(0, TaskPriority::DefaultEndpoint));
-
 	auto feed = data->uidChangeFeed.find(req.rangeID);
 	if (feed == data->uidChangeFeed.end() || feed->second->removing) {
 		req.reply.sendError(unknown_change_feed());
@@ -2640,7 +2943,7 @@ ACTOR Future<Void> stopChangeFeedOnMove(StorageServer* data, ChangeFeedStreamReq
 		}
 		return Void();
 	}
-	TEST(true); // Change feed moved away cancelling queries
+	CODE_PROBE(true, "Change feed moved away cancelling queries");
 	// DO NOT call req.reply.onReady before sending - we need to propagate this error through regardless of how far
 	// behind client is
 	req.reply.sendError(wrong_shard_server());
@@ -2652,17 +2955,30 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 	state bool atLatest = false;
 	state bool removeUID = false;
 	state Optional<Version> blockedVersion;
-	if (req.replyBufferSize <= 0) {
-		req.reply.setByteLimit(SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES);
-	} else {
-		req.reply.setByteLimit(std::min((int64_t)req.replyBufferSize, SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES));
-	}
-
-	++data->counters.feedStreamQueries;
-
-	wait(delay(0, TaskPriority::DefaultEndpoint));
 
 	try {
+		++data->counters.feedStreamQueries;
+
+		// FIXME: do something more sophisticated here besides hard limit
+		// Allow other storage servers fetching feeds to go above this limit. currently, req.canReadPopped == read is a
+		// fetch from another ss
+		if (!req.canReadPopped && (data->activeFeedQueries >= SERVER_KNOBS->STORAGE_FEED_QUERY_HARD_LIMIT ||
+		                           (g_network->isSimulated() && BUGGIFY_WITH_PROB(0.005)))) {
+			req.reply.sendError(storage_too_many_feed_streams());
+			++data->counters.rejectedFeedStreamQueries;
+			return Void();
+		}
+
+		data->activeFeedQueries++;
+
+		if (req.replyBufferSize <= 0) {
+			req.reply.setByteLimit(SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES);
+		} else {
+			req.reply.setByteLimit(std::min((int64_t)req.replyBufferSize, SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES));
+		}
+
+		wait(delay(0, TaskPriority::DefaultEndpoint));
+
 		if (DEBUG_CF_TRACE) {
 			TraceEvent(SevDebug, "TraceChangeFeedStreamStart", data->thisServerID)
 			    .detail("FeedID", req.rangeID)
@@ -2670,9 +2986,11 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 			    .detail("Range", req.range)
 			    .detail("Begin", req.begin)
 			    .detail("End", req.end)
-			    .detail("CanReadPopped", req.canReadPopped);
+			    .detail("CanReadPopped", req.canReadPopped)
+			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 		}
-		data->activeFeedQueries++;
+
+		wait(success(waitForVersionNoTooOld(data, req.begin)));
 
 		// send an empty version at begin - 1 to establish the stream quickly
 		ChangeFeedStreamReply emptyInitialReply;
@@ -2691,7 +3009,8 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 			    .detail("Begin", req.begin)
 			    .detail("End", req.end)
 			    .detail("CanReadPopped", req.canReadPopped)
-			    .detail("Version", req.begin - 1);
+			    .detail("Version", req.begin - 1)
+			    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 		}
 
 		loop {
@@ -2707,7 +3026,8 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 					    .detail("Begin", req.begin)
 					    .detail("End", req.end)
 					    .detail("CanReadPopped", req.canReadPopped)
-					    .detail("Version", blockedVersion.present() ? blockedVersion.get() : data->prevVersion);
+					    .detail("Version", blockedVersion.present() ? blockedVersion.get() : data->prevVersion)
+					    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 				}
 				removeUID = true;
 			}
@@ -2727,7 +3047,8 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
 					    .detail("Begin", req.begin)
 					    .detail("End", req.end)
 					    .detail("CanReadPopped", req.canReadPopped)
-					    .detail("Version", blockedVersion.present() ? blockedVersion.get() : data->prevVersion);
+					    .detail("Version", blockedVersion.present() ? blockedVersion.get() : data->prevVersion)
+					    .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
 				}
 			}
 			std::pair<ChangeFeedStreamReply, bool> _feedReply = wait(feedReplyFuture);
@@ -2849,7 +3170,11 @@ ACTOR Future<Void> getShardState_impl(StorageServer* data, GetShardStateRequest
 		}
 
 		if (!onChange.size()) {
-			req.reply.send(GetShardStateReply{ data->version.get(), data->durableVersion.get() });
+			GetShardStateReply rep(data->version.get(), data->durableVersion.get());
+			if (req.includePhysicalShard) {
+				rep.shards = data->getStorageServerShards(req.keys);
+			}
+			req.reply.send(rep);
 			return Void();
 		}
 
@@ -2952,7 +3277,9 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
                                                     // To provide span context, tags, debug ID to underlying lookups.
                                                     GetMappedKeyValuesRequest* pOriginalReq) {
 	state GetValueReqAndResultRef getValue;
+	state double getValueStart = g_network->timer();
 	getValue.key = key;
+	state Optional<ReadOptions> options = pOriginalReq->options;
 
 	if (data->shards[key]->isReadable()) {
 		try {
@@ -2962,7 +3289,7 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
 			                    key,
 			                    version,
 			                    pOriginalReq->tags,
-			                    pOriginalReq->debugID,
+			                    options,
 			                    VersionVector());
 			// Note that it does not use readGuard to avoid server being overloaded here. Throttling is enforced at the
 			// original request level, rather than individual underlying lookups. The reason is that throttle any
@@ -2972,6 +3299,8 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
 			if (!reply.error.present()) {
 				++data->counters.quickGetValueHit;
 				copyOptionalValue(a, getValue, reply.value);
+				const double duration = g_network->timer() - getValueStart;
+				data->counters.mappedRangeLocalSample.addMeasurement(duration);
 				return getValue;
 			}
 			// Otherwise fallback.
@@ -2983,7 +3312,7 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
 
 	++data->counters.quickGetValueMiss;
 	if (SERVER_KNOBS->QUICK_GET_VALUE_FALLBACK) {
-		state Transaction tr(data->cx, pOriginalReq->tenantInfo.name);
+		state Transaction tr(data->cx, pOriginalReq->tenantInfo.name.castTo<TenantName>());
 		tr.setVersion(version);
 		// TODO: is DefaultPromiseEndpoint the best priority for this?
 		tr.trState->taskID = TaskPriority::DefaultPromiseEndpoint;
@@ -2991,6 +3320,8 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
 		// TODO: async in case it needs to read from other servers.
 		Optional<Value> valueOption = wait(valueFuture);
 		copyOptionalValue(a, getValue, valueOption);
+		double duration = g_network->timer() - getValueStart;
+		data->counters.mappedRangeRemoteSample.addMeasurement(duration);
 		return getValue;
 	} else {
 		throw quick_get_value_miss();
@@ -3005,7 +3336,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
                                           int limit,
                                           int* pLimitBytes,
                                           SpanContext parentSpan,
-                                          IKeyValueStore::ReadType type,
+                                          Optional<ReadOptions> options,
                                           Optional<Key> tenantPrefix) {
 	state GetKeyValuesReply result;
 	state StorageServer::VersionedData::ViewAtVersion view = data->data().at(version);
@@ -3081,7 +3412,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			// Read the data on disk up to vCurrent (or the end of the range)
 			readEnd = vCurrent ? std::min(vCurrent.key(), range.end) : range.end;
 			RangeResult atStorageVersion =
-			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, type));
+			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, options));
 			data->counters.kvScanBytes += atStorageVersion.logicalSize();
 
 			ASSERT(atStorageVersion.size() <= limit);
@@ -3176,7 +3507,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			readBegin = vCurrent ? std::max(vCurrent->isClearTo() ? vCurrent->getEndKey() : vCurrent.key(), range.begin)
 			                     : range.begin;
 			RangeResult atStorageVersion =
-			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, type));
+			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, options));
 			data->counters.kvScanBytes += atStorageVersion.logicalSize();
 
 			ASSERT(atStorageVersion.size() <= -limit);
@@ -3246,7 +3577,7 @@ ACTOR Future<Key> findKey(StorageServer* data,
                           KeyRange range,
                           int* pOffset,
                           SpanContext parentSpan,
-                          IKeyValueStore::ReadType type)
+                          Optional<ReadOptions> options)
 // Attempts to find the key indicated by sel in the data at version, within range.
 // Precondition: selectorInRange(sel, range)
 // If it is found, offset is set to 0 and a key is returned which falls inside range.
@@ -3274,7 +3605,7 @@ ACTOR Future<Key> findKey(StorageServer* data,
 	if (sel.offset <= 1 && sel.offset >= 0)
 		maxBytes = std::numeric_limits<int>::max();
 	else
-		maxBytes = (g_network->isSimulated() && g_simulator.tssMode == ISimulator::TSSMode::Disabled && BUGGIFY)
+		maxBytes = (g_network->isSimulated() && g_simulator->tssMode == ISimulator::TSSMode::Disabled && BUGGIFY)
 		               ? SERVER_KNOBS->BUGGIFY_LIMIT_BYTES
 		               : SERVER_KNOBS->STORAGE_LIMIT_BYTES;
 
@@ -3285,14 +3616,14 @@ ACTOR Future<Key> findKey(StorageServer* data,
 	              (distance + skipEqualKey) * sign,
 	              &maxBytes,
 	              span.context,
-	              type,
+	              options,
 	              Optional<Key>()));
 	state bool more = rep.more && rep.data.size() != distance + skipEqualKey;
 
 	// If we get only one result in the reverse direction as a result of the data being too large, we could get stuck in
 	// a loop
 	if (more && !forward && rep.data.size() == 1) {
-		TEST(true); // Reverse key selector returned only one result in range read
+		CODE_PROBE(true, "Reverse key selector returned only one result in range read");
 		maxBytes = std::numeric_limits<int>::max();
 		GetKeyValuesReply rep2 = wait(readRange(data,
 		                                        version,
@@ -3300,7 +3631,7 @@ ACTOR Future<Key> findKey(StorageServer* data,
 		                                        -2,
 		                                        &maxBytes,
 		                                        span.context,
-		                                        type,
+		                                        options,
 		                                        Optional<Key>()));
 		rep = rep2;
 		more = rep.more && rep.data.size() != distance + skipEqualKey;
@@ -3333,7 +3664,7 @@ ACTOR Future<Key> findKey(StorageServer* data,
 			*pOffset = -*pOffset;
 
 		if (more) {
-			TEST(true); // Key selector read range had more results
+			CODE_PROBE(true, "Key selector read range had more results");
 
 			ASSERT(rep.data.size());
 			Key returnKey = forward ? keyAfter(rep.data.back().key) : rep.data.back().key;
@@ -3354,10 +3685,28 @@ KeyRange getShardKeyRange(StorageServer* data, const KeySelectorRef& sel)
 {
 	auto i = sel.isBackward() ? data->shards.rangeContainingKeyBefore(sel.getKey())
 	                          : data->shards.rangeContaining(sel.getKey());
+	auto fullRange = data->shards.ranges();
 	if (!i->value()->isReadable())
 		throw wrong_shard_server();
 	ASSERT(selectorInRange(sel, i->range()));
-	return i->range();
+	Key begin, end;
+	if (sel.isBackward()) {
+		end = i->range().end;
+		while (i != fullRange.begin() && i.value()->isReadable()) {
+			begin = i->range().begin;
+			--i;
+		}
+		if (i.value()->isReadable()) {
+			begin = i->range().begin;
+		}
+	} else {
+		begin = i->range().begin;
+		while (i != fullRange.end() && i.value()->isReadable()) {
+			end = i->range().end;
+			++i;
+		}
+	}
+	return KeyRangeRef(begin, end);
 }
 
 ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
@@ -3366,8 +3715,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 {
 	state Span span("SS:getKeyValues"_loc, req.spanContext);
 	state int64_t resultSize = 0;
-	state IKeyValueStore::ReadType type =
-	    req.isFetchKeys ? IKeyValueStore::ReadType::FETCH : IKeyValueStore::ReadType::NORMAL;
+	state Optional<ReadOptions> options = req.options;
 
 	if (req.tenantInfo.name.present()) {
 		span.addAttribute("tenant"_sr, req.tenantInfo.name.get());
@@ -3377,24 +3725,29 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 
 	++data->counters.getRangeQueries;
 	++data->counters.allQueries;
-	++data->readQueueSizeMetric;
 	data->maxQueryQueue = std::max<int>(
 	    data->maxQueryQueue, data->counters.allQueries.getValue() - data->counters.finishedQueries.getValue());
 
 	// Active load balancing runs at a very high priority (to obtain accurate queue lengths)
 	// so we need to downgrade here
-	if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.isFetchKeys) {
+	if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.options.present() && req.options.get().type == ReadType::FETCH) {
 		wait(delay(0, TaskPriority::FetchKeys));
 	} else {
 		wait(data->getQueryDelay());
 	}
 
+	// Track time from requestTime through now as read queueing wait time
+	state double queueWaitEnd = g_network->timer();
+	data->counters.readQueueWaitSample.addMeasurement(queueWaitEnd - req.requestTime());
+
 	try {
-		if (req.debugID.present())
-			g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.Before");
+		if (req.options.present() && req.options.get().debugID.present())
+			g_traceBatch.addEvent(
+			    "TransactionDebug", req.options.get().debugID.get().first(), "storageserver.getKeyValues.Before");
 
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
 		state Version version = wait(waitForVersion(data, commitVersion, req.version, span.context));
+		data->counters.readVersionWaitSample.addMeasurement(g_network->timer() - queueWaitEnd);
 
 		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(version, req.tenantInfo);
 		state Optional<Key> tenantPrefix = tenantEntry.map<Key>([](TenantMapEntry e) { return e.prefix; });
@@ -3407,9 +3760,9 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 		//		try {
 		state KeyRange shard = getShardKeyRange(data, req.begin);
 
-		if (req.debugID.present())
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent(
-			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.AfterVersion");
+			    "TransactionDebug", req.options.get().debugID.get().first(), "storageserver.getKeyValues.AfterVersion");
 		//.detail("ShardBegin", shard.begin).detail("ShardEnd", shard.end);
 		//} catch (Error& e) { TraceEvent("WrongShardServer", data->thisServerID).detail("Begin",
 		// req.begin.toString()).detail("End", req.end.toString()).detail("Version", version).detail("Shard",
@@ -3426,18 +3779,19 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 
 		state int offset1 = 0;
 		state int offset2;
-		state Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
-		                               ? Future<Key>(req.begin.getKey())
-		                               : findKey(data, req.begin, version, searchRange, &offset1, span.context, type);
+		state Future<Key> fBegin =
+		    req.begin.isFirstGreaterOrEqual()
+		        ? Future<Key>(req.begin.getKey())
+		        : findKey(data, req.begin, version, searchRange, &offset1, span.context, options);
 		state Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
 		                             ? Future<Key>(req.end.getKey())
-		                             : findKey(data, req.end, version, searchRange, &offset2, span.context, type);
+		                             : findKey(data, req.end, version, searchRange, &offset2, span.context, options);
 		state Key begin = wait(fBegin);
 		state Key end = wait(fEnd);
 
-		if (req.debugID.present())
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent(
-			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.AfterKeys");
+			    "TransactionDebug", req.options.get().debugID.get().first(), "storageserver.getKeyValues.AfterKeys");
 		//.detail("Off1",offset1).detail("Off2",offset2).detail("ReqBegin",req.begin.getKey()).detail("ReqEnd",req.end.getKey());
 
 		// Offsets of zero indicate begin/end keys in this shard, which obviously means we can answer the query
@@ -3445,7 +3799,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 		// end the last actual key returned must be from this shard. A begin offset of 1 is also OK because then either
 		// begin is past end or equal to end (so the result is definitely empty)
 		if ((offset1 && offset1 != 1) || (offset2 && offset2 != 1)) {
-			TEST(true); // wrong_shard_server due to offset
+			CODE_PROBE(true, "wrong_shard_server due to offset");
 			// We could detect when offset1 takes us off the beginning of the database or offset2 takes us off the end,
 			// and return a clipped range rather than an error (since that is what the NativeAPI.getRange will do anyway
 			// via its "slow path"), but we would have to add some flags to the response to encode whether we went off
@@ -3455,8 +3809,9 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 		}
 
 		if (begin >= end) {
-			if (req.debugID.present())
-				g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.Send");
+			if (req.options.present() && req.options.get().debugID.present())
+				g_traceBatch.addEvent(
+				    "TransactionDebug", req.options.get().debugID.get().first(), "storageserver.getKeyValues.Send");
 			//.detail("Begin",begin).detail("End",end);
 
 			GetKeyValuesReply none;
@@ -3477,13 +3832,14 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			                                      req.limit,
 			                                      &remainingLimitBytes,
 			                                      span.context,
-			                                      type,
+			                                      options,
 			                                      tenantPrefix));
 			GetKeyValuesReply r = _r;
 
-			if (req.debugID.present())
-				g_traceBatch.addEvent(
-				    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.AfterReadRange");
+			if (req.options.present() && req.options.get().debugID.present())
+				g_traceBatch.addEvent("TransactionDebug",
+				                      req.options.get().debugID.get().first(),
+				                      "storageserver.getKeyValues.AfterReadRange");
 			//.detail("Begin",begin).detail("End",end).detail("SizeOf",r.data.size());
 			data->checkChangeCounter(
 			    changeCounter,
@@ -3532,10 +3888,10 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 
 	data->transactionTagCounter.addRequest(req.tags, resultSize);
 	++data->counters.finishedQueries;
-	--data->readQueueSizeMetric;
 
 	double duration = g_network->timer() - req.requestTime();
 	data->counters.readLatencySample.addMeasurement(duration);
+	data->counters.readRangeLatencySample.addMeasurement(duration);
 	if (data->latencyBandConfig.present()) {
 		int maxReadBytes =
 		    data->latencyBandConfig.get().readConfig.maxReadBytes.orDefault(std::numeric_limits<int>::max());
@@ -3558,10 +3914,15 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
     // To provide span context, tags, debug ID to underlying lookups.
     GetMappedKeyValuesRequest* pOriginalReq) {
 	state GetRangeReqAndResultRef getRange;
+	state double getValuesStart = g_network->timer();
 	getRange.begin = firstGreaterOrEqual(KeyRef(*a, prefix));
 	getRange.end = firstGreaterOrEqual(strinc(prefix, *a));
+	if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
+		g_traceBatch.addEvent("TransactionDebug",
+		                      pOriginalReq->options.get().debugID.get().first(),
+		                      "storageserver.quickGetKeyValues.Before");
 	try {
-		// TODO: Use a lower level API may be better? Or tweak priorities?
+		// TODO: Use a lower level API may be better?
 		GetKeyValuesRequest req;
 		req.spanContext = pOriginalReq->spanContext;
 		req.arena = *a;
@@ -3573,10 +3934,10 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 		// TODO: Use remainingLimit, remainingLimitBytes rather than separate knobs.
 		req.limit = SERVER_KNOBS->QUICK_GET_KEY_VALUES_LIMIT;
 		req.limitBytes = SERVER_KNOBS->QUICK_GET_KEY_VALUES_LIMIT_BYTES;
-		req.isFetchKeys = false;
+		req.options = pOriginalReq->options;
+		// TODO: tweak priorities in req.options.get().type?
 		req.tags = pOriginalReq->tags;
 		req.ssLatestCommitVersions = VersionVector();
-		req.debugID = pOriginalReq->debugID;
 
 		// Note that it does not use readGuard to avoid server being overloaded here. Throttling is enforced at the
 		// original request level, rather than individual underlying lookups. The reason is that throttle any individual
@@ -3588,6 +3949,12 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 			// Convert GetKeyValuesReply to RangeResult.
 			a->dependsOn(reply.arena);
 			getRange.result = RangeResultRef(reply.data, reply.more);
+			const double duration = g_network->timer() - getValuesStart;
+			data->counters.mappedRangeLocalSample.addMeasurement(duration);
+			if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
+				g_traceBatch.addEvent("TransactionDebug",
+				                      pOriginalReq->options.get().debugID.get().first(),
+				                      "storageserver.quickGetKeyValues.AfterLocalFetch");
 			return getRange;
 		}
 		// Otherwise fallback.
@@ -3597,8 +3964,11 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 
 	++data->counters.quickGetKeyValuesMiss;
 	if (SERVER_KNOBS->QUICK_GET_KEY_VALUES_FALLBACK) {
-		state Transaction tr(data->cx, pOriginalReq->tenantInfo.name);
+		state Transaction tr(data->cx, pOriginalReq->tenantInfo.name.castTo<TenantName>());
 		tr.setVersion(version);
+		if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present()) {
+			tr.debugTransaction(pOriginalReq->options.get().debugID.get());
+		}
 		// TODO: is DefaultPromiseEndpoint the best priority for this?
 		tr.trState->taskID = TaskPriority::DefaultPromiseEndpoint;
 		Future<RangeResult> rangeResultFuture =
@@ -3607,6 +3977,12 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 		RangeResult rangeResult = wait(rangeResultFuture);
 		a->dependsOn(rangeResult.arena());
 		getRange.result = rangeResult;
+		const double duration = g_network->timer() - getValuesStart;
+		data->counters.mappedRangeRemoteSample.addMeasurement(duration);
+		if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
+			g_traceBatch.addEvent("TransactionDebug",
+			                      pOriginalReq->options.get().debugID.get().first(),
+			                      "storageserver.quickGetKeyValues.AfterRemoteFetch");
 		return getRange;
 	} else {
 		throw quick_get_key_values_miss();
@@ -3678,9 +4054,7 @@ void preprocessMappedKey(Tuple& mappedKeyFormatTuple, std::vector<Optional<Tuple
 			bool escaped = unescapeLiterals(s, "{{", "{");
 			escaped = unescapeLiterals(s, "}}", "}") || escaped;
 			if (escaped) {
-				Tuple escapedTuple;
-				escapedTuple.append(s);
-				vt.emplace_back(escapedTuple);
+				vt.emplace_back(Tuple::makeTuple(s));
 			} else if (singleKeyOrValue(s, sz)) {
 				// when it is SingleKeyOrValue, insert an empty Tuple to vector as placeholder
 				vt.emplace_back(Tuple());
@@ -3705,14 +4079,12 @@ void preprocessMappedKey(Tuple& mappedKeyFormatTuple, std::vector<Optional<Tuple
 	}
 }
 
-Key constructMappedKey(KeyValueRef* keyValue,
-                       std::vector<Optional<Tuple>>& vec,
-                       Tuple& mappedKeyTuple,
-                       Tuple& mappedKeyFormatTuple) {
+Key constructMappedKey(KeyValueRef* keyValue, std::vector<Optional<Tuple>>& vec, Tuple& mappedKeyFormatTuple) {
 	// Lazily parse key and/or value to tuple because they may not need to be a tuple if not used.
 	Optional<Tuple> keyTuple;
 	Optional<Tuple> valueTuple;
-	mappedKeyTuple.clear();
+	Tuple mappedKeyTuple;
+
 	mappedKeyTuple.reserve(vec.size());
 
 	for (int i = 0; i < vec.size(); i++) {
@@ -3752,73 +4124,61 @@ Key constructMappedKey(KeyValueRef* keyValue,
 }
 
 TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
-	Key key = Tuple().append("key-0"_sr).append("key-1"_sr).append("key-2"_sr).getDataAsStandalone();
-	Value value = Tuple().append("value-0"_sr).append("value-1"_sr).append("value-2"_sr).getDataAsStandalone();
+	Key key = Tuple::makeTuple("key-0"_sr, "key-1"_sr, "key-2"_sr).getDataAsStandalone();
+	Value value = Tuple::makeTuple("value-0"_sr, "value-1"_sr, "value-2"_sr).getDataAsStandalone();
 	state KeyValueRef kvr(key, value);
 	{
-		Tuple mappedKeyFormatTuple = Tuple()
-		                                 .append("normal"_sr)
-		                                 .append("{{escaped}}"_sr)
-		                                 .append("{K[2]}"_sr)
-		                                 .append("{V[0]}"_sr)
-		                                 .append("{...}"_sr);
+		Tuple mappedKeyFormatTuple =
+		    Tuple::makeTuple("normal"_sr, "{{escaped}}"_sr, "{K[2]}"_sr, "{V[0]}"_sr, "{...}"_sr);
 
-		Tuple mappedKeyTuple;
 		std::vector<Optional<Tuple>> vt;
 		bool isRangeQuery = false;
 		preprocessMappedKey(mappedKeyFormatTuple, vt, isRangeQuery);
 
-		Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyTuple, mappedKeyFormatTuple);
+		Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyFormatTuple);
 
-		Key expectedMappedKey = Tuple()
-		                            .append("normal"_sr)
-		                            .append("{escaped}"_sr)
-		                            .append("key-2"_sr)
-		                            .append("value-0"_sr)
-		                            .getDataAsStandalone();
+		Key expectedMappedKey =
+		    Tuple::makeTuple("normal"_sr, "{escaped}"_sr, "key-2"_sr, "value-0"_sr).getDataAsStandalone();
 		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
 		ASSERT(mappedKey.compare(expectedMappedKey) == 0);
 		ASSERT(isRangeQuery == true);
 	}
 
 	{
-		Tuple mappedKeyFormatTuple = Tuple().append("{{{{}}"_sr).append("}}"_sr);
+		Tuple mappedKeyFormatTuple = Tuple::makeTuple("{{{{}}"_sr, "}}"_sr);
 
-		Tuple mappedKeyTuple;
 		std::vector<Optional<Tuple>> vt;
 		bool isRangeQuery = false;
 		preprocessMappedKey(mappedKeyFormatTuple, vt, isRangeQuery);
-		Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyTuple, mappedKeyFormatTuple);
+		Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyFormatTuple);
 
-		Key expectedMappedKey = Tuple().append("{{}"_sr).append("}"_sr).getDataAsStandalone();
+		Key expectedMappedKey = Tuple::makeTuple("{{}"_sr, "}"_sr).getDataAsStandalone();
 		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
 		ASSERT(mappedKey.compare(expectedMappedKey) == 0);
 		ASSERT(isRangeQuery == false);
 	}
 	{
-		Tuple mappedKeyFormatTuple = Tuple().append("{{{{}}"_sr).append("}}"_sr);
+		Tuple mappedKeyFormatTuple = Tuple::makeTuple("{{{{}}"_sr, "}}"_sr);
 
-		Tuple mappedKeyTuple;
 		std::vector<Optional<Tuple>> vt;
 		bool isRangeQuery = false;
 		preprocessMappedKey(mappedKeyFormatTuple, vt, isRangeQuery);
-		Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyTuple, mappedKeyFormatTuple);
+		Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyFormatTuple);
 
-		Key expectedMappedKey = Tuple().append("{{}"_sr).append("}"_sr).getDataAsStandalone();
+		Key expectedMappedKey = Tuple::makeTuple("{{}"_sr, "}"_sr).getDataAsStandalone();
 		//		std::cout << printable(mappedKey) << " == " << printable(expectedMappedKey) << std::endl;
 		ASSERT(mappedKey.compare(expectedMappedKey) == 0);
 		ASSERT(isRangeQuery == false);
 	}
 	{
-		Tuple mappedKeyFormatTuple = Tuple().append("{K[100]}"_sr);
+		Tuple mappedKeyFormatTuple = Tuple::makeTuple("{K[100]}"_sr);
 		state bool throwException = false;
 		try {
-			Tuple mappedKeyTuple;
 			std::vector<Optional<Tuple>> vt;
 			bool isRangeQuery = false;
 			preprocessMappedKey(mappedKeyFormatTuple, vt, isRangeQuery);
 
-			Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyTuple, mappedKeyFormatTuple);
+			Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyFormatTuple);
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_mapper_bad_index);
 			throwException = true;
@@ -3826,15 +4186,14 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		ASSERT(throwException);
 	}
 	{
-		Tuple mappedKeyFormatTuple = Tuple().append("{...}"_sr).append("last-element"_sr);
+		Tuple mappedKeyFormatTuple = Tuple::makeTuple("{...}"_sr, "last-element"_sr);
 		state bool throwException2 = false;
 		try {
-			Tuple mappedKeyTuple;
 			std::vector<Optional<Tuple>> vt;
 			bool isRangeQuery = false;
 			preprocessMappedKey(mappedKeyFormatTuple, vt, isRangeQuery);
 
-			Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyTuple, mappedKeyFormatTuple);
+			Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyFormatTuple);
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_mapper_bad_range_decriptor);
 			throwException2 = true;
@@ -3842,15 +4201,14 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 		ASSERT(throwException2);
 	}
 	{
-		Tuple mappedKeyFormatTuple = Tuple().append("{K[not-a-number]}"_sr);
+		Tuple mappedKeyFormatTuple = Tuple::makeTuple("{K[not-a-number]}"_sr);
 		state bool throwException3 = false;
 		try {
-			Tuple mappedKeyTuple;
 			std::vector<Optional<Tuple>> vt;
 			bool isRangeQuery = false;
 			preprocessMappedKey(mappedKeyFormatTuple, vt, isRangeQuery);
 
-			Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyTuple, mappedKeyFormatTuple);
+			Key mappedKey = constructMappedKey(&kvr, vt, mappedKeyFormatTuple);
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_mapper_bad_index);
 			throwException3 = true;
@@ -3904,14 +4262,15 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 	result.arena.dependsOn(input.arena);
 
 	result.data.reserve(result.arena, input.data.size());
-
+	if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
+		g_traceBatch.addEvent(
+		    "TransactionDebug", pOriginalReq->options.get().debugID.get().first(), "storageserver.mapKeyValues.Start");
 	state Tuple mappedKeyFormatTuple;
-	state Tuple mappedKeyTuple;
 
 	try {
 		mappedKeyFormatTuple = Tuple::unpack(mapper);
 	} catch (Error& e) {
-		TraceEvent("MapperNotTuple").error(e).detail("Mapper", mapper.printable());
+		TraceEvent("MapperNotTuple").error(e).detail("Mapper", mapper);
 		throw mapper_not_tuple();
 	}
 	state std::vector<Optional<Tuple>> vt;
@@ -3923,6 +4282,10 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 	state std::vector<MappedKeyValueRef> kvms(k);
 	state std::vector<Future<Void>> subqueries;
 	state int offset = 0;
+	if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
+		g_traceBatch.addEvent("TransactionDebug",
+		                      pOriginalReq->options.get().debugID.get().first(),
+		                      "storageserver.mapKeyValues.BeforeLoop");
 	for (; offset < sz; offset += SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE) {
 		// Divide into batches of MAX_PARALLEL_QUICK_GET_VALUE subqueries
 		for (int i = 0; i + offset < sz && i < SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE; i++) {
@@ -3939,7 +4302,7 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 				kvm->value = ""_sr;
 			}
 
-			Key mappedKey = constructMappedKey(it, vt, mappedKeyTuple, mappedKeyFormatTuple);
+			Key mappedKey = constructMappedKey(it, vt, mappedKeyFormatTuple);
 			// Make sure the mappedKey is always available, so that it's good even we want to get key asynchronously.
 			result.arena.dependsOn(mappedKey.arena());
 
@@ -3958,11 +4321,19 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 			                                 mappedKey));
 		}
 		wait(waitForAll(subqueries));
+		if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
+			g_traceBatch.addEvent("TransactionDebug",
+			                      pOriginalReq->options.get().debugID.get().first(),
+			                      "storageserver.mapKeyValues.AfterBatch");
 		subqueries.clear();
 		for (int i = 0; i + offset < sz && i < SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE; i++) {
 			result.data.push_back(result.arena, kvms[i]);
 		}
 	}
+	if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
+		g_traceBatch.addEvent("TransactionDebug",
+		                      pOriginalReq->options.get().debugID.get().first(),
+		                      "storageserver.mapKeyValues.AfterAll");
 	return result;
 }
 
@@ -3987,11 +4358,13 @@ bool rangeIntersectsAnyTenant(TenantPrefixIndex& prefixIndex, KeyRangeRef range,
 }
 
 TEST_CASE("/fdbserver/storageserver/rangeIntersectsAnyTenant") {
-	std::map<TenantName, TenantMapEntry> entries = { std::make_pair("tenant0"_sr, TenantMapEntry(0, ""_sr)),
-		                                             std::make_pair("tenant2"_sr, TenantMapEntry(2, ""_sr)),
-		                                             std::make_pair("tenant3"_sr, TenantMapEntry(3, ""_sr)),
-		                                             std::make_pair("tenant4"_sr, TenantMapEntry(4, ""_sr)),
-		                                             std::make_pair("tenant6"_sr, TenantMapEntry(6, ""_sr)) };
+	std::map<TenantName, TenantMapEntry> entries = {
+		std::make_pair("tenant0"_sr, TenantMapEntry(0, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION)),
+		std::make_pair("tenant2"_sr, TenantMapEntry(2, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION)),
+		std::make_pair("tenant3"_sr, TenantMapEntry(3, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION)),
+		std::make_pair("tenant4"_sr, TenantMapEntry(4, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION)),
+		std::make_pair("tenant6"_sr, TenantMapEntry(6, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION))
+	};
 	TenantPrefixIndex index;
 	index.createNewVersion(1);
 	for (auto entry : entries) {
@@ -4086,8 +4459,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 {
 	state Span span("SS:getMappedKeyValues"_loc, req.spanContext);
 	state int64_t resultSize = 0;
-	state IKeyValueStore::ReadType type =
-	    req.isFetchKeys ? IKeyValueStore::ReadType::FETCH : IKeyValueStore::ReadType::NORMAL;
+	state Optional<ReadOptions> options = req.options;
 
 	if (req.tenantInfo.name.present()) {
 		span.addAttribute("tenant"_sr, req.tenantInfo.name.get());
@@ -4097,25 +4469,29 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 
 	++data->counters.getMappedRangeQueries;
 	++data->counters.allQueries;
-	++data->readQueueSizeMetric;
 	data->maxQueryQueue = std::max<int>(
 	    data->maxQueryQueue, data->counters.allQueries.getValue() - data->counters.finishedQueries.getValue());
 
 	// Active load balancing runs at a very high priority (to obtain accurate queue lengths)
 	// so we need to downgrade here
-	if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.isFetchKeys) {
+	if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.options.present() && req.options.get().type == ReadType::FETCH) {
 		wait(delay(0, TaskPriority::FetchKeys));
 	} else {
 		wait(data->getQueryDelay());
 	}
 
+	// Track time from requestTime through now as read queueing wait time
+	state double queueWaitEnd = g_network->timer();
+	data->counters.readQueueWaitSample.addMeasurement(queueWaitEnd - req.requestTime());
+
 	try {
-		if (req.debugID.present())
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent(
-			    "TransactionDebug", req.debugID.get().first(), "storageserver.getMappedKeyValues.Before");
+			    "TransactionDebug", req.options.get().debugID.get().first(), "storageserver.getMappedKeyValues.Before");
 		// VERSION_VECTOR change
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
 		state Version version = wait(waitForVersion(data, commitVersion, req.version, span.context));
+		data->counters.readVersionWaitSample.addMeasurement(g_network->timer() - queueWaitEnd);
 
 		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(req.version, req.tenantInfo);
 		state Optional<Key> tenantPrefix = tenantEntry.map<Key>([](TenantMapEntry e) { return e.prefix; });
@@ -4128,9 +4504,10 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 		//		try {
 		state KeyRange shard = getShardKeyRange(data, req.begin);
 
-		if (req.debugID.present())
-			g_traceBatch.addEvent(
-			    "TransactionDebug", req.debugID.get().first(), "storageserver.getMappedKeyValues.AfterVersion");
+		if (req.options.present() && req.options.get().debugID.present())
+			g_traceBatch.addEvent("TransactionDebug",
+			                      req.options.get().debugID.get().first(),
+			                      "storageserver.getMappedKeyValues.AfterVersion");
 		//.detail("ShardBegin", shard.begin).detail("ShardEnd", shard.end);
 		//} catch (Error& e) { TraceEvent("WrongShardServer", data->thisServerID).detail("Begin",
 		// req.begin.toString()).detail("End", req.end.toString()).detail("Version", version).detail("Shard",
@@ -4147,12 +4524,13 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 
 		state int offset1 = 0;
 		state int offset2;
-		state Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
-		                               ? Future<Key>(req.begin.getKey())
-		                               : findKey(data, req.begin, version, searchRange, &offset1, span.context, type);
+		state Future<Key> fBegin =
+		    req.begin.isFirstGreaterOrEqual()
+		        ? Future<Key>(req.begin.getKey())
+		        : findKey(data, req.begin, version, searchRange, &offset1, span.context, options);
 		state Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
 		                             ? Future<Key>(req.end.getKey())
-		                             : findKey(data, req.end, version, searchRange, &offset2, span.context, type);
+		                             : findKey(data, req.end, version, searchRange, &offset2, span.context, options);
 		state Key begin = wait(fBegin);
 		state Key end = wait(fEnd);
 
@@ -4164,14 +4542,15 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 				throw tenant_name_required();
 			}
 
-			if (rangeIntersectsAnyTenant(data->tenantPrefixIndex, KeyRangeRef(begin, end), req.version)) {
+			if (rangeIntersectsAnyTenant(*(data->tenantPrefixIndex), KeyRangeRef(begin, end), req.version)) {
 				throw tenant_name_required();
 			}
 		}
 
-		if (req.debugID.present())
-			g_traceBatch.addEvent(
-			    "TransactionDebug", req.debugID.get().first(), "storageserver.getMappedKeyValues.AfterKeys");
+		if (req.options.present() && req.options.get().debugID.present())
+			g_traceBatch.addEvent("TransactionDebug",
+			                      req.options.get().debugID.get().first(),
+			                      "storageserver.getMappedKeyValues.AfterKeys");
 		//.detail("Off1",offset1).detail("Off2",offset2).detail("ReqBegin",req.begin.getKey()).detail("ReqEnd",req.end.getKey());
 
 		// Offsets of zero indicate begin/end keys in this shard, which obviously means we can answer the query
@@ -4179,7 +4558,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 		// end the last actual key returned must be from this shard. A begin offset of 1 is also OK because then either
 		// begin is past end or equal to end (so the result is definitely empty)
 		if ((offset1 && offset1 != 1) || (offset2 && offset2 != 1)) {
-			TEST(true); // wrong_shard_server due to offset in getMappedKeyValuesQ
+			CODE_PROBE(true, "wrong_shard_server due to offset in getMappedKeyValuesQ");
 			// We could detect when offset1 takes us off the beginning of the database or offset2 takes us off the end,
 			// and return a clipped range rather than an error (since that is what the NativeAPI.getRange will do anyway
 			// via its "slow path"), but we would have to add some flags to the response to encode whether we went off
@@ -4189,9 +4568,10 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 		}
 
 		if (begin >= end) {
-			if (req.debugID.present())
-				g_traceBatch.addEvent(
-				    "TransactionDebug", req.debugID.get().first(), "storageserver.getMappedKeyValues.Send");
+			if (req.options.present() && req.options.get().debugID.present())
+				g_traceBatch.addEvent("TransactionDebug",
+				                      req.options.get().debugID.get().first(),
+				                      "storageserver.getMappedKeyValues.Send");
 			//.detail("Begin",begin).detail("End",end);
 
 			GetMappedKeyValuesReply none;
@@ -4212,7 +4592,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 			                                                     req.limit,
 			                                                     &remainingLimitBytes,
 			                                                     span.context,
-			                                                     type,
+			                                                     options,
 			                                                     tenantPrefix));
 
 			state GetMappedKeyValuesReply r;
@@ -4226,9 +4606,10 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 				throw;
 			}
 
-			if (req.debugID.present())
-				g_traceBatch.addEvent(
-				    "TransactionDebug", req.debugID.get().first(), "storageserver.getMappedKeyValues.AfterReadRange");
+			if (req.options.present() && req.options.get().debugID.present())
+				g_traceBatch.addEvent("TransactionDebug",
+				                      req.options.get().debugID.get().first(),
+				                      "storageserver.getMappedKeyValues.AfterReadRange");
 			//.detail("Begin",begin).detail("End",end).detail("SizeOf",r.data.size());
 			data->checkChangeCounter(
 			    changeCounter,
@@ -4270,10 +4651,10 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 
 	data->transactionTagCounter.addRequest(req.tags, resultSize);
 	++data->counters.finishedQueries;
-	--data->readQueueSizeMetric;
 
 	double duration = g_network->timer() - req.requestTime();
 	data->counters.readLatencySample.addMeasurement(duration);
+	data->counters.mappedRangeSample.addMeasurement(duration);
 	if (data->latencyBandConfig.present()) {
 		int maxReadBytes =
 		    data->latencyBandConfig.get().readConfig.maxReadBytes.orDefault(std::numeric_limits<int>::max());
@@ -4294,8 +4675,7 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 {
 	state Span span("SS:getKeyValuesStream"_loc, req.spanContext);
 	state int64_t resultSize = 0;
-	state IKeyValueStore::ReadType type =
-	    req.isFetchKeys ? IKeyValueStore::ReadType::FETCH : IKeyValueStore::ReadType::NORMAL;
+	state Optional<ReadOptions> options = req.options;
 
 	if (req.tenantInfo.name.present()) {
 		span.addAttribute("tenant"_sr, req.tenantInfo.name.get());
@@ -4304,22 +4684,21 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 	req.reply.setByteLimit(SERVER_KNOBS->RANGESTREAM_LIMIT_BYTES);
 	++data->counters.getRangeStreamQueries;
 	++data->counters.allQueries;
-	++data->readQueueSizeMetric;
 	data->maxQueryQueue = std::max<int>(
 	    data->maxQueryQueue, data->counters.allQueries.getValue() - data->counters.finishedQueries.getValue());
 
 	// Active load balancing runs at a very high priority (to obtain accurate queue lengths)
 	// so we need to downgrade here
-	if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.isFetchKeys) {
+	if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.options.present() && req.options.get().type == ReadType::FETCH) {
 		wait(delay(0, TaskPriority::FetchKeys));
 	} else {
 		wait(delay(0, TaskPriority::DefaultEndpoint));
 	}
 
 	try {
-		if (req.debugID.present())
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent(
-			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesStream.Before");
+			    "TransactionDebug", req.options.get().debugID.get().first(), "storageserver.getKeyValuesStream.Before");
 
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
 		state Version version = wait(waitForVersion(data, commitVersion, req.version, span.context));
@@ -4335,9 +4714,10 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 		//		try {
 		state KeyRange shard = getShardKeyRange(data, req.begin);
 
-		if (req.debugID.present())
-			g_traceBatch.addEvent(
-			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesStream.AfterVersion");
+		if (req.options.present() && req.options.get().debugID.present())
+			g_traceBatch.addEvent("TransactionDebug",
+			                      req.options.get().debugID.get().first(),
+			                      "storageserver.getKeyValuesStream.AfterVersion");
 		//.detail("ShardBegin", shard.begin).detail("ShardEnd", shard.end);
 		//} catch (Error& e) { TraceEvent("WrongShardServer", data->thisServerID).detail("Begin",
 		// req.begin.toString()).detail("End", req.end.toString()).detail("Version", version).detail("Shard",
@@ -4354,17 +4734,19 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 
 		state int offset1 = 0;
 		state int offset2;
-		state Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
-		                               ? Future<Key>(req.begin.getKey())
-		                               : findKey(data, req.begin, version, searchRange, &offset1, span.context, type);
+		state Future<Key> fBegin =
+		    req.begin.isFirstGreaterOrEqual()
+		        ? Future<Key>(req.begin.getKey())
+		        : findKey(data, req.begin, version, searchRange, &offset1, span.context, options);
 		state Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
 		                             ? Future<Key>(req.end.getKey())
-		                             : findKey(data, req.end, version, searchRange, &offset2, span.context, type);
+		                             : findKey(data, req.end, version, searchRange, &offset2, span.context, options);
 		state Key begin = wait(fBegin);
 		state Key end = wait(fEnd);
-		if (req.debugID.present())
-			g_traceBatch.addEvent(
-			    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesStream.AfterKeys");
+		if (req.options.present() && req.options.get().debugID.present())
+			g_traceBatch.addEvent("TransactionDebug",
+			                      req.options.get().debugID.get().first(),
+			                      "storageserver.getKeyValuesStream.AfterKeys");
 		//.detail("Off1",offset1).detail("Off2",offset2).detail("ReqBegin",req.begin.getKey()).detail("ReqEnd",req.end.getKey());
 
 		// Offsets of zero indicate begin/end keys in this shard, which obviously means we can answer the query
@@ -4372,7 +4754,7 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 		// end the last actual key returned must be from this shard. A begin offset of 1 is also OK because then either
 		// begin is past end or equal to end (so the result is definitely empty)
 		if ((offset1 && offset1 != 1) || (offset2 && offset2 != 1)) {
-			TEST(true); // wrong_shard_server due to offset in rangeStream
+			CODE_PROBE(true, "wrong_shard_server due to offset in rangeStream");
 			// We could detect when offset1 takes us off the beginning of the database or offset2 takes us off the end,
 			// and return a clipped range rather than an error (since that is what the NativeAPI.getRange will do anyway
 			// via its "slow path"), but we would have to add some flags to the response to encode whether we went off
@@ -4382,9 +4764,10 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 		}
 
 		if (begin >= end) {
-			if (req.debugID.present())
-				g_traceBatch.addEvent(
-				    "TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValuesStream.Send");
+			if (req.options.present() && req.options.get().debugID.present())
+				g_traceBatch.addEvent("TransactionDebug",
+				                      req.options.get().debugID.get().first(),
+				                      "storageserver.getKeyValuesStream.Send");
 			//.detail("Begin",begin).detail("End",end);
 
 			GetKeyValuesStreamReply none;
@@ -4406,17 +4789,29 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 
 				// Even if TSS mode is Disabled, this may be the second test in a restarting test where the first run
 				// had it enabled.
-				state int byteLimit = (BUGGIFY && g_simulator.tssMode == ISimulator::TSSMode::Disabled &&
-				                       !data->isTss() && !data->isSSWithTSSPair())
-				                          ? 1
-				                          : CLIENT_KNOBS->REPLY_BYTE_LIMIT;
-				GetKeyValuesReply _r = wait(readRange(
-				    data, version, KeyRangeRef(begin, end), req.limit, &byteLimit, span.context, type, tenantPrefix));
+				state int byteLimit =
+				    (BUGGIFY && g_network->isSimulated() && g_simulator->tssMode == ISimulator::TSSMode::Disabled &&
+				     !data->isTss() && !data->isSSWithTSSPair())
+				        ? 1
+				        : CLIENT_KNOBS->REPLY_BYTE_LIMIT;
+				TraceEvent(SevDebug, "SSGetKeyValueStreamLimits")
+				    .detail("ByteLimit", byteLimit)
+				    .detail("ReqLimit", req.limit)
+				    .detail("Begin", begin.printable())
+				    .detail("End", end.printable());
+				GetKeyValuesReply _r = wait(readRange(data,
+				                                      version,
+				                                      KeyRangeRef(begin, end),
+				                                      req.limit,
+				                                      &byteLimit,
+				                                      span.context,
+				                                      options,
+				                                      tenantPrefix));
 				GetKeyValuesStreamReply r(_r);
 
-				if (req.debugID.present())
+				if (req.options.present() && req.options.get().debugID.present())
 					g_traceBatch.addEvent("TransactionDebug",
-					                      req.debugID.get().first(),
+					                      req.options.get().debugID.get().first(),
 					                      "storageserver.getKeyValuesStream.AfterReadRange");
 				//.detail("Begin",begin).detail("End",end).detail("SizeOf",r.data.size());
 				data->checkChangeCounter(
@@ -4471,7 +4866,8 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 					end = lastKey;
 				}
 
-				if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.isFetchKeys) {
+				if (SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY && req.options.present() &&
+				    req.options.get().type == ReadType::FETCH) {
 					wait(delay(0, TaskPriority::FetchKeys));
 				} else {
 					wait(delay(0, TaskPriority::DefaultEndpoint));
@@ -4490,7 +4886,6 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 
 	data->transactionTagCounter.addRequest(req.tags, resultSize);
 	++data->counters.finishedQueries;
-	--data->readQueueSizeMetric;
 
 	return Void();
 }
@@ -4501,11 +4896,14 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 		span.addAttribute("tenant"_sr, req.tenantInfo.name.get());
 	}
 	state int64_t resultSize = 0;
+	state ReadOptions options;
+	if (req.options.present()) {
+		options = req.options.get();
+	}
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.traceID;
 
 	++data->counters.getKeyQueries;
 	++data->counters.allQueries;
-	++data->readQueueSizeMetric;
 	data->maxQueryQueue = std::max<int>(
 	    data->maxQueryQueue, data->counters.allQueries.getValue() - data->counters.finishedQueries.getValue());
 
@@ -4513,9 +4911,14 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 	// so we need to downgrade here
 	wait(data->getQueryDelay());
 
+	// Track time from requestTime through now as read queueing wait time
+	state double queueWaitEnd = g_network->timer();
+	data->counters.readQueueWaitSample.addMeasurement(queueWaitEnd - req.requestTime());
+
 	try {
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
 		state Version version = wait(waitForVersion(data, commitVersion, req.version, req.spanContext));
+		data->counters.readVersionWaitSample.addMeasurement(g_network->timer() - queueWaitEnd);
 
 		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(version, req.tenantInfo);
 		if (tenantEntry.present()) {
@@ -4527,8 +4930,13 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 		KeyRangeRef searchRange = data->clampRangeToTenant(shard, tenantEntry, req.arena);
 
 		state int offset;
-		Key absoluteKey = wait(
-		    findKey(data, req.sel, version, searchRange, &offset, req.spanContext, IKeyValueStore::ReadType::NORMAL));
+		Key absoluteKey = wait(findKey(data,
+		                               req.sel,
+		                               version,
+		                               searchRange,
+		                               &offset,
+		                               req.spanContext,
+		                               req.options.present() ? options : Optional<ReadOptions>()));
 
 		data->checkChangeCounter(changeCounter,
 		                         KeyRangeRef(std::min<KeyRef>(req.sel.getKey(), absoluteKey),
@@ -4558,7 +4966,7 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 		auto cached = data->cachedRangeMap[absoluteKey];
 		// if (cached)
 		//	TraceEvent(SevDebug, "SSGetKeyCached").detail("Key", k).detail("Begin",
-		// shard.begin.printable()).detail("End", shard.end.printable());
+		// shard.begin).detail("End", shard.end);
 
 		GetKeyReply reply(updated, cached);
 		reply.penalty = data->getPenalty();
@@ -4577,10 +4985,11 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 	data->transactionTagCounter.addRequest(req.tags, resultSize);
 
 	++data->counters.finishedQueries;
-	--data->readQueueSizeMetric;
 
 	double duration = g_network->timer() - req.requestTime();
 	data->counters.readLatencySample.addMeasurement(duration);
+	data->counters.readKeyLatencySample.addMeasurement(duration);
+
 	if (data->latencyBandConfig.present()) {
 		int maxReadBytes =
 		    data->latencyBandConfig.get().readConfig.maxReadBytes.orDefault(std::numeric_limits<int>::max());
@@ -4624,11 +5033,12 @@ void getQueuingMetrics(StorageServer* self, StorageQueuingMetricsRequest const&
 
 ACTOR Future<Void> doEagerReads(StorageServer* data, UpdateEagerReadInfo* eager) {
 	eager->finishKeyBegin();
-
+	state ReadOptions options;
+	options.type = ReadType::EAGER;
 	if (SERVER_KNOBS->ENABLE_CLEAR_RANGE_EAGER_READS) {
 		std::vector<Future<Key>> keyEnd(eager->keyBegin.size());
 		for (int i = 0; i < keyEnd.size(); i++)
-			keyEnd[i] = data->storage.readNextKeyInclusive(eager->keyBegin[i], IKeyValueStore::ReadType::EAGER);
+			keyEnd[i] = data->storage.readNextKeyInclusive(eager->keyBegin[i], options);
 		data->counters.eagerReadsKeys += keyEnd.size();
 
 		state Future<std::vector<Key>> futureKeyEnds = getAll(keyEnd);
@@ -4641,8 +5051,7 @@ ACTOR Future<Void> doEagerReads(StorageServer* data, UpdateEagerReadInfo* eager)
 
 	std::vector<Future<Optional<Value>>> value(eager->keys.size());
 	for (int i = 0; i < value.size(); i++)
-		value[i] =
-		    data->storage.readValuePrefix(eager->keys[i].first, eager->keys[i].second, IKeyValueStore::ReadType::EAGER);
+		value[i] = data->storage.readValuePrefix(eager->keys[i].first, eager->keys[i].second, options);
 
 	state Future<std::vector<Optional<Value>>> futureValues = getAll(value);
 	std::vector<Optional<Value>> optionalValues = wait(futureValues);
@@ -4704,6 +5113,17 @@ bool changeDurableVersion(StorageServer* data, Version desiredDurableVersion) {
 		data->counters.bytesDurable += bytesDurable;
 	}
 
+	int64_t feedBytesDurable = 0;
+	while (!data->feedMemoryBytesByVersion.empty() &&
+	       data->feedMemoryBytesByVersion.front().first <= desiredDurableVersion) {
+		feedBytesDurable += data->feedMemoryBytesByVersion.front().second;
+		data->feedMemoryBytesByVersion.pop_front();
+	}
+	data->changeFeedMemoryBytes -= feedBytesDurable;
+	if (SERVER_KNOBS->STORAGE_INCLUDE_FEED_STORAGE_QUEUE) {
+		data->counters.bytesDurable += feedBytesDurable;
+	}
+
 	if (EXPENSIVE_VALIDATION) {
 		// Check that the above loop did its job
 		auto view = data->data().atLatest();
@@ -4747,7 +5167,7 @@ bool convertAtomicOp(MutationRef& m, StorageServer::VersionedData const& data, U
 		if (it != data.atLatest().end() && it->isValue() && it.key() == m.param1)
 			oldVal = it->getValue();
 		else if (it != data.atLatest().end() && it->isClearTo() && it->getEndKey() > m.param1) {
-			TEST(true); // Atomic op right after a clear.
+			CODE_PROBE(true, "Atomic op right after a clear.");
 		} else {
 			Optional<Value>& oldThing = eager->getValue(m.param1);
 			if (oldThing.present())
@@ -4889,14 +5309,17 @@ void applyChangeFeedMutation(StorageServer* self, MutationRef const& m, Version
 				}
 				it->mutations.back().mutations.push_back_deep(it->mutations.back().arena(), m);
 				self->currentChangeFeeds.insert(it->id);
+				self->addFeedBytesAtVersion(m.totalSize(), version);
 
 				DEBUG_MUTATION("ChangeFeedWriteSet", version, m, self->thisServerID)
 				    .detail("Range", it->range)
 				    .detail("ChangeFeedID", it->id);
+
+				++self->counters.changeFeedMutations;
 			} else {
-				TEST(version <= it->emptyVersion); // Skip CF write because version <= emptyVersion
-				TEST(it->removing); // Skip CF write because removing
-				TEST(version >= it->stopVersion); // Skip CF write because stopped
+				CODE_PROBE(version <= it->emptyVersion, "Skip CF write because version <= emptyVersion");
+				CODE_PROBE(it->removing, "Skip CF write because removing");
+				CODE_PROBE(version >= it->stopVersion, "Skip CF write because stopped");
 				DEBUG_MUTATION("ChangeFeedWriteSetIgnore", version, m, self->thisServerID)
 				    .detail("Range", it->range)
 				    .detail("ChangeFeedID", it->id)
@@ -4915,13 +5338,16 @@ void applyChangeFeedMutation(StorageServer* self, MutationRef const& m, Version
 					}
 					it->mutations.back().mutations.push_back_deep(it->mutations.back().arena(), m);
 					self->currentChangeFeeds.insert(it->id);
+					self->addFeedBytesAtVersion(m.totalSize(), version);
+
 					DEBUG_MUTATION("ChangeFeedWriteClear", version, m, self->thisServerID)
 					    .detail("Range", it->range)
 					    .detail("ChangeFeedID", it->id);
+					++self->counters.changeFeedMutations;
 				} else {
-					TEST(version <= it->emptyVersion); // Skip CF clear because version <= emptyVersion
-					TEST(it->removing); // Skip CF clear because removing
-					TEST(version >= it->stopVersion); // Skip CF clear because stopped
+					CODE_PROBE(version <= it->emptyVersion, "Skip CF clear because version <= emptyVersion");
+					CODE_PROBE(it->removing, "Skip CF clear because removing");
+					CODE_PROBE(version >= it->stopVersion, "Skip CF clear because stopped");
 					DEBUG_MUTATION("ChangeFeedWriteClearIgnore", version, m, self->thisServerID)
 					    .detail("Range", it->range)
 					    .detail("ChangeFeedID", it->id)
@@ -4978,6 +5404,34 @@ void removeDataRange(StorageServer* ss,
 
 void setAvailableStatus(StorageServer* self, KeyRangeRef keys, bool available);
 void setAssignedStatus(StorageServer* self, KeyRangeRef keys, bool nowAssigned);
+void updateStorageShard(StorageServer* self, StorageServerShard shard);
+
+void coalescePhysicalShards(StorageServer* data, KeyRangeRef keys) {
+	auto shardRanges = data->shards.intersectingRanges(keys);
+	auto fullRange = data->shards.ranges();
+
+	auto iter = shardRanges.begin();
+	if (iter != fullRange.begin()) {
+		--iter;
+	}
+	auto iterEnd = shardRanges.end();
+	if (iterEnd != fullRange.end()) {
+		++iterEnd;
+	}
+
+	KeyRangeMap<Reference<ShardInfo>>::iterator lastShard = iter;
+	++iter;
+
+	for (; iter != iterEnd; ++iter) {
+		if (ShardInfo::canMerge(lastShard.value().getPtr(), iter->value().getPtr())) {
+			ShardInfo* newShard = lastShard.value().extractPtr();
+			ASSERT(newShard->mergeWith(iter->value().getPtr()));
+			data->addShard(newShard);
+			iter = data->shards.rangeContaining(newShard->keys.begin);
+		}
+		lastShard = iter;
+	}
+}
 
 void coalesceShards(StorageServer* data, KeyRangeRef keys) {
 	auto shardRanges = data->shards.intersectingRanges(keys);
@@ -5057,8 +5511,8 @@ ACTOR Future<Void> logFetchKeysWarning(AddingShard* shard) {
 		TraceEvent(traceEventLevel, "FetchKeysTooLong")
 		    .detail("Duration", now() - startTime)
 		    .detail("Phase", shard->phase)
-		    .detail("Begin", shard->keys.begin.printable())
-		    .detail("End", shard->keys.end.printable());
+		    .detail("Begin", shard->keys.begin)
+		    .detail("End", shard->keys.end);
 	}
 }
 
@@ -5109,6 +5563,11 @@ public:
 };
 
 ACTOR Future<Void> tryGetRange(PromiseStream<RangeResult> results, Transaction* tr, KeyRange keys) {
+	if (SERVER_KNOBS->FETCH_USING_STREAMING) {
+		wait(tr->getRangeStream(results, keys, GetRangeLimits(), Snapshot::True));
+		return Void();
+	}
+
 	state KeySelectorRef begin = firstGreaterOrEqual(keys.begin);
 	state KeySelectorRef end = firstGreaterOrEqual(keys.end);
 
@@ -5142,25 +5601,99 @@ ACTOR Future<Void> tryGetRange(PromiseStream<RangeResult> results, Transaction*
 	}
 }
 
+// Read blob granules mapping from system keyspace. It keeps retrying until reaching maxRetryCount.
+ACTOR Future<Standalone<VectorRef<BlobGranuleChunkRef>>> tryReadBlobGranules(Transaction* tr,
+                                                                             KeyRange keys,
+                                                                             Version fetchVersion,
+                                                                             int maxRetryCount = 10) {
+	state int retryCount = 0;
+	state Version readVersion = fetchVersion;
+	loop {
+		try {
+			Standalone<VectorRef<BlobGranuleChunkRef>> chunks = wait(tr->readBlobGranules(keys, 0, readVersion));
+			return chunks;
+		} catch (Error& e) {
+			if (retryCount >= maxRetryCount) {
+				throw e;
+			}
+			wait(tr->onError(e));
+			retryCount += 1;
+		}
+	}
+}
+
+// Read keys from blob storage if they exist. Fail back to tryGetRange, which reads keys
+// from storage servers with locally attached disks
+ACTOR Future<Void> tryGetRangeFromBlob(PromiseStream<RangeResult> results,
+                                       Transaction* tr,
+                                       KeyRange keys,
+                                       Version fetchVersion,
+                                       Reference<BlobConnectionProvider> blobConn) {
+	ASSERT(blobConn.isValid());
+	try {
+
+		state Standalone<VectorRef<BlobGranuleChunkRef>> chunks = wait(tryReadBlobGranules(tr, keys, fetchVersion));
+
+		if (chunks.size() == 0) {
+			throw blob_granule_transaction_too_old(); // no data on blob
+		}
+
+		if (!isRangeFullyCovered(keys, chunks)) {
+			throw blob_granule_transaction_too_old();
+		}
+
+		for (const BlobGranuleChunkRef& chunk : chunks) {
+			state KeyRangeRef chunkRange = chunk.keyRange;
+			state RangeResult rows = wait(readBlobGranule(chunk, keys, 0, fetchVersion, blobConn));
+			TraceEvent("ReadBlobData")
+			    .detail("Rows", rows.size())
+			    .detail("ChunkRange", chunkRange.toString())
+			    .detail("Keys", keys.toString());
+
+			if (rows.size() == 0) {
+				rows.readThrough = KeyRef(rows.arena(), chunkRange.end);
+			}
+			results.send(rows);
+		}
+		results.sendError(end_of_stream()); // end of range read
+	} catch (Error& e) {
+		TraceEvent(SevWarn, "ReadBlobDataFailure")
+		    .suppressFor(5.0)
+		    .detail("Keys", keys.toString())
+		    .detail("FetchVersion", fetchVersion)
+		    .detail("Error", e.what());
+		tr->reset();
+		tr->setVersion(fetchVersion);
+		tr->trState->taskID = TaskPriority::FetchKeys;
+		wait(tryGetRange(results, tr, keys)); // fail back to storage server
+	}
+	return Void();
+}
+
 // We have to store the version the change feed was stopped at in the SS instead of just the stopped status
 // In addition to simplifying stopping logic, it enables communicating stopped status when fetching change feeds
 // from other SS correctly
-const Value changeFeedSSValue(KeyRangeRef const& range, Version popVersion, Version stopVersion) {
+const Value changeFeedSSValue(KeyRangeRef const& range,
+                              Version popVersion,
+                              Version stopVersion,
+                              Version metadataVersion) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withChangeFeed()));
 	wr << range;
 	wr << popVersion;
 	wr << stopVersion;
+	wr << metadataVersion;
 	return wr.toValue();
 }
 
-std::tuple<KeyRange, Version, Version> decodeChangeFeedSSValue(ValueRef const& value) {
+std::tuple<KeyRange, Version, Version, Version> decodeChangeFeedSSValue(ValueRef const& value) {
 	KeyRange range;
-	Version popVersion, stopVersion;
+	Version popVersion, stopVersion, metadataVersion;
 	BinaryReader reader(value, IncludeVersion());
 	reader >> range;
 	reader >> popVersion;
 	reader >> stopVersion;
-	return std::make_tuple(range, popVersion, stopVersion);
+	reader >> metadataVersion;
+	return std::make_tuple(range, popVersion, stopVersion, metadataVersion);
 }
 
 ACTOR Future<Void> changeFeedPopQ(StorageServer* self, ChangeFeedPopRequest req) {
@@ -5179,7 +5712,7 @@ ACTOR Future<Void> changeFeedPopQ(StorageServer* self, ChangeFeedPopRequest req)
 	}
 
 	TraceEvent(SevDebug, "ChangeFeedPopQuery", self->thisServerID)
-	    .detail("RangeID", req.rangeID.printable())
+	    .detail("FeedID", req.rangeID)
 	    .detail("Version", req.version)
 	    .detail("SSVersion", self->version.get())
 	    .detail("Range", req.range);
@@ -5194,10 +5727,12 @@ ACTOR Future<Void> changeFeedPopQ(StorageServer* self, ChangeFeedPopRequest req)
 			auto& mLV = self->addVersionToMutationLog(durableVersion);
 			self->addMutationToMutationLog(
 			    mLV,
-			    MutationRef(
-			        MutationRef::SetValue,
-			        persistChangeFeedKeys.begin.toString() + feed->second->id.toString(),
-			        changeFeedSSValue(feed->second->range, feed->second->emptyVersion + 1, feed->second->stopVersion)));
+			    MutationRef(MutationRef::SetValue,
+			                persistChangeFeedKeys.begin.toString() + feed->second->id.toString(),
+			                changeFeedSSValue(feed->second->range,
+			                                  feed->second->emptyVersion + 1,
+			                                  feed->second->stopVersion,
+			                                  feed->second->metadataVersion)));
 			if (feed->second->storageVersion != invalidVersion) {
 				++self->counters.kvSystemClearRanges;
 				self->addMutationToMutationLog(mLV,
@@ -5234,10 +5769,10 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 	ASSERT(startVersion >= 0);
 
 	if (startVersion >= endVersion || (changeFeedInfo->removing)) {
-		TEST(true); // Change Feed popped before fetch
+		CODE_PROBE(true, "Change Feed popped before fetch");
 		TraceEvent(SevDebug, "FetchChangeFeedNoOp", data->thisServerID)
-		    .detail("RangeID", rangeId.printable())
-		    .detail("Range", range.toString())
+		    .detail("FeedID", rangeId)
+		    .detail("Range", range)
 		    .detail("StartVersion", startVersion)
 		    .detail("EndVersion", endVersion)
 		    .detail("Removing", changeFeedInfo->removing);
@@ -5252,17 +5787,9 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 	state Version lastVersion = invalidVersion;
 	state int64_t versionsFetched = 0;
 
-	state PromiseStream<Standalone<MutationsAndVersionRef>> localResults;
+	// ensure SS is at least caught up to begin version, to maintain behavior with old fetch
+	wait(data->version.whenAtLeast(startVersion));
 
-	// Add 1 to fetch version to make sure the local stream will have more versions in the stream than the remote stream
-	// to avoid edge cases in the merge logic
-
-	state Future<Void> localStream =
-	    localChangeFeedStream(data, localResults, rangeId, startVersion, endVersion + 1, range);
-	state Standalone<MutationsAndVersionRef> localResult;
-
-	Standalone<MutationsAndVersionRef> _localResult = waitNext(localResults.getFuture());
-	localResult = _localResult;
 	try {
 		loop {
 			while (data->fetchKeysBudgetUsed.get()) {
@@ -5272,10 +5799,14 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 			state Standalone<VectorRef<MutationsAndVersionRef>> remoteResult =
 			    waitNext(feedResults->mutations.getFuture());
 			state int remoteLoc = 0;
+			// ensure SS is at least caught up to begin version, to maintain behavior with old fetch
+			if (!remoteResult.empty()) {
+				wait(data->version.whenAtLeast(remoteResult.back().version));
+			}
 
 			while (remoteLoc < remoteResult.size()) {
 				if (feedResults->popVersion - 1 > changeFeedInfo->emptyVersion) {
-					TEST(true); // CF fetched updated popped version from src SS
+					CODE_PROBE(true, "CF fetched updated popped version from src SS");
 					changeFeedInfo->emptyVersion = feedResults->popVersion - 1;
 					// pop mutations
 					while (!changeFeedInfo->mutations.empty() &&
@@ -5289,7 +5820,8 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 					                persistChangeFeedKeys.begin.toString() + changeFeedInfo->id.toString(),
 					                changeFeedSSValue(changeFeedInfo->range,
 					                                  changeFeedInfo->emptyVersion + 1,
-					                                  changeFeedInfo->stopVersion)));
+					                                  changeFeedInfo->stopVersion,
+					                                  changeFeedInfo->metadataVersion)));
 					data->addMutationToMutationLog(
 					    mLV,
 					    MutationRef(MutationRef::ClearRange,
@@ -5298,80 +5830,53 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 					++data->counters.kvSystemClearRanges;
 				}
 
-				Version localVersion = localResult.version;
 				Version remoteVersion = remoteResult[remoteLoc].version;
-
-				if (remoteVersion <= localVersion) {
-					if (remoteVersion > changeFeedInfo->emptyVersion) {
-						// merge if same version
-						if (remoteVersion == localVersion && remoteResult[remoteLoc].mutations.size() &&
-						    remoteResult[remoteLoc].mutations.back().param1 != lastEpochEndPrivateKey) {
-							int remoteSize = remoteResult[remoteLoc].mutations.size();
-							ASSERT(localResult.mutations.size());
-							remoteResult[remoteLoc].mutations.append(
-							    remoteResult.arena(), localResult.mutations.begin(), localResult.mutations.size());
-							if (MUTATION_TRACKING_ENABLED) {
-								int midx = 0;
-								for (auto& m : remoteResult[remoteLoc].mutations) {
-									DEBUG_MUTATION("ChangeFeedWriteMoveMerge", remoteVersion, m, data->thisServerID)
-									    .detail("Range", range)
-									    .detail("FromLocal", midx >= remoteSize)
-									    .detail("ChangeFeedID", rangeId);
-									midx++;
-								}
-							}
-						} else {
-							if (MUTATION_TRACKING_ENABLED) {
-								for (auto& m : remoteResult[remoteLoc].mutations) {
-									DEBUG_MUTATION("ChangeFeedWriteMove", remoteVersion, m, data->thisServerID)
-									    .detail("Range", range)
-									    .detail("ChangeFeedID", rangeId);
-								}
-							}
-						}
-
-						data->storage.writeKeyValue(
-						    KeyValueRef(changeFeedDurableKey(rangeId, remoteVersion),
-						                changeFeedDurableValue(remoteResult[remoteLoc].mutations,
-						                                       remoteResult[remoteLoc].knownCommittedVersion)));
-						++data->counters.kvSystemClearRanges;
-						changeFeedInfo->fetchVersion = std::max(changeFeedInfo->fetchVersion, remoteVersion);
-
-						if (firstVersion == invalidVersion) {
-							firstVersion = remoteVersion;
-						}
-						lastVersion = remoteVersion;
-						versionsFetched++;
-					} else {
-						TEST(true); // Change feed ignoring write on move because it was popped concurrently
-						if (MUTATION_TRACKING_ENABLED) {
-							for (auto& m : remoteResult[remoteLoc].mutations) {
-								DEBUG_MUTATION("ChangeFeedWriteMoveIgnore", remoteVersion, m, data->thisServerID)
-								    .detail("Range", range)
-								    .detail("ChangeFeedID", rangeId)
-								    .detail("EmptyVersion", changeFeedInfo->emptyVersion);
-							}
-						}
-						if (versionsFetched > 0) {
-							ASSERT(firstVersion != invalidVersion);
-							ASSERT(lastVersion != invalidVersion);
-							data->storage.clearRange(
-							    KeyRangeRef(changeFeedDurableKey(changeFeedInfo->id, firstVersion),
-							                changeFeedDurableKey(changeFeedInfo->id, lastVersion + 1)));
-							++data->counters.kvSystemClearRanges;
-							firstVersion = invalidVersion;
-							lastVersion = invalidVersion;
-							versionsFetched = 0;
+				// ensure SS is at least caught up to this version, to maintain behavior with old fetch
+				ASSERT(remoteVersion <= data->version.get());
+				if (remoteVersion > changeFeedInfo->emptyVersion) {
+					if (MUTATION_TRACKING_ENABLED) {
+						for (auto& m : remoteResult[remoteLoc].mutations) {
+							DEBUG_MUTATION("ChangeFeedWriteMove", remoteVersion, m, data->thisServerID)
+							    .detail("Range", range)
+							    .detail("ChangeFeedID", rangeId);
 						}
 					}
-					remoteLoc++;
-				}
-				if (localVersion <= remoteVersion) {
-					// Do this once per wait instead of once per version for efficiency
-					data->fetchingChangeFeeds.insert(changeFeedInfo->id);
-					Standalone<MutationsAndVersionRef> _localResult = waitNext(localResults.getFuture());
-					localResult = _localResult;
+
+					data->storage.writeKeyValue(
+					    KeyValueRef(changeFeedDurableKey(rangeId, remoteVersion),
+					                changeFeedDurableValue(remoteResult[remoteLoc].mutations,
+					                                       remoteResult[remoteLoc].knownCommittedVersion)));
+					++data->counters.kvSystemClearRanges;
+					changeFeedInfo->fetchVersion = std::max(changeFeedInfo->fetchVersion, remoteVersion);
+
+					if (firstVersion == invalidVersion) {
+						firstVersion = remoteVersion;
+					}
+					lastVersion = remoteVersion;
+					versionsFetched++;
+				} else {
+					CODE_PROBE(true, "Change feed ignoring write on move because it was popped concurrently");
+					if (MUTATION_TRACKING_ENABLED) {
+						for (auto& m : remoteResult[remoteLoc].mutations) {
+							DEBUG_MUTATION("ChangeFeedWriteMoveIgnore", remoteVersion, m, data->thisServerID)
+							    .detail("Range", range)
+							    .detail("ChangeFeedID", rangeId)
+							    .detail("EmptyVersion", changeFeedInfo->emptyVersion);
+						}
+					}
+					if (versionsFetched > 0) {
+						ASSERT(firstVersion != invalidVersion);
+						ASSERT(lastVersion != invalidVersion);
+						data->storage.clearRange(
+						    KeyRangeRef(changeFeedDurableKey(changeFeedInfo->id, firstVersion),
+						                changeFeedDurableKey(changeFeedInfo->id, lastVersion + 1)));
+						++data->counters.kvSystemClearRanges;
+						firstVersion = invalidVersion;
+						lastVersion = invalidVersion;
+						versionsFetched = 0;
+					}
 				}
+				remoteLoc++;
 			}
 			// Do this once per wait instead of once per version for efficiency
 			data->fetchingChangeFeeds.insert(changeFeedInfo->id);
@@ -5387,8 +5892,8 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 		if (e.code() != error_code_end_of_stream) {
 			TraceEvent(SevDebug, "FetchChangeFeedError", data->thisServerID)
 			    .errorUnsuppressed(e)
-			    .detail("RangeID", rangeId.printable())
-			    .detail("Range", range.toString())
+			    .detail("FeedID", rangeId)
+			    .detail("Range", range)
 			    .detail("EndVersion", endVersion)
 			    .detail("Removing", changeFeedInfo->removing)
 			    .detail("Destroyed", changeFeedInfo->destroyed);
@@ -5397,7 +5902,7 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 	}
 
 	if (feedResults->popVersion - 1 > changeFeedInfo->emptyVersion) {
-		TEST(true); // CF fetched updated popped version from src SS at end
+		CODE_PROBE(true, "CF fetched updated popped version from src SS at end");
 		changeFeedInfo->emptyVersion = feedResults->popVersion - 1;
 		while (!changeFeedInfo->mutations.empty() &&
 		       changeFeedInfo->mutations.front().version <= changeFeedInfo->emptyVersion) {
@@ -5408,8 +5913,10 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 		    mLV,
 		    MutationRef(MutationRef::SetValue,
 		                persistChangeFeedKeys.begin.toString() + changeFeedInfo->id.toString(),
-		                changeFeedSSValue(
-		                    changeFeedInfo->range, changeFeedInfo->emptyVersion + 1, changeFeedInfo->stopVersion)));
+		                changeFeedSSValue(changeFeedInfo->range,
+		                                  changeFeedInfo->emptyVersion + 1,
+		                                  changeFeedInfo->stopVersion,
+		                                  changeFeedInfo->metadataVersion)));
 		data->addMutationToMutationLog(mLV,
 		                               MutationRef(MutationRef::ClearRange,
 		                                           changeFeedDurableKey(changeFeedInfo->id, 0),
@@ -5419,7 +5926,7 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 
 	// if we were popped or removed while fetching but it didn't pass the fetch version while writing, clean up here
 	if (versionsFetched > 0 && startVersion < changeFeedInfo->emptyVersion) {
-		TEST(true); // Change feed cleaning up popped data after move
+		CODE_PROBE(true, "Change feed cleaning up popped data after move");
 		ASSERT(firstVersion != invalidVersion);
 		ASSERT(lastVersion != invalidVersion);
 		Version endClear = std::min(lastVersion + 1, changeFeedInfo->emptyVersion);
@@ -5434,8 +5941,8 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
 	}
 
 	TraceEvent(SevDebug, "FetchChangeFeedDone", data->thisServerID)
-	    .detail("RangeID", rangeId.printable())
-	    .detail("Range", range.toString())
+	    .detail("FeedID", rangeId)
+	    .detail("Range", range)
 	    .detail("StartVersion", startVersion)
 	    .detail("EndVersion", endVersion)
 	    .detail("EmptyVersion", changeFeedInfo->emptyVersion)
@@ -5453,22 +5960,18 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
                                       Version endVersion) {
 	wait(delay(0)); // allow this actor to be cancelled by removals
 
-	// bound active change feed fetches
-	wait(data->fetchChangeFeedParallelismLock.take(TaskPriority::DefaultYield));
-	state FlowLock::Releaser holdingFCFPL(data->fetchChangeFeedParallelismLock);
-
 	TraceEvent(SevDebug, "FetchChangeFeed", data->thisServerID)
-	    .detail("RangeID", changeFeedInfo->id.printable())
-	    .detail("Range", changeFeedInfo->range.toString())
+	    .detail("FeedID", changeFeedInfo->id)
+	    .detail("Range", changeFeedInfo->range)
 	    .detail("BeginVersion", beginVersion)
 	    .detail("EndVersion", endVersion);
 
 	auto cleanupPending = data->changeFeedCleanupDurable.find(changeFeedInfo->id);
 	if (cleanupPending != data->changeFeedCleanupDurable.end()) {
-		TEST(true); // Change feed waiting for dirty previous move to finish
+		CODE_PROBE(true, "Change feed waiting for dirty previous move to finish");
 		TraceEvent(SevDebug, "FetchChangeFeedWaitCleanup", data->thisServerID)
-		    .detail("RangeID", changeFeedInfo->id.printable())
-		    .detail("Range", changeFeedInfo->range.toString())
+		    .detail("FeedID", changeFeedInfo->id)
+		    .detail("Range", changeFeedInfo->range)
 		    .detail("CleanupVersion", cleanupPending->second)
 		    .detail("EmptyVersion", changeFeedInfo->emptyVersion)
 		    .detail("BeginVersion", beginVersion)
@@ -5480,8 +5983,8 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
 		if (cleanupPendingAfter != data->changeFeedCleanupDurable.end()) {
 			ASSERT(cleanupPendingAfter->second >= endVersion);
 			TraceEvent(SevDebug, "FetchChangeFeedCancelledByCleanup", data->thisServerID)
-			    .detail("RangeID", changeFeedInfo->id.printable())
-			    .detail("Range", changeFeedInfo->range.toString())
+			    .detail("FeedID", changeFeedInfo->id)
+			    .detail("Range", changeFeedInfo->range)
 			    .detail("BeginVersion", beginVersion)
 			    .detail("EndVersion", endVersion);
 			return invalidVersion;
@@ -5506,13 +6009,6 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
 			}
 		}
 
-		/*fmt::print("DBG: SS {} Feed {} possibly destroyed {}, {} metadata create, {} desired committed\n",
-		           data->thisServerID.toString().substr(0, 4),
-		           changeFeedInfo->id.printable(),
-		           changeFeedInfo->possiblyDestroyed,
-		           changeFeedInfo->metadataCreateVersion,
-		           data->desiredOldestVersion.get());*/
-
 		// There are two reasons for change_feed_not_registered:
 		//   1. The feed was just created, but the ss mutation stream is ahead of the GRV that fetchChangeFeedApplier
 		//   uses to read the change feed data from the database. In this case we need to wait and retry
@@ -5524,12 +6020,12 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
 			Version cleanupVersion = data->data().getLatestVersion();
 
 			TraceEvent(SevDebug, "DestroyingChangeFeedFromFetch", data->thisServerID)
-			    .detail("RangeID", changeFeedInfo->id.printable())
-			    .detail("Range", changeFeedInfo->range.toString())
+			    .detail("FeedID", changeFeedInfo->id)
+			    .detail("Range", changeFeedInfo->range)
 			    .detail("Version", cleanupVersion);
 
 			if (g_network->isSimulated()) {
-				ASSERT(g_simulator.validationData.allDestroyedChangeFeedIDs.count(changeFeedInfo->id.toString()));
+				ASSERT(g_simulator->validationData.allDestroyedChangeFeedIDs.count(changeFeedInfo->id.toString()));
 			}
 
 			Key beginClearKey = changeFeedInfo->id.withPrefix(persistChangeFeedKeys.begin);
@@ -5551,7 +6047,7 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
 				data->changeFeedCleanupDurable[changeFeedInfo->id] = cleanupVersion;
 			}
 
-			for (auto& it : data->changeFeedRemovals) {
+			for (auto& it : data->changeFeedDestroys) {
 				it.second.send(changeFeedInfo->id);
 			}
 
@@ -5567,7 +6063,7 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
 
 ACTOR Future<std::vector<Key>> fetchChangeFeedMetadata(StorageServer* data,
                                                        KeyRange keys,
-                                                       PromiseStream<Key> removals,
+                                                       PromiseStream<Key> destroyedFeeds,
                                                        UID fetchKeysID) {
 
 	// Wait for current TLog batch to finish to ensure that we're fetching metadata at a version >= the version of the
@@ -5581,106 +6077,112 @@ ACTOR Future<std::vector<Key>> fetchChangeFeedMetadata(StorageServer* data,
 	    .detail("FetchVersion", fetchVersion)
 	    .detail("FKID", fetchKeysID);
 
-	state std::set<Key> refreshedFeedIds;
-	state std::set<Key> destroyedFeedIds;
-	// before fetching feeds from other SS's, refresh any feeds we already have that are being marked as removed
+	state OverlappingChangeFeedsInfo feedMetadata = wait(data->cx->getOverlappingChangeFeeds(keys, fetchVersion));
+	// rest of this actor needs to happen without waits that might yield to scheduler, to avoid races in feed metadata.
+
+	// Find set of feeds we currently have that were not present in fetch, to infer that they may have been destroyed.
+	state std::unordered_map<Key, Version> missingFeeds;
 	auto ranges = data->keyChangeFeed.intersectingRanges(keys);
 	for (auto& r : ranges) {
 		for (auto& cfInfo : r.value()) {
-			auto feedCleanup = data->changeFeedCleanupDurable.find(cfInfo->id);
-			if (feedCleanup != data->changeFeedCleanupDurable.end() && cfInfo->removing && !cfInfo->destroyed) {
-				TEST(true); // re-fetching feed scheduled for deletion! Un-mark it as removing
-				destroyedFeedIds.insert(cfInfo->id);
-
-				cfInfo->removing = false;
-				// because we now have a gap in the metadata, it's possible this feed was destroyed
-				cfInfo->possiblyDestroyed = true;
-				// Set refreshInProgress, so that if this actor is replaced by an expanded move actor, the new actor
-				// picks up the refresh
-				cfInfo->refreshInProgress = true;
-				// reset fetch versions because everything previously fetched was cleaned up
-				cfInfo->fetchVersion = invalidVersion;
-
-				cfInfo->durableFetchVersion = NotifiedVersion();
-
-				TraceEvent(SevDebug, "ResetChangeFeedInfo", data->thisServerID)
-				    .detail("RangeID", cfInfo->id.printable())
-				    .detail("Range", cfInfo->range)
-				    .detail("FetchVersion", fetchVersion)
-				    .detail("EmptyVersion", cfInfo->emptyVersion)
-				    .detail("StopVersion", cfInfo->stopVersion)
-				    .detail("FKID", fetchKeysID);
-			} else if (cfInfo->refreshInProgress) {
-				TEST(true); // Racing refreshes for same change feed in fetch
-				destroyedFeedIds.insert(cfInfo->id);
+			if (cfInfo->removing && !cfInfo->destroyed) {
+				missingFeeds.insert({ cfInfo->id, cfInfo->metadataVersion });
 			}
 		}
 	}
 
-	state std::vector<OverlappingChangeFeedEntry> feeds = wait(data->cx->getOverlappingChangeFeeds(keys, fetchVersion));
-	// handle change feeds removed while fetching overlapping
-	while (removals.getFuture().isReady()) {
-		Key remove = waitNext(removals.getFuture());
-		for (int i = 0; i < feeds.size(); i++) {
-			if (feeds[i].rangeId == remove) {
-				swapAndPop(&feeds, i--);
+	// handle change feeds destroyed while fetching overlapping info
+	while (destroyedFeeds.getFuture().isReady()) {
+		Key destroyed = waitNext(destroyedFeeds.getFuture());
+		for (int i = 0; i < feedMetadata.feeds.size(); i++) {
+			if (feedMetadata.feeds[i].feedId == destroyed) {
+				missingFeeds.erase(destroyed); // feed definitely destroyed, no need to infer
+				swapAndPop(&feedMetadata.feeds, i--);
 			}
 		}
 	}
+	// FIXME: might want to inject delay here sometimes in simulation, so that races that would only happen when a feed
+	// destroy causes a wait are more prominent?
 
 	std::vector<Key> feedIds;
-	feedIds.reserve(feeds.size());
+	feedIds.reserve(feedMetadata.feeds.size());
 	// create change feed metadata if it does not exist
-	for (auto& cfEntry : feeds) {
-		auto cleanupEntry = data->changeFeedCleanupDurable.find(cfEntry.rangeId);
+	for (auto& cfEntry : feedMetadata.feeds) {
+		auto cleanupEntry = data->changeFeedCleanupDurable.find(cfEntry.feedId);
 		bool cleanupPending = cleanupEntry != data->changeFeedCleanupDurable.end();
-		feedIds.push_back(cfEntry.rangeId);
-		auto existingEntry = data->uidChangeFeed.find(cfEntry.rangeId);
+		auto existingEntry = data->uidChangeFeed.find(cfEntry.feedId);
 		bool existing = existingEntry != data->uidChangeFeed.end();
 
 		TraceEvent(SevDebug, "FetchedChangeFeedInfo", data->thisServerID)
-		    .detail("RangeID", cfEntry.rangeId.printable())
+		    .detail("FeedID", cfEntry.feedId)
 		    .detail("Range", cfEntry.range)
 		    .detail("FetchVersion", fetchVersion)
 		    .detail("EmptyVersion", cfEntry.emptyVersion)
 		    .detail("StopVersion", cfEntry.stopVersion)
+		    .detail("FeedMetadataVersion", cfEntry.feedMetadataVersion)
 		    .detail("Existing", existing)
+		    .detail("ExistingMetadataVersion", existing ? existingEntry->second->metadataVersion : invalidVersion)
 		    .detail("CleanupPendingVersion", cleanupPending ? cleanupEntry->second : invalidVersion)
 		    .detail("FKID", fetchKeysID);
 
 		bool addMutationToLog = false;
 		Reference<ChangeFeedInfo> changeFeedInfo;
 
-		auto fid = destroyedFeedIds.find(cfEntry.rangeId);
-		if (fid != destroyedFeedIds.end()) {
-			refreshedFeedIds.insert(cfEntry.rangeId);
-			destroyedFeedIds.erase(fid);
-		}
-
 		if (!existing) {
-			TEST(cleanupPending); // Fetch change feed which is cleanup pending. This means there was a move away and a
-			// move back, this will remake the metadata
+			CODE_PROBE(cleanupPending,
+			           "Fetch change feed which is cleanup pending. This means there was a move away and a move back, "
+			           "this will remake the metadata");
 
 			changeFeedInfo = Reference<ChangeFeedInfo>(new ChangeFeedInfo());
 			changeFeedInfo->range = cfEntry.range;
-			changeFeedInfo->id = cfEntry.rangeId;
+			changeFeedInfo->id = cfEntry.feedId;
 
 			changeFeedInfo->emptyVersion = cfEntry.emptyVersion;
 			changeFeedInfo->stopVersion = cfEntry.stopVersion;
-			data->uidChangeFeed[cfEntry.rangeId] = changeFeedInfo;
+			data->uidChangeFeed[cfEntry.feedId] = changeFeedInfo;
 			auto rs = data->keyChangeFeed.modify(cfEntry.range);
 			for (auto r = rs.begin(); r != rs.end(); ++r) {
 				r->value().push_back(changeFeedInfo);
 			}
-			data->keyChangeFeed.coalesce(cfEntry.range.contents());
+			data->keyChangeFeed.coalesce(cfEntry.range);
 
 			addMutationToLog = true;
 		} else {
 			changeFeedInfo = existingEntry->second;
 
+			CODE_PROBE(cfEntry.feedMetadataVersion > data->version.get(),
+			           "Change Feed fetched future metadata version");
+
+			auto fid = missingFeeds.find(cfEntry.feedId);
+			if (fid != missingFeeds.end()) {
+				missingFeeds.erase(fid);
+				ASSERT(!changeFeedInfo->destroyed);
+				// could possibly be not removing because it was reset  while
+				// waiting on destroyedFeeds by a private mutation or another fetch
+				if (changeFeedInfo->removing) {
+					TraceEvent(SevDebug, "ResetChangeFeedInfoFromFetch", data->thisServerID)
+					    .detail("FeedID", changeFeedInfo->id.printable())
+					    .detail("Range", changeFeedInfo->range)
+					    .detail("FetchVersion", fetchVersion)
+					    .detail("EmptyVersion", changeFeedInfo->emptyVersion)
+					    .detail("StopVersion", changeFeedInfo->stopVersion)
+					    .detail("PreviousMetadataVersion", changeFeedInfo->metadataVersion)
+					    .detail("NewMetadataVersion", cfEntry.feedMetadataVersion)
+					    .detail("FKID", fetchKeysID);
+
+					CODE_PROBE(true, "re-fetching feed scheduled for deletion! Un-mark it as removing");
+
+					// TODO only reset data if feed is still removing
+					changeFeedInfo->removing = false;
+					// reset fetch versions because everything previously fetched was cleaned up
+					changeFeedInfo->fetchVersion = invalidVersion;
+					changeFeedInfo->durableFetchVersion = NotifiedVersion();
+					addMutationToLog = true;
+				}
+			}
+
 			if (changeFeedInfo->destroyed) {
-				// race where multiple feeds fetched overlapping change feed, one realized feed was missing and marked
-				// it removed+destroyed, then this one fetched the same info
+				CODE_PROBE(true, "Change feed fetched and destroyed by other fetch while fetching metadata");
 				continue;
 			}
 
@@ -5688,94 +6190,75 @@ ACTOR Future<std::vector<Key>> fetchChangeFeedMetadata(StorageServer* data,
 			// this actor would have been cancelled if a later remove happened
 			ASSERT(!changeFeedInfo->removing);
 			if (cfEntry.stopVersion < changeFeedInfo->stopVersion) {
-				TEST(true); // Change feed updated stop version from fetch metadata
+				CODE_PROBE(true, "Change feed updated stop version from fetch metadata");
 				changeFeedInfo->stopVersion = cfEntry.stopVersion;
 				addMutationToLog = true;
 			}
 
 			// don't update empty version past SS version if SS is behind, it can cause issues
 			if (cfEntry.emptyVersion < data->version.get() && cfEntry.emptyVersion > changeFeedInfo->emptyVersion) {
-				TEST(true); // Change feed updated empty version from fetch metadata
+				CODE_PROBE(true, "Change feed updated empty version from fetch metadata");
 				changeFeedInfo->emptyVersion = cfEntry.emptyVersion;
 				addMutationToLog = true;
 			}
 		}
+		feedIds.push_back(cfEntry.feedId);
+		addMutationToLog |= changeFeedInfo->updateMetadataVersion(cfEntry.feedMetadataVersion);
 		if (addMutationToLog) {
 			ASSERT(changeFeedInfo.isValid());
-			auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion());
+			Version logV = data->data().getLatestVersion();
+			auto& mLV = data->addVersionToMutationLog(logV);
 			data->addMutationToMutationLog(
 			    mLV,
-			    MutationRef(
-			        MutationRef::SetValue,
-			        persistChangeFeedKeys.begin.toString() + cfEntry.rangeId.toString(),
-			        changeFeedSSValue(cfEntry.range, changeFeedInfo->emptyVersion + 1, changeFeedInfo->stopVersion)));
+			    MutationRef(MutationRef::SetValue,
+			                persistChangeFeedKeys.begin.toString() + cfEntry.feedId.toString(),
+			                changeFeedSSValue(cfEntry.range,
+			                                  changeFeedInfo->emptyVersion + 1,
+			                                  changeFeedInfo->stopVersion,
+			                                  changeFeedInfo->metadataVersion)));
 			// if we updated pop version, remove mutations
 			while (!changeFeedInfo->mutations.empty() &&
 			       changeFeedInfo->mutations.front().version <= changeFeedInfo->emptyVersion) {
 				changeFeedInfo->mutations.pop_front();
 			}
+			if (BUGGIFY) {
+				data->maybeInjectTargetedRestart(logV);
+			}
 		}
 	}
 
-	TEST(!refreshedFeedIds.empty()); // Feed refreshed between move away and move back
-	TEST(!destroyedFeedIds.empty()); // Feed destroyed between move away and move back
-	for (auto& feedId : refreshedFeedIds) {
-		auto existingEntry = data->uidChangeFeed.find(feedId);
-		if (existingEntry == data->uidChangeFeed.end() || existingEntry->second->destroyed ||
-		    !existingEntry->second->refreshInProgress) {
-			TEST(true); // feed refreshed
+	for (auto& feed : missingFeeds) {
+		auto existingEntry = data->uidChangeFeed.find(feed.first);
+		ASSERT(existingEntry != data->uidChangeFeed.end());
+		ASSERT(existingEntry->second->removing);
+		ASSERT(!existingEntry->second->destroyed);
+
+		Version fetchedMetadataVersion = feedMetadata.getFeedMetadataVersion(existingEntry->second->range);
+		Version lastMetadataVersion = feed.second;
+		// Look for case where feed's range was moved away, feed was destroyed, and then feed's range was moved back.
+		// This happens where feed is removing, the fetch metadata is higher than the moved away version, and the feed
+		// isn't in the fetched response. In that case, the feed must have been destroyed between lastMetadataVersion
+		// and fetchedMetadataVersion
+		if (lastMetadataVersion >= fetchedMetadataVersion) {
+			CODE_PROBE(true, "Change Feed fetched higher metadata version before moved away");
 			continue;
 		}
 
-		// Since cleanup put a mutation in the log to delete the change feed data, put one in the log to restore
-		// it
-		// We may just want to refactor this so updateStorage does explicit deletes based on
-		// changeFeedCleanupDurable and not use the mutation log at all for the change feed metadata cleanup.
-		// Then we wouldn't have to reset anything here or above
-		// Do the mutation log update here instead of above to ensure we only add it back to the mutation log if we're
-		// sure it wasn't deleted in the metadata gap
-		Version metadataVersion = data->data().getLatestVersion();
-		auto& mLV = data->addVersionToMutationLog(metadataVersion);
-		data->addMutationToMutationLog(
-		    mLV,
-		    MutationRef(MutationRef::SetValue,
-		                persistChangeFeedKeys.begin.toString() + existingEntry->second->id.toString(),
-		                changeFeedSSValue(existingEntry->second->range,
-		                                  existingEntry->second->emptyVersion + 1,
-		                                  existingEntry->second->stopVersion)));
-		TraceEvent(SevDebug, "PersistingResetChangeFeedInfo", data->thisServerID)
-		    .detail("RangeID", existingEntry->second->id.printable())
-		    .detail("Range", existingEntry->second->range)
-		    .detail("FetchVersion", fetchVersion)
-		    .detail("EmptyVersion", existingEntry->second->emptyVersion)
-		    .detail("StopVersion", existingEntry->second->stopVersion)
-		    .detail("FKID", fetchKeysID)
-		    .detail("MetadataVersion", metadataVersion);
-		existingEntry->second->refreshInProgress = false;
-	}
-	for (auto& feedId : destroyedFeedIds) {
-		auto existingEntry = data->uidChangeFeed.find(feedId);
-		if (existingEntry == data->uidChangeFeed.end() || existingEntry->second->destroyed) {
-			TEST(true); // feed refreshed but then destroyed elsewhere
-			continue;
-		}
-
-		/*fmt::print("DBG: SS {} fetching feed {} was refreshed but not present!! assuming destroyed\n",
-		           data->thisServerID.toString().substr(0, 4),
-		           feedId.printable());*/
 		Version cleanupVersion = data->data().getLatestVersion();
 
+		CODE_PROBE(true, "Destroying change feed from fetch metadata"); //
 		TraceEvent(SevDebug, "DestroyingChangeFeedFromFetchMetadata", data->thisServerID)
-		    .detail("RangeID", feedId.printable())
+		    .detail("FeedID", feed.first)
 		    .detail("Range", existingEntry->second->range)
 		    .detail("Version", cleanupVersion)
 		    .detail("FKID", fetchKeysID);
 
 		if (g_network->isSimulated()) {
-			ASSERT(g_simulator.validationData.allDestroyedChangeFeedIDs.count(feedId.toString()));
+			// verify that the feed was actually destroyed and it's not an error in this inference logic
+			ASSERT(g_simulator->validationData.allDestroyedChangeFeedIDs.count(feed.first.toString()));
 		}
 
-		Key beginClearKey = feedId.withPrefix(persistChangeFeedKeys.begin);
+		Key beginClearKey = feed.first.withPrefix(persistChangeFeedKeys.begin);
 
 		auto& mLV = data->addVersionToMutationLog(cleanupVersion);
 		data->addMutationToMutationLog(mLV,
@@ -5783,15 +6266,18 @@ ACTOR Future<std::vector<Key>> fetchChangeFeedMetadata(StorageServer* data,
 		++data->counters.kvSystemClearRanges;
 		data->addMutationToMutationLog(mLV,
 		                               MutationRef(MutationRef::ClearRange,
-		                                           changeFeedDurableKey(feedId, 0),
-		                                           changeFeedDurableKey(feedId, cleanupVersion)));
+		                                           changeFeedDurableKey(feed.first, 0),
+		                                           changeFeedDurableKey(feed.first, cleanupVersion)));
 		++data->counters.kvSystemClearRanges;
 
 		existingEntry->second->destroy(cleanupVersion);
-		data->changeFeedCleanupDurable[feedId] = cleanupVersion;
+		data->changeFeedCleanupDurable[feed.first] = cleanupVersion;
 
-		for (auto& it : data->changeFeedRemovals) {
-			it.second.send(feedId);
+		for (auto& it : data->changeFeedDestroys) {
+			it.second.send(feed.first);
+		}
+		if (BUGGIFY) {
+			data->maybeInjectTargetedRestart(cleanupVersion);
 		}
 	}
 	return feedIds;
@@ -5804,7 +6290,7 @@ ACTOR Future<std::unordered_map<Key, Version>> dispatchChangeFeeds(StorageServer
                                                                    KeyRange keys,
                                                                    Version beginVersion,
                                                                    Version endVersion,
-                                                                   PromiseStream<Key> removals,
+                                                                   PromiseStream<Key> destroyedFeeds,
                                                                    std::vector<Key>* feedIds,
                                                                    std::unordered_set<Key> newFeedIds) {
 	state std::unordered_map<Key, Version> feedMaxFetched;
@@ -5833,7 +6319,7 @@ ACTOR Future<std::unordered_map<Key, Version>> dispatchChangeFeeds(StorageServer
 
 		loop {
 			Future<Version> nextFeed = Never();
-			if (!removals.getFuture().isReady()) {
+			if (!destroyedFeeds.getFuture().isReady()) {
 				bool done = true;
 				while (!feedFetches.empty()) {
 					if (feedFetches.begin()->second.isReady()) {
@@ -5853,11 +6339,11 @@ ACTOR Future<std::unordered_map<Key, Version>> dispatchChangeFeeds(StorageServer
 				}
 			}
 			choose {
-				when(state Key remove = waitNext(removals.getFuture())) {
+				when(state Key destroyed = waitNext(destroyedFeeds.getFuture())) {
 					wait(delay(0));
-					feedFetches.erase(remove);
+					feedFetches.erase(destroyed);
 					for (int i = 0; i < feedIds->size(); i++) {
-						if ((*feedIds)[i] == remove) {
+						if ((*feedIds)[i] == destroyed) {
 							swapAndPop(feedIds, i--);
 						}
 					}
@@ -5868,7 +6354,7 @@ ACTOR Future<std::unordered_map<Key, Version>> dispatchChangeFeeds(StorageServer
 
 	} catch (Error& e) {
 		if (!data->shuttingDown) {
-			data->changeFeedRemovals.erase(fetchKeysID);
+			data->changeFeedDestroys.erase(fetchKeysID);
 		}
 		throw;
 	}
@@ -5881,6 +6367,8 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 	state Future<Void> warningLogger = logFetchKeysWarning(shard);
 	state const double startTime = now();
 	state Version fetchVersion = invalidVersion;
+
+	state PromiseStream<Key> destroyedFeeds;
 	state FetchKeysMetricReporter metricReporter(fetchKeysID,
 	                                             startTime,
 	                                             keys,
@@ -5889,17 +6377,27 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 	                                             data->counters.bytesFetched,
 	                                             data->counters.kvFetched);
 
+	// need to set this at the very start of the fetch, to handle any private change feed destroy mutations we get for
+	// this key range, that apply to change feeds we don't know about yet because their metadata hasn't been fetched yet
+	data->changeFeedDestroys[fetchKeysID] = destroyedFeeds;
+
 	// delay(0) to force a return to the run loop before the work of fetchKeys is started.
 	//  This allows adding->start() to be called inline with CSK.
-	wait(data->coreStarted.getFuture() && delay(0));
+	try {
+		wait(data->coreStarted.getFuture() && delay(0));
 
-	// On SS Reboot, durableVersion == latestVersion, so any mutations we add to the mutation log would be skipped if
-	// added before latest version advances.
-	// To ensure this doesn't happen, we wait for version to increase by one if this fetchKeys was initiated by a
-	// changeServerKeys from restoreDurableState
-	if (data->version.get() == data->durableVersion.get()) {
-		wait(data->version.whenAtLeast(data->version.get() + 1));
-		wait(delay(0));
+		// On SS Reboot, durableVersion == latestVersion, so any mutations we add to the mutation log would be skipped
+		// if added before latest version advances. To ensure this doesn't happen, we wait for version to increase by
+		// one if this fetchKeys was initiated by a changeServerKeys from restoreDurableState
+		if (data->version.get() == data->durableVersion.get()) {
+			wait(data->version.whenAtLeast(data->version.get() + 1));
+			wait(delay(0));
+		}
+	} catch (Error& e) {
+		if (!data->shuttingDown) {
+			data->changeFeedDestroys.erase(fetchKeysID);
+		}
+		throw e;
 	}
 
 	try {
@@ -5911,9 +6409,8 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		    .detail("Version", data->version.get())
 		    .detail("FKID", fetchKeysID);
 
-		state PromiseStream<Key> removals;
-		data->changeFeedRemovals[fetchKeysID] = removals;
-		state Future<std::vector<Key>> fetchCFMetadata = fetchChangeFeedMetadata(data, keys, removals, fetchKeysID);
+		state Future<std::vector<Key>> fetchCFMetadata =
+		    fetchChangeFeedMetadata(data, keys, destroyedFeeds, fetchKeysID);
 
 		validate(data);
 
@@ -5930,12 +6427,15 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			lastAvailable = std::max(lastAvailable, r->value());
 
 		if (lastAvailable != invalidVersion && lastAvailable >= data->durableVersion.get()) {
-			TEST(true); // FetchKeys waits for previous available version to be durable
+			CODE_PROBE(true, "FetchKeys waits for previous available version to be durable");
 			wait(data->durableVersion.whenAtLeast(lastAvailable + 1));
 		}
 
 		TraceEvent(SevDebug, "FetchKeysVersionSatisfied", data->thisServerID).detail("FKID", interval.pairID);
 
+		wait(data->fetchKeysParallelismFullLock.take(TaskPriority::DefaultYield));
+		state FlowLock::Releaser holdingFullFKPL(data->fetchKeysParallelismFullLock);
+
 		wait(data->fetchKeysParallelismLock.take(TaskPriority::DefaultYield));
 		state FlowLock::Releaser holdingFKPL(data->fetchKeysParallelismLock);
 
@@ -5965,12 +6465,20 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		state int debug_nextRetryToLog = 1;
 		state Error lastError;
 
+		// TODO: update to FETCH once the priority multi lock is used.
+		// leaving the readtype off for now to prevent data fetches stall under heavy load
+		// it is used to inform the storage that the rangeRead is for Fetch
+		// state ReadOptions options = ReadOptions(Optional<UID>(), ReadType::FETCH);
+		state ReadOptions options = ReadOptions(Optional<UID>(), ReadType::NORMAL);
+
 		// FIXME: The client cache does not notice when servers are added to a team. To read from a local storage server
 		// we must refresh the cache manually.
 		data->cx->invalidateCache(Key(), keys);
 
 		loop {
 			state Transaction tr(data->cx);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			// fetchVersion = data->version.get();
 			// A quick fix:
 			// By default, we use data->version as the fetchVersion.
@@ -5984,11 +6492,30 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			// At iteration 1, dest SS selects GRV as fetchVersion and (suppose) can read the data from src SS.
 			// Then dest SS waits its version catch up with this GRV version and write the data to disk.
 			// Note that dest SS waits outside the fetchKeysParallelismLock.
+			fetchVersion = std::max(shard->fetchVersion, data->version.get());
+			if (g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01)) {
+				// Test using GRV version for fetchKey.
+				lastError = transaction_too_old();
+			}
 			if (lastError.code() == error_code_transaction_too_old) {
-				Version grvVersion = wait(tr.getRawReadVersion());
-				fetchVersion = std::max(grvVersion, data->version.get());
-			} else {
-				fetchVersion = std::max(shard->fetchVersion, data->version.get());
+				try {
+					Version grvVersion = wait(tr.getRawReadVersion());
+					if (g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01)) {
+						// Test failed GRV request.
+						throw grv_proxy_memory_limit_exceeded();
+					}
+					fetchVersion = std::max(grvVersion, fetchVersion);
+				} catch (Error& e) {
+					if (e.code() == error_code_actor_cancelled) {
+						throw e;
+					}
+
+					// Note that error in getting GRV doesn't affect any storage server state. Therefore, we catch all
+					// errors here without failing the storage server. When error happens, fetchVersion fall back to
+					// the above computed fetchVersion.
+					TraceEvent(SevWarn, "FetchKeyGRVError", data->thisServerID).error(e);
+					lastError = e;
+				}
 			}
 			ASSERT(fetchVersion >= shard->fetchVersion); // at this point, shard->fetchVersion is the last fetchVersion
 			shard->fetchVersion = fetchVersion;
@@ -6000,15 +6527,20 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 				shard->updates.pop_front();
 			tr.setVersion(fetchVersion);
 			tr.trState->taskID = TaskPriority::FetchKeys;
+			tr.trState->readOptions = options;
 			state PromiseStream<RangeResult> results;
-			state Future<Void> hold = SERVER_KNOBS->FETCH_USING_STREAMING
-			                              ? tr.getRangeStream(results, keys, GetRangeLimits(), Snapshot::True)
-			                              : tryGetRange(results, &tr, keys);
+			state Future<Void> hold;
+			if (SERVER_KNOBS->FETCH_USING_BLOB) {
+				hold = tryGetRangeFromBlob(results, &tr, keys, fetchVersion, data->blobConn);
+			} else {
+				hold = tryGetRange(results, &tr, keys);
+			}
+
 			state Key nfk = keys.begin;
 
 			try {
 				loop {
-					TEST(true); // Fetching keys for transferred shard
+					CODE_PROBE(true, "Fetching keys for transferred shard");
 					while (data->fetchKeysBudgetUsed.get()) {
 						wait(data->fetchKeysBudgetUsed.onChange());
 					}
@@ -6099,8 +6631,16 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 					// This actor finishes committing the keys [keys.begin,nfk) that we already fetched.
 					// The remaining unfetched keys [nfk,keys.end) will become a separate AddingShard with its own
 					// fetchKeys.
-					shard->server->addShard(ShardInfo::addingSplitLeft(KeyRangeRef(keys.begin, nfk), shard));
-					shard->server->addShard(ShardInfo::newAdding(data, KeyRangeRef(nfk, keys.end)));
+					if (data->shardAware) {
+						StorageServerShard rightShard = data->shards[keys.begin]->toStorageServerShard();
+						rightShard.range = KeyRangeRef(nfk, keys.end);
+						shard->server->addShard(ShardInfo::addingSplitLeft(KeyRangeRef(keys.begin, nfk), shard));
+						shard->server->addShard(ShardInfo::newShard(data, rightShard));
+						data->shards[keys.begin]->populateShard(rightShard);
+					} else {
+						shard->server->addShard(ShardInfo::addingSplitLeft(KeyRangeRef(keys.begin, nfk), shard));
+						shard->server->addShard(ShardInfo::newAdding(data, KeyRangeRef(nfk, keys.end)));
+					}
 					shard = data->shards.rangeContaining(keys.begin).value()->adding.get();
 					warningLogger = logFetchKeysWarning(shard);
 					AddingShard* otherShard = data->shards.rangeContaining(nfk).value()->adding.get();
@@ -6116,8 +6656,8 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 						splitMutations(data, data->shards, *u);
 					}
 
-					TEST(true); // fetchkeys has more
-					TEST(shard->updates.size()); // Shard has updates
+					CODE_PROBE(true, "fetchkeys has more");
+					CODE_PROBE(shard->updates.size(), "Shard has updates");
 					ASSERT(otherShard->updates.empty());
 				}
 				break;
@@ -6125,8 +6665,10 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		}
 
 		// FIXME: remove when we no longer support upgrades from 5.X
-		data->cx->enableLocalityLoadBalance = EnableLocalityLoadBalance::True;
-		TraceEvent(SevWarnAlways, "FKReenableLB").detail("FKID", fetchKeysID);
+		if (!data->cx->enableLocalityLoadBalance) {
+			data->cx->enableLocalityLoadBalance = EnableLocalityLoadBalance::True;
+			TraceEvent(SevWarnAlways, "FKReenableLB").detail("FKID", fetchKeysID);
+		}
 
 		// We have completed the fetch and write of the data, now we wait for MVCC window to pass.
 		//  As we have finished this work, we will allow more work to start...
@@ -6140,8 +6682,14 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		// being recovered. Instead we wait for the updateStorage loop to commit something (and consequently also what
 		// we have written)
 
-		state Future<std::unordered_map<Key, Version>> feedFetchMain = dispatchChangeFeeds(
-		    data, fetchKeysID, keys, 0, fetchVersion + 1, removals, &changeFeedsToFetch, std::unordered_set<Key>());
+		state Future<std::unordered_map<Key, Version>> feedFetchMain = dispatchChangeFeeds(data,
+		                                                                                   fetchKeysID,
+		                                                                                   keys,
+		                                                                                   0,
+		                                                                                   fetchVersion + 1,
+		                                                                                   destroyedFeeds,
+		                                                                                   &changeFeedsToFetch,
+		                                                                                   std::unordered_set<Key>());
 
 		state Future<Void> fetchDurable = data->durableVersion.whenAtLeast(data->storageVersion() + 1);
 		state Future<Void> dataArrive = data->version.whenAtLeast(fetchVersion);
@@ -6186,7 +6734,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		std::unordered_set<Key> newChangeFeeds;
 		for (auto& r : ranges) {
 			for (auto& cfInfo : r.value()) {
-				TEST(true); // SS fetching new change feed that didn't exist when fetch started
+				CODE_PROBE(true, "SS fetching new change feed that didn't exist when fetch started");
 				if (!cfInfo->removing) {
 					newChangeFeeds.insert(cfInfo->id);
 				}
@@ -6204,7 +6752,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		                        keys,
 		                        fetchVersion + 1,
 		                        shard->transferredVersion,
-		                        removals,
+		                        destroyedFeeds,
 		                        &changeFeedsToFetch,
 		                        newChangeFeeds);
 
@@ -6228,7 +6776,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		}
 
 		int startSize = batch->changes.size();
-		TEST(startSize); // Adding fetch data to a batch which already has changes
+		CODE_PROBE(startSize, "Adding fetch data to a batch which already has changes");
 		batch->changes.resize(batch->changes.size() + shard->updates.size());
 
 		// FIXME: pass the deque back rather than copy the data
@@ -6258,7 +6806,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			}
 		}
 
-		data->changeFeedRemovals.erase(fetchKeysID);
+		data->changeFeedDestroys.erase(fetchKeysID);
 
 		shard->phase = AddingShard::Waiting;
 
@@ -6270,6 +6818,13 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		    .detail("Version", feedTransferredVersion)
 		    .detail("StorageVersion", data->storageVersion());
 
+		state StorageServerShard newShard;
+		if (data->shardAware) {
+			newShard = data->shards[keys.begin]->toStorageServerShard();
+			ASSERT(newShard.range == keys);
+			ASSERT(newShard.getShardState() == StorageServerShard::ReadWritePending);
+			updateStorageShard(data, newShard);
+		}
 		setAvailableStatus(data,
 		                   keys,
 		                   true); // keys will be available when getLatestVersion()==transferredVersion is durable
@@ -6277,6 +6832,9 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		// Note that since it receives a pointer to FetchInjectionInfo, the thread does not leave this actor until this
 		// point.
 
+		// At this point change feed fetching and mutation injection is complete, so full fetch is finished.
+		holdingFullFKPL.release();
+
 		// Wait for the transferred version (and therefore the shard data) to be committed and durable.
 		wait(data->durableVersion.whenAtLeast(feedTransferredVersion));
 
@@ -6285,8 +6843,14 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		           shard->keys); // We aren't changing whether the shard is assigned
 		data->newestAvailableVersion.insert(shard->keys, latestVersion);
 		shard->readWrite.send(Void());
-		data->addShard(ShardInfo::newReadWrite(shard->keys, data)); // invalidates shard!
-		coalesceShards(data, keys);
+		if (data->shardAware) {
+			newShard.setShardState(StorageServerShard::ReadWrite);
+			data->addShard(ShardInfo::newShard(data, newShard)); // invalidates shard!
+			coalescePhysicalShards(data, keys);
+		} else {
+			data->addShard(ShardInfo::newReadWrite(shard->keys, data)); // invalidates shard!
+			coalesceShards(data, keys);
+		}
 
 		validate(data);
 
@@ -6299,7 +6863,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		    .errorUnsuppressed(e)
 		    .detail("Version", data->version.get());
 		if (!data->shuttingDown) {
-			data->changeFeedRemovals.erase(fetchKeysID);
+			data->changeFeedDestroys.erase(fetchKeysID);
 		}
 		if (e.code() == error_code_actor_cancelled && !data->shuttingDown && shard->phase >= AddingShard::Fetching) {
 			if (shard->phase < AddingShard::FetchingCF) {
@@ -6382,9 +6946,129 @@ void ShardInfo::addMutation(Version version, bool fromFetch, MutationRef const&
 	}
 }
 
-enum ChangeServerKeysContext { CSK_UPDATE, CSK_RESTORE, CSK_ASSIGN_EMPTY };
-const char* changeServerKeysContextName[] = { "Update", "Restore" };
+ACTOR Future<Void> restoreShards(StorageServer* data,
+                                 Version version,
+                                 RangeResult storageShards,
+                                 RangeResult assignedShards,
+                                 RangeResult availableShards) {
+	TraceEvent(SevInfo, "StorageServerRestoreShardsBegin", data->thisServerID)
+	    .detail("StorageShard", storageShards.size())
+	    .detail("Version", version);
 
+	state int mLoc;
+	for (mLoc = 0; mLoc < storageShards.size(); ++mLoc) {
+		const KeyRangeRef shardRange(storageShards[mLoc].key.removePrefix(persistStorageServerShardKeys.begin),
+		                             mLoc + 1 == storageShards.size() ? allKeys.end
+		                                                              : storageShards[mLoc + 1].key.removePrefix(
+		                                                                    persistStorageServerShardKeys.begin));
+		StorageServerShard shard =
+		    ObjectReader::fromStringRef<StorageServerShard>(storageShards[mLoc].value, IncludeVersion());
+		shard.range = shardRange;
+		TraceEvent(SevVerbose, "RestoreShardsStorageShard", data->thisServerID)
+		    .detail("Range", shardRange)
+		    .detail("StorageShard", shard.toString());
+
+		// TODO(psm): Schedule deletion of finished moveInShard.
+		const StorageServerShard::ShardState shardState = shard.getShardState();
+		auto existingShards = data->shards.intersectingRanges(shardRange);
+		for (auto it = existingShards.begin(); it != existingShards.end(); ++it) {
+			TraceEvent(SevVerbose, "RestoreShardsIntersectingRange", data->thisServerID)
+			    .detail("StorageShard", shard.toString())
+			    .detail("IntersectingShardRange", it->value()->keys)
+			    .detail("IntersectingShardState", it->value()->debugDescribeState())
+			    .log();
+			ASSERT(it->value()->notAssigned());
+		}
+
+		if (shardState == StorageServerShard::NotAssigned) {
+			continue;
+		}
+
+		auto ranges = data->shards.getAffectedRangesAfterInsertion(shard.range, Reference<ShardInfo>());
+		for (int i = 0; i < ranges.size(); i++) {
+			KeyRangeRef& range = static_cast<KeyRangeRef&>(ranges[i]);
+			TraceEvent(SevVerbose, "RestoreShardsInstallPhysicalShard", data->thisServerID)
+			    .detail("Shard", shard.toString())
+			    .detail("Range", range);
+			if (range == shard.range) {
+				data->addShard(ShardInfo::newShard(data, shard));
+			} else {
+				StorageServerShard rightShard = ranges[i].value->toStorageServerShard();
+				rightShard.range = range;
+				data->addShard(ShardInfo::newShard(data, rightShard));
+			}
+		}
+
+		const bool nowAvailable = shard.getShardState() == StorageServerShard::ReadWrite;
+		data->newestAvailableVersion.insert(shard.range, nowAvailable ? latestVersion : invalidVersion);
+
+		wait(yield());
+	}
+
+	state int availableLoc;
+	for (availableLoc = 0; availableLoc < availableShards.size(); availableLoc++) {
+		KeyRangeRef shardRange(
+		    availableShards[availableLoc].key.removePrefix(persistShardAvailableKeys.begin),
+		    availableLoc + 1 == availableShards.size()
+		        ? allKeys.end
+		        : availableShards[availableLoc + 1].key.removePrefix(persistShardAvailableKeys.begin));
+		ASSERT(!shardRange.empty());
+
+		const bool nowAvailable = availableShards[availableLoc].value != "0"_sr;
+		auto existingShards = data->shards.intersectingRanges(shardRange);
+		for (auto it = existingShards.begin(); it != existingShards.end(); ++it) {
+			TraceEvent(SevVerbose, "RestoreShardsValidateAvailable", data->thisServerID)
+			    .detail("Range", shardRange)
+			    .detail("Available", nowAvailable)
+			    .detail("IntersectingShardRange", it->value()->keys)
+			    .detail("IntersectingShardState", it->value()->debugDescribeState())
+			    .log();
+			if (nowAvailable) {
+				ASSERT(it->value()->isReadable());
+				ASSERT(data->newestAvailableVersion.allEqual(shardRange, latestVersion));
+			}
+		}
+
+		wait(yield());
+	}
+
+	state int assignedLoc;
+	for (assignedLoc = 0; assignedLoc < assignedShards.size(); ++assignedLoc) {
+		KeyRangeRef shardRange(assignedShards[assignedLoc].key.removePrefix(persistShardAssignedKeys.begin),
+		                       assignedLoc + 1 == assignedShards.size()
+		                           ? allKeys.end
+		                           : assignedShards[assignedLoc + 1].key.removePrefix(persistShardAssignedKeys.begin));
+		ASSERT(!shardRange.empty());
+		const bool nowAssigned = assignedShards[assignedLoc].value != "0"_sr;
+
+		auto existingShards = data->shards.intersectingRanges(shardRange);
+		for (auto it = existingShards.begin(); it != existingShards.end(); ++it) {
+			TraceEvent(SevVerbose, "RestoreShardsValidateAssigned", data->thisServerID)
+			    .detail("Range", shardRange)
+			    .detail("Assigned", nowAssigned)
+			    .detail("IntersectingShardRange", it->value()->keys)
+			    .detail("IntersectingShardState", it->value()->debugDescribeState())
+			    .log();
+
+			ASSERT_EQ(it->value()->assigned(), nowAssigned);
+			if (!nowAssigned) {
+				ASSERT(data->newestAvailableVersion.allEqual(shardRange, invalidVersion));
+			}
+		}
+
+		wait(yield());
+	}
+
+	TraceEvent(SevVerbose, "StorageServerRestoreShardsCoalesce", data->thisServerID).detail("Version", version);
+	coalescePhysicalShards(data, allKeys);
+	TraceEvent(SevVerbose, "StorageServerRestoreShardsValidate", data->thisServerID).detail("Version", version);
+	validate(data, /*force=*/true);
+	TraceEvent(SevInfo, "StorageServerRestoreShardsEnd", data->thisServerID).detail("Version", version);
+
+	return Void();
+}
+
+// Finds any change feeds that no longer have shards on this server, and clean them up
 void cleanUpChangeFeeds(StorageServer* data, const KeyRangeRef& keys, Version version) {
 	std::map<Key, KeyRange> candidateFeeds;
 	auto ranges = data->keyChangeFeed.intersectingRanges(keys);
@@ -6429,11 +7113,15 @@ void cleanUpChangeFeeds(StorageServer* data, const KeyRangeRef& keys, Version ve
 
 			auto feed = data->uidChangeFeed.find(f.first);
 			if (feed != data->uidChangeFeed.end()) {
+				feed->second->updateMetadataVersion(version);
 				feed->second->removing = true;
-				feed->second->refreshInProgress = false;
 				feed->second->moved(feed->second->range);
 				feed->second->newMutations.trigger();
 			}
+
+			if (BUGGIFY) {
+				data->maybeInjectTargetedRestart(durableVersion);
+			}
 		} else {
 			// if just part of feed's range is moved away
 			auto feed = data->uidChangeFeed.find(f.first);
@@ -6456,7 +7144,7 @@ void changeServerKeys(StorageServer* data,
 	//     .detail("KeyEnd", keys.end)
 	//     .detail("NowAssigned", nowAssigned)
 	//     .detail("Version", version)
-	//     .detail("Context", changeServerKeysContextName[(int)context]);
+	//     .detail("Context", changeServerKeysContextName(context));
 	validate(data);
 
 	// TODO(alexmiller): Figure out how to selectively enable spammy data distribution events.
@@ -6498,7 +7186,7 @@ void changeServerKeys(StorageServer* data,
 		else {
 			ASSERT(ranges[i].value->adding);
 			data->addShard(ShardInfo::newAdding(data, ranges[i]));
-			TEST(true); // ChangeServerKeys reFetchKeys
+			CODE_PROBE(true, "ChangeServerKeys reFetchKeys");
 		}
 	}
 
@@ -6586,6 +7274,229 @@ void changeServerKeys(StorageServer* data,
 	}
 	validate(data);
 
+	if (!nowAssigned) {
+		cleanUpChangeFeeds(data, keys, version);
+	}
+}
+
+void changeServerKeysWithPhysicalShards(StorageServer* data,
+                                        const KeyRangeRef& keys,
+                                        const UID& dataMoveId,
+                                        bool nowAssigned,
+                                        Version version,
+                                        ChangeServerKeysContext context) {
+	ASSERT(!keys.empty());
+	TraceEvent(SevDebug, "ChangeServerKeysWithPhysicalShards", data->thisServerID)
+	    .detail("DataMoveID", dataMoveId)
+	    .detail("Range", keys)
+	    .detail("NowAssigned", nowAssigned)
+	    .detail("Version", version)
+	    .detail("Context", changeServerKeysContextName(context));
+
+	validate(data);
+
+	DEBUG_KEY_RANGE(nowAssigned ? "KeysAssigned" : "KeysUnassigned", version, keys, data->thisServerID);
+
+	const uint64_t desiredId = dataMoveId.first();
+	const Version cVer = version + 1;
+	ASSERT(data->data().getLatestVersion() == cVer);
+
+	// Save a backup of the ShardInfo references before we start messing with shards, in order to defer fetchKeys
+	// cancellation (and its potential call to removeDataRange()) until shards is again valid
+	std::vector<Reference<ShardInfo>> oldShards;
+	auto os = data->shards.intersectingRanges(keys);
+	for (auto r = os.begin(); r != os.end(); ++r) {
+		oldShards.push_back(r->value());
+	}
+
+	auto ranges = data->shards.getAffectedRangesAfterInsertion(
+	    keys,
+	    Reference<ShardInfo>()); // null reference indicates the range being changed
+	for (int i = 0; i < ranges.size(); i++) {
+		const Reference<ShardInfo> currentShard = ranges[i].value;
+		const KeyRangeRef currentRange = static_cast<KeyRangeRef>(ranges[i]);
+		if (currentShard.isValid()) {
+			TraceEvent(SevVerbose, "OverlappingPhysicalShard", data->thisServerID)
+			    .detail("PhysicalShard", currentShard->toStorageServerShard().toString());
+		}
+		if (!currentShard.isValid()) {
+			ASSERT(currentRange == keys); // there shouldn't be any nulls except for the range being inserted
+		} else if (currentShard->notAssigned()) {
+			ASSERT(nowAssigned); // Adding a new range to the server.
+			StorageServerShard newShard = currentShard->toStorageServerShard();
+			newShard.range = currentRange;
+			data->addShard(ShardInfo::newShard(data, newShard));
+			TraceEvent(SevVerbose, "SSSplitShardNotAssigned", data->thisServerID)
+			    .detail("Range", keys)
+			    .detail("NowAssigned", nowAssigned)
+			    .detail("Version", cVer)
+			    .detail("ResultingShard", newShard.toString());
+		} else if (currentShard->isReadable()) {
+			StorageServerShard newShard = currentShard->toStorageServerShard();
+			newShard.range = currentRange;
+			data->addShard(ShardInfo::newShard(data, newShard));
+			TraceEvent(SevVerbose, "SSSplitShardReadable", data->thisServerID)
+			    .detail("Range", keys)
+			    .detail("NowAssigned", nowAssigned)
+			    .detail("Version", cVer)
+			    .detail("ResultingShard", newShard.toString());
+		} else if (currentShard->adding) {
+			ASSERT(!nowAssigned);
+			StorageServerShard newShard = currentShard->toStorageServerShard();
+			newShard.range = currentRange;
+			// TODO(psm): Cancel or update the moving-in shard when physical shard move is enabled.
+			data->addShard(ShardInfo::newShard(data, newShard));
+			TraceEvent(SevVerbose, "SSSplitShardAdding", data->thisServerID)
+			    .detail("Range", keys)
+			    .detail("NowAssigned", nowAssigned)
+			    .detail("Version", cVer)
+			    .detail("ResultingShard", newShard.toString());
+		} else {
+			ASSERT(false);
+		}
+	}
+
+	auto vr = data->shards.intersectingRanges(keys);
+	std::vector<std::pair<KeyRange, Version>> changeNewestAvailable;
+	std::vector<KeyRange> removeRanges;
+	std::vector<KeyRange> newEmptyRanges;
+	std::vector<StorageServerShard> updatedShards;
+	int totalAssignedAtVer = 0;
+	for (auto r = vr.begin(); r != vr.end(); ++r) {
+		KeyRangeRef range = keys & r->range();
+		const bool dataAvailable = r->value()->isReadable();
+		TraceEvent(SevVerbose, "CSKPhysicalShard", data->thisServerID)
+		    .detail("Range", range)
+		    .detail("ExistingShardRange", r->range())
+		    .detail("Available", dataAvailable)
+		    .detail("NowAssigned", nowAssigned)
+		    .detail("ShardState", r->value()->debugDescribeState());
+		ASSERT(keys.contains(r->range()));
+		if (context == CSK_ASSIGN_EMPTY && !dataAvailable) {
+			ASSERT(nowAssigned);
+			TraceEvent(SevVerbose, "ChangeServerKeysAddEmptyRange", data->thisServerID)
+			    .detail("Range", range)
+			    .detail("Version", cVer);
+			newEmptyRanges.push_back(range);
+			updatedShards.emplace_back(range, cVer, desiredId, desiredId, StorageServerShard::ReadWrite);
+		} else if (!nowAssigned) {
+			if (dataAvailable) {
+				ASSERT(data->newestAvailableVersion[range.begin] ==
+				       latestVersion); // Not that we care, but this used to be checked instead of dataAvailable
+				ASSERT(data->mutableData().getLatestVersion() > version || context == CSK_RESTORE);
+				changeNewestAvailable.emplace_back(range, version);
+				removeRanges.push_back(range);
+			}
+			updatedShards.push_back(StorageServerShard::notAssigned(range, cVer));
+			data->watches.triggerRange(range.begin, range.end);
+			data->pendingRemoveRanges[cVer].push_back(range);
+			// TODO(psm): When fetchKeys() is not used, make sure KV will remove the data, otherwise, removeDataShard()
+			// here.
+			TraceEvent(SevVerbose, "SSUnassignShard", data->thisServerID)
+			    .detail("Range", range)
+			    .detail("NowAssigned", nowAssigned)
+			    .detail("Version", cVer)
+			    .detail("NewShard", updatedShards.back().toString());
+		} else if (!dataAvailable) {
+			if (version == data->initialClusterVersion - 1) {
+				TraceEvent(SevVerbose, "CSKWithPhysicalShardsSeedRange", data->thisServerID)
+				    .detail("ShardID", desiredId)
+				    .detail("Range", range);
+				changeNewestAvailable.emplace_back(range, latestVersion);
+				updatedShards.push_back(
+				    StorageServerShard(range, version, desiredId, desiredId, StorageServerShard::ReadWrite));
+				setAvailableStatus(data, range, true);
+				// Note: The initial range is available, however, the shard won't be created in the storage engine
+				// untill version is committed.
+				data->pendingAddRanges[cVer].emplace_back(desiredId, range);
+				TraceEvent(SevVerbose, "SSInitialShard", data->thisServerID)
+				    .detail("Range", range)
+				    .detail("NowAssigned", nowAssigned)
+				    .detail("Version", cVer)
+				    .detail("NewShard", updatedShards.back().toString());
+			} else {
+				auto& shard = data->shards[range.begin];
+				if (!shard->assigned()) {
+					updatedShards.push_back(
+					    StorageServerShard(range, cVer, desiredId, desiredId, StorageServerShard::MovingIn));
+					data->pendingAddRanges[cVer].emplace_back(desiredId, range);
+					data->newestDirtyVersion.insert(range, cVer);
+					TraceEvent(SevVerbose, "SSAssignShard", data->thisServerID)
+					    .detail("Range", range)
+					    .detail("NowAssigned", nowAssigned)
+					    .detail("Version", cVer)
+					    .detail("TotalAssignedAtVer", ++totalAssignedAtVer)
+					    .detail("NewShard", updatedShards.back().toString());
+				} else {
+					ASSERT(shard->adding != nullptr);
+					if (shard->desiredShardId != desiredId) {
+						TraceEvent(SevError, "CSKConflictingMoveInShards", data->thisServerID)
+						    .detail("DataMoveID", dataMoveId)
+						    .detail("Range", range)
+						    .detail("TargetShard", desiredId)
+						    .detail("CurrentShard", shard->desiredShardId)
+						    .detail("Version", cVer);
+						// TODO(psm): Replace this with moveInShard->cancel().
+						ASSERT(false);
+					} else {
+						TraceEvent(SevVerbose, "CSKMoveInToSameShard", data->thisServerID)
+						    .detail("DataMoveID", dataMoveId)
+						    .detailf("TargetShard", "%016llx", desiredId)
+						    .detail("MoveRange", keys)
+						    .detail("Range", range)
+						    .detail("ExistingShardRange", shard->keys)
+						    .detail("Version", cVer);
+					}
+				}
+			}
+		} else {
+			updatedShards.push_back(StorageServerShard(
+			    range, cVer, data->shards[range.begin]->shardId, desiredId, StorageServerShard::ReadWrite));
+			changeNewestAvailable.emplace_back(range, latestVersion);
+			TraceEvent(SevVerbose, "SSAssignShardAlreadyAvailable", data->thisServerID)
+			    .detail("Range", range)
+			    .detail("NowAssigned", nowAssigned)
+			    .detail("Version", cVer)
+			    .detail("NewShard", updatedShards.back().toString());
+		}
+	}
+
+	for (const auto& shard : updatedShards) {
+		data->addShard(ShardInfo::newShard(data, shard));
+		updateStorageShard(data, shard);
+	}
+
+	// Update newestAvailableVersion when a shard becomes (un)available (in a separate loop to avoid invalidating vr
+	// above)
+	for (auto r = changeNewestAvailable.begin(); r != changeNewestAvailable.end(); ++r) {
+		data->newestAvailableVersion.insert(r->first, r->second);
+	}
+
+	if (!nowAssigned) {
+		data->metrics.notifyNotReadable(keys);
+	}
+
+	coalescePhysicalShards(data, KeyRangeRef(ranges[0].begin, ranges[ranges.size() - 1].end));
+
+	// Now it is OK to do removeDataRanges, directly and through fetchKeys cancellation (and we have to do so before
+	// validate())
+	oldShards.clear();
+	ranges.clear();
+	for (auto r = removeRanges.begin(); r != removeRanges.end(); ++r) {
+		removeDataRange(data, data->addVersionToMutationLog(data->data().getLatestVersion()), data->shards, *r);
+		setAvailableStatus(data, *r, false);
+	}
+
+	// Clear the moving-in empty range, and set it available at the latestVersion.
+	for (const auto& range : newEmptyRanges) {
+		MutationRef clearRange(MutationRef::ClearRange, range.begin, range.end);
+		data->addMutation(data->data().getLatestVersion(), true, clearRange, range, data->updateEagerReads);
+		data->newestAvailableVersion.insert(range, latestVersion);
+		setAvailableStatus(data, range, true);
+		++data->counters.kvSystemClearRanges;
+	}
+	validate(data);
+
 	// find any change feeds that no longer have shards on this server, and clean them up
 	if (!nowAssigned) {
 		cleanUpChangeFeeds(data, keys, version);
@@ -6593,7 +7504,7 @@ void changeServerKeys(StorageServer* data,
 }
 
 void rollback(StorageServer* data, Version rollbackVersion, Version nextVersion) {
-	TEST(true); // call to shard rollback
+	CODE_PROBE(true, "call to shard rollback");
 	DEBUG_KEY_RANGE("Rollback", rollbackVersion, allKeys, data->thisServerID);
 
 	// We used to do a complicated dance to roll back in MVCC history.  It's much simpler, and more testable,
@@ -6716,15 +7627,24 @@ private:
 
 			// ignore data movements for tss in quarantine
 			if (!data->isTSSInQuarantine()) {
-				// add changes in shard assignment to the mutation log
-				setAssignedStatus(data, keys, nowAssigned);
-
-				// The changes for version have already been received (and are being processed now).  We need to fetch
-				// the data for change.version-1 (changes from versions < change.version)
-				// If emptyRange, treat the shard as empty, see removeKeysFromFailedServer() for more details about this
-				// scenario.
 				const ChangeServerKeysContext context = emptyRange ? CSK_ASSIGN_EMPTY : CSK_UPDATE;
-				changeServerKeys(data, keys, nowAssigned, currentVersion - 1, context);
+				if (data->shardAware) {
+					setAssignedStatus(data, keys, nowAssigned);
+					TraceEvent(SevDebug, "SSSetAssignedStatus", data->thisServerID)
+					    .detail("Range", keys)
+					    .detail("NowAssigned", nowAssigned)
+					    .detail("Version", ver);
+					changeServerKeysWithPhysicalShards(
+					    data, keys, dataMoveId, nowAssigned, currentVersion - 1, context);
+				} else {
+					// add changes in shard assignment to the mutation log
+					setAssignedStatus(data, keys, nowAssigned);
+
+					// The changes for version have already been received (and are being processed now).  We need to
+					// fetch the data for change.version-1 (changes from versions < change.version) If emptyRange, treat
+					// the shard as empty, see removeKeysFromFailedServer() for more details about this scenario.
+					changeServerKeys(data, keys, nowAssigned, currentVersion - 1, context);
+				}
 			}
 
 			processedStartKey = false;
@@ -6745,7 +7665,7 @@ private:
 			br >> rollbackVersion;
 
 			if (rollbackVersion < fromVersion && rollbackVersion > restoredVersion) {
-				TEST(true); // ShardApplyPrivateData shard rollback
+				CODE_PROBE(true, "ShardApplyPrivateData shard rollback");
 				TraceEvent(SevWarn, "Rollback", data->thisServerID)
 				    .detail("FromVersion", fromVersion)
 				    .detail("ToVersion", rollbackVersion)
@@ -6819,14 +7739,14 @@ private:
 			auto feed = data->uidChangeFeed.find(changeFeedId);
 
 			TraceEvent(SevDebug, "ChangeFeedPrivateMutation", data->thisServerID)
-			    .detail("RangeID", changeFeedId.printable())
-			    .detail("Range", changeFeedRange.toString())
+			    .detail("FeedID", changeFeedId)
+			    .detail("Range", changeFeedRange)
 			    .detail("Version", currentVersion)
 			    .detail("PopVersion", popVersion)
 			    .detail("Status", status);
 
 			// Because of data moves, we can get mutations operating on a change feed we don't yet know about, because
-			// the fetch hasn't started yet
+			// the metadata fetch hasn't started yet
 			bool createdFeed = false;
 			if (feed == data->uidChangeFeed.end() && status != ChangeFeedStatus::CHANGE_FEED_DESTROY) {
 				createdFeed = true;
@@ -6838,7 +7758,7 @@ private:
 					// for a create, the empty version should be now, otherwise it will be set in a later pop
 					changeFeedInfo->emptyVersion = currentVersion - 1;
 				} else {
-					TEST(true); // SS got non-create change feed private mutation before move created its metadata
+					CODE_PROBE(true, "SS got non-create change feed private mutation before move created its metadata");
 					changeFeedInfo->emptyVersion = invalidVersion;
 				}
 				changeFeedInfo->metadataCreateVersion = currentVersion;
@@ -6848,8 +7768,8 @@ private:
 				ASSERT(feed != data->uidChangeFeed.end());
 
 				TraceEvent(SevDebug, "AddingChangeFeed", data->thisServerID)
-				    .detail("RangeID", changeFeedId.printable())
-				    .detail("Range", changeFeedRange.toString())
+				    .detail("FeedID", changeFeedId)
+				    .detail("Range", changeFeedRange)
 				    .detail("EmptyVersion", feed->second->emptyVersion);
 
 				auto rs = data->keyChangeFeed.modify(changeFeedRange);
@@ -6857,6 +7777,32 @@ private:
 					r->value().push_back(changeFeedInfo);
 				}
 				data->keyChangeFeed.coalesce(changeFeedRange.contents());
+			} else if (feed != data->uidChangeFeed.end() && feed->second->removing && !feed->second->destroyed &&
+			           status != ChangeFeedStatus::CHANGE_FEED_DESTROY) {
+				// Because we got a private mutation for this change feed, the feed must have moved back after being
+				// moved away. Normally we would later find out about this via a fetch, but in the particular case where
+				// the private mutation is the creation of the change feed, and the following race occurred, we must
+				// refresh it here:
+				// 1. This SS found out about the feed from a fetch, from a SS with a higher version that already got
+				// the feed create mutation
+				// 2. The shard was moved away
+				// 3. The shard was moved back, and this SS fetched change feed metadata from a different SS that did
+				// not yet recieve the private mutation, so the feed was not refreshed
+				// 4. This SS gets the private mutation, the feed is still marked as removing
+				TraceEvent(SevDebug, "ResetChangeFeedInfoFromPrivateMutation", data->thisServerID)
+				    .detail("FeedID", changeFeedId)
+				    .detail("Range", changeFeedRange)
+				    .detail("Version", currentVersion);
+
+				CODE_PROBE(true, "private mutation for feed scheduled for deletion! Un-mark it as removing");
+
+				feed->second->removing = false;
+				// reset fetch versions because everything previously fetched was cleaned up
+				feed->second->fetchVersion = invalidVersion;
+				feed->second->durableFetchVersion = NotifiedVersion();
+			}
+			if (feed != data->uidChangeFeed.end()) {
+				feed->second->updateMetadataVersion(currentVersion);
 			}
 
 			bool popMutationLog = false;
@@ -6886,24 +7832,24 @@ private:
 
 			} else if (status == ChangeFeedStatus::CHANGE_FEED_CREATE && createdFeed) {
 				TraceEvent(SevDebug, "CreatingChangeFeed", data->thisServerID)
-				    .detail("RangeID", changeFeedId.printable())
-				    .detail("Range", changeFeedRange.toString())
+				    .detail("FeedID", changeFeedId)
+				    .detail("Range", changeFeedRange)
 				    .detail("Version", currentVersion);
 				// no-op, already created metadata
 				addMutationToLog = true;
 			}
 			if (status == ChangeFeedStatus::CHANGE_FEED_STOP && currentVersion < feed->second->stopVersion) {
 				TraceEvent(SevDebug, "StoppingChangeFeed", data->thisServerID)
-				    .detail("RangeID", changeFeedId.printable())
-				    .detail("Range", changeFeedRange.toString())
+				    .detail("FeedID", changeFeedId)
+				    .detail("Range", changeFeedRange)
 				    .detail("Version", currentVersion);
 				feed->second->stopVersion = currentVersion;
 				addMutationToLog = true;
 			}
 			if (status == ChangeFeedStatus::CHANGE_FEED_DESTROY && !createdFeed && feed != data->uidChangeFeed.end()) {
 				TraceEvent(SevDebug, "DestroyingChangeFeed", data->thisServerID)
-				    .detail("RangeID", changeFeedId.printable())
-				    .detail("Range", changeFeedRange.toString())
+				    .detail("FeedID", changeFeedId)
+				    .detail("Range", changeFeedRange)
 				    .detail("Version", currentVersion);
 				Key beginClearKey = changeFeedId.withPrefix(persistChangeFeedKeys.begin);
 				Version cleanupVersion = data->data().getLatestVersion();
@@ -6919,22 +7865,29 @@ private:
 
 				feed->second->destroy(currentVersion);
 				data->changeFeedCleanupDurable[feed->first] = cleanupVersion;
+
+				if (BUGGIFY) {
+					data->maybeInjectTargetedRestart(cleanupVersion);
+				}
 			}
 
 			if (status == ChangeFeedStatus::CHANGE_FEED_DESTROY) {
-				for (auto& it : data->changeFeedRemovals) {
+				for (auto& it : data->changeFeedDestroys) {
 					it.second.send(changeFeedId);
 				}
 			}
 
 			if (addMutationToLog) {
-				auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion());
+				Version logV = data->data().getLatestVersion();
+				auto& mLV = data->addVersionToMutationLog(logV);
 				data->addMutationToMutationLog(
 				    mLV,
 				    MutationRef(MutationRef::SetValue,
 				                persistChangeFeedKeys.begin.toString() + changeFeedId.toString(),
-				                changeFeedSSValue(
-				                    feed->second->range, feed->second->emptyVersion + 1, feed->second->stopVersion)));
+				                changeFeedSSValue(feed->second->range,
+				                                  feed->second->emptyVersion + 1,
+				                                  feed->second->stopVersion,
+				                                  feed->second->metadataVersion)));
 				if (popMutationLog) {
 					++data->counters.kvSystemClearRanges;
 					data->addMutationToMutationLog(mLV,
@@ -6942,14 +7895,18 @@ private:
 					                                           changeFeedDurableKey(feed->second->id, 0),
 					                                           changeFeedDurableKey(feed->second->id, popVersion)));
 				}
+				if (BUGGIFY) {
+					data->maybeInjectTargetedRestart(logV);
+				}
 			}
 		} else if ((m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) &&
-		           m.param1.startsWith(tenantMapPrivatePrefix)) {
+		           m.param1.startsWith(TenantMetadata::tenantMapPrivatePrefix())) {
 			if (m.type == MutationRef::SetValue) {
-				data->insertTenant(m.param1.removePrefix(tenantMapPrivatePrefix), m.param2, currentVersion, true);
+				data->insertTenant(
+				    m.param1.removePrefix(TenantMetadata::tenantMapPrivatePrefix()), m.param2, currentVersion);
 			} else if (m.type == MutationRef::ClearRange) {
-				data->clearTenants(m.param1.removePrefix(tenantMapPrivatePrefix),
-				                   m.param2.removePrefix(tenantMapPrivatePrefix),
+				data->clearTenants(m.param1.removePrefix(TenantMetadata::tenantMapPrivatePrefix()),
+				                   m.param2.removePrefix(TenantMetadata::tenantMapPrivatePrefix()),
 				                   currentVersion);
 			}
 		} else if (m.param1.substr(1).startsWith(tssMappingKeys.begin) &&
@@ -6978,7 +7935,7 @@ private:
 				UID ssId = decodeTssQuarantineKey(m.param1.substr(1));
 				ASSERT(ssId == data->thisServerID);
 				if (m.type == MutationRef::SetValue) {
-					TEST(true); // Putting TSS in quarantine
+					CODE_PROBE(true, "Putting TSS in quarantine");
 					TraceEvent(SevWarn, "TSSQuarantineStart", data->thisServerID).log();
 					data->startTssQuarantine();
 				} else {
@@ -7044,39 +8001,39 @@ private:
 	}
 };
 
-void StorageServer::insertTenant(TenantNameRef tenantName,
-                                 ValueRef value,
-                                 Version version,
-                                 bool insertIntoMutationLog) {
+bool StorageServer::insertTenant(TenantNameRef tenantName, TenantMapEntry tenantEntry, Version version) {
 	if (version >= tenantMap.getLatestVersion()) {
 		tenantMap.createNewVersion(version);
-		tenantPrefixIndex.createNewVersion(version);
-
-		TenantMapEntry tenantEntry = TenantMapEntry::decode(value);
+		tenantPrefixIndex->createNewVersion(version);
 
 		tenantMap.insert(tenantName, tenantEntry);
-		tenantPrefixIndex.insert(tenantEntry.prefix, tenantName);
-
-		if (insertIntoMutationLog) {
-			auto& mLV = addVersionToMutationLog(version);
-			addMutationToMutationLog(
-			    mLV, MutationRef(MutationRef::SetValue, tenantName.withPrefix(persistTenantMapKeys.begin), value));
-		}
+		tenantPrefixIndex->insert(tenantEntry.prefix, tenantName);
 
 		TraceEvent("InsertTenant", thisServerID).detail("Tenant", tenantName).detail("Version", version);
+		return true;
+	} else {
+		return false;
+	}
+}
+
+void StorageServer::insertTenant(TenantNameRef tenantName, ValueRef value, Version version) {
+	if (insertTenant(tenantName, TenantMapEntry::decode(value), version)) {
+		auto& mLV = addVersionToMutationLog(version);
+		addMutationToMutationLog(
+		    mLV, MutationRef(MutationRef::SetValue, tenantName.withPrefix(persistTenantMapKeys.begin), value));
 	}
 }
 
 void StorageServer::clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version) {
 	if (version >= tenantMap.getLatestVersion()) {
 		tenantMap.createNewVersion(version);
-		tenantPrefixIndex.createNewVersion(version);
+		tenantPrefixIndex->createNewVersion(version);
 
 		auto view = tenantMap.at(version);
 		for (auto itr = view.lower_bound(startTenant); itr != view.lower_bound(endTenant); ++itr) {
 			// Trigger any watches on the prefix associated with the tenant.
 			watches.triggerRange(itr->prefix, strinc(itr->prefix));
-			tenantPrefixIndex.erase(itr->prefix);
+			tenantPrefixIndex->erase(itr->prefix);
 			TraceEvent("EraseTenant", thisServerID).detail("Tenant", itr.key()).detail("Version", version);
 		}
 
@@ -7093,7 +8050,7 @@ void StorageServer::clearTenants(TenantNameRef startTenant, TenantNameRef endTen
 ACTOR Future<Void> tssDelayForever() {
 	loop {
 		wait(delay(5.0));
-		if (g_simulator.speedUpSimulation) {
+		if (g_simulator->speedUpSimulation) {
 			return Void();
 		}
 	}
@@ -7106,19 +8063,23 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 		// If we are disk bound and durableVersion is very old, we need to block updates or we could run out of
 		// memory. This is often referred to as the storage server e-brake (emergency brake)
 
-		// We allow the storage server to make some progress between e-brake periods, referreed to as "overage", in
+		// We allow the storage server to make some progress between e-brake periods, referred to as "overage", in
 		// order to ensure that it advances desiredOldestVersion enough for updateStorage to make enough progress on
-		// freeing up queue size.
+		// freeing up queue size. We also increase these limits if speed up simulation was set IF they were buggified to
+		// a very small value.
+		state int64_t hardLimit = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES;
+		state int64_t hardLimitOverage = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_OVERAGE;
+		if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
+			hardLimit = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_SPEED_UP_SIM;
+			hardLimitOverage = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_OVERAGE_SPEED_UP_SIM;
+		}
 		state double waitStartT = 0;
-		if (data->queueSize() >= SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES &&
-		    data->durableVersion.get() < data->desiredOldestVersion.get() &&
+		if (data->queueSize() >= hardLimit && data->durableVersion.get() < data->desiredOldestVersion.get() &&
 		    ((data->desiredOldestVersion.get() - SERVER_KNOBS->STORAGE_HARD_LIMIT_VERSION_OVERAGE >
 		      data->lastDurableVersionEBrake) ||
-		     (data->counters.bytesInput.getValue() - SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_OVERAGE >
-		      data->lastBytesInputEBrake))) {
+		     (data->counters.bytesInput.getValue() - hardLimitOverage > data->lastBytesInputEBrake))) {
 
-			while (data->queueSize() >= SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES &&
-			       data->durableVersion.get() < data->desiredOldestVersion.get()) {
+			while (data->queueSize() >= hardLimit && data->durableVersion.get() < data->desiredOldestVersion.get()) {
 				if (now() - waitStartT >= 1) {
 					TraceEvent(SevWarn, "StorageServerUpdateLag", data->thisServerID)
 					    .detail("Version", data->version.get())
@@ -7137,8 +8098,8 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 			data->lastDurableVersionEBrake = data->durableVersion.get();
 		}
 
-		if (g_network->isSimulated() && data->isTss() && g_simulator.tssMode == ISimulator::TSSMode::EnabledAddDelay &&
-		    !g_simulator.speedUpSimulation && data->tssFaultInjectTime.present() &&
+		if (g_network->isSimulated() && data->isTss() && g_simulator->tssMode == ISimulator::TSSMode::EnabledAddDelay &&
+		    !g_simulator->speedUpSimulation && data->tssFaultInjectTime.present() &&
 		    data->tssFaultInjectTime.get() < now()) {
 			if (deterministicRandom()->random01() < 0.01) {
 				TraceEvent(SevWarnAlways, "TSSInjectDelayForever", data->thisServerID).log();
@@ -7153,6 +8114,10 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 			}
 		}
 
+		if (data->maybeInjectDelay()) {
+			wait(delay(deterministicRandom()->random01() * 10.0));
+		}
+
 		while (data->byteSampleClearsTooLarge.get()) {
 			wait(data->byteSampleClearsTooLarge.onChange());
 		}
@@ -7197,7 +8162,7 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 		start = now();
 		state UpdateEagerReadInfo eager;
 		state FetchInjectionInfo fii;
-		state Reference<ILogSystem::IPeekCursor> cloneCursor2;
+		state Reference<ILogSystem::IPeekCursor> cloneCursor2 = cursor->cloneNoMore();
 		state Optional<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> cipherKeys;
 		state bool collectingCipherKeys = false;
 
@@ -7212,8 +8177,7 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 
 			std::unordered_set<BlobCipherDetails> cipherDetails;
 
-			Reference<ILogSystem::IPeekCursor> cloneCursor1 = cursor->cloneNoMore();
-			cloneCursor2 = cursor->cloneNoMore();
+			Reference<ILogSystem::IPeekCursor> cloneCursor1 = cloneCursor2->cloneNoMore();
 
 			cloneCursor1->setProtocolVersion(data->logProtocol);
 
@@ -7234,23 +8198,18 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 				           OTELSpanContextMessage::isNextIn(cloneReader)) {
 					OTELSpanContextMessage scm;
 					cloneReader >> scm;
-				} else if (cloneReader.protocolVersion().hasEncryptionAtRest() &&
-				           EncryptedMutationMessage::isNextIn(cloneReader) && !cipherKeys.present()) {
-					// Encrypted mutation found, but cipher keys haven't been fetch.
-					// Collect cipher details to fetch cipher keys in one batch.
-					EncryptedMutationMessage emm;
-					cloneReader >> emm;
-					cipherDetails.insert(emm.header.cipherTextDetails);
-					cipherDetails.insert(emm.header.cipherHeaderDetails);
-					collectingCipherKeys = true;
 				} else {
 					MutationRef msg;
-					if (cloneReader.protocolVersion().hasEncryptionAtRest() &&
-					    EncryptedMutationMessage::isNextIn(cloneReader)) {
-						assert(cipherKeys.present());
-						msg = EncryptedMutationMessage::decrypt(cloneReader, eager.arena, cipherKeys.get());
-					} else {
-						cloneReader >> msg;
+					cloneReader >> msg;
+					if (msg.isEncrypted()) {
+						if (!cipherKeys.present()) {
+							const BlobCipherEncryptHeader* header = msg.encryptionHeader();
+							cipherDetails.insert(header->cipherTextDetails);
+							cipherDetails.insert(header->cipherHeaderDetails);
+							collectingCipherKeys = true;
+						} else {
+							msg = msg.decrypt(cipherKeys.get(), eager.arena, BlobCipherMetrics::TLOG);
+						}
 					}
 					// TraceEvent(SevDebug, "SSReadingLog", data->thisServerID).detail("Mutation", msg);
 
@@ -7272,7 +8231,7 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 
 			if (collectingCipherKeys) {
 				std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> getCipherKeysResult =
-				    wait(getEncryptCipherKeys(data->db, cipherDetails));
+				    wait(getEncryptCipherKeys(data->db, cipherDetails, BlobCipherMetrics::TLOG));
 				cipherKeys = getCipherKeysResult;
 				collectingCipherKeys = false;
 				eager = UpdateEagerReadInfo();
@@ -7294,11 +8253,13 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 				wait(doEagerReads(data, &eager));
 				if (data->shardChangeCounter == changeCounter)
 					break;
-				TEST(true); // A fetchKeys completed while we were doing this, so eager might be outdated.  Read it
-				            // again.
+				CODE_PROBE(
+				    true,
+				    "A fetchKeys completed while we were doing this, so eager might be outdated.  Read it again.");
 				// SOMEDAY: Theoretically we could check the change counters of individual shards and retry the reads
 				// only selectively
 				eager = UpdateEagerReadInfo();
+				cloneCursor2 = cursor->cloneNoMore();
 			}
 		}
 		data->eagerReadsLatencyHistogram->sampleSeconds(now() - start);
@@ -7379,24 +8340,23 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 			} else if (rd.protocolVersion().hasSpanContext() && SpanContextMessage::isNextIn(rd)) {
 				SpanContextMessage scm;
 				rd >> scm;
-				TEST(true); // storageserveractor converting SpanContextMessage into OTEL SpanContext
+				CODE_PROBE(true, "storageserveractor converting SpanContextMessage into OTEL SpanContext");
 				spanContext =
 				    SpanContext(UID(scm.spanContext.first(), scm.spanContext.second()),
 				                0,
 				                scm.spanContext.first() != 0 && scm.spanContext.second() != 0 ? TraceFlags::sampled
 				                                                                              : TraceFlags::unsampled);
 			} else if (rd.protocolVersion().hasOTELSpanContext() && OTELSpanContextMessage::isNextIn(rd)) {
-				TEST(true); // storageserveractor reading OTELSpanContextMessage
+				CODE_PROBE(true, "storageserveractor reading OTELSpanContextMessage");
 				OTELSpanContextMessage scm;
 				rd >> scm;
 				spanContext = scm.spanContext;
 			} else {
 				MutationRef msg;
-				if (rd.protocolVersion().hasEncryptionAtRest() && EncryptedMutationMessage::isNextIn(rd)) {
+				rd >> msg;
+				if (msg.isEncrypted()) {
 					ASSERT(cipherKeys.present());
-					msg = EncryptedMutationMessage::decrypt(rd, rd.arena(), cipherKeys.get());
-				} else {
-					rd >> msg;
+					msg = msg.decrypt(cipherKeys.get(), rd.arena(), BlobCipherMetrics::TLOG);
 				}
 
 				Span span("SS:update"_loc, spanContext);
@@ -7404,8 +8364,8 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 
 				// Drop non-private mutations if TSS fault injection is enabled in simulation, or if this is a TSS in
 				// quarantine.
-				if (g_network->isSimulated() && data->isTss() && !g_simulator.speedUpSimulation &&
-				    g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations &&
+				if (g_network->isSimulated() && data->isTss() && !g_simulator->speedUpSimulation &&
+				    g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations &&
 				    data->tssFaultInjectTime.present() && data->tssFaultInjectTime.get() < now() &&
 				    (msg.type == MutationRef::SetValue || msg.type == MutationRef::ClearRange) &&
 				    (msg.param1.size() < 2 || msg.param1[0] != 0xff || msg.param1[1] != 0xff) &&
@@ -7523,7 +8483,7 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 				data->otherError.getFuture().get();
 
 			Version maxVersionsInMemory =
-			    (g_network->isSimulated() && g_simulator.speedUpSimulation)
+			    (g_network->isSimulated() && g_simulator->speedUpSimulation)
 			        ? std::max(5 * SERVER_KNOBS->VERSIONS_PER_SECOND, SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS)
 			        : SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS;
 			for (int i = 0; i < data->recoveryVersionSkips.size(); i++) {
@@ -7637,11 +8597,13 @@ ACTOR Future<Void> createCheckpoint(StorageServer* data, CheckpointMetaData meta
 }
 
 ACTOR Future<Void> updateStorage(StorageServer* data) {
+	state UnlimitedCommitBytes unlimitedCommitBytes = UnlimitedCommitBytes::False;
 	loop {
+		unlimitedCommitBytes = UnlimitedCommitBytes::False;
 		ASSERT(data->durableVersion.get() == data->storageVersion());
 		if (g_network->isSimulated()) {
 			double endTime =
-			    g_simulator.checkDisabled(format("%s/updateStorage", data->thisServerID.toString().c_str()));
+			    g_simulator->checkDisabled(format("%s/updateStorage", data->thisServerID.toString().c_str()));
 			if (endTime > now()) {
 				wait(delay(endTime - now(), TaskPriority::UpdateStorage));
 			}
@@ -7676,7 +8638,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 		if (!data->pendingCheckpoints.empty()) {
 			const Version cVer = data->pendingCheckpoints.begin()->first;
 			if (cVer <= desiredVersion) {
-				TraceEvent("CheckpointVersionSatisfied", data->thisServerID)
+				TraceEvent(SevDebug, "CheckpointVersionSatisfied", data->thisServerID)
 				    .detail("DesiredVersion", desiredVersion)
 				    .detail("DurableVersion", data->durableVersion.get())
 				    .detail("CheckPointVersion", cVer);
@@ -7685,13 +8647,59 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 			}
 		}
 
+		state bool removeKVSRanges = false;
+		if (!data->pendingRemoveRanges.empty()) {
+			const Version aVer = data->pendingRemoveRanges.begin()->first;
+			if (aVer <= desiredVersion) {
+				TraceEvent(SevDebug, "RemoveRangeVersionSatisfied", data->thisServerID)
+				    .detail("DesiredVersion", desiredVersion)
+				    .detail("DurableVersion", data->durableVersion.get())
+				    .detail("RemoveRangeVersion", aVer);
+				desiredVersion = aVer;
+				removeKVSRanges = true;
+			}
+		}
+
+		state bool addedRanges = false;
+		if (!data->pendingAddRanges.empty()) {
+			const Version aVer = data->pendingAddRanges.begin()->first;
+			if (aVer <= desiredVersion) {
+				TraceEvent(SevDebug, "AddRangeVersionSatisfied", data->thisServerID)
+				    .detail("DesiredVersion", desiredVersion)
+				    .detail("DurableVersion", data->durableVersion.get())
+				    .detail("AddRangeVersion", aVer);
+				desiredVersion = aVer;
+				ASSERT(!data->pendingAddRanges.begin()->second.empty());
+				TraceEvent(SevVerbose, "SSAddKVSRangeBegin", data->thisServerID)
+				    .detail("Version", data->pendingAddRanges.begin()->first)
+				    .detail("DurableVersion", data->durableVersion.get())
+				    .detail("NewRanges", describe(data->pendingAddRanges.begin()->second));
+				state std::vector<Future<Void>> fAddRanges;
+				for (const auto& shard : data->pendingAddRanges.begin()->second) {
+					TraceEvent(SevInfo, "SSAddKVSRange", data->thisServerID)
+					    .detail("Range", shard.range)
+					    .detail("PhysicalShardID", shard.shardId);
+					fAddRanges.push_back(data->storage.addRange(shard.range, shard.shardId));
+				}
+				wait(waitForAll(fAddRanges));
+				TraceEvent(SevVerbose, "SSAddKVSRangeEnd", data->thisServerID)
+				    .detail("Version", data->pendingAddRanges.begin()->first)
+				    .detail("DurableVersion", data->durableVersion.get());
+				addedRanges = true;
+				// Remove commit byte limit to make sure the private mutaiton(s) associated with the
+				// `addRange` are committed.
+				unlimitedCommitBytes = UnlimitedCommitBytes::True;
+			}
+		}
+
 		// Write mutations to storage until we reach the desiredVersion or have written too much (bytesleft)
 		state double beforeStorageUpdates = now();
 		loop {
-			state bool done = data->storage.makeVersionMutationsDurable(newOldestVersion, desiredVersion, bytesLeft);
+			state bool done = data->storage.makeVersionMutationsDurable(
+			    newOldestVersion, desiredVersion, bytesLeft, unlimitedCommitBytes);
 			if (data->tenantMap.getLatestVersion() < newOldestVersion) {
 				data->tenantMap.createNewVersion(newOldestVersion);
-				data->tenantPrefixIndex.createNewVersion(newOldestVersion);
+				data->tenantPrefixIndex->createNewVersion(newOldestVersion);
 			}
 			// We want to forget things from these data structures atomically with changing oldestVersion (and "before",
 			// since oldestVersion.set() may trigger waiting actors) forgetVersionsBeforeAsync visibly forgets
@@ -7699,7 +8707,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 			Future<Void> finishedForgetting =
 			    data->mutableData().forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage) &&
 			    data->tenantMap.forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage) &&
-			    data->tenantPrefixIndex.forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage);
+			    data->tenantPrefixIndex->forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage);
 			data->oldestVersion.set(newOldestVersion);
 			wait(finishedForgetting);
 			wait(yield(TaskPriority::UpdateStorage));
@@ -7707,6 +8715,32 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 				break;
 		}
 
+		if (addedRanges) {
+			TraceEvent(SevVerbose, "SSAddKVSRangeMetaData", data->thisServerID)
+			    .detail("NewDurableVersion", newOldestVersion)
+			    .detail("DesiredVersion", desiredVersion)
+			    .detail("OldestRemoveKVSRangesVersion", data->pendingAddRanges.begin()->first);
+			ASSERT(newOldestVersion == data->pendingAddRanges.begin()->first);
+			ASSERT(newOldestVersion == desiredVersion);
+			for (const auto& shard : data->pendingAddRanges.begin()->second) {
+				data->storage.persistRangeMapping(shard.range, true);
+			}
+			data->pendingAddRanges.erase(data->pendingAddRanges.begin());
+		}
+
+		if (removeKVSRanges) {
+			TraceEvent(SevDebug, "RemoveKVSRangesVersionDurable", data->thisServerID)
+			    .detail("NewDurableVersion", newOldestVersion)
+			    .detail("DesiredVersion", desiredVersion)
+			    .detail("OldestRemoveKVSRangesVersion", data->pendingRemoveRanges.begin()->first);
+			ASSERT(newOldestVersion <= data->pendingRemoveRanges.begin()->first);
+			if (newOldestVersion == data->pendingRemoveRanges.begin()->first) {
+				for (const auto& range : data->pendingRemoveRanges.begin()->second) {
+					data->storage.persistRangeMapping(range, false);
+				}
+			}
+		}
+
 		std::set<Key> modifiedChangeFeeds = data->fetchingChangeFeeds;
 		data->fetchingChangeFeeds.clear();
 		while (!data->changeFeedVersions.empty() && data->changeFeedVersions.front().second <= newOldestVersion) {
@@ -7719,6 +8753,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 
 		state std::vector<Key> updatedChangeFeeds(modifiedChangeFeeds.begin(), modifiedChangeFeeds.end());
 		state int curFeed = 0;
+		state int64_t durableChangeFeedMutations = 0;
 		while (curFeed < updatedChangeFeeds.size()) {
 			auto info = data->uidChangeFeed.find(updatedChangeFeeds[curFeed]);
 			if (info != data->uidChangeFeed.end()) {
@@ -7737,6 +8772,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 					// in the stream. We should fix this assert to be strictly > and re-enable it
 					ASSERT(it.version >= info->second->storageVersion);
 					info->second->storageVersion = it.version;
+					durableChangeFeedMutations++;
 				}
 
 				if (info->second->fetchVersion != invalidVersion && !info->second->removing) {
@@ -7778,6 +8814,31 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 
 		debug_advanceMinCommittedVersion(data->thisServerID, newOldestVersion);
 
+		if (removeKVSRanges) {
+			TraceEvent(SevDebug, "RemoveKVSRangesComitted", data->thisServerID)
+			    .detail("NewDurableVersion", newOldestVersion)
+			    .detail("DesiredVersion", desiredVersion)
+			    .detail("OldestRemoveKVSRangesVersion", data->pendingRemoveRanges.begin()->first);
+			ASSERT(newOldestVersion <= data->pendingRemoveRanges.begin()->first);
+			if (newOldestVersion == data->pendingRemoveRanges.begin()->first) {
+				state std::vector<std::string> emptyShardIds;
+				for (const auto& range : data->pendingRemoveRanges.begin()->second) {
+					auto ids = data->storage.removeRange(range);
+					emptyShardIds.insert(emptyShardIds.end(), ids.begin(), ids.end());
+					TraceEvent(SevVerbose, "RemoveKVSRange", data->thisServerID).detail("Range", range);
+				}
+				if (emptyShardIds.size() > 0) {
+					state double start = now();
+					wait(data->storage.cleanUpShardsIfNeeded(emptyShardIds));
+					TraceEvent(SevInfo, "RemoveEmptyPhysicalShards", data->thisServerID)
+					    .detail("NumShards", emptyShardIds.size())
+					    .detail("TimeSpent", now() - start);
+				}
+				data->pendingRemoveRanges.erase(data->pendingRemoveRanges.begin());
+			}
+			removeKVSRanges = false;
+		}
+
 		if (requireCheckpoint) {
 			// `pendingCheckpoints` is a queue of checkpoint requests ordered by their versoins, and
 			// `newOldestVersion` is chosen such that it is no larger than the smallest pending checkpoing
@@ -7809,6 +8870,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 			TraceEvent("RebootWhenDurableTriggered", data->thisServerID)
 			    .detail("NewOldestVersion", newOldestVersion)
 			    .detail("RebootAfterDurableVersion", data->rebootAfterDurableVersion);
+			CODE_PROBE(true, "SS rebooting after durable");
 			// To avoid brokenPromise error, which is caused by the sender of the durableInProgress (i.e., this
 			// process) never sets durableInProgress, we should set durableInProgress before send the
 			// please_reboot() error. Otherwise, in the race situation when storage server receives both reboot and
@@ -7872,7 +8934,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 
 					data->uidChangeFeed.erase(feed);
 				} else {
-					TEST(true); // Feed re-fetched after remove
+					CODE_PROBE(true, "Feed re-fetched after remove");
 				}
 				cfCleanup = data->changeFeedCleanupDurable.erase(cfCleanup);
 			} else {
@@ -7880,6 +8942,8 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 			}
 		}
 
+		data->counters.changeFeedMutationsDurable += durableChangeFeedMutations;
+
 		durableInProgress.send(Void());
 		wait(delay(0, TaskPriority::UpdateStorage)); // Setting durableInProgess could cause the storage server to
 		                                             // shut down, so delay to check for cancellation
@@ -7905,6 +8969,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 
 		//TraceEvent("StorageServerDurable", data->thisServerID).detail("Version", newOldestVersion);
 		data->fetchKeysBytesBudget = SERVER_KNOBS->STORAGE_FETCH_BYTES;
+
 		data->fetchKeysBudgetUsed.set(false);
 		if (!data->fetchKeysBudgetUsed.get()) {
 			wait(durableDelay || data->fetchKeysBudgetUsed.onChange());
@@ -7921,8 +8986,12 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 #pragma region StorageServerDisk
 #endif
 
-void StorageServerDisk::makeNewStorageServerDurable() {
-	storage->set(persistFormat);
+void StorageServerDisk::makeNewStorageServerDurable(const bool shardAware) {
+	if (shardAware) {
+		storage->set(persistShardAwareFormat);
+	} else {
+		storage->set(persistFormat);
+	}
 	storage->set(KeyValueRef(persistID, BinaryWriter::toValue(data->thisServerID, Unversioned())));
 	if (data->tssPairID.present()) {
 		storage->set(KeyValueRef(persistTssPairID, BinaryWriter::toValue(data->tssPairID.get(), Unversioned())));
@@ -7931,8 +9000,14 @@ void StorageServerDisk::makeNewStorageServerDurable() {
 	storage->set(
 	    KeyValueRef(persistClusterIdKey, BinaryWriter::toValue(data->clusterId.getFuture().get(), Unversioned())));
 	storage->set(KeyValueRef(persistVersion, BinaryWriter::toValue(data->version.get(), Unversioned())));
-	storage->set(KeyValueRef(persistShardAssignedKeys.begin.toString(), LiteralStringRef("0")));
-	storage->set(KeyValueRef(persistShardAvailableKeys.begin.toString(), LiteralStringRef("0")));
+
+	if (shardAware) {
+		storage->set(KeyValueRef(persistStorageServerShardKeys.begin.toString(),
+		                         ObjectWriter::toValue(StorageServerShard::notAssigned(allKeys, 0), IncludeVersion())));
+	} else {
+		storage->set(KeyValueRef(persistShardAssignedKeys.begin.toString(), "0"_sr));
+		storage->set(KeyValueRef(persistShardAvailableKeys.begin.toString(), "0"_sr));
+	}
 
 	auto view = data->tenantMap.atLatest();
 	for (auto itr = view.begin(); itr != view.end(); ++itr) {
@@ -7944,7 +9019,8 @@ void setAvailableStatus(StorageServer* self, KeyRangeRef keys, bool available) {
 	// ASSERT( self->debug_inApplyUpdate );
 	ASSERT(!keys.empty());
 
-	auto& mLV = self->addVersionToMutationLog(self->data().getLatestVersion());
+	Version logV = self->data().getLatestVersion();
+	auto& mLV = self->addVersionToMutationLog(logV);
 
 	KeyRange availableKeys = KeyRangeRef(persistShardAvailableKeys.begin.toString() + keys.begin.toString(),
 	                                     persistShardAvailableKeys.begin.toString() + keys.end.toString());
@@ -7952,16 +9028,12 @@ void setAvailableStatus(StorageServer* self, KeyRangeRef keys, bool available) {
 
 	self->addMutationToMutationLog(mLV, MutationRef(MutationRef::ClearRange, availableKeys.begin, availableKeys.end));
 	++self->counters.kvSystemClearRanges;
-	self->addMutationToMutationLog(mLV,
-	                               MutationRef(MutationRef::SetValue,
-	                                           availableKeys.begin,
-	                                           available ? LiteralStringRef("1") : LiteralStringRef("0")));
+	self->addMutationToMutationLog(
+	    mLV, MutationRef(MutationRef::SetValue, availableKeys.begin, available ? "1"_sr : "0"_sr));
 	if (keys.end != allKeys.end) {
 		bool endAvailable = self->shards.rangeContaining(keys.end)->value()->isCFInVersionedData();
-		self->addMutationToMutationLog(mLV,
-		                               MutationRef(MutationRef::SetValue,
-		                                           availableKeys.end,
-		                                           endAvailable ? LiteralStringRef("1") : LiteralStringRef("0")));
+		self->addMutationToMutationLog(
+		    mLV, MutationRef(MutationRef::SetValue, availableKeys.end, endAvailable ? "1"_sr : "0"_sr));
 	}
 
 	// When a shard is moved out, delete all related checkpoints created for data move.
@@ -7980,26 +9052,63 @@ void setAvailableStatus(StorageServer* self, KeyRangeRef keys, bool available) {
 			    .detail("DeleteVersion", mLV.version + 1);
 		}
 	}
+
+	if (BUGGIFY) {
+		self->maybeInjectTargetedRestart(logV);
+	}
+}
+
+void updateStorageShard(StorageServer* data, StorageServerShard shard) {
+	if (shard.getShardState() == StorageServerShard::ReadWritePending) {
+		shard.setShardState(StorageServerShard::ReadWrite);
+	}
+
+	auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion());
+
+	KeyRange shardKeys = KeyRangeRef(persistStorageServerShardKeys.begin.toString() + shard.range.begin.toString(),
+	                                 persistStorageServerShardKeys.begin.toString() + shard.range.end.toString());
+	TraceEvent(SevVerbose, "UpdateStorageServerShard", data->thisServerID)
+	    .detail("Version", mLV.version)
+	    .detail("Shard", shard.toString())
+	    .detail("ShardKey", shardKeys.begin);
+
+	data->addMutationToMutationLog(mLV, MutationRef(MutationRef::ClearRange, shardKeys.begin, shardKeys.end));
+	++data->counters.kvSystemClearRanges;
+	data->addMutationToMutationLog(
+	    mLV, MutationRef(MutationRef::SetValue, shardKeys.begin, ObjectWriter::toValue(shard, IncludeVersion())));
+	if (shard.range.end != allKeys.end) {
+		StorageServerShard endShard = data->shards.rangeContaining(shard.range.end)->value()->toStorageServerShard();
+		if (endShard.getShardState() == StorageServerShard::ReadWritePending) {
+			endShard.setShardState(StorageServerShard::ReadWrite);
+		}
+		TraceEvent(SevVerbose, "UpdateStorageServerShardEndShard", data->thisServerID)
+		    .detail("Version", mLV.version)
+		    .detail("Shard", endShard.toString())
+		    .detail("ShardKey", shardKeys.end);
+		data->addMutationToMutationLog(
+		    mLV, MutationRef(MutationRef::SetValue, shardKeys.end, ObjectWriter::toValue(endShard, IncludeVersion())));
+	}
 }
 
 void setAssignedStatus(StorageServer* self, KeyRangeRef keys, bool nowAssigned) {
 	ASSERT(!keys.empty());
-	auto& mLV = self->addVersionToMutationLog(self->data().getLatestVersion());
+	Version logV = self->data().getLatestVersion();
+	auto& mLV = self->addVersionToMutationLog(logV);
 	KeyRange assignedKeys = KeyRangeRef(persistShardAssignedKeys.begin.toString() + keys.begin.toString(),
 	                                    persistShardAssignedKeys.begin.toString() + keys.end.toString());
 	//TraceEvent("SetAssignedStatus", self->thisServerID).detail("Version", mLV.version).detail("RangeBegin", assignedKeys.begin).detail("RangeEnd", assignedKeys.end);
 	self->addMutationToMutationLog(mLV, MutationRef(MutationRef::ClearRange, assignedKeys.begin, assignedKeys.end));
 	++self->counters.kvSystemClearRanges;
-	self->addMutationToMutationLog(mLV,
-	                               MutationRef(MutationRef::SetValue,
-	                                           assignedKeys.begin,
-	                                           nowAssigned ? LiteralStringRef("1") : LiteralStringRef("0")));
+	self->addMutationToMutationLog(
+	    mLV, MutationRef(MutationRef::SetValue, assignedKeys.begin, nowAssigned ? "1"_sr : "0"_sr));
 	if (keys.end != allKeys.end) {
 		bool endAssigned = self->shards.rangeContaining(keys.end)->value()->assigned();
-		self->addMutationToMutationLog(mLV,
-		                               MutationRef(MutationRef::SetValue,
-		                                           assignedKeys.end,
-		                                           endAssigned ? LiteralStringRef("1") : LiteralStringRef("0")));
+		self->addMutationToMutationLog(
+		    mLV, MutationRef(MutationRef::SetValue, assignedKeys.end, endAssigned ? "1"_sr : "0"_sr));
+	}
+
+	if (BUGGIFY) {
+		self->maybeInjectTargetedRestart(logV);
 	}
 }
 
@@ -8041,8 +9150,9 @@ void StorageServerDisk::writeMutations(const VectorRef<MutationRef>& mutations,
 
 bool StorageServerDisk::makeVersionMutationsDurable(Version& prevStorageVersion,
                                                     Version newStorageVersion,
-                                                    int64_t& bytesLeft) {
-	if (bytesLeft <= 0)
+                                                    int64_t& bytesLeft,
+                                                    UnlimitedCommitBytes unlimitedCommitBytes) {
+	if (!unlimitedCommitBytes && bytesLeft <= 0)
 		return true;
 
 	// Apply mutations from the mutationLog
@@ -8075,7 +9185,7 @@ void StorageServerDisk::makeVersionDurable(Version version) {
 
 // Update data->storage to persist tss quarantine state
 void StorageServerDisk::makeTssQuarantineDurable() {
-	storage->set(KeyValueRef(persistTssQuarantine, LiteralStringRef("1")));
+	storage->set(KeyValueRef(persistTssQuarantine, "1"_sr));
 }
 
 void StorageServerDisk::changeLogProtocol(Version version, ProtocolVersion protocol) {
@@ -8124,7 +9234,7 @@ ACTOR Future<Void> applyByteSampleResult(StorageServer* data,
 		} else {
 			data->byteSampleClears.insert(KeyRangeRef(begin.removePrefix(persistByteSampleKeys.begin),
 			                                          end == persistByteSampleKeys.end
-			                                              ? LiteralStringRef("\xff\xff\xff")
+			                                              ? "\xff\xff\xff"_sr
 			                                              : end.removePrefix(persistByteSampleKeys.begin)),
 			                              true);
 			data->byteSampleClearsTooLarge.set(data->byteSampleClears.size() >
@@ -8200,6 +9310,7 @@ ACTOR Future<UID> getClusterId(StorageServer* self) {
 		try {
 			self->cx->invalidateCache(Key(), systemKeys);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			Optional<Value> clusterId = wait(tr.get(clusterIdKey));
 			ASSERT(clusterId.present());
@@ -8252,6 +9363,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	state Future<RangeResult> fPendingCheckpoints = storage->readRange(persistPendingCheckpointKeys);
 	state Future<RangeResult> fCheckpoints = storage->readRange(persistCheckpointKeys);
 	state Future<RangeResult> fTenantMap = storage->readRange(persistTenantMapKeys);
+	state Future<RangeResult> fStorageShards = storage->readRange(persistStorageServerShardKeys);
 
 	state Promise<Void> byteSampleSampleRecovered;
 	state Promise<Void> startByteSampleRestore;
@@ -8261,8 +9373,13 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	TraceEvent("ReadingDurableState", data->thisServerID).log();
 	wait(waitForAll(std::vector{
 	    fFormat, fID, fClusterID, ftssPairID, fssPairID, fTssQuarantine, fVersion, fLogProtocol, fPrimaryLocality }));
-	wait(waitForAll(
-	    std::vector{ fShardAssigned, fShardAvailable, fChangeFeeds, fPendingCheckpoints, fCheckpoints, fTenantMap }));
+	wait(waitForAll(std::vector{ fShardAssigned,
+	                             fShardAvailable,
+	                             fChangeFeeds,
+	                             fPendingCheckpoints,
+	                             fCheckpoints,
+	                             fTenantMap,
+	                             fStorageShards }));
 	wait(byteSampleSampleRecovered.getFuture());
 	TraceEvent("RestoringDurableState", data->thisServerID).log();
 
@@ -8273,6 +9390,9 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		data->thisServerID = UID();
 		data->sk = Key();
 		return false;
+	} else {
+		TraceEvent(SevVerbose, "RestoringStorageServerWithPhysicalShards", data->thisServerID).log();
+		data->shardAware = fFormat.get().get() == persistShardAwareFormat.value;
 	}
 	data->bytesRestored += fFormat.get().expectedSize();
 	if (!persistFormatReadableRange.contains(fFormat.get().get())) {
@@ -8297,7 +9417,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		data->clusterId.send(BinaryReader::fromStringRef<UID>(fClusterID.get().get(), Unversioned()));
 		data->bytesRestored += fClusterID.get().expectedSize();
 	} else {
-		TEST(true); // storage server upgraded to version supporting cluster IDs
+		CODE_PROBE(true, "storage server upgraded to version supporting cluster IDs");
 		data->actors.add(persistClusterId(data));
 	}
 
@@ -8305,7 +9425,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	// state means the storage engine already had a durability or correctness error, but it should get
 	// re-quarantined very quickly because of a mismatch if it starts trying to do things again
 	if (fTssQuarantine.get().present()) {
-		TEST(true); // TSS restarted while quarantined
+		CODE_PROBE(true, "TSS restarted while quarantined");
 		data->tssInQuarantine = true;
 		data->bytesRestored += fTssQuarantine.get().expectedSize();
 	}
@@ -8357,7 +9477,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		                     : available[availableLoc + 1].key.removePrefix(persistShardAvailableKeys.begin));
 		ASSERT(!keys.empty());
 
-		bool nowAvailable = available[availableLoc].value != LiteralStringRef("0");
+		bool nowAvailable = available[availableLoc].value != "0"_sr;
 		/*if(nowAvailable)
 		  TraceEvent("AvailableShard", data->thisServerID).detail("RangeBegin", keys.begin).detail("RangeEnd", keys.end);*/
 		data->newestAvailableVersion.insert(keys, nowAvailable ? latestVersion : invalidVersion);
@@ -8367,20 +9487,25 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	state RangeResult assigned = fShardAssigned.get();
 	data->bytesRestored += assigned.logicalSize();
 	state int assignedLoc;
-	for (assignedLoc = 0; assignedLoc < assigned.size(); assignedLoc++) {
-		KeyRangeRef keys(assigned[assignedLoc].key.removePrefix(persistShardAssignedKeys.begin),
-		                 assignedLoc + 1 == assigned.size()
-		                     ? allKeys.end
-		                     : assigned[assignedLoc + 1].key.removePrefix(persistShardAssignedKeys.begin));
-		ASSERT(!keys.empty());
-		bool nowAssigned = assigned[assignedLoc].value != LiteralStringRef("0");
-		/*if(nowAssigned)
-		  TraceEvent("AssignedShard", data->thisServerID).detail("RangeBegin", keys.begin).detail("RangeEnd", keys.end);*/
-		changeServerKeys(data, keys, nowAssigned, version, CSK_RESTORE);
+	if (data->shardAware) {
+		// TODO(psm): Avoid copying RangeResult around.
+		wait(restoreShards(data, version, fStorageShards.get(), assigned, available));
+	} else {
+		for (assignedLoc = 0; assignedLoc < assigned.size(); assignedLoc++) {
+			KeyRangeRef keys(assigned[assignedLoc].key.removePrefix(persistShardAssignedKeys.begin),
+			                 assignedLoc + 1 == assigned.size()
+			                     ? allKeys.end
+			                     : assigned[assignedLoc + 1].key.removePrefix(persistShardAssignedKeys.begin));
+			ASSERT(!keys.empty());
+			bool nowAssigned = assigned[assignedLoc].value != "0"_sr;
+			/*if(nowAssigned)
+			  TraceEvent("AssignedShard", data->thisServerID).detail("RangeBegin", keys.begin).detail("RangeEnd", keys.end);*/
+			changeServerKeys(data, keys, nowAssigned, version, CSK_RESTORE);
 
-		if (!nowAssigned)
-			ASSERT(data->newestAvailableVersion.allEqual(keys, invalidVersion));
-		wait(yield());
+			if (!nowAssigned)
+				ASSERT(data->newestAvailableVersion.allEqual(keys, invalidVersion));
+			wait(yield());
+		}
 	}
 
 	state RangeResult changeFeeds = fChangeFeeds.get();
@@ -8389,13 +9514,15 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	for (feedLoc = 0; feedLoc < changeFeeds.size(); feedLoc++) {
 		Key changeFeedId = changeFeeds[feedLoc].key.removePrefix(persistChangeFeedKeys.begin);
 		KeyRange changeFeedRange;
-		Version popVersion, stopVersion;
-		std::tie(changeFeedRange, popVersion, stopVersion) = decodeChangeFeedSSValue(changeFeeds[feedLoc].value);
+		Version popVersion, stopVersion, metadataVersion;
+		std::tie(changeFeedRange, popVersion, stopVersion, metadataVersion) =
+		    decodeChangeFeedSSValue(changeFeeds[feedLoc].value);
 		TraceEvent(SevDebug, "RestoringChangeFeed", data->thisServerID)
-		    .detail("RangeID", changeFeedId.printable())
-		    .detail("Range", changeFeedRange.toString())
+		    .detail("FeedID", changeFeedId)
+		    .detail("Range", changeFeedRange)
 		    .detail("StopVersion", stopVersion)
-		    .detail("PopVer", popVersion);
+		    .detail("PopVer", popVersion)
+		    .detail("MetadataVersion", metadataVersion);
 		Reference<ChangeFeedInfo> changeFeedInfo(new ChangeFeedInfo());
 		changeFeedInfo->range = changeFeedRange;
 		changeFeedInfo->id = changeFeedId;
@@ -8403,6 +9530,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		changeFeedInfo->storageVersion = version;
 		changeFeedInfo->emptyVersion = popVersion - 1;
 		changeFeedInfo->stopVersion = stopVersion;
+		changeFeedInfo->metadataVersion = metadataVersion;
 		data->uidChangeFeed[changeFeedId] = changeFeedInfo;
 		auto rs = data->keyChangeFeed.modify(changeFeedRange);
 		for (auto r = rs.begin(); r != rs.end(); ++r) {
@@ -8420,7 +9548,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		TenantMapEntry tenantEntry = TenantMapEntry::decode(result.value);
 
 		data->tenantMap.insert(tenantName, tenantEntry);
-		data->tenantPrefixIndex.insert(tenantEntry.prefix, tenantName);
+		data->tenantPrefixIndex->insert(tenantEntry.prefix, tenantName);
 
 		TraceEvent("RestoringTenant", data->thisServerID)
 		    .detail("Key", tenantMap[tenantMapLoc].key)
@@ -8580,7 +9708,7 @@ ACTOR Future<Void> waitMetrics(StorageServerMetrics* self, WaitMetricsRequest re
 	state bool timedout = false;
 
 	if (!req.min.allLessOrEqual(metrics) || !metrics.allLessOrEqual(req.max)) {
-		TEST(true); // ShardWaitMetrics return case 1 (quickly)
+		CODE_PROBE(true, "ShardWaitMetrics return case 1 (quickly)");
 		req.reply.send(metrics);
 		return Void();
 	}
@@ -8621,7 +9749,7 @@ ACTOR Future<Void> waitMetrics(StorageServerMetrics* self, WaitMetricsRequest re
 			}
 
 			if (timedout) {
-				TEST(true); // ShardWaitMetrics return on timeout
+				CODE_PROBE(true, "ShardWaitMetrics return on timeout");
 				// FIXME: instead of using random chance, send wrong_shard_server when the call in from
 				// waitMetricsMultiple (requires additional information in the request)
 				if (deterministicRandom()->random01() < SERVER_KNOBS->WAIT_METRICS_WRONG_SHARD_CHANCE) {
@@ -8633,7 +9761,7 @@ ACTOR Future<Void> waitMetrics(StorageServerMetrics* self, WaitMetricsRequest re
 			}
 
 			if (!req.min.allLessOrEqual(metrics) || !metrics.allLessOrEqual(req.max)) {
-				TEST(true); // ShardWaitMetrics return case 2 (delayed)
+				CODE_PROBE(true, "ShardWaitMetrics return case 2 (delayed)");
 				req.reply.send(metrics);
 				break;
 			}
@@ -8657,7 +9785,7 @@ ACTOR Future<Void> waitMetrics(StorageServerMetrics* self, WaitMetricsRequest re
 	if (error.code() != error_code_success) {
 		if (error.code() != error_code_wrong_shard_server)
 			throw error;
-		TEST(true); // ShardWaitMetrics delayed wrong_shard_server()
+		CODE_PROBE(true, "ShardWaitMetrics delayed wrong_shard_server()");
 		req.reply.sendError(error);
 	}
 
@@ -8715,7 +9843,7 @@ ACTOR Future<Void> metricsCore(StorageServer* self, StorageServerInterface ssi)
 		choose {
 			when(WaitMetricsRequest req = waitNext(ssi.waitMetrics.getFuture())) {
 				if (!self->isReadable(req.keys)) {
-					TEST(true); // waitMetrics immediate wrong_shard_server()
+					CODE_PROBE(true, "waitMetrics immediate wrong_shard_server()");
 					self->sendErrorWithPenalty(req.reply, wrong_shard_server(), self->getPenalty());
 				} else {
 					self->actors.add(
@@ -8724,7 +9852,7 @@ ACTOR Future<Void> metricsCore(StorageServer* self, StorageServerInterface ssi)
 			}
 			when(SplitMetricsRequest req = waitNext(ssi.splitMetrics.getFuture())) {
 				if (!self->isReadable(req.keys)) {
-					TEST(true); // splitMetrics immediate wrong_shard_server()
+					CODE_PROBE(true, "splitMetrics immediate wrong_shard_server()");
 					self->sendErrorWithPenalty(req.reply, wrong_shard_server(), self->getPenalty());
 				} else {
 					self->metrics.splitMetrics(req);
@@ -8737,7 +9865,7 @@ ACTOR Future<Void> metricsCore(StorageServer* self, StorageServerInterface ssi)
 			}
 			when(ReadHotSubRangeRequest req = waitNext(ssi.getReadHotRanges.getFuture())) {
 				if (!self->isReadable(req.keys)) {
-					TEST(true); // readHotSubRanges immediate wrong_shard_server()
+					CODE_PROBE(true, "readHotSubRanges immediate wrong_shard_server()");
 					self->sendErrorWithPenalty(req.reply, wrong_shard_server(), self->getPenalty());
 				} else {
 					self->metrics.getReadHotRanges(req);
@@ -8745,7 +9873,7 @@ ACTOR Future<Void> metricsCore(StorageServer* self, StorageServerInterface ssi)
 			}
 			when(SplitRangeRequest req = waitNext(ssi.getRangeSplitPoints.getFuture())) {
 				if (!self->isReadable(req.keys)) {
-					TEST(true); // getSplitPoints immediate wrong_shard_server()
+					CODE_PROBE(true, "getSplitPoints immediate wrong_shard_server()");
 					self->sendErrorWithPenalty(req.reply, wrong_shard_server(), self->getPenalty());
 				} else {
 					self->getSplitPoints(req);
@@ -8777,6 +9905,8 @@ ACTOR Future<Void> checkBehind(StorageServer* self) {
 		state Transaction tr(self->cx);
 		loop {
 			try {
+				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				Version readVersion = wait(tr.getRawReadVersion());
 				if (readVersion > self->version.get() + SERVER_KNOBS->BEHIND_CHECK_VERSIONS) {
 					behindCount++;
@@ -8798,9 +9928,9 @@ ACTOR Future<Void> serveGetValueRequests(StorageServer* self, FutureStream<GetVa
 		GetValueRequest req = waitNext(getValue);
 		// Warning: This code is executed at extremely high priority (TaskPriority::LoadBalancedEndpoint), so
 		// downgrade before doing real work
-		if (req.debugID.present())
+		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("GetValueDebug",
-			                      req.debugID.get().first(),
+			                      req.options.get().debugID.get().first(),
 			                      "storageServer.received"); //.detail("TaskID", g_network->getCurrentTask());
 
 		if (SHORT_CIRCUT_ACTUAL_STORAGE && normalKeys.contains(req.key))
@@ -8885,6 +10015,7 @@ ACTOR Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream
 		state Reference<ServerWatchMetadata> metadata = self->getWatchMetadata(req.key.contents());
 		state Span span("SS:serveWatchValueRequestsImpl"_loc, req.spanContext);
 		getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.traceID;
+		state ReadOptions options;
 
 		// case 1: no watch set for the current key
 		if (!metadata.isValid()) {
@@ -8916,22 +10047,19 @@ ACTOR Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream
 		}
 		// case 4: version in the map is higher so immediately trigger watch
 		else if (req.version < metadata->version) {
-			TEST(true); // watch version in map is higher so trigger watch (case 4)
+			CODE_PROBE(true, "watch version in map is higher so trigger watch (case 4)");
 			req.reply.send(WatchValueReply{ metadata->version });
 		}
 		// case 5: watch value differs but their versions are the same (rare case) so check with the SS
 		else {
-			TEST(true); // watch version in the map is the same but value is different (case 5)
+			CODE_PROBE(true, "watch version in the map is the same but value is different (case 5)");
 			loop {
 				try {
 					state Version latest = self->version.get();
-					GetValueRequest getReq(span.context,
-					                       TenantInfo(),
-					                       metadata->key,
-					                       latest,
-					                       metadata->tags,
-					                       metadata->debugID,
-					                       VersionVector());
+					options.debugID = metadata->debugID;
+
+					GetValueRequest getReq(
+					    span.context, TenantInfo(), metadata->key, latest, metadata->tags, options, VersionVector());
 					state Future<Void> getValue = getValueQ(self, getReq);
 					GetValueReply reply = wait(getReq.reply.getFuture());
 					metadata = self->getWatchMetadata(req.key.contents());
@@ -8961,7 +10089,7 @@ ACTOR Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream
 						self->sendErrorWithPenalty(req.reply, e, self->getPenalty());
 						break;
 					}
-					TEST(true); // Reading a watched key failed with transaction_too_old case 5
+					CODE_PROBE(true, "Reading a watched key failed with transaction_too_old case 5");
 				}
 			}
 		}
@@ -9101,7 +10229,7 @@ ACTOR Future<Void> storageServerCore(StorageServer* self, StorageServerInterface
 				}
 			}
 			when(wait(dbInfoChange)) {
-				TEST(self->logSystem); // shardServer dbInfo changed
+				CODE_PROBE(self->logSystem, "shardServer dbInfo changed");
 				dbInfoChange = self->db->onChange();
 				if (self->db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
 					self->logSystem = ILogSystem::fromServerDBInfo(self->thisServerID, self->db->get());
@@ -9169,6 +10297,35 @@ ACTOR Future<Void> storageServerCore(StorageServer* self, StorageServerInterface
 			when(FetchCheckpointRequest req = waitNext(ssi.fetchCheckpoint.getFuture())) {
 				self->actors.add(fetchCheckpointQ(self, req));
 			}
+			when(UpdateCommitCostRequest req = waitNext(ssi.updateCommitCostRequest.getFuture())) {
+				// Ratekeeper might change with a new ID. In this case, always accept the data.
+				if (req.ratekeeperID != self->busiestWriteTagContext.ratekeeperID) {
+					TraceEvent("RatekeeperIDChange")
+					    .detail("OldID", self->busiestWriteTagContext.ratekeeperID)
+					    .detail("OldLastUpdateTime", self->busiestWriteTagContext.lastUpdateTime)
+					    .detail("NewID", req.ratekeeperID)
+					    .detail("LastUpdateTime", req.postTime);
+					self->busiestWriteTagContext.ratekeeperID = req.ratekeeperID;
+					self->busiestWriteTagContext.lastUpdateTime = -1;
+				}
+				// In case we received an old request/duplicate request, due to, e.g. network problem
+				ASSERT(req.postTime > 0);
+				if (req.postTime < self->busiestWriteTagContext.lastUpdateTime) {
+					continue;
+				}
+
+				self->busiestWriteTagContext.lastUpdateTime = req.postTime;
+				TraceEvent("BusiestWriteTag", self->thisServerID)
+				    .detail("Elapsed", req.elapsed)
+				    .detail("Tag", printable(req.busiestTag))
+				    .detail("TagOps", req.opsSum)
+				    .detail("TagCost", req.costSum)
+				    .detail("TotalCost", req.totalWriteCosts)
+				    .detail("Reported", req.reported)
+				    .trackLatest(self->busiestWriteTagContext.busiestWriteTagTrackingKey);
+
+				req.reply.send(Void());
+			}
 			when(FetchCheckpointKeyValuesRequest req = waitNext(ssi.fetchCheckpointKeyValues.getFuture())) {
 				self->actors.add(fetchCheckpointKeyValuesQ(self, req));
 			}
@@ -9215,7 +10372,7 @@ ACTOR Future<Void> memoryStoreRecover(IKeyValueStore* store, Reference<IClusterC
 	}
 
 	// create a temp client connect to DB
-	Database cx = Database::createDatabase(connRecord, Database::API_VERSION_LATEST);
+	Database cx = Database::createDatabase(connRecord, ApiVersion::LATEST_VERSION);
 
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
 	state int noCanRemoveCount = 0;
@@ -9226,8 +10383,9 @@ ACTOR Future<Void> memoryStoreRecover(IKeyValueStore* store, Reference<IClusterC
 
 			state bool canRemove = wait(canRemoveStorageServer(tr, id));
 			if (!canRemove) {
-				TEST(true); // it's possible that the caller had a transaction in flight that assigned keys to the
-				            // server. Wait for it to reverse its mistake.
+				CODE_PROBE(true,
+				           "it's possible that the caller had a transaction in flight that assigned keys to the "
+				           "server. Wait for it to reverse its mistake.");
 				wait(delayJittered(SERVER_KNOBS->REMOVE_RETRY_DELAY, TaskPriority::UpdateStorage));
 				tr->reset();
 				TraceEvent("RemoveStorageServerRetrying")
@@ -9251,17 +10409,22 @@ ACTOR Future<Void> initTenantMap(StorageServer* self) {
 	loop {
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			state Version version = wait(tr->getReadVersion());
 			// This limits the number of tenants, but eventually we shouldn't need to do this at all
 			// when SSs store only the local tenants
-			RangeResult entries = wait(tr->getRange(tenantMapKeys, CLIENT_KNOBS->TOO_MANY));
+			KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> entries =
+			    wait(TenantMetadata::tenantMap().getRange(
+			        tr, Optional<TenantName>(), Optional<TenantName>(), CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1));
+			ASSERT(entries.results.size() <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER && !entries.more);
 
 			TraceEvent("InitTenantMap", self->thisServerID)
 			    .detail("Version", version)
-			    .detail("NumTenants", entries.size());
+			    .detail("NumTenants", entries.results.size());
 
-			for (auto kv : entries) {
-				self->insertTenant(kv.key.removePrefix(tenantMapPrefix), kv.value, version, false);
+			for (auto entry : entries.results) {
+				self->insertTenant(entry.first, entry.second, version);
 			}
 			break;
 		} catch (Error& e) {
@@ -9291,6 +10454,7 @@ ACTOR Future<Void> replaceInterface(StorageServer* self, StorageServerInterface
 				try {
 					tr.reset();
 					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 					tr.setVersion(rep.version);
 
 					tr.addReadConflictRange(singleKeyRange(serverListKeyFor(ssi.id())));
@@ -9376,11 +10540,12 @@ ACTOR Future<Void> replaceTSSInterface(StorageServer* self, StorageServerInterfa
 			tr->reset();
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			Optional<Value> pairTagValue = wait(tr->get(serverTagKeyFor(self->tssPairID.get())));
 
 			if (!pairTagValue.present()) {
-				TEST(true); // Race where tss was down, pair was removed, tss starts back up
+				CODE_PROBE(true, "Race where tss was down, pair was removed, tss starts back up");
 				TraceEvent("StorageServerWorkerRemoved", self->thisServerID).detail("Reason", "TssPairMissing");
 				throw worker_removed();
 			}
@@ -9440,8 +10605,10 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
                                  Version tssSeedVersion,
                                  ReplyPromise<InitializeStorageReply> recruitReply,
                                  Reference<AsyncVar<ServerDBInfo> const> db,
-                                 std::string folder) {
-	state StorageServer self(persistentData, db, ssi);
+                                 std::string folder,
+                                 Reference<IPageEncryptionKeyProvider> encryptionKeyProvider) {
+	state StorageServer self(persistentData, db, ssi, encryptionKeyProvider);
+	self.shardAware = SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && persistentData->shardAware();
 	state Future<Void> ssCore;
 	self.clusterId.send(clusterId);
 	self.initialClusterVersion = startVersion;
@@ -9478,7 +10645,8 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 			self.tag = seedTag;
 		}
 
-		self.storage.makeNewStorageServerDurable();
+		self.encryptionKeyProvider->setTenantPrefixIndex(self.tenantPrefixIndex);
+		self.storage.makeNewStorageServerDurable(self.shardAware);
 		wait(self.storage.commit());
 		++self.counters.kvCommits;
 
@@ -9528,8 +10696,9 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
                                  Reference<AsyncVar<ServerDBInfo> const> db,
                                  std::string folder,
                                  Promise<Void> recovered,
-                                 Reference<IClusterConnectionRecord> connRecord) {
-	state StorageServer self(persistentData, db, ssi);
+                                 Reference<IClusterConnectionRecord> connRecord,
+                                 Reference<IPageEncryptionKeyProvider> encryptionKeyProvider) {
+	state StorageServer self(persistentData, db, ssi, encryptionKeyProvider);
 	state Future<Void> ssCore;
 	self.folder = folder;
 
@@ -9556,6 +10725,13 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 				recovered.send(Void());
 			return Void();
 		}
+		// Pass a reference of tenantPrefixIndex to the storage engine to support per-tenant data encryption,
+		// after the tenant map is recovered in restoreDurableState. In case of a storage server reboot,
+		// it is possible that the storage engine is still holding a pre-reboot tenantPrefixIndex, and use that
+		// for its own recovery, before we set the tenantPrefixIndex here.
+		if (self.encryptionKeyProvider.isValid()) {
+			self.encryptionKeyProvider->setTenantPrefixIndex(self.tenantPrefixIndex);
+		}
 		TraceEvent("SSTimeRestoreDurableState", self.thisServerID).detail("TimeTaken", now() - start);
 
 		// if this is a tss storage file, use that as source of truth for this server being a tss instead of the
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 2c63378341..fb1974f819 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -26,6 +26,7 @@
 #include <toml.hpp>
 
 #include "flow/ActorCollection.h"
+#include "flow/DeterministicRandom.h"
 #include "fdbrpc/sim_validation.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/ClusterInterface.h"
@@ -49,7 +50,7 @@ WorkloadContext::WorkloadContext() {}
 
 WorkloadContext::WorkloadContext(const WorkloadContext& r)
   : options(r.options), clientId(r.clientId), clientCount(r.clientCount), sharedRandomNumber(r.sharedRandomNumber),
-    dbInfo(r.dbInfo), ccr(r.ccr) {}
+    dbInfo(r.dbInfo), ccr(r.ccr), defaultTenant(r.defaultTenant) {}
 
 WorkloadContext::~WorkloadContext() {}
 
@@ -153,7 +154,7 @@ Value getOption(VectorRef<KeyValueRef> options, Key key, Value defaultValue) {
 	for (int i = 0; i < options.size(); i++)
 		if (options[i].key == key) {
 			Value value = options[i].value;
-			options[i].value = LiteralStringRef("");
+			options[i].value = ""_sr;
 			return value;
 		}
 
@@ -165,7 +166,7 @@ int getOption(VectorRef<KeyValueRef> options, Key key, int defaultValue) {
 		if (options[i].key == key) {
 			int r;
 			if (sscanf(options[i].value.toString().c_str(), "%d", &r)) {
-				options[i].value = LiteralStringRef("");
+				options[i].value = ""_sr;
 				return r;
 			} else {
 				TraceEvent(SevError, "InvalidTestOption").detail("OptionName", key);
@@ -181,7 +182,7 @@ uint64_t getOption(VectorRef<KeyValueRef> options, Key key, uint64_t defaultValu
 		if (options[i].key == key) {
 			uint64_t r;
 			if (sscanf(options[i].value.toString().c_str(), "%" SCNd64, &r)) {
-				options[i].value = LiteralStringRef("");
+				options[i].value = ""_sr;
 				return r;
 			} else {
 				TraceEvent(SevError, "InvalidTestOption").detail("OptionName", key);
@@ -197,7 +198,7 @@ int64_t getOption(VectorRef<KeyValueRef> options, Key key, int64_t defaultValue)
 		if (options[i].key == key) {
 			int64_t r;
 			if (sscanf(options[i].value.toString().c_str(), "%" SCNd64, &r)) {
-				options[i].value = LiteralStringRef("");
+				options[i].value = ""_sr;
 				return r;
 			} else {
 				TraceEvent(SevError, "InvalidTestOption").detail("OptionName", key);
@@ -213,7 +214,7 @@ double getOption(VectorRef<KeyValueRef> options, Key key, double defaultValue) {
 		if (options[i].key == key) {
 			float r;
 			if (sscanf(options[i].value.toString().c_str(), "%f", &r)) {
-				options[i].value = LiteralStringRef("");
+				options[i].value = ""_sr;
 				return r;
 			}
 		}
@@ -222,10 +223,10 @@ double getOption(VectorRef<KeyValueRef> options, Key key, double defaultValue) {
 }
 
 bool getOption(VectorRef<KeyValueRef> options, Key key, bool defaultValue) {
-	Value p = getOption(options, key, defaultValue ? LiteralStringRef("true") : LiteralStringRef("false"));
-	if (p == LiteralStringRef("true"))
+	Value p = getOption(options, key, defaultValue ? "true"_sr : "false"_sr);
+	if (p == "true"_sr)
 		return true;
-	if (p == LiteralStringRef("false"))
+	if (p == "false"_sr)
 		return false;
 	ASSERT(false);
 	return false; // Assure that compiler is fine with the function
@@ -242,7 +243,7 @@ std::vector<std::string> getOption(VectorRef<KeyValueRef> options, Key key, std:
 					begin = c + 1;
 				}
 			v.push_back(options[i].value.substr(begin).toString());
-			options[i].value = LiteralStringRef("");
+			options[i].value = ""_sr;
 			return v;
 		}
 	return defaultValue;
@@ -259,7 +260,7 @@ bool hasOption(VectorRef<KeyValueRef> options, Key key) {
 
 // returns unconsumed options
 Standalone<VectorRef<KeyValueRef>> checkAllOptionsConsumed(VectorRef<KeyValueRef> options) {
-	static StringRef nothing = LiteralStringRef("");
+	static StringRef nothing = ""_sr;
 	Standalone<VectorRef<KeyValueRef>> unconsumed;
 	for (int i = 0; i < options.size(); i++)
 		if (!(options[i].value == nothing)) {
@@ -271,111 +272,197 @@ Standalone<VectorRef<KeyValueRef>> checkAllOptionsConsumed(VectorRef<KeyValueRef
 	return unconsumed;
 }
 
-struct CompoundWorkload : TestWorkload {
-	std::vector<Reference<TestWorkload>> workloads;
+CompoundWorkload::CompoundWorkload(WorkloadContext& wcx) : TestWorkload(wcx) {}
 
-	CompoundWorkload(WorkloadContext& wcx) : TestWorkload(wcx) {}
-	CompoundWorkload* add(Reference<TestWorkload>&& w) {
-		workloads.push_back(std::move(w));
-		return this;
-	}
+CompoundWorkload* CompoundWorkload::add(Reference<TestWorkload>&& w) {
+	workloads.push_back(std::move(w));
+	return this;
+}
 
-	std::string description() const override {
-		std::string d;
-		for (int w = 0; w < workloads.size(); w++)
-			d += workloads[w]->description() + (w == workloads.size() - 1 ? "" : ";");
-		return d;
+std::string CompoundWorkload::description() const {
+	std::vector<std::string> names;
+	names.reserve(workloads.size());
+	for (auto const& w : workloads) {
+		names.push_back(w->description());
 	}
-	Future<Void> setup(Database const& cx) override {
-		std::vector<Future<Void>> all;
-		all.reserve(workloads.size());
-		for (int w = 0; w < workloads.size(); w++)
-			all.push_back(workloads[w]->setup(cx));
-		return waitForAll(all);
+	return fmt::format("{}", fmt::join(std::move(names), ";"));
+}
+Future<Void> CompoundWorkload::setup(Database const& cx) {
+	std::vector<Future<Void>> all;
+	all.reserve(workloads.size());
+	for (int w = 0; w < workloads.size(); w++)
+		all.push_back(workloads[w]->setup(cx));
+	auto done = waitForAll(all);
+	if (failureInjection.empty()) {
+		return done;
 	}
-	Future<Void> start(Database const& cx) override {
-		std::vector<Future<Void>> all;
-		all.reserve(workloads.size());
-		auto wCount = std::make_shared<unsigned>(0);
-		for (int i = 0; i < workloads.size(); i++) {
-			std::string workloadName = workloads[i]->description();
-			++(*wCount);
-			TraceEvent("WorkloadRunStatus")
-			    .detail("Name", workloadName)
-			    .detail("Count", *wCount)
-			    .detail("Phase", "Start");
-			all.push_back(fmap(
-			    [workloadName, wCount](Void value) {
-				    --(*wCount);
-				    TraceEvent("WorkloadRunStatus")
-				        .detail("Name", workloadName)
-				        .detail("Remaining", *wCount)
-				        .detail("Phase", "End");
-				    return Void();
-			    },
-			    workloads[i]->start(cx)));
+	std::vector<Future<Void>> res;
+	res.reserve(failureInjection.size());
+	for (auto& f : failureInjection) {
+		res.push_back(f->setupInjectionWorkload(cx, done));
+	}
+	return waitForAll(res);
+}
+
+Future<Void> CompoundWorkload::start(Database const& cx) {
+	std::vector<Future<Void>> all;
+	all.reserve(workloads.size() + failureInjection.size());
+	auto wCount = std::make_shared<unsigned>(0);
+	auto startWorkload = [&](TestWorkload& workload) -> Future<Void> {
+		auto workloadName = workload.description();
+		++(*wCount);
+		TraceEvent("WorkloadRunStatus").detail("Name", workloadName).detail("Count", *wCount).detail("Phase", "Start");
+		return fmap(
+		    [workloadName, wCount](Void value) {
+			    --(*wCount);
+			    TraceEvent("WorkloadRunStatus")
+			        .detail("Name", workloadName)
+			        .detail("Remaining", *wCount)
+			        .detail("Phase", "End");
+			    return Void();
+		    },
+		    workload.start(cx));
+	};
+	for (auto& workload : workloads) {
+		all.push_back(startWorkload(*workload));
+	}
+	for (auto& workload : failureInjection) {
+		all.push_back(startWorkload(*workload));
+	}
+	return waitForAll(all);
+}
+
+Future<bool> CompoundWorkload::check(Database const& cx) {
+	std::vector<Future<bool>> all;
+	all.reserve(workloads.size() + failureInjection.size());
+	auto wCount = std::make_shared<unsigned>(0);
+	auto starter = [&](TestWorkload& workload) -> Future<bool> {
+		++(*wCount);
+		std::string workloadName = workload.description();
+		TraceEvent("WorkloadCheckStatus")
+		    .detail("Name", workloadName)
+		    .detail("Count", *wCount)
+		    .detail("Phase", "Start");
+		return fmap(
+		    [workloadName, wCount](bool ret) {
+			    --(*wCount);
+			    TraceEvent("WorkloadCheckStatus")
+			        .detail("Name", workloadName)
+			        .detail("Remaining", *wCount)
+			        .detail("Phase", "End");
+			    return ret;
+		    },
+		    workload.check(cx));
+	};
+	for (auto& workload : workloads) {
+		all.push_back(starter(*workload));
+	}
+	for (auto& workload : failureInjection) {
+		all.push_back(starter(*workload));
+	}
+	return allTrue(all);
+}
+
+ACTOR Future<std::vector<PerfMetric>> getMetricsCompoundWorkload(CompoundWorkload* self) {
+	state std::vector<Future<std::vector<PerfMetric>>> results;
+	for (int w = 0; w < self->workloads.size(); w++) {
+		std::vector<PerfMetric> p;
+		results.push_back(self->workloads[w]->getMetrics());
+	}
+	wait(waitForAll(results));
+	std::vector<PerfMetric> res;
+	for (int i = 0; i < results.size(); ++i) {
+		auto const& p = results[i].get();
+		for (auto const& m : p) {
+			res.push_back(m.withPrefix(self->workloads[i]->description() + "."));
 		}
-		return waitForAll(all);
-	}
-	Future<bool> check(Database const& cx) override {
-		std::vector<Future<bool>> all;
-		all.reserve(workloads.size());
-		auto wCount = std::make_shared<unsigned>(0);
-		for (int i = 0; i < workloads.size(); i++) {
-			++(*wCount);
-			std::string workloadName = workloads[i]->description();
-			TraceEvent("WorkloadCheckStatus")
-			    .detail("Name", workloadName)
-			    .detail("Count", *wCount)
-			    .detail("Phase", "Start");
-			all.push_back(fmap(
-			    [workloadName, wCount](bool ret) {
-				    --(*wCount);
-				    TraceEvent("WorkloadCheckStatus")
-				        .detail("Name", workloadName)
-				        .detail("Remaining", *wCount)
-				        .detail("Phase", "End");
-				    return true;
-			    },
-			    workloads[i]->check(cx)));
-		}
-		return allTrue(all);
 	}
+	return res;
+}
 
-	ACTOR static Future<std::vector<PerfMetric>> getMetrics(CompoundWorkload* self) {
-		state std::vector<Future<std::vector<PerfMetric>>> results;
-		for (int w = 0; w < self->workloads.size(); w++) {
-			std::vector<PerfMetric> p;
-			results.push_back(self->workloads[w]->getMetrics());
-		}
-		wait(waitForAll(results));
-		std::vector<PerfMetric> res;
-		for (int i = 0; i < results.size(); ++i) {
-			auto const& p = results[i].get();
-			for (auto const& m : p) {
-				res.push_back(m.withPrefix(self->workloads[i]->description() + "."));
-			}
-		}
-		return res;
+void CompoundWorkload::addFailureInjection(WorkloadRequest& work) {
+	if (!work.runFailureWorkloads) {
+		return;
 	}
-
-	Future<std::vector<PerfMetric>> getMetrics() override { return getMetrics(this); }
-	double getCheckTimeout() const override {
-		double m = 0;
-		for (int w = 0; w < workloads.size(); w++)
-			m = std::max(workloads[w]->getCheckTimeout(), m);
-		return m;
+	// Some workloads won't work with some failure injection workloads
+	std::set<std::string> disabledWorkloads;
+	for (auto const& w : workloads) {
+		w->disableFailureInjectionWorkloads(disabledWorkloads);
 	}
+	if (disabledWorkloads.count("all") > 0) {
+		return;
+	}
+	auto& factories = IFailureInjectorFactory::factories();
+	DeterministicRandom random(sharedRandomNumber);
+	for (auto& factory : factories) {
+		auto workload = factory->create(*this);
+		if (disabledWorkloads.count(workload->description()) > 0) {
+			continue;
+		}
+		while (shouldInjectFailure(random, work, workload)) {
+			workload->initFailureInjectionMode(random);
+			failureInjection.push_back(workload);
+			workload = factory->create(*this);
+		}
+	}
+}
 
-	void getMetrics(std::vector<PerfMetric>&) override { ASSERT(false); }
-};
+bool CompoundWorkload::shouldInjectFailure(DeterministicRandom& random,
+                                           const WorkloadRequest& work,
+                                           Reference<FailureInjectionWorkload> failure) const {
+	auto desc = failure->description();
+	unsigned alreadyAdded =
+	    std::count_if(workloads.begin(), workloads.end(), [&desc](auto const& w) { return w->description() == desc; });
+	alreadyAdded += std::count_if(
+	    failureInjection.begin(), failureInjection.end(), [&desc](auto const& w) { return w->description() == desc; });
+	return failure->shouldInject(random, work, alreadyAdded);
+}
+
+Future<std::vector<PerfMetric>> CompoundWorkload::getMetrics() {
+	return getMetricsCompoundWorkload(this);
+}
+
+double CompoundWorkload::getCheckTimeout() const {
+	double m = 0;
+	for (int w = 0; w < workloads.size(); w++)
+		m = std::max(workloads[w]->getCheckTimeout(), m);
+	return m;
+}
+
+void CompoundWorkload::getMetrics(std::vector<PerfMetric>&) {
+	ASSERT(false);
+}
+
+void TestWorkload::disableFailureInjectionWorkloads(std::set<std::string>& out) const {}
+
+FailureInjectionWorkload::FailureInjectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+
+void FailureInjectionWorkload::initFailureInjectionMode(DeterministicRandom& random) {}
+
+bool FailureInjectionWorkload::shouldInject(DeterministicRandom& random,
+                                            const WorkloadRequest& work,
+                                            const unsigned alreadyAdded) const {
+	return alreadyAdded < 3 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
+}
+
+Future<Void> FailureInjectionWorkload::setupInjectionWorkload(const Database& cx, Future<Void> done) {
+	return holdWhile(this->setup(cx), done);
+}
+
+Future<Void> FailureInjectionWorkload::startInjectionWorkload(const Database& cx, Future<Void> done) {
+	return holdWhile(this->start(cx), done);
+}
+
+Future<bool> FailureInjectionWorkload::checkInjectionWorkload(const Database& cx, Future<bool> done) {
+	return holdWhile(this->check(cx), done);
+}
 
 ACTOR Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
                                                        Reference<IClusterConnectionRecord> ccr,
                                                        VectorRef<KeyValueRef> options,
                                                        Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
 	state Reference<TestWorkload> workload;
-	state Value testName = getOption(options, LiteralStringRef("testName"), LiteralStringRef("no-test-specified"));
+	state Value testName = getOption(options, "testName"_sr, "no-test-specified"_sr);
 	WorkloadContext wcx;
 	wcx.clientId = work.clientId;
 	wcx.clientCount = work.clientCount;
@@ -383,9 +470,12 @@ ACTOR Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
 	wcx.dbInfo = dbInfo;
 	wcx.options = options;
 	wcx.sharedRandomNumber = work.sharedRandomNumber;
+	wcx.defaultTenant = work.defaultTenant.castTo<TenantName>();
 
 	workload = IWorkloadFactory::create(testName.toString(), wcx);
-	wait(workload->initialized());
+	if (workload) {
+		wait(workload->initialized());
+	}
 
 	auto unconsumedOptions = checkAllOptionsConsumed(workload ? workload->options : VectorRef<KeyValueRef>());
 	if (!workload || unconsumedOptions.size()) {
@@ -420,14 +510,13 @@ ACTOR Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
 		fprintf(stderr, "ERROR: No options were provided for workload.\n");
 		throw test_specification_invalid();
 	}
-	if (work.options.size() == 1) {
-		Reference<TestWorkload> res = wait(getWorkloadIface(work, ccr, work.options[0], dbInfo));
-		return res;
-	}
 
 	wcx.clientId = work.clientId;
 	wcx.clientCount = work.clientCount;
 	wcx.sharedRandomNumber = work.sharedRandomNumber;
+	wcx.ccr = ccr;
+	wcx.dbInfo = dbInfo;
+	wcx.defaultTenant = work.defaultTenant.castTo<TenantName>();
 	// FIXME: Other stuff not filled in; why isn't this constructed here and passed down to the other
 	// getWorkloadIface()?
 	for (int i = 0; i < work.options.size(); i++) {
@@ -438,6 +527,7 @@ ACTOR Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
 	for (int i = 0; i < work.options.size(); i++) {
 		compound->add(ifaces[i].getValue());
 	}
+	compound->addFailureInjection(work);
 	return compound;
 }
 
@@ -449,7 +539,7 @@ void printSimulatedTopology() {
 	if (!g_network->isSimulated()) {
 		return;
 	}
-	auto processes = g_simulator.getAllProcesses();
+	auto processes = g_simulator->getAllProcesses();
 	std::sort(processes.begin(), processes.end(), [](ISimulator::ProcessInfo* lhs, ISimulator::ProcessInfo* rhs) {
 		auto l = lhs->locality;
 		auto r = rhs->locality;
@@ -682,7 +772,7 @@ ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
 		startRole(Role::TESTER, workIface.id(), UID(), details);
 
 		if (work.useDatabase) {
-			cx = Database::createDatabase(ccr, -1, IsInternal::True, locality);
+			cx = Database::createDatabase(ccr, ApiVersion::LATEST_VERSION, IsInternal::True, locality);
 			cx->defaultTenant = work.defaultTenant.castTo<TenantName>();
 			wait(delay(1.0));
 		}
@@ -708,6 +798,7 @@ ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
 
 		endRole(Role::TESTER, workIface.id(), "Complete");
 	} catch (Error& e) {
+		TraceEvent(SevDebug, "TesterWorkloadFailed").errorUnsuppressed(e);
 		if (!replied) {
 			if (e.code() == error_code_test_specification_invalid)
 				work.reply.sendError(e);
@@ -733,7 +824,7 @@ ACTOR Future<Void> testerServerCore(TesterInterface interf,
 	state PromiseStream<Future<Void>> addWorkload;
 	state Future<Void> workerFatalError = actorCollection(addWorkload.getFuture());
 
-	TraceEvent("StartingTesterServerCore", interf.id());
+	TraceEvent("StartingTesterServerCore", interf.id()).log();
 	loop choose {
 		when(wait(workerFatalError)) {}
 		when(WorkloadRequest work = waitNext(interf.recruitments.getFuture())) {
@@ -880,6 +971,7 @@ ACTOR Future<DistributedTestResults> runWorkload(Database cx,
 		WorkloadRequest req;
 		req.title = spec.title;
 		req.useDatabase = spec.useDB;
+		req.runFailureWorkloads = spec.runFailureWorkloads;
 		req.timeout = spec.timeout;
 		req.databasePingDelay = spec.useDB ? spec.databasePingDelay : 0.0;
 		req.options = spec.options;
@@ -972,10 +1064,10 @@ ACTOR Future<DistributedTestResults> runWorkload(Database cx,
 ACTOR Future<Void> changeConfiguration(Database cx, std::vector<TesterInterface> testers, StringRef configMode) {
 	state TestSpec spec;
 	Standalone<VectorRef<KeyValueRef>> options;
-	spec.title = LiteralStringRef("ChangeConfig");
-	options.push_back_deep(options.arena(),
-	                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ChangeConfig")));
-	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("configMode"), configMode));
+	spec.title = "ChangeConfig"_sr;
+	spec.runFailureWorkloads = false;
+	options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "ChangeConfig"_sr));
+	options.push_back_deep(options.arena(), KeyValueRef("configMode"_sr, configMode));
 	spec.options.push_back_deep(spec.options.arena(), options);
 
 	DistributedTestResults testResults = wait(runWorkload(cx, testers, spec, Optional<TenantName>()));
@@ -998,37 +1090,37 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	state double connectionFailures;
 	if (g_network->isSimulated()) {
 		// NOTE: the value will be reset after consistency check
-		connectionFailures = g_simulator.connectionFailuresDisableDuration;
-		g_simulator.connectionFailuresDisableDuration = 1e6;
-		g_simulator.speedUpSimulation = true;
+		connectionFailures = g_simulator->connectionFailuresDisableDuration;
+		g_simulator->connectionFailuresDisableDuration = 1e6;
+		g_simulator->speedUpSimulation = true;
 	}
 
 	Standalone<VectorRef<KeyValueRef>> options;
-	StringRef performQuiescent = LiteralStringRef("false");
-	StringRef performCacheCheck = LiteralStringRef("false");
-	StringRef performTSSCheck = LiteralStringRef("false");
+	StringRef performQuiescent = "false"_sr;
+	StringRef performCacheCheck = "false"_sr;
+	StringRef performTSSCheck = "false"_sr;
 	if (doQuiescentCheck) {
-		performQuiescent = LiteralStringRef("true");
+		performQuiescent = "true"_sr;
 		spec.restorePerpetualWiggleSetting = false;
 	}
 	if (doCacheCheck) {
-		performCacheCheck = LiteralStringRef("true");
+		performCacheCheck = "true"_sr;
 	}
 	if (doTSSCheck) {
-		performTSSCheck = LiteralStringRef("true");
+		performTSSCheck = "true"_sr;
 	}
-	spec.title = LiteralStringRef("ConsistencyCheck");
+	spec.title = "ConsistencyCheck"_sr;
 	spec.databasePingDelay = databasePingDelay;
+	spec.runFailureWorkloads = false;
 	spec.timeout = 32000;
-	options.push_back_deep(options.arena(),
-	                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
-	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("performQuiescentChecks"), performQuiescent));
-	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("performCacheCheck"), performCacheCheck));
-	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("performTSSCheck"), performTSSCheck));
-	options.push_back_deep(options.arena(),
-	                       KeyValueRef(LiteralStringRef("quiescentWaitTimeout"),
-	                                   ValueRef(options.arena(), format("%f", quiescentWaitTimeout))));
-	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
+	options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "ConsistencyCheck"_sr));
+	options.push_back_deep(options.arena(), KeyValueRef("performQuiescentChecks"_sr, performQuiescent));
+	options.push_back_deep(options.arena(), KeyValueRef("performCacheCheck"_sr, performCacheCheck));
+	options.push_back_deep(options.arena(), KeyValueRef("performTSSCheck"_sr, performTSSCheck));
+	options.push_back_deep(
+	    options.arena(),
+	    KeyValueRef("quiescentWaitTimeout"_sr, ValueRef(options.arena(), format("%f", quiescentWaitTimeout))));
+	options.push_back_deep(options.arena(), KeyValueRef("distributed"_sr, "false"_sr));
 	spec.options.push_back_deep(spec.options.arena(), options);
 
 	state double start = now();
@@ -1037,13 +1129,12 @@ ACTOR Future<Void> checkConsistency(Database cx,
 		DistributedTestResults testResults = wait(runWorkload(cx, testers, spec, Optional<TenantName>()));
 		if (testResults.ok() || lastRun) {
 			if (g_network->isSimulated()) {
-				g_simulator.connectionFailuresDisableDuration = connectionFailures;
+				g_simulator->connectionFailuresDisableDuration = connectionFailures;
 			}
 			return Void();
 		}
 		if (now() - start > softTimeLimit) {
-			spec.options[0].push_back_deep(spec.options.arena(),
-			                               KeyValueRef(LiteralStringRef("failureIsError"), LiteralStringRef("true")));
+			spec.options[0].push_back_deep(spec.options.arena(), KeyValueRef("failureIsError"_sr, "true"_sr));
 			lastRun = true;
 		}
 
@@ -1143,7 +1234,13 @@ ACTOR Future<bool> runTest(Database cx,
 std::map<std::string, std::function<void(const std::string&)>> testSpecGlobalKeys = {
 	// These are read by SimulatedCluster and used before testers exist.  Thus, they must
 	// be recognized and accepted, but there's no point in placing them into a testSpec.
-	{ "extraDB", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDB", ""); } },
+	// testClass and testPriority are only used for TestHarness, we'll ignore those here
+	{ "testClass", [](std::string const&) {} },
+	{ "testPriority", [](std::string const&) {} },
+	{ "extraDatabaseMode",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseMode", ""); } },
+	{ "extraDatabaseCount",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseCount", ""); } },
 	{ "configureLocked",
 	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedConfigureLocked", ""); } },
 	{ "minimumReplication",
@@ -1280,7 +1377,7 @@ std::map<std::string, std::function<void(const std::string& value, TestSpec* spe
 	      ASSERT(connectionFailuresDisableDuration >= 0);
 	      spec->simConnectionFailuresDisableDuration = connectionFailuresDisableDuration;
 	      if (g_network->isSimulated())
-		      g_simulator.connectionFailuresDisableDuration = spec->simConnectionFailuresDisableDuration;
+		      g_simulator->connectionFailuresDisableDuration = spec->simConnectionFailuresDisableDuration;
 	      TraceEvent("TestParserTest")
 	          .detail("ParsedSimConnectionFailuresDisableDuration", spec->simConnectionFailuresDisableDuration);
 	  } },
@@ -1308,6 +1405,8 @@ std::map<std::string, std::function<void(const std::string& value, TestSpec* spe
 	      if (value == "false")
 		      spec->restorePerpetualWiggleSetting = false;
 	  } },
+	{ "runFailureWorkloads",
+	  [](const std::string& value, TestSpec* spec) { spec->runFailureWorkloads = (value == "true"); } },
 };
 
 std::vector<TestSpec> readTests(std::ifstream& ifs) {
@@ -1409,6 +1508,9 @@ KnobKeyValuePairs getOverriddenKnobKeyValues(const toml::value& context) {
 				if (std::get_if<NoKnobFound>(&parsedValue)) {
 					parsedValue = SERVER_KNOBS->parseKnobValue(key, value);
 				}
+				if (std::get_if<NoKnobFound>(&parsedValue)) {
+					parsedValue = FLOW_KNOBS->parseKnobValue(key, value);
+				}
 				if (std::get_if<NoKnobFound>(&parsedValue)) {
 					TraceEvent(SevError, "TestSpecUnrecognizedKnob")
 					    .detail("KnobName", key)
@@ -1529,6 +1631,24 @@ ACTOR Future<Void> monitorServerDBInfo(Reference<AsyncVar<Optional<ClusterContro
 	}
 }
 
+ACTOR Future<Void> initializeSimConfig(Database db) {
+	state Transaction tr(db);
+	ASSERT(g_network->isSimulated());
+	loop {
+		try {
+			DatabaseConfiguration dbConfig = wait(getDatabaseConfiguration(&tr));
+			g_simulator->storagePolicy = dbConfig.storagePolicy;
+			g_simulator->tLogPolicy = dbConfig.tLogPolicy;
+			g_simulator->tLogWriteAntiQuorum = dbConfig.tLogWriteAntiQuorum;
+			g_simulator->remoteTLogPolicy = dbConfig.getRemoteTLogPolicy();
+			g_simulator->usableRegions = dbConfig.usableRegions;
+			return Void();
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
 /**
  * \brief Test orchestrator: sends test specification to testers in the right order and collects the results.
  *
@@ -1554,7 +1674,8 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
                             StringRef startingConfiguration,
                             LocalityData locality,
                             Optional<TenantName> defaultTenant,
-                            Standalone<VectorRef<TenantNameRef>> tenantsToCreate) {
+                            Standalone<VectorRef<TenantNameRef>> tenantsToCreate,
+                            bool restartingTest) {
 	state Database cx;
 	state Reference<AsyncVar<ServerDBInfo>> dbInfo(new AsyncVar<ServerDBInfo>);
 	state Future<Void> ccMonitor = monitorServerDBInfo(cc, LocalityData(), dbInfo); // FIXME: locality
@@ -1588,12 +1709,12 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 		if (iter->simDrAgents != ISimulator::BackupAgentType::NoBackupAgents) {
 			simDrAgents = iter->simDrAgents;
 		}
-		enableDD = enableDD || getOption(iter->options[0], LiteralStringRef("enableDD"), false);
+		enableDD = enableDD || getOption(iter->options[0], "enableDD"_sr, false);
 	}
 
 	if (g_network->isSimulated()) {
-		g_simulator.backupAgents = simBackupAgents;
-		g_simulator.drAgents = simDrAgents;
+		g_simulator->backupAgents = simBackupAgents;
+		g_simulator->drAgents = simDrAgents;
 	}
 
 	// turn off the database ping functionality if the suite of tests are not going to be using the database
@@ -1606,9 +1727,15 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 	}
 
 	state Future<Void> disabler = disableConnectionFailuresAfter(FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, "Tester");
+	state Future<Void> repairDataCenter;
+	if (useDB) {
+		Future<Void> reconfigure = reconfigureAfter(cx, FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, dbInfo, "Tester");
+		repairDataCenter = reconfigure;
+	}
 
 	// Change the configuration (and/or create the database) if necessary
 	printf("startingConfiguration:%s start\n", startingConfiguration.toString().c_str());
+	fmt::print("useDB: {}\n", useDB);
 	printSimulatedTopology();
 	if (useDB && startingConfiguration != StringRef()) {
 		try {
@@ -1630,14 +1757,87 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 		}
 	}
 
+	// Read cluster configuration
+	if (useDB) {
+		state Transaction tr = Transaction(cx);
+		state DatabaseConfiguration configuration;
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+				RangeResult results = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY));
+				ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
+				configuration.fromKeyValues((VectorRef<KeyValueRef>)results);
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		g_simulator->storagePolicy = configuration.storagePolicy;
+		g_simulator->tLogPolicy = configuration.tLogPolicy;
+		g_simulator->tLogWriteAntiQuorum = configuration.tLogWriteAntiQuorum;
+		g_simulator->remoteTLogPolicy = configuration.remoteTLogPolicy;
+		g_simulator->usableRegions = configuration.usableRegions;
+		if (configuration.regions.size() > 0) {
+			g_simulator->primaryDcId = configuration.regions[0].dcId;
+			g_simulator->hasSatelliteReplication = configuration.regions[0].satelliteTLogReplicationFactor > 0;
+			if (configuration.regions[0].satelliteTLogUsableDcsFallback > 0) {
+				g_simulator->satelliteTLogPolicyFallback = configuration.regions[0].satelliteTLogPolicyFallback;
+				g_simulator->satelliteTLogWriteAntiQuorumFallback =
+				    configuration.regions[0].satelliteTLogWriteAntiQuorumFallback;
+			} else {
+				g_simulator->satelliteTLogPolicyFallback = configuration.regions[0].satelliteTLogPolicy;
+				g_simulator->satelliteTLogWriteAntiQuorumFallback =
+				    configuration.regions[0].satelliteTLogWriteAntiQuorum;
+			}
+			g_simulator->satelliteTLogPolicy = configuration.regions[0].satelliteTLogPolicy;
+			g_simulator->satelliteTLogWriteAntiQuorum = configuration.regions[0].satelliteTLogWriteAntiQuorum;
+
+			for (auto s : configuration.regions[0].satellites) {
+				g_simulator->primarySatelliteDcIds.push_back(s.dcId);
+			}
+		} else {
+			g_simulator->hasSatelliteReplication = false;
+			g_simulator->satelliteTLogWriteAntiQuorum = 0;
+		}
+
+		if (configuration.regions.size() == 2) {
+			g_simulator->remoteDcId = configuration.regions[1].dcId;
+			ASSERT((!configuration.regions[0].satelliteTLogPolicy && !configuration.regions[1].satelliteTLogPolicy) ||
+			       configuration.regions[0].satelliteTLogPolicy->info() ==
+			           configuration.regions[1].satelliteTLogPolicy->info());
+
+			for (auto s : configuration.regions[1].satellites) {
+				g_simulator->remoteSatelliteDcIds.push_back(s.dcId);
+			}
+		}
+
+		if (restartingTest || g_simulator->usableRegions < 2 || !g_simulator->hasSatelliteReplication) {
+			g_simulator->allowLogSetKills = false;
+		}
+
+		ASSERT(g_simulator->storagePolicy && g_simulator->tLogPolicy);
+		ASSERT(!g_simulator->hasSatelliteReplication || g_simulator->satelliteTLogPolicy);
+	}
+
 	if (useDB) {
 		std::vector<Future<Void>> tenantFutures;
 		for (auto tenant : tenantsToCreate) {
-			TraceEvent("CreatingTenant").detail("Tenant", tenant);
-			tenantFutures.push_back(success(TenantAPI::createTenant(cx.getReference(), tenant)));
+			TenantMapEntry entry;
+			if (deterministicRandom()->coinflip()) {
+				entry.tenantGroup = "TestTenantGroup"_sr;
+			}
+			entry.encrypted = SERVER_KNOBS->ENABLE_ENCRYPTION;
+			TraceEvent("CreatingTenant").detail("Tenant", tenant).detail("TenantGroup", entry.tenantGroup);
+			tenantFutures.push_back(success(TenantAPI::createTenant(cx.getReference(), tenant, entry)));
 		}
 
 		wait(waitForAll(tenantFutures));
+		if (g_network->isSimulated()) {
+			wait(initializeSimConfig(cx));
+		}
 	}
 
 	if (useDB && waitForQuiescenceBegin) {
@@ -1657,6 +1857,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 		if (perpetualWiggleEnabled) { // restore the enabled perpetual storage wiggle setting
 			printf("Set perpetual_storage_wiggle=1 ...\n");
 			Version cVer = wait(setPerpetualStorageWiggle(cx, true, LockAware::True));
+			(void)cVer;
 			printf("Set perpetual_storage_wiggle=1 Done.\n");
 		}
 	}
@@ -1720,7 +1921,8 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
                             StringRef startingConfiguration,
                             LocalityData locality,
                             Optional<TenantName> defaultTenant,
-                            Standalone<VectorRef<TenantNameRef>> tenantsToCreate) {
+                            Standalone<VectorRef<TenantNameRef>> tenantsToCreate,
+                            bool restartingTest) {
 	state int flags = (at == TEST_ON_SERVERS ? 0 : GetWorkersRequest::TESTER_CLASS_ONLY) |
 	                  GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
 	state Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
@@ -1751,7 +1953,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 	for (int i = 0; i < workers.size(); i++)
 		ts.push_back(workers[i].interf.testerInterface);
 
-	wait(runTests(cc, ci, ts, tests, startingConfiguration, locality, defaultTenant, tenantsToCreate));
+	wait(runTests(cc, ci, ts, tests, startingConfiguration, locality, defaultTenant, tenantsToCreate, restartingTest));
 	return Void();
 }
 
@@ -1790,7 +1992,8 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
                             LocalityData locality,
                             UnitTestParameters testOptions,
                             Optional<TenantName> defaultTenant,
-                            Standalone<VectorRef<TenantNameRef>> tenantsToCreate) {
+                            Standalone<VectorRef<TenantNameRef>> tenantsToCreate,
+                            bool restartingTest) {
 	state TestSet testSet;
 	state std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup(nullptr);
 	auto cc = makeReference<AsyncVar<Optional<ClusterControllerFullInterface>>>();
@@ -1804,36 +2007,31 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 	if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK) {
 		TestSpec spec;
 		Standalone<VectorRef<KeyValueRef>> options;
-		spec.title = LiteralStringRef("ConsistencyCheck");
+		spec.title = "ConsistencyCheck"_sr;
+		spec.runFailureWorkloads = false;
 		spec.databasePingDelay = 0;
 		spec.timeout = 0;
 		spec.waitForQuiescenceBegin = false;
 		spec.waitForQuiescenceEnd = false;
 		std::string rateLimitMax = format("%d", CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX);
-		options.push_back_deep(options.arena(),
-		                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
-		options.push_back_deep(options.arena(),
-		                       KeyValueRef(LiteralStringRef("performQuiescentChecks"), LiteralStringRef("false")));
-		options.push_back_deep(options.arena(),
-		                       KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
-		options.push_back_deep(options.arena(),
-		                       KeyValueRef(LiteralStringRef("failureIsError"), LiteralStringRef("true")));
-		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("indefinite"), LiteralStringRef("true")));
-		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("rateLimitMax"), StringRef(rateLimitMax)));
-		options.push_back_deep(options.arena(),
-		                       KeyValueRef(LiteralStringRef("shuffleShards"), LiteralStringRef("true")));
+		options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "ConsistencyCheck"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("performQuiescentChecks"_sr, "false"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("distributed"_sr, "false"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("failureIsError"_sr, "true"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("indefinite"_sr, "true"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("rateLimitMax"_sr, StringRef(rateLimitMax)));
+		options.push_back_deep(options.arena(), KeyValueRef("shuffleShards"_sr, "true"_sr));
 		spec.options.push_back_deep(spec.options.arena(), options);
 		testSet.testSpecs.push_back(spec);
 	} else if (whatToRun == TEST_TYPE_UNIT_TESTS) {
 		TestSpec spec;
 		Standalone<VectorRef<KeyValueRef>> options;
-		spec.title = LiteralStringRef("UnitTests");
+		spec.title = "UnitTests"_sr;
 		spec.startDelay = 0;
 		spec.useDB = false;
 		spec.timeout = 0;
-		options.push_back_deep(options.arena(),
-		                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("UnitTests")));
-		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("testsMatching"), fileName));
+		options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "UnitTests"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("testsMatching"_sr, fileName));
 		// Add unit test options as test spec options
 		for (auto& kv : testOptions.params) {
 			options.push_back_deep(options.arena(), KeyValueRef(kv.first, kv.second));
@@ -1874,8 +2072,15 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		actors.push_back(
 		    reportErrors(monitorServerDBInfo(cc, LocalityData(), db), "MonitorServerDBInfo")); // FIXME: Locality
 		actors.push_back(reportErrors(testerServerCore(iTesters[0], connRecord, db, locality), "TesterServerCore"));
-		tests = runTests(
-		    cc, ci, iTesters, testSet.testSpecs, startingConfiguration, locality, defaultTenant, tenantsToCreate);
+		tests = runTests(cc,
+		                 ci,
+		                 iTesters,
+		                 testSet.testSpecs,
+		                 startingConfiguration,
+		                 locality,
+		                 defaultTenant,
+		                 tenantsToCreate,
+		                 restartingTest);
 	} else {
 		tests = reportErrors(runTests(cc,
 		                              ci,
@@ -1885,7 +2090,8 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		                              startingConfiguration,
 		                              locality,
 		                              defaultTenant,
-		                              tenantsToCreate),
+		                              tenantsToCreate,
+		                              restartingTest),
 		                     "RunTests");
 	}
 
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 5621754a73..fb0b68edc0 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -23,6 +23,7 @@
 #include <boost/lexical_cast.hpp>
 
 #include "fdbclient/FDBTypes.h"
+#include "flow/ApiVersion.h"
 #include "flow/IAsyncFile.h"
 #include "fdbrpc/Locality.h"
 #include "fdbclient/GlobalConfig.actor.h"
@@ -41,7 +42,7 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/MetricLogger.actor.h"
 #include "fdbserver/BackupInterface.h"
-#include "fdbserver/EncryptKeyProxyInterface.h"
+#include "fdbclient/EncryptKeyProxyInterface.h"
 #include "fdbserver/RoleLineage.actor.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbserver/IKeyValueStore.h"
@@ -190,10 +191,10 @@ Error checkIOTimeout(Error const& e) {
 	// In simulation, have to check global timed out flag for both this process and the machine process on which IO is
 	// done
 	if (g_network->isSimulated() && !timeoutOccurred)
-		timeoutOccurred = g_pSimulator->getCurrentProcess()->machine->machineProcess->global(INetwork::enASIOTimedOut);
+		timeoutOccurred = g_simulator->getCurrentProcess()->machine->machineProcess->global(INetwork::enASIOTimedOut);
 
 	if (timeoutOccurred) {
-		TEST(true); // Timeout occurred
+		CODE_PROBE(true, "Timeout occurred");
 		Error timeout = io_timeout();
 		// Preserve injectedness of error
 		if (e.isInjectedFault())
@@ -226,9 +227,9 @@ ACTOR Future<Void> handleIOErrors(Future<Void> actor, IClosable* store, UID id,
 			if (e.isError() && e.getError().code() == error_code_broken_promise && !storeError.isReady()) {
 				wait(delay(0.00001 + FLOW_KNOBS->MAX_BUGGIFIED_DELAY));
 			}
-			if (storeError.isReady() &&
-			    !((storeError.get().isError() && storeError.get().getError().code() == error_code_file_not_found))) {
-				throw storeError.get().isError() ? storeError.get().getError() : actor_cancelled();
+			if (storeError.isReady() && storeError.isError() &&
+			    storeError.getError().code() != error_code_file_not_found) {
+				throw storeError.get().getError();
 			}
 			if (e.isError()) {
 				throw e.getError();
@@ -236,18 +237,18 @@ ACTOR Future<Void> handleIOErrors(Future<Void> actor, IClosable* store, UID id,
 				return e.get();
 		}
 		when(ErrorOr<Void> e = wait(storeError)) {
-			// for remote kv store, worker can terminate without an error, so throws actor_cancelled
-			// (there's probably a better way tho)
-			TraceEvent("WorkerTerminatingByIOError", id)
-			    .errorUnsuppressed(e.isError() ? e.getError() : actor_cancelled());
+			TraceEvent("WorkerTerminatingByIOError", id).errorUnsuppressed(e.getError());
 			actor.cancel();
 			// file_not_found can occur due to attempting to open a partially deleted DiskQueue, which should not be
 			// reported SevError.
-			if (e.isError() && e.getError().code() == error_code_file_not_found) {
-				TEST(true); // Worker terminated with file_not_found error
+			if (e.getError().code() == error_code_file_not_found) {
+				CODE_PROBE(true, "Worker terminated with file_not_found error");
 				return Void();
+			} else if (e.getError().code() == error_code_lock_file_failure) {
+				CODE_PROBE(true, "Unable to lock file");
+				throw please_reboot_kv_store();
 			}
-			throw e.isError() ? e.getError() : actor_cancelled();
+			throw e.getError();
 		}
 	}
 }
@@ -269,9 +270,10 @@ ACTOR Future<Void> workerHandleErrors(FutureStream<ErrorInfo> errors) {
 			endRole(err.role, err.id, "Error", ok, err.error);
 
 			if (err.error.code() == error_code_please_reboot ||
-			    err.error.code() == error_code_please_reboot_remote_kv_store ||
 			    (err.role == Role::SHARED_TRANSACTION_LOG &&
-			     (err.error.code() == error_code_io_error || err.error.code() == error_code_io_timeout)))
+			     (err.error.code() == error_code_io_error || err.error.code() == error_code_io_timeout)) ||
+			    (SERVER_KNOBS->STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT && err.role == Role::STORAGE_SERVER &&
+			     err.error.code() == error_code_io_timeout))
 				throw err.error;
 		}
 	}
@@ -298,18 +300,18 @@ ACTOR Future<Void> loadedPonger(FutureStream<LoadedPingRequest> pings) {
 	loop {
 		LoadedPingRequest pong = waitNext(pings);
 		LoadedReply rep;
-		rep.payload = (pong.loadReply ? payloadBack : LiteralStringRef(""));
+		rep.payload = (pong.loadReply ? payloadBack : ""_sr);
 		rep.id = pong.id;
 		pong.reply.send(rep);
 	}
 }
 
-StringRef fileStoragePrefix = LiteralStringRef("storage-");
-StringRef testingStoragePrefix = LiteralStringRef("testingstorage-");
-StringRef fileLogDataPrefix = LiteralStringRef("log-");
-StringRef fileVersionedLogDataPrefix = LiteralStringRef("log2-");
-StringRef fileLogQueuePrefix = LiteralStringRef("logqueue-");
-StringRef tlogQueueExtension = LiteralStringRef("fdq");
+StringRef fileStoragePrefix = "storage-"_sr;
+StringRef testingStoragePrefix = "testingstorage-"_sr;
+StringRef fileLogDataPrefix = "log-"_sr;
+StringRef fileVersionedLogDataPrefix = "log2-"_sr;
+StringRef fileLogQueuePrefix = "logqueue-"_sr;
+StringRef tlogQueueExtension = "fdq"_sr;
 
 enum class FilesystemCheck {
 	FILES_ONLY,
@@ -386,12 +388,12 @@ struct TLogOptions {
 			if (key.size() != 0 && value.size() == 0)
 				return default_error_or();
 
-			if (key == LiteralStringRef("V")) {
+			if (key == "V"_sr) {
 				ErrorOr<TLogVersion> tLogVersion = TLogVersion::FromStringRef(value);
 				if (tLogVersion.isError())
 					return tLogVersion.getError();
 				options.version = tLogVersion.get();
-			} else if (key == LiteralStringRef("LS")) {
+			} else if (key == "LS"_sr) {
 				ErrorOr<TLogSpillType> tLogSpillType = TLogSpillType::FromStringRef(value);
 				if (tLogSpillType.isError())
 					return tLogSpillType.getError();
@@ -560,11 +562,13 @@ ACTOR Future<Void> registrationClient(
     Reference<AsyncVar<Optional<RatekeeperInterface>> const> rkInterf,
     Reference<AsyncVar<Optional<std::pair<int64_t, BlobManagerInterface>>> const> bmInterf,
     Reference<AsyncVar<Optional<EncryptKeyProxyInterface>> const> ekpInterf,
+    Reference<AsyncVar<Optional<ConsistencyScanInterface>> const> csInterf,
     Reference<AsyncVar<bool> const> degraded,
     Reference<IClusterConnectionRecord> connRecord,
     Reference<AsyncVar<std::set<std::string>> const> issues,
     Reference<ConfigNode> configNode,
     Reference<LocalConfiguration> localConfig,
+    ConfigBroadcastInterface configBroadcastInterface,
     Reference<AsyncVar<ServerDBInfo>> dbInfo,
     Promise<Void> recoveredDiskFiles) {
 	// Keeps the cluster controller (as it may be re-elected) informed that this worker exists
@@ -589,27 +593,29 @@ ACTOR Future<Void> registrationClient(
 			incorrectTime = Optional<double>();
 		}
 
-		RegisterWorkerRequest request(interf,
-		                              initialClass,
-		                              processClass,
-		                              asyncPriorityInfo->get(),
-		                              requestGeneration++,
-		                              ddInterf->get(),
-		                              rkInterf->get(),
-		                              bmInterf->get().present() ? bmInterf->get().get().second
-		                                                        : Optional<BlobManagerInterface>(),
-		                              ekpInterf->get(),
-		                              degraded->get(),
-		                              localConfig->lastSeenVersion(),
-		                              localConfig->configClassSet(),
-		                              recoveredDiskFiles.isSet());
+		RegisterWorkerRequest request(
+		    interf,
+		    initialClass,
+		    processClass,
+		    asyncPriorityInfo->get(),
+		    requestGeneration++,
+		    ddInterf->get(),
+		    rkInterf->get(),
+		    bmInterf->get().present() ? bmInterf->get().get().second : Optional<BlobManagerInterface>(),
+		    ekpInterf->get(),
+		    csInterf->get(),
+		    degraded->get(),
+		    localConfig.isValid() ? localConfig->lastSeenVersion() : Optional<Version>(),
+		    localConfig.isValid() ? localConfig->configClassSet() : Optional<ConfigClassSet>(),
+		    recoveredDiskFiles.isSet(),
+		    configBroadcastInterface);
 
 		for (auto const& i : issues->get()) {
 			request.issues.push_back_deep(request.issues.arena(), i);
 		}
 
 		if (!upToDate) {
-			request.issues.push_back_deep(request.issues.arena(), LiteralStringRef("incorrect_cluster_file_contents"));
+			request.issues.push_back_deep(request.issues.arena(), "incorrect_cluster_file_contents"_sr);
 			std::string connectionString = connRecord->getConnectionString().toString();
 			if (!incorrectTime.present()) {
 				incorrectTime = now();
@@ -666,6 +672,7 @@ ACTOR Future<Void> registrationClient(
 			when(wait(ccInterface->onChange())) { break; }
 			when(wait(ddInterf->onChange())) { break; }
 			when(wait(rkInterf->onChange())) { break; }
+			when(wait(csInterf->onChange())) { break; }
 			when(wait(bmInterf->onChange())) { break; }
 			when(wait(ekpInterf->onChange())) { break; }
 			when(wait(degraded->onChange())) { break; }
@@ -692,6 +699,10 @@ bool addressInDbAndPrimaryDc(const NetworkAddress& address, Reference<AsyncVar<S
 		return true;
 	}
 
+	if (dbi.consistencyScan.present() && dbi.consistencyScan.get().address() == address) {
+		return true;
+	}
+
 	if (dbi.blobManager.present() && dbi.blobManager.get().address() == address) {
 		return true;
 	}
@@ -752,7 +763,7 @@ TEST_CASE("/fdbserver/worker/addressInDbAndPrimaryDc") {
 	// Setup a ServerDBInfo for test.
 	ServerDBInfo testDbInfo;
 	LocalityData testLocal;
-	testLocal.set(LiteralStringRef("dcid"), StringRef(std::to_string(1)));
+	testLocal.set("dcid"_sr, StringRef(std::to_string(1)));
 	testDbInfo.master.locality = testLocal;
 
 	// Manually set up a master address.
@@ -768,7 +779,7 @@ TEST_CASE("/fdbserver/worker/addressInDbAndPrimaryDc") {
 	// Create a remote TLog. Although the remote TLog also uses the local address, it shouldn't be considered as
 	// in primary DC given the remote locality.
 	LocalityData fakeRemote;
-	fakeRemote.set(LiteralStringRef("dcid"), StringRef(std::to_string(2)));
+	fakeRemote.set("dcid"_sr, StringRef(std::to_string(2)));
 	TLogInterface remoteTlog(fakeRemote);
 	remoteTlog.initEndpoints();
 	testDbInfo.logSystemConfig.tLogs.back().tLogs.push_back(OptionalInterface(remoteTlog));
@@ -833,7 +844,7 @@ TEST_CASE("/fdbserver/worker/addressInDbAndPrimarySatelliteDc") {
 	// Setup a ServerDBInfo for test.
 	ServerDBInfo testDbInfo;
 	LocalityData testLocal;
-	testLocal.set(LiteralStringRef("dcid"), StringRef(std::to_string(1)));
+	testLocal.set("dcid"_sr, StringRef(std::to_string(1)));
 	testDbInfo.master.locality = testLocal;
 
 	// First, create an empty TLogInterface, and check that it shouldn't be considered as in satellite DC.
@@ -865,7 +876,7 @@ TEST_CASE("/fdbserver/worker/addressInDbAndPrimarySatelliteDc") {
 	// Create a remote TLog, and it should be considered as in remote DC.
 	NetworkAddress remoteTLogAddress(IPAddress(0x37373737), 1);
 	LocalityData fakeRemote;
-	fakeRemote.set(LiteralStringRef("dcid"), StringRef(std::to_string(2)));
+	fakeRemote.set("dcid"_sr, StringRef(std::to_string(2)));
 	TLogInterface remoteTLog(fakeRemote);
 	remoteTLog.initEndpoints();
 	remoteTLog.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ remoteTLogAddress }, UID(1, 2)));
@@ -914,7 +925,7 @@ TEST_CASE("/fdbserver/worker/addressInDbAndRemoteDc") {
 	// Setup a ServerDBInfo for test.
 	ServerDBInfo testDbInfo;
 	LocalityData testLocal;
-	testLocal.set(LiteralStringRef("dcid"), StringRef(std::to_string(1)));
+	testLocal.set("dcid"_sr, StringRef(std::to_string(1)));
 	testDbInfo.master.locality = testLocal;
 
 	// First, create an empty TLogInterface, and check that it shouldn't be considered as in remote DC.
@@ -930,7 +941,7 @@ TEST_CASE("/fdbserver/worker/addressInDbAndRemoteDc") {
 
 	// Create a remote TLog, and it should be considered as in remote DC.
 	LocalityData fakeRemote;
-	fakeRemote.set(LiteralStringRef("dcid"), StringRef(std::to_string(2)));
+	fakeRemote.set("dcid"_sr, StringRef(std::to_string(2)));
 	TLogInterface remoteTlog(fakeRemote);
 	remoteTlog.initEndpoints();
 
@@ -996,48 +1007,74 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 						continue;
 					}
 					bool degradedPeer = false;
+					bool disconnectedPeer = false;
 					if ((workerLocation == Primary && addressInDbAndPrimaryDc(address, dbInfo)) ||
 					    (workerLocation == Remote && addressInDbAndRemoteDc(address, dbInfo))) {
 						// Monitors intra DC latencies between servers that in the primary or remote DC's transaction
 						// systems. Note that currently we are not monitor storage servers, since lagging in storage
 						// servers today already can trigger server exclusion by data distributor.
 
-						if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT ||
-						    peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE) >
-						        SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD ||
-						    peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) >
-						        SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) {
+						if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
+							disconnectedPeer = true;
+						} else if (peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE) >
+						               SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD ||
+						           peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) >
+						               SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) {
 							degradedPeer = true;
 						}
+						if (disconnectedPeer || degradedPeer) {
+							TraceEvent("HealthMonitorDetectDegradedPeer")
+							    .detail("Peer", address)
+							    .detail("Elapsed", now() - peer->lastLoggedTime)
+							    .detail("Disconnected", disconnectedPeer)
+							    .detail("MinLatency", peer->pingLatencies.min())
+							    .detail("MaxLatency", peer->pingLatencies.max())
+							    .detail("MeanLatency", peer->pingLatencies.mean())
+							    .detail("MedianLatency", peer->pingLatencies.median())
+							    .detail("CheckedPercentile", SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE)
+							    .detail(
+							        "CheckedPercentileLatency",
+							        peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE))
+							    .detail("PingCount", peer->pingLatencies.getPopulationSize())
+							    .detail("PingTimeoutCount", peer->timeoutCount)
+							    .detail("ConnectionFailureCount", peer->connectFailedCount);
+						}
 					} else if (workerLocation == Primary && addressInDbAndPrimarySatelliteDc(address, dbInfo)) {
 						// Monitors inter DC latencies between servers in primary and primary satellite DC. Note that
 						// TLog workers in primary satellite DC are on the critical path of serving a commit.
-						if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT ||
-						    peer->pingLatencies.percentile(
-						        SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE) >
-						        SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD_SATELLITE ||
-						    peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) >
-						        SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) {
+						if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
+							disconnectedPeer = true;
+						} else if (peer->pingLatencies.percentile(
+						               SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE) >
+						               SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD_SATELLITE ||
+						           peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) >
+						               SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) {
 							degradedPeer = true;
 						}
+
+						if (disconnectedPeer || degradedPeer) {
+							TraceEvent("HealthMonitorDetectDegradedPeer")
+							    .detail("Peer", address)
+							    .detail("Elapsed", now() - peer->lastLoggedTime)
+							    .detail("Disconnected", disconnectedPeer)
+							    .detail("MinLatency", peer->pingLatencies.min())
+							    .detail("MaxLatency", peer->pingLatencies.max())
+							    .detail("MeanLatency", peer->pingLatencies.mean())
+							    .detail("MedianLatency", peer->pingLatencies.median())
+							    .detail("CheckedPercentile",
+							            SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE)
+							    .detail("CheckedPercentileLatency",
+							            peer->pingLatencies.percentile(
+							                SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE))
+							    .detail("PingCount", peer->pingLatencies.getPopulationSize())
+							    .detail("PingTimeoutCount", peer->timeoutCount)
+							    .detail("ConnectionFailureCount", peer->connectFailedCount);
+						}
 					}
 
-					if (degradedPeer) {
-						TraceEvent("HealthMonitorDetectDegradedPeer")
-						    .suppressFor(30)
-						    .detail("Peer", address)
-						    .detail("Elapsed", now() - peer->lastLoggedTime)
-						    .detail("MinLatency", peer->pingLatencies.min())
-						    .detail("MaxLatency", peer->pingLatencies.max())
-						    .detail("MeanLatency", peer->pingLatencies.mean())
-						    .detail("MedianLatency", peer->pingLatencies.median())
-						    .detail("CheckedPercentile", SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE)
-						    .detail("CheckedPercentileLatency",
-						            peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE))
-						    .detail("PingCount", peer->pingLatencies.getPopulationSize())
-						    .detail("PingTimeoutCount", peer->timeoutCount)
-						    .detail("ConnectionFailureCount", peer->connectFailedCount);
-
+					if (disconnectedPeer) {
+						req.disconnectedPeers.push_back(address);
+					} else if (degradedPeer) {
 						req.degradedPeers.push_back(address);
 					}
 				}
@@ -1058,13 +1095,13 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 						    (workerLocation == Remote && addressInDbAndRemoteDc(address, dbInfo)) ||
 						    (workerLocation == Primary && addressInDbAndPrimarySatelliteDc(address, dbInfo))) {
 							TraceEvent("HealthMonitorDetectRecentClosedPeer").suppressFor(30).detail("Peer", address);
-							req.degradedPeers.push_back(address);
+							req.disconnectedPeers.push_back(address);
 						}
 					}
 				}
 			}
 
-			if (!req.degradedPeers.empty()) {
+			if (!req.disconnectedPeers.empty() || !req.degradedPeers.empty()) {
 				req.address = FlowTransport::transport().getLocalAddress();
 				ccInterface->get().get().updateWorkerHealth.send(req);
 			}
@@ -1236,13 +1273,9 @@ struct TrackRunningStorage {
 	                    KeyValueStoreType storeType,
 	                    std::set<std::pair<UID, KeyValueStoreType>>* runningStorages)
 	  : self(self), storeType(storeType), runningStorages(runningStorages) {
-		TraceEvent(SevDebug, "TrackingRunningStorageConstruction").detail("StorageID", self);
 		runningStorages->emplace(self, storeType);
 	}
-	~TrackRunningStorage() {
-		runningStorages->erase(std::make_pair(self, storeType));
-		TraceEvent(SevDebug, "TrackingRunningStorageDesctruction").detail("StorageID", self);
-	};
+	~TrackRunningStorage() { runningStorages->erase(std::make_pair(self, storeType)); };
 };
 
 ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValueStoreType>>* runningStorages,
@@ -1256,16 +1289,48 @@ ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValu
                                                  std::string folder,
                                                  ActorCollection* filesClosed,
                                                  int64_t memoryLimit,
-                                                 IKeyValueStore* store) {
+                                                 IKeyValueStore* store,
+                                                 bool validateDataFiles,
+                                                 Promise<Void>* rebootKVStore,
+                                                 Reference<IPageEncryptionKeyProvider> encryptionKeyProvider) {
 	state TrackRunningStorage _(id, storeType, runningStorages);
 	loop {
 		ErrorOr<Void> e = wait(errorOr(prevStorageServer));
 		if (!e.isError())
 			return Void();
-		else if (e.getError().code() != error_code_please_reboot)
+		else if (e.getError().code() != error_code_please_reboot &&
+		         e.getError().code() != error_code_please_reboot_kv_store)
 			throw e.getError();
 
-		TraceEvent("StorageServerRequestedReboot", id).log();
+		TraceEvent("StorageServerRequestedReboot", id)
+		    .detail("RebootStorageEngine", e.getError().code() == error_code_please_reboot_kv_store)
+		    .log();
+
+		if (e.getError().code() == error_code_please_reboot_kv_store) {
+			// Add the to actorcollection to make sure filesClosed not return
+			filesClosed->add(rebootKVStore->getFuture());
+			wait(delay(SERVER_KNOBS->REBOOT_KV_STORE_DELAY));
+			// reopen KV store
+			store = openKVStore(
+			    storeType,
+			    filename,
+			    id,
+			    memoryLimit,
+			    false,
+			    validateDataFiles,
+			    SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled */
+			        (g_network->isSimulated()
+			             ? (/* Disable for RocksDB */ storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
+			                deterministicRandom()->coinflip())
+			             : true));
+			Promise<Void> nextRebootKVStorePromise;
+			filesClosed->add(store->onClosed() ||
+			                 nextRebootKVStorePromise
+			                     .getFuture() /* clear the onClosed() Future in actorCollection when rebooting */);
+			// remove the original onClosed signal from the actorCollection
+			rebootKVStore->send(Void());
+			rebootKVStore->swap(nextRebootKVStorePromise);
+		}
 
 		StorageServerInterface recruited;
 		recruited.uniqueID = id;
@@ -1294,8 +1359,13 @@ ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValu
 		DUMPTOKEN(recruited.changeFeedPop);
 		DUMPTOKEN(recruited.changeFeedVersionUpdate);
 
-		prevStorageServer =
-		    storageServer(store, recruited, db, folder, Promise<Void>(), Reference<IClusterConnectionRecord>(nullptr));
+		prevStorageServer = storageServer(store,
+		                                  recruited,
+		                                  db,
+		                                  folder,
+		                                  Promise<Void>(),
+		                                  Reference<IClusterConnectionRecord>(nullptr),
+		                                  encryptionKeyProvider);
 		prevStorageServer = handleIOErrors(prevStorageServer, store, id, store->onClosed());
 	}
 }
@@ -1376,10 +1446,10 @@ void startRole(const Role& role,
 
 	// Update roles map, log Roles metrics
 	g_roles.insert({ role.roleName, roleId.shortString() });
-	StringMetricHandle(LiteralStringRef("Roles")) = roleString(g_roles, false);
-	StringMetricHandle(LiteralStringRef("RolesWithIDs")) = roleString(g_roles, true);
+	StringMetricHandle("Roles"_sr) = roleString(g_roles, false);
+	StringMetricHandle("RolesWithIDs"_sr) = roleString(g_roles, true);
 	if (g_network->isSimulated())
-		g_simulator.addRole(g_network->getLocalAddress(), role.roleName);
+		g_simulator->addRole(g_network->getLocalAddress(), role.roleName);
 }
 
 void endRole(const Role& role, UID id, std::string reason, bool ok, Error e) {
@@ -1406,10 +1476,10 @@ void endRole(const Role& role, UID id, std::string reason, bool ok, Error e) {
 
 	// Update roles map, log Roles metrics
 	g_roles.erase({ role.roleName, id.shortString() });
-	StringMetricHandle(LiteralStringRef("Roles")) = roleString(g_roles, false);
-	StringMetricHandle(LiteralStringRef("RolesWithIDs")) = roleString(g_roles, true);
+	StringMetricHandle("Roles"_sr) = roleString(g_roles, false);
+	StringMetricHandle("RolesWithIDs"_sr) = roleString(g_roles, true);
 	if (g_network->isSimulated())
-		g_simulator.removeRole(g_network->getLocalAddress(), role.roleName);
+		g_simulator->removeRole(g_network->getLocalAddress(), role.roleName);
 
 	if (role.includeInTraceRoles) {
 		removeTraceRole(role.abbreviation);
@@ -1580,12 +1650,18 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 	state Reference<AsyncVar<Optional<RatekeeperInterface>>> rkInterf(new AsyncVar<Optional<RatekeeperInterface>>());
 	state Reference<AsyncVar<Optional<std::pair<int64_t, BlobManagerInterface>>>> bmEpochAndInterf(
 	    new AsyncVar<Optional<std::pair<int64_t, BlobManagerInterface>>>());
+	state UID lastBMRecruitRequestId;
 	state Reference<AsyncVar<Optional<EncryptKeyProxyInterface>>> ekpInterf(
 	    new AsyncVar<Optional<EncryptKeyProxyInterface>>());
+	state Reference<AsyncVar<Optional<ConsistencyScanInterface>>> csInterf(
+	    new AsyncVar<Optional<ConsistencyScanInterface>>());
 	state Future<Void> handleErrors = workerHandleErrors(errors.getFuture()); // Needs to be stopped last
 	state ActorCollection errorForwarders(false);
 	state Future<Void> loggingTrigger = Void();
 	state double loggingDelay = SERVER_KNOBS->WORKER_LOGGING_INTERVAL;
+	// These two promises are destroyed after the "filesClosed" below to avoid broken_promise
+	state Promise<Void> rebootKVSPromise;
+	state Promise<Void> rebootKVSPromise2;
 	state ActorCollection filesClosed(true);
 	state Promise<Void> stopping;
 	state WorkerCache<InitializeStorageReply> storageCache;
@@ -1610,7 +1686,6 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 	state std::string coordFolder = abspath(_coordFolder);
 
 	state WorkerInterface interf(locality);
-	interf.configBroadcastInterface = configBroadcastInterface;
 	state std::set<std::pair<UID, KeyValueStoreType>> runningStorages;
 	interf.initEndpoints();
 
@@ -1627,7 +1702,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 		if (metricsConnFile.size() > 0) {
 			try {
 				state Database db =
-				    Database::createDatabase(metricsConnFile, Database::API_VERSION_LATEST, IsInternal::True, locality);
+				    Database::createDatabase(metricsConnFile, ApiVersion::LATEST_VERSION, IsInternal::True, locality);
 				metricsLogger = runMetrics(db, KeyRef(metricsPrefix));
 				db->globalConfig->trigger(samplingFrequency, samplingProfilerUpdateFrequency);
 			} catch (Error& e) {
@@ -1688,6 +1763,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 			if (s.storedComponent == DiskStore::Storage) {
 				LocalLineage _;
 				getCurrentLineage()->modify(&RoleLineage::role) = ProcessClass::ClusterRole::Storage;
+				Reference<IPageEncryptionKeyProvider> encryptionKeyProvider =
+				    makeReference<TenantAwareEncryptionKeyProvider>(dbInfo);
 				IKeyValueStore* kv = openKVStore(
 				    s.storeType,
 				    s.filename,
@@ -1700,8 +1777,11 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 				             ? (/* Disable for RocksDB */ s.storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
 				                s.storeType != KeyValueStoreType::SSD_SHARDED_ROCKSDB &&
 				                deterministicRandom()->coinflip())
-				             : true));
-				Future<Void> kvClosed = kv->onClosed();
+				             : true),
+				    encryptionKeyProvider);
+				Future<Void> kvClosed =
+				    kv->onClosed() ||
+				    rebootKVSPromise.getFuture() /* clear the onClosed() Future in actorCollection when rebooting */;
 				filesClosed.add(kvClosed);
 
 				// std::string doesn't have startsWith
@@ -1746,7 +1826,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 				DUMPTOKEN(recruited.changeFeedVersionUpdate);
 
 				Promise<Void> recovery;
-				Future<Void> f = storageServer(kv, recruited, dbInfo, folder, recovery, connRecord);
+				Future<Void> f =
+				    storageServer(kv, recruited, dbInfo, folder, recovery, connRecord, encryptionKeyProvider);
 				recoveries.push_back(recovery.getFuture());
 				f = handleIOErrors(f, kv, s.storeID, kvClosed);
 				f = storageServerRollbackRebooter(&runningStorages,
@@ -1760,7 +1841,10 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 				                                  folder,
 				                                  &filesClosed,
 				                                  memoryLimit,
-				                                  kv);
+				                                  kv,
+				                                  validateDataFiles,
+				                                  &rebootKVSPromise,
+				                                  encryptionKeyProvider);
 				errorForwarders.add(forwardError(errors, ssRole, recruited.id(), f));
 			} else if (s.storedComponent == DiskStore::TLogData) {
 				LocalLineage _;
@@ -1774,7 +1858,6 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					logQueueBasename = fileLogQueuePrefix.toString() + optionsString.toString() + "-";
 				}
 				ASSERT_WE_THINK(abspath(parentDirectory(s.filename)) == folder);
-				// TraceEvent(SevDebug, "openRemoteKVStore").detail("storeType", "TlogData");
 				IKeyValueStore* kv = openKVStore(s.storeType, s.filename, s.storeID, memoryLimit, validateDataFiles);
 				const DiskQueueVersion dqv = s.tLogOptions.getDiskQueueVersion();
 				const int64_t diskQueueWarnSize =
@@ -1894,16 +1977,18 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 		                                       rkInterf,
 		                                       bmEpochAndInterf,
 		                                       ekpInterf,
+		                                       csInterf,
 		                                       degraded,
 		                                       connRecord,
 		                                       issues,
 		                                       configNode,
 		                                       localConfig,
+		                                       configBroadcastInterface,
 		                                       dbInfo,
 		                                       recoveredDiskFiles));
 
 		if (configNode.isValid()) {
-			errorForwarders.add(localConfig->consume(interf.configBroadcastInterface));
+			errorForwarders.add(brokenPromiseToNever(localConfig->consume(configBroadcastInterface)));
 		}
 
 		if (SERVER_KNOBS->ENABLE_WORKER_HEALTH_MONITOR) {
@@ -2044,7 +2129,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 
 				if (ddInterf->get().present()) {
 					recruited = ddInterf->get().get();
-					TEST(true); // Recruited while already a data distributor.
+					CODE_PROBE(true, "Recruited while already a data distributor.");
 				} else {
 					startRole(Role::DATA_DISTRIBUTOR, recruited.id(), interf.id());
 					DUMPTOKEN(recruited.waitFailure);
@@ -2068,7 +2153,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 
 				if (rkInterf->get().present()) {
 					recruited = rkInterf->get().get();
-					TEST(true); // Recruited while already a ratekeeper.
+					CODE_PROBE(true, "Recruited while already a ratekeeper.");
 				} else {
 					startRole(Role::RATEKEEPER, recruited.id(), interf.id());
 					DUMPTOKEN(recruited.waitFailure);
@@ -2087,16 +2172,49 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 				TraceEvent("Ratekeeper_InitRequest", req.reqId).detail("RatekeeperId", recruited.id());
 				req.reply.send(recruited);
 			}
+			when(InitializeConsistencyScanRequest req = waitNext(interf.consistencyScan.getFuture())) {
+				LocalLineage _;
+				getCurrentLineage()->modify(&RoleLineage::role) = ProcessClass::ClusterRole::ConsistencyScan;
+				ConsistencyScanInterface recruited(locality, req.reqId);
+				recruited.initEndpoints();
+
+				if (csInterf->get().present()) {
+					recruited = csInterf->get().get();
+					CODE_PROBE(true, "Recovered while already a consistencyscan");
+				} else {
+					startRole(Role::CONSISTENCYSCAN, recruited.id(), interf.id());
+					DUMPTOKEN(recruited.waitFailure);
+					DUMPTOKEN(recruited.haltConsistencyScan);
+
+					Future<Void> consistencyScanProcess = consistencyScan(recruited, dbInfo);
+					errorForwarders.add(forwardError(
+					    errors,
+					    Role::CONSISTENCYSCAN,
+					    recruited.id(),
+					    setWhenDoneOrError(consistencyScanProcess, csInterf, Optional<ConsistencyScanInterface>())));
+					csInterf->set(Optional<ConsistencyScanInterface>(recruited));
+				}
+				TraceEvent("ConsistencyScanReceived", req.reqId).detail("ConsistencyScanId", recruited.id());
+				req.reply.send(recruited);
+			}
 			when(InitializeBlobManagerRequest req = waitNext(interf.blobManager.getFuture())) {
 				LocalLineage _;
 				getCurrentLineage()->modify(&RoleLineage::role) = ProcessClass::ClusterRole::BlobManager;
-				BlobManagerInterface recruited(locality, req.reqId);
+				BlobManagerInterface recruited(locality, req.reqId, req.epoch);
 				recruited.initEndpoints();
 
 				if (bmEpochAndInterf->get().present() && bmEpochAndInterf->get().get().first == req.epoch) {
+					ASSERT(req.reqId == lastBMRecruitRequestId);
 					recruited = bmEpochAndInterf->get().get().second;
 
-					TEST(true); // Recruited while already a blob manager.
+					CODE_PROBE(true, "Recruited while already a blob manager.");
+				} else if (lastBMRecruitRequestId == req.reqId && !bmEpochAndInterf->get().present()) {
+					// The previous blob manager WAS present, like the above case, but it died before the CC got the
+					// response to the recruitment request, so the CC retried to recruit the same blob manager id/epoch
+					// from the same reqId. To keep epoch safety between different managers, instead of restarting the
+					// same manager id at the same epoch, we should just tell it the original request succeeded, and let
+					// it realize this manager died via failure detection and start a new one.
+					CODE_PROBE(true, "Recruited while formerly the same blob manager.");
 				} else {
 					// TODO: it'd be more optimal to halt the last manager if present here, but it will figure it out
 					// via the epoch check
@@ -2109,6 +2227,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					DUMPTOKEN(recruited.haltBlobGranules);
 					DUMPTOKEN(recruited.blobManagerExclCheckReq);
 
+					lastBMRecruitRequestId = req.reqId;
+
 					Future<Void> blobManagerProcess = blobManager(recruited, dbInfo, req.epoch);
 					errorForwarders.add(
 					    forwardError(errors,
@@ -2151,7 +2271,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 
 				if (ekpInterf->get().present()) {
 					recruited = ekpInterf->get().get();
-					TEST(true); // Recruited while already a encryptKeyProxy server.
+					CODE_PROBE(true, "Recruited while already a encryptKeyProxy server.");
 				} else {
 					startRole(Role::ENCRYPT_KEY_PROXY, recruited.id(), interf.id());
 					DUMPTOKEN(recruited.waitFailure);
@@ -2200,7 +2320,6 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					    req.logVersion > TLogVersion::V2 ? fileVersionedLogDataPrefix : fileLogDataPrefix;
 					std::string filename =
 					    filenameFromId(req.storeType, folder, prefix.toString() + tLogOptions.toPrefix(), logId);
-					// TraceEvent(SevDebug, "openRemoteKVStore").detail("storeType", "3");
 					IKeyValueStore* data = openKVStore(req.storeType, filename, logId, memoryLimit);
 					const DiskQueueVersion dqv = tLogOptions.getDiskQueueVersion();
 					IDiskQueue* queue = openDiskQueue(
@@ -2281,14 +2400,14 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					DUMPTOKEN(recruited.changeFeedStream);
 					DUMPTOKEN(recruited.changeFeedPop);
 					DUMPTOKEN(recruited.changeFeedVersionUpdate);
-					// printf("Recruited as storageServer\n");
 
 					std::string filename =
 					    filenameFromId(req.storeType,
 					                   folder,
 					                   isTss ? testingStoragePrefix.toString() : fileStoragePrefix.toString(),
 					                   recruited.id());
-
+					Reference<IPageEncryptionKeyProvider> encryptionKeyProvider =
+					    makeReference<TenantAwareEncryptionKeyProvider>(dbInfo);
 					IKeyValueStore* data = openKVStore(
 					    req.storeType,
 					    filename,
@@ -2301,9 +2420,13 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					             ? (/* Disable for RocksDB */ req.storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
 					                req.storeType != KeyValueStoreType::SSD_SHARDED_ROCKSDB &&
 					                deterministicRandom()->coinflip())
-					             : true));
+					             : true),
+					    encryptionKeyProvider);
 
-					Future<Void> kvClosed = data->onClosed();
+					Future<Void> kvClosed =
+					    data->onClosed() ||
+					    rebootKVSPromise2
+					        .getFuture() /* clear the onClosed() Future in actorCollection when rebooting */;
 					filesClosed.add(kvClosed);
 					ReplyPromise<InitializeStorageReply> storageReady = req.reply;
 					storageCache.set(req.reqId, storageReady.getFuture());
@@ -2315,7 +2438,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					                               isTss ? req.tssPairIDAndVersion.get().second : 0,
 					                               storageReady,
 					                               dbInfo,
-					                               folder);
+					                               folder,
+					                               encryptionKeyProvider);
 					s = handleIOErrors(s, data, recruited.id(), kvClosed);
 					s = storageCache.removeOnReady(req.reqId, s);
 					s = storageServerRollbackRebooter(&runningStorages,
@@ -2329,7 +2453,10 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					                                  folder,
 					                                  &filesClosed,
 					                                  memoryLimit,
-					                                  data);
+					                                  data,
+					                                  false,
+					                                  &rebootKVSPromise2,
+					                                  encryptionKeyProvider);
 					errorForwarders.add(forwardError(errors, ssRole, recruited.id(), s));
 				} else if (storageCache.exists(req.reqId)) {
 					forwardPromise(req.reply, storageCache.get(req.reqId));
@@ -2354,6 +2481,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					DUMPTOKEN(recruited.granuleAssignmentsRequest);
 					DUMPTOKEN(recruited.granuleStatusStreamRequest);
 					DUMPTOKEN(recruited.haltBlobWorker);
+					DUMPTOKEN(recruited.minBlobVersionRequest);
 
 					ReplyPromise<InitializeBlobWorkerReply> blobWorkerReady = req.reply;
 					Future<Void> bw = blobWorker(recruited, blobWorkerReady, dbInfo);
@@ -2543,28 +2671,30 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 				loggingTrigger = delay(loggingDelay, TaskPriority::FlushTrace);
 			}
 			when(state WorkerSnapRequest snapReq = waitNext(interf.workerSnapReq.getFuture())) {
-				std::string snapUID = snapReq.snapUID.toString() + snapReq.role.toString();
-				if (snapReqResultMap.count(snapUID)) {
-					TEST(true); // Worker received a duplicate finished snap request
-					auto result = snapReqResultMap[snapUID];
+				std::string snapReqKey = snapReq.snapUID.toString() + snapReq.role.toString();
+				if (snapReqResultMap.count(snapReqKey)) {
+					CODE_PROBE(true, "Worker received a duplicate finished snapshot request");
+					auto result = snapReqResultMap[snapReqKey];
 					result.isError() ? snapReq.reply.sendError(result.getError()) : snapReq.reply.send(result.get());
 					TraceEvent("RetryFinishedWorkerSnapRequest")
-					    .detail("SnapUID", snapUID)
+					    .detail("SnapUID", snapReq.snapUID.toString())
 					    .detail("Role", snapReq.role)
-					    .detail("Result", result.isError() ? result.getError().code() : 0);
-				} else if (snapReqMap.count(snapUID)) {
-					TEST(true); // Worker received a duplicate ongoing snap request
-					TraceEvent("RetryOngoingWorkerSnapRequest").detail("SnapUID", snapUID).detail("Role", snapReq.role);
-					ASSERT(snapReq.role == snapReqMap[snapUID].role);
-					ASSERT(snapReq.snapPayload == snapReqMap[snapUID].snapPayload);
-					snapReqMap[snapUID] = snapReq;
+					    .detail("Result", result.isError() ? result.getError().code() : success().code());
+				} else if (snapReqMap.count(snapReqKey)) {
+					CODE_PROBE(true, "Worker received a duplicate ongoing snapshot request");
+					TraceEvent("RetryOngoingWorkerSnapRequest")
+					    .detail("SnapUID", snapReq.snapUID.toString())
+					    .detail("Role", snapReq.role);
+					ASSERT(snapReq.role == snapReqMap[snapReqKey].role);
+					ASSERT(snapReq.snapPayload == snapReqMap[snapReqKey].snapPayload);
+					snapReqMap[snapReqKey] = snapReq;
 				} else {
-					snapReqMap[snapUID] = snapReq; // set map point to the request
+					snapReqMap[snapReqKey] = snapReq; // set map point to the request
 					if (g_network->isSimulated() && (now() - lastSnapTime) < SERVER_KNOBS->SNAP_MINIMUM_TIME_GAP) {
 						// only allow duplicate snapshots on same process in a short time for different roles
 						auto okay = (lastSnapReq.snapUID == snapReq.snapUID) && lastSnapReq.role != snapReq.role;
 						TraceEvent(okay ? SevInfo : SevError, "RapidSnapRequestsOnSameProcess")
-						    .detail("CurrSnapUID", snapUID)
+						    .detail("CurrSnapUID", snapReqKey)
 						    .detail("PrevSnapUID", lastSnapReq.snapUID)
 						    .detail("CurrRole", snapReq.role)
 						    .detail("PrevRole", lastSnapReq.role)
@@ -2576,8 +2706,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					                                     &snapReqResultMap));
 					auto* snapReqResultMapPtr = &snapReqResultMap;
 					errorForwarders.add(fmap(
-					    [snapReqResultMapPtr, snapUID](Void _) {
-						    snapReqResultMapPtr->erase(snapUID);
+					    [snapReqResultMapPtr, snapReqKey](Void _) {
+						    snapReqResultMapPtr->erase(snapReqKey);
 						    return Void();
 					    },
 					    delay(SERVER_KNOBS->SNAP_MINIMUM_TIME_GAP)));
@@ -2591,26 +2721,21 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 			when(wait(handleErrors)) {}
 		}
 	} catch (Error& err) {
-		TraceEvent(SevDebug, "WorkerServer").detail("Error", err.code()).backtrace();
 		// Make sure actors are cancelled before "recovery" promises are destructed.
 		for (auto f : recoveries)
 			f.cancel();
 		state Error e = err;
 		bool ok = e.code() == error_code_please_reboot || e.code() == error_code_actor_cancelled ||
-		          e.code() == error_code_please_reboot_delete || e.code() == error_code_please_reboot_remote_kv_store;
+		          e.code() == error_code_please_reboot_delete;
 
 		endRole(Role::WORKER, interf.id(), "WorkerError", ok, e);
 		errorForwarders.clear(false);
 		sharedLogs.clear();
 
-		if (e.code() != error_code_actor_cancelled && e.code() != error_code_please_reboot_remote_kv_store) {
+		if (e.code() != error_code_actor_cancelled) {
 			// actor_cancelled:
 			// We get cancelled e.g. when an entire simulation times out, but in that case
 			// we won't be restarted and don't need to wait for shutdown
-			// reboot_remote_kv_store:
-			// The child process running the storage engine died abnormally,
-			// the current solution is to reboot the worker.
-			// Some refactoring work in the future can make it only reboot the storage server
 			stopping.send(Void());
 			wait(filesClosed.getResult()); // Wait for complete shutdown of KV stores
 			wait(delay(0.0)); // Unwind the callstack to make sure that IAsyncFile references are all gone
@@ -2808,7 +2933,8 @@ ACTOR Future<Void> updateNewestSoftwareVersion(std::string folder,
 }
 
 ACTOR Future<Void> testAndUpdateSoftwareVersionCompatibility(std::string dataFolder, UID processIDUid) {
-	ErrorOr<SWVersion> swVersion = wait(errorOr(testSoftwareVersionCompatibility(dataFolder, currentProtocolVersion)));
+	ErrorOr<SWVersion> swVersion =
+	    wait(errorOr(testSoftwareVersionCompatibility(dataFolder, currentProtocolVersion())));
 	if (swVersion.isError()) {
 		TraceEvent(SevWarnAlways, "SWVersionCompatibilityCheckError", processIDUid).error(swVersion.getError());
 		throw swVersion.getError();
@@ -2817,16 +2943,16 @@ ACTOR Future<Void> testAndUpdateSoftwareVersionCompatibility(std::string dataFol
 	TraceEvent(SevInfo, "SWVersionCompatible", processIDUid).detail("SWVersion", swVersion.get());
 
 	if (!swVersion.get().isValid() ||
-	    currentProtocolVersion > ProtocolVersion(swVersion.get().newestProtocolVersion())) {
+	    currentProtocolVersion() > ProtocolVersion(swVersion.get().newestProtocolVersion())) {
 		ErrorOr<Void> updatedSWVersion = wait(errorOr(updateNewestSoftwareVersion(
-		    dataFolder, currentProtocolVersion, currentProtocolVersion, minCompatibleProtocolVersion)));
+		    dataFolder, currentProtocolVersion(), currentProtocolVersion(), minCompatibleProtocolVersion)));
 		if (updatedSWVersion.isError()) {
 			throw updatedSWVersion.getError();
 		}
-	} else if (currentProtocolVersion < ProtocolVersion(swVersion.get().newestProtocolVersion())) {
+	} else if (currentProtocolVersion() < ProtocolVersion(swVersion.get().newestProtocolVersion())) {
 		ErrorOr<Void> updatedSWVersion = wait(
 		    errorOr(updateNewestSoftwareVersion(dataFolder,
-		                                        currentProtocolVersion,
+		                                        currentProtocolVersion(),
 		                                        ProtocolVersion(swVersion.get().newestProtocolVersion()),
 		                                        ProtocolVersion(swVersion.get().lowestCompatibleProtocolVersion()))));
 		if (updatedSWVersion.isError()) {
@@ -2835,7 +2961,7 @@ ACTOR Future<Void> testAndUpdateSoftwareVersionCompatibility(std::string dataFol
 	}
 
 	ErrorOr<SWVersion> newSWVersion =
-	    wait(errorOr(testSoftwareVersionCompatibility(dataFolder, currentProtocolVersion)));
+	    wait(errorOr(testSoftwareVersionCompatibility(dataFolder, currentProtocolVersion())));
 	if (newSWVersion.isError()) {
 		TraceEvent(SevWarnAlways, "SWVersionCompatibilityCheckError", processIDUid).error(newSWVersion.getError());
 		throw newSWVersion.getError();
@@ -3287,8 +3413,11 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 	state std::vector<Future<Void>> actors;
 	state Promise<Void> recoveredDiskFiles;
 	state Reference<ConfigNode> configNode;
-	state Reference<LocalConfiguration> localConfig =
-	    makeReference<LocalConfiguration>(dataFolder, configPath, manualKnobOverrides);
+	state Reference<LocalConfiguration> localConfig;
+	if (configDBType != ConfigDBType::DISABLED) {
+		localConfig = makeReference<LocalConfiguration>(
+		    dataFolder, configPath, manualKnobOverrides, IsTest(g_network->isSimulated()));
+	}
 	// setupStackSignal();
 	getCurrentLineage()->modify(&RoleLineage::role) = ProcessClass::Worker;
 
@@ -3296,18 +3425,11 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 		configNode = makeReference<ConfigNode>(dataFolder);
 	}
 
-	// FIXME: Initializing here causes simulation issues, these must be fixed
-	/*
-	if (configDBType != ConfigDBType::DISABLED) {
-	    wait(localConfig->initialize());
-	}
-	*/
-
 	actors.push_back(serveProtocolInfo());
 	actors.push_back(serveProcess());
 
 	try {
-		ServerCoordinators coordinators(connRecord);
+		ServerCoordinators coordinators(connRecord, configDBType);
 		if (g_network->isSimulated()) {
 			whitelistBinPaths = ",, random_path,  /bin/snap_create.sh,,";
 		}
@@ -3316,7 +3438,8 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 		    .detail("MachineId", localities.machineId())
 		    .detail("DiskPath", dataFolder)
 		    .detail("CoordPath", coordFolder)
-		    .detail("WhiteListBinPath", whitelistBinPaths);
+		    .detail("WhiteListBinPath", whitelistBinPaths)
+		    .detail("ConfigDBType", configDBType);
 
 		state ConfigBroadcastInterface configBroadcastInterface;
 		// SOMEDAY: start the services on the machine in a staggered fashion in simulation?
@@ -3335,12 +3458,20 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 
 		wait(testAndUpdateSoftwareVersionCompatibility(dataFolder, processIDUid));
 
+		if (configDBType != ConfigDBType::DISABLED) {
+			wait(localConfig->initialize());
+		}
+
 		std::string fitnessFilePath = joinPath(dataFolder, "fitness");
 		auto cc = makeReference<AsyncVar<Optional<ClusterControllerFullInterface>>>();
 		auto ci = makeReference<AsyncVar<Optional<ClusterInterface>>>();
 		auto asyncPriorityInfo =
 		    makeReference<AsyncVar<ClusterControllerPriorityInfo>>(getCCPriorityInfo(fitnessFilePath, processClass));
-		auto dbInfo = makeReference<AsyncVar<ServerDBInfo>>();
+		auto serverDBInfo = ServerDBInfo();
+		serverDBInfo.client.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
+		serverDBInfo.myLocality = localities;
+		auto dbInfo = makeReference<AsyncVar<ServerDBInfo>>(serverDBInfo);
+		TraceEvent("MyLocality").detail("Locality", dbInfo->get().myLocality.toString());
 
 		actors.push_back(reportErrors(monitorAndWriteCCPriorityInfo(fitnessFilePath, asyncPriorityInfo),
 		                              "MonitorAndWriteCCPriorityInfo"));
@@ -3418,3 +3549,4 @@ const Role Role::STORAGE_CACHE("StorageCache", "SC");
 const Role Role::COORDINATOR("Coordinator", "CD");
 const Role Role::BACKUP("Backup", "BK");
 const Role Role::ENCRYPT_KEY_PROXY("EncryptKeyProxy", "EP");
+const Role Role::CONSISTENCYSCAN("ConsistencyScan", "CS");
diff --git a/fdbserver/workloads/ApiCorrectness.actor.cpp b/fdbserver/workloads/ApiCorrectness.actor.cpp
index 246af9cde1..3777e8ea2a 100644
--- a/fdbserver/workloads/ApiCorrectness.actor.cpp
+++ b/fdbserver/workloads/ApiCorrectness.actor.cpp
@@ -35,7 +35,7 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 private:
 // Enable to track the activity on a particular key
 #if CENABLED(0, NOT_IN_CLEAN)
-#define targetKey LiteralStringRef( ??? )
+#define targetKey "???"_sr
 
 	void debugKey(KeyRef key, std::string context) {
 		if (key == targetKey)
@@ -99,21 +99,21 @@ public:
 
 	ApiCorrectnessWorkload(WorkloadContext const& wcx)
 	  : ApiWorkload(wcx), numRandomOperations("Num Random Operations") {
-		numGets = getOption(options, LiteralStringRef("numGets"), 1000);
-		numGetRanges = getOption(options, LiteralStringRef("numGetRanges"), 100);
-		numGetRangeSelectors = getOption(options, LiteralStringRef("numGetRangeSelectors"), 100);
-		numGetKeys = getOption(options, LiteralStringRef("numGetKeys"), 100);
-		numClears = getOption(options, LiteralStringRef("numClears"), 100);
-		numClearRanges = getOption(options, LiteralStringRef("numClearRanges"), 100);
-		minSizeAfterClear = getOption(options, LiteralStringRef("minSizeAfterClear"), (int)(0.1 * numKeys));
+		numGets = getOption(options, "numGets"_sr, 1000);
+		numGetRanges = getOption(options, "numGetRanges"_sr, 100);
+		numGetRangeSelectors = getOption(options, "numGetRangeSelectors"_sr, 100);
+		numGetKeys = getOption(options, "numGetKeys"_sr, 100);
+		numClears = getOption(options, "numClears"_sr, 100);
+		numClearRanges = getOption(options, "numClearRanges"_sr, 100);
+		minSizeAfterClear = getOption(options, "minSizeAfterClear"_sr, (int)(0.1 * numKeys));
 
-		maxRandomTestKeys = getOption(options, LiteralStringRef("maxRandomTestKeys"), numKeys);
-		randomTestDuration = getOption(options, LiteralStringRef("randomTestDuration"), 60.0);
+		maxRandomTestKeys = getOption(options, "maxRandomTestKeys"_sr, numKeys);
+		randomTestDuration = getOption(options, "randomTestDuration"_sr, 60.0);
 
-		int maxTransactionBytes = getOption(options, LiteralStringRef("maxTransactionBytes"), 500000);
+		int maxTransactionBytes = getOption(options, "maxTransactionBytes"_sr, 500000);
 		maxKeysPerTransaction = std::max(1, maxTransactionBytes / (maxValueLength + maxLongKeyLength));
 
-		resetDBTimeout = getOption(options, LiteralStringRef("resetDBTimeout"), 1800.0);
+		resetDBTimeout = getOption(options, "resetDBTimeout"_sr, 1800.0);
 
 		if (maxTransactionBytes > 500000) {
 			TraceEvent("RemapEventSeverity")
@@ -502,7 +502,7 @@ public:
 
 				if (keys[i].startsWith(StringRef(self->clientPrefix)) ||
 				    (keys[i].size() == 0 && self->clientPrefixInt == 0) ||
-				    (keys[i].startsWith(LiteralStringRef("\xff")) && self->clientPrefixInt == self->clientCount - 1)) {
+				    (keys[i].startsWith("\xff"_sr) && self->clientPrefixInt == self->clientCount - 1)) {
 					break;
 				}
 
@@ -553,7 +553,7 @@ public:
 				if (endKey == self->store.endKey()) {
 					for (int i = 0; i < range.size(); i++) {
 						// Don't include results in the 0xFF key-space
-						if (!range[i].key.startsWith(LiteralStringRef("\xff")))
+						if (!range[i].key.startsWith("\xff"_sr))
 							dbResults.push_back_deep(dbResults.arena(), range[i]);
 					}
 					if (reverse && dbResults.size() < storeResults.size()) {
diff --git a/fdbserver/workloads/ApiWorkload.actor.cpp b/fdbserver/workloads/ApiWorkload.actor.cpp
index e19b3d05dc..326e72c6ca 100644
--- a/fdbserver/workloads/ApiWorkload.actor.cpp
+++ b/fdbserver/workloads/ApiWorkload.actor.cpp
@@ -307,7 +307,7 @@ ACTOR Future<Void> chooseTransactionFactory(Database cx, std::vector<Transaction
 		    new TransactionFactory<ThreadTransactionWrapper, Reference<IDatabase>>(dbHandle, dbHandle, false));
 	} else if (transactionType == MULTI_VERSION) {
 		printf("client %d: Running Multi-Version Transactions\n", self->clientPrefixInt);
-		MultiVersionApi::api->selectApiVersion(cx->apiVersion);
+		MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
 		Reference<IDatabase> threadSafeHandle =
 		    wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
 		Reference<IDatabase> dbHandle = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
diff --git a/fdbserver/workloads/AsyncFile.cpp b/fdbserver/workloads/AsyncFile.cpp
index 96c7c239e1..0dae37423f 100644
--- a/fdbserver/workloads/AsyncFile.cpp
+++ b/fdbserver/workloads/AsyncFile.cpp
@@ -62,11 +62,11 @@ const int AsyncFileWorkload::_PAGE_SIZE = 4096;
 AsyncFileWorkload::AsyncFileWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), fileHandle(nullptr) {
 	// Only run on one client
 	enabled = clientId == 0;
-	testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-	unbufferedIO = getOption(options, LiteralStringRef("unbufferedIO"), false);
-	uncachedIO = getOption(options, LiteralStringRef("uncachedIO"), false);
-	fillRandom = getOption(options, LiteralStringRef("fillRandom"), false);
-	path = getOption(options, LiteralStringRef("fileName"), LiteralStringRef("")).toString();
+	testDuration = getOption(options, "testDuration"_sr, 10.0);
+	unbufferedIO = getOption(options, "unbufferedIO"_sr, false);
+	uncachedIO = getOption(options, "uncachedIO"_sr, false);
+	fillRandom = getOption(options, "fillRandom"_sr, false);
+	path = getOption(options, "fileName"_sr, ""_sr).toString();
 }
 
 Reference<AsyncFileBuffer> AsyncFileWorkload::allocateBuffer(size_t size) {
diff --git a/fdbserver/workloads/AsyncFileCorrectness.actor.cpp b/fdbserver/workloads/AsyncFileCorrectness.actor.cpp
index 3eff9d3a05..5dd5137819 100644
--- a/fdbserver/workloads/AsyncFileCorrectness.actor.cpp
+++ b/fdbserver/workloads/AsyncFileCorrectness.actor.cpp
@@ -75,9 +75,9 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 
 	AsyncFileCorrectnessWorkload(WorkloadContext const& wcx)
 	  : AsyncFileWorkload(wcx), memoryFile(nullptr), success(true), numOperations("Num Operations") {
-		maxOperationSize = getOption(options, LiteralStringRef("maxOperationSize"), 4096);
-		numSimultaneousOperations = getOption(options, LiteralStringRef("numSimultaneousOperations"), 10);
-		targetFileSize = getOption(options, LiteralStringRef("targetFileSize"), (uint64_t)163840);
+		maxOperationSize = getOption(options, "maxOperationSize"_sr, 4096);
+		numSimultaneousOperations = getOption(options, "numSimultaneousOperations"_sr, 10);
+		targetFileSize = getOption(options, "targetFileSize"_sr, (uint64_t)163840);
 
 		if (unbufferedIO)
 			maxOperationSize = std::max(_PAGE_SIZE, maxOperationSize);
@@ -377,7 +377,7 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 			}
 		} else if (info.operation == WRITE) {
 			info.data = self->allocateBuffer(info.length);
-			generateRandomData(reinterpret_cast<uint8_t*>(info.data->buffer), info.length);
+			deterministicRandom()->randomBytes(reinterpret_cast<uint8_t*>(info.data->buffer), info.length);
 			memcpy(&self->memoryFile->buffer[info.offset], info.data->buffer, info.length);
 			memset(&self->fileValidityMask[info.offset], 0xFF, info.length);
 
diff --git a/fdbserver/workloads/AsyncFileRead.actor.cpp b/fdbserver/workloads/AsyncFileRead.actor.cpp
index f4321ca2f8..17267adf6c 100644
--- a/fdbserver/workloads/AsyncFileRead.actor.cpp
+++ b/fdbserver/workloads/AsyncFileRead.actor.cpp
@@ -173,15 +173,14 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload {
 
 	AsyncFileReadWorkload(WorkloadContext const& wcx) : AsyncFileWorkload(wcx), bytesRead("Bytes Read"), ioLog(0) {
 		// Only run on one client
-		numParallelReads = getOption(options, LiteralStringRef("numParallelReads"), 0);
-		readSize = getOption(options, LiteralStringRef("readSize"), _PAGE_SIZE);
-		fileSize =
-		    getOption(options, LiteralStringRef("fileSize"), (int64_t)0); // 0 = use existing, else, change file size
-		unbatched = getOption(options, LiteralStringRef("unbatched"), false);
-		sequential = getOption(options, LiteralStringRef("sequential"), true);
-		writeFraction = getOption(options, LiteralStringRef("writeFraction"), 0.0);
-		randomData = getOption(options, LiteralStringRef("randomData"), true);
-		fixedRate = getOption(options, LiteralStringRef("fixedRate"), 0.0);
+		numParallelReads = getOption(options, "numParallelReads"_sr, 0);
+		readSize = getOption(options, "readSize"_sr, _PAGE_SIZE);
+		fileSize = getOption(options, "fileSize"_sr, (int64_t)0); // 0 = use existing, else, change file size
+		unbatched = getOption(options, "unbatched"_sr, false);
+		sequential = getOption(options, "sequential"_sr, true);
+		writeFraction = getOption(options, "writeFraction"_sr, 0.0);
+		randomData = getOption(options, "randomData"_sr, true);
+		fixedRate = getOption(options, "fixedRate"_sr, 0.0);
 	}
 
 	~AsyncFileReadWorkload() override {}
diff --git a/fdbserver/workloads/AsyncFileWrite.actor.cpp b/fdbserver/workloads/AsyncFileWrite.actor.cpp
index fe6a1448ac..45352c946d 100644
--- a/fdbserver/workloads/AsyncFileWrite.actor.cpp
+++ b/fdbserver/workloads/AsyncFileWrite.actor.cpp
@@ -47,10 +47,10 @@ struct AsyncFileWriteWorkload : public AsyncFileWorkload {
 
 	AsyncFileWriteWorkload(WorkloadContext const& wcx)
 	  : AsyncFileWorkload(wcx), writeBuffer(nullptr), bytesWritten("Bytes Written") {
-		numParallelWrites = getOption(options, LiteralStringRef("numParallelWrites"), 0);
-		writeSize = getOption(options, LiteralStringRef("writeSize"), _PAGE_SIZE);
-		fileSize = getOption(options, LiteralStringRef("fileSize"), 10002432);
-		sequential = getOption(options, LiteralStringRef("sequential"), true);
+		numParallelWrites = getOption(options, "numParallelWrites"_sr, 0);
+		writeSize = getOption(options, "writeSize"_sr, _PAGE_SIZE);
+		fileSize = getOption(options, "fileSize"_sr, 10002432);
+		sequential = getOption(options, "sequential"_sr, true);
 	}
 
 	std::string description() const override { return "AsyncFileWrite"; }
diff --git a/fdbserver/workloads/AtomicOps.actor.cpp b/fdbserver/workloads/AtomicOps.actor.cpp
index 2cc2f1020f..cbde2560d8 100644
--- a/fdbserver/workloads/AtomicOps.actor.cpp
+++ b/fdbserver/workloads/AtomicOps.actor.cpp
@@ -39,11 +39,11 @@ struct AtomicOpsWorkload : TestWorkload {
 	uint64_t lbsum, ubsum; // The lower bound and upper bound sum of operations when opType = AddValue
 
 	AtomicOpsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), opNum(0) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 600.0);
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 5000.0) / clientCount;
-		actorCount = getOption(options, LiteralStringRef("actorsPerClient"), transactionsPerSecond / 5);
-		opType = getOption(options, LiteralStringRef("opType"), -1);
-		nodeCount = getOption(options, LiteralStringRef("nodeCount"), 1000);
+		testDuration = getOption(options, "testDuration"_sr, 600.0);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0) / clientCount;
+		actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5);
+		opType = getOption(options, "opType"_sr, -1);
+		nodeCount = getOption(options, "nodeCount"_sr, 1000);
 		// Atomic OPs Min and And have modified behavior from api version 510. Hence allowing testing for older version
 		// (500) with a 10% probability Actual change of api Version happens in setup
 		apiVersion500 = ((sharedRandomNumber % 10) == 0);
@@ -58,47 +58,47 @@ struct AtomicOpsWorkload : TestWorkload {
 
 		switch (opType) {
 		case 0:
-			TEST(true); // Testing atomic AddValue
+			CODE_PROBE(true, "Testing atomic AddValue");
 			opType = MutationRef::AddValue;
 			break;
 		case 1:
-			TEST(true); // Testing atomic And
+			CODE_PROBE(true, "Testing atomic And");
 			opType = MutationRef::And;
 			break;
 		case 2:
-			TEST(true); // Testing atomic Or
+			CODE_PROBE(true, "Testing atomic Or");
 			opType = MutationRef::Or;
 			break;
 		case 3:
-			TEST(true); // Testing atomic Xor
+			CODE_PROBE(true, "Testing atomic Xor");
 			opType = MutationRef::Xor;
 			break;
 		case 4:
-			TEST(true); // Testing atomic Max
+			CODE_PROBE(true, "Testing atomic Max");
 			opType = MutationRef::Max;
 			break;
 		case 5:
-			TEST(true); // Testing atomic Min
+			CODE_PROBE(true, "Testing atomic Min");
 			opType = MutationRef::Min;
 			break;
 		case 6:
-			TEST(true); // Testing atomic ByteMin
+			CODE_PROBE(true, "Testing atomic ByteMin");
 			opType = MutationRef::ByteMin;
 			break;
 		case 7:
-			TEST(true); // Testing atomic ByteMax
+			CODE_PROBE(true, "Testing atomic ByteMax");
 			opType = MutationRef::ByteMax;
 			break;
 		case 8:
-			TEST(true); // Testing atomic MinV2
+			CODE_PROBE(true, "Testing atomic MinV2");
 			opType = MutationRef::MinV2;
 			break;
 		case 9:
-			TEST(true); // Testing atomic AndV2
+			CODE_PROBE(true, "Testing atomic AndV2");
 			opType = MutationRef::AndV2;
 			break;
 		// case 10:
-		// 	TEST(true); // Testing atomic CompareAndClear Not supported yet
+		// 	CODE_PROBE(true, "Testing atomic CompareAndClear Not supported yet");
 		// 	opType = MutationRef::CompareAndClear
 		//  break;
 		default:
@@ -111,7 +111,7 @@ struct AtomicOpsWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override {
 		if (apiVersion500)
-			cx->apiVersion = 500;
+			cx->apiVersion = ApiVersion(500);
 
 		if (clientId != 0)
 			return Void();
@@ -319,7 +319,7 @@ struct AtomicOpsWorkload : TestWorkload {
 		for (auto& kv : log) {
 			uint64_t intValue = 0;
 			memcpy(&intValue, kv.value.begin(), kv.value.size());
-			logVal[kv.key.removePrefix(LiteralStringRef("log")).withPrefix(LiteralStringRef("debug"))] = intValue;
+			logVal[kv.key.removePrefix("log"_sr).withPrefix("debug"_sr)] = intValue;
 		}
 
 		// Get opsKeys and validate if it has correct value
@@ -372,12 +372,11 @@ struct AtomicOpsWorkload : TestWorkload {
 						RangeResult log_ = wait(tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY));
 						log = log_;
 						uint64_t zeroValue = 0;
-						tr.set(LiteralStringRef("xlogResult"),
-						       StringRef((const uint8_t*)&zeroValue, sizeof(zeroValue)));
+						tr.set("xlogResult"_sr, StringRef((const uint8_t*)&zeroValue, sizeof(zeroValue)));
 						for (auto& kv : log) {
 							uint64_t intValue = 0;
 							memcpy(&intValue, kv.value.begin(), kv.value.size());
-							tr.atomicOp(LiteralStringRef("xlogResult"), kv.value, self->opType);
+							tr.atomicOp("xlogResult"_sr, kv.value, self->opType);
 						}
 					}
 
@@ -386,18 +385,16 @@ struct AtomicOpsWorkload : TestWorkload {
 						Key begin(format("ops%08x", g));
 						RangeResult ops = wait(tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY));
 						uint64_t zeroValue = 0;
-						tr.set(LiteralStringRef("xopsResult"),
-						       StringRef((const uint8_t*)&zeroValue, sizeof(zeroValue)));
+						tr.set("xopsResult"_sr, StringRef((const uint8_t*)&zeroValue, sizeof(zeroValue)));
 						for (auto& kv : ops) {
 							uint64_t intValue = 0;
 							memcpy(&intValue, kv.value.begin(), kv.value.size());
-							tr.atomicOp(LiteralStringRef("xopsResult"), kv.value, self->opType);
+							tr.atomicOp("xopsResult"_sr, kv.value, self->opType);
 						}
 
-						if (tr.get(LiteralStringRef("xlogResult")).get() !=
-						    tr.get(LiteralStringRef("xopsResult")).get()) {
-							Optional<Standalone<StringRef>> logResult = tr.get(LiteralStringRef("xlogResult")).get();
-							Optional<Standalone<StringRef>> opsResult = tr.get(LiteralStringRef("xopsResult")).get();
+						if (tr.get("xlogResult"_sr).get() != tr.get("xopsResult"_sr).get()) {
+							Optional<Standalone<StringRef>> logResult = tr.get("xlogResult"_sr).get();
+							Optional<Standalone<StringRef>> opsResult = tr.get("xopsResult"_sr).get();
 							ASSERT(logResult.present());
 							ASSERT(opsResult.present());
 							TraceEvent(SevError, "LogMismatch")
@@ -408,7 +405,7 @@ struct AtomicOpsWorkload : TestWorkload {
 
 						if (self->opType == MutationRef::AddValue) {
 							uint64_t opsResult = 0;
-							Key opsResultStr = tr.get(LiteralStringRef("xopsResult")).get().get();
+							Key opsResultStr = tr.get("xopsResult"_sr).get().get();
 							memcpy(&opsResult, opsResultStr.begin(), opsResultStr.size());
 							uint64_t logResult = 0;
 							for (auto& kv : log) {
diff --git a/fdbserver/workloads/AtomicOpsApiCorrectness.actor.cpp b/fdbserver/workloads/AtomicOpsApiCorrectness.actor.cpp
index 4acdb7eddc..bd10514d5d 100644
--- a/fdbserver/workloads/AtomicOpsApiCorrectness.actor.cpp
+++ b/fdbserver/workloads/AtomicOpsApiCorrectness.actor.cpp
@@ -29,9 +29,9 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 	uint32_t opType;
 
 private:
-	static int getApiVersion(const Database& cx) { return cx->apiVersion; }
+	static int getApiVersion(const Database& cx) { return cx->apiVersion.version(); }
 
-	static void setApiVersion(Database* cx, int version) { (*cx)->apiVersion = version; }
+	static void setApiVersion(Database* cx, int version) { (*cx)->apiVersion = ApiVersion(version); }
 
 	Key getTestKey(std::string prefix) {
 		std::string key = prefix + std::to_string(clientId);
@@ -40,7 +40,7 @@ private:
 
 public:
 	AtomicOpsApiCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		opType = getOption(options, LiteralStringRef("opType"), -1);
+		opType = getOption(options, "opType"_sr, -1);
 	}
 
 	std::string description() const override { return "AtomicOpsApiCorrectness"; }
@@ -53,31 +53,31 @@ public:
 
 		switch (opType) {
 		case 0:
-			TEST(true); // Testing atomic Min
+			CODE_PROBE(true, "Testing atomic Min");
 			return testMin(cx->clone(), this);
 		case 1:
-			TEST(true); // Testing atomic And
+			CODE_PROBE(true, "Testing atomic And");
 			return testAnd(cx->clone(), this);
 		case 2:
-			TEST(true); // Testing atomic ByteMin
+			CODE_PROBE(true, "Testing atomic ByteMin");
 			return testByteMin(cx->clone(), this);
 		case 3:
-			TEST(true); // Testing atomic ByteMax
+			CODE_PROBE(true, "Testing atomic ByteMax");
 			return testByteMax(cx->clone(), this);
 		case 4:
-			TEST(true); // Testing atomic Or
+			CODE_PROBE(true, "Testing atomic Or");
 			return testOr(cx->clone(), this);
 		case 5:
-			TEST(true); // Testing atomic Max
+			CODE_PROBE(true, "Testing atomic Max");
 			return testMax(cx->clone(), this);
 		case 6:
-			TEST(true); // Testing atomic Xor
+			CODE_PROBE(true, "Testing atomic Xor");
 			return testXor(cx->clone(), this);
 		case 7:
-			TEST(true); // Testing atomic Add
+			CODE_PROBE(true, "Testing atomic Add");
 			return testAdd(cx->clone(), this);
 		case 8:
-			TEST(true); // Testing atomic CompareAndClear
+			CODE_PROBE(true, "Testing atomic CompareAndClear");
 			return testCompareAndClear(cx->clone(), this);
 		default:
 			ASSERT(false);
diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.actor.cpp
index 86d90e1093..d7f8a328be 100644
--- a/fdbserver/workloads/AtomicRestore.actor.cpp
+++ b/fdbserver/workloads/AtomicRestore.actor.cpp
@@ -20,6 +20,7 @@
 
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/RestoreCommon.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
@@ -37,15 +38,20 @@ struct AtomicRestoreWorkload : TestWorkload {
 
 	AtomicRestoreWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 
-		startAfter = getOption(options, LiteralStringRef("startAfter"), 10.0);
-		restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 20.0);
-		fastRestore = getOption(options, LiteralStringRef("fastRestore"), false);
-		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
-		usePartitionedLogs.set(getOption(
-		    options, LiteralStringRef("usePartitionedLogs"), deterministicRandom()->random01() < 0.5 ? true : false));
+		startAfter = getOption(options, "startAfter"_sr, 10.0);
+		restoreAfter = getOption(options, "restoreAfter"_sr, 20.0);
+		fastRestore = getOption(options, "fastRestore"_sr, false);
+		if (!fastRestore) {
+			addDefaultBackupRanges(backupRanges);
+		} else {
+			// Fast restore doesn't support multiple ranges yet
+			backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
+		}
+		usePartitionedLogs.set(
+		    getOption(options, "usePartitionedLogs"_sr, deterministicRandom()->random01() < 0.5 ? true : false));
 
-		addPrefix = getOption(options, LiteralStringRef("addPrefix"), LiteralStringRef(""));
-		removePrefix = getOption(options, LiteralStringRef("removePrefix"), LiteralStringRef(""));
+		addPrefix = getOption(options, "addPrefix"_sr, ""_sr);
+		removePrefix = getOption(options, "removePrefix"_sr, ""_sr);
 
 		// Correctness is not clean for addPrefix feature yet. Uncomment below to enable the test
 		// Generate addPrefix
@@ -81,7 +87,7 @@ struct AtomicRestoreWorkload : TestWorkload {
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	bool hasPrefix() const { return addPrefix != LiteralStringRef("") || removePrefix != LiteralStringRef(""); }
+	bool hasPrefix() const { return addPrefix != ""_sr || removePrefix != ""_sr; }
 
 	ACTOR static Future<Void> _start(Database cx, AtomicRestoreWorkload* self) {
 		state FileBackupAgent backupAgent;
@@ -98,6 +104,7 @@ struct AtomicRestoreWorkload : TestWorkload {
 			                              deterministicRandom()->randomInt(0, 100),
 			                              BackupAgentBase::getDefaultTagName(),
 			                              self->backupRanges,
+			                              false,
 			                              StopWhenDone::False,
 			                              self->usePartitionedLogs));
 		} catch (Error& e) {
@@ -117,17 +124,9 @@ struct AtomicRestoreWorkload : TestWorkload {
 			    cx, BackupAgentBase::getDefaultTag(), self->backupRanges, self->addPrefix, self->removePrefix));
 		} else { // Old style restore
 			loop {
-				std::vector<Future<Version>> restores;
-				if (deterministicRandom()->random01() < 0.5) {
-					for (auto& range : self->backupRanges)
-						restores.push_back(backupAgent.atomicRestore(
-						    cx, BackupAgentBase::getDefaultTag(), range, StringRef(), StringRef()));
-				} else {
-					restores.push_back(backupAgent.atomicRestore(
-					    cx, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef()));
-				}
 				try {
-					wait(waitForAll(restores));
+					wait(success(backupAgent.atomicRestore(
+					    cx, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef())));
 					break;
 				} catch (Error& e) {
 					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
@@ -138,8 +137,8 @@ struct AtomicRestoreWorkload : TestWorkload {
 		}
 
 		// SOMEDAY: Remove after backup agents can exist quiescently
-		if (g_simulator.backupAgents == ISimulator::BackupAgentType::BackupToFile) {
-			g_simulator.backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
+		if (g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) {
+			g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
 		}
 
 		TraceEvent("AtomicRestore_Done").log();
diff --git a/fdbserver/workloads/AtomicSwitchover.actor.cpp b/fdbserver/workloads/AtomicSwitchover.actor.cpp
index 0337da51b4..df31da1294 100644
--- a/fdbserver/workloads/AtomicSwitchover.actor.cpp
+++ b/fdbserver/workloads/AtomicSwitchover.actor.cpp
@@ -33,14 +33,14 @@ struct AtomicSwitchoverWorkload : TestWorkload {
 
 	AtomicSwitchoverWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 
-		switch1delay = getOption(options, LiteralStringRef("switch1delay"), 50.0);
-		switch2delay = getOption(options, LiteralStringRef("switch2delay"), 50.0);
-		stopDelay = getOption(options, LiteralStringRef("stopDelay"), 50.0);
+		switch1delay = getOption(options, "switch1delay"_sr, 50.0);
+		switch2delay = getOption(options, "switch2delay"_sr, 50.0);
+		stopDelay = getOption(options, "stopDelay"_sr, 50.0);
 
-		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
+		addDefaultBackupRanges(backupRanges);
 
-		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-		extraDB = Database::createDatabase(extraFile, -1);
+		ASSERT(g_simulator->extraDatabases.size() == 1);
+		extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], wcx.defaultTenant);
 	}
 
 	std::string description() const override { return "AtomicSwitchover"; }
@@ -191,8 +191,8 @@ struct AtomicSwitchoverWorkload : TestWorkload {
 		TraceEvent("AS_Done").log();
 
 		// SOMEDAY: Remove after backup agents can exist quiescently
-		if (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) {
-			g_simulator.drAgents = ISimulator::BackupAgentType::NoBackupAgents;
+		if (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
+			g_simulator->drAgents = ISimulator::BackupAgentType::NoBackupAgents;
 		}
 
 		return Void();
diff --git a/fdbserver/workloads/BackgroundSelectors.actor.cpp b/fdbserver/workloads/BackgroundSelectors.actor.cpp
index 702b40054a..63ee6511cf 100644
--- a/fdbserver/workloads/BackgroundSelectors.actor.cpp
+++ b/fdbserver/workloads/BackgroundSelectors.actor.cpp
@@ -39,13 +39,12 @@ struct BackgroundSelectorWorkload : TestWorkload {
 
 	BackgroundSelectorWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), operations("Operations"), checks("Checks"), retries("Retries") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		actorsPerClient = std::max(getOption(options, LiteralStringRef("actorsPerClient"), 1), 1);
-		maxDiff = std::max(getOption(options, LiteralStringRef("maxDiff"), 100), 2);
-		minDrift = getOption(options, LiteralStringRef("minDiff"), -10);
-		maxDrift = getOption(options, LiteralStringRef("minDiff"), 100);
-		transactionsPerSecond =
-		    getOption(options, LiteralStringRef("transactionsPerSecond"), 10.0) / (clientCount * actorsPerClient);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		actorsPerClient = std::max(getOption(options, "actorsPerClient"_sr, 1), 1);
+		maxDiff = std::max(getOption(options, "maxDiff"_sr, 100), 2);
+		minDrift = getOption(options, "minDiff"_sr, -10);
+		maxDrift = getOption(options, "minDiff"_sr, 100);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 10.0) / (clientCount * actorsPerClient);
 		resultLimit = 10 * maxDiff;
 	}
 
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index c1bcd5c5d0..9d8d9cbeba 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -51,34 +51,33 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 
 	BackupAndParallelRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
-		backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0);
-		restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 35.0);
-		performRestore = getOption(options, LiteralStringRef("performRestore"), true);
-		backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
-		backupRangesCount = getOption(options, LiteralStringRef("backupRangesCount"), 5);
-		backupRangeLengthMax = getOption(options, LiteralStringRef("backupRangeLengthMax"), 1);
+		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
+		restoreAfter = getOption(options, "restoreAfter"_sr, 35.0);
+		performRestore = getOption(options, "performRestore"_sr, true);
+		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
+		backupRangesCount = getOption(options, "backupRangesCount"_sr, 5);
+		backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1);
 		abortAndRestartAfter =
 		    getOption(options,
-		              LiteralStringRef("abortAndRestartAfter"),
+		              "abortAndRestartAfter"_sr,
 		              deterministicRandom()->random01() < 0.5
 		                  ? deterministicRandom()->random01() * (restoreAfter - backupAfter) + backupAfter
 		                  : 0.0);
-		differentialBackup = getOption(
-		    options, LiteralStringRef("differentialBackup"), deterministicRandom()->random01() < 0.5 ? true : false);
+		differentialBackup =
+		    getOption(options, "differentialBackup"_sr, deterministicRandom()->random01() < 0.5 ? true : false);
 		stopDifferentialAfter =
 		    getOption(options,
-		              LiteralStringRef("stopDifferentialAfter"),
+		              "stopDifferentialAfter"_sr,
 		              differentialBackup ? deterministicRandom()->random01() *
 		                                           (restoreAfter - std::max(abortAndRestartAfter, backupAfter)) +
 		                                       std::max(abortAndRestartAfter, backupAfter)
 		                                 : 0.0);
-		agentRequest = getOption(options, LiteralStringRef("simBackupAgents"), true);
-		allowPauses = getOption(options, LiteralStringRef("allowPauses"), true);
-		shareLogRange = getOption(options, LiteralStringRef("shareLogRange"), false);
-		usePartitionedLogs.set(
-		    getOption(options, LiteralStringRef("usePartitionedLogs"), deterministicRandom()->coinflip()));
-		addPrefix = getOption(options, LiteralStringRef("addPrefix"), LiteralStringRef(""));
-		removePrefix = getOption(options, LiteralStringRef("removePrefix"), LiteralStringRef(""));
+		agentRequest = getOption(options, "simBackupAgents"_sr, true);
+		allowPauses = getOption(options, "allowPauses"_sr, true);
+		shareLogRange = getOption(options, "shareLogRange"_sr, false);
+		usePartitionedLogs.set(getOption(options, "usePartitionedLogs"_sr, deterministicRandom()->coinflip()));
+		addPrefix = getOption(options, "addPrefix"_sr, ""_sr);
+		removePrefix = getOption(options, "removePrefix"_sr, ""_sr);
 
 		KeyRef beginRange;
 		KeyRef endRange;
@@ -108,11 +107,10 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 		if (shareLogRange) {
 			bool beforePrefix = sharedRandomNumber & 1;
 			if (beforePrefix)
-				backupRanges.push_back_deep(backupRanges.arena(),
-				                            KeyRangeRef(normalKeys.begin, LiteralStringRef("\xfe\xff\xfe")));
+				backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(normalKeys.begin, "\xfe\xff\xfe"_sr));
 			else
 				backupRanges.push_back_deep(backupRanges.arena(),
-				                            KeyRangeRef(strinc(LiteralStringRef("\x00\x00\x01")), normalKeys.end));
+				                            KeyRangeRef(strinc("\x00\x00\x01"_sr), normalKeys.end));
 		} else if (backupRangesCount <= 0) {
 			backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
 		} else {
@@ -161,7 +159,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 		return _start(cx, this);
 	}
 
-	bool hasPrefix() const { return addPrefix != LiteralStringRef("") || removePrefix != LiteralStringRef(""); }
+	bool hasPrefix() const { return addPrefix != ""_sr || removePrefix != ""_sr; }
 
 	Future<bool> check(Database const& cx) override { return true; }
 
@@ -227,6 +225,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 			                               deterministicRandom()->randomInt(0, 100),
 			                               tag.toString(),
 			                               backupRanges,
+			                               false,
 			                               StopWhenDone{ !stopDifferentialDelay },
 			                               self->usePartitionedLogs));
 		} catch (Error& e) {
@@ -239,7 +238,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 
 		// Stop the differential backup, if enabled
 		if (stopDifferentialDelay) {
-			TEST(!stopDifferentialFuture.isReady()); // Restore starts at specified time - stopDifferential not ready
+			CODE_PROBE(!stopDifferentialFuture.isReady(),
+			           "Restore starts at specified time - stopDifferential not ready");
 			wait(stopDifferentialFuture);
 			TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID)
 			    .detail("Tag", printable(tag))
@@ -479,12 +479,13 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 					// Note the "partitionedLog" must be false, because we change
 					// the configuration to disable backup workers before restore.
 					extraBackup = backupAgent.submitBackup(cx,
-					                                       LiteralStringRef("file://simfdb/backups/"),
+					                                       "file://simfdb/backups/"_sr,
 					                                       {},
 					                                       deterministicRandom()->randomInt(0, 60),
 					                                       deterministicRandom()->randomInt(0, 100),
 					                                       self->backupTag.toString(),
 					                                       self->backupRanges,
+					                                       false,
 					                                       StopWhenDone::True,
 					                                       UsePartitionedLog::False);
 				} catch (Error& e) {
@@ -496,7 +497,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				}
 			}
 
-			TEST(!startRestore.isReady()); // Restore starts at specified time
+			CODE_PROBE(!startRestore.isReady(), "Restore starts at specified time");
 			wait(startRestore);
 
 			if (lastBackupContainer && self->performRestore) {
@@ -765,7 +766,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 			}
 
 			if (displaySystemKeys) {
-				wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
+				wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
 			}
 
 			TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag));
@@ -776,9 +777,9 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 			}
 
 			// SOMEDAY: Remove after backup agents can exist quiescently
-			if ((g_simulator.backupAgents == ISimulator::BackupAgentType::BackupToFile) &&
+			if ((g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) &&
 			    (!BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests)) {
-				g_simulator.backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
+				g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupAndParallelRestoreCorrectness").error(e).GetLastError();
diff --git a/fdbserver/workloads/BackupCorrectness.actor.cpp b/fdbserver/workloads/BackupCorrectness.actor.cpp
index 595eca5470..7f0427387f 100644
--- a/fdbserver/workloads/BackupCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupCorrectness.actor.cpp
@@ -18,40 +18,51 @@
  * limitations under the License.
  */
 
+#include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
+#include "fdbclient/TenantManagement.actor.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
+#include "flow/IRandom.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // A workload which test the correctness of backup and restore process
 struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 	double backupAfter, restoreAfter, abortAndRestartAfter;
+	double minBackupAfter;
 	double backupStartAt, restoreStartAfterBackupFinished, stopDifferentialAfter;
 	Key backupTag;
 	int backupRangesCount, backupRangeLengthMax;
 	bool differentialBackup, performRestore, agentRequest;
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
-	std::vector<std::string> restorePrefixesToInclude;
-	std::vector<Standalone<KeyRangeRef>> skippedRestoreRanges;
+	std::vector<KeyRange> skippedRestoreRanges;
 	Standalone<VectorRef<KeyRangeRef>> restoreRanges;
 	static int backupAgentRequests;
 	LockDB locked{ false };
 	bool allowPauses;
 	bool shareLogRange;
 	bool shouldSkipRestoreRanges;
+	bool enableBackupEncryption;
+	bool defaultBackup;
 	Optional<std::string> encryptionKeyFileName;
 
 	BackupAndRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
 		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
+		double minBackupAfter = getOption(options, "minBackupAfter"_sr, backupAfter);
+		if (backupAfter > minBackupAfter) {
+			backupAfter = deterministicRandom()->random01() * (backupAfter - minBackupAfter) + minBackupAfter;
+		}
 		restoreAfter = getOption(options, "restoreAfter"_sr, 35.0);
 		performRestore = getOption(options, "performRestore"_sr, true);
 		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
 		backupRangesCount = getOption(options, "backupRangesCount"_sr, 5);
 		backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1);
+		enableBackupEncryption = getOption(options, "enableBackupEncryption"_sr, false);
 		abortAndRestartAfter =
 		    getOption(options,
 		              "abortAndRestartAfter"_sr,
@@ -70,16 +81,22 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 		agentRequest = getOption(options, "simBackupAgents"_sr, true);
 		allowPauses = getOption(options, "allowPauses"_sr, true);
 		shareLogRange = getOption(options, "shareLogRange"_sr, false);
-		restorePrefixesToInclude = getOption(options, "restorePrefixesToInclude"_sr, std::vector<std::string>());
+		defaultBackup = getOption(options, "defaultBackup"_sr, false);
+
+		std::vector<std::string> restorePrefixesToInclude =
+		    getOption(options, "restorePrefixesToInclude"_sr, std::vector<std::string>());
+
 		shouldSkipRestoreRanges = deterministicRandom()->random01() < 0.3 ? true : false;
 		if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.1)) {
-			encryptionKeyFileName = "simfdb/test_encryption_key_file";
+			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
 		}
 
 		TraceEvent("BARW_ClientId").detail("Id", wcx.clientId);
 		UID randomID = nondeterministicRandom()->randomUniqueID();
 		TraceEvent("BARW_PerformRestore", randomID).detail("Value", performRestore);
-		if (shareLogRange) {
+		if (defaultBackup) {
+			addDefaultBackupRanges(backupRanges);
+		} else if (shareLogRange) {
 			bool beforePrefix = sharedRandomNumber & 1;
 			if (beforePrefix)
 				backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(normalKeys.begin, "\xfe\xff\xfe"_sr));
@@ -155,7 +172,60 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 
 	std::string description() const override { return "BackupAndRestoreCorrectness"; }
 
-	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> setup(Database const& cx) override {
+		if (clientId != 0) {
+			return Void();
+		}
+
+		return _setup(cx, this);
+	}
+
+	ACTOR Future<Void> _setup(Database cx, BackupAndRestoreCorrectnessWorkload* self) {
+		state bool adjusted = false;
+		state TenantMapEntry entry;
+
+		if (!self->defaultBackup && (cx->defaultTenant.present() || BUGGIFY)) {
+			if (cx->defaultTenant.present()) {
+				wait(store(entry, TenantAPI::getTenant(cx.getReference(), cx->defaultTenant.get())));
+
+				// If we are specifying sub-ranges (or randomly, if backing up normal keys), adjust them to be relative
+				// to the tenant
+				if (self->backupRanges.size() != 1 || self->backupRanges[0] != normalKeys ||
+				    deterministicRandom()->coinflip()) {
+					adjusted = true;
+					Standalone<VectorRef<KeyRangeRef>> modifiedBackupRanges;
+					for (int i = 0; i < self->backupRanges.size(); ++i) {
+						modifiedBackupRanges.push_back_deep(
+						    modifiedBackupRanges.arena(),
+						    self->backupRanges[i].withPrefix(entry.prefix, self->backupRanges.arena()));
+					}
+					self->backupRanges = modifiedBackupRanges;
+				}
+			}
+			for (auto r : getSystemBackupRanges()) {
+				self->backupRanges.push_back_deep(self->backupRanges.arena(), r);
+			}
+
+			if (adjusted) {
+				Standalone<VectorRef<KeyRangeRef>> modifiedRestoreRanges;
+				for (int i = 0; i < self->restoreRanges.size(); ++i) {
+					modifiedRestoreRanges.push_back_deep(
+					    modifiedRestoreRanges.arena(),
+					    self->restoreRanges[i].withPrefix(entry.prefix, self->restoreRanges.arena()));
+				}
+				self->restoreRanges = modifiedRestoreRanges;
+
+				for (int i = 0; i < self->skippedRestoreRanges.size(); ++i) {
+					self->skippedRestoreRanges[i] = self->skippedRestoreRanges[i].withPrefix(entry.prefix);
+				}
+			}
+			for (auto r : getSystemBackupRanges()) {
+				self->restoreRanges.push_back_deep(self->restoreRanges.arena(), r);
+			}
+		}
+
+		return Void();
+	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
@@ -220,6 +290,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 	ACTOR static Future<Void> statusLoop(Database cx, std::string tag) {
 		state FileBackupAgent agent;
 		loop {
+			bool active = wait(agent.checkActive(cx));
+			TraceEvent("BARW_AgentActivityCheck").detail("IsActive", active);
 			std::string status = wait(agent.getStatus(cx, ShowErrors::True, tag));
 			puts(status.c_str());
 			std::string statusJSON = wait(agent.getStatusJSON(cx, tag));
@@ -268,9 +340,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 			                               StringRef(backupContainer),
 			                               {},
 			                               deterministicRandom()->randomInt(0, 60),
-			                               deterministicRandom()->randomInt(0, 100),
+			                               deterministicRandom()->randomInt(0, 2000),
 			                               tag.toString(),
 			                               backupRanges,
+			                               self->enableBackupEncryption && SERVER_KNOBS->ENABLE_ENCRYPTION,
 			                               StopWhenDone{ !stopDifferentialDelay },
 			                               UsePartitionedLog::False,
 			                               IncrementalBackupOnly::False,
@@ -285,7 +358,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 
 		// Stop the differential backup, if enabled
 		if (stopDifferentialDelay) {
-			TEST(!stopDifferentialFuture.isReady()); // Restore starts at specified time - stopDifferential not ready
+			CODE_PROBE(!stopDifferentialFuture.isReady(),
+			           "Restore starts at specified time - stopDifferential not ready");
 			wait(stopDifferentialFuture);
 			TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID)
 			    .detail("Tag", printable(tag))
@@ -523,14 +597,16 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 			if (!self->locked && BUGGIFY) {
 				TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
 				try {
-					extraBackup = backupAgent.submitBackup(cx,
-					                                       "file://simfdb/backups/"_sr,
-					                                       {},
-					                                       deterministicRandom()->randomInt(0, 60),
-					                                       deterministicRandom()->randomInt(0, 100),
-					                                       self->backupTag.toString(),
-					                                       self->backupRanges,
-					                                       StopWhenDone::True);
+					extraBackup =
+					    backupAgent.submitBackup(cx,
+					                             "file://simfdb/backups/"_sr,
+					                             {},
+					                             deterministicRandom()->randomInt(0, 60),
+					                             deterministicRandom()->randomInt(0, 100),
+					                             self->backupTag.toString(),
+					                             self->backupRanges,
+					                             self->enableBackupEncryption && SERVER_KNOBS->ENABLE_ENCRYPTION,
+					                             StopWhenDone::True);
 				} catch (Error& e) {
 					TraceEvent("BARW_SubmitBackup2Exception", randomID)
 					    .error(e)
@@ -540,7 +616,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 				}
 			}
 
-			TEST(!startRestore.isReady()); // Restore starts at specified time
+			CODE_PROBE(!startRestore.isReady(), "Restore starts at specified time");
 			wait(startRestore);
 
 			if (lastBackupContainer && self->performRestore) {
@@ -549,6 +625,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 					    self, cx, &backupAgent, StringRef(lastBackupContainer->getURL()), randomID));
 				}
 				wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					for (auto& kvrange : self->backupRanges)
 						tr->clear(kvrange);
 					return Void();
@@ -645,6 +722,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 						// was even able to start.  Only run a new restore if the previous one was actually aborted.
 						if (rs == FileBackupAgent::ERestoreState::ABORTED) {
 							wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+								tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 								for (auto& range : self->restoreRanges)
 									tr->clear(range);
 								return Void();
@@ -676,6 +754,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 							if (rs == FileBackupAgent::ERestoreState::ABORTED) {
 								wait(
 								    runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+									    tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 									    tr->clear(self->restoreRanges[restoreIndex]);
 									    return Void();
 								    }));
@@ -743,8 +822,6 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 				TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(self->backupTag));
 
 				try {
-					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-
 					// Check the left over tasks
 					// We have to wait for the list to empty since an abort and get status
 					// can leave extra tasks in the queue
@@ -761,7 +838,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 						wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef()));
 					}
 
-					loop {
+					while (taskCount > 0) {
 						waitCycles++;
 
 						TraceEvent("BARW_NonzeroTaskWait", randomID)
@@ -775,22 +852,9 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 						       (long long)taskCount);
 
 						wait(delay(5.0));
+
 						tr = makeReference<ReadYourWritesTransaction>(cx);
-						int64_t _taskCount = wait(backupAgent.getTaskCount(tr));
-						taskCount = _taskCount;
-
-						if (!taskCount) {
-							break;
-						}
-					}
-
-					if (taskCount) {
-						displaySystemKeys++;
-						TraceEvent(SevError, "BARW_NonzeroTaskCount", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
-						    .detail("TaskCount", taskCount)
-						    .detail("WaitCycles", waitCycles);
-						printf("BackupCorrectnessLeftOverLogTasks: %ld\n", (long)taskCount);
+						wait(store(taskCount, backupAgent.getTaskCount(tr)));
 					}
 
 					RangeResult agentValues =
@@ -868,9 +932,9 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 			}
 
 			// SOMEDAY: Remove after backup agents can exist quiescently
-			if ((g_simulator.backupAgents == ISimulator::BackupAgentType::BackupToFile) &&
+			if ((g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) &&
 			    (!BackupAndRestoreCorrectnessWorkload::backupAgentRequests)) {
-				g_simulator.backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
+				g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupAndRestoreCorrectness").error(e).GetLastError();
@@ -882,5 +946,9 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 
 int BackupAndRestoreCorrectnessWorkload::backupAgentRequests = 0;
 
+std::string getTestEncryptionFileName() {
+	return "test_encryption_key_file";
+}
+
 WorkloadFactory<BackupAndRestoreCorrectnessWorkload> BackupAndRestoreCorrectnessWorkloadFactory(
     "BackupAndRestoreCorrectness");
diff --git a/fdbserver/workloads/BackupToBlob.actor.cpp b/fdbserver/workloads/BackupToBlob.actor.cpp
index 480ae62466..382256ee3d 100644
--- a/fdbserver/workloads/BackupToBlob.actor.cpp
+++ b/fdbserver/workloads/BackupToBlob.actor.cpp
@@ -21,6 +21,7 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/workloads/BlobStoreWorkload.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -35,15 +36,12 @@ struct BackupToBlobWorkload : TestWorkload {
 	static constexpr const char* DESCRIPTION = "BackupToBlob";
 
 	BackupToBlobWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0);
-		backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
-		auto backupURLString =
-		    getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000")).toString();
-		auto accessKeyEnvVar =
-		    getOption(options, LiteralStringRef("accessKeyVar"), LiteralStringRef("BLOB_ACCESS_KEY")).toString();
-		auto secretKeyEnvVar =
-		    getOption(options, LiteralStringRef("secretKeyVar"), LiteralStringRef("BLOB_SECRET_KEY")).toString();
-		bool provideKeys = getOption(options, LiteralStringRef("provideKeys"), false);
+		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
+		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
+		auto backupURLString = getOption(options, "backupURL"_sr, "http://0.0.0.0:10000"_sr).toString();
+		auto accessKeyEnvVar = getOption(options, "accessKeyVar"_sr, "BLOB_ACCESS_KEY"_sr).toString();
+		auto secretKeyEnvVar = getOption(options, "secretKeyVar"_sr, "BLOB_SECRET_KEY"_sr).toString();
+		bool provideKeys = getOption(options, "provideKeys"_sr, false);
 		if (provideKeys) {
 			updateBackupURL(backupURLString, accessKeyEnvVar, "<access_key>", secretKeyEnvVar, "<secret_key>");
 		}
@@ -57,7 +55,8 @@ struct BackupToBlobWorkload : TestWorkload {
 	ACTOR static Future<Void> _start(Database cx, BackupToBlobWorkload* self) {
 		state FileBackupAgent backupAgent;
 		state Standalone<VectorRef<KeyRangeRef>> backupRanges;
-		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
+
+		addDefaultBackupRanges(backupRanges);
 
 		wait(delay(self->backupAfter));
 		wait(backupAgent.submitBackup(cx,
@@ -66,7 +65,8 @@ struct BackupToBlobWorkload : TestWorkload {
 		                              self->initSnapshotInterval,
 		                              self->snapshotInterval,
 		                              self->backupTag.toString(),
-		                              backupRanges));
+		                              backupRanges,
+		                              false));
 		EBackupState backupStatus = wait(backupAgent.waitBackup(cx, self->backupTag.toString(), StopWhenDone::True));
 		TraceEvent("BackupToBlob_BackupStatus").detail("Status", BackupAgentBase::getStateText(backupStatus));
 		return Void();
diff --git a/fdbserver/workloads/BackupToDBAbort.actor.cpp b/fdbserver/workloads/BackupToDBAbort.actor.cpp
index a505d8e02e..3fa244e123 100644
--- a/fdbserver/workloads/BackupToDBAbort.actor.cpp
+++ b/fdbserver/workloads/BackupToDBAbort.actor.cpp
@@ -23,6 +23,7 @@
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
+#include "flow/ApiVersion.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct BackupToDBAbort : TestWorkload {
@@ -32,12 +33,12 @@ struct BackupToDBAbort : TestWorkload {
 	UID lockid;
 
 	explicit BackupToDBAbort(const WorkloadContext& wcx) : TestWorkload(wcx) {
-		abortDelay = getOption(options, LiteralStringRef("abortDelay"), 50.0);
+		abortDelay = getOption(options, "abortDelay"_sr, 50.0);
 
-		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
+		addDefaultBackupRanges(backupRanges);
 
-		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-		extraDB = Database::createDatabase(extraFile, -1);
+		ASSERT(g_simulator->extraDatabases.size() == 1);
+		extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], wcx.defaultTenant);
 
 		lockid = UID(0xbeeffeed, 0xdecaf00d);
 	}
@@ -91,8 +92,8 @@ struct BackupToDBAbort : TestWorkload {
 		TraceEvent("BDBA_End").log();
 
 		// SOMEDAY: Remove after backup agents can exist quiescently
-		if (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) {
-			g_simulator.drAgents = ISimulator::BackupAgentType::NoBackupAgents;
+		if (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
+			g_simulator->drAgents = ISimulator::BackupAgentType::NoBackupAgents;
 		}
 
 		return Void();
diff --git a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp b/fdbserver/workloads/BackupToDBCorrectness.actor.cpp
index 415744a677..58bbf36f77 100644
--- a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupToDBCorrectness.actor.cpp
@@ -21,8 +21,10 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
+#include "fdbclient/TenantManagement.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
+#include "flow/ApiVersion.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // A workload which test the correctness of backup and restore process. The
@@ -33,7 +35,6 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 	double backupStartAt, restoreStartAfterBackupFinished, stopDifferentialAfter;
 	Key backupTag, restoreTag;
 	Key backupPrefix, extraPrefix;
-	bool beforePrefix;
 	int backupRangesCount, backupRangeLengthMax;
 	bool differentialBackup, performRestore, agentRequest;
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
@@ -41,63 +42,72 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 	Database extraDB;
 	LockDB locked{ false };
 	bool shareLogRange;
+	bool defaultBackup;
 	UID destUid;
 
 	BackupToDBCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
-		backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0);
-		restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 35.0);
-		performRestore = getOption(options, LiteralStringRef("performRestore"), true);
-		backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
-		restoreTag = getOption(options, LiteralStringRef("restoreTag"), LiteralStringRef("restore"));
-		backupPrefix = getOption(options, LiteralStringRef("backupPrefix"), StringRef());
+		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
+		double minBackupAfter = getOption(options, "minBackupAfter"_sr, backupAfter);
+		if (backupAfter > minBackupAfter) {
+			backupAfter = deterministicRandom()->random01() * (backupAfter - minBackupAfter) + minBackupAfter;
+		}
+		restoreAfter = getOption(options, "restoreAfter"_sr, 35.0);
+		performRestore = getOption(options, "performRestore"_sr, true);
+		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
+		restoreTag = getOption(options, "restoreTag"_sr, "restore"_sr);
+		backupPrefix = getOption(options, "backupPrefix"_sr, StringRef());
 		backupRangesCount = getOption(options,
-		                              LiteralStringRef("backupRangesCount"),
+		                              "backupRangesCount"_sr,
 		                              5); // tests can hangs if set higher than 1 + BACKUP_MAP_KEY_LOWER_LIMIT
-		backupRangeLengthMax = getOption(options, LiteralStringRef("backupRangeLengthMax"), 1);
+		backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1);
 		abortAndRestartAfter =
 		    getOption(options,
-		              LiteralStringRef("abortAndRestartAfter"),
+		              "abortAndRestartAfter"_sr,
 		              (!locked && deterministicRandom()->random01() < 0.5)
 		                  ? deterministicRandom()->random01() * (restoreAfter - backupAfter) + backupAfter
 		                  : 0.0);
-		differentialBackup = getOption(
-		    options, LiteralStringRef("differentialBackup"), deterministicRandom()->random01() < 0.5 ? true : false);
+		differentialBackup =
+		    getOption(options, "differentialBackup"_sr, deterministicRandom()->random01() < 0.5 ? true : false);
 		stopDifferentialAfter =
 		    getOption(options,
-		              LiteralStringRef("stopDifferentialAfter"),
+		              "stopDifferentialAfter"_sr,
 		              differentialBackup ? deterministicRandom()->random01() *
 		                                           (restoreAfter - std::max(abortAndRestartAfter, backupAfter)) +
 		                                       std::max(abortAndRestartAfter, backupAfter)
 		                                 : 0.0);
-		agentRequest = getOption(options, LiteralStringRef("simDrAgents"), true);
-		shareLogRange = getOption(options, LiteralStringRef("shareLogRange"), false);
+		agentRequest = getOption(options, "simDrAgents"_sr, true);
+		shareLogRange = getOption(options, "shareLogRange"_sr, false);
+		defaultBackup = getOption(options, "defaultBackup"_sr, false);
 
 		// Use sharedRandomNumber if shareLogRange is true so that we can ensure backup and DR both backup the same
 		// range
-		beforePrefix = shareLogRange ? (sharedRandomNumber & 1) : (deterministicRandom()->random01() < 0.5);
+		bool beforePrefix = shareLogRange ? (sharedRandomNumber & 1) : (deterministicRandom()->random01() < 0.5);
 
-		if (beforePrefix) {
-			extraPrefix = backupPrefix.withPrefix(LiteralStringRef("\xfe\xff\xfe"));
-			backupPrefix = backupPrefix.withPrefix(LiteralStringRef("\xfe\xff\xff"));
-		} else {
-			extraPrefix = backupPrefix.withPrefix(LiteralStringRef("\x00\x00\x01"));
-			backupPrefix = backupPrefix.withPrefix(LiteralStringRef("\x00\x00\00"));
+		if (!defaultBackup) {
+			if (beforePrefix) {
+				extraPrefix = backupPrefix.withPrefix("\xfe\xff\xfe"_sr);
+				backupPrefix = backupPrefix.withPrefix("\xfe\xff\xff"_sr);
+			} else {
+				extraPrefix = backupPrefix.withPrefix("\x00\x00\x01"_sr);
+				backupPrefix = backupPrefix.withPrefix("\x00\x00\00"_sr);
+			}
+
+			ASSERT(backupPrefix != StringRef());
 		}
 
-		ASSERT(backupPrefix != StringRef());
-
 		KeyRef beginRange;
 		KeyRef endRange;
 		UID randomID = nondeterministicRandom()->randomUniqueID();
 
-		if (shareLogRange) {
+		if (defaultBackup) {
+			addDefaultBackupRanges(backupRanges);
+		} else if (shareLogRange) {
 			if (beforePrefix)
-				backupRanges.push_back_deep(backupRanges.arena(),
-				                            KeyRangeRef(normalKeys.begin, LiteralStringRef("\xfe\xff\xfe")));
+				backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(normalKeys.begin, "\xfe\xff\xfe"_sr));
 			else
 				backupRanges.push_back_deep(backupRanges.arena(),
-				                            KeyRangeRef(strinc(LiteralStringRef("\x00\x00\x01")), normalKeys.end));
+				                            KeyRangeRef(strinc("\x00\x00\x01"_sr), normalKeys.end));
 		} else if (backupRangesCount <= 0) {
 			if (beforePrefix)
 				backupRanges.push_back_deep(backupRanges.arena(),
@@ -128,15 +138,46 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 			}
 		}
 
-		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-		extraDB = Database::createDatabase(extraFile, -1);
+		ASSERT(g_simulator->extraDatabases.size() == 1);
+		extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], wcx.defaultTenant);
 
 		TraceEvent("BARW_Start").detail("Locked", locked);
 	}
 
 	std::string description() const override { return "BackupToDBCorrectness"; }
 
-	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> setup(Database const& cx) override {
+		if (clientId != 0) {
+			return Void();
+		}
+		return _setup(cx, this);
+	}
+
+	ACTOR Future<Void> _setup(Database cx, BackupToDBCorrectnessWorkload* self) {
+		if (!self->defaultBackup && (cx->defaultTenant.present() || BUGGIFY)) {
+			if (cx->defaultTenant.present()) {
+				TenantMapEntry entry = wait(TenantAPI::getTenant(cx.getReference(), cx->defaultTenant.get()));
+
+				// If we are specifying sub-ranges (or randomly, if backing up normal keys), adjust them to be relative
+				// to the tenant
+				if (self->backupRanges.size() != 1 || self->backupRanges[0] != normalKeys ||
+				    deterministicRandom()->coinflip()) {
+					Standalone<VectorRef<KeyRangeRef>> modifiedBackupRanges;
+					for (int i = 0; i < self->backupRanges.size(); ++i) {
+						modifiedBackupRanges.push_back_deep(
+						    modifiedBackupRanges.arena(),
+						    self->backupRanges[i].withPrefix(entry.prefix, self->backupRanges.arena()));
+					}
+					self->backupRanges = modifiedBackupRanges;
+				}
+			}
+			for (auto r : getSystemBackupRanges()) {
+				self->backupRanges.push_back_deep(self->backupRanges.arena(), r);
+			}
+		}
+
+		return Void();
+	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
@@ -327,7 +368,8 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 
 		// Stop the differential backup, if enabled
 		if (stopDifferentialDelay) {
-			TEST(!stopDifferentialFuture.isReady()); // Restore starts at specified time - stopDifferential not ready
+			CODE_PROBE(!stopDifferentialFuture.isReady(),
+			           "Restore starts at specified time - stopDifferential not ready");
 			wait(stopDifferentialFuture);
 			TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID)
 			    .detail("Tag", printable(tag))
@@ -421,9 +463,6 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 			TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(tag));
 
 			try {
-				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-
 				// Check the left over tasks
 				// We have to wait for the list to empty since an abort and get status
 				// can leave extra tasks in the queue
@@ -437,10 +476,10 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 					    .detail("TaskCount", taskCount)
 					    .detail("WaitCycles", waitCycles);
 					printf("EndingNonZeroTasks: %ld\n", (long)taskCount);
-					wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
+					wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
 				}
 
-				loop {
+				while (taskCount > 0) {
 					waitCycles++;
 
 					TraceEvent("BARW_NonzeroTaskWait", randomID)
@@ -455,21 +494,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 
 					wait(delay(5.0));
 					tr = makeReference<ReadYourWritesTransaction>(cx);
-					int64_t _taskCount = wait(backupAgent->getTaskCount(tr));
-					taskCount = _taskCount;
-
-					if (!taskCount) {
-						break;
-					}
-				}
-
-				if (taskCount) {
-					displaySystemKeys++;
-					TraceEvent(SevError, "BARW_NonzeroTaskCount", randomID)
-					    .detail("BackupTag", printable(tag))
-					    .detail("TaskCount", taskCount)
-					    .detail("WaitCycles", waitCycles);
-					printf("BackupCorrectnessLeftoverLogTasks: %ld\n", (long)taskCount);
+					wait(store(taskCount, backupAgent->getTaskCount(tr)));
 				}
 
 				RangeResult agentValues =
@@ -545,7 +570,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 		}
 
 		if (displaySystemKeys) {
-			wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
+			wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
 		}
 		return Void();
 	}
@@ -612,7 +637,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 			state UID logUid = wait(backupAgent.getLogUid(self->extraDB, self->backupTag));
 
 			// Occasionally start yet another backup that might still be running when we restore
-			if (!self->locked && BUGGIFY) {
+			if (!self->locked && self->extraPrefix != self->backupPrefix && BUGGIFY) {
 				TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
 				try {
 					extraBackup = backupAgent.submitBackup(self->extraDB,
@@ -632,7 +657,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 				}
 			}
 
-			TEST(!startRestore.isReady()); // Restore starts at specified time
+			CODE_PROBE(!startRestore.isReady(), "Restore starts at specified time");
 			wait(startRestore);
 
 			if (self->performRestore) {
@@ -733,9 +758,9 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 			}
 
 			// SOMEDAY: Remove after backup agents can exist quiescently
-			if ((g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) &&
+			if ((g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) &&
 			    (!BackupToDBCorrectnessWorkload::drAgentRequests)) {
-				g_simulator.drAgents = ISimulator::BackupAgentType::NoBackupAgents;
+				g_simulator->drAgents = ISimulator::BackupAgentType::NoBackupAgents;
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupAndRestoreCorrectness").error(e);
diff --git a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
index 6ccb691c9d..abf10ca90c 100644
--- a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
+++ b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
@@ -18,12 +18,15 @@
  * limitations under the License.
  */
 
+#include "fdbclient/FDBOptions.g.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
+#include "fdbclient/TenantManagement.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
 #include "fdbclient/ManagementAPI.actor.h"
+#include "flow/ApiVersion.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // A workload which test the correctness of upgrading DR from 5.1 to 5.2
@@ -35,15 +38,15 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 	Database extraDB;
 
 	BackupToDBUpgradeWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		backupAfter = getOption(options, LiteralStringRef("backupAfter"), deterministicRandom()->random01() * 10.0);
-		backupPrefix = getOption(options, LiteralStringRef("backupPrefix"), StringRef());
-		backupRangeLengthMax = getOption(options, LiteralStringRef("backupRangeLengthMax"), 1);
-		stopDifferentialAfter = getOption(options, LiteralStringRef("stopDifferentialAfter"), 60.0);
-		backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
-		restoreTag = getOption(options, LiteralStringRef("restoreTag"), LiteralStringRef("restore"));
-		backupRangesCount = getOption(options, LiteralStringRef("backupRangesCount"), 5);
-		extraPrefix = backupPrefix.withPrefix(LiteralStringRef("\xfe\xff\xfe"));
-		backupPrefix = backupPrefix.withPrefix(LiteralStringRef("\xfe\xff\xff"));
+		backupAfter = getOption(options, "backupAfter"_sr, deterministicRandom()->random01() * 10.0);
+		backupPrefix = getOption(options, "backupPrefix"_sr, StringRef());
+		backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1);
+		stopDifferentialAfter = getOption(options, "stopDifferentialAfter"_sr, 60.0);
+		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
+		restoreTag = getOption(options, "restoreTag"_sr, "restore"_sr);
+		backupRangesCount = getOption(options, "backupRangesCount"_sr, 5);
+		extraPrefix = backupPrefix.withPrefix("\xfe\xff\xfe"_sr);
+		backupPrefix = backupPrefix.withPrefix("\xfe\xff\xff"_sr);
 
 		ASSERT(backupPrefix != StringRef());
 
@@ -76,8 +79,8 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 			}
 		}
 
-		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-		extraDB = Database::createDatabase(extraFile, -1);
+		ASSERT(g_simulator->extraDatabases.size() == 1);
+		extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], wcx.defaultTenant);
 
 		TraceEvent("DRU_Start").log();
 	}
@@ -159,9 +162,6 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 			TraceEvent("DRU_CheckLeftoverkeys").detail("BackupTag", printable(tag));
 
 			try {
-				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-
 				// Check the left over tasks
 				// We have to wait for the list to empty since an abort and get status
 				// can leave extra tasks in the queue
@@ -175,10 +175,10 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 					    .detail("TaskCount", taskCount)
 					    .detail("WaitCycles", waitCycles);
 					printf("EndingNonZeroTasks: %ld\n", (long)taskCount);
-					wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
+					wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
 				}
 
-				loop {
+				while (taskCount > 0) {
 					waitCycles++;
 
 					TraceEvent("DRU_NonzeroTaskWait")
@@ -189,21 +189,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 
 					wait(delay(20.0));
 					tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
-					int64_t _taskCount = wait(backupAgent->getTaskCount(tr));
-					taskCount = _taskCount;
-
-					if (!taskCount) {
-						break;
-					}
-				}
-
-				if (taskCount) {
-					displaySystemKeys++;
-					TraceEvent(SevError, "DRU_NonzeroTaskCount")
-					    .detail("BackupTag", printable(tag))
-					    .detail("TaskCount", taskCount)
-					    .detail("WaitCycles", waitCycles);
-					printf("BackupCorrectnessLeftoverLogTasks: %ld\n", (long)taskCount);
+					wait(store(taskCount, backupAgent->getTaskCount(tr)));
 				}
 
 				RangeResult agentValues =
@@ -279,7 +265,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 		}
 
 		if (displaySystemKeys) {
-			wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
+			wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
 		}
 
 		return Void();
@@ -288,6 +274,28 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 	ACTOR static Future<Void> _setup(Database cx, BackupToDBUpgradeWorkload* self) {
 		state DatabaseBackupAgent backupAgent(cx);
 
+		if (cx->defaultTenant.present() || BUGGIFY) {
+			if (cx->defaultTenant.present()) {
+				TenantMapEntry entry = wait(TenantAPI::getTenant(cx.getReference(), cx->defaultTenant.get()));
+
+				// If we are specifying sub-ranges (or randomly, if backing up normal keys), adjust them to be relative
+				// to the tenant
+				if (self->backupRanges.size() != 1 || self->backupRanges[0] != normalKeys ||
+				    deterministicRandom()->coinflip()) {
+					Standalone<VectorRef<KeyRangeRef>> modifiedBackupRanges;
+					for (int i = 0; i < self->backupRanges.size(); ++i) {
+						modifiedBackupRanges.push_back_deep(
+						    modifiedBackupRanges.arena(),
+						    self->backupRanges[i].withPrefix(entry.prefix, self->backupRanges.arena()));
+					}
+					self->backupRanges = modifiedBackupRanges;
+				}
+			}
+			for (auto r : getSystemBackupRanges()) {
+				self->backupRanges.push_back_deep(self->backupRanges.arena(), r);
+			}
+		}
+
 		try {
 			wait(delay(self->backupAfter));
 
@@ -298,7 +306,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 			wait(b);
 			TraceEvent("DRU_DoBackupWaitEnd").detail("BackupTag", printable(self->backupTag));
 		} catch (Error& e) {
-			TraceEvent(SevError, "BackupToDBUpgradeSetuEerror").error(e);
+			TraceEvent(SevError, "BackupToDBUpgradeSetupError").error(e);
 			throw;
 		}
 
@@ -322,7 +330,9 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 				try {
 					loop {
 						tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+						tr.setOption(FDBTransactionOptions::RAW_ACCESS);
 						tr2.setOption(FDBTransactionOptions::LOCK_AWARE);
+						tr2.setOption(FDBTransactionOptions::RAW_ACCESS);
 						state Future<RangeResult> srcFuture = tr.getRange(KeyRangeRef(begin, range.end), 1000);
 						state Future<RangeResult> bkpFuture =
 						    tr2.getRange(KeyRangeRef(begin, range.end).withPrefix(backupPrefix), 1000);
@@ -517,8 +527,8 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 
 			TraceEvent("DRU_Complete").detail("BackupTag", printable(self->backupTag));
 
-			if (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) {
-				g_simulator.drAgents = ISimulator::BackupAgentType::NoBackupAgents;
+			if (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
+				g_simulator->drAgents = ISimulator::BackupAgentType::NoBackupAgents;
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupAndRestoreCorrectnessError").error(e);
diff --git a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
index b73b135e83..e185592e1e 100644
--- a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
@@ -31,6 +31,7 @@
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/TenantManagement.actor.h"
+#include "fdbclient/Tuple.h"
 #include "fdbserver/BlobGranuleServerCommon.actor.h"
 #include "fdbserver/BlobGranuleValidation.actor.h"
 #include "fdbserver/Knobs.h"
@@ -43,6 +44,7 @@
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 #define BGW_DEBUG true
+#define BGW_TUPLE_KEY_SIZE 2
 
 struct WriteData {
 	Version writeVersion;
@@ -89,6 +91,9 @@ struct ThreadData : ReferenceCounted<ThreadData>, NonCopyable {
 	Promise<Void> firstWriteSuccessful;
 	Version minSuccessfulReadVersion = MAX_VERSION;
 
+	Future<Void> summaryClient;
+	Promise<Void> triggerSummaryComplete;
+
 	// stats
 	int64_t errors = 0;
 	int64_t mismatches = 0;
@@ -111,7 +116,7 @@ struct ThreadData : ReferenceCounted<ThreadData>, NonCopyable {
 
 		nextKeySequential = deterministicRandom()->random01() < 0.5;
 		reuseKeyProb = 0.1 + (deterministicRandom()->random01() * 0.8);
-		targetIDsPerKey = 1 + deterministicRandom()->randomInt(1, 10);
+		targetIDsPerKey = 1 + deterministicRandom()->randomInt(10, 100);
 
 		if (BGW_DEBUG) {
 			fmt::print("Directory {0} initialized with the following parameters:\n", directoryID);
@@ -124,7 +129,44 @@ struct ThreadData : ReferenceCounted<ThreadData>, NonCopyable {
 	}
 
 	// TODO could make keys variable length?
-	Key getKey(uint32_t key, uint32_t id) { return StringRef(format("%08x/%08x", key, id)); }
+	Key getKey(uint32_t key, uint32_t id) {
+		std::stringstream ss;
+		ss << std::setw(32) << std::setfill('0') << id;
+		Standalone<StringRef> str(ss.str());
+		Tuple::UserTypeStr udt(0x41, str);
+		return Tuple::makeTuple((int64_t)key, udt).pack();
+	}
+
+	void validateGranuleBoundary(Key k, Key e, Key lastKey) {
+		if (k == allKeys.begin || k == allKeys.end) {
+			return;
+		}
+
+		// Fully formed tuples are inserted. The expectation is boundaries should be a
+		// sub-tuple of the inserted key.
+		Tuple t = Tuple::unpackUserType(k, true);
+		if (SERVER_KNOBS->BG_KEY_TUPLE_TRUNCATE_OFFSET) {
+			Tuple t2;
+			try {
+				t2 = Tuple::unpackUserType(lastKey);
+			} catch (Error& e) {
+				// Ignore being unable to parse lastKey as it may be a dummy key.
+			}
+
+			if (t2.size() > 0 && t.getInt(0) != t2.getInt(0)) {
+				if (t.size() > BGW_TUPLE_KEY_SIZE - SERVER_KNOBS->BG_KEY_TUPLE_TRUNCATE_OFFSET) {
+					fmt::print("Tenant: {0}, K={1}, E={2}, LK={3}. {4} != {5}\n",
+					           tenant.prefix.printable(),
+					           k.printable(),
+					           e.printable(),
+					           lastKey.printable(),
+					           t.getInt(0),
+					           t2.getInt(0));
+				}
+				ASSERT(t.size() <= BGW_TUPLE_KEY_SIZE - SERVER_KNOBS->BG_KEY_TUPLE_TRUNCATE_OFFSET);
+			}
+		}
+	}
 };
 
 // For debugging mismatches on what data should be and why
@@ -149,6 +191,7 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 
 	// parameters global across all clients
 	int64_t targetByteRate;
+	bool doMergeCheckAtEnd;
 
 	std::vector<Reference<ThreadData>> directories;
 	std::vector<Future<Void>> clients;
@@ -156,12 +199,15 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 
 	BlobGranuleCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		doSetup = !clientId; // only do this on the "first" client
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 120.0);
+		testDuration = getOption(options, "testDuration"_sr, 120.0);
 
 		// randomize global test settings based on shared parameter to get similar workload across tests, but then vary
 		// different parameters within those constraints
 		int64_t randomness = sharedRandomNumber;
 
+		doMergeCheckAtEnd = randomness % 10 == 0;
+		randomness /= 10;
+
 		// randomize between low and high directory count
 		int64_t targetDirectories = 1 + (randomness % 8);
 		randomness /= 8;
@@ -208,13 +254,15 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 			fmt::print("Setting up blob granule range for tenant {0}\n", name.printable());
 		}
 
-		TenantMapEntry entry = wait(TenantAPI::createTenant(cx.getReference(), name));
+		Optional<TenantMapEntry> entry = wait(TenantAPI::createTenant(cx.getReference(), name));
+		ASSERT(entry.present());
 
 		if (BGW_DEBUG) {
-			fmt::print("Set up blob granule range for tenant {0}: {1}\n", name.printable(), entry.prefix.printable());
+			fmt::print(
+			    "Set up blob granule range for tenant {0}: {1}\n", name.printable(), entry.get().prefix.printable());
 		}
 
-		return entry;
+		return entry.get();
 	}
 
 	std::string description() const override { return "BlobGranuleCorrectnessWorkload"; }
@@ -241,6 +289,8 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 			self->directories[directoryIdx]->directoryRange =
 			    KeyRangeRef(tenantEntry.prefix, tenantEntry.prefix.withSuffix(normalKeys.end));
 			tenants.push_back({ self->directories[directoryIdx]->tenantName, tenantEntry });
+			bool _success = wait(cx->blobbifyRange(self->directories[directoryIdx]->directoryRange));
+			ASSERT(_success);
 		}
 		tenantData.addTenants(tenants);
 
@@ -431,18 +481,20 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 		beginVersionByChunk.insert(normalKeys, 0);
 		int beginCollapsed = 0;
 		int beginNotCollapsed = 0;
+		Key lastBeginKey = ""_sr;
 		for (auto& chunk : blob.second) {
+			KeyRange beginVersionRange;
+			if (chunk.tenantPrefix.present()) {
+				beginVersionRange = KeyRangeRef(chunk.keyRange.begin.removePrefix(chunk.tenantPrefix.get()),
+				                                chunk.keyRange.end.removePrefix(chunk.tenantPrefix.get()));
+			} else {
+				beginVersionRange = chunk.keyRange;
+			}
+
 			if (!chunk.snapshotFile.present()) {
 				ASSERT(beginVersion > 0);
 				ASSERT(chunk.snapshotVersion == invalidVersion);
 				beginCollapsed++;
-				KeyRange beginVersionRange;
-				if (chunk.tenantPrefix.present()) {
-					beginVersionRange = KeyRangeRef(chunk.keyRange.begin.removePrefix(chunk.tenantPrefix.get()),
-					                                chunk.keyRange.end.removePrefix(chunk.tenantPrefix.get()));
-				} else {
-					beginVersionRange = chunk.keyRange;
-				}
 
 				beginVersionByChunk.insert(beginVersionRange, beginVersion);
 			} else {
@@ -451,11 +503,15 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 					beginNotCollapsed++;
 				}
 			}
+
+			// Validate boundary alignment.
+			threadData->validateGranuleBoundary(beginVersionRange.begin, beginVersionRange.end, lastBeginKey);
+			lastBeginKey = beginVersionRange.begin;
 		}
-		TEST(beginCollapsed > 0); // BGCorrectness got collapsed request with beginVersion > 0
-		TEST(beginNotCollapsed > 0); // BGCorrectness got un-collapsed request with beginVersion > 0
-		TEST(beginCollapsed > 0 &&
-		     beginNotCollapsed > 0); // BGCorrectness got both collapsed and uncollapsed in the same request!
+		CODE_PROBE(beginCollapsed > 0, "BGCorrectness got collapsed request with beginVersion > 0");
+		CODE_PROBE(beginNotCollapsed > 0, "BGCorrectness got un-collapsed request with beginVersion > 0");
+		CODE_PROBE(beginCollapsed > 0 && beginNotCollapsed > 0,
+		           "BGCorrectness got both collapsed and uncollapsed in the same request!");
 
 		while (checkIt != threadData->keyData.end() && checkIt->first < endKeyExclusive) {
 			uint32_t key = checkIt->first;
@@ -839,6 +895,7 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 		for (auto& it : directories) {
 			// Wait for blob worker to initialize snapshot before starting test for that range
 			Future<Void> start = waitFirstSnapshot(this, cx, it, true);
+			it->summaryClient = validateGranuleSummaries(cx, normalKeys, it->tenantName, it->triggerSummaryComplete);
 			clients.push_back(timeout(writeWorker(this, start, cx, it), testDuration, Void()));
 			clients.push_back(timeout(readWorker(this, start, cx, it), testDuration, Void()));
 		}
@@ -854,8 +911,8 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 		loop {
 			state Transaction tr(cx, threadData->tenantName);
 			try {
-				Standalone<VectorRef<KeyRangeRef>> ranges = wait(tr.getBlobGranuleRanges(normalKeys));
-				ASSERT(ranges.size() >= 1);
+				Standalone<VectorRef<KeyRangeRef>> ranges = wait(tr.getBlobGranuleRanges(normalKeys, 1000000));
+				ASSERT(ranges.size() >= 1 && ranges.size() < 1000000);
 				ASSERT(ranges.front().begin == normalKeys.begin);
 				ASSERT(ranges.back().end == normalKeys.end);
 				for (int i = 0; i < ranges.size() - 1; i++) {
@@ -872,6 +929,9 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 	                                  BlobGranuleCorrectnessWorkload* self,
 	                                  Reference<ThreadData> threadData) {
 
+		if (threadData->triggerSummaryComplete.canBeSet()) {
+			threadData->triggerSummaryComplete.send(Void());
+		}
 		state bool result = true;
 		state int finalRowsValidated;
 		if (threadData->writeVersions.empty()) {
@@ -910,7 +970,7 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 		}
 		wait(self->checkTenantRanges(self, cx, threadData));
 
-		bool initialCheck = result;
+		state bool initialCheck = result;
 		result &= threadData->mismatches == 0 && (threadData->timeTravelTooOld == 0);
 
 		fmt::print("Blob Granule Workload Directory {0} {1}:\n", threadData->directoryID, result ? "passed" : "failed");
@@ -933,6 +993,14 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 		// For some reason simulation is still passing when this fails?.. so assert for now
 		ASSERT(result);
 
+		if (self->clientId == 0 && SERVER_KNOBS->BG_ENABLE_MERGING && self->doMergeCheckAtEnd) {
+			CODE_PROBE(true, "BGCorrectness clearing database and awaiting merge");
+			wait(clearAndAwaitMerge(cx, threadData->directoryRange));
+		}
+
+		// validate that summary completes without error
+		wait(threadData->summaryClient);
+
 		return result;
 	}
 
diff --git a/fdbserver/workloads/BlobGranuleRangesWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleRangesWorkload.actor.cpp
new file mode 100644
index 0000000000..40268caa18
--- /dev/null
+++ b/fdbserver/workloads/BlobGranuleRangesWorkload.actor.cpp
@@ -0,0 +1,678 @@
+/*
+ * BlobGranuleRangesWorkload.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/SystemData.h"
+#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "flow/Arena.h"
+#include "flow/IRandom.h"
+#include "flow/Trace.h"
+#include "flow/Util.h"
+#include "flow/serialize.h"
+#include <cstring>
+#include <limits>
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+#define BGRW_DEBUG true
+
+// FIXME: need to do multiple changes per commit to properly exercise future change feed logic
+// A workload specifically designed to stress the blob range management of the blob manager + blob worker, and test the
+// blob database api functions
+struct BlobGranuleRangesWorkload : TestWorkload {
+	// test settings
+	double testDuration;
+	int operationsPerSecond;
+	int targetRanges;
+	bool sequential;
+	int sequentialGap;
+
+	Future<Void> client;
+	Future<Void> unitClient;
+	bool stopUnitClient;
+
+	int32_t nextKey;
+
+	std::vector<KeyRange> inactiveRanges;
+	std::vector<KeyRange> activeRanges;
+
+	BlobGranuleRangesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		testDuration = getOption(options, "testDuration"_sr, 30.0);
+		operationsPerSecond = getOption(options, "opsPerSecond"_sr, deterministicRandom()->randomInt(1, 100));
+		operationsPerSecond /= clientCount;
+		if (operationsPerSecond <= 0) {
+			operationsPerSecond = 1;
+		}
+
+		int64_t rand = wcx.sharedRandomNumber;
+		targetRanges = deterministicRandom()->randomExp(1, 1 + rand % 10);
+		targetRanges *= (0.8 + (deterministicRandom()->random01() * 0.4));
+		targetRanges /= clientCount;
+		if (targetRanges <= 0) {
+			targetRanges = 1;
+		}
+		rand /= 10;
+
+		sequential = rand % 2;
+		rand /= 2;
+
+		sequentialGap = 1 + rand % 2;
+		rand /= 2;
+
+		nextKey = 10000000 * clientId;
+
+		stopUnitClient = false;
+
+		TraceEvent("BlobGranuleRangesWorkloadInit").detail("TargetRanges", targetRanges);
+	}
+
+	std::string description() const override { return "BlobGranuleRangesWorkload"; }
+	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
+
+	std::string newKey() {
+		if (sequential) {
+			nextKey += sequentialGap;
+			return format("%08x", nextKey);
+		} else {
+			return deterministicRandom()->randomUniqueID().toString();
+		}
+	}
+
+	ACTOR Future<bool> setRange(Database cx, KeyRange range, bool active) {
+		if (active) {
+			bool success = wait(cx->blobbifyRange(range));
+			return success;
+		} else {
+			bool success = wait(cx->unblobbifyRange(range));
+			return success;
+		}
+	}
+
+	ACTOR Future<Void> registerNewRange(Database cx, BlobGranuleRangesWorkload* self) {
+		std::string nextRangeKey = "R_" + self->newKey();
+		state KeyRange range(KeyRangeRef(StringRef(nextRangeKey), strinc(StringRef(nextRangeKey))));
+		if (BGRW_DEBUG) {
+			fmt::print("Registering new range [{0} - {1})\n", range.begin.printable(), range.end.printable());
+		}
+
+		// don't put in active ranges until AFTER set range command succeeds, to avoid checking a range that maybe
+		// wasn't initialized
+		bool success = wait(self->setRange(cx, range, true));
+		ASSERT(success);
+
+		if (BGRW_DEBUG) {
+			fmt::print("Registered new range [{0} - {1})\n", range.begin.printable(), range.end.printable());
+		}
+
+		self->activeRanges.push_back(range);
+		return Void();
+	}
+
+	ACTOR Future<Key> versionedForcePurge(Database cx, KeyRange range, Optional<TenantName> tenantName) {
+		Version rv = deterministicRandom()->coinflip() ? latestVersion : 1;
+		Key purgeKey = wait(cx->purgeBlobGranules(range, rv, tenantName, true));
+
+		return purgeKey;
+	}
+
+	ACTOR Future<Void> unregisterRandomRange(Database cx, BlobGranuleRangesWorkload* self) {
+		int randomRangeIdx = deterministicRandom()->randomInt(0, self->activeRanges.size());
+		state KeyRange range = self->activeRanges[randomRangeIdx];
+		// remove range from active BEFORE committing txn but add to remove AFTER, to avoid checking a range that could
+		// potentially be in either state
+		swapAndPop(&self->activeRanges, randomRangeIdx);
+
+		if (BGRW_DEBUG) {
+			fmt::print("Unregistering new range [{0} - {1})\n", range.begin.printable(), range.end.printable());
+		}
+
+		if (deterministicRandom()->coinflip()) {
+			if (BGRW_DEBUG) {
+				fmt::print("Force purging range before un-registering: [{0} - {1})\n",
+				           range.begin.printable(),
+				           range.end.printable());
+			}
+			Key purgeKey = wait(self->versionedForcePurge(cx, range, {}));
+			wait(cx->waitPurgeGranulesComplete(purgeKey));
+		}
+		bool success = wait(self->setRange(cx, range, false));
+		ASSERT(success);
+
+		if (BGRW_DEBUG) {
+			fmt::print("Unregistered new range [{0} - {1})\n", range.begin.printable(), range.end.printable());
+		}
+
+		self->inactiveRanges.push_back(range);
+
+		return Void();
+	}
+
+	ACTOR Future<Void> _setup(Database cx, BlobGranuleRangesWorkload* self) {
+		// create initial target ranges
+		TraceEvent("BlobGranuleRangesSetup").detail("InitialRanges", self->targetRanges).log();
+		// set up blob granules
+		wait(success(ManagementAPI::changeConfig(cx.getReference(), "blob_granules_enabled=1", true)));
+
+		state int i;
+		std::vector<Future<Void>> createInitialRanges;
+		for (i = 0; i < self->targetRanges; i++) {
+			wait(self->registerNewRange(cx, self));
+		}
+		TraceEvent("BlobGranuleRangesSetupComplete");
+		return Void();
+	}
+
+	Future<Void> start(Database const& cx) override {
+		client = blobGranuleRangesClient(cx->clone(), this);
+		if (clientId == 0) {
+			unitClient = blobGranuleRangesUnitTests(cx->clone(), this);
+		} else {
+			unitClient = Future<Void>(Void());
+		}
+		return delay(testDuration);
+	}
+
+	Future<bool> check(Database const& cx) override {
+		client = Future<Void>();
+		stopUnitClient = true;
+		return _check(cx, this);
+	}
+
+	ACTOR Future<bool> isRangeActive(Database cx, KeyRange range) {
+		Optional<Version> rv;
+		if (deterministicRandom()->coinflip()) {
+			rv = latestVersion;
+		}
+		state Version v = wait(cx->verifyBlobRange(range, rv));
+		return v != invalidVersion;
+	}
+
+	ACTOR Future<Void> checkRange(Database cx, BlobGranuleRangesWorkload* self, KeyRange range, bool isActive) {
+		// Check that a read completes for the range. If not loop around and try again
+		loop {
+			bool completed = wait(self->isRangeActive(cx, range));
+
+			if (completed == isActive) {
+				break;
+			}
+
+			if (BGRW_DEBUG) {
+				fmt::print("CHECK: {0} range [{1} - {2}) failed!\n",
+				           isActive ? "Active" : "Inactive",
+				           range.begin.printable(),
+				           range.end.printable());
+			}
+
+			wait(delay(1.0));
+		}
+
+		Standalone<VectorRef<KeyRangeRef>> blobRanges = wait(cx->listBlobbifiedRanges(range, 1000000));
+		if (isActive) {
+			ASSERT(blobRanges.size() == 1);
+			ASSERT(blobRanges[0].begin <= range.begin);
+			ASSERT(blobRanges[0].end >= range.end);
+		} else {
+			ASSERT(blobRanges.empty());
+		}
+
+		state Transaction tr(cx);
+		loop {
+			try {
+				Standalone<VectorRef<KeyRangeRef>> granules = wait(tr.getBlobGranuleRanges(range, 1000000));
+				if (isActive) {
+					ASSERT(granules.size() >= 1);
+					ASSERT(granules.front().begin <= range.begin);
+					ASSERT(granules.back().end >= range.end);
+					for (int i = 0; i < granules.size() - 1; i++) {
+						ASSERT(granules[i].end == granules[i + 1].begin);
+					}
+				} else {
+					if (BGRW_DEBUG) {
+						fmt::print("Granules for [{0} - {1}) not empty! ({2}):\n",
+						           range.begin.printable(),
+						           range.end.printable(),
+						           granules.size());
+						for (auto& it : granules) {
+							fmt::print("  [{0} - {1})\n", it.begin.printable(), it.end.printable());
+						}
+					}
+					ASSERT(granules.empty());
+				}
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<bool> _check(Database cx, BlobGranuleRangesWorkload* self) {
+		TraceEvent("BlobGranuleRangesCheck")
+		    .detail("ActiveRanges", self->activeRanges.size())
+		    .detail("InactiveRanges", self->inactiveRanges.size())
+		    .log();
+		if (BGRW_DEBUG) {
+			fmt::print("Checking {0} active and {1} inactive ranges\n",
+			           self->activeRanges.size(),
+			           self->inactiveRanges.size());
+		}
+		state std::vector<Future<Void>> checks;
+		for (int i = 0; i < self->activeRanges.size(); i++) {
+			checks.push_back(self->checkRange(cx, self, self->activeRanges[i], true));
+		}
+
+		// FIXME: re-enable! if we don't force purge there are weird races that cause granules to technically still
+		// exist
+		/*for (int i = 0; i < self->inactiveRanges.size(); i++) {
+		    checks.push_back(self->checkRange(cx, self, self->inactiveRanges[i], false));
+		}*/
+		wait(waitForAll(checks));
+		wait(self->unitClient);
+		TraceEvent("BlobGranuleRangesCheckComplete");
+		return true;
+	}
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	ACTOR Future<Void> blobGranuleRangesClient(Database cx, BlobGranuleRangesWorkload* self) {
+		state double last = now();
+		loop {
+			state Future<Void> waitNextOp = poisson(&last, 1.0 / self->operationsPerSecond);
+
+			if (self->activeRanges.empty() || deterministicRandom()->coinflip()) {
+				wait(self->registerNewRange(cx, self));
+			} else {
+				wait(self->unregisterRandomRange(cx, self));
+			}
+
+			wait(waitNextOp);
+		}
+	}
+
+	ACTOR Future<Void> tearDownRangeAfterUnit(Database cx, BlobGranuleRangesWorkload* self, KeyRange range) {
+		if (BGRW_DEBUG) {
+			fmt::print("Tearing down [{0} - {1}) after unit!\n", range.begin.printable(), range.end.printable());
+		}
+
+		// tear down range at end
+		Key purgeKey = wait(self->versionedForcePurge(cx, range, {}));
+		wait(cx->waitPurgeGranulesComplete(purgeKey));
+		bool success = wait(self->setRange(cx, range, false));
+		ASSERT(success);
+
+		if (BGRW_DEBUG) {
+			fmt::print("Range [{0} - {1}) torn down.\n", range.begin.printable(), range.end.printable());
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<Void> verifyRangeUnit(Database cx, BlobGranuleRangesWorkload* self, KeyRange range) {
+		state KeyRange activeRange(KeyRangeRef(range.begin.withSuffix("A"_sr), range.begin.withSuffix("B"_sr)));
+		state Key middleKey = range.begin.withSuffix("AF"_sr);
+
+		if (BGRW_DEBUG) {
+			fmt::print("VerifyRangeUnit: [{0} - {1})\n", range.begin.printable(), range.end.printable());
+		}
+		bool setSuccess = wait(self->setRange(cx, activeRange, true));
+		ASSERT(setSuccess);
+		wait(self->checkRange(cx, self, activeRange, true));
+
+		bool success1 = wait(self->isRangeActive(cx, KeyRangeRef(activeRange.begin, middleKey)));
+		ASSERT(success1);
+
+		bool success2 = wait(self->isRangeActive(cx, KeyRangeRef(middleKey, activeRange.end)));
+		ASSERT(success2);
+
+		bool fail1 = wait(self->isRangeActive(cx, range));
+		ASSERT(!fail1);
+
+		bool fail2 = wait(self->isRangeActive(cx, KeyRangeRef(range.begin, activeRange.begin)));
+		ASSERT(!fail2);
+
+		bool fail3 = wait(self->isRangeActive(cx, KeyRangeRef(activeRange.end, range.end)));
+		ASSERT(!fail3);
+
+		bool fail4 = wait(self->isRangeActive(cx, KeyRangeRef(range.begin, middleKey)));
+		ASSERT(!fail4);
+
+		bool fail5 = wait(self->isRangeActive(cx, KeyRangeRef(middleKey, range.end)));
+		ASSERT(!fail5);
+
+		bool fail6 = wait(self->isRangeActive(cx, KeyRangeRef(range.begin, activeRange.end)));
+		ASSERT(!fail6);
+
+		bool fail7 = wait(self->isRangeActive(cx, KeyRangeRef(activeRange.begin, range.end)));
+		ASSERT(!fail7);
+
+		wait(self->tearDownRangeAfterUnit(cx, self, activeRange));
+
+		return Void();
+	}
+
+	ACTOR Future<Void> verifyRangeGapUnit(Database cx, BlobGranuleRangesWorkload* self, KeyRange range) {
+		state std::vector<Key> boundaries;
+		boundaries.push_back(range.begin);
+		state int rangeCount = deterministicRandom()->randomExp(3, 6) + 1;
+		for (int i = 0; i < rangeCount - 1; i++) {
+			std::string suffix = format("%04x", i);
+			boundaries.push_back(range.begin.withSuffix(suffix));
+		}
+		boundaries.push_back(range.end);
+
+		ASSERT(boundaries.size() - 1 == rangeCount);
+
+		state int rangeToNotBlobbify = deterministicRandom()->randomInt(0, rangeCount);
+		state int i;
+		for (i = 0; i < rangeCount; i++) {
+			state KeyRange subRange(KeyRangeRef(boundaries[i], boundaries[i + 1]));
+			if (i != rangeToNotBlobbify) {
+				bool setSuccess = wait(self->setRange(cx, subRange, true));
+				ASSERT(setSuccess);
+				wait(self->checkRange(cx, self, subRange, true));
+			} else {
+				wait(self->checkRange(cx, self, subRange, false));
+			}
+		}
+
+		bool success = wait(self->isRangeActive(cx, range));
+		ASSERT(!success);
+
+		if (rangeToNotBlobbify != 0) {
+			wait(self->tearDownRangeAfterUnit(cx, self, KeyRangeRef(boundaries[0], boundaries[rangeToNotBlobbify])));
+		}
+		if (rangeToNotBlobbify != rangeCount - 1) {
+			wait(self->tearDownRangeAfterUnit(
+			    cx, self, KeyRangeRef(boundaries[rangeToNotBlobbify + 1], boundaries.back())));
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<Void> checkRangesMisaligned(Database cx,
+	                                         BlobGranuleRangesWorkload* self,
+	                                         KeyRange expectedRange,
+	                                         KeyRange queryRange) {
+		Standalone<VectorRef<KeyRangeRef>> blobRanges = wait(cx->listBlobbifiedRanges(queryRange, 1000000));
+		ASSERT(blobRanges.size() == 1);
+		ASSERT(blobRanges[0] == expectedRange);
+
+		state Transaction tr(cx);
+		loop {
+			try {
+				Standalone<VectorRef<KeyRangeRef>> granules = wait(tr.getBlobGranuleRanges(queryRange, 1000000));
+				ASSERT(granules.size() == 1);
+				ASSERT(granules[0] == expectedRange);
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<Void> rangesMisalignedUnit(Database cx, BlobGranuleRangesWorkload* self, KeyRange range) {
+		bool setSuccess = wait(self->setRange(cx, range, true));
+		ASSERT(setSuccess);
+		state KeyRange subRange(KeyRangeRef(range.begin.withSuffix("A"_sr), range.begin.withSuffix("B"_sr)));
+
+		// validate range set up correctly
+		wait(self->checkRange(cx, self, range, true));
+		wait(self->checkRangesMisaligned(cx, self, range, range));
+
+		// getBlobGranules and getBlobRanges on sub ranges- should return actual granule/range instead of clipped
+		wait(self->checkRange(cx, self, subRange, true));
+		wait(self->checkRangesMisaligned(cx, self, range, subRange));
+		wait(self->checkRangesMisaligned(cx, self, range, KeyRangeRef(range.begin, subRange.end)));
+		wait(self->checkRangesMisaligned(cx, self, range, KeyRangeRef(subRange.begin, range.end)));
+
+		try {
+			wait(success(cx->purgeBlobGranules(subRange, 1, {}, false)));
+			ASSERT(false);
+		} catch (Error& e) {
+			if (e.code() == error_code_operation_cancelled) {
+				throw e;
+			}
+			ASSERT(e.code() == error_code_unsupported_operation);
+		}
+
+		try {
+			wait(success(cx->purgeBlobGranules(subRange, 1, {}, true)));
+			ASSERT(false);
+		} catch (Error& e) {
+			if (e.code() == error_code_operation_cancelled) {
+				throw e;
+			}
+			ASSERT(e.code() == error_code_unsupported_operation);
+		}
+
+		// ensure range still there after unaligned purges
+		wait(self->checkRange(cx, self, range, true));
+		wait(self->checkRangesMisaligned(cx, self, range, range));
+
+		wait(self->tearDownRangeAfterUnit(cx, self, range));
+		return Void();
+	}
+
+	ACTOR Future<Void> blobbifyIdempotentUnit(Database cx, BlobGranuleRangesWorkload* self, KeyRange range) {
+		state KeyRange activeRange(KeyRangeRef(range.begin.withSuffix("A"_sr), range.begin.withSuffix("B"_sr)));
+		state Key middleKey = range.begin.withSuffix("AF"_sr);
+		state Key middleKey2 = range.begin.withSuffix("AG"_sr);
+
+		if (BGRW_DEBUG) {
+			fmt::print("IdempotentUnit: [{0} - {1})\n", range.begin.printable(), range.end.printable());
+		}
+
+		// unblobbifying range that already doesn't exist should be no-op
+		if (deterministicRandom()->coinflip()) {
+			bool unblobbifyStartSuccess = wait(self->setRange(cx, activeRange, false));
+			ASSERT(unblobbifyStartSuccess);
+		}
+
+		bool success = wait(self->setRange(cx, activeRange, true));
+		ASSERT(success);
+		wait(self->checkRange(cx, self, activeRange, true));
+
+		// check that re-blobbifying same range is successful
+		bool retrySuccess = wait(self->setRange(cx, activeRange, true));
+		ASSERT(retrySuccess);
+		wait(self->checkRange(cx, self, activeRange, true));
+
+		// check that blobbifying range that overlaps but does not match existing blob range fails
+		bool fail1 = wait(self->setRange(cx, range, true));
+		ASSERT(!fail1);
+
+		bool fail2 = wait(self->setRange(cx, KeyRangeRef(range.begin, activeRange.end), true));
+		ASSERT(!fail2);
+
+		bool fail3 = wait(self->setRange(cx, KeyRangeRef(activeRange.begin, range.end), true));
+		ASSERT(!fail3);
+
+		bool fail4 = wait(self->setRange(cx, KeyRangeRef(range.begin, middleKey), true));
+		ASSERT(!fail4);
+
+		bool fail5 = wait(self->setRange(cx, KeyRangeRef(middleKey, range.end), true));
+		ASSERT(!fail5);
+
+		bool fail6 = wait(self->setRange(cx, KeyRangeRef(activeRange.begin, middleKey), true));
+		ASSERT(!fail6);
+
+		bool fail7 = wait(self->setRange(cx, KeyRangeRef(middleKey, activeRange.end), true));
+		ASSERT(!fail7);
+
+		bool fail8 = wait(self->setRange(cx, KeyRangeRef(middleKey, middleKey2), true));
+		ASSERT(!fail8);
+
+		{
+			Standalone<VectorRef<KeyRangeRef>> blobRanges = wait(cx->listBlobbifiedRanges(range, 1000000));
+			ASSERT(blobRanges.size() == 1);
+			ASSERT(blobRanges[0] == activeRange);
+
+			state Transaction tr(cx);
+			loop {
+				try {
+					Standalone<VectorRef<KeyRangeRef>> granules = wait(tr.getBlobGranuleRanges(range, 1000000));
+					ASSERT(granules.size() == 1);
+					ASSERT(granules[0] == activeRange);
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+
+			// tear down + check that un-blobbifying at a non-aligned range also doesn't work
+			Key purgeKey = wait(self->versionedForcePurge(cx, activeRange, {}));
+			wait(cx->waitPurgeGranulesComplete(purgeKey));
+
+			if (deterministicRandom()->coinflip()) {
+				// force purge again and ensure it is idempotent
+				Key purgeKeyAgain = wait(cx->purgeBlobGranules(activeRange, 1, {}, true));
+				wait(cx->waitPurgeGranulesComplete(purgeKeyAgain));
+			}
+		}
+
+		// Check that the blob range is still listed
+		{
+			Standalone<VectorRef<KeyRangeRef>> blobRanges = wait(cx->listBlobbifiedRanges(range, 1000000));
+			ASSERT(blobRanges.size() == 1);
+			ASSERT(blobRanges[0] == activeRange);
+
+			bool unblobbifyFail1 = wait(self->setRange(cx, range, false));
+			ASSERT(!unblobbifyFail1);
+
+			bool unblobbifyFail2 = wait(self->setRange(cx, KeyRangeRef(range.begin, activeRange.end), false));
+			ASSERT(!unblobbifyFail2);
+
+			bool unblobbifyFail3 = wait(self->setRange(cx, KeyRangeRef(activeRange.begin, range.end), false));
+			ASSERT(!unblobbifyFail3);
+
+			bool unblobbifyFail4 = wait(self->setRange(cx, KeyRangeRef(activeRange.begin, middleKey), false));
+			ASSERT(!unblobbifyFail4);
+
+			bool unblobbifyFail5 = wait(self->setRange(cx, KeyRangeRef(middleKey, activeRange.end), false));
+			ASSERT(!unblobbifyFail5);
+
+			bool unblobbifyFail6 = wait(self->setRange(cx, KeyRangeRef(activeRange.begin, middleKey), false));
+			ASSERT(!unblobbifyFail6);
+
+			bool unblobbifyFail7 = wait(self->setRange(cx, KeyRangeRef(middleKey, activeRange.end), false));
+			ASSERT(!unblobbifyFail7);
+
+			bool unblobbifyFail8 = wait(self->setRange(cx, KeyRangeRef(middleKey, middleKey2), false));
+			ASSERT(!unblobbifyFail8);
+
+			bool unblobbifySuccess = wait(self->setRange(cx, activeRange, true));
+			ASSERT(unblobbifySuccess);
+
+			bool unblobbifySuccessAgain = wait(self->setRange(cx, activeRange, true));
+			ASSERT(unblobbifySuccessAgain);
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<Void> reBlobbifyUnit(Database cx, BlobGranuleRangesWorkload* self, KeyRange range) {
+		bool setSuccess = wait(self->setRange(cx, range, true));
+		ASSERT(setSuccess);
+		wait(self->checkRange(cx, self, range, true));
+
+		// force purge range
+		Key purgeKey = wait(self->versionedForcePurge(cx, range, {}));
+		wait(cx->waitPurgeGranulesComplete(purgeKey));
+		wait(self->checkRange(cx, self, range, false));
+
+		bool unsetSuccess = wait(self->setRange(cx, range, false));
+		ASSERT(unsetSuccess);
+		wait(self->checkRange(cx, self, range, false));
+
+		bool reSetSuccess = wait(self->setRange(cx, range, true));
+		ASSERT(reSetSuccess);
+		wait(self->checkRange(cx, self, range, true));
+
+		wait(self->tearDownRangeAfterUnit(cx, self, range));
+
+		return Void();
+	}
+
+	enum UnitTestTypes {
+		VERIFY_RANGE_UNIT,
+		VERIFY_RANGE_GAP_UNIT,
+		RANGES_MISALIGNED,
+		BLOBBIFY_IDEMPOTENT,
+		RE_BLOBBIFY,
+		OP_COUNT = 5 /* keep this last */
+	};
+
+	ACTOR Future<Void> blobGranuleRangesUnitTests(Database cx, BlobGranuleRangesWorkload* self) {
+		loop {
+			if (self->stopUnitClient) {
+				return Void();
+			}
+			std::set<UnitTestTypes> excludedTypes;
+			excludedTypes.insert(OP_COUNT);
+
+			// FIXME: fix bugs and enable these tests!
+			excludedTypes.insert(RE_BLOBBIFY); // TODO - fix is non-trivial, is desired behavior eventually
+
+			std::string nextRangeKey = "U_" + self->newKey();
+			state KeyRange range(KeyRangeRef(StringRef(nextRangeKey), strinc(StringRef(nextRangeKey))));
+			// prevent infinite loop
+			int loopTries = 1000;
+			int op = OP_COUNT;
+			loop {
+				op = deterministicRandom()->randomInt(0, OP_COUNT);
+				if (!excludedTypes.count((UnitTestTypes)op)) {
+					break;
+				}
+				loopTries--;
+				ASSERT(loopTries >= 0);
+			}
+			if (BGRW_DEBUG) {
+				fmt::print(
+				    "Selected range [{0} - {1}) for unit {2}.\n", range.begin.printable(), range.end.printable(), op);
+			}
+
+			if (op == VERIFY_RANGE_UNIT) {
+				wait(self->verifyRangeUnit(cx, self, range));
+			} else if (op == VERIFY_RANGE_GAP_UNIT) {
+				wait(self->verifyRangeGapUnit(cx, self, range));
+			} else if (op == RANGES_MISALIGNED) {
+				wait(self->rangesMisalignedUnit(cx, self, range));
+			} else if (op == BLOBBIFY_IDEMPOTENT) {
+				wait(self->blobbifyIdempotentUnit(cx, self, range));
+			} else if (op == RE_BLOBBIFY) {
+				wait(self->reBlobbifyUnit(cx, self, range));
+			} else {
+				ASSERT(false);
+			}
+
+			wait(delay(1.0));
+		}
+	}
+};
+
+WorkloadFactory<BlobGranuleRangesWorkload> BlobGranuleRangesWorkloadFactory("BlobGranuleRanges");
diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
index 8b95d20b93..0575fece6d 100644
--- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp
@@ -29,8 +29,10 @@
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/SystemData.h"
 #include "fdbserver/BlobGranuleValidation.actor.h"
+#include "fdbserver/BlobGranuleServerCommon.actor.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/QuietDatabase.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
@@ -49,8 +51,6 @@
  */
 struct BlobGranuleVerifierWorkload : TestWorkload {
 	bool doSetup;
-	double minDelay;
-	double maxDelay;
 	double testDuration;
 	double timeTravelLimit;
 	uint64_t timeTravelBufferSize;
@@ -65,35 +65,83 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 	int64_t purges = 0;
 	std::vector<Future<Void>> clients;
 	bool enablePurging;
+	bool initAtEnd;
+	bool strictPurgeChecking;
+	bool doForcePurge;
+	bool purgeAtLatest;
+	bool clearAndMergeCheck;
 
 	DatabaseConfiguration config;
 
 	Reference<BlobConnectionProvider> bstore;
 	AsyncVar<Standalone<VectorRef<KeyRangeRef>>> granuleRanges;
 
+	bool startedForcePurge;
+	Optional<Key> forcePurgeKey;
+	Version forcePurgeVersion;
+
+	std::vector<std::tuple<KeyRange, Version, UID, Future<GranuleFiles>>> purgedDataToCheck;
+
+	Future<Void> summaryClient;
+	Promise<Void> triggerSummaryComplete;
+
 	BlobGranuleVerifierWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		doSetup = !clientId; // only do this on the "first" client
-		// FIXME: don't do the delay in setup, as that delays the start of all workloads
-		minDelay = getOption(options, LiteralStringRef("minDelay"), 0.0);
-		maxDelay = getOption(options, LiteralStringRef("maxDelay"), 0.0);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 120.0);
-		timeTravelLimit = getOption(options, LiteralStringRef("timeTravelLimit"), testDuration);
-		timeTravelBufferSize = getOption(options, LiteralStringRef("timeTravelBufferSize"), 100000000);
-		threads = getOption(options, LiteralStringRef("threads"), 1);
-		enablePurging = getOption(options, LiteralStringRef("enablePurging"), false /*sharedRandomNumber % 2 == 0*/);
-		ASSERT(threads >= 1);
+		testDuration = getOption(options, "testDuration"_sr, 120.0);
+		timeTravelLimit = getOption(options, "timeTravelLimit"_sr, testDuration);
+		timeTravelBufferSize = getOption(options, "timeTravelBufferSize"_sr, 100000000);
+		threads = getOption(options, "threads"_sr, 1);
 
-		if (BGV_DEBUG) {
-			printf("Initializing Blob Granule Verifier s3 stuff\n");
+		enablePurging = getOption(options, "enablePurging"_sr, sharedRandomNumber % 3 == 0);
+		sharedRandomNumber /= 3;
+		// FIXME: re-enable this! There exist several bugs with purging active granules where a small amount of state
+		// won't be cleaned up.
+		strictPurgeChecking = getOption(options, "strictPurgeChecking"_sr, false /*sharedRandomNumber % 2 == 0*/);
+		sharedRandomNumber /= 2;
+
+		doForcePurge = getOption(options, "doForcePurge"_sr, sharedRandomNumber % 3 == 0);
+		sharedRandomNumber /= 3;
+
+		purgeAtLatest = getOption(options, "purgeAtLatest"_sr, sharedRandomNumber % 3 == 0);
+		sharedRandomNumber /= 3;
+
+		// randomly some tests write data first and then turn on blob granules later, to test conversion of existing DB
+		initAtEnd = !enablePurging && sharedRandomNumber % 10 == 0;
+		sharedRandomNumber /= 10;
+
+		clearAndMergeCheck = getOption(options, "clearAndMergeCheck"_sr, sharedRandomNumber % 10 == 0);
+		sharedRandomNumber /= 10;
+
+		// don't do strictPurgeChecking or forcePurge if !enablePurging
+		if (!enablePurging) {
+			strictPurgeChecking = false;
+			doForcePurge = false;
+			purgeAtLatest = false;
 		}
+
+		if (doForcePurge) {
+			purgeAtLatest = false;
+		}
+
+		if (purgeAtLatest) {
+			strictPurgeChecking = false;
+		}
+
+		startedForcePurge = false;
+
+		if (doSetup && BGV_DEBUG) {
+			fmt::print("BlobGranuleVerifier starting\n");
+			fmt::print("  enablePurging={0}\n", enablePurging);
+			fmt::print("  purgeAtLatest={0}\n", purgeAtLatest);
+			fmt::print("  strictPurgeChecking={0}\n", strictPurgeChecking);
+			fmt::print("  doForcePurge={0}\n", doForcePurge);
+			fmt::print("  initAtEnd={0}\n", initAtEnd);
+			fmt::print("  clearAndMergeCheck={0}\n", clearAndMergeCheck);
+		}
+
+		ASSERT(threads >= 1);
 		try {
-			if (BGV_DEBUG) {
-				printf("Blob Granule Verifier constructing backup container from %s\n", SERVER_KNOBS->BG_URL.c_str());
-			}
 			bstore = BlobConnectionProvider::newBlobConnectionProvider(SERVER_KNOBS->BG_URL);
-			if (BGV_DEBUG) {
-				printf("Blob Granule Verifier constructed backup container\n");
-			}
 		} catch (Error& e) {
 			if (BGV_DEBUG) {
 				printf("Blob Granule Verifier got backup container init error %s\n", e.name());
@@ -104,15 +152,14 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 
 	// FIXME: run the actual FDBCLI command instead of copy/pasting its implementation
 	// Sets the whole user keyspace to be blobified
-	ACTOR Future<Void> setUpBlobRange(Database cx, Future<Void> waitForStart) {
+	ACTOR Future<Void> setUpBlobRange(Database cx) {
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		wait(waitForStart);
 		loop {
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr->set(blobRangeChangeKey, deterministicRandom()->randomUniqueID().toString());
-				wait(krmSetRange(tr, blobRangeKeys.begin, KeyRange(normalKeys), LiteralStringRef("1")));
+				wait(krmSetRange(tr, blobRangeKeys.begin, KeyRange(normalKeys), "1"_sr));
 				wait(tr->commit());
 				if (BGV_DEBUG) {
 					printf("Successfully set up blob granule range for normalKeys\n");
@@ -136,11 +183,9 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 
 		wait(success(ManagementAPI::changeConfig(cx.getReference(), "blob_granules_enabled=1", true)));
 
-		double initialDelay = deterministicRandom()->random01() * (self->maxDelay - self->minDelay) + self->minDelay;
-		if (BGV_DEBUG) {
-			printf("BGW setup initial delay of %.3f\n", initialDelay);
+		if (!self->initAtEnd) {
+			wait(self->setUpBlobRange(cx));
 		}
-		wait(self->setUpBlobRange(cx, delay(initialDelay)));
 		return Void();
 	}
 
@@ -149,7 +194,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 			state Transaction tr(cx);
 			loop {
 				try {
-					Standalone<VectorRef<KeyRangeRef>> allGranules = wait(tr.getBlobGranuleRanges(normalKeys));
+					Standalone<VectorRef<KeyRangeRef>> allGranules = wait(tr.getBlobGranuleRanges(normalKeys, 1000000));
 					self->granuleRanges.set(allGranules);
 					break;
 				} catch (Error& e) {
@@ -210,6 +255,48 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 		}
 	}
 
+	// TODO refactor more generally
+	ACTOR Future<Void> loadGranuleMetadataBeforeForcePurge(Database cx, BlobGranuleVerifierWorkload* self) {
+		// load all granule history entries that intersect purged range
+		state KeyRange cur = blobGranuleHistoryKeys;
+		state Transaction tr(cx);
+		loop {
+			try {
+				RangeResult history = wait(tr.getRange(cur, 100));
+				for (auto& it : history) {
+					KeyRange keyRange;
+					Version version;
+					std::tie(keyRange, version) = decodeBlobGranuleHistoryKey(it.key);
+					// TODO: filter by key range for partial key range purge
+					Standalone<BlobGranuleHistoryValue> historyValue = decodeBlobGranuleHistoryValue(it.value);
+
+					Future<GranuleFiles> fileFuture = loadHistoryFiles(cx, historyValue.granuleID);
+					self->purgedDataToCheck.push_back({ keyRange, version, historyValue.granuleID, fileFuture });
+				}
+				if (!history.empty() && history.more) {
+					cur = KeyRangeRef(keyAfter(history.back().key), cur.end);
+				} else {
+					break;
+				}
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		// wait for file loads to finish
+		state int i;
+		for (i = 0; i < self->purgedDataToCheck.size(); i++) {
+			wait(success(std::get<3>(self->purgedDataToCheck[i])));
+		}
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV loaded {0} granules metadata before force purge\n", self->purgedDataToCheck.size());
+		}
+
+		ASSERT(!self->purgedDataToCheck.empty());
+		return Void();
+	}
+
 	ACTOR Future<Void> verifyGranules(Database cx, BlobGranuleVerifierWorkload* self, bool allowPurging) {
 		state double last = now();
 		state double endTime = last + self->testDuration;
@@ -217,77 +304,99 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 		state int64_t timeTravelChecksMemory = 0;
 		state Version prevPurgeVersion = -1;
 		state UID dbgId = debugRandom()->randomUniqueID();
+		state Version newPurgeVersion = 0;
 
 		TraceEvent("BlobGranuleVerifierStart");
 		if (BGV_DEBUG) {
-			printf("BGV thread starting\n");
+			fmt::print("BGV {0}) thread starting\n", self->clientId);
 		}
 
 		// wait for first set of ranges to be loaded
 		wait(self->granuleRanges.onChange());
 
 		if (BGV_DEBUG) {
-			printf("BGV got ranges\n");
+			fmt::print("BGV {0}) got ranges\n", self->clientId);
 		}
 
 		loop {
 			try {
 				state double currentTime = now();
 				state std::map<double, OldRead>::iterator timeTravelIt = timeTravelChecks.begin();
+				newPurgeVersion = 0;
 				while (timeTravelIt != timeTravelChecks.end() && currentTime >= timeTravelIt->first) {
 					state OldRead oldRead = timeTravelIt->second;
 					timeTravelChecksMemory -= oldRead.oldResult.expectedSize();
+					// advance iterator before doing read, so if it gets error we don't retry it
 					timeTravelIt = timeTravelChecks.erase(timeTravelIt);
 					if (prevPurgeVersion == -1) {
 						prevPurgeVersion = oldRead.v;
 					}
-					// advance iterator before doing read, so if it gets error we don't retry it
 
-					try {
-						state Version newPurgeVersion = 0;
-						state bool doPurging = allowPurging && deterministicRandom()->random01() < 0.5;
-						if (doPurging) {
-							Version maxPurgeVersion = oldRead.v;
-							for (auto& it : timeTravelChecks) {
-								maxPurgeVersion = std::min(it.second.v, maxPurgeVersion);
-							}
-							if (prevPurgeVersion < maxPurgeVersion) {
-								newPurgeVersion = deterministicRandom()->randomInt64(prevPurgeVersion, maxPurgeVersion);
-								prevPurgeVersion = std::max(prevPurgeVersion, newPurgeVersion);
-								Key purgeKey = wait(cx->purgeBlobGranules(normalKeys, newPurgeVersion, {}, false));
-								wait(cx->waitPurgeGranulesComplete(purgeKey));
-								self->purges++;
-							} else {
-								doPurging = false;
-							}
+					// before doing read, purge just before read version
+					state bool doPurging =
+					    allowPurging && !self->purgeAtLatest && deterministicRandom()->random01() < 0.5;
+					state bool forcePurge = doPurging && self->doForcePurge && deterministicRandom()->random01() < 0.25;
+					if (doPurging) {
+						CODE_PROBE(true, "BGV considering purge");
+						Version maxPurgeVersion = oldRead.v;
+						for (auto& it : timeTravelChecks) {
+							maxPurgeVersion = std::min(it.second.v, maxPurgeVersion);
 						}
+						if (prevPurgeVersion < maxPurgeVersion) {
+							CODE_PROBE(true, "BGV doing purge");
+							newPurgeVersion = deterministicRandom()->randomInt64(prevPurgeVersion, maxPurgeVersion);
+							prevPurgeVersion = std::max(prevPurgeVersion, newPurgeVersion);
+							if (BGV_DEBUG) {
+								fmt::print("BGV Purging @ {0}{1}\n", newPurgeVersion, forcePurge ? " (force)" : "");
+							}
+							try {
+								if (forcePurge) {
+									wait(self->loadGranuleMetadataBeforeForcePurge(cx, self));
+									self->startedForcePurge = true;
+								}
+								Key purgeKey = wait(cx->purgeBlobGranules(normalKeys, newPurgeVersion, {}, forcePurge));
+								if (forcePurge) {
+									self->forcePurgeKey = purgeKey;
+									self->forcePurgeVersion = newPurgeVersion;
+									if (BGV_DEBUG) {
+										fmt::print("BGV Force purge registered, stopping\n");
+									}
+									return Void();
+								}
+								if (BGV_DEBUG) {
+									fmt::print("BGV Purged @ {0}, waiting\n", newPurgeVersion);
+								}
+								wait(cx->waitPurgeGranulesComplete(purgeKey));
+							} catch (Error& e) {
+								if (e.code() == error_code_operation_cancelled) {
+									throw e;
+								}
+								// purging shouldn't error, it should retry.
+								if (BGV_DEBUG) {
+									fmt::print("Unexpected error {0} purging @ {1}!\n", e.name(), newPurgeVersion);
+								}
+								ASSERT(false);
+							}
+							CODE_PROBE(true, "BGV purge complete");
+							if (BGV_DEBUG) {
+								fmt::print("BGV Purge complete @ {0}\n", newPurgeVersion);
+							}
+							self->purges++;
+						} else {
+							doPurging = false;
+						}
+					}
+
+					// do time travel read
+					try {
 						std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> reReadResult =
 						    wait(readFromBlob(cx, self->bstore, oldRead.range, 0, oldRead.v));
 						if (!compareFDBAndBlob(oldRead.oldResult, reReadResult, oldRead.range, oldRead.v, BGV_DEBUG)) {
 							self->mismatches++;
 						}
 						self->timeTravelReads++;
-
-						if (doPurging) {
-							wait(self->killBlobWorkers(cx, self));
-							std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> versionRead =
-							    wait(readFromBlob(cx, self->bstore, oldRead.range, 0, prevPurgeVersion));
-							try {
-								Version minSnapshotVersion = newPurgeVersion;
-								for (auto& it : versionRead.second) {
-									minSnapshotVersion = std::min(minSnapshotVersion, it.snapshotVersion);
-								}
-								std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> versionRead =
-								    wait(readFromBlob(cx, self->bstore, oldRead.range, 0, minSnapshotVersion - 1));
-								ASSERT(false);
-							} catch (Error& e) {
-								if (e.code() == error_code_actor_cancelled) {
-									throw;
-								}
-								ASSERT(e.code() == error_code_blob_granule_transaction_too_old);
-							}
-						}
 					} catch (Error& e) {
+						fmt::print("Error TT: {0}\n", e.name());
 						if (e.code() == error_code_blob_granule_transaction_too_old) {
 							self->timeTravelTooOld++;
 							// TODO: add debugging info for when this is a failure
@@ -297,6 +406,55 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 							           oldRead.v);
 						}
 					}
+
+					// if purged just before read, verify that purge cleaned up data by restarting blob workers and
+					// reading older than the purge version
+					if (doPurging) {
+						if (self->strictPurgeChecking) {
+							wait(self->killBlobWorkers(cx, self));
+							if (BGV_DEBUG) {
+								fmt::print("BGV Reading post-purge [{0} - {1}) @ {2}\n",
+								           oldRead.range.begin.printable(),
+								           oldRead.range.end.printable(),
+								           prevPurgeVersion);
+							}
+						}
+						// ensure purge version exactly is still readable
+						std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> versionRead1 =
+						    wait(readFromBlob(cx, self->bstore, oldRead.range, 0, prevPurgeVersion));
+						if (BGV_DEBUG) {
+							fmt::print("BGV Post-purge first read:\n");
+							printGranuleChunks(versionRead1.second);
+						}
+						if (self->strictPurgeChecking) {
+							try {
+								// read at purgeVersion - 1, should NOT be readable
+								Version minSnapshotVersion = newPurgeVersion;
+								for (auto& it : versionRead1.second) {
+									minSnapshotVersion = std::min(minSnapshotVersion, it.snapshotVersion);
+								}
+								if (BGV_DEBUG) {
+									fmt::print("BGV Reading post-purge again [{0} - {1}) @ {2}\n",
+									           oldRead.range.begin.printable(),
+									           oldRead.range.end.printable(),
+									           minSnapshotVersion - 1);
+								}
+								std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> versionRead2 =
+								    wait(readFromBlob(cx, self->bstore, oldRead.range, 0, minSnapshotVersion - 1));
+								if (BGV_DEBUG) {
+									fmt::print("BGV ERROR: data not purged! Read successful!!\n");
+									printGranuleChunks(versionRead2.second);
+								}
+								ASSERT(false);
+							} catch (Error& e) {
+								if (e.code() == error_code_actor_cancelled) {
+									throw;
+								}
+								ASSERT(e.code() == error_code_blob_granule_transaction_too_old);
+								CODE_PROBE(true, "BGV verified too old after purge");
+							}
+						}
+					}
 				}
 
 				// pick a random range
@@ -304,11 +462,33 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 				state KeyRange range = self->granuleRanges.get()[rIndex];
 
 				state std::pair<RangeResult, Version> fdb = wait(readFromFDB(cx, range));
-				std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> blob =
+				state std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> blob =
 				    wait(readFromBlob(cx, self->bstore, range, 0, fdb.second));
+				if (self->purgeAtLatest && timeTravelChecks.empty() && deterministicRandom()->random01() < 0.25) {
+					// purge at this version, and make sure it's still readable after on our immediate re-read
+					try {
+						Key purgeKey = wait(cx->purgeBlobGranules(normalKeys, fdb.second, {}, false));
+						if (BGV_DEBUG) {
+							fmt::print("BGV {0}) Purged Latest @ {1}, waiting\n", self->clientId, fdb.second);
+						}
+						wait(cx->waitPurgeGranulesComplete(purgeKey));
+					} catch (Error& e) {
+						if (e.code() == error_code_operation_cancelled) {
+							throw e;
+						}
+						// purging shouldn't error, it should retry.
+						if (BGV_DEBUG) {
+							fmt::print("Unexpected error {0} purging latest @ {1}!\n", e.name(), newPurgeVersion);
+						}
+						ASSERT(false);
+					}
+					self->purges++;
+				}
 				if (compareFDBAndBlob(fdb.first, blob, range, fdb.second, BGV_DEBUG)) {
-					// TODO: bias for immediately re-reading to catch rollback cases
-					double reReadTime = currentTime + deterministicRandom()->random01() * self->timeTravelLimit;
+					bool rereadImmediately = self->purgeAtLatest || deterministicRandom()->random01() < 0.25;
+					double reReadTime =
+					    currentTime +
+					    (rereadImmediately ? 0.0 : deterministicRandom()->random01() * self->timeTravelLimit);
 					int memory = fdb.first.expectedSize();
 					if (reReadTime <= endTime &&
 					    timeTravelChecksMemory + memory <= (self->timeTravelBufferSize / self->threads)) {
@@ -327,11 +507,10 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 					throw;
 				}
 				if (e.code() != error_code_blob_granule_transaction_too_old && BGV_DEBUG) {
-					printf("BGVerifier got unexpected error %s\n", e.name());
+					fmt::print("BGVerifier {0} got unexpected error {1}\n", self->clientId, e.name());
 				}
 				self->errors++;
 			}
-			// wait(poisson(&last, 5.0));
 			wait(poisson(&last, 0.1));
 		}
 	}
@@ -348,6 +527,11 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 				    reportErrors(verifyGranules(cx, this, false), "BlobGranuleVerifier"), testDuration, Void()));
 			}
 		}
+		if (!enablePurging) {
+			summaryClient = validateGranuleSummaries(cx, normalKeys, {}, triggerSummaryComplete);
+		} else {
+			summaryClient = Future<Void>(Void());
+		}
 		return delay(testDuration);
 	}
 
@@ -365,10 +549,486 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<bool> _check(Database cx, BlobGranuleVerifierWorkload* self) {
-		// check error counts, and do an availability check at the end
+	ACTOR Future<bool> checkGranuleMetadataPurged(Transaction* tr,
+	                                              KeyRange granuleRange,
+	                                              Version historyVersion,
+	                                              UID granuleId,
+	                                              bool strictMetadataCheck,
+	                                              bool possiblyInFlight) {
+		// change feed
+		Optional<Value> changeFeed = wait(tr->get(granuleIDToCFKey(granuleId).withPrefix(changeFeedPrefix)));
+		if (possiblyInFlight && changeFeed.present()) {
+			fmt::print("WARN: Change Feed for [{0} - {1}): {2} not purged, retrying\n",
+			           granuleRange.begin.printable(),
+			           granuleRange.end.printable(),
+			           granuleId.toString().substr(0, 6));
+			return false;
+		}
+		ASSERT(!changeFeed.present());
+
+		// file metadata
+		RangeResult fileMetadata = wait(tr->getRange(blobGranuleFileKeyRangeFor(granuleId), 1));
+		if (possiblyInFlight && !fileMetadata.empty()) {
+			fmt::print("WARN: File metadata for [{0} - {1}): {2} not purged, retrying\n",
+			           granuleRange.begin.printable(),
+			           granuleRange.end.printable(),
+			           granuleId.toString().substr(0, 6));
+			return false;
+		}
+		ASSERT(fileMetadata.empty());
+
+		if (strictMetadataCheck) {
+			// lock
+			Optional<Value> lock = wait(tr->get(blobGranuleLockKeyFor(granuleRange)));
+			if (possiblyInFlight && lock.present()) {
+				return false;
+			}
+			ASSERT(!lock.present());
+
+			// history entry
+			Optional<Value> history = wait(tr->get(blobGranuleHistoryKeyFor(granuleRange, historyVersion)));
+			if (possiblyInFlight && history.present()) {
+				return false;
+			}
+			ASSERT(!history.present());
+
+			// split state
+			RangeResult splitData = wait(tr->getRange(blobGranuleSplitKeyRangeFor(granuleId), 1));
+			if (possiblyInFlight && !splitData.empty()) {
+				return false;
+			}
+			ASSERT(splitData.empty());
+
+			// merge state
+			Optional<Value> merge = wait(tr->get(blobGranuleMergeKeyFor(granuleId)));
+			if (possiblyInFlight && merge.present()) {
+				return false;
+			}
+			ASSERT(!merge.present());
+
+			// FIXME: add merge boundaries!
+		}
+
+		return true;
+	}
+
+	ACTOR Future<Void> checkPurgedHistoryEntries(Database cx,
+	                                             BlobGranuleVerifierWorkload* self,
+	                                             KeyRange purgeRange,
+	                                             bool strictMetadataCheck) {
+		// quick check to make sure we didn't miss any new granules generated between the purge metadata load time and
+		// the actual purge, by checking for any new history keys in the range
+		// FIXME: fix this check! The BW granule check is really the important one, this finds occasional leftover
+		// metadata from boundary changes that can race with a force purge, but as long as no blob worker acts on the
+		// boundary changes, a bit of leftover metadata is a much smaller problem. To confirm that it was a race with
+		// force purging, we check that the history version > the force purge version
+		state Transaction tr(cx);
+		state KeyRange cur = blobGranuleHistoryKeys;
+		state std::vector<std::tuple<KeyRange, Version, UID>> granulesToCheck;
+		loop {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			try {
+				RangeResult history = wait(tr.getRange(cur, 1000));
+				for (auto& it : history) {
+					KeyRange keyRange;
+					Version version;
+					std::tie(keyRange, version) = decodeBlobGranuleHistoryKey(it.key);
+					if (purgeRange.intersects(keyRange)) {
+						if (BGV_DEBUG) {
+							fmt::print("Found range [{0} - {1}) @ {2} that avoided force purge [{3} - {4}) @ {5}!!\n",
+							           keyRange.begin.printable(),
+							           keyRange.end.printable(),
+							           version,
+							           purgeRange.begin.printable(),
+							           purgeRange.end.printable(),
+							           self->forcePurgeVersion);
+						}
+						if (strictMetadataCheck) {
+							ASSERT(!purgeRange.intersects(keyRange));
+						} else {
+							Standalone<BlobGranuleHistoryValue> historyValue = decodeBlobGranuleHistoryValue(it.value);
+							granulesToCheck.emplace_back(keyRange, version, historyValue.granuleID);
+						}
+					}
+				}
+				if (!history.empty() && history.more) {
+					cur = KeyRangeRef(keyAfter(history.back().key), cur.end);
+				} else {
+					break;
+				}
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		tr.reset();
+		state int i;
+		if (BGV_DEBUG && !granulesToCheck.empty()) {
+			fmt::print("Checking metadata for {0} non-purged ranges\n", granulesToCheck.size());
+		}
+		for (i = 0; i < granulesToCheck.size(); i++) {
+			loop {
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				try {
+					bool success = wait(self->checkGranuleMetadataPurged(&tr,
+					                                                     std::get<0>(granulesToCheck[i]),
+					                                                     std::get<1>(granulesToCheck[i]),
+					                                                     std::get<2>(granulesToCheck[i]),
+					                                                     strictMetadataCheck,
+					                                                     true));
+					if (success) {
+						break;
+					}
+					wait(delay(5.0));
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<bool> checkPurgedChangeFeeds(Database cx, BlobGranuleVerifierWorkload* self, KeyRange purgeRange) {
+		// quick check to make sure we didn't miss any change feeds
+		state Transaction tr(cx);
+		state KeyRange cur = changeFeedKeys;
+		loop {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			try {
+				RangeResult feeds = wait(tr.getRange(cur, 1000));
+				for (auto& it : feeds) {
+					KeyRange keyRange;
+					Version version;
+					ChangeFeedStatus status;
+					std::tie(keyRange, version, status) = decodeChangeFeedValue(it.value);
+					if (purgeRange.intersects(keyRange)) {
+						Key feedId = it.key.removePrefix(changeFeedKeys.begin);
+						if (BGV_DEBUG) {
+							fmt::print(
+							    "Found Change Feed {0}: [{1} - {2}) that avoided force purge [{3} - {4}) @ {5}!!\n",
+							    feedId.printable(),
+							    keyRange.begin.printable(),
+							    keyRange.end.printable(),
+							    purgeRange.begin.printable(),
+							    purgeRange.end.printable(),
+							    self->forcePurgeVersion);
+						}
+						// FIXME!!: there is a known race with the existing force purge algorithm that would require a
+						// bit of a redesign. This is mostly an edge case though that we don't anticipate seeing much in
+						// actual use, and the impact of these leaked change feeds is limited because the range is
+						// purged anyway.
+						bool foundAnyHistoryForRange = false;
+						for (auto& purgedData : self->purgedDataToCheck) {
+							KeyRange granuleRange = std::get<0>(purgedData);
+							if (granuleRange.intersects(keyRange)) {
+								foundAnyHistoryForRange = true;
+								break;
+							}
+						}
+
+						if (!foundAnyHistoryForRange) {
+							// if range never existed in blob, and was doing the initial snapshot,  it could have a
+							// change feed but not a history entry/snapshot
+							CODE_PROBE(true, "not failing test for leaked feed with no history");
+							fmt::print("Not failing test b/c feed never had history!\n");
+						}
+						return !foundAnyHistoryForRange;
+					}
+					// ASSERT(!purgeRange.intersects(keyRange));
+				}
+				if (!feeds.empty() && feeds.more) {
+					cur = KeyRangeRef(keyAfter(feeds.back().key), cur.end);
+				} else {
+					break;
+				}
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		return true;
+	}
+
+	ACTOR Future<Void> validateForcePurge(Database cx, BlobGranuleVerifierWorkload* self, KeyRange purgeRange) {
+		// FIXME: enable!! right now leaking metadata isn't nearly as bad as leaking data/change feeds
+		state bool strictMetadataCheck = false;
+		// first, wait for force purge to complete
+		if (BGV_DEBUG) {
+			fmt::print("BGV waiting for force purge to complete\n");
+		}
+		wait(cx->waitPurgeGranulesComplete(self->forcePurgeKey.get()));
+		if (BGV_DEBUG) {
+			fmt::print("BGV force purge completed, checking\n");
+		}
 
 		state Transaction tr(cx);
+
+		// check that force purge range is set and that data is not readable
+		loop {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			try {
+				ForcedPurgeState forcePurgedState = wait(getForcePurgedState(&tr, purgeRange));
+				ASSERT(forcePurgedState == ForcedPurgeState::AllPurged);
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV force purge checked state\n");
+		}
+
+		// make sure blob read fails
+		tr.reset();
+		loop {
+			try {
+				Version readVersion = wait(self->doGrv(&tr));
+				wait(success(readFromBlob(cx, self->bstore, purgeRange, 0, readVersion)));
+				ASSERT(false);
+			} catch (Error& e) {
+				if (e.code() == error_code_operation_cancelled) {
+					throw e;
+				}
+				ASSERT(e.code() == error_code_blob_granule_transaction_too_old);
+				break;
+			}
+		}
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV force purge checked read\n");
+		}
+
+		// check that metadata is gone for each granule
+		if (BGV_DEBUG) {
+			fmt::print("BGV checking metadata deleted\n");
+		}
+		state int i;
+		state int64_t filesChecked = 0;
+		for (i = 0; i < self->purgedDataToCheck.size(); i++) {
+			state KeyRange granuleRange = std::get<0>(self->purgedDataToCheck[i]);
+			state Version historyVersion = std::get<1>(self->purgedDataToCheck[i]);
+			state UID granuleId = std::get<2>(self->purgedDataToCheck[i]);
+			state GranuleFiles oldFiles = wait(std::get<3>(self->purgedDataToCheck[i]));
+			fmt::print("  Checking [{0} - {1}): {2}\n",
+			           granuleRange.begin.printable(),
+			           granuleRange.end.printable(),
+			           granuleId.toString().substr(0, 6));
+			loop {
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				try {
+					bool success = wait(self->checkGranuleMetadataPurged(
+					    &tr, granuleRange, historyVersion, granuleId, strictMetadataCheck, false));
+					ASSERT(success);
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+
+			// ensure all old files were deleted from blob storage
+			state int fileIdx;
+			for (fileIdx = 0; fileIdx < oldFiles.snapshotFiles.size() + oldFiles.deltaFiles.size(); fileIdx++) {
+				std::string fname = (fileIdx >= oldFiles.snapshotFiles.size())
+				                        ? oldFiles.deltaFiles[fileIdx - oldFiles.snapshotFiles.size()].filename
+				                        : oldFiles.snapshotFiles[fileIdx].filename;
+				state Reference<BackupContainerFileSystem> bstore = self->bstore->getForRead(fname);
+				try {
+					wait(success(bstore->readFile(fname)));
+					ASSERT(false);
+				} catch (Error& e) {
+					if (e.code() == error_code_operation_cancelled) {
+						throw e;
+					}
+					ASSERT(e.code() == error_code_file_not_found);
+					filesChecked++;
+				}
+			}
+		}
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV force purge checked {0} old granules and {1} old files cleaned up\n",
+			           self->purgedDataToCheck.size(),
+			           filesChecked);
+		}
+
+		wait(self->checkPurgedHistoryEntries(cx, self, purgeRange, strictMetadataCheck));
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV force purge checked for new granule history entries\n");
+		}
+
+		loop {
+			bool success = wait(self->checkPurgedChangeFeeds(cx, self, purgeRange));
+			if (success) {
+				break;
+			}
+		}
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV force purge checked for leaked change feeds\n");
+		}
+
+		// ask all workers for all of their open granules and make sure none are in the force purge range
+
+		// Because there could be ranges assigned that havne't yet finished opening to check for purge, some BWs might
+		// still temporarily have ranges assigned. To address this, we just retry the check after a bit
+		loop {
+			state bool anyRangesLeft = false;
+			state std::vector<BlobWorkerInterface> blobWorkers = wait(getBlobWorkers(cx));
+
+			if (BGV_DEBUG) {
+				fmt::print("BGV force purge checking {0} blob worker mappings\n", blobWorkers.size());
+			}
+
+			for (i = 0; i < blobWorkers.size(); i++) {
+				GetGranuleAssignmentsRequest req;
+				req.managerEpoch = -1; // not manager
+				Optional<GetGranuleAssignmentsReply> assignments =
+				    wait(timeout(brokenPromiseToNever(blobWorkers[i].granuleAssignmentsRequest.getReply(req)),
+				                 SERVER_KNOBS->BLOB_WORKER_TIMEOUT));
+				if (assignments.present()) {
+					for (auto& it : assignments.get().assignments) {
+						if (purgeRange.intersects(it.range)) {
+							if (BGV_DEBUG) {
+								fmt::print("BW {0} still has range [{1} - {2})\n",
+								           blobWorkers[i].id().toString(),
+								           it.range.begin.printable(),
+								           it.range.end.printable());
+							}
+							anyRangesLeft = true;
+						}
+					}
+				} else {
+					if (BGV_DEBUG) {
+						fmt::print("BGV mapping check failed to reach BW {0}\n", blobWorkers[i].id().toString());
+					}
+					// if BW timed out, we don't for sure know it didn't still have some range
+					anyRangesLeft = true;
+				}
+			}
+			if (anyRangesLeft) {
+				wait(delay(10.0));
+			} else {
+				break;
+			}
+		}
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV force purge check complete\n");
+		}
+
+		return Void();
+	}
+
+	// Check database against blob granules. This is especially important because during chaos phase this can error, and
+	// initAtEnd doesn't get data checked otherwise
+	ACTOR Future<bool> checkAllData(Database cx, BlobGranuleVerifierWorkload* self) {
+		state Transaction tr(cx);
+		state KeyRange keyRange = normalKeys;
+		state bool gotEOS = false;
+		state int64_t totalRows = 0;
+		loop {
+			state RangeResult output;
+			state Version readVersion = invalidVersion;
+			state int64_t bufferedBytes = 0;
+			try {
+				Version ver = wait(tr.getReadVersion());
+				readVersion = ver;
+
+				state PromiseStream<Standalone<RangeResultRef>> results;
+				state Future<Void> stream = tr.getRangeStream(results, keyRange, GetRangeLimits());
+
+				loop {
+					Standalone<RangeResultRef> res = waitNext(results.getFuture());
+					output.arena().dependsOn(res.arena());
+					output.append(output.arena(), res.begin(), res.size());
+					bufferedBytes += res.expectedSize();
+					// force checking if we have enough data
+					if (bufferedBytes >= 10 * SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES) {
+						break;
+					}
+				}
+			} catch (Error& e) {
+				if (e.code() == error_code_operation_cancelled) {
+					throw e;
+				}
+				if (e.code() == error_code_end_of_stream) {
+					gotEOS = true;
+				} else {
+					wait(tr.onError(e));
+				}
+			}
+
+			if (!output.empty()) {
+				state KeyRange rangeToCheck = KeyRangeRef(keyRange.begin, keyAfter(output.back().key));
+				try {
+					std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> blob =
+					    wait(readFromBlob(cx, self->bstore, rangeToCheck, 0, readVersion));
+					if (!compareFDBAndBlob(output, blob, rangeToCheck, readVersion, BGV_DEBUG)) {
+						return false;
+					}
+				} catch (Error& e) {
+					if (BGV_DEBUG && e.code() == error_code_blob_granule_transaction_too_old) {
+						fmt::print("CheckAllData got BG_TTO for [{0} - {1}) @ {2}\n",
+						           rangeToCheck.begin.printable(),
+						           rangeToCheck.end.printable(),
+						           readVersion);
+					}
+					ASSERT(e.code() != error_code_blob_granule_transaction_too_old);
+					throw e;
+				}
+				totalRows += output.size();
+				keyRange = KeyRangeRef(rangeToCheck.end, keyRange.end);
+			}
+			if (gotEOS) {
+				break;
+			}
+		}
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV {0}) Final data check complete, checked {1} rows\n", self->clientId, totalRows);
+		}
+
+		return true;
+	}
+
+	ACTOR Future<bool> _check(Database cx, BlobGranuleVerifierWorkload* self) {
+		if (self->triggerSummaryComplete.canBeSet()) {
+			self->triggerSummaryComplete.send(Void());
+		}
+		state Transaction tr(cx);
+		if (self->doForcePurge) {
+			if (self->startedForcePurge) {
+				if (self->forcePurgeKey.present()) {
+					wait(self->validateForcePurge(cx, self, normalKeys));
+				} // else if we had already started purge during the test but aren't sure whether it was registered or
+				  // not,
+				// don't validate that data was purged since it may never be
+				return true;
+			}
+		} else if (self->enablePurging && self->purgeAtLatest && deterministicRandom()->coinflip()) {
+			Version latestPurgeVersion = wait(self->doGrv(&tr));
+			if (BGV_DEBUG) {
+				fmt::print("BGV {0}) Purging Latest @ {1} before final availability check\n",
+				           self->clientId,
+				           latestPurgeVersion);
+			}
+			Key purgeKey = wait(cx->purgeBlobGranules(normalKeys, latestPurgeVersion, {}, false));
+			wait(cx->waitPurgeGranulesComplete(purgeKey));
+			if (BGV_DEBUG) {
+				fmt::print("BGV {0}) Purged Latest before final availability check complete\n", self->clientId);
+			}
+			self->purges++;
+		}
+
+		// check error counts, and do an availability check at the end
+
+		if (self->doSetup && self->initAtEnd) {
+			wait(self->setUpBlobRange(cx));
+		}
+
 		state Version readVersion = wait(self->doGrv(&tr));
 		state Version startReadVersion = readVersion;
 		state int checks = 0;
@@ -377,27 +1037,39 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 		state bool availabilityPassed = true;
 
 		state Standalone<VectorRef<KeyRangeRef>> allRanges;
-		if (self->granuleRanges.get().empty()) {
-			if (BGV_DEBUG) {
-				fmt::print("Waiting to get granule ranges for check\n");
-			}
-			state Future<Void> rangeFetcher = self->findGranules(cx, self);
-			loop {
-				wait(self->granuleRanges.onChange());
-				if (!self->granuleRanges.get().empty()) {
+
+		state Future<Void> rangeFetcher = self->findGranules(cx, self);
+		loop {
+			// wait until entire keyspace has granules
+			if (!self->granuleRanges.get().empty()) {
+				bool haveAll = true;
+				if (self->granuleRanges.get().front().begin != normalKeys.begin ||
+				    self->granuleRanges.get().back().end != normalKeys.end) {
+					haveAll = false;
+				}
+				for (int i = 0; haveAll && i < self->granuleRanges.get().size() - 1; i++) {
+					if (self->granuleRanges.get()[i].end != self->granuleRanges.get()[i + 1].begin) {
+						haveAll = false;
+					}
+				}
+				if (haveAll) {
 					break;
 				}
 			}
-			rangeFetcher.cancel();
 			if (BGV_DEBUG) {
-				fmt::print("Got granule ranges for check\n");
+				fmt::print("Waiting to get granule ranges for check\n");
 			}
+			wait(self->granuleRanges.onChange());
 		}
+
+		rangeFetcher.cancel();
+
 		allRanges = self->granuleRanges.get();
 		for (auto& range : allRanges) {
 			state KeyRange r = range;
 			if (BGV_DEBUG) {
-				fmt::print("Final availability check [{0} - {1}) @ {2}\n",
+				fmt::print("BGV {0}) Final availability check [{1} - {2}) @ {3}\n",
+				           self->clientId,
 				           r.begin.printable(),
 				           r.end.printable(),
 				           readVersion);
@@ -451,7 +1123,11 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 		if (BGV_DEBUG && startReadVersion != readVersion) {
 			fmt::print("Availability check updated read version from {0} to {1}\n", startReadVersion, readVersion);
 		}
-		bool result = availabilityPassed && self->mismatches == 0 && (checks > 0) && (self->timeTravelTooOld == 0);
+
+		state bool dataPassed = wait(self->checkAllData(cx, self));
+
+		state bool result =
+		    availabilityPassed && dataPassed && self->mismatches == 0 && (checks > 0) && (self->timeTravelTooOld == 0);
 		fmt::print("Blob Granule Verifier {0} {1}:\n", self->clientId, result ? "passed" : "failed");
 		fmt::print("  {} successful final granule checks\n", checks);
 		fmt::print("  {} failed final granule checks\n", availabilityPassed ? 0 : 1);
@@ -463,6 +1139,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 		fmt::print("  {} rows\n", self->rowsRead);
 		fmt::print("  {} bytes\n", self->bytesRead);
 		fmt::print("  {} purges\n", self->purges);
+		fmt::print("  {} final data check\n", dataPassed ? "passed" : "failed");
 		// FIXME: add above as details to trace event
 
 		TraceEvent("BlobGranuleVerifierChecked").detail("Result", result);
@@ -470,10 +1147,83 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
 		// For some reason simulation is still passing when this fails?.. so assert for now
 		ASSERT(result);
 
+		if (self->doForcePurge) {
+			// if granules are available, and we didn't do a force purge during the test, do it now
+			ASSERT(!self->startedForcePurge);
+			Version rv = wait(self->doGrv(&tr));
+			self->forcePurgeVersion = rv;
+			self->purgedDataToCheck.clear(); //  in case we started but didn't finish loading it, reset it
+			wait(self->loadGranuleMetadataBeforeForcePurge(cx, self));
+			Key purgeKey = wait(cx->purgeBlobGranules(normalKeys, self->forcePurgeVersion, {}, true));
+			self->forcePurgeKey = purgeKey;
+			wait(self->validateForcePurge(cx, self, normalKeys));
+
+			return true;
+		} else if (self->enablePurging && self->purgeAtLatest && deterministicRandom()->coinflip()) {
+			Version latestPurgeVersion = wait(self->doGrv(&tr));
+			if (BGV_DEBUG) {
+				fmt::print("BGV {0}) Purging Latest @ {1} after final availability check, waiting\n",
+				           self->clientId,
+				           latestPurgeVersion);
+			}
+			Key purgeKey = wait(cx->purgeBlobGranules(normalKeys, latestPurgeVersion, {}, false));
+			wait(cx->waitPurgeGranulesComplete(purgeKey));
+			if (BGV_DEBUG) {
+				fmt::print("BGV {0}) Purged Latest after final availability check complete\n", self->clientId);
+			}
+		}
+
+		if (self->clientId == 0 && SERVER_KNOBS->BG_ENABLE_MERGING && self->clearAndMergeCheck) {
+			CODE_PROBE(true, "BGV clearing database and awaiting merge");
+			wait(clearAndAwaitMerge(cx, normalKeys));
+
+			if (self->enablePurging && self->purgeAtLatest && deterministicRandom()->coinflip()) {
+				Version latestPurgeVersion = wait(self->doGrv(&tr));
+				if (BGV_DEBUG) {
+					fmt::print("BGV {0}) Purging Latest @ {1} after clearAndAwaitMerge, waiting\n",
+					           self->clientId,
+					           latestPurgeVersion);
+				}
+				Key purgeKey = wait(cx->purgeBlobGranules(normalKeys, latestPurgeVersion, {}, false));
+				wait(cx->waitPurgeGranulesComplete(purgeKey));
+				if (BGV_DEBUG) {
+					fmt::print("BGV {0}) Purged Latest after clearAndAwaitMerge complete\n", self->clientId);
+				}
+			}
+
+			if (BGV_DEBUG) {
+				fmt::print("BGV {0}) Checking data after merge\n", self->clientId);
+			}
+
+			// read after merge to make sure it completed, granules are available, and data is empty
+			bool dataCheckAfterMerge = wait(self->checkAllData(cx, self));
+			ASSERT(dataCheckAfterMerge);
+
+			if (BGV_DEBUG) {
+				fmt::print("BGV {0}) Checked data after merge\n", self->clientId);
+			}
+		}
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV {0}) check waiting on summarizer to complete\n", self->clientId);
+		}
+
+		// validate that summary completes without error
+		wait(self->summaryClient);
+
+		if (BGV_DEBUG) {
+			fmt::print("BGV {0}) check done\n", self->clientId);
+		}
+
 		return result;
 	}
 
-	Future<bool> check(Database const& cx) override { return _check(cx, this); }
+	Future<bool> check(Database const& cx) override {
+		if (clientId == 0 || (!doForcePurge && !purgeAtLatest)) {
+			return _check(cx, this);
+		}
+		return true;
+	}
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
diff --git a/fdbserver/workloads/BulkLoad.actor.cpp b/fdbserver/workloads/BulkLoad.actor.cpp
index d10a909a04..e5726cdfcb 100644
--- a/fdbserver/workloads/BulkLoad.actor.cpp
+++ b/fdbserver/workloads/BulkLoad.actor.cpp
@@ -38,13 +38,13 @@ struct BulkLoadWorkload : TestWorkload {
 	BulkLoadWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), clientCount(wcx.clientCount), transactions("Transactions"), retries("Retries"),
 	    latencies(2000) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		actorCount = getOption(options, LiteralStringRef("actorCount"), 20);
-		writesPerTransaction = getOption(options, LiteralStringRef("writesPerTransaction"), 10);
-		valueBytes = std::max(getOption(options, LiteralStringRef("valueBytes"), 96), 16);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		actorCount = getOption(options, "actorCount"_sr, 20);
+		writesPerTransaction = getOption(options, "writesPerTransaction"_sr, 10);
+		valueBytes = std::max(getOption(options, "valueBytes"_sr, 96), 16);
 		value = Value(std::string(valueBytes, '.'));
-		targetBytes = getOption(options, LiteralStringRef("targetBytes"), std::numeric_limits<uint64_t>::max());
-		keyPrefix = getOption(options, LiteralStringRef("keyPrefix"), LiteralStringRef(""));
+		targetBytes = getOption(options, "targetBytes"_sr, std::numeric_limits<uint64_t>::max());
+		keyPrefix = getOption(options, "keyPrefix"_sr, ""_sr);
 		keyPrefix = unprintable(keyPrefix.toString());
 	}
 
diff --git a/fdbserver/workloads/BulkSetup.actor.cpp b/fdbserver/workloads/BulkSetup.actor.cpp
new file mode 100644
index 0000000000..db797722a5
--- /dev/null
+++ b/fdbserver/workloads/BulkSetup.actor.cpp
@@ -0,0 +1,75 @@
+/*
+ * BulkSetup.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbrpc/ContinuousSample.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct BulkSetupWorkload : TestWorkload {
+
+	std::vector<TenantName> tenantNames;
+	int nodeCount;
+	double transactionsPerSecond;
+	Key keyPrefix;
+
+	BulkSetupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0) / clientCount;
+		nodeCount = getOption(options, "nodeCount"_sr, transactionsPerSecond * clientCount);
+		keyPrefix = unprintable(getOption(options, "keyPrefix"_sr, ""_sr).toString());
+		std::vector<std::string> tenants = getOption(options, "tenants"_sr, std::vector<std::string>());
+		for (std::string tenant : tenants) {
+			tenantNames.push_back(TenantName(tenant));
+		}
+	}
+
+	std::string description() const override { return "BulkSetup"; }
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	Key keyForIndex(int n) { return key(n); }
+	Key key(int n) { return doubleToTestKey((double)n / nodeCount, keyPrefix); }
+	Value value(int n) { return doubleToTestKey(n, keyPrefix); }
+
+	Standalone<KeyValueRef> operator()(int n) { return KeyValueRef(key(n), value((n + 1) % nodeCount)); }
+
+	Future<Void> start(Database const& cx) override {
+		return bulkSetup(cx,
+		                 this,
+		                 nodeCount,
+		                 Promise<double>(),
+		                 false,
+		                 0.0,
+		                 1e12,
+		                 std::vector<uint64_t>(),
+		                 Promise<std::vector<std::pair<uint64_t, double>>>(),
+		                 0,
+		                 0.1,
+		                 0,
+		                 0,
+		                 this->tenantNames);
+	}
+
+	Future<bool> check(Database const& cx) override { return true; }
+};
+
+WorkloadFactory<BulkSetupWorkload> BulkSetupWorkloadFactory("BulkSetup");
diff --git a/fdbserver/workloads/Cache.actor.cpp b/fdbserver/workloads/Cache.actor.cpp
index 2b107a07e1..23bd9e3d92 100644
--- a/fdbserver/workloads/Cache.actor.cpp
+++ b/fdbserver/workloads/Cache.actor.cpp
@@ -27,7 +27,7 @@ struct CacheWorkload : TestWorkload {
 	Key keyPrefix;
 
 	CacheWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		keyPrefix = unprintable(getOption(options, LiteralStringRef("keyPrefix"), LiteralStringRef("")).toString());
+		keyPrefix = unprintable(getOption(options, "keyPrefix"_sr, ""_sr).toString());
 	}
 
 	std::string description() const override { return "CacheWorkload"; }
diff --git a/fdbserver/workloads/ChangeConfig.actor.cpp b/fdbserver/workloads/ChangeConfig.actor.cpp
index 8f9949ec52..5e2af183b9 100644
--- a/fdbserver/workloads/ChangeConfig.actor.cpp
+++ b/fdbserver/workloads/ChangeConfig.actor.cpp
@@ -26,23 +26,31 @@
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/Schemas.h"
+#include "flow/ApiVersion.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct ChangeConfigWorkload : TestWorkload {
 	double minDelayBeforeChange, maxDelayBeforeChange;
 	std::string configMode; //<\"single\"|\"double\"|\"triple\">
 	std::string networkAddresses; // comma separated list e.g. "127.0.0.1:4000,127.0.0.1:4001"
+	int coordinatorChanges; // number of times to change coordinators. Only applied if `coordinators` is set to `auto`
 
 	ChangeConfigWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		minDelayBeforeChange = getOption(options, LiteralStringRef("minDelayBeforeChange"), 0);
-		maxDelayBeforeChange = getOption(options, LiteralStringRef("maxDelayBeforeChange"), 0);
+		minDelayBeforeChange = getOption(options, "minDelayBeforeChange"_sr, 0);
+		maxDelayBeforeChange = getOption(options, "maxDelayBeforeChange"_sr, 0);
 		ASSERT(maxDelayBeforeChange >= minDelayBeforeChange);
-		configMode = getOption(options, LiteralStringRef("configMode"), StringRef()).toString();
-		networkAddresses = getOption(options, LiteralStringRef("coordinators"), StringRef()).toString();
+		configMode = getOption(options, "configMode"_sr, StringRef()).toString();
+		networkAddresses = getOption(options, "coordinators"_sr, StringRef()).toString();
+		coordinatorChanges = getOption(options, "coordinatorChanges"_sr, 1);
+		if (networkAddresses != "auto") {
+			coordinatorChanges = 1;
+		}
 	}
 
 	std::string description() const override { return "ChangeConfig"; }
 
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
+
 	Future<Void> start(Database const& cx) override {
 		if (this->clientId != 0)
 			return Void();
@@ -53,37 +61,44 @@ struct ChangeConfigWorkload : TestWorkload {
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	// When simulated two clusters for DR tests, this actor sets the starting configuration
-	// for the extra cluster.
-	ACTOR Future<Void> extraDatabaseConfigure(ChangeConfigWorkload* self) {
-		if (g_network->isSimulated() && g_simulator.extraDB) {
-			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-			state Database extraDB = Database::createDatabase(extraFile, -1);
-
-			wait(delay(5 * deterministicRandom()->random01()));
-			if (self->configMode.size()) {
-				if (g_simulator.startingDisabledConfiguration != "") {
-					// It is not safe to allow automatic failover to a region which is not fully replicated,
-					// so wait for both regions to be fully replicated before enabling failover
-					wait(success(ManagementAPI::changeConfig(
-					    extraDB.getReference(), g_simulator.startingDisabledConfiguration, true)));
-					TraceEvent("WaitForReplicasExtra").log();
-					wait(waitForFullReplication(extraDB));
-					TraceEvent("WaitForReplicasExtraEnd").log();
-				}
-				wait(success(ManagementAPI::changeConfig(extraDB.getReference(), self->configMode, true)));
+	ACTOR Future<Void> configureExtraDatabase(ChangeConfigWorkload* self, Database db) {
+		wait(delay(5 * deterministicRandom()->random01()));
+		if (self->configMode.size()) {
+			if (g_simulator->startingDisabledConfiguration != "") {
+				// It is not safe to allow automatic failover to a region which is not fully replicated,
+				// so wait for both regions to be fully replicated before enabling failover
+				wait(success(
+				    ManagementAPI::changeConfig(db.getReference(), g_simulator->startingDisabledConfiguration, true)));
+				TraceEvent("WaitForReplicasExtra").log();
+				wait(waitForFullReplication(db));
+				TraceEvent("WaitForReplicasExtraEnd").log();
 			}
-			if (self->networkAddresses.size()) {
-				if (self->networkAddresses == "auto")
-					wait(CoordinatorsChangeActor(extraDB, self, true));
-				else
-					wait(CoordinatorsChangeActor(extraDB, self));
-			}
-			wait(delay(5 * deterministicRandom()->random01()));
+			wait(success(ManagementAPI::changeConfig(db.getReference(), self->configMode, true)));
 		}
+		if (self->networkAddresses.size()) {
+			if (self->networkAddresses == "auto")
+				wait(CoordinatorsChangeActor(db, self, true));
+			else
+				wait(CoordinatorsChangeActor(db, self));
+		}
+
+		wait(delay(5 * deterministicRandom()->random01()));
 		return Void();
 	}
 
+	// When simulating multiple clusters, this actor sets the starting configuration
+	// for the extra clusters.
+	Future<Void> configureExtraDatabases(ChangeConfigWorkload* self) {
+		std::vector<Future<Void>> futures;
+		if (g_network->isSimulated()) {
+			for (auto extraDatabase : g_simulator->extraDatabases) {
+				Database db = Database::createSimulatedExtraDatabase(extraDatabase);
+				futures.push_back(configureExtraDatabase(self, db));
+			}
+		}
+		return waitForAll(futures);
+	}
+
 	// Either changes the database configuration, or changes the coordinators based on the parameters
 	// of the workload.
 	ACTOR Future<Void> ChangeConfigClient(Database cx, ChangeConfigWorkload* self) {
@@ -93,30 +108,36 @@ struct ChangeConfigWorkload : TestWorkload {
 		state bool extraConfigureBefore = deterministicRandom()->random01() < 0.5;
 
 		if (extraConfigureBefore) {
-			wait(self->extraDatabaseConfigure(self));
+			wait(self->configureExtraDatabases(self));
 		}
 
 		if (self->configMode.size()) {
-			if (g_network->isSimulated() && g_simulator.startingDisabledConfiguration != "") {
+			if (g_network->isSimulated() && g_simulator->startingDisabledConfiguration != "") {
 				// It is not safe to allow automatic failover to a region which is not fully replicated,
 				// so wait for both regions to be fully replicated before enabling failover
 				wait(success(
-				    ManagementAPI::changeConfig(cx.getReference(), g_simulator.startingDisabledConfiguration, true)));
+				    ManagementAPI::changeConfig(cx.getReference(), g_simulator->startingDisabledConfiguration, true)));
 				TraceEvent("WaitForReplicas").log();
 				wait(waitForFullReplication(cx));
 				TraceEvent("WaitForReplicasEnd").log();
 			}
 			wait(success(ManagementAPI::changeConfig(cx.getReference(), self->configMode, true)));
 		}
-		if (self->networkAddresses.size()) {
-			if (self->networkAddresses == "auto")
-				wait(CoordinatorsChangeActor(cx, self, true));
-			else
-				wait(CoordinatorsChangeActor(cx, self));
+		if ((g_network->isSimulated() && g_simulator->configDBType != ConfigDBType::SIMPLE) ||
+		    !g_network->isSimulated()) {
+			if (self->networkAddresses.size()) {
+				state int i;
+				for (i = 0; i < self->coordinatorChanges; ++i) {
+					if (i > 0) {
+						wait(delay(20));
+					}
+					wait(CoordinatorsChangeActor(cx, self, self->networkAddresses == "auto"));
+				}
+			}
 		}
 
 		if (!extraConfigureBefore) {
-			wait(self->extraDatabaseConfigure(self));
+			wait(self->configureExtraDatabases(self));
 		}
 
 		return Void();
@@ -131,9 +152,11 @@ struct ChangeConfigWorkload : TestWorkload {
 		if (autoChange) { // if auto, we first get the desired addresses by read \xff\xff/management/auto_coordinators
 			loop {
 				try {
-					Optional<Value> newCoordinatorsKey = wait(tr.get(
-					    LiteralStringRef("auto_coordinators")
-					        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
+					// Set RAW_ACCESS to explicitly avoid using tenants because
+					// access to management keys is denied for tenant transactions
+					tr.setOption(FDBTransactionOptions::RAW_ACCESS);
+					Optional<Value> newCoordinatorsKey = wait(tr.get("auto_coordinators"_sr.withPrefix(
+					    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
 					ASSERT(newCoordinatorsKey.present());
 					desiredCoordinatorsKey = newCoordinatorsKey.get().toString();
 					tr.reset();
@@ -170,8 +193,7 @@ struct ChangeConfigWorkload : TestWorkload {
 		loop {
 			try {
 				tr.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-				tr.set(LiteralStringRef("processes")
-				           .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
+				tr.set("processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
 				       Value(desiredCoordinatorsKey));
 				TraceEvent(SevDebug, "CoordinatorsChangeBeforeCommit")
 				    .detail("Auto", autoChange)
diff --git a/fdbserver/workloads/ChangeFeedOperations.actor.cpp b/fdbserver/workloads/ChangeFeedOperations.actor.cpp
new file mode 100644
index 0000000000..a0b1bf9923
--- /dev/null
+++ b/fdbserver/workloads/ChangeFeedOperations.actor.cpp
@@ -0,0 +1,775 @@
+/*
+ * ChangeFeedOperations.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/SystemData.h"
+#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "flow/Arena.h"
+#include "flow/IRandom.h"
+#include "flow/Trace.h"
+#include "flow/Util.h"
+#include "flow/serialize.h"
+#include <cstring>
+#include <limits>
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+// enable to debug specific operations for a given change feed
+#define DEBUG_KEY ""_sr
+
+#define DEBUG_CF(feedKey) (feedKey.printable() == DEBUG_KEY)
+
+ACTOR Future<Void> doPop(Database cx, Key key, Key feedID, Version version, Version* doneOut) {
+	wait(cx->popChangeFeedMutations(feedID, version));
+	if (*doneOut < version) {
+		*doneOut = version;
+	}
+	if (DEBUG_CF(key)) {
+		fmt::print("DBG) {0} Popped through {1}\n", key.printable(), version);
+	}
+	// TODO: could strengthen pop checking by validating that a read immediately after the pop completes has no data
+	return Void();
+}
+
+struct FeedTestData : ReferenceCounted<FeedTestData>, NonCopyable {
+	Key key;
+	KeyRange keyRange;
+	Key feedID;
+	int nextVal;
+	Future<Void> liveReader;
+	bool lastCleared = false;
+
+	std::vector<Future<Void>> pops;
+	Version poppingVersion;
+	Version poppedVersion;
+	Optional<Version> stopVersion;
+	bool destroying;
+	bool destroyed;
+	bool complete;
+
+	int popWindow;
+	int popDelayWindow;
+
+	std::deque<std::pair<Version, Optional<Value>>> writesByVersion;
+
+	// these were all committed
+	std::deque<std::pair<Version, Optional<Value>>> pendingCheck;
+	NotifiedVersion checkVersion;
+
+	FeedTestData(Key key, bool doPops)
+	  : key(key), keyRange(KeyRangeRef(key, keyAfter(key))), feedID(key.withPrefix("CF"_sr)), nextVal(0),
+	    lastCleared(false), poppingVersion(0), poppedVersion(0), destroying(false), destroyed(false), complete(false),
+	    checkVersion(0) {
+		if (doPops) {
+			popWindow = deterministicRandom()->randomExp(1, 8);
+			popDelayWindow = deterministicRandom()->randomInt(0, 2) * deterministicRandom()->randomExp(1, 4);
+		} else {
+			popWindow = -1;
+			popDelayWindow = -1;
+		}
+	}
+
+	Value nextValue() {
+		std::string v = std::to_string(nextVal);
+		nextVal++;
+		return Value(v);
+	}
+
+	void update(Version version, Optional<Value> value) {
+		if (!stopVersion.present()) {
+			// if feed is stopped, value should not get read
+			writesByVersion.push_back({ version, value });
+			pendingCheck.push_back(writesByVersion.back());
+			checkVersion.set(version);
+		}
+	}
+
+	void testComplete() {
+		complete = true;
+		checkVersion.set(checkVersion.get() + 1);
+	}
+
+	void pop(Database cx, Version v) {
+		if (DEBUG_CF(key)) {
+			fmt::print("DBG) {0} Popping through {1}\n", key.printable(), v);
+		}
+		ASSERT(poppingVersion < v);
+		poppingVersion = v;
+		while (!writesByVersion.empty() && v > writesByVersion.front().first) {
+			writesByVersion.pop_front();
+		}
+		while (!pendingCheck.empty() && v > pendingCheck.front().first) {
+			pendingCheck.pop_front();
+		}
+		pops.push_back(doPop(cx, key, feedID, v, &poppedVersion));
+	}
+};
+
+static void rollbackFeed(Key key,
+                         std::deque<Standalone<MutationsAndVersionRef>>& buffered,
+                         Version version,
+                         MutationRef rollbackMutation) {
+	Version rollbackVersion;
+	BinaryReader br(rollbackMutation.param2, Unversioned());
+	br >> rollbackVersion;
+	TraceEvent("ChangeFeedRollback").detail("Key", key).detail("Ver", version).detail("RollbackVer", rollbackVersion);
+	if (DEBUG_CF(key)) {
+		fmt::print("DBG) {0} Rolling back {1} -> {2}\n", key.printable(), version, rollbackVersion);
+	}
+	while (!buffered.empty() && buffered.back().version > rollbackVersion) {
+		TraceEvent("ChangeFeedRollbackVer").detail("Ver", buffered.back().version);
+		buffered.pop_back();
+	}
+}
+
+static void checkNextResult(Key key,
+                            std::deque<Standalone<MutationsAndVersionRef>>& buffered,
+                            std::deque<std::pair<Version, Optional<Value>>>& checkData) {
+	// First asserts are checking data is in the form the test is supposed to produce
+	ASSERT(!buffered.empty());
+	ASSERT(buffered.front().mutations.size() == 1);
+	ASSERT(buffered.front().mutations[0].param1 == key);
+
+	// Below asserts are correctness of change feed invariants.
+
+	// Handle case where txn retried and wrote same value twice. checkData's version is the committed one, so the same
+	// update may appear at an earlier version. This is fine, as long as it then actually appears at the committed
+	// version
+	// TODO: could strengthen this check a bit and only allow it to appear at the lower version if the txn retried on
+	// commit_unknown_result?
+	if (checkData.front().first < buffered.front().version) {
+		fmt::print("ERROR. {0} Check version {1} != {2}.\n  Check: {3} {4}\n  Buffered: {5} {6}\n",
+		           key.printable(),
+		           checkData.front().first,
+		           buffered.front().version,
+		           checkData.front().second.present() ? "SET" : "CLEAR",
+		           checkData.front().second.present() ? checkData.front().second.get().printable()
+		                                              : keyAfter(key).printable(),
+		           buffered.front().mutations[0].type == MutationRef::SetValue ? "SET" : "CLEAR",
+		           buffered.front().mutations[0].param2.printable());
+	}
+	ASSERT(checkData.front().first >= buffered.front().version);
+
+	if (checkData.front().second.present()) {
+		ASSERT(buffered.front().mutations[0].type == MutationRef::SetValue);
+		ASSERT(buffered.front().mutations[0].param2 == checkData.front().second.get());
+	} else {
+		ASSERT(buffered.front().mutations[0].type == MutationRef::ClearRange);
+		ASSERT(buffered.front().mutations[0].param2 == keyAfter(key));
+	}
+
+	if (checkData.front().first == buffered.front().version) {
+		checkData.pop_front();
+	}
+	buffered.pop_front();
+}
+
+ACTOR Future<Void> liveReader(Database cx, Reference<FeedTestData> data, Version begin) {
+	state Version lastCheckVersion = 0;
+	state Version nextCheckVersion = 0;
+	state std::deque<Standalone<MutationsAndVersionRef>> buffered;
+	state Reference<ChangeFeedData> results = makeReference<ChangeFeedData>();
+	state Future<Void> stream =
+	    cx->getChangeFeedStream(results, data->feedID, begin, std::numeric_limits<Version>::max(), data->keyRange);
+	try {
+		loop {
+			if (data->complete && data->pendingCheck.empty()) {
+				return Void();
+			}
+			nextCheckVersion = data->pendingCheck.empty() ? invalidVersion : data->pendingCheck.front().first;
+			choose {
+				when(Standalone<VectorRef<MutationsAndVersionRef>> res = waitNext(results->mutations.getFuture())) {
+					for (auto& it : res) {
+						if (it.mutations.size() == 1 && it.mutations.back().param1 == lastEpochEndPrivateKey) {
+							rollbackFeed(data->key, buffered, it.version, it.mutations.back());
+						} else {
+							if (it.mutations.size() == 0) {
+								// FIXME: THIS SHOULD NOT HAPPEN
+								// FIXME: these are also getting sent past stopVersion!!
+							} else {
+								if (data->stopVersion.present()) {
+									if (it.version > data->stopVersion.get()) {
+										fmt::print("DBG) {0} Read data with version {1} > stop version {2} ({3})\n",
+										           data->key.printable(),
+										           it.version,
+										           data->stopVersion.get(),
+										           it.mutations.size());
+									}
+									ASSERT(it.version <= data->stopVersion.get());
+								}
+								buffered.push_back(Standalone<MutationsAndVersionRef>(it));
+								if (DEBUG_CF(data->key)) {
+									fmt::print("DBG) {0} Live read through {1} ({2})\n",
+									           data->key.printable(),
+									           it.version,
+									           it.mutations.size());
+								}
+							}
+						}
+					}
+				}
+				when(wait(data->checkVersion.whenAtLeast(lastCheckVersion + 1))) {
+					// wake loop and start new whenAtLeast whenever checkVersion is set
+					lastCheckVersion = data->checkVersion.get();
+				}
+				when(wait(data->pendingCheck.empty() ? Never()
+				                                     : results->whenAtLeast(data->pendingCheck.front().first))) {
+
+					if (data->pendingCheck.empty() || data->pendingCheck.front().first > nextCheckVersion) {
+						// pendingCheck wasn't empty before whenAtLeast, and nextCheckVersion = the front version, so if
+						// either of these are true, the data was popped concurrently and we can move on to checking the
+						// next value
+						CODE_PROBE(true, "popped while waiting for whenAtLeast to check next value");
+						continue;
+					}
+					while (!buffered.empty() && buffered.front().version < data->poppingVersion) {
+						CODE_PROBE(true, "live reader ignoring data that is being popped");
+						buffered.pop_front();
+					}
+					if (buffered.empty()) {
+						if (data->poppingVersion < data->pendingCheck.front().first && !data->destroying) {
+							fmt::print("DBG) {0} Buffered empty after ready for check, and data not popped! popped "
+							           "{1}, popping {2}, check {3}\n",
+							           data->key.printable(),
+							           data->poppedVersion,
+							           data->poppingVersion,
+							           data->pendingCheck.front().first);
+						}
+						ASSERT(data->poppingVersion >= data->pendingCheck.front().first || data->destroying);
+						data->pendingCheck.pop_front();
+					} else {
+						Version v = buffered.front().version;
+						if (DEBUG_CF(data->key)) {
+							fmt::print("DBG) {0} Live checking through {1}\n",
+							           data->key.printable(),
+							           data->pendingCheck.front().first);
+						}
+						checkNextResult(data->key, buffered, data->pendingCheck);
+						if (DEBUG_CF(data->key)) {
+							fmt::print("DBG) {0} Live Checked through {1}\n", data->key.printable(), v);
+						}
+
+						if (data->popDelayWindow >= 0 && data->popWindow >= 0 &&
+						    data->writesByVersion.size() == data->popWindow + data->popDelayWindow) {
+							data->pop(cx, data->writesByVersion[data->popWindow - 1].first + 1);
+							ASSERT(data->writesByVersion.size() == data->popDelayWindow);
+						}
+					}
+				}
+			}
+		}
+	} catch (Error& e) {
+		throw e;
+	}
+}
+
+ACTOR Future<Void> historicReader(Database cx,
+                                  Reference<FeedTestData> data,
+                                  Version begin,
+                                  Version end,
+                                  bool skipPopped) {
+	state std::deque<std::pair<Version, Optional<Value>>> checkData;
+	state std::deque<Standalone<MutationsAndVersionRef>> buffered;
+	state Reference<ChangeFeedData> results = makeReference<ChangeFeedData>();
+	state Future<Void> stream = cx->getChangeFeedStream(results, data->feedID, begin, end, data->keyRange);
+	state Version poppedVersionAtStart = data->poppedVersion;
+
+	if (DEBUG_CF(data->key)) {
+		fmt::print("DBG) {0} Starting historical read {1} - {2}\n", data->key.printable(), begin, end);
+	}
+
+	// TODO could cpu optimize this
+	for (auto& it : data->writesByVersion) {
+		if (it.first >= end) {
+			break;
+		}
+		if (it.first >= begin) {
+			checkData.push_back(it);
+		}
+	}
+
+	try {
+		loop {
+			Standalone<VectorRef<MutationsAndVersionRef>> res = waitNext(results->mutations.getFuture());
+			for (auto& it : res) {
+				if (it.mutations.size() == 1 && it.mutations.back().param1 == lastEpochEndPrivateKey) {
+					rollbackFeed(data->key, buffered, it.version, it.mutations.back());
+				} else {
+					if (it.mutations.size() == 0) {
+						// FIXME: THIS SHOULD NOT HAPPEN
+						// FIXME: these are also getting sent past stopVersion!!
+					} else {
+						if (data->stopVersion.present()) {
+							ASSERT(it.version <= data->stopVersion.get());
+						}
+						buffered.push_back(Standalone<MutationsAndVersionRef>(it));
+					}
+				}
+			}
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_end_of_stream) {
+			throw;
+		}
+	}
+
+	if (skipPopped) {
+		while (!buffered.empty() && buffered.front().version < data->poppingVersion) {
+			// ignore data
+			buffered.pop_front();
+		}
+		while (!checkData.empty() && checkData.front().first < data->poppingVersion) {
+			checkData.pop_front();
+		}
+	}
+
+	while (!checkData.empty() && !buffered.empty()) {
+		checkNextResult(data->key, buffered, checkData);
+	}
+	// Change feed missing data it should have
+	ASSERT(checkData.empty());
+	// Change feed read extra data it shouldn't have
+	ASSERT(buffered.empty());
+
+	// check pop version of cursor
+	// TODO: this check might not always work if read is for old data and SS is way behind
+	// FIXME: this check doesn't work for now, probably due to above comment
+	/*if (data->poppingVersion != 0) {
+	    ASSERT(results->popVersion >= poppedVersionAtStart && results->popVersion <= data->poppingVersion);
+	}*/
+
+	return Void();
+}
+
+enum Op {
+	CREATE_DELETE = 0,
+	READ = 1,
+	UPDATE_CLEAR = 2,
+	STOP = 3,
+	POP = 4,
+	OP_COUNT = 5 /* keep this last */
+};
+
+struct ChangeFeedOperationsWorkload : TestWorkload {
+	// test settings
+	double testDuration;
+	int operationsPerSecond;
+	int targetFeeds;
+	bool clientsDisjointKeyspace;
+	bool clearKeyWhenDestroy;
+	double clearFrequency;
+	int popMode;
+
+	int opWeights[Op::OP_COUNT];
+	int totalOpWeight;
+
+	Future<Void> client;
+	std::unordered_set<Key> usedKeys;
+	std::vector<Reference<FeedTestData>> data;
+
+	ChangeFeedOperationsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		testDuration = getOption(options, "testDuration"_sr, 60.0);
+		operationsPerSecond = getOption(options, "opsPerSecond"_sr, 100.0);
+		int64_t rand = wcx.sharedRandomNumber;
+		targetFeeds = deterministicRandom()->randomExp(1, 1 + rand % 10);
+		targetFeeds *= (0.8 + (deterministicRandom()->random01() * 0.4));
+		targetFeeds = std::max(1, targetFeeds / clientCount);
+		rand /= 10;
+		clientsDisjointKeyspace = rand % 2;
+		rand /= 2;
+		clearKeyWhenDestroy = rand % 2;
+		rand /= 2;
+		bool doStops = rand % 2;
+		rand /= 2;
+		bool noCreateDelete = rand % 10 == 0;
+		rand /= 10;
+		popMode = rand % 3; // 0=none, 1=read-driven, 2=op-driven
+		rand /= 3;
+
+		ASSERT(clientId >= 0);
+		ASSERT(clientId < clientCount);
+		ASSERT(clientCount < 255);
+
+		clearFrequency = deterministicRandom()->random01();
+
+		for (int i = 0; i < Op::OP_COUNT; i++) {
+			int randWeight = deterministicRandom()->randomExp(0, 5);
+			ASSERT(randWeight > 0);
+			opWeights[i] = randWeight;
+		}
+
+		if (!doStops) {
+			opWeights[Op::STOP] = 0;
+		}
+		if (noCreateDelete) {
+			opWeights[Op::CREATE_DELETE] = 0;
+		}
+		if (popMode != 2) {
+			opWeights[Op::POP] = 0;
+		}
+
+		std::string weightString = "|";
+		totalOpWeight = 0;
+		for (int i = 0; i < Op::OP_COUNT; i++) {
+			totalOpWeight += opWeights[i];
+			weightString += std::to_string(opWeights[i]) + "|";
+		}
+
+		TraceEvent("ChangeFeedOperationsInit")
+		    .detail("TargetFeeds", targetFeeds)
+		    .detail("DisjointKeyspace", clientsDisjointKeyspace)
+		    .detail("ClearWhenDestroy", clearKeyWhenDestroy)
+		    .detail("DoStops", doStops)
+		    .detail("NoCreateDelete", noCreateDelete)
+		    .detail("Weights", weightString);
+	}
+
+	Key unusedNewRandomKey() {
+		while (true) {
+			Key k = newRandomKey();
+			if (usedKeys.insert(k).second) {
+				return k;
+			}
+		}
+	}
+
+	Key newRandomKey() {
+		if (clientsDisjointKeyspace) {
+			double keyspaceRange = (1.0 / clientCount);
+			double randPartOfRange = deterministicRandom()->random01() * (keyspaceRange - 0.0001);
+			double randomDouble = clientId * keyspaceRange + 0.0001 + randPartOfRange;
+			return doubleToTestKey(randomDouble);
+		} else {
+			// this is kinda hacky but it guarantees disjoint keys per client
+			Key ret = doubleToTestKey(deterministicRandom()->random01());
+			std::string str = ret.toString();
+			str.back() = (uint8_t)clientId;
+			return Key(str);
+		}
+	}
+
+	// Pick op with weighted average
+	Op pickRandomOp() {
+		int r = deterministicRandom()->randomInt(0, totalOpWeight);
+		int i = 0;
+		while (i < Op::OP_COUNT && (opWeights[i] <= r || opWeights[i] == 0)) {
+			r -= opWeights[i];
+			i++;
+		}
+		ASSERT(i < Op::OP_COUNT);
+		return (Op)i;
+	}
+
+	ACTOR Future<Void> createNewFeed(Database cx, ChangeFeedOperationsWorkload* self) {
+		state Transaction tr(cx);
+		state Key key = self->unusedNewRandomKey();
+		state Reference<FeedTestData> feedData = makeReference<FeedTestData>(key, self->popMode == 1);
+		state Value initialValue = feedData->nextValue();
+
+		if (DEBUG_CF(key)) {
+			fmt::print("DBG) Creating {0}\n", key.printable());
+		}
+
+		loop {
+			try {
+				tr.set(key, initialValue);
+				wait(updateChangeFeed(&tr, feedData->feedID, ChangeFeedStatus::CHANGE_FEED_CREATE, feedData->keyRange));
+				wait(tr.commit());
+
+				Version createVersion = tr.getCommittedVersion();
+				if (DEBUG_CF(key)) {
+					fmt::print("DBG) Created {0} @ {1}\n", key.printable(), createVersion);
+				}
+				feedData->update(createVersion, initialValue);
+				feedData->liveReader = liveReader(cx, feedData, createVersion);
+
+				self->data.push_back(feedData);
+
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	std::string description() const override { return "ChangeFeedOperationsWorkload"; }
+	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
+
+	ACTOR Future<Void> _setup(Database cx, ChangeFeedOperationsWorkload* self) {
+		// create initial targetFeeds feeds
+		TraceEvent("ChangeFeedOperationsSetup").detail("InitialFeeds", self->targetFeeds).log();
+		state int i;
+		for (i = 0; i < self->targetFeeds; i++) {
+			wait(self->createNewFeed(cx, self));
+		}
+		TraceEvent("ChangeFeedOperationsSetupComplete");
+		return Void();
+	}
+
+	Future<Void> start(Database const& cx) override {
+		client = changeFeedOperationsClient(cx->clone(), this);
+		return delay(testDuration);
+	}
+	Future<bool> check(Database const& cx) override {
+		client = Future<Void>();
+		return _check(cx, this);
+	}
+
+	ACTOR Future<Void> checkFeed(Database cx, ChangeFeedOperationsWorkload* self, Reference<FeedTestData> feedData) {
+		state int popIdx;
+		feedData->testComplete();
+
+		if (DEBUG_CF(feedData->key)) {
+			fmt::print("Final check {0} waiting on live reader\n", feedData->key.printable());
+		}
+		// wait on live reader and pops to make sure they complete without error
+		wait(feedData->liveReader);
+		if (DEBUG_CF(feedData->key)) {
+			fmt::print("Final check {0} waiting on {1} pops\n", feedData->key.printable(), feedData->pops.size());
+		}
+		for (popIdx = 0; popIdx < feedData->pops.size(); popIdx++) {
+			wait(feedData->pops[popIdx]);
+		}
+
+		// do final check, read everything not popped
+		if (DEBUG_CF(feedData->key)) {
+			fmt::print("Final check {0} waiting on data check\n", feedData->key.printable(), feedData->pops.size());
+		}
+		wait(self->doRead(cx, feedData, feedData->writesByVersion.size()));
+
+		// ensure reading [0, poppedVersion) returns no results
+		if (feedData->poppedVersion > 0) {
+			if (DEBUG_CF(feedData->key)) {
+				fmt::print(
+				    "Final check {0} waiting on read popped check\n", feedData->key.printable(), feedData->pops.size());
+			}
+			// FIXME: re-enable checking for popped data by changing skipPopped back to false!
+			wait(historicReader(cx, feedData, 0, feedData->poppedVersion, true));
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<bool> _check(Database cx, ChangeFeedOperationsWorkload* self) {
+		TraceEvent("ChangeFeedOperationsCheck").detail("FeedCount", self->data.size()).log();
+		fmt::print("Checking {0} feeds\n", self->data.size()); // TODO REMOVE
+		state std::vector<Future<Void>> feedChecks;
+		for (int i = 0; i < self->data.size(); i++) {
+			if (self->data[i]->destroying) {
+				continue;
+			}
+			if (DEBUG_CF(self->data[i]->key)) {
+				fmt::print("Final check {0}\n", self->data[i]->key.printable());
+			}
+			feedChecks.push_back(self->checkFeed(cx, self, self->data[i]));
+		}
+		wait(waitForAll(feedChecks));
+		// FIXME: check that all destroyed feeds are actually destroyed?
+		TraceEvent("ChangeFeedOperationsCheckComplete");
+		return true;
+	}
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	ACTOR Future<Void> stopFeed(Database cx, Reference<FeedTestData> feedData) {
+		state Transaction tr(cx);
+		if (DEBUG_CF(feedData->key)) {
+			fmt::print("DBG) {0} Stopping\n", feedData->key.printable());
+		}
+		loop {
+			try {
+				wait(updateChangeFeed(&tr, feedData->feedID, ChangeFeedStatus::CHANGE_FEED_STOP, feedData->keyRange));
+				wait(tr.commit());
+
+				Version stopVersion = tr.getCommittedVersion();
+				if (!feedData->stopVersion.present()) {
+					feedData->stopVersion = stopVersion;
+				}
+				if (DEBUG_CF(feedData->key)) {
+					fmt::print("DBG) {0} Stopped @ {1}\n", feedData->key.printable(), stopVersion);
+				}
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	void popFeed(Database cx, Reference<FeedTestData> feedData) {
+		if (!feedData->writesByVersion.empty()) {
+			feedData->pop(cx, feedData->writesByVersion.front().first + 1);
+		}
+	}
+
+	ACTOR Future<Void> destroyFeed(Database cx, ChangeFeedOperationsWorkload* self, int feedIdx) {
+		state Reference<FeedTestData> feedData = self->data[feedIdx];
+		state Transaction tr(cx);
+		feedData->destroying = true;
+		if (DEBUG_CF(feedData->key)) {
+			fmt::print("DBG) {0} Destroying\n", feedData->key.printable());
+		}
+		loop {
+			try {
+				wait(
+				    updateChangeFeed(&tr, feedData->feedID, ChangeFeedStatus::CHANGE_FEED_DESTROY, feedData->keyRange));
+				if (self->clearKeyWhenDestroy) {
+					tr.clear(feedData->key);
+				}
+				wait(tr.commit());
+
+				feedData->destroyed = true;
+				// remove feed from list
+				ASSERT(self->data[feedIdx]->key == feedData->key);
+				swapAndPop(&self->data, feedIdx);
+				if (DEBUG_CF(feedData->key)) {
+					fmt::print("DBG) {0} Destroyed @ {1}\n", feedData->key.printable(), tr.getCommittedVersion());
+				}
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR Future<Void> doRead(Database cx, Reference<FeedTestData> feedData, int targetReadWidth) {
+		if (feedData->writesByVersion.empty()) {
+			return Void();
+		}
+		Version beginVersion;
+		Version endVersion;
+		if (targetReadWidth >= feedData->writesByVersion.size()) {
+			beginVersion = feedData->writesByVersion.front().first;
+			endVersion = feedData->writesByVersion.back().first + 1;
+		} else {
+			// either up to or including end
+			int randStart = deterministicRandom()->randomInt(0, feedData->writesByVersion.size() - targetReadWidth);
+			beginVersion = feedData->writesByVersion[randStart].first;
+			int end = randStart + targetReadWidth;
+			if (end == feedData->writesByVersion.size()) {
+				endVersion = feedData->writesByVersion.back().first + 1;
+			} else {
+				// Make sure last included value (end version -1) is a committed version for checking
+				endVersion = feedData->writesByVersion[end].first + 1;
+			}
+		}
+
+		if (DEBUG_CF(feedData->key)) {
+			fmt::print("DBG) {0} Reading @ {1} - {2}\n", feedData->key.printable(), beginVersion, endVersion);
+		}
+
+		// FIXME: this sometimes reads popped data!
+		wait(historicReader(cx, feedData, beginVersion, endVersion, true));
+
+		if (DEBUG_CF(feedData->key)) {
+			fmt::print("DBG) {0} Read complete\n", feedData->key.printable());
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<Void> doUpdateClear(Database cx,
+	                                 ChangeFeedOperationsWorkload* self,
+	                                 Reference<FeedTestData> feedData) {
+		state Transaction tr(cx);
+		state Optional<Value> updateValue;
+
+		// FIXME: right now there is technically a bug in the change feed contract (mutations can appear in the stream
+		// at a higher version than the stop version) But because stopping a feed is sort of just an optimization, and
+		// no current user of change feeds currently relies on the stop version for correctness, it's fine to not test
+		// this for now
+		if (feedData->stopVersion.present()) {
+			return Void();
+		}
+
+		// if value is already not set, don't do a clear, otherwise pick either
+		if (feedData->lastCleared || deterministicRandom()->random01() > self->clearFrequency) {
+			updateValue = feedData->nextValue();
+			if (DEBUG_CF(feedData->key)) {
+				fmt::print("DBG) {0} Setting {1}\n", feedData->key.printable(), updateValue.get().printable());
+			}
+		} else if (DEBUG_CF(feedData->key)) {
+			fmt::print("DBG) {0} Clearing\n", feedData->key.printable());
+		}
+		loop {
+			try {
+				if (updateValue.present()) {
+					tr.set(feedData->key, updateValue.get());
+				} else {
+					tr.clear(feedData->key);
+				}
+
+				wait(tr.commit());
+
+				Version writtenVersion = tr.getCommittedVersion();
+
+				if (DEBUG_CF(feedData->key) && updateValue.present()) {
+					fmt::print("DBG) {0} Set {1} @ {2}\n",
+					           feedData->key.printable(),
+					           updateValue.get().printable(),
+					           writtenVersion);
+				}
+				if (DEBUG_CF(feedData->key) && !updateValue.present()) {
+					fmt::print("DBG) {0} Cleared @ {1}\n", feedData->key.printable(), writtenVersion);
+				}
+
+				feedData->update(writtenVersion, updateValue);
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR Future<Void> changeFeedOperationsClient(Database cx, ChangeFeedOperationsWorkload* self) {
+		state double last = now();
+		loop {
+			state Future<Void> waitNextOp = poisson(&last, 1.0 / self->operationsPerSecond);
+			Op op = self->pickRandomOp();
+			int feedIdx = deterministicRandom()->randomInt(0, self->data.size());
+			if (op == Op::CREATE_DELETE) {
+				// bundle these together so random creates/deletes keep about the target number of feeds
+				if (deterministicRandom()->random01() < 0.5 || self->data.size() == 1) {
+					wait(self->createNewFeed(cx, self));
+				} else {
+					wait(self->destroyFeed(cx, self, feedIdx));
+				}
+			} else if (op == Op::READ) {
+				// relatively small random read
+				wait(self->doRead(cx, self->data[feedIdx], deterministicRandom()->randomExp(2, 8)));
+			} else if (op == Op::UPDATE_CLEAR) {
+				wait(self->doUpdateClear(cx, self, self->data[feedIdx]));
+			} else if (op == Op::STOP) {
+				wait(self->stopFeed(cx, self->data[feedIdx]));
+			} else if (op == Op::POP) {
+				self->popFeed(cx, self->data[feedIdx]);
+			} else {
+				ASSERT(false);
+			}
+
+			wait(waitNextOp);
+		}
+	}
+};
+
+WorkloadFactory<ChangeFeedOperationsWorkload> ChangeFeedOperationsWorkloadFactory("ChangeFeedOperations");
diff --git a/fdbserver/workloads/ChangeFeeds.actor.cpp b/fdbserver/workloads/ChangeFeeds.actor.cpp
index 530aa54e33..580a07072a 100644
--- a/fdbserver/workloads/ChangeFeeds.actor.cpp
+++ b/fdbserver/workloads/ChangeFeeds.actor.cpp
@@ -108,7 +108,7 @@ Standalone<VectorRef<KeyValueRef>> advanceData(Standalone<VectorRef<KeyValueRef>
                                                Standalone<VectorRef<MutationsAndVersionRef>> mutations,
                                                Version begin,
                                                Version end) {
-	StringRef dbgKey = LiteralStringRef("");
+	StringRef dbgKey = ""_sr;
 	std::map<KeyRef, ValueRef> data;
 	for (auto& kv : source) {
 		if (kv.key == dbgKey)
diff --git a/fdbserver/workloads/ClearSingleRange.actor.cpp b/fdbserver/workloads/ClearSingleRange.actor.cpp
index bc4ab66d34..38565bed4c 100644
--- a/fdbserver/workloads/ClearSingleRange.actor.cpp
+++ b/fdbserver/workloads/ClearSingleRange.actor.cpp
@@ -30,9 +30,9 @@ struct ClearSingleRange : TestWorkload {
 	double startDelay;
 
 	ClearSingleRange(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		begin = getOption(options, LiteralStringRef("begin"), normalKeys.begin);
-		end = getOption(options, LiteralStringRef("end"), normalKeys.end);
-		startDelay = getOption(options, LiteralStringRef("beginClearRange"), 10.0);
+		begin = getOption(options, "begin"_sr, normalKeys.begin);
+		end = getOption(options, "end"_sr, normalKeys.end);
+		startDelay = getOption(options, "beginClearRange"_sr, 10.0);
 	}
 
 	std::string description() const override { return "ClearSingleRangeWorkload"; }
diff --git a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp
index 64da60cf7a..6366d0060b 100644
--- a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp
+++ b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp
@@ -26,8 +26,8 @@
 #include "fdbclient/Tuple.h"
 #include "flow/actorcompiler.h" // has to be last include
 
-static const Key CLIENT_LATENCY_INFO_PREFIX = LiteralStringRef("client_latency/");
-static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = LiteralStringRef("client_latency_counter/");
+static const Key CLIENT_LATENCY_INFO_PREFIX = "client_latency/"_sr;
+static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = "client_latency_counter/"_sr;
 
 /*
 FF               - 2 bytes \xff\x02
@@ -37,8 +37,7 @@ NNNN             - 4 Bytes Chunk number (Big Endian)
 TTTT             - 4 Bytes Total number of chunks (Big Endian)
 XXXX             - Variable length user provided transaction identifier
 */
-StringRef sampleTrInfoKey =
-    LiteralStringRef("\xff\x02/fdbClientInfo/client_latency/SSSSSSSSSS/RRRRRRRRRRRRRRRR/NNNNTTTT/XXXX/");
+StringRef sampleTrInfoKey = "\xff\x02/fdbClientInfo/client_latency/SSSSSSSSSS/RRRRRRRRRRRRRRRR/NNNNTTTT/XXXX/"_sr;
 static const auto chunkNumStartIndex = sampleTrInfoKey.toString().find('N');
 static const auto numChunksStartIndex = sampleTrInfoKey.toString().find('T');
 static const int chunkFormatSize = 4;
@@ -203,10 +202,10 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
 
 	ClientTransactionProfileCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		samplingProbability = getOption(options,
-		                                LiteralStringRef("samplingProbability"),
+		                                "samplingProbability"_sr,
 		                                deterministicRandom()->random01() / 10); // rand range 0 - 0.1
 		trInfoSizeLimit = getOption(options,
-		                            LiteralStringRef("trInfoSizeLimit"),
+		                            "trInfoSizeLimit"_sr,
 		                            deterministicRandom()->randomInt(100 * 1024, 10 * 1024 * 1024)); // 100 KB - 10 MB
 		TraceEvent(SevInfo, "ClientTransactionProfilingSetup")
 		    .detail("ClientId", clientId)
@@ -294,8 +293,8 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
 		wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			Tuple rate = Tuple().appendDouble(sampleProbability);
-			Tuple size = Tuple().append(sizeLimit);
+			Tuple rate = Tuple::makeTuple(sampleProbability);
+			Tuple size = Tuple::makeTuple(sizeLimit);
 			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack());
 			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack());
 			return Void();
diff --git a/fdbserver/workloads/ClientWorkload.actor.cpp b/fdbserver/workloads/ClientWorkload.actor.cpp
index 1ff537cbc6..73dbb0ea75 100644
--- a/fdbserver/workloads/ClientWorkload.actor.cpp
+++ b/fdbserver/workloads/ClientWorkload.actor.cpp
@@ -24,6 +24,7 @@
 
 #include <fmt/format.h>
 
+#include "flow/ApiVersion.h"
 #include "flow/actorcompiler.h" // has to be last include
 
 class WorkloadProcessState {
@@ -36,11 +37,11 @@ class WorkloadProcessState {
 
 	~WorkloadProcessState() {
 		TraceEvent("ShutdownClientForWorkload", id).log();
-		g_simulator.destroyProcess(childProcess);
+		g_simulator->destroyProcess(childProcess);
 	}
 
 	ACTOR static Future<Void> initializationDone(WorkloadProcessState* self, ISimulator::ProcessInfo* parent) {
-		wait(g_simulator.onProcess(parent, TaskPriority::DefaultYield));
+		wait(g_simulator->onProcess(parent, TaskPriority::DefaultYield));
 		self->init.send(Void());
 		wait(Never());
 		ASSERT(false); // does not happen
@@ -48,7 +49,7 @@ class WorkloadProcessState {
 	}
 
 	ACTOR static Future<Void> processStart(WorkloadProcessState* self) {
-		state ISimulator::ProcessInfo* parent = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
 		state std::vector<Future<Void>> futures;
 		if (parent->address.isV6()) {
 			self->childAddress =
@@ -64,22 +65,22 @@ class WorkloadProcessState {
 		TraceEvent("StartingClientWorkloadProcess", self->id)
 		    .detail("Name", self->processName)
 		    .detail("Address", self->childAddress);
-		self->childProcess = g_simulator.newProcess(self->processName.c_str(),
-		                                            self->childAddress,
-		                                            1,
-		                                            parent->address.isTLS(),
-		                                            1,
-		                                            locality,
-		                                            ProcessClass(ProcessClass::TesterClass, ProcessClass::AutoSource),
-		                                            dataFolder.c_str(),
-		                                            parent->coordinationFolder.c_str(),
-		                                            parent->protocolVersion);
+		self->childProcess = g_simulator->newProcess(self->processName.c_str(),
+		                                             self->childAddress,
+		                                             1,
+		                                             parent->address.isTLS(),
+		                                             1,
+		                                             locality,
+		                                             ProcessClass(ProcessClass::TesterClass, ProcessClass::AutoSource),
+		                                             dataFolder.c_str(),
+		                                             parent->coordinationFolder.c_str(),
+		                                             parent->protocolVersion);
 		self->childProcess->excludeFromRestarts = true;
-		wait(g_simulator.onProcess(self->childProcess, TaskPriority::DefaultYield));
+		wait(g_simulator->onProcess(self->childProcess, TaskPriority::DefaultYield));
 		try {
 			FlowTransport::createInstance(true, 1, WLTOKEN_RESERVED_COUNT);
 			Sim2FileSystem::newFileSystem();
-			auto addr = g_simulator.getCurrentProcess()->address;
+			auto addr = g_simulator->getCurrentProcess()->address;
 			futures.push_back(FlowTransport::transport().bind(addr, addr));
 			futures.push_back(success((self->childProcess->onShutdown())));
 			TraceEvent("ClientWorkloadProcessInitialized", self->id).log();
@@ -119,6 +120,7 @@ public:
 
 struct WorkloadProcess {
 	WorkloadProcessState* processState;
+	WorkloadContext childWorkloadContext;
 	UID id = deterministicRandom()->randomUniqueID();
 	Database cx;
 	Future<Void> databaseOpened;
@@ -129,7 +131,7 @@ struct WorkloadProcess {
 		try {
 			child = childCreator(wcx);
 			TraceEvent("ClientWorkloadOpenDatabase", id).detail("ClusterFileLocation", child->ccr->getLocation());
-			cx = Database::createDatabase(child->ccr, -1);
+			cx = Database::createDatabase(child->ccr, ApiVersion::LATEST_VERSION);
 			desc = child->description();
 		} catch (Error&) {
 			throw;
@@ -141,18 +143,18 @@ struct WorkloadProcess {
 	ACTOR static Future<Void> openDatabase(WorkloadProcess* self,
 	                                       ClientWorkload::CreateWorkload childCreator,
 	                                       WorkloadContext wcx) {
-		state ISimulator::ProcessInfo* parent = g_simulator.getCurrentProcess();
+		state ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
 		state Optional<Error> err;
 		wcx.dbInfo = Reference<AsyncVar<struct ServerDBInfo> const>();
 		wait(self->processState->initialized());
-		wait(g_simulator.onProcess(self->childProcess(), TaskPriority::DefaultYield));
+		wait(g_simulator->onProcess(self->childProcess(), TaskPriority::DefaultYield));
 		try {
 			self->createDatabase(childCreator, wcx);
 		} catch (Error& e) {
 			ASSERT(e.code() != error_code_actor_cancelled);
 			err = e;
 		}
-		wait(g_simulator.onProcess(parent, TaskPriority::DefaultYield));
+		wait(g_simulator->onProcess(parent, TaskPriority::DefaultYield));
 		if (err.present()) {
 			throw err.get();
 		}
@@ -166,37 +168,57 @@ struct WorkloadProcess {
 	WorkloadProcess(ClientWorkload::CreateWorkload const& childCreator, WorkloadContext const& wcx)
 	  : processState(WorkloadProcessState::instance(wcx.clientId)) {
 		TraceEvent("StartingClinetWorkload", id).detail("OnClientProcess", processState->id);
-		databaseOpened = openDatabase(this, childCreator, wcx);
+		childWorkloadContext.clientCount = wcx.clientCount;
+		childWorkloadContext.clientId = wcx.clientId;
+		childWorkloadContext.ccr = wcx.ccr;
+		childWorkloadContext.options = wcx.options;
+		childWorkloadContext.sharedRandomNumber = wcx.sharedRandomNumber;
+		databaseOpened = openDatabase(this, childCreator, childWorkloadContext);
 	}
 
 	ACTOR static void destroy(WorkloadProcess* self) {
-		state ISimulator::ProcessInfo* parent = g_simulator.getCurrentProcess();
-		wait(g_simulator.onProcess(self->childProcess(), TaskPriority::DefaultYield));
+		state ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
+		wait(g_simulator->onProcess(self->childProcess(), TaskPriority::DefaultYield));
+		TraceEvent("DeleteWorkloadProcess").backtrace();
 		delete self;
-		wait(g_simulator.onProcess(parent, TaskPriority::DefaultYield));
+		wait(g_simulator->onProcess(parent, TaskPriority::DefaultYield));
 	}
 
 	std::string description() { return desc; }
 
+	// This actor will keep a reference to a future alive, switch to another process and then return. If the future
+	// count of `f` is 1, this will cause the future to be destroyed in the process `process`
+	ACTOR template <class T>
+	static void cancelChild(ISimulator::ProcessInfo* process, Future<T> f) {
+		wait(g_simulator->onProcess(process, TaskPriority::DefaultYield));
+	}
+
 	ACTOR template <class Ret, class Fun>
 	Future<Ret> runActor(WorkloadProcess* self, Optional<TenantName> defaultTenant, Fun f) {
 		state Optional<Error> err;
 		state Ret res;
-		state ISimulator::ProcessInfo* parent = g_simulator.getCurrentProcess();
+		state Future<Ret> fut;
+		state ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
 		wait(self->databaseOpened);
-		wait(g_simulator.onProcess(self->childProcess(), TaskPriority::DefaultYield));
+		wait(g_simulator->onProcess(self->childProcess(), TaskPriority::DefaultYield));
 		self->cx->defaultTenant = defaultTenant;
 		try {
-			Ret r = wait(f(self->cx));
+			fut = f(self->cx);
+			Ret r = wait(fut);
 			res = r;
 		} catch (Error& e) {
+			// if we're getting cancelled, we could run in the scope of the parent process, but we're not allowed to
+			// cancel `fut` in any other process than the child process. So we're going to pass the future to an
+			// uncancellable actor (it has to be uncancellable because if we got cancelled here we can't wait on
+			// anything) which will then destroy the future on the child process.
+			cancelChild(self->childProcess(), fut);
 			if (e.code() == error_code_actor_cancelled) {
-				ASSERT(g_simulator.getCurrentProcess() == parent);
-				throw;
+				throw e;
 			}
 			err = e;
 		}
-		wait(g_simulator.onProcess(parent, TaskPriority::DefaultYield));
+		fut = Future<Ret>();
+		wait(g_simulator->onProcess(parent, TaskPriority::DefaultYield));
 		if (err.present()) {
 			throw err.get();
 		}
@@ -208,6 +230,7 @@ ClientWorkload::ClientWorkload(CreateWorkload const& childCreator, WorkloadConte
   : TestWorkload(wcx), impl(new WorkloadProcess(childCreator, wcx)) {}
 
 ClientWorkload::~ClientWorkload() {
+	TraceEvent(SevDebug, "DestroyClientWorkload").backtrace();
 	WorkloadProcess::destroy(impl);
 }
 
diff --git a/fdbserver/workloads/ClogSingleConnection.actor.cpp b/fdbserver/workloads/ClogSingleConnection.actor.cpp
index 0fe828ca6a..0852a562e0 100644
--- a/fdbserver/workloads/ClogSingleConnection.actor.cpp
+++ b/fdbserver/workloads/ClogSingleConnection.actor.cpp
@@ -60,10 +60,10 @@ public:
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
 	void clogRandomPair() {
-		auto m1 = deterministicRandom()->randomChoice(g_simulator.getAllProcesses());
-		auto m2 = deterministicRandom()->randomChoice(g_simulator.getAllProcesses());
+		auto m1 = deterministicRandom()->randomChoice(g_simulator->getAllProcesses());
+		auto m2 = deterministicRandom()->randomChoice(g_simulator->getAllProcesses());
 		if (m1->address.ip != m2->address.ip) {
-			g_simulator.clogPair(m1->address.ip, m2->address.ip, clogDuration.orDefault(10000));
+			g_simulator->clogPair(m1->address.ip, m2->address.ip, clogDuration.orDefault(10000));
 		}
 	}
 };
diff --git a/fdbserver/workloads/CommitBugCheck.actor.cpp b/fdbserver/workloads/CommitBugCheck.actor.cpp
index 4da04cd7bf..89a6f3b67e 100644
--- a/fdbserver/workloads/CommitBugCheck.actor.cpp
+++ b/fdbserver/workloads/CommitBugCheck.actor.cpp
@@ -36,8 +36,8 @@ struct CommitBugWorkload : TestWorkload {
 
 	ACTOR Future<Void> bug1(Database cx, CommitBugWorkload* self) {
 		state Key key = StringRef(format("B1Key%d", self->clientId));
-		state Value val1 = LiteralStringRef("Value1");
-		state Value val2 = LiteralStringRef("Value2");
+		state Value val1 = "Value1"_sr;
+		state Value val2 = "Value2"_sr;
 
 		loop {
 			state Transaction tr(cx);
@@ -49,7 +49,7 @@ struct CommitBugWorkload : TestWorkload {
 					break;
 				} catch (Error& e) {
 					TraceEvent("CommitBugSetVal1Error").error(e);
-					TEST(e.code() == error_code_commit_unknown_result); // Commit unknown result
+					CODE_PROBE(e.code() == error_code_commit_unknown_result, "Commit unknown result");
 					wait(tr.onError(e));
 				}
 			}
@@ -140,7 +140,7 @@ struct CommitBugWorkload : TestWorkload {
 
 						break;
 					} else {
-						TEST(true); // Commit conflict
+						CODE_PROBE(true, "Commit conflict");
 
 						TraceEvent("CommitBug2Error").error(e).detail("AttemptedNum", i + 1);
 						wait(tr.onError(e));
diff --git a/fdbserver/workloads/ConfigIncrement.actor.cpp b/fdbserver/workloads/ConfigIncrement.actor.cpp
index dd805224e2..3d709b03c4 100644
--- a/fdbserver/workloads/ConfigIncrement.actor.cpp
+++ b/fdbserver/workloads/ConfigIncrement.actor.cpp
@@ -36,20 +36,17 @@ class ConfigIncrementWorkload : public TestWorkload {
 
 	PerfIntCounter transactions, retries, commitUnknownResult;
 
-	static Key getConfigKey() {
-		Tuple tuple;
-		tuple.appendNull(); // config class
-		tuple << testKnobName;
-		return tuple.pack();
-	}
+	static Key getConfigKey() { return Tuple::makeTuple(/* config class */ nullptr, testKnobName).pack(); }
 
 	ACTOR static Future<int> get(Reference<ISingleThreadTransaction> tr) {
-		TraceEvent(SevDebug, "ConfigIncrementGet");
+		state TraceEvent te(SevDebug, "ConfigIncrementGet");
 		Optional<Value> serializedValue = wait(tr->get(getConfigKey()));
 		if (!serializedValue.present()) {
 			return 0;
 		} else {
-			return BinaryReader::fromStringRef<int>(serializedValue.get(), Unversioned());
+			int value = BinaryReader::fromStringRef<int>(serializedValue.get(), Unversioned());
+			te.detail("Value", value);
+			return value;
 		}
 	}
 
@@ -104,9 +101,11 @@ class ConfigIncrementWorkload : public TestWorkload {
 	}
 
 	ACTOR static Future<bool> check(ConfigIncrementWorkload* self, Database cx) {
-		state Reference<ISingleThreadTransaction> tr = self->getTransaction(cx);
+		state Reference<ISingleThreadTransaction> tr;
 		loop {
+			tr.clear();
 			try {
+				tr = self->getTransaction(cx);
 				state int currentValue = wait(get(tr));
 				auto expectedValue = self->incrementActors * self->incrementsPerActor;
 				TraceEvent("ConfigIncrementCheck")
@@ -121,9 +120,9 @@ class ConfigIncrementWorkload : public TestWorkload {
 
 	Reference<ISingleThreadTransaction> getTransaction(Database cx) const {
 		ASSERT(g_network->isSimulated()); // TODO: Enforce elsewhere
-		ASSERT(g_simulator.configDBType != ConfigDBType::DISABLED);
-		auto type = (g_simulator.configDBType == ConfigDBType::SIMPLE) ? ISingleThreadTransaction::Type::SIMPLE_CONFIG
-		                                                               : ISingleThreadTransaction::Type::PAXOS_CONFIG;
+		ASSERT(g_simulator->configDBType != ConfigDBType::DISABLED);
+		auto type = (g_simulator->configDBType == ConfigDBType::SIMPLE) ? ISingleThreadTransaction::Type::SIMPLE_CONFIG
+		                                                                : ISingleThreadTransaction::Type::PAXOS_CONFIG;
 		return ISingleThreadTransaction::create(type, cx);
 	}
 
diff --git a/fdbserver/workloads/ConfigureDatabase.actor.cpp b/fdbserver/workloads/ConfigureDatabase.actor.cpp
index 2b60f6d2f2..76fce0eb3d 100644
--- a/fdbserver/workloads/ConfigureDatabase.actor.cpp
+++ b/fdbserver/workloads/ConfigureDatabase.actor.cpp
@@ -52,9 +52,9 @@ static const char* backupTypes[] = { "backup_worker_enabled:=0", "backup_worker_
 
 std::string generateRegions() {
 	std::string result;
-	if (g_simulator.physicalDatacenters == 1 ||
-	    (g_simulator.physicalDatacenters == 2 && deterministicRandom()->random01() < 0.25) ||
-	    g_simulator.physicalDatacenters == 3) {
+	if (g_simulator->physicalDatacenters == 1 ||
+	    (g_simulator->physicalDatacenters == 2 && deterministicRandom()->random01() < 0.25) ||
+	    g_simulator->physicalDatacenters == 3) {
 		return " usable_regions=1 regions=\"\"";
 	}
 
@@ -87,7 +87,7 @@ std::string generateRegions() {
 	StatusArray remoteDcArr;
 	remoteDcArr.push_back(remoteDcObj);
 
-	if (g_simulator.physicalDatacenters > 3 && deterministicRandom()->random01() < 0.5) {
+	if (g_simulator->physicalDatacenters > 3 && deterministicRandom()->random01() < 0.5) {
 		StatusObject primarySatelliteObj;
 		primarySatelliteObj["id"] = "2";
 		primarySatelliteObj["priority"] = 1;
@@ -104,7 +104,7 @@ std::string generateRegions() {
 			remoteSatelliteObj["satellite_logs"] = deterministicRandom()->randomInt(1, 7);
 		remoteDcArr.push_back(remoteSatelliteObj);
 
-		if (g_simulator.physicalDatacenters > 5 && deterministicRandom()->random01() < 0.5) {
+		if (g_simulator->physicalDatacenters > 5 && deterministicRandom()->random01() < 0.5) {
 			StatusObject primarySatelliteObjB;
 			primarySatelliteObjB["id"] = "4";
 			primarySatelliteObjB["priority"] = 1;
@@ -124,17 +124,17 @@ std::string generateRegions() {
 			int satellite_replication_type = deterministicRandom()->randomInt(0, 3);
 			switch (satellite_replication_type) {
 			case 0: {
-				TEST(true); // Simulated cluster using no satellite redundancy mode
+				CODE_PROBE(true, "Simulated cluster using no satellite redundancy mode");
 				break;
 			}
 			case 1: {
-				TEST(true); // Simulated cluster using two satellite fast redundancy mode
+				CODE_PROBE(true, "Simulated cluster using two satellite fast redundancy mode");
 				primaryObj["satellite_redundancy_mode"] = "two_satellite_fast";
 				remoteObj["satellite_redundancy_mode"] = "two_satellite_fast";
 				break;
 			}
 			case 2: {
-				TEST(true); // Simulated cluster using two satellite safe redundancy mode
+				CODE_PROBE(true, "Simulated cluster using two satellite safe redundancy mode");
 				primaryObj["satellite_redundancy_mode"] = "two_satellite_safe";
 				remoteObj["satellite_redundancy_mode"] = "two_satellite_safe";
 				break;
@@ -147,21 +147,21 @@ std::string generateRegions() {
 			switch (satellite_replication_type) {
 			case 0: {
 				// FIXME: implement
-				TEST(true); // Simulated cluster using custom satellite redundancy mode
+				CODE_PROBE(true, "Simulated cluster using custom satellite redundancy mode");
 				break;
 			}
 			case 1: {
-				TEST(true); // Simulated cluster using no satellite redundancy mode (<5 datacenters)
+				CODE_PROBE(true, "Simulated cluster using no satellite redundancy mode (<5 datacenters)");
 				break;
 			}
 			case 2: {
-				TEST(true); // Simulated cluster using single satellite redundancy mode
+				CODE_PROBE(true, "Simulated cluster using single satellite redundancy mode");
 				primaryObj["satellite_redundancy_mode"] = "one_satellite_single";
 				remoteObj["satellite_redundancy_mode"] = "one_satellite_single";
 				break;
 			}
 			case 3: {
-				TEST(true); // Simulated cluster using double satellite redundancy mode
+				CODE_PROBE(true, "Simulated cluster using double satellite redundancy mode");
 				primaryObj["satellite_redundancy_mode"] = "one_satellite_double";
 				remoteObj["satellite_redundancy_mode"] = "one_satellite_double";
 				break;
@@ -180,20 +180,20 @@ std::string generateRegions() {
 		switch (remote_replication_type) {
 		case 0: {
 			// FIXME: implement
-			TEST(true); // Simulated cluster using custom remote redundancy mode
+			CODE_PROBE(true, "Simulated cluster using custom remote redundancy mode");
 			break;
 		}
 		case 1: {
-			TEST(true); // Simulated cluster using default remote redundancy mode
+			CODE_PROBE(true, "Simulated cluster using default remote redundancy mode");
 			break;
 		}
 		case 2: {
-			TEST(true); // Simulated cluster using single remote redundancy mode
+			CODE_PROBE(true, "Simulated cluster using single remote redundancy mode");
 			result += " remote_single";
 			break;
 		}
 		case 3: {
-			TEST(true); // Simulated cluster using double remote redundancy mode
+			CODE_PROBE(true, "Simulated cluster using double remote redundancy mode");
 			result += " remote_double";
 			break;
 		}
@@ -235,19 +235,23 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 	PerfIntCounter retries;
 
 	ConfigureDatabaseWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), retries("Retries") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 200.0);
+		testDuration = getOption(options, "testDuration"_sr, 200.0);
 		allowDescriptorChange =
-		    getOption(options, LiteralStringRef("allowDescriptorChange"), SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT);
+		    getOption(options, "allowDescriptorChange"_sr, SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT);
 		allowTestStorageMigration =
-		    getOption(options, "allowTestStorageMigration"_sr, false) && g_simulator.allowStorageMigrationTypeChange;
+		    getOption(options, "allowTestStorageMigration"_sr, false) && g_simulator->allowStorageMigrationTypeChange;
 		storageMigrationCompatibleConf = getOption(options, "storageMigrationCompatibleConf"_sr, false);
 		waitStoreTypeCheck = getOption(options, "waitStoreTypeCheck"_sr, false);
 		downgradeTest1 = getOption(options, "downgradeTest1"_sr, false);
-		g_simulator.usableRegions = 1;
+		g_simulator->usableRegions = 1;
 	}
 
 	std::string description() const override { return "DestroyDatabaseWorkload"; }
 
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
+		out.insert("MachineAttritionWorkload");
+	}
+
 	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
 
 	Future<Void> start(Database const& cx) override { return _start(this, cx); }
@@ -284,6 +288,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 	}
 
 	ACTOR Future<bool> _check(ConfigureDatabaseWorkload* self, Database cx) {
+		wait(delay(30.0));
 		// only storage_migration_type=gradual && perpetual_storage_wiggle=1 need this check because in QuietDatabase
 		// perpetual wiggle will be forced to close For other cases, later ConsistencyCheck will check KV store type
 		// there
@@ -346,7 +351,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 	ACTOR Future<Void> singleDB(ConfigureDatabaseWorkload* self, Database cx) {
 		state Transaction tr;
 		loop {
-			if (g_simulator.speedUpSimulation) {
+			if (g_simulator->speedUpSimulation) {
 				return Void();
 			}
 			state int randomChoice;
@@ -362,7 +367,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 			if (randomChoice == 0) {
 				wait(success(
 				    runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
-					    return tr->get(LiteralStringRef("This read is only to ensure that the database recovered"));
+					    return tr->get("This read is only to ensure that the database recovered"_sr);
 				    })));
 				wait(delay(20 + 10 * deterministicRandom()->random01()));
 			} else if (randomChoice < 3) {
@@ -372,14 +377,14 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 			} else if (randomChoice == 3) {
 				//TraceEvent("ConfigureTestConfigureBegin").detail("NewConfig", newConfig);
 				int maxRedundancies = sizeof(redundancies) / sizeof(redundancies[0]);
-				if (g_simulator.physicalDatacenters == 2 || g_simulator.physicalDatacenters > 3) {
+				if (g_simulator->physicalDatacenters == 2 || g_simulator->physicalDatacenters > 3) {
 					maxRedundancies--; // There are not enough machines for triple replication in fearless
 					                   // configurations
 				}
 				int redundancy = deterministicRandom()->randomInt(0, maxRedundancies);
 				std::string config = redundancies[redundancy];
 
-				if (config == "triple" && g_simulator.physicalDatacenters == 3) {
+				if (config == "triple" && g_simulator->physicalDatacenters == 3) {
 					config = "three_data_hall ";
 				}
 
@@ -437,7 +442,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 				    false)));
 			} else if (randomChoice == 8) {
 				if (self->allowTestStorageMigration) {
-					TEST(true); // storage migration type change
+					CODE_PROBE(true, "storage migration type change");
 
 					// randomly configuring perpetual_storage_wiggle_locality
 					state std::string randomPerpetualWiggleLocality;
diff --git a/fdbserver/workloads/ConflictRange.actor.cpp b/fdbserver/workloads/ConflictRange.actor.cpp
index c95ad02320..daf45740fc 100644
--- a/fdbserver/workloads/ConflictRange.actor.cpp
+++ b/fdbserver/workloads/ConflictRange.actor.cpp
@@ -39,12 +39,12 @@ struct ConflictRangeWorkload : TestWorkload {
 
 	ConflictRangeWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), withConflicts("WithConflicts"), withoutConflicts("withoutConflicts"), retries("Retries") {
-		minOperationsPerTransaction = getOption(options, LiteralStringRef("minOperationsPerTransaction"), 2);
-		maxOperationsPerTransaction = getOption(options, LiteralStringRef("minOperationsPerTransaction"), 4);
-		maxKeySpace = getOption(options, LiteralStringRef("maxKeySpace"), 100);
-		maxOffset = getOption(options, LiteralStringRef("maxOffset"), 5);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		testReadYourWrites = getOption(options, LiteralStringRef("testReadYourWrites"), false);
+		minOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 2);
+		maxOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 4);
+		maxKeySpace = getOption(options, "maxKeySpace"_sr, 100);
+		maxOffset = getOption(options, "maxOffset"_sr, 5);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		testReadYourWrites = getOption(options, "testReadYourWrites"_sr, false);
 	}
 
 	std::string description() const override { return "ConflictRange"; }
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 83479c2beb..bf1d110e10 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -98,16 +98,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	Future<Void> monitorConsistencyCheckSettingsActor;
 
 	ConsistencyCheckWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		performQuiescentChecks = getOption(options, LiteralStringRef("performQuiescentChecks"), false);
-		performCacheCheck = getOption(options, LiteralStringRef("performCacheCheck"), false);
-		performTSSCheck = getOption(options, LiteralStringRef("performTSSCheck"), true);
-		quiescentWaitTimeout = getOption(options, LiteralStringRef("quiescentWaitTimeout"), 600.0);
-		distributed = getOption(options, LiteralStringRef("distributed"), true);
-		shardSampleFactor = std::max(getOption(options, LiteralStringRef("shardSampleFactor"), 1), 1);
-		failureIsError = getOption(options, LiteralStringRef("failureIsError"), false);
-		rateLimitMax = getOption(options, LiteralStringRef("rateLimitMax"), 0);
-		shuffleShards = getOption(options, LiteralStringRef("shuffleShards"), false);
-		indefinite = getOption(options, LiteralStringRef("indefinite"), false);
+		performQuiescentChecks = getOption(options, "performQuiescentChecks"_sr, false);
+		performCacheCheck = getOption(options, "performCacheCheck"_sr, false);
+		performTSSCheck = getOption(options, "performTSSCheck"_sr, true);
+		quiescentWaitTimeout = getOption(options, "quiescentWaitTimeout"_sr, 600.0);
+		distributed = getOption(options, "distributed"_sr, true);
+		shardSampleFactor = std::max(getOption(options, "shardSampleFactor"_sr, 1), 1);
+		failureIsError = getOption(options, "failureIsError"_sr, false);
+		rateLimitMax = getOption(options, "rateLimitMax"_sr, 0);
+		shuffleShards = getOption(options, "shuffleShards"_sr, false);
+		indefinite = getOption(options, "indefinite"_sr, false);
 		suspendConsistencyCheck.set(true);
 
 		success = true;
@@ -206,8 +206,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	}
 
 	ACTOR Future<Void> runCheck(Database cx, ConsistencyCheckWorkload* self) {
-		TEST(self->performQuiescentChecks); // Quiescent consistency check
-		TEST(!self->performQuiescentChecks); // Non-quiescent consistency check
+		CODE_PROBE(self->performQuiescentChecks, "Quiescent consistency check");
+		CODE_PROBE(!self->performQuiescentChecks, "Non-quiescent consistency check");
 
 		if (self->firstClient || self->distributed) {
 			try {
@@ -321,19 +321,41 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 				// Get a list of key servers; verify that the TLogs and master all agree about who the key servers are
 				state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
-				bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise, keyServersKeys));
+				bool keyServerResult =
+				    wait(getKeyServers(cx, keyServerPromise, keyServersKeys, self->performQuiescentChecks));
 				if (keyServerResult) {
 					state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
 					    keyServerPromise.getFuture().get();
 
 					// Get the locations of all the shards in the database
 					state Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
-					bool keyLocationResult = wait(self->getKeyLocations(cx, keyServers, self, keyLocationPromise));
+					bool keyLocationResult =
+					    wait(getKeyLocations(cx, keyServers, keyLocationPromise, self->performQuiescentChecks));
 					if (keyLocationResult) {
 						state Standalone<VectorRef<KeyValueRef>> keyLocations = keyLocationPromise.getFuture().get();
 
 						// Check that each shard has the same data on all storage servers that it resides on
-						wait(::success(self->checkDataConsistency(cx, keyLocations, configuration, tssMapping, self)));
+						wait(::success(
+						    checkDataConsistency(cx,
+						                         keyLocations,
+						                         configuration,
+						                         tssMapping,
+						                         self->performQuiescentChecks,
+						                         self->performTSSCheck,
+						                         self->firstClient,
+						                         self->failureIsError,
+						                         self->clientId,
+						                         self->clientCount,
+						                         self->distributed,
+						                         self->shuffleShards,
+						                         self->shardSampleFactor,
+						                         self->sharedRandomNumber,
+						                         self->repetitions,
+						                         &(self->bytesReadInPreviousRound),
+						                         true,
+						                         self->rateLimitMax,
+						                         CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,
+						                         KeyRef())));
 
 						// Cache consistency check
 						if (self->performCacheCheck)
@@ -343,11 +365,12 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			} catch (Error& e) {
 				if (e.code() == error_code_transaction_too_old || e.code() == error_code_future_version ||
 				    e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
-				    e.code() == error_code_process_behind)
+				    e.code() == error_code_process_behind || e.code() == error_code_actor_cancelled) {
 					TraceEvent("ConsistencyCheck_Retry")
 					    .error(e); // FIXME: consistency check does not retry in this case
-				else
+				} else {
 					self->testFailure(format("Error %d - %s", e.code(), e.name()));
+				}
 			}
 		}
 
@@ -526,7 +549,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					lastSampleKey = lastStartSampleKey;
 
 					// Get the min version of the storage servers
-					Version version = wait(self->getVersion(cx, self));
+					Version version = wait(getVersion(cx));
 
 					state GetKeyValuesRequest req;
 					req.begin = begin;
@@ -744,7 +767,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	                                        bool removePrefix) {
 		// get shards paired with corresponding storage servers
 		state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
-		bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise, range));
+		bool keyServerResult = wait(getKeyServers(cx, keyServerPromise, range, self->performQuiescentChecks));
 		if (!keyServerResult)
 			return false;
 		state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> shards =
@@ -762,7 +785,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		for (i = 0; i < shards.size(); i++) {
 			while (beginKey < std::min<KeyRef>(shards[i].first.end, endKey)) {
 				try {
-					Version version = wait(self->getVersion(cx, self));
+					Version version = wait(getVersion(cx));
 
 					GetKeyValuesRequest req;
 					req.begin = firstGreaterOrEqual(beginKey);
@@ -846,884 +869,12 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		return true;
 	}
 
-	// Gets a version at which to read from the storage servers
-	ACTOR Future<Version> getVersion(Database cx, ConsistencyCheckWorkload* self) {
-		loop {
-			state Transaction tr(cx);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			try {
-				Version version = wait(tr.getReadVersion());
-				return version;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-
-	// Get a list of storage servers(persisting keys within range "kr") from the master and compares them with the
-	// TLogs. If this is a quiescent check, then each commit proxy needs to respond, otherwise only one needs to
-	// respond. Returns false if there is a failure (in this case, keyServersPromise will never be set)
-	ACTOR Future<bool> getKeyServers(
-	    Database cx,
-	    ConsistencyCheckWorkload* self,
-	    Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise,
-	    KeyRangeRef kr) {
-		state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers;
-
-		// Try getting key server locations from the master proxies
-		state std::vector<Future<ErrorOr<GetKeyServerLocationsReply>>> keyServerLocationFutures;
-		state Key begin = kr.begin;
-		state Key end = kr.end;
-		state int limitKeyServers = BUGGIFY ? 1 : 100;
-		state Span span(SpanContext(deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUInt64()),
-		                "WL:ConsistencyCheck"_loc);
-
-		while (begin < end) {
-			state Reference<CommitProxyInfo> commitProxyInfo =
-			    wait(cx->getCommitProxiesFuture(UseProvisionalProxies::False));
-			keyServerLocationFutures.clear();
-			for (int i = 0; i < commitProxyInfo->size(); i++)
-				keyServerLocationFutures.push_back(
-				    commitProxyInfo->get(i, &CommitProxyInterface::getKeyServersLocations)
-				        .getReplyUnlessFailedFor(GetKeyServerLocationsRequest(span.context,
-				                                                              Optional<TenantNameRef>(),
-				                                                              begin,
-				                                                              end,
-				                                                              limitKeyServers,
-				                                                              false,
-				                                                              latestVersion,
-				                                                              Arena()),
-				                                 2,
-				                                 0));
-
-			state bool keyServersInsertedForThisIteration = false;
-			choose {
-				when(wait(waitForAll(keyServerLocationFutures))) {
-					// Read the key server location results
-					for (int i = 0; i < keyServerLocationFutures.size(); i++) {
-						ErrorOr<GetKeyServerLocationsReply> shards = keyServerLocationFutures[i].get();
-
-						// If performing quiescent check, then all master proxies should be reachable.  Otherwise, only
-						// one needs to be reachable
-						if (self->performQuiescentChecks && !shards.present()) {
-							TraceEvent("ConsistencyCheck_CommitProxyUnavailable")
-							    .detail("CommitProxyID", commitProxyInfo->getId(i));
-							self->testFailure("Commit proxy unavailable");
-							return false;
-						}
-
-						// Get the list of shards if one was returned.  If not doing a quiescent check, we can break if
-						// it is. If we are doing a quiescent check, then we only need to do this for the first shard.
-						if (shards.present() && !keyServersInsertedForThisIteration) {
-							keyServers.insert(
-							    keyServers.end(), shards.get().results.begin(), shards.get().results.end());
-							keyServersInsertedForThisIteration = true;
-							begin = shards.get().results.back().first.end;
-
-							if (!self->performQuiescentChecks)
-								break;
-						}
-					} // End of For
-				}
-				when(wait(cx->onProxiesChanged())) {}
-			} // End of choose
-
-			if (!keyServersInsertedForThisIteration) // Retry the entire workflow
-				wait(delay(1.0));
-
-		} // End of while
-
-		keyServersPromise.send(keyServers);
-		return true;
-	}
-
-	// Retrieves the locations of all shards in the database
-	// Returns false if there is a failure (in this case, keyLocationPromise will never be set)
-	ACTOR Future<bool> getKeyLocations(Database cx,
-	                                   std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> shards,
-	                                   ConsistencyCheckWorkload* self,
-	                                   Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise) {
-		state Standalone<VectorRef<KeyValueRef>> keyLocations;
-		state Key beginKey = allKeys.begin.withPrefix(keyServersPrefix);
-		state Key endKey = allKeys.end.withPrefix(keyServersPrefix);
-		state int i = 0;
-		state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
-
-		// If the responses are too big, we may use multiple requests to get the key locations.  Each request begins
-		// where the last left off
-		for (; i < shards.size(); i++) {
-			while (beginKey < std::min<KeyRef>(shards[i].first.end, endKey)) {
-				try {
-					Version version = wait(self->getVersion(cx, self));
-
-					GetKeyValuesRequest req;
-					req.begin = firstGreaterOrEqual(beginKey);
-					req.end = firstGreaterOrEqual(std::min<KeyRef>(shards[i].first.end, endKey));
-					req.limit = SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT;
-					req.limitBytes = SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES;
-					req.version = version;
-					req.tags = TagSet();
-
-					// Try getting the shard locations from the key servers
-					state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
-					for (const auto& kv : shards[i].second) {
-						resetReply(req);
-						keyValueFutures.push_back(kv.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
-					}
-
-					wait(waitForAll(keyValueFutures));
-
-					int firstValidStorageServer = -1;
-
-					// Read the shard location results
-					for (int j = 0; j < keyValueFutures.size(); j++) {
-						ErrorOr<GetKeyValuesReply> reply = keyValueFutures[j].get();
-
-						if (!reply.present() || reply.get().error.present()) {
-							// If the storage server didn't reply in a quiescent database, then the check fails
-							if (self->performQuiescentChecks) {
-								TraceEvent("ConsistencyCheck_KeyServerUnavailable")
-								    .detail("StorageServer", shards[i].second[j].id().toString().c_str());
-								self->testFailure("Key server unavailable");
-								return false;
-							}
-
-							// If no storage servers replied, then throw all_alternatives_failed to force a retry
-							else if (firstValidStorageServer < 0 && j == keyValueFutures.size() - 1)
-								throw all_alternatives_failed();
-						}
-
-						// If this is the first storage server, store the locations to send back to the caller
-						else if (firstValidStorageServer < 0) {
-							firstValidStorageServer = j;
-
-							// Otherwise, compare the data to the results from the first storage server.  If they are
-							// different, then the check fails
-						} else if (reply.get().data != keyValueFutures[firstValidStorageServer].get().get().data ||
-						           reply.get().more != keyValueFutures[firstValidStorageServer].get().get().more) {
-							TraceEvent("ConsistencyCheck_InconsistentKeyServers")
-							    .detail("StorageServer1", shards[i].second[firstValidStorageServer].id())
-							    .detail("StorageServer2", shards[i].second[j].id());
-							self->testFailure("Key servers inconsistent", true);
-							return false;
-						}
-					}
-
-					auto keyValueResponse = keyValueFutures[firstValidStorageServer].get().get();
-					RangeResult currentLocations = krmDecodeRanges(
-					    keyServersPrefix,
-					    KeyRangeRef(beginKey.removePrefix(keyServersPrefix),
-					                std::min<KeyRef>(shards[i].first.end, endKey).removePrefix(keyServersPrefix)),
-					    RangeResultRef(keyValueResponse.data, keyValueResponse.more));
-
-					if (keyValueResponse.data.size() && beginKey == keyValueResponse.data[0].key) {
-						keyLocations.push_back_deep(keyLocations.arena(), currentLocations[0]);
-					}
-
-					if (currentLocations.size() > 2) {
-						keyLocations.append_deep(
-						    keyLocations.arena(), &currentLocations[1], currentLocations.size() - 2);
-					}
-
-					// Next iteration should pick up where we left off
-					ASSERT(currentLocations.size() > 1);
-					if (!keyValueResponse.more) {
-						beginKey = shards[i].first.end;
-					} else {
-						beginKey = keyValueResponse.data.end()[-1].key;
-					}
-
-					// If this is the last iteration, then push the allKeys.end KV pair
-					if (beginKey >= endKey)
-						keyLocations.push_back_deep(keyLocations.arena(), currentLocations.end()[-1]);
-				} catch (Error& e) {
-					state Error err = e;
-					wait(onErrorTr.onError(err));
-					TraceEvent("ConsistencyCheck_RetryGetKeyLocations").error(err);
-				}
-			}
-		}
-
-		keyLocationPromise.send(keyLocations);
-		return true;
-	}
-
-	// Retrieves a vector of the storage servers' estimates for the size of a particular shard
-	// If a storage server can't be reached, its estimate will be -1
-	// If there is an error, then the returned vector will have 0 size
-	ACTOR Future<std::vector<int64_t>> getStorageSizeEstimate(std::vector<StorageServerInterface> storageServers,
-	                                                          KeyRangeRef shard) {
-		state std::vector<int64_t> estimatedBytes;
-
-		state WaitMetricsRequest req;
-		req.keys = shard;
-		req.max.bytes = -1;
-		req.min.bytes = 0;
-
-		state std::vector<Future<ErrorOr<StorageMetrics>>> metricFutures;
-
-		try {
-			// Check the size of the shard on each storage server
-			for (int i = 0; i < storageServers.size(); i++) {
-				resetReply(req);
-				metricFutures.push_back(storageServers[i].waitMetrics.getReplyUnlessFailedFor(req, 2, 0));
-			}
-
-			// Wait for the storage servers to respond
-			wait(waitForAll(metricFutures));
-
-			int firstValidStorageServer = -1;
-
-			// Retrieve the size from the storage server responses
-			for (int i = 0; i < storageServers.size(); i++) {
-				ErrorOr<StorageMetrics> reply = metricFutures[i].get();
-
-				// If the storage server doesn't reply, then return -1
-				if (!reply.present()) {
-					TraceEvent("ConsistencyCheck_FailedToFetchMetrics")
-					    .error(reply.getError())
-					    .detail("Begin", printable(shard.begin))
-					    .detail("End", printable(shard.end))
-					    .detail("StorageServer", storageServers[i].id())
-					    .detail("IsTSS", storageServers[i].isTss() ? "True" : "False");
-					estimatedBytes.push_back(-1);
-				}
-
-				// Add the result to the list of estimates
-				else if (reply.present()) {
-					int64_t numBytes = reply.get().bytes;
-					estimatedBytes.push_back(numBytes);
-					if (firstValidStorageServer < 0)
-						firstValidStorageServer = i;
-					else if (estimatedBytes[firstValidStorageServer] != numBytes) {
-						TraceEvent("ConsistencyCheck_InconsistentStorageMetrics")
-						    .detail("ByteEstimate1", estimatedBytes[firstValidStorageServer])
-						    .detail("ByteEstimate2", numBytes)
-						    .detail("Begin", shard.begin)
-						    .detail("End", shard.end)
-						    .detail("StorageServer1", storageServers[firstValidStorageServer].id())
-						    .detail("StorageServer2", storageServers[i].id())
-						    .detail("IsTSS",
-						            storageServers[i].isTss() || storageServers[firstValidStorageServer].isTss()
-						                ? "True"
-						                : "False");
-					}
-				}
-			}
-		} catch (Error& e) {
-			TraceEvent("ConsistencyCheck_ErrorFetchingMetrics")
-			    .error(e)
-			    .detail("Begin", printable(shard.begin))
-			    .detail("End", printable(shard.end));
-			estimatedBytes.clear();
-		}
-
-		return estimatedBytes;
-	}
-
 	// Comparison function used to compare map elements by value
 	template <class K, class T>
 	static bool compareByValue(std::pair<K, T> a, std::pair<K, T> b) {
 		return a.second < b.second;
 	}
 
-	ACTOR Future<int64_t> getDatabaseSize(Database cx) {
-		state Transaction tr(cx);
-		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-		loop {
-			try {
-				StorageMetrics metrics =
-				    wait(tr.getDatabase()->getStorageMetrics(KeyRangeRef(allKeys.begin, keyServersPrefix), 100000));
-				return metrics.bytes;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-
-	// Checks that the data in each shard is the same on each storage server that it resides on.  Also performs some
-	// sanity checks on the sizes of shards and storage servers. Returns false if there is a failure
-	ACTOR Future<bool> checkDataConsistency(Database cx,
-	                                        VectorRef<KeyValueRef> keyLocations,
-	                                        DatabaseConfiguration configuration,
-	                                        std::map<UID, StorageServerInterface> tssMapping,
-	                                        ConsistencyCheckWorkload* self) {
-		// Stores the total number of bytes on each storage server
-		// In a distributed test, this will be an estimated size
-		state std::map<UID, int64_t> storageServerSizes;
-
-		// Iterate through each shard, checking its values on all of its storage servers
-		// If shardSampleFactor > 1, then not all shards are processed
-		// Also, in a distributed data consistency check, each client processes a subset of the shards
-		// Note: this may cause some shards to be processed more than once or not at all in a non-quiescent database
-		state int effectiveClientCount = (self->distributed) ? self->clientCount : 1;
-		state int i = self->clientId * (self->shardSampleFactor + 1);
-		state int increment =
-		    (self->distributed && !self->firstClient) ? effectiveClientCount * self->shardSampleFactor : 1;
-		state int rateLimitForThisRound =
-		    self->bytesReadInPreviousRound == 0
-		        ? self->rateLimitMax
-		        : std::min(
-		              self->rateLimitMax,
-		              static_cast<int>(ceil(self->bytesReadInPreviousRound /
-		                                    (float)CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME)));
-		ASSERT(rateLimitForThisRound >= 0 && rateLimitForThisRound <= self->rateLimitMax);
-		TraceEvent("ConsistencyCheck_RateLimitForThisRound").detail("RateLimit", rateLimitForThisRound);
-		state Reference<IRateControl> rateLimiter = Reference<IRateControl>(new SpeedLimit(rateLimitForThisRound, 1));
-		state double rateLimiterStartTime = now();
-		state int64_t bytesReadInthisRound = 0;
-
-		state double dbSize = 100e12;
-		if (g_network->isSimulated()) {
-			// This call will get all shard ranges in the database, which is too expensive on real clusters.
-			int64_t _dbSize = wait(self->getDatabaseSize(cx));
-			dbSize = _dbSize;
-		}
-
-		state std::vector<KeyRangeRef> ranges;
-
-		for (int k = 0; k < keyLocations.size() - 1; k++) {
-			KeyRangeRef range(keyLocations[k].key, keyLocations[k + 1].key);
-			ranges.push_back(range);
-		}
-
-		state std::vector<int> shardOrder;
-		shardOrder.reserve(ranges.size());
-		for (int k = 0; k < ranges.size(); k++)
-			shardOrder.push_back(k);
-		if (self->shuffleShards) {
-			uint32_t seed = self->sharedRandomNumber + self->repetitions;
-			DeterministicRandom sharedRandom(seed == 0 ? 1 : seed);
-			sharedRandom.randomShuffle(shardOrder);
-		}
-
-		for (; i < ranges.size(); i += increment) {
-			state int shard = shardOrder[i];
-
-			state KeyRangeRef range = ranges[shard];
-			state std::vector<UID> sourceStorageServers;
-			state std::vector<UID> destStorageServers;
-			state Transaction tr(cx);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			state int bytesReadInRange = 0;
-
-			RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
-			ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
-			decodeKeyServersValue(
-			    UIDtoTagMap, keyLocations[shard].value, sourceStorageServers, destStorageServers, false);
-
-			// If the destStorageServers is non-empty, then this shard is being relocated
-			state bool isRelocating = destStorageServers.size() > 0;
-
-			// This check was disabled because we now disable data distribution during the consistency check,
-			// which can leave shards with dest storage servers.
-
-			// Disallow relocations in a quiescent database
-			/*if(self->firstClient && self->performQuiescentChecks && isRelocating)
-			{
-			    TraceEvent("ConsistencyCheck_QuiescentShardRelocation").detail("ShardBegin", printable(range.start)).detail("ShardEnd", printable(range.end));
-			    self->testFailure("Shard is being relocated in quiescent database");
-			    return false;
-			}*/
-
-			// In a quiescent database, check that the team size is the same as the desired team size
-			if (self->firstClient && self->performQuiescentChecks &&
-			    sourceStorageServers.size() != configuration.usableRegions * configuration.storageTeamSize) {
-				TraceEvent("ConsistencyCheck_InvalidTeamSize")
-				    .detail("ShardBegin", printable(range.begin))
-				    .detail("ShardEnd", printable(range.end))
-				    .detail("SourceTeamSize", sourceStorageServers.size())
-				    .detail("DestServerSize", destStorageServers.size())
-				    .detail("ConfigStorageTeamSize", configuration.storageTeamSize)
-				    .detail("UsableRegions", configuration.usableRegions);
-				// Record the server reponsible for the problematic shards
-				int i = 0;
-				for (auto& id : sourceStorageServers) {
-					TraceEvent("IncorrectSizeTeamInfo").detail("ServerUID", id).detail("TeamIndex", i++);
-				}
-				self->testFailure("Invalid team size");
-				return false;
-			}
-
-			state std::vector<UID> storageServers = (isRelocating) ? destStorageServers : sourceStorageServers;
-			state std::vector<StorageServerInterface> storageServerInterfaces;
-
-			//TraceEvent("ConsistencyCheck_GetStorageInfo").detail("StorageServers", storageServers.size());
-			loop {
-				try {
-					std::vector<Future<Optional<Value>>> serverListEntries;
-					serverListEntries.reserve(storageServers.size());
-					for (int s = 0; s < storageServers.size(); s++)
-						serverListEntries.push_back(tr.get(serverListKeyFor(storageServers[s])));
-					state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
-					for (int s = 0; s < serverListValues.size(); s++) {
-						if (serverListValues[s].present())
-							storageServerInterfaces.push_back(decodeServerListValue(serverListValues[s].get()));
-						else if (self->performQuiescentChecks)
-							self->testFailure("/FF/serverList changing in a quiescent database");
-					}
-
-					break;
-				} catch (Error& e) {
-					wait(tr.onError(e));
-				}
-			}
-
-			// add TSS to end of list, if configured and if not relocating
-			if (!isRelocating && self->performTSSCheck) {
-				int initialSize = storageServers.size();
-				for (int i = 0; i < initialSize; i++) {
-					auto tssPair = tssMapping.find(storageServers[i]);
-					if (tssPair != tssMapping.end()) {
-						TEST(true); // TSS checked in consistency check
-						storageServers.push_back(tssPair->second.id());
-						storageServerInterfaces.push_back(tssPair->second);
-					}
-				}
-			}
-
-			state std::vector<int64_t> estimatedBytes =
-			    wait(self->getStorageSizeEstimate(storageServerInterfaces, range));
-
-			// Gets permitted size range of shard
-			int64_t maxShardSize = getMaxShardSize(dbSize);
-			state ShardSizeBounds shardBounds = getShardSizeBounds(range, maxShardSize);
-
-			if (self->firstClient) {
-				// If there was an error retrieving shard estimated size
-				if (self->performQuiescentChecks && estimatedBytes.size() == 0)
-					self->testFailure("Error fetching storage metrics");
-
-				// If running a distributed test, storage server size is an accumulation of shard estimates
-				else if (self->distributed && self->firstClient)
-					for (int j = 0; j < storageServers.size(); j++)
-						storageServerSizes[storageServers[j]] += std::max(estimatedBytes[j], (int64_t)0);
-			}
-
-			// The first client may need to skip the rest of the loop contents if it is just processing this shard to
-			// get a size estimate
-			if (!self->firstClient || shard % (effectiveClientCount * self->shardSampleFactor) == 0) {
-				state int shardKeys = 0;
-				state int shardBytes = 0;
-				state int sampledBytes = 0;
-				state int splitBytes = 0;
-				state int firstKeySampledBytes = 0;
-				state int sampledKeys = 0;
-				state int sampledKeysWithProb = 0;
-				state double shardVariance = 0;
-				state bool canSplit = false;
-				state Key lastSampleKey;
-				state Key lastStartSampleKey;
-				state int64_t totalReadAmount = 0;
-
-				state KeySelector begin = firstGreaterOrEqual(range.begin);
-				state Transaction onErrorTr(
-				    cx); // This transaction exists only to access onError and its backoff behavior
-
-				// Read a limited number of entries at a time, repeating until all keys in the shard have been read
-				loop {
-					try {
-						lastSampleKey = lastStartSampleKey;
-
-						// Get the min version of the storage servers
-						Version version = wait(self->getVersion(cx, self));
-
-						state GetKeyValuesRequest req;
-						req.begin = begin;
-						req.end = firstGreaterOrEqual(range.end);
-						req.limit = 1e4;
-						req.limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
-						req.version = version;
-						req.tags = TagSet();
-
-						// Try getting the entries in the specified range
-						state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
-						state int j = 0;
-						for (j = 0; j < storageServerInterfaces.size(); j++) {
-							resetReply(req);
-							keyValueFutures.push_back(
-							    storageServerInterfaces[j].getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
-						}
-
-						wait(waitForAll(keyValueFutures));
-
-						// Read the resulting entries
-						state int firstValidServer = -1;
-						totalReadAmount = 0;
-						for (j = 0; j < keyValueFutures.size(); j++) {
-							ErrorOr<GetKeyValuesReply> rangeResult = keyValueFutures[j].get();
-
-							// Compare the results with other storage servers
-							if (rangeResult.present() && !rangeResult.get().error.present()) {
-								state GetKeyValuesReply current = rangeResult.get();
-								totalReadAmount += current.data.expectedSize();
-								// If we haven't encountered a valid storage server yet, then mark this as the baseline
-								// to compare against
-								if (firstValidServer == -1)
-									firstValidServer = j;
-
-								// Compare this shard against the first
-								else {
-									GetKeyValuesReply reference = keyValueFutures[firstValidServer].get().get();
-
-									if (current.data != reference.data || current.more != reference.more) {
-										// Be especially verbose if in simulation
-										if (g_network->isSimulated()) {
-											int invalidIndex = -1;
-											printf("\n%sSERVER %d (%s); shard = %s - %s:\n",
-											       storageServerInterfaces[j].isTss() ? "TSS " : "",
-											       j,
-											       storageServerInterfaces[j].address().toString().c_str(),
-											       printable(req.begin.getKey()).c_str(),
-											       printable(req.end.getKey()).c_str());
-											for (int k = 0; k < current.data.size(); k++) {
-												printf("%d. %s => %s\n",
-												       k,
-												       printable(current.data[k].key).c_str(),
-												       printable(current.data[k].value).c_str());
-												if (invalidIndex < 0 &&
-												    (k >= reference.data.size() ||
-												     current.data[k].key != reference.data[k].key ||
-												     current.data[k].value != reference.data[k].value))
-													invalidIndex = k;
-											}
-
-											printf(
-											    "\n%sSERVER %d (%s); shard = %s - %s:\n",
-											    storageServerInterfaces[firstValidServer].isTss() ? "TSS " : "",
-											    firstValidServer,
-											    storageServerInterfaces[firstValidServer].address().toString().c_str(),
-											    printable(req.begin.getKey()).c_str(),
-											    printable(req.end.getKey()).c_str());
-											for (int k = 0; k < reference.data.size(); k++) {
-												printf("%d. %s => %s\n",
-												       k,
-												       printable(reference.data[k].key).c_str(),
-												       printable(reference.data[k].value).c_str());
-												if (invalidIndex < 0 &&
-												    (k >= current.data.size() ||
-												     reference.data[k].key != current.data[k].key ||
-												     reference.data[k].value != current.data[k].value))
-													invalidIndex = k;
-											}
-
-											printf("\nMISMATCH AT %d\n\n", invalidIndex);
-										}
-
-										// Data for trace event
-										// The number of keys unique to the current shard
-										int currentUniques = 0;
-										// The number of keys unique to the reference shard
-										int referenceUniques = 0;
-										// The number of keys in both shards with conflicting values
-										int valueMismatches = 0;
-										// The number of keys in both shards with matching values
-										int matchingKVPairs = 0;
-										// Last unique key on the current shard
-										KeyRef currentUniqueKey;
-										// Last unique key on the reference shard
-										KeyRef referenceUniqueKey;
-										// Last value mismatch
-										KeyRef valueMismatchKey;
-
-										// Loop indeces
-										int currentI = 0;
-										int referenceI = 0;
-										while (currentI < current.data.size() || referenceI < reference.data.size()) {
-											if (currentI >= current.data.size()) {
-												referenceUniqueKey = reference.data[referenceI].key;
-												referenceUniques++;
-												referenceI++;
-											} else if (referenceI >= reference.data.size()) {
-												currentUniqueKey = current.data[currentI].key;
-												currentUniques++;
-												currentI++;
-											} else {
-												KeyValueRef currentKV = current.data[currentI];
-												KeyValueRef referenceKV = reference.data[referenceI];
-
-												if (currentKV.key == referenceKV.key) {
-													if (currentKV.value == referenceKV.value)
-														matchingKVPairs++;
-													else {
-														valueMismatchKey = currentKV.key;
-														valueMismatches++;
-													}
-
-													currentI++;
-													referenceI++;
-												} else if (currentKV.key < referenceKV.key) {
-													currentUniqueKey = currentKV.key;
-													currentUniques++;
-													currentI++;
-												} else {
-													referenceUniqueKey = referenceKV.key;
-													referenceUniques++;
-													referenceI++;
-												}
-											}
-										}
-
-										TraceEvent("ConsistencyCheck_DataInconsistent")
-										    .detail(format("StorageServer%d", j).c_str(), storageServers[j].toString())
-										    .detail(format("StorageServer%d", firstValidServer).c_str(),
-										            storageServers[firstValidServer].toString())
-										    .detail("ShardBegin", req.begin.getKey())
-										    .detail("ShardEnd", req.end.getKey())
-										    .detail("VersionNumber", req.version)
-										    .detail(format("Server%dUniques", j).c_str(), currentUniques)
-										    .detail(format("Server%dUniqueKey", j).c_str(), currentUniqueKey)
-										    .detail(format("Server%dUniques", firstValidServer).c_str(),
-										            referenceUniques)
-										    .detail(format("Server%dUniqueKey", firstValidServer).c_str(),
-										            referenceUniqueKey)
-										    .detail("ValueMismatches", valueMismatches)
-										    .detail("ValueMismatchKey", valueMismatchKey)
-										    .detail("MatchingKVPairs", matchingKVPairs)
-										    .detail("IsTSS",
-										            storageServerInterfaces[j].isTss() ||
-										                    storageServerInterfaces[firstValidServer].isTss()
-										                ? "True"
-										                : "False");
-
-										if ((g_network->isSimulated() &&
-										     g_simulator.tssMode != ISimulator::TSSMode::EnabledDropMutations) ||
-										    (!storageServerInterfaces[j].isTss() &&
-										     !storageServerInterfaces[firstValidServer].isTss())) {
-											self->testFailure("Data inconsistent", true);
-											return false;
-										}
-									}
-								}
-							}
-
-							// If the data is not available and we aren't relocating this shard
-							else if (!isRelocating) {
-								Error e =
-								    rangeResult.isError() ? rangeResult.getError() : rangeResult.get().error.get();
-
-								TraceEvent("ConsistencyCheck_StorageServerUnavailable")
-								    .errorUnsuppressed(e)
-								    .suppressFor(1.0)
-								    .detail("StorageServer", storageServers[j])
-								    .detail("ShardBegin", printable(range.begin))
-								    .detail("ShardEnd", printable(range.end))
-								    .detail("Address", storageServerInterfaces[j].address())
-								    .detail("UID", storageServerInterfaces[j].id())
-								    .detail("GetKeyValuesToken",
-								            storageServerInterfaces[j].getKeyValues.getEndpoint().token)
-								    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False");
-
-								// All shards should be available in quiscence
-								if (self->performQuiescentChecks && !storageServerInterfaces[j].isTss()) {
-									self->testFailure("Storage server unavailable");
-									return false;
-								}
-							}
-						}
-
-						if (firstValidServer >= 0) {
-							VectorRef<KeyValueRef> data = keyValueFutures[firstValidServer].get().get().data;
-							// Calculate the size of the shard, the variance of the shard size estimate, and the correct
-							// shard size estimate
-							for (int k = 0; k < data.size(); k++) {
-								ByteSampleInfo sampleInfo = isKeyValueInSample(data[k]);
-								shardBytes += sampleInfo.size;
-								double itemProbability = ((double)sampleInfo.size) / sampleInfo.sampledSize;
-								if (itemProbability < 1)
-									shardVariance += itemProbability * (1 - itemProbability) *
-									                 pow((double)sampleInfo.sampledSize, 2);
-
-								if (sampleInfo.inSample) {
-									sampledBytes += sampleInfo.sampledSize;
-									if (!canSplit && sampledBytes >= shardBounds.min.bytes &&
-									    data[k].key.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT &&
-									    sampledBytes <= shardBounds.max.bytes *
-									                        CLIENT_KNOBS->STORAGE_METRICS_UNFAIR_SPLIT_LIMIT / 2) {
-										canSplit = true;
-										splitBytes = sampledBytes;
-									}
-
-									/*TraceEvent("ConsistencyCheck_ByteSample").detail("ShardBegin", printable(range.begin)).detail("ShardEnd", printable(range.end))
-									    .detail("SampledBytes", sampleInfo.sampledSize).detail("Key",
-									   printable(data[k].key)).detail("KeySize", data[k].key.size()).detail("ValueSize",
-									   data[k].value.size());*/
-
-									// In data distribution, the splitting process ignores the first key in a shard.
-									// Thus, we shouldn't consider it when validating the upper bound of estimated shard
-									// sizes
-									if (k == 0)
-										firstKeySampledBytes += sampleInfo.sampledSize;
-
-									sampledKeys++;
-									if (itemProbability < 1) {
-										sampledKeysWithProb++;
-									}
-								}
-							}
-
-							// Accumulate number of keys in this shard
-							shardKeys += data.size();
-						}
-						// after requesting each shard, enforce rate limit based on how much data will likely be read
-						if (rateLimitForThisRound > 0) {
-							wait(rateLimiter->getAllowance(totalReadAmount));
-							// Set ratelimit to max allowed if current round has been going on for a while
-							if (now() - rateLimiterStartTime >
-							        1.1 * CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME &&
-							    rateLimitForThisRound != self->rateLimitMax) {
-								rateLimitForThisRound = self->rateLimitMax;
-								rateLimiter = Reference<IRateControl>(new SpeedLimit(rateLimitForThisRound, 1));
-								rateLimiterStartTime = now();
-								TraceEvent(SevInfo, "ConsistencyCheck_RateLimitSetMaxForThisRound")
-								    .detail("RateLimit", rateLimitForThisRound);
-							}
-						}
-						bytesReadInRange += totalReadAmount;
-						bytesReadInthisRound += totalReadAmount;
-
-						// Advance to the next set of entries
-						if (firstValidServer >= 0 && keyValueFutures[firstValidServer].get().get().more) {
-							VectorRef<KeyValueRef> result = keyValueFutures[firstValidServer].get().get().data;
-							ASSERT(result.size() > 0);
-							begin = firstGreaterThan(result[result.size() - 1].key);
-							ASSERT(begin.getKey() != allKeys.end);
-							lastStartSampleKey = lastSampleKey;
-						} else
-							break;
-					} catch (Error& e) {
-						state Error err = e;
-						wait(onErrorTr.onError(err));
-						TraceEvent("ConsistencyCheck_RetryDataConsistency").error(err);
-					}
-				}
-
-				canSplit = canSplit && sampledBytes - splitBytes >= shardBounds.min.bytes && sampledBytes > splitBytes;
-
-				// Update the size of all storage servers containing this shard
-				// This is only done in a non-distributed consistency check; the distributed check uses shard size
-				// estimates
-				if (!self->distributed)
-					for (int j = 0; j < storageServers.size(); j++)
-						storageServerSizes[storageServers[j]] += shardBytes;
-
-				// FIXME: Where is this intended to be used?
-				[[maybe_unused]] bool hasValidEstimate = estimatedBytes.size() > 0;
-
-				// If the storage servers' sampled estimate of shard size is different from ours
-				if (self->performQuiescentChecks) {
-					for (int j = 0; j < estimatedBytes.size(); j++) {
-						if (estimatedBytes[j] >= 0 && estimatedBytes[j] != sampledBytes) {
-							TraceEvent("ConsistencyCheck_IncorrectEstimate")
-							    .detail("EstimatedBytes", estimatedBytes[j])
-							    .detail("CorrectSampledBytes", sampledBytes)
-							    .detail("StorageServer", storageServers[j])
-							    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False");
-
-							if (!storageServerInterfaces[j].isTss()) {
-								self->testFailure("Storage servers had incorrect sampled estimate");
-							}
-
-							hasValidEstimate = false;
-
-							break;
-						} else if (estimatedBytes[j] < 0 &&
-						           ((g_network->isSimulated() &&
-						             g_simulator.tssMode <= ISimulator::TSSMode::EnabledNormal) ||
-						            !storageServerInterfaces[j].isTss())) {
-							// Ignore a non-responding TSS outside of simulation, or if tss fault injection is enabled
-							hasValidEstimate = false;
-							break;
-						}
-					}
-				}
-
-				// Compute the difference between the shard size estimate and its actual size.  If it is sufficiently
-				// large, then fail
-				double stdDev = sqrt(shardVariance);
-
-				double failErrorNumStdDev = 7;
-				int estimateError = abs(shardBytes - sampledBytes);
-
-				// Only perform the check if there are sufficient keys to get a distribution that should resemble a
-				// normal distribution
-				if (sampledKeysWithProb > 30 && estimateError > failErrorNumStdDev * stdDev) {
-					double numStdDev = estimateError / sqrt(shardVariance);
-					TraceEvent("ConsistencyCheck_InaccurateShardEstimate")
-					    .detail("Min", shardBounds.min.bytes)
-					    .detail("Max", shardBounds.max.bytes)
-					    .detail("Estimate", sampledBytes)
-					    .detail("Actual", shardBytes)
-					    .detail("NumStdDev", numStdDev)
-					    .detail("Variance", shardVariance)
-					    .detail("StdDev", stdDev)
-					    .detail("ShardBegin", printable(range.begin))
-					    .detail("ShardEnd", printable(range.end))
-					    .detail("NumKeys", shardKeys)
-					    .detail("NumSampledKeys", sampledKeys)
-					    .detail("NumSampledKeysWithProb", sampledKeysWithProb);
-
-					self->testFailure(format("Shard size is more than %f std dev from estimate", failErrorNumStdDev));
-				}
-
-				// In a quiescent database, check that the (estimated) size of the shard is within permitted bounds
-				// Min and max shard sizes have a 3 * shardBounds.permittedError.bytes cushion for error since shard
-				// sizes are not precise Shard splits ignore the first key in a shard, so its size shouldn't be
-				// considered when checking the upper bound 0xff shards are not checked
-				if (canSplit && sampledKeys > 5 && self->performQuiescentChecks &&
-				    !range.begin.startsWith(keyServersPrefix) &&
-				    (sampledBytes < shardBounds.min.bytes - 3 * shardBounds.permittedError.bytes ||
-				     sampledBytes - firstKeySampledBytes >
-				         shardBounds.max.bytes + 3 * shardBounds.permittedError.bytes)) {
-					TraceEvent("ConsistencyCheck_InvalidShardSize")
-					    .detail("Min", shardBounds.min.bytes)
-					    .detail("Max", shardBounds.max.bytes)
-					    .detail("Size", shardBytes)
-					    .detail("EstimatedSize", sampledBytes)
-					    .detail("ShardBegin", printable(range.begin))
-					    .detail("ShardEnd", printable(range.end))
-					    .detail("ShardCount", ranges.size())
-					    .detail("SampledKeys", sampledKeys);
-					self->testFailure(format("Shard size in quiescent database is too %s",
-					                         (sampledBytes < shardBounds.min.bytes) ? "small" : "large"));
-					return false;
-				}
-			}
-
-			if (bytesReadInRange > 0) {
-				TraceEvent("ConsistencyCheck_ReadRange")
-				    .suppressFor(1.0)
-				    .detail("Range", range)
-				    .detail("BytesRead", bytesReadInRange);
-			}
-		}
-
-		// SOMEDAY: when background data distribution is implemented, include this test
-		// In a quiescent database, check that the sizes of storage servers are roughly the same
-		/*if(self->performQuiescentChecks)
-		{
-		    auto minStorageServer = std::min_element(storageServerSizes.begin(), storageServerSizes.end(),
-		ConsistencyCheckWorkload::compareByValue<UID, int64_t>); auto maxStorageServer =
-		std::max_element(storageServerSizes.begin(), storageServerSizes.end(),
-		ConsistencyCheckWorkload::compareByValue<UID, int64_t>);
-
-		    int bias = SERVER_KNOBS->MIN_SHARD_BYTES;
-		    if(1.1 * (minStorageServer->second + SERVER_KNOBS->MIN_SHARD_BYTES) < maxStorageServer->second +
-		SERVER_KNOBS->MIN_SHARD_BYTES)
-		    {
-		        TraceEvent("ConsistencyCheck_InconsistentStorageServerSizes").detail("MinSize", minStorageServer->second).detail("MaxSize", maxStorageServer->second)
-		            .detail("MinStorageServer", minStorageServer->first).detail("MaxStorageServer",
-		maxStorageServer->first);
-
-		        self->testFailure(format("Storage servers differ significantly in size by a factor of %f",
-		((double)maxStorageServer->second) / minStorageServer->second)); return false;
-		    }
-		}*/
-
-		self->bytesReadInPreviousRound = bytesReadInthisRound;
-		return true;
-	}
-
 	// Returns true if any storage servers have the exact same network address or are not using the correct key value
 	// store type
 	ACTOR Future<bool> checkForUndesirableServers(Database cx,
@@ -1933,7 +1084,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					// FIXME: this is hiding the fact that we can recruit a new storage server on a location the has
 					// files left behind by a previous failure
 					// this means that the process is wasting disk space until the process is rebooting
-					ISimulator::ProcessInfo* p = g_simulator.getProcessByAddress(itr->interf.address());
+					ISimulator::ProcessInfo* p = g_simulator->getProcessByAddress(itr->interf.address());
 					// Note: itr->interf.address() may not equal to p->address() because role's endpoint's primary
 					// addr can be swapped by choosePrimaryAddress() based on its peer's tls config.
 					TraceEvent("ConsistencyCheck_RebootProcess")
@@ -1942,14 +1093,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("ProcessPrimaryAddress", p->address)
 					    .detail("ProcessAddresses", p->addresses.toString())
 					    .detail("DataStoreID", id)
-					    .detail("Protected", g_simulator.protectedAddresses.count(itr->interf.address()))
+					    .detail("Protected", g_simulator->protectedAddresses.count(itr->interf.address()))
 					    .detail("Reliable", p->isReliable())
 					    .detail("ReliableInfo", p->getReliableInfo())
 					    .detail("KillOrRebootProcess", p->address);
 					if (p->isReliable()) {
-						g_simulator.rebootProcess(p, ISimulator::RebootProcess);
+						g_simulator->rebootProcess(p, ISimulator::RebootProcess);
 					} else {
-						g_simulator.killProcess(p, ISimulator::KillInstantly);
+						g_simulator->killProcess(p, ISimulator::KillInstantly);
 					}
 				}
 
@@ -1958,6 +1109,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 
 		if (foundExtraDataStore) {
+			wait(delay(10)); // let the cluster get to fully_recovered after the reboot before retrying
 			self->testFailure("Extra data stores present on workers");
 			return false;
 		}
@@ -2039,15 +1191,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	}
 
 	ACTOR Future<bool> checkWorkerList(Database cx, ConsistencyCheckWorkload* self) {
-		if (g_simulator.extraDB)
+		if (!g_simulator->extraDatabases.empty()) {
 			return true;
+		}
 
 		std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
 		std::set<NetworkAddress> workerAddresses;
 
 		for (const auto& it : workers) {
 			NetworkAddress addr = it.interf.tLog.getEndpoint().addresses.getTLSAddress();
-			ISimulator::ProcessInfo* info = g_simulator.getProcessByAddress(addr);
+			ISimulator::ProcessInfo* info = g_simulator->getProcessByAddress(addr);
 			if (!info || info->failed) {
 				TraceEvent("ConsistencyCheck_FailedWorkerInList").detail("Addr", it.interf.address());
 				return false;
@@ -2055,7 +1208,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			workerAddresses.insert(NetworkAddress(addr.ip, addr.port, true, addr.isTLS()));
 		}
 
-		std::vector<ISimulator::ProcessInfo*> all = g_simulator.getAllProcesses();
+		std::vector<ISimulator::ProcessInfo*> all = g_simulator->getAllProcesses();
 		for (int i = 0; i < all.size(); i++) {
 			if (all[i]->isReliable() && all[i]->name == std::string("Server") &&
 			    all[i]->startingClass != ProcessClass::TesterClass &&
@@ -2185,10 +1338,10 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 		// Check if master and cluster controller are in the desired DC for fearless cluster when running under
 		// simulation
-		// FIXME: g_simulator.datacenterDead could return false positives. Relaxing checks until it is fixed.
-		if (g_network->isSimulated() && config.usableRegions > 1 && g_simulator.primaryDcId.present() &&
-		    !g_simulator.datacenterDead(g_simulator.primaryDcId) &&
-		    !g_simulator.datacenterDead(g_simulator.remoteDcId)) {
+		// FIXME: g_simulator->datacenterDead could return false positives. Relaxing checks until it is fixed.
+		if (g_network->isSimulated() && config.usableRegions > 1 && g_simulator->primaryDcId.present() &&
+		    !g_simulator->datacenterDead(g_simulator->primaryDcId) &&
+		    !g_simulator->datacenterDead(g_simulator->remoteDcId)) {
 			expectedPrimaryDcId = config.regions[0].dcId;
 			expectedRemoteDcId = config.regions[1].dcId;
 			// If the priorities are equal, either could be the primary
@@ -2307,9 +1460,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 
 		// Check LogRouter
-		if (g_network->isSimulated() && config.usableRegions > 1 && g_simulator.primaryDcId.present() &&
-		    !g_simulator.datacenterDead(g_simulator.primaryDcId) &&
-		    !g_simulator.datacenterDead(g_simulator.remoteDcId)) {
+		if (g_network->isSimulated() && config.usableRegions > 1 && g_simulator->primaryDcId.present() &&
+		    !g_simulator->datacenterDead(g_simulator->primaryDcId) &&
+		    !g_simulator->datacenterDead(g_simulator->remoteDcId)) {
 			for (auto& tlogSet : db.logSystemConfig.tLogs) {
 				if (!tlogSet.isLocal && tlogSet.logRouters.size()) {
 					for (auto& logRouter : tlogSet.logRouters) {
diff --git a/fdbserver/workloads/CpuProfiler.actor.cpp b/fdbserver/workloads/CpuProfiler.actor.cpp
index bc7cbfea15..f5ebbf2174 100644
--- a/fdbserver/workloads/CpuProfiler.actor.cpp
+++ b/fdbserver/workloads/CpuProfiler.actor.cpp
@@ -42,9 +42,9 @@ struct CpuProfilerWorkload : TestWorkload {
 	std::vector<WorkerInterface> profilingWorkers;
 
 	CpuProfilerWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		initialDelay = getOption(options, LiteralStringRef("initialDelay"), 0.0);
-		duration = getOption(options, LiteralStringRef("duration"), -1.0);
-		roles = getOption(options, LiteralStringRef("roles"), std::vector<std::string>());
+		initialDelay = getOption(options, "initialDelay"_sr, 0.0);
+		duration = getOption(options, "duration"_sr, -1.0);
+		roles = getOption(options, "roles"_sr, std::vector<std::string>());
 		success = true;
 	}
 
diff --git a/fdbserver/workloads/CreateTenant.actor.cpp b/fdbserver/workloads/CreateTenant.actor.cpp
new file mode 100644
index 0000000000..624ebce177
--- /dev/null
+++ b/fdbserver/workloads/CreateTenant.actor.cpp
@@ -0,0 +1,62 @@
+/*
+ * CreateTenant.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdint>
+
+#include "fdbclient/TenantManagement.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct CreateTenantWorkload : TestWorkload {
+	TenantName tenant;
+
+	CreateTenantWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		tenant = getOption(options, "name"_sr, "DefaultTenant"_sr);
+	}
+
+	std::string description() const override { return "CreateTenant"; }
+	Future<Void> setup(Database const& cx) override {
+		if (clientId == 0) {
+			return _setup(this, cx);
+		}
+		return Void();
+	}
+
+	Future<Void> start(Database const& cx) override { return Void(); }
+	Future<bool> check(Database const& cx) override { return true; }
+	virtual void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	ACTOR static Future<Void> _setup(CreateTenantWorkload* self, Database db) {
+		try {
+			Optional<TenantMapEntry> entry = wait(TenantAPI::createTenant(db.getReference(), self->tenant));
+			ASSERT(entry.present());
+		} catch (Error& e) {
+			TraceEvent(SevError, "TenantCreationFailed").error(e);
+			if (e.code() == error_code_actor_cancelled) {
+				throw;
+			}
+			ASSERT(false);
+		}
+		return Void();
+	}
+};
+
+WorkloadFactory<CreateTenantWorkload> CreateTenantWorkload("CreateTenant");
diff --git a/fdbserver/workloads/Cycle.actor.cpp b/fdbserver/workloads/Cycle.actor.cpp
index 6d2fb054f3..3ceaead9f0 100644
--- a/fdbserver/workloads/Cycle.actor.cpp
+++ b/fdbserver/workloads/Cycle.actor.cpp
@@ -20,19 +20,34 @@
 
 #include <cstring>
 
+#include "flow/Arena.h"
+#include "flow/IRandom.h"
+#include "flow/Trace.h"
+#include "flow/serialize.h"
+#include "fdbrpc/simulator.h"
+#include "fdbrpc/TokenSign.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/Arena.h"
-#include "flow/IRandom.h"
-#include "flow/Trace.h"
-#include "flow/serialize.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-struct CycleWorkload : TestWorkload {
+template <bool MultiTenancy>
+struct CycleMembers {};
+
+template <>
+struct CycleMembers<true> {
+	Arena arena;
+	TenantName tenant;
+	authz::jwt::TokenRef token;
+	StringRef signedToken;
+	bool useToken;
+};
+
+template <bool MultiTenancy>
+struct CycleWorkload : TestWorkload, CycleMembers<MultiTenancy> {
 	int actorCount, nodeCount;
 	double testDuration, transactionsPerSecond, minExpectedTransactionsPerSecond, traceParentProbability;
 	Key keyPrefix;
@@ -48,20 +63,57 @@ struct CycleWorkload : TestWorkload {
 		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0) / clientCount;
 		actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5);
 		nodeCount = getOption(options, "nodeCount"_sr, transactionsPerSecond * clientCount);
-		keyPrefix = unprintable(getOption(options, "keyPrefix"_sr, LiteralStringRef("")).toString());
+		keyPrefix = unprintable(getOption(options, "keyPrefix"_sr, ""_sr).toString());
 		traceParentProbability = getOption(options, "traceParentProbability"_sr, 0.01);
 		minExpectedTransactionsPerSecond = transactionsPerSecond * getOption(options, "expectedRate"_sr, 0.7);
+		if constexpr (MultiTenancy) {
+			ASSERT(g_network->isSimulated());
+			this->useToken = getOption(options, "useToken"_sr, true);
+			auto k = g_simulator->authKeys.begin();
+			this->tenant = getOption(options, "tenant"_sr, "CycleTenant"_sr);
+			// make it comfortably longer than the timeout of the workload
+			auto currentTime = uint64_t(lround(g_network->timer()));
+			this->token.algorithm = authz::Algorithm::ES256;
+			this->token.issuedAtUnixTime = currentTime;
+			this->token.expiresAtUnixTime =
+			    currentTime + uint64_t(std::lround(getCheckTimeout())) + uint64_t(std::lround(testDuration)) + 100;
+			this->token.keyId = k->first;
+			this->token.notBeforeUnixTime = currentTime - 10;
+			VectorRef<StringRef> tenants;
+			tenants.push_back_deep(this->arena, this->tenant);
+			this->token.tenants = tenants;
+			// we currently don't support this workload to be run outside of simulation
+			this->signedToken = authz::jwt::signToken(this->arena, this->token, k->second);
+		}
 	}
 
-	std::string description() const override { return "CycleWorkload"; }
-	Future<Void> setup(Database const& cx) override { return bulkSetup(cx, this, nodeCount, Promise<double>()); }
+	std::string description() const override {
+		if constexpr (MultiTenancy) {
+			return "TenantCycleWorkload";
+		} else {
+			return "CycleWorkload";
+		}
+	}
+
+	Future<Void> setup(Database const& cx) override {
+		if constexpr (MultiTenancy) {
+			cx->defaultTenant = this->tenant;
+		}
+		return bulkSetup(cx, this, nodeCount, Promise<double>());
+	}
 	Future<Void> start(Database const& cx) override {
+		if constexpr (MultiTenancy) {
+			cx->defaultTenant = this->tenant;
+		}
 		for (int c = 0; c < actorCount; c++)
 			clients.push_back(
 			    timeout(cycleClient(cx->clone(), this, actorCount / transactionsPerSecond), testDuration, Void()));
 		return delay(testDuration);
 	}
 	Future<bool> check(Database const& cx) override {
+		if constexpr (MultiTenancy) {
+			cx->defaultTenant = this->tenant;
+		}
 		int errors = 0;
 		for (int c = 0; c < clients.size(); c++)
 			errors += clients[c].isError();
@@ -95,6 +147,15 @@ struct CycleWorkload : TestWorkload {
 		    .detailf("From", "%016llx", debug_lastLoadBalanceResultEndpointToken);
 	}
 
+	template <bool B = MultiTenancy>
+	std::enable_if_t<B> setAuthToken(Transaction& tr) const {
+		if (this->useToken)
+			tr.setOption(FDBTransactionOptions::AUTHORIZATION_TOKEN, this->signedToken);
+	}
+
+	template <bool B = MultiTenancy>
+	std::enable_if_t<!B> setAuthToken(Transaction& tr) const {}
+
 	ACTOR Future<Void> cycleClient(Database cx, CycleWorkload* self, double delay) {
 		state double lastTime = now();
 		try {
@@ -104,6 +165,7 @@ struct CycleWorkload : TestWorkload {
 				state double tstart = now();
 				state int r = deterministicRandom()->randomInt(0, self->nodeCount);
 				state Transaction tr(cx);
+				self->setAuthToken(tr);
 				if (deterministicRandom()->random01() <= self->traceParentProbability) {
 					state Span span("CycleClient"_loc);
 					TraceEvent("CycleTracingTransaction", span.context.traceID).log();
@@ -231,6 +293,7 @@ struct CycleWorkload : TestWorkload {
 		}
 		return true;
 	}
+
 	ACTOR Future<bool> cycleCheck(Database cx, CycleWorkload* self, bool ok) {
 		if (self->transactions.getMetric().value() < self->testDuration * self->minExpectedTransactionsPerSecond) {
 			TraceEvent(SevWarnAlways, "TestFailure")
@@ -249,6 +312,7 @@ struct CycleWorkload : TestWorkload {
 			// One client checks the validity of the cycle
 			state Transaction tr(cx);
 			state int retryCount = 0;
+			self->setAuthToken(tr);
 			loop {
 				try {
 					state Version v = wait(tr.getReadVersion());
@@ -261,9 +325,9 @@ struct CycleWorkload : TestWorkload {
 					retryCount++;
 					TraceEvent(retryCount > 20 ? SevWarnAlways : SevWarn, "CycleCheckError").error(e);
 					if (g_network->isSimulated() && retryCount > 50) {
-						TEST(true); // Cycle check enable speedUpSimulation because too many transaction_too_old()
+						CODE_PROBE(true, "Cycle check enable speedUpSimulation because too many transaction_too_old()");
 						// try to make the read window back to normal size (5 * version_per_sec)
-						g_simulator.speedUpSimulation = true;
+						g_simulator->speedUpSimulation = true;
 					}
 					wait(tr.onError(e));
 				}
@@ -273,4 +337,5 @@ struct CycleWorkload : TestWorkload {
 	}
 };
 
-WorkloadFactory<CycleWorkload> CycleWorkloadFactory("Cycle", true);
+WorkloadFactory<CycleWorkload<false>> CycleWorkloadFactory("Cycle", false);
+WorkloadFactory<CycleWorkload<true>> TenantCycleWorkloadFactory("TenantCycle", true);
diff --git a/fdbserver/workloads/DDBalance.actor.cpp b/fdbserver/workloads/DDBalance.actor.cpp
index 62b9a03ccd..1b7b992e4a 100644
--- a/fdbserver/workloads/DDBalance.actor.cpp
+++ b/fdbserver/workloads/DDBalance.actor.cpp
@@ -36,17 +36,17 @@ struct DDBalanceWorkload : TestWorkload {
 
 	DDBalanceWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), bin_shifts("Bin_Shifts"), operations("Operations"), retries("Retries"), latencies(2000) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		binCount = getOption(options, LiteralStringRef("binCount"), 1000);
-		writesPerTransaction = getOption(options, LiteralStringRef("writesPerTransaction"), 1);
-		keySpaceDriftFactor = getOption(options, LiteralStringRef("keySpaceDriftFactor"), 1);
-		moversPerClient = std::max(getOption(options, LiteralStringRef("moversPerClient"), 10), 1);
-		actorsPerClient = std::max(getOption(options, LiteralStringRef("actorsPerClient"), 100), 1);
-		int nodes = getOption(options, LiteralStringRef("nodes"), 10000);
-		discardEdgeMeasurements = getOption(options, LiteralStringRef("discardEdgeMeasurements"), true);
-		warmingDelay = getOption(options, LiteralStringRef("warmingDelay"), 0.0);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		binCount = getOption(options, "binCount"_sr, 1000);
+		writesPerTransaction = getOption(options, "writesPerTransaction"_sr, 1);
+		keySpaceDriftFactor = getOption(options, "keySpaceDriftFactor"_sr, 1);
+		moversPerClient = std::max(getOption(options, "moversPerClient"_sr, 10), 1);
+		actorsPerClient = std::max(getOption(options, "actorsPerClient"_sr, 100), 1);
+		int nodes = getOption(options, "nodes"_sr, 10000);
+		discardEdgeMeasurements = getOption(options, "discardEdgeMeasurements"_sr, true);
+		warmingDelay = getOption(options, "warmingDelay"_sr, 0.0);
 		transactionsPerSecond =
-		    getOption(options, LiteralStringRef("transactionsPerSecond"), 5000.0) / (clientCount * moversPerClient);
+		    getOption(options, "transactionsPerSecond"_sr, 5000.0) / (clientCount * moversPerClient);
 
 		nodesPerActor = nodes / (actorsPerClient * clientCount);
 
diff --git a/fdbserver/workloads/DDMetrics.actor.cpp b/fdbserver/workloads/DDMetrics.actor.cpp
index 916e03c7cc..7fcb018a6f 100644
--- a/fdbserver/workloads/DDMetrics.actor.cpp
+++ b/fdbserver/workloads/DDMetrics.actor.cpp
@@ -30,7 +30,7 @@ struct DDMetricsWorkload : TestWorkload {
 	double startDelay, ddDone;
 
 	DDMetricsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ddDone(0.0) {
-		startDelay = getOption(options, LiteralStringRef("beginPoll"), 10.0);
+		startDelay = getOption(options, "beginPoll"_sr, 10.0);
 	}
 
 	std::string description() const override { return "Data Distribution Metrics"; }
@@ -39,8 +39,8 @@ struct DDMetricsWorkload : TestWorkload {
 		WorkerInterface masterWorker = wait(getMasterWorker(cx, self->dbInfo));
 
 		TraceEvent("GetHighPriorityReliocationsInFlight").detail("Stage", "ContactingMaster");
-		TraceEventFields md = wait(
-		    timeoutError(masterWorker.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("MovingData"))), 1.0));
+		TraceEventFields md =
+		    wait(timeoutError(masterWorker.eventLogRequest.getReply(EventLogRequest("MovingData"_sr)), 1.0));
 		int relocations;
 		sscanf(md.getValue("UnhealthyRelocations").c_str(), "%d", &relocations);
 		return relocations;
diff --git a/fdbserver/workloads/DDMetricsExclude.actor.cpp b/fdbserver/workloads/DDMetricsExclude.actor.cpp
index ff35a73f21..9aca58b570 100644
--- a/fdbserver/workloads/DDMetricsExclude.actor.cpp
+++ b/fdbserver/workloads/DDMetricsExclude.actor.cpp
@@ -38,8 +38,8 @@ struct DDMetricsExcludeWorkload : TestWorkload {
 	DDMetricsExcludeWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), ddDone(0.0), peakMovingData(0.0), peakInQueue(0.0), peakInFlight(0.0),
 	    movingDataPerSec(0.0) {
-		excludeIp = getOption(options, LiteralStringRef("excludeIp"), Value(LiteralStringRef("127.0.0.1")));
-		excludePort = getOption(options, LiteralStringRef("excludePort"), 4500);
+		excludeIp = getOption(options, "excludeIp"_sr, Value("127.0.0.1"_sr));
+		excludePort = getOption(options, "excludePort"_sr, 4500);
 	}
 
 	static Value getRandomValue() {
diff --git a/fdbserver/workloads/DataDistributionMetrics.actor.cpp b/fdbserver/workloads/DataDistributionMetrics.actor.cpp
index 4cb59c5768..cdaf2cb5b5 100644
--- a/fdbserver/workloads/DataDistributionMetrics.actor.cpp
+++ b/fdbserver/workloads/DataDistributionMetrics.actor.cpp
@@ -29,7 +29,7 @@
 struct DataDistributionMetricsWorkload : KVWorkload {
 
 	int numShards, readPerTx, writePerTx;
-	int64_t avgBytes;
+	int64_t avgBytes, transactionTimeLimit;
 	double testDuration;
 	std::string keyPrefix;
 	PerfIntCounter commits, errors;
@@ -37,11 +37,13 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 
 	DataDistributionMetricsWorkload(WorkloadContext const& wcx)
 	  : KVWorkload(wcx), numShards(0), avgBytes(0), commits("Commits"), errors("Errors") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		keyPrefix = getOption(options, LiteralStringRef("keyPrefix"), LiteralStringRef("DDMetrics")).toString();
-		readPerTx = getOption(options, LiteralStringRef("readPerTransaction"), 1);
-		writePerTx = getOption(options, LiteralStringRef("writePerTransaction"), 5 * readPerTx);
-		delayPerLoop = getOption(options, LiteralStringRef("delayPerLoop"), 0.1); // throttling dd rpc calls
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		// transaction time out duration(ms)
+		transactionTimeLimit = getOption(options, "transactionTimeLimit"_sr, 10000);
+		keyPrefix = getOption(options, "keyPrefix"_sr, "DDMetrics"_sr).toString();
+		readPerTx = getOption(options, "readPerTransaction"_sr, 1);
+		writePerTx = getOption(options, "writePerTransaction"_sr, 5 * readPerTx);
+		delayPerLoop = getOption(options, "delayPerLoop"_sr, 0.1); // throttling dd rpc calls
 		ASSERT(nodeCount > 1);
 	}
 
@@ -73,6 +75,9 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 	ACTOR Future<Void> resultConsistencyCheckClient(Database cx, DataDistributionMetricsWorkload* self) {
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
 		loop {
+			tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+			tr->setOption(FDBTransactionOptions::TIMEOUT,
+			              StringRef((uint8_t*)&self->transactionTimeLimit, sizeof(int64_t)));
 			try {
 				wait(delay(self->delayPerLoop));
 				int startIndex = deterministicRandom()->randomInt(0, self->nodeCount - 1);
@@ -88,7 +93,7 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 				// the range. If we didn't read through the end of the range, then the second last key
 				// in the result will be the last key less than endKey. (Condition #2)
 				state KeySelector end = KeySelectorRef(endKey.withPrefix(ddStatsRange.begin, endKey.arena()), false, 2);
-				RangeResult result = wait(tr->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->SHARD_COUNT_LIMIT)));
+				RangeResult result = wait(tr->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
 				// Condition #1 and #2 can be broken if multiple rpc calls happened in one getRange
 				if (result.size() > 1) {
 					if (result[0].key > begin.getKey() || result[1].key <= begin.getKey()) {
@@ -100,13 +105,14 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 						    .detail("SecondKey", result[1].key)
 						    .detail("BeginKeySelector", begin);
 					}
-					if (result[result.size() - 1].key < end.getKey() || result[result.size() - 2].key >= end.getKey()) {
+					if (!result.readThroughEnd && (result[result.size() - 1].key < end.getKey() ||
+					                               result[result.size() - 2].key >= end.getKey())) {
 						++self->errors;
 						TraceEvent(SevError, "TestFailure")
 						    .detail("Reason", "Result mismatches the given end selector")
 						    .detail("Size", result.size())
-						    .detail("FirstKey", result[result.size() - 1].key)
-						    .detail("SecondKey", result[result.size() - 2].key)
+						    .detail("LastKey", result[result.size() - 1].key)
+						    .detail("SecondLastKey", result[result.size() - 2].key)
 						    .detail("EndKeySelector", end);
 					}
 					// Debugging traces
@@ -123,9 +129,10 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 				}
 			} catch (Error& e) {
 				// Ignore timed_out error and cross_module_read, the end key selector may read through the end
-				if (e.code() == error_code_timed_out || e.code() == error_code_special_keys_cross_module_read)
+				if (e.code() == error_code_timed_out || e.code() == error_code_transaction_timed_out) {
+					tr->reset();
 					continue;
-				TraceEvent(SevDebug, "FailedToRetrieveDDMetrics").error(e);
+				}
 				wait(tr->onError(e));
 			}
 		}
@@ -139,38 +146,45 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 		// TODO : find why this not work
 		// wait(quietDatabase(cx, self->dbInfo, "PopulateTPCC"));
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		try {
-			state RangeResult result = wait(tr->getRange(ddStatsRange, CLIENT_KNOBS->SHARD_COUNT_LIMIT));
-			ASSERT(!result.more);
-			self->numShards = result.size();
-			if (self->numShards < 1)
-				return false;
-			state int64_t totalBytes = 0;
-			auto schema = readJSONStrictly(JSONSchemas::dataDistributionStatsSchema.toString()).get_obj();
-			for (int i = 0; i < result.size(); ++i) {
-				ASSERT(result[i].key.startsWith(ddStatsRange.begin));
-				std::string errorStr;
-				auto valueObj = readJSONStrictly(result[i].value.toString()).get_obj();
-				TEST(true); // data_distribution_stats schema validation
-				if (!schemaMatch(schema, valueObj, errorStr, SevError, true)) {
-					TraceEvent(SevError, "DataDistributionStatsSchemaValidationFailed")
-					    .detail("ErrorStr", errorStr.c_str())
-					    .detail("JSON", json_spirit::write_string(json_spirit::mValue(result[i].value.toString())));
-					return false;
+		state int i;
+		state int retries = 0;
+		loop {
+			tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+			tr->setOption(FDBTransactionOptions::TIMEOUT,
+			              StringRef((uint8_t*)&self->transactionTimeLimit, sizeof(int64_t)));
+			try {
+				state RangeResult result = wait(tr->getRange(ddStatsRange, CLIENT_KNOBS->TOO_MANY));
+				ASSERT(!result.more);
+				self->numShards = result.size();
+				// There's no guarantee that #shards <= CLIENT_KNOBS->SHARD_COUNT_LIMIT all the time
+				ASSERT(self->numShards >= 1);
+				state int64_t totalBytes = 0;
+				auto schema = readJSONStrictly(JSONSchemas::dataDistributionStatsSchema.toString()).get_obj();
+				for (i = 0; i < result.size(); ++i) {
+					ASSERT(result[i].key.startsWith(ddStatsRange.begin));
+					std::string errorStr;
+					auto valueObj = readJSONStrictly(result[i].value.toString()).get_obj();
+					CODE_PROBE(true, "data_distribution_stats schema validation");
+					if (!schemaMatch(schema, valueObj, errorStr, SevError, true)) {
+						TraceEvent(SevError, "DataDistributionStatsSchemaValidationFailed")
+						    .detail("ErrorStr", errorStr.c_str())
+						    .detail("JSON", json_spirit::write_string(json_spirit::mValue(result[i].value.toString())));
+						return false;
+					}
+					totalBytes += valueObj["shard_bytes"].get_int64();
 				}
-				totalBytes += valueObj["shard_bytes"].get_int64();
+				self->avgBytes = totalBytes / self->numShards;
+				break;
+			} catch (Error& e) {
+				if (e.code() == error_code_timed_out || e.code() == error_code_transaction_timed_out) {
+					tr->reset();
+					// The RPC call may in some corner cases get no response
+					if (++retries > 10)
+						break;
+					continue;
+				}
+				wait(tr->onError(e));
 			}
-			self->avgBytes = totalBytes / self->numShards;
-			// fetch data-distribution stats for a smaller range
-			ASSERT(result.size());
-			state int idx = deterministicRandom()->randomInt(0, result.size());
-			RangeResult res = wait(tr->getRange(
-			    KeyRangeRef(result[idx].key, idx + 1 < result.size() ? result[idx + 1].key : ddStatsRange.end), 100));
-			ASSERT_WE_THINK(res.size() == 1 && res[0] == result[idx]); // It works good now. However, not sure in any
-			                                                           // case of data-distribution, the number changes
-		} catch (Error& e) {
-			TraceEvent(SevError, "FailedToRetrieveDDMetrics").detail("Error", e.what());
-			throw;
 		}
 		return true;
 	}
@@ -188,7 +202,12 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 	std::string description() const override { return "DataDistributionMetrics"; }
 	Future<Void> setup(Database const& cx) override { return Void(); }
 	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-	Future<bool> check(Database const& cx) override { return _check(cx, this); }
+
+	Future<bool> check(Database const& cx) override {
+		if (clientId == 0)
+			return _check(cx, this);
+		return true;
+	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {
 		m.emplace_back("NumShards", numShards, Averaged::True);
diff --git a/fdbserver/workloads/DataLossRecovery.actor.cpp b/fdbserver/workloads/DataLossRecovery.actor.cpp
index 048b6d4bec..996c403efb 100644
--- a/fdbserver/workloads/DataLossRecovery.actor.cpp
+++ b/fdbserver/workloads/DataLossRecovery.actor.cpp
@@ -63,6 +63,8 @@ struct DataLossRecoveryWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("MoveKeysWorkload"); }
+
 	Future<Void> start(Database const& cx) override {
 		if (!enabled) {
 			return Void();
@@ -188,7 +190,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
 			std::vector<StorageServerInterface> interfs = wait(getStorageServers(cx));
 			if (!interfs.empty()) {
 				const auto& interf = interfs[deterministicRandom()->randomInt(0, interfs.size())];
-				if (g_simulator.protectedAddresses.count(interf.address()) == 0) {
+				if (g_simulator->protectedAddresses.count(interf.address()) == 0) {
 					dest.push_back(interf.uniqueID);
 					addr = interf.address();
 				}
@@ -213,18 +215,18 @@ struct DataLossRecoveryWorkload : TestWorkload {
 
 				TraceEvent("DataLossRecovery").detail("Phase", "StartMoveKeys");
 				wait(moveKeys(cx,
-				              deterministicRandom()->randomUniqueID(),
-				              keys,
-				              dest,
-				              dest,
-				              moveKeysLock,
-				              Promise<Void>(),
-				              &self->startMoveKeysParallelismLock,
-				              &self->finishMoveKeysParallelismLock,
-				              false,
-				              UID(), // for logging only
-				              &ddEnabledState,
-				              CancelConflictingDataMoves::True));
+				              MoveKeysParams{ deterministicRandom()->randomUniqueID(),
+				                              keys,
+				                              dest,
+				                              dest,
+				                              moveKeysLock,
+				                              Promise<Void>(),
+				                              &self->startMoveKeysParallelismLock,
+				                              &self->finishMoveKeysParallelismLock,
+				                              false,
+				                              UID(), // for logging only
+				                              &ddEnabledState,
+				                              CancelConflictingDataMoves::True }));
 				break;
 			} catch (Error& e) {
 				TraceEvent("DataLossRecovery").error(e).detail("Phase", "MoveRangeError");
@@ -256,9 +258,9 @@ struct DataLossRecoveryWorkload : TestWorkload {
 	}
 
 	void killProcess(DataLossRecoveryWorkload* self, const NetworkAddress& addr) {
-		ISimulator::ProcessInfo* process = g_simulator.getProcessByAddress(addr);
+		ISimulator::ProcessInfo* process = g_simulator->getProcessByAddress(addr);
 		ASSERT(process->addresses.contains(addr));
-		g_simulator.killProcess(process, ISimulator::KillInstantly);
+		g_simulator->killProcess(process, ISimulator::KillInstantly);
 		TraceEvent("TestTeamKilled").detail("Address", addr);
 	}
 
@@ -267,4 +269,4 @@ struct DataLossRecoveryWorkload : TestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
-WorkloadFactory<DataLossRecoveryWorkload> DataLossRecoveryWorkloadFactory("DataLossRecovery");
\ No newline at end of file
+WorkloadFactory<DataLossRecoveryWorkload> DataLossRecoveryWorkloadFactory("DataLossRecovery");
diff --git a/fdbserver/workloads/DifferentClustersSameRV.actor.cpp b/fdbserver/workloads/DifferentClustersSameRV.actor.cpp
index 9598607830..a3edd32df5 100644
--- a/fdbserver/workloads/DifferentClustersSameRV.actor.cpp
+++ b/fdbserver/workloads/DifferentClustersSameRV.actor.cpp
@@ -23,6 +23,7 @@
 #include "fdbclient/RunTransaction.actor.h"
 #include "fdbrpc/simulator.h"
 #include "fdbserver/workloads/workloads.actor.h"
+#include "flow/ApiVersion.h"
 #include "flow/genericactors.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -37,13 +38,14 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 	bool switchComplete = false;
 
 	DifferentClustersSameRVWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		ASSERT(g_simulator.extraDB != nullptr);
-		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-		extraDB = Database::createDatabase(extraFile, -1);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 100.0);
-		switchAfter = getOption(options, LiteralStringRef("switchAfter"), 50.0);
-		keyToRead = getOption(options, LiteralStringRef("keyToRead"), LiteralStringRef("someKey"));
-		keyToWatch = getOption(options, LiteralStringRef("keyToWatch"), LiteralStringRef("anotherKey"));
+		ASSERT(g_simulator->extraDatabases.size() == 1);
+		auto extraFile =
+		    makeReference<ClusterConnectionMemoryRecord>(ClusterConnectionString(g_simulator->extraDatabases[0]));
+		extraDB = Database::createDatabase(extraFile, ApiVersion::LATEST_VERSION);
+		testDuration = getOption(options, "testDuration"_sr, 100.0);
+		switchAfter = getOption(options, "switchAfter"_sr, 50.0);
+		keyToRead = getOption(options, "keyToRead"_sr, "someKey"_sr);
+		keyToWatch = getOption(options, "keyToWatch"_sr, "anotherKey"_sr);
 	}
 
 	std::string description() const override { return "DifferentClustersSameRV"; }
@@ -163,7 +165,7 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 		wait(unlockDatabase(self->extraDB, lockUid));
 		TraceEvent("DifferentClusters_UnlockedExtraDB").log();
 		ASSERT(!watchFuture.isReady() || watchFuture.isError());
-		wait(doWrite(self->extraDB, self->keyToWatch, Optional<Value>{ LiteralStringRef("") }));
+		wait(doWrite(self->extraDB, self->keyToWatch, Optional<Value>{ ""_sr }));
 		TraceEvent("DifferentClusters_WaitingForWatch").log();
 		try {
 			wait(timeoutError(watchFuture, (self->testDuration - self->switchAfter) / 2));
@@ -216,7 +218,7 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 				state Future<Optional<Value>> val2 = tr2.get(self->keyToRead);
 				wait(success(val1) && success(val2));
 				// We're reading from different db's with the same read version. We can get a different value.
-				TEST(val1.get() != val2.get()); // reading from different dbs with the same version
+				CODE_PROBE(val1.get() != val2.get(), "reading from different dbs with the same version");
 			} catch (Error& e) {
 				wait(tr1.onError(e) && tr2.onError(e));
 			}
diff --git a/fdbserver/workloads/DiskDurability.actor.cpp b/fdbserver/workloads/DiskDurability.actor.cpp
index bbbbbf0b15..0d5048e6b4 100644
--- a/fdbserver/workloads/DiskDurability.actor.cpp
+++ b/fdbserver/workloads/DiskDurability.actor.cpp
@@ -87,14 +87,14 @@ struct DiskDurabilityWorkload : public AsyncFileWorkload {
 	double syncInterval;
 
 	DiskDurabilityWorkload(WorkloadContext const& wcx) : AsyncFileWorkload(wcx) {
-		writers = getOption(options, LiteralStringRef("writers"), 1);
-		filePages = getOption(options, LiteralStringRef("filePages"), 1000000);
+		writers = getOption(options, "writers"_sr, 1);
+		filePages = getOption(options, "filePages"_sr, 1000000);
 		fileSize = filePages * _PAGE_SIZE;
 		unbufferedIO = true;
 		uncachedIO = true;
 		fillRandom = false;
-		pagesPerWrite = getOption(options, LiteralStringRef("pagesPerWrite"), 1);
-		syncInterval = (double)(getOption(options, LiteralStringRef("syncIntervalMs"), 2000)) / 1000;
+		pagesPerWrite = getOption(options, "pagesPerWrite"_sr, 1);
+		syncInterval = (double)(getOption(options, "syncIntervalMs"_sr, 2000)) / 1000;
 	}
 
 	~DiskDurabilityWorkload() override {}
diff --git a/fdbserver/workloads/DiskDurabilityTest.actor.cpp b/fdbserver/workloads/DiskDurabilityTest.actor.cpp
index 4752e8a7b1..836f7217f2 100644
--- a/fdbserver/workloads/DiskDurabilityTest.actor.cpp
+++ b/fdbserver/workloads/DiskDurabilityTest.actor.cpp
@@ -32,9 +32,9 @@ struct DiskDurabilityTest : TestWorkload {
 
 	DiskDurabilityTest(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
-		filename = getOption(options, LiteralStringRef("filename"), LiteralStringRef("durability_test.bin")).toString();
-		auto prefix = getOption(options, LiteralStringRef("prefix"), LiteralStringRef("/DiskDurabilityTest/"));
-		range = prefixRange(LiteralStringRef("S").withPrefix(prefix));
+		filename = getOption(options, "filename"_sr, "durability_test.bin"_sr).toString();
+		auto prefix = getOption(options, "prefix"_sr, "/DiskDurabilityTest/"_sr);
+		range = prefixRange("S"_sr.withPrefix(prefix));
 		metrics = prefixRange(prefix);
 	}
 
@@ -142,10 +142,10 @@ struct DiskDurabilityTest : TestWorkload {
 						tr.clear(self->encodeKey(targetPages[i]));
 
 					if (!first) {
-						Optional<Value> v = wait(tr.get(LiteralStringRef("syncs").withPrefix(self->metrics.begin)));
+						Optional<Value> v = wait(tr.get("syncs"_sr.withPrefix(self->metrics.begin)));
 						int64_t count = v.present() ? self->decodeValue(v.get()) : 0;
 						count++;
-						tr.set(LiteralStringRef("syncs").withPrefix(self->metrics.begin), self->encodeValue(count));
+						tr.set("syncs"_sr.withPrefix(self->metrics.begin), self->encodeValue(count));
 					}
 
 					wait(tr.commit());
diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp
index 810bd801c1..319b9849f7 100644
--- a/fdbserver/workloads/DiskFailureInjection.actor.cpp
+++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp
@@ -28,44 +28,48 @@
 #include "fdbserver/Status.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-struct DiskFailureInjectionWorkload : TestWorkload {
+struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
 	bool enabled;
-	double testDuration;
-	double startDelay;
-	bool throttleDisk;
-	int workersToThrottle;
-	double stallInterval;
-	double stallPeriod;
-	double throttlePeriod;
-	bool corruptFile;
-	int workersToCorrupt;
-	double percentBitFlips;
-	double periodicBroadcastInterval;
+	double testDuration = 60.0;
+	double startDelay = 0.0;
+	bool throttleDisk = false;
+	int workersToThrottle = 3;
+	double stallInterval = 0.0;
+	double stallPeriod = 60.0;
+	double throttlePeriod = 60.0;
+	bool corruptFile = false;
+	int workersToCorrupt = 1;
+	double percentBitFlips = 10;
+	double periodicBroadcastInterval = 5.0;
 	std::vector<NetworkAddress> chosenWorkers;
 	std::vector<Future<Void>> clients;
 	// Verification Mode: We run the workload indefinitely in this mode.
 	// The idea is to keep going until we get a non-zero chaosMetric to ensure
 	// that we haven't lost the chaos event. testDuration is ignored in this mode
-	bool verificationMode;
+	bool verificationMode = false;
 
-	DiskFailureInjectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	DiskFailureInjectionWorkload(WorkloadContext const& wcx, NoOptions) : FailureInjectionWorkload(wcx) {}
+
+	DiskFailureInjectionWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
-		startDelay = getOption(options, LiteralStringRef("startDelay"), 0.0);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0);
-		verificationMode = getOption(options, LiteralStringRef("verificationMode"), false);
-		throttleDisk = getOption(options, LiteralStringRef("throttleDisk"), false);
-		workersToThrottle = getOption(options, LiteralStringRef("workersToThrottle"), 3);
-		stallInterval = getOption(options, LiteralStringRef("stallInterval"), 0.0);
-		stallPeriod = getOption(options, LiteralStringRef("stallPeriod"), 60.0);
-		throttlePeriod = getOption(options, LiteralStringRef("throttlePeriod"), 60.0);
-		corruptFile = getOption(options, LiteralStringRef("corruptFile"), false);
-		workersToCorrupt = getOption(options, LiteralStringRef("workersToCorrupt"), 1);
-		percentBitFlips = getOption(options, LiteralStringRef("percentBitFlips"), 10.0);
-		periodicBroadcastInterval = getOption(options, LiteralStringRef("periodicBroadcastInterval"), 5.0);
+		startDelay = getOption(options, "startDelay"_sr, startDelay);
+		testDuration = getOption(options, "testDuration"_sr, testDuration);
+		verificationMode = getOption(options, "verificationMode"_sr, verificationMode);
+		throttleDisk = getOption(options, "throttleDisk"_sr, throttleDisk);
+		workersToThrottle = getOption(options, "workersToThrottle"_sr, workersToThrottle);
+		stallInterval = getOption(options, "stallInterval"_sr, stallInterval);
+		stallPeriod = getOption(options, "stallPeriod"_sr, stallPeriod);
+		throttlePeriod = getOption(options, "throttlePeriod"_sr, throttlePeriod);
+		corruptFile = getOption(options, "corruptFile"_sr, corruptFile);
+		workersToCorrupt = getOption(options, "workersToCorrupt"_sr, workersToCorrupt);
+		percentBitFlips = getOption(options, "percentBitFlips"_sr, percentBitFlips);
+		periodicBroadcastInterval = getOption(options, "periodicBroadcastInterval"_sr, periodicBroadcastInterval);
 	}
 
+	void initFailureInjectionMode(DeterministicRandom& random) override { enabled = clientId == 0; }
+
 	std::string description() const override {
-		if (&g_simulator == g_network)
+		if (g_simulator == g_network)
 			return "DiskFailureInjection";
 		else
 			return "NoSimDiskFailureInjection";
@@ -177,8 +181,8 @@ struct DiskFailureInjectionWorkload : TestWorkload {
 			if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle))
 				self->injectDiskDelays(machine, self->stallInterval, self->stallPeriod, self->throttlePeriod);
 			if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) {
-				if (&g_simulator == g_network)
-					g_simulator.corruptWorkerMap[machine.address()] = true;
+				if (g_simulator == g_network)
+					g_simulator->corruptWorkerMap[machine.address()] = true;
 				self->injectBitFlips(machine, self->percentBitFlips);
 			}
 		}
@@ -200,8 +204,8 @@ struct DiskFailureInjectionWorkload : TestWorkload {
 				if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle))
 					self->injectDiskDelays(itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod);
 				if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) {
-					if (&g_simulator == g_network)
-						g_simulator.corruptWorkerMap[workerAddress] = true;
+					if (g_simulator == g_network)
+						g_simulator->corruptWorkerMap[workerAddress] = true;
 					self->injectBitFlips(itr->second, self->percentBitFlips);
 				}
 			}
@@ -275,3 +279,4 @@ struct DiskFailureInjectionWorkload : TestWorkload {
 	}
 };
 WorkloadFactory<DiskFailureInjectionWorkload> DiskFailureInjectionWorkloadFactory("DiskFailureInjection");
+FailureInjectorFactory<DiskFailureInjectionWorkload> DiskFailureInjectionWorkloadFailureInjectionFactory;
diff --git a/fdbserver/workloads/DummyWorkload.actor.cpp b/fdbserver/workloads/DummyWorkload.actor.cpp
index 99af0a2947..a908085b3e 100644
--- a/fdbserver/workloads/DummyWorkload.actor.cpp
+++ b/fdbserver/workloads/DummyWorkload.actor.cpp
@@ -27,8 +27,8 @@ struct DummyWorkload : TestWorkload {
 	double displayDelay;
 
 	DummyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		displayWorkers = getOption(options, LiteralStringRef("displayWorkers"), true);
-		displayDelay = getOption(options, LiteralStringRef("displayDelay"), 0.0);
+		displayWorkers = getOption(options, "displayWorkers"_sr, true);
+		displayDelay = getOption(options, "displayDelay"_sr, 0.0);
 	}
 
 	std::string description() const override { return "DummyWorkload"; }
@@ -43,7 +43,7 @@ struct DummyWorkload : TestWorkload {
 	ACTOR static Future<Void> _start(DummyWorkload* self, Database cx) {
 		if (self->displayDelay > 0.0)
 			wait(delay(self->displayDelay));
-		g_simulator.displayWorkers();
+		g_simulator->displayWorkers();
 		return Void();
 	}
 
diff --git a/fdbserver/workloads/EncryptKeyProxyTest.actor.cpp b/fdbserver/workloads/EncryptKeyProxyTest.actor.cpp
index 3cd575db11..7b00e15b6d 100644
--- a/fdbserver/workloads/EncryptKeyProxyTest.actor.cpp
+++ b/fdbserver/workloads/EncryptKeyProxyTest.actor.cpp
@@ -19,7 +19,7 @@
  */
 
 #include "fdbrpc/Locality.h"
-#include "fdbserver/EncryptKeyProxyInterface.h"
+#include "fdbclient/EncryptKeyProxyInterface.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/ServerDBInfo.actor.h"
 #include "fdbserver/WorkerInterface.actor.h"
@@ -57,7 +57,7 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
 			enableTest = true;
 			minDomainId = 1000 + (++seed * 30) + 1;
 			maxDomainId = deterministicRandom()->randomInt(minDomainId, minDomainId + 50) + 5;
-			TraceEvent("EKPTest_Init").detail("MinDomainId", minDomainId).detail("MaxDomainId", maxDomainId);
+			TraceEvent("EKPTestInit").detail("MinDomainId", minDomainId).detail("MaxDomainId", maxDomainId);
 		}
 	}
 
@@ -111,7 +111,7 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
 			}
 		}
 
-		TraceEvent("SimEmptyDomainIdCache_Done").log();
+		TraceEvent("SimEmptyDomainIdCacheDone").log();
 		return Void();
 	}
 
@@ -119,7 +119,7 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
 		state int expectedHits;
 		state int expectedMisses;
 
-		TraceEvent("SimPartialDomainIdCache_Start").log();
+		TraceEvent("SimPartialDomainIdCacheStart").log();
 
 		self->domainInfos.clear();
 
@@ -178,14 +178,14 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
 		}
 		self->domainInfos.clear();
 
-		TraceEvent("SimPartialDomainIdCache_Done").log();
+		TraceEvent("SimPartialDomainIdCacheDone").log();
 		return Void();
 	}
 
 	ACTOR Future<Void> simRandomBaseCipherIdCache(EncryptKeyProxyTestWorkload* self) {
 		state int expectedHits;
 
-		TraceEvent("SimRandomDomainIdCache_Start").log();
+		TraceEvent("SimRandomDomainIdCacheStart").log();
 
 		self->domainInfos.clear();
 		for (int i = 0; i < self->numDomains; i++) {
@@ -271,14 +271,14 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
 			}
 		}
 
-		TraceEvent("SimRandomDomainIdCache_Done").log();
+		TraceEvent("SimRandomDomainIdCacheDone").log();
 		return Void();
 	}
 
 	ACTOR Future<Void> simLookupInvalidKeyId(EncryptKeyProxyTestWorkload* self) {
 		Arena arena;
 
-		TraceEvent("SimLookupInvalidKeyId_Start").log();
+		TraceEvent("SimLookupInvalidKeyIdStart").log();
 
 		// Prepare a lookup with valid and invalid keyIds - SimEncryptKmsProxy should throw encrypt_key_not_found()
 		EKPGetBaseCipherKeysByIdsRequest req;
@@ -294,7 +294,7 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
 		ASSERT(rep.error.present());
 		ASSERT_EQ(rep.error.get().code(), error_code_encrypt_key_not_found);
 
-		TraceEvent("SimLookupInvalidKeyId_Done").log();
+		TraceEvent("SimLookupInvalidKeyIdDone").log();
 		return Void();
 	}
 
@@ -329,7 +329,7 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
 	}
 
 	Future<Void> start(Database const& cx) override {
-		TEST(true); // Testing
+		CODE_PROBE(true, "Testing");
 		if (!enableTest) {
 			return Void();
 		}
diff --git a/fdbserver/workloads/EncryptionOps.actor.cpp b/fdbserver/workloads/EncryptionOps.actor.cpp
index eb86ec12fc..623e7c4433 100644
--- a/fdbserver/workloads/EncryptionOps.actor.cpp
+++ b/fdbserver/workloads/EncryptionOps.actor.cpp
@@ -18,17 +18,20 @@
  * limitations under the License.
  */
 
+#include "fdbclient/BlobCipher.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "flow/EncryptUtils.h"
+#include "flow/Error.h"
 #include "flow/IRandom.h"
-#include "flow/BlobCipher.h"
 #include "fdbserver/workloads/workloads.actor.h"
+#include "flow/flow.h"
 #include "flow/ITrace.h"
 #include "flow/Trace.h"
 
 #include <chrono>
 #include <cstring>
+#include <limits>
 #include <memory>
 #include <random>
 
@@ -111,6 +114,7 @@ struct EncryptionOpsWorkload : TestWorkload {
 	int pageSize;
 	int maxBufSize;
 	std::unique_ptr<uint8_t[]> buff;
+	int enableTTLTest;
 
 	Arena arena;
 	std::unique_ptr<WorkloadMetrics> metrics;
@@ -121,11 +125,11 @@ struct EncryptionOpsWorkload : TestWorkload {
 	EncryptCipherBaseKeyId headerBaseCipherId;
 	EncryptCipherRandomSalt headerRandomSalt;
 
-	EncryptionOpsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		mode = getOption(options, LiteralStringRef("fixedSize"), 1);
-		numIterations = getOption(options, LiteralStringRef("numIterations"), 10);
-		pageSize = getOption(options, LiteralStringRef("pageSize"), 4096);
-		maxBufSize = getOption(options, LiteralStringRef("maxBufSize"), 512 * 1024);
+	EncryptionOpsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), enableTTLTest(false) {
+		mode = getOption(options, "fixedSize"_sr, 1);
+		numIterations = getOption(options, "numIterations"_sr, 10);
+		pageSize = getOption(options, "pageSize"_sr, 4096);
+		maxBufSize = getOption(options, "maxBufSize"_sr, 512 * 1024);
 		buff = std::make_unique<uint8_t[]>(maxBufSize);
 
 		// assign unique encryptionDomainId range per workload clients
@@ -136,13 +140,18 @@ struct EncryptionOpsWorkload : TestWorkload {
 
 		metrics = std::make_unique<WorkloadMetrics>();
 
+		if (wcx.clientId == 0 && mode == 1) {
+			enableTTLTest = true;
+		}
+
 		TraceEvent("EncryptionOpsWorkload")
 		    .detail("Mode", getModeStr())
 		    .detail("MinDomainId", minDomainId)
-		    .detail("MaxDomainId", maxDomainId);
+		    .detail("MaxDomainId", maxDomainId)
+		    .detail("EnableTTL", enableTTLTest);
 	}
 
-	~EncryptionOpsWorkload() { TraceEvent("EncryptionOpsWorkload_Done").log(); }
+	~EncryptionOpsWorkload() { TraceEvent("EncryptionOpsWorkloadDone").log(); }
 
 	bool isFixedSizePayload() { return mode == 1; }
 
@@ -159,20 +168,25 @@ struct EncryptionOpsWorkload : TestWorkload {
 	void generateRandomBaseCipher(const int maxLen, uint8_t* buff, int* retLen) {
 		memset(buff, 0, maxLen);
 		*retLen = deterministicRandom()->randomInt(maxLen / 2, maxLen);
-		generateRandomData(buff, *retLen);
+		deterministicRandom()->randomBytes(buff, *retLen);
 	}
 
 	void setupCipherEssentials() {
 		Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
 
-		TraceEvent("SetupCipherEssentials_Start").detail("MinDomainId", minDomainId).detail("MaxDomainId", maxDomainId);
+		TraceEvent("SetupCipherEssentialsStart").detail("MinDomainId", minDomainId).detail("MaxDomainId", maxDomainId);
 
 		uint8_t buff[AES_256_KEY_LENGTH];
 		std::vector<Reference<BlobCipherKey>> cipherKeys;
 		int cipherLen = 0;
 		for (EncryptCipherDomainId id = minDomainId; id <= maxDomainId; id++) {
 			generateRandomBaseCipher(AES_256_KEY_LENGTH, &buff[0], &cipherLen);
-			cipherKeyCache->insertCipherKey(id, minBaseCipherId, buff, cipherLen);
+			cipherKeyCache->insertCipherKey(id,
+			                                minBaseCipherId,
+			                                buff,
+			                                cipherLen,
+			                                std::numeric_limits<int64_t>::max(),
+			                                std::numeric_limits<int64_t>::max());
 
 			ASSERT(cipherLen > 0 && cipherLen <= AES_256_KEY_LENGTH);
 
@@ -183,13 +197,18 @@ struct EncryptionOpsWorkload : TestWorkload {
 		// insert the Encrypt Header cipherKey; record cipherDetails as getLatestCipher() may not work with multiple
 		// test clients
 		generateRandomBaseCipher(AES_256_KEY_LENGTH, &buff[0], &cipherLen);
-		cipherKeyCache->insertCipherKey(ENCRYPT_HEADER_DOMAIN_ID, headerBaseCipherId, buff, cipherLen);
+		cipherKeyCache->insertCipherKey(ENCRYPT_HEADER_DOMAIN_ID,
+		                                headerBaseCipherId,
+		                                buff,
+		                                cipherLen,
+		                                std::numeric_limits<int64_t>::max(),
+		                                std::numeric_limits<int64_t>::max());
 		Reference<BlobCipherKey> latestCipher = cipherKeyCache->getLatestCipherKey(ENCRYPT_HEADER_DOMAIN_ID);
 		ASSERT_EQ(latestCipher->getBaseCipherId(), headerBaseCipherId);
 		ASSERT_EQ(memcmp(latestCipher->rawBaseCipher(), buff, cipherLen), 0);
 		headerRandomSalt = latestCipher->getSalt();
 
-		TraceEvent("SetupCipherEssentials_Done")
+		TraceEvent("SetupCipherEssentialsDone")
 		    .detail("MinDomainId", minDomainId)
 		    .detail("MaxDomainId", maxDomainId)
 		    .detail("HeaderBaseCipherId", headerBaseCipherId)
@@ -197,10 +216,18 @@ struct EncryptionOpsWorkload : TestWorkload {
 	}
 
 	void resetCipherEssentials() {
-		Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
-		cipherKeyCache->cleanup();
+		TraceEvent("ResetCipherEssentialsStart").detail("Min", minDomainId).detail("Max", maxDomainId);
 
-		TraceEvent("ResetCipherEssentials_Done").log();
+		Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
+		for (EncryptCipherDomainId id = minDomainId; id <= maxDomainId; id++) {
+			cipherKeyCache->resetEncryptDomainId(id);
+		}
+
+		cipherKeyCache->resetEncryptDomainId(FDB_DEFAULT_ENCRYPT_DOMAIN_ID);
+		cipherKeyCache->resetEncryptDomainId(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID);
+		cipherKeyCache->resetEncryptDomainId(ENCRYPT_HEADER_DOMAIN_ID);
+
+		TraceEvent("ResetCipherEssentialsDone");
 	}
 
 	void updateLatestBaseCipher(const EncryptCipherDomainId encryptDomainId,
@@ -232,7 +259,9 @@ struct EncryptionOpsWorkload : TestWorkload {
 			                                baseCipherId,
 			                                cipherKey->rawBaseCipher(),
 			                                cipherKey->getBaseCipherLen(),
-			                                cipherKey->getSalt());
+			                                cipherKey->getSalt(),
+			                                std::numeric_limits<int64_t>::max(),
+			                                std::numeric_limits<int64_t>::max());
 			// Ensure the update was a NOP
 			Reference<BlobCipherKey> cKey = cipherKeyCache->getCipherKey(domainId, baseCipherId, salt);
 			ASSERT(cKey->isEqual(cipherKey));
@@ -245,10 +274,12 @@ struct EncryptionOpsWorkload : TestWorkload {
 	                                   uint8_t* payload,
 	                                   int len,
 	                                   const EncryptAuthTokenMode authMode,
+	                                   const EncryptAuthTokenAlgo authAlgo,
 	                                   BlobCipherEncryptHeader* header) {
 		uint8_t iv[AES_256_IV_LENGTH];
-		generateRandomData(&iv[0], AES_256_IV_LENGTH);
-		EncryptBlobCipherAes265Ctr encryptor(textCipherKey, headerCipherKey, &iv[0], AES_256_IV_LENGTH, authMode);
+		deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH);
+		EncryptBlobCipherAes265Ctr encryptor(
+		    textCipherKey, headerCipherKey, &iv[0], AES_256_IV_LENGTH, authMode, authAlgo, BlobCipherMetrics::TEST);
 
 		auto start = std::chrono::high_resolution_clock::now();
 		Reference<EncryptBuf> encrypted = encryptor.encrypt(payload, len, header, arena);
@@ -279,8 +310,10 @@ struct EncryptionOpsWorkload : TestWorkload {
 		                                                            header.cipherHeaderDetails.salt);
 		ASSERT(cipherKey.isValid());
 		ASSERT(cipherKey->isEqual(orgCipherKey));
+		ASSERT(headerCipherKey.isValid() ||
+		       header.flags.authTokenMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE);
 
-		DecryptBlobCipherAes256Ctr decryptor(cipherKey, headerCipherKey, header.iv);
+		DecryptBlobCipherAes256Ctr decryptor(cipherKey, headerCipherKey, header.iv, BlobCipherMetrics::TEST);
 		const bool validateHeaderAuthToken = deterministicRandom()->randomInt(0, 100) < 65;
 
 		auto start = std::chrono::high_resolution_clock::now();
@@ -297,11 +330,7 @@ struct EncryptionOpsWorkload : TestWorkload {
 		metrics->updateDecryptionTime(std::chrono::duration<double, std::nano>(end - start).count());
 	}
 
-	Future<Void> setup(Database const& ctx) override { return Void(); }
-
-	std::string description() const override { return "EncryptionOps"; }
-
-	Future<Void> start(Database const& cx) override {
+	void testBlobCipherKeyCacheOps() {
 		uint8_t baseCipher[AES_256_KEY_LENGTH];
 		int baseCipherLen = 0;
 		EncryptCipherBaseKeyId nextBaseCipherId;
@@ -322,7 +351,12 @@ struct EncryptionOpsWorkload : TestWorkload {
 			if (updateBaseCipher) {
 				// simulate baseCipherId getting refreshed/updated
 				updateLatestBaseCipher(encryptDomainId, &baseCipher[0], &baseCipherLen, &nextBaseCipherId);
-				cipherKeyCache->insertCipherKey(encryptDomainId, nextBaseCipherId, &baseCipher[0], baseCipherLen);
+				cipherKeyCache->insertCipherKey(encryptDomainId,
+				                                nextBaseCipherId,
+				                                &baseCipher[0],
+				                                baseCipherLen,
+				                                std::numeric_limits<int64_t>::max(),
+				                                std::numeric_limits<int64_t>::max());
 			}
 
 			auto start = std::chrono::high_resolution_clock::now();
@@ -341,16 +375,18 @@ struct EncryptionOpsWorkload : TestWorkload {
 			}
 
 			int dataLen = isFixedSizePayload() ? pageSize : deterministicRandom()->randomInt(100, maxBufSize);
-			generateRandomData(buff.get(), dataLen);
+			deterministicRandom()->randomBytes(buff.get(), dataLen);
 
 			// Encrypt the payload - generates BlobCipherEncryptHeader to assist decryption later
 			BlobCipherEncryptHeader header;
-			const EncryptAuthTokenMode authMode = deterministicRandom()->randomInt(0, 100) < 50
-			                                          ? ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE
-			                                          : ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI;
+			const EncryptAuthTokenMode authMode = getRandomAuthTokenMode();
+			const EncryptAuthTokenAlgo authAlgo = authMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE
+			                                          ? EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE
+			                                          : getRandomAuthTokenAlgo();
+
 			try {
 				Reference<EncryptBuf> encrypted =
-				    doEncryption(cipherKey, headerCipherKey, buff.get(), dataLen, authMode, &header);
+				    doEncryption(cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &header);
 
 				// Decrypt the payload - parses the BlobCipherEncryptHeader, fetch corresponding cipherKey and
 				// decrypt
@@ -359,7 +395,8 @@ struct EncryptionOpsWorkload : TestWorkload {
 				TraceEvent("Failed")
 				    .detail("DomainId", encryptDomainId)
 				    .detail("BaseCipherId", cipherKey->getBaseCipherId())
-				    .detail("AuthMode", authMode);
+				    .detail("AuthTokenMode", authMode)
+				    .detail("AuthTokenAlgo", authAlgo);
 				throw;
 			}
 
@@ -368,6 +405,103 @@ struct EncryptionOpsWorkload : TestWorkload {
 
 		// Cleanup cipherKeys
 		resetCipherEssentials();
+	}
+
+	static void compareCipherDetails(Reference<BlobCipherKey> cipherKey,
+	                                 const EncryptCipherDomainId domId,
+	                                 const EncryptCipherBaseKeyId baseCipherId,
+	                                 const uint8_t* baseCipher,
+	                                 const int baseCipherLen,
+	                                 const int64_t refreshAt,
+	                                 const int64_t expAt) {
+		ASSERT(cipherKey.isValid());
+		ASSERT_EQ(cipherKey->getDomainId(), domId);
+		ASSERT_EQ(cipherKey->getBaseCipherId(), baseCipherId);
+		ASSERT_EQ(memcmp(cipherKey->rawBaseCipher(), baseCipher, baseCipherLen), 0);
+		ASSERT_EQ(cipherKey->getRefreshAtTS(), refreshAt);
+		ASSERT_EQ(cipherKey->getExpireAtTS(), expAt);
+	}
+
+	ACTOR Future<Void> testBlobCipherKeyCacheTTL(EncryptionOpsWorkload* self) {
+		state Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
+
+		state EncryptCipherDomainId domId = deterministicRandom()->randomInt(120000, 150000);
+		state EncryptCipherBaseKeyId baseCipherId = deterministicRandom()->randomInt(786, 1024);
+		state std::unique_ptr<uint8_t[]> baseCipher = std::make_unique<uint8_t[]>(AES_256_KEY_LENGTH);
+		state Reference<BlobCipherKey> cipherKey;
+		state EncryptCipherRandomSalt salt;
+		state int64_t refreshAt;
+		state int64_t expAt;
+
+		TraceEvent("TestBlobCipherCacheTTLStart").detail("DomId", domId);
+
+		deterministicRandom()->randomBytes(baseCipher.get(), AES_256_KEY_LENGTH);
+
+		// Validate 'non-revocable' cipher with no expiration
+		refreshAt = std::numeric_limits<int64_t>::max();
+		expAt = std::numeric_limits<int64_t>::max();
+		cipherKeyCache->insertCipherKey(domId, baseCipherId, baseCipher.get(), AES_256_KEY_LENGTH, refreshAt, expAt);
+		cipherKey = cipherKeyCache->getLatestCipherKey(domId);
+		compareCipherDetails(cipherKey, domId, baseCipherId, baseCipher.get(), AES_256_KEY_LENGTH, refreshAt, expAt);
+
+		TraceEvent("TestBlobCipherCacheTTLNonRevocableNoExpiry").detail("DomId", domId);
+
+		// Validate 'non-revocable' cipher with expiration
+		state EncryptCipherBaseKeyId baseCipherId_1 = baseCipherId + 1;
+		refreshAt = now() + 5;
+		cipherKeyCache->insertCipherKey(domId, baseCipherId_1, baseCipher.get(), AES_256_KEY_LENGTH, refreshAt, expAt);
+		cipherKey = cipherKeyCache->getLatestCipherKey(domId);
+		ASSERT(cipherKey.isValid());
+		compareCipherDetails(cipherKey, domId, baseCipherId_1, baseCipher.get(), AES_256_KEY_LENGTH, refreshAt, expAt);
+		salt = cipherKey->getSalt();
+		wait(delayUntil(refreshAt));
+		// Ensure that latest cipherKey needs refresh, however, cipher lookup works (non-revocable)
+		cipherKey = cipherKeyCache->getLatestCipherKey(domId);
+		ASSERT(!cipherKey.isValid());
+		cipherKey = cipherKeyCache->getCipherKey(domId, baseCipherId_1, salt);
+		ASSERT(cipherKey.isValid());
+		compareCipherDetails(cipherKey, domId, baseCipherId_1, baseCipher.get(), AES_256_KEY_LENGTH, refreshAt, expAt);
+
+		TraceEvent("TestBlobCipherCacheTTLNonRevocableWithExpiry").detail("DomId", domId);
+
+		// Validate 'revocable' cipher with expiration
+		state EncryptCipherBaseKeyId baseCipherId_2 = baseCipherId + 2;
+		refreshAt = now() + 5;
+		expAt = refreshAt + 5;
+		cipherKeyCache->insertCipherKey(domId, baseCipherId_2, baseCipher.get(), AES_256_KEY_LENGTH, refreshAt, expAt);
+		cipherKey = cipherKeyCache->getLatestCipherKey(domId);
+		ASSERT(cipherKey.isValid());
+		compareCipherDetails(cipherKey, domId, baseCipherId_2, baseCipher.get(), AES_256_KEY_LENGTH, refreshAt, expAt);
+		salt = cipherKey->getSalt();
+		wait(delayUntil(refreshAt));
+		// Ensure that latest cipherKey needs refresh, however, cipher lookup works (non-revocable)
+		cipherKey = cipherKeyCache->getLatestCipherKey(domId);
+		ASSERT(!cipherKey.isValid());
+		cipherKey = cipherKeyCache->getCipherKey(domId, baseCipherId_2, salt);
+		ASSERT(cipherKey.isValid());
+		compareCipherDetails(cipherKey, domId, baseCipherId_2, baseCipher.get(), AES_256_KEY_LENGTH, refreshAt, expAt);
+		wait(delayUntil(expAt));
+		// Ensure that cipherKey lookup doesn't work after expiry
+		cipherKey = cipherKeyCache->getLatestCipherKey(domId);
+		ASSERT(!cipherKey.isValid());
+		cipherKey = cipherKeyCache->getCipherKey(domId, baseCipherId_2, salt);
+		ASSERT(!cipherKey.isValid());
+
+		TraceEvent("TestBlobCipherCacheTTLEnd").detail("DomId", domId);
+		return Void();
+	}
+
+	Future<Void> setup(Database const& ctx) override { return Void(); }
+
+	std::string description() const override { return "EncryptionOps"; }
+
+	Future<Void> start(Database const& cx) override { return _start(cx, this); }
+
+	ACTOR Future<Void> _start(Database cx, EncryptionOpsWorkload* self) {
+		self->testBlobCipherKeyCacheOps();
+		if (self->enableTTLTest) {
+			wait(self->testBlobCipherKeyCacheTTL(self));
+		}
 		return Void();
 	}
 
diff --git a/fdbserver/workloads/ExternalWorkload.actor.cpp b/fdbserver/workloads/ExternalWorkload.actor.cpp
index e652674347..dde3c04c47 100644
--- a/fdbserver/workloads/ExternalWorkload.actor.cpp
+++ b/fdbserver/workloads/ExternalWorkload.actor.cpp
@@ -128,9 +128,9 @@ struct ExternalWorkload : TestWorkload, FDBWorkloadContext {
 	}
 
 	explicit ExternalWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		libraryName = ::getOption(options, LiteralStringRef("libraryName"), LiteralStringRef("")).toString();
-		libraryPath = ::getOption(options, LiteralStringRef("libraryPath"), Value(getDefaultLibraryPath())).toString();
-		auto wName = ::getOption(options, LiteralStringRef("workloadName"), LiteralStringRef(""));
+		libraryName = ::getOption(options, "libraryName"_sr, ""_sr).toString();
+		libraryPath = ::getOption(options, "libraryPath"_sr, Value(getDefaultLibraryPath())).toString();
+		auto wName = ::getOption(options, "workloadName"_sr, ""_sr);
 		auto fullPath = joinPath(libraryPath, toLibName(libraryName));
 		TraceEvent("ExternalWorkloadLoad")
 		    .detail("LibraryName", libraryName)
@@ -185,7 +185,7 @@ struct ExternalWorkload : TestWorkload, FDBWorkloadContext {
 		keepAlive(f, database);
 		workloadImpl->setup(reinterpret_cast<FDBDatabase*>(database.getPtr()),
 		                    GenericPromise<bool>(new FDBPromiseImpl(promise)));
-		return assertTrue(LiteralStringRef("setup"), f);
+		return assertTrue("setup"_sr, f);
 	}
 
 	Future<Void> start(Database const& cx) override {
@@ -200,7 +200,7 @@ struct ExternalWorkload : TestWorkload, FDBWorkloadContext {
 		keepAlive(f, database);
 		workloadImpl->start(reinterpret_cast<FDBDatabase*>(database.getPtr()),
 		                    GenericPromise<bool>(new FDBPromiseImpl(promise)));
-		return assertTrue(LiteralStringRef("start"), f);
+		return assertTrue("start"_sr, f);
 	}
 	Future<bool> check(Database const& cx) override {
 		if (!success) {
@@ -242,14 +242,14 @@ struct ExternalWorkload : TestWorkload, FDBWorkloadContext {
 	}
 	uint64_t getProcessID() const override {
 		if (g_network->isSimulated()) {
-			return reinterpret_cast<uint64_t>(g_simulator.getCurrentProcess());
+			return reinterpret_cast<uint64_t>(g_simulator->getCurrentProcess());
 		} else {
 			return 0ul;
 		}
 	}
 	void setProcessID(uint64_t processID) override {
 		if (g_network->isSimulated()) {
-			g_simulator.currentProcess = reinterpret_cast<ISimulator::ProcessInfo*>(processID);
+			g_simulator->currentProcess = reinterpret_cast<ISimulator::ProcessInfo*>(processID);
 		}
 	}
 	double now() const override { return g_network->now(); }
diff --git a/fdbserver/workloads/FastTriggeredWatches.actor.cpp b/fdbserver/workloads/FastTriggeredWatches.actor.cpp
index 8d1e635a47..cafdda2fc8 100644
--- a/fdbserver/workloads/FastTriggeredWatches.actor.cpp
+++ b/fdbserver/workloads/FastTriggeredWatches.actor.cpp
@@ -36,10 +36,10 @@ struct FastTriggeredWatchesWorkload : TestWorkload {
 
 	FastTriggeredWatchesWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), operations("Operations"), retries("Retries") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 600.0);
-		nodes = getOption(options, LiteralStringRef("nodes"), 100);
+		testDuration = getOption(options, "testDuration"_sr, 600.0);
+		nodes = getOption(options, "nodes"_sr, 100);
 		defaultValue = StringRef(format("%010d", deterministicRandom()->randomInt(0, 1000)));
-		keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 16);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
 	}
 
 	std::string description() const override { return "Watches"; }
diff --git a/fdbserver/workloads/FileSystem.actor.cpp b/fdbserver/workloads/FileSystem.actor.cpp
index a5fa90af4e..b5340e9c12 100644
--- a/fdbserver/workloads/FileSystem.actor.cpp
+++ b/fdbserver/workloads/FileSystem.actor.cpp
@@ -44,23 +44,21 @@ struct FileSystemWorkload : TestWorkload {
 
 	FileSystemWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), queries("Queries"), writes("Latency"), latencies(2500), writeLatencies(1000) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 5000.0) / clientCount;
-		double allowedLatency = getOption(options, LiteralStringRef("allowedLatency"), 0.250);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0) / clientCount;
+		double allowedLatency = getOption(options, "allowedLatency"_sr, 0.250);
 		actorCount = transactionsPerSecond * allowedLatency;
-		fileCount = getOption(options, LiteralStringRef("fileCount"), 100000);
-		pathMinChars = std::max(getOption(options, LiteralStringRef("pathMinChars"), 32), 8);
-		pathCharRange =
-		    std::max(getOption(options, LiteralStringRef("pathMaxChars"), 128), pathMinChars) - pathMinChars;
-		discardEdgeMeasurements = getOption(options, LiteralStringRef("discardEdgeMeasurements"), true);
-		deletedFilesRatio = getOption(options, LiteralStringRef("deletedFilesRatio"), 0.01);
-		serverCount = getOption(options, LiteralStringRef("serverCount"), 32);
-		userIDCount = getOption(options, LiteralStringRef("userIDCount"), std::max(100, fileCount / 3000));
-		operationName =
-		    getOption(options, LiteralStringRef("operationName"), LiteralStringRef("modificationQuery")).toString();
-		performingWrites = getOption(options, LiteralStringRef("performingWrites"), false);
-		writeActorCount = getOption(options, LiteralStringRef("writeActorCount"), 4);
-		loggingQueries = getOption(options, LiteralStringRef("loggingQueries"), false);
+		fileCount = getOption(options, "fileCount"_sr, 100000);
+		pathMinChars = std::max(getOption(options, "pathMinChars"_sr, 32), 8);
+		pathCharRange = std::max(getOption(options, "pathMaxChars"_sr, 128), pathMinChars) - pathMinChars;
+		discardEdgeMeasurements = getOption(options, "discardEdgeMeasurements"_sr, true);
+		deletedFilesRatio = getOption(options, "deletedFilesRatio"_sr, 0.01);
+		serverCount = getOption(options, "serverCount"_sr, 32);
+		userIDCount = getOption(options, "userIDCount"_sr, std::max(100, fileCount / 3000));
+		operationName = getOption(options, "operationName"_sr, "modificationQuery"_sr).toString();
+		performingWrites = getOption(options, "performingWrites"_sr, false);
+		writeActorCount = getOption(options, "writeActorCount"_sr, 4);
+		loggingQueries = getOption(options, "loggingQueries"_sr, false);
 	}
 
 	std::string description() const override { return "ReadWrite"; }
@@ -105,7 +103,7 @@ struct FileSystemWorkload : TestWorkload {
 		std::string keyStr(key.toString());
 		tr->set(keyStr + "/size", format("%d", deterministicRandom()->randomInt(0, std::numeric_limits<int>::max())));
 		tr->set(keyStr + "/server", format("%d", deterministicRandom()->randomInt(0, self->serverCount)));
-		tr->set(keyStr + "/deleted", deleted ? LiteralStringRef("1") : LiteralStringRef("0"));
+		tr->set(keyStr + "/deleted", deleted ? "1"_sr : "0"_sr);
 		tr->set(keyStr + "/server", format("%d", serverID));
 		tr->set(keyStr + "/created", doubleToTestKey(time));
 		tr->set(keyStr + "/lastupdated", doubleToTestKey(time));
@@ -250,10 +248,10 @@ struct FileSystemWorkload : TestWorkload {
 						ASSERT(serverStr.present());
 						int serverID = testKeyToInt(serverStr.get());
 						if (deleted.get().toString() == "1") {
-							tr.set(keyStr + "/deleted", LiteralStringRef("0"));
+							tr.set(keyStr + "/deleted", "0"_sr);
 							tr.clear(format("/files/server/%08x/deleted/%016llx", serverID, fileID));
 						} else {
-							tr.set(keyStr + "/deleted", LiteralStringRef("1"));
+							tr.set(keyStr + "/deleted", "1"_sr);
 							tr.set(format("/files/server/%08x/deleted/%016llx", serverID, fileID),
 							       doubleToTestKey(time));
 						}
diff --git a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp b/fdbserver/workloads/FuzzApiCorrectness.actor.cpp
index 390ff6f295..8342e35f9f 100644
--- a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp
+++ b/fdbserver/workloads/FuzzApiCorrectness.actor.cpp
@@ -24,6 +24,7 @@
 #include <sstream>
 
 #include "fdbclient/FDBOptions.g.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbclient/GenericManagementAPI.actor.h"
 #include "fdbclient/TenantManagement.actor.h"
@@ -62,7 +63,7 @@ struct ExceptionContract {
 		    e.code() == error_code_future_version || e.code() == error_code_transaction_cancelled ||
 		    e.code() == error_code_key_too_large || e.code() == error_code_value_too_large ||
 		    e.code() == error_code_process_behind || e.code() == error_code_batch_transaction_throttled ||
-		    e.code() == error_code_tag_throttled) {
+		    e.code() == error_code_tag_throttled || e.code() == error_code_grv_proxy_memory_limit_exceeded) {
 			return;
 		}
 
@@ -131,6 +132,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	std::vector<Reference<ITenant>> tenants;
 	std::set<TenantName> createdTenants;
 	int numTenants;
+	int numTenantGroups;
 
 	// Map from tenant number to key prefix
 	std::map<int, std::string> keyPrefixes;
@@ -138,11 +140,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	FuzzApiCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), operationId(0), success(true) {
 		std::call_once(onceFlag, [&]() { addTestCases(); });
 
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0);
-		numOps = getOption(options, LiteralStringRef("numOps"), 21);
-		rarelyCommit = getOption(options, LiteralStringRef("rarelyCommit"), false);
-		maximumTotalData = getOption(options, LiteralStringRef("maximumTotalData"), 15e6);
-		minNode = getOption(options, LiteralStringRef("minNode"), 0);
+		testDuration = getOption(options, "testDuration"_sr, 60.0);
+		numOps = getOption(options, "numOps"_sr, 21);
+		rarelyCommit = getOption(options, "rarelyCommit"_sr, false);
+		maximumTotalData = getOption(options, "maximumTotalData"_sr, 15e6);
+		minNode = getOption(options, "minNode"_sr, 0);
 		adjacentKeys = deterministicRandom()->coinflip();
 		useSystemKeys = deterministicRandom()->coinflip();
 		initialKeyDensity = deterministicRandom()->random01(); // This fraction of keys are present before the first
@@ -154,6 +156,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 		int maxTenants = getOption(options, "numTenants"_sr, 4);
 		numTenants = deterministicRandom()->randomInt(0, maxTenants + 1);
 
+		int maxTenantGroups = getOption(options, "numTenantGroups"_sr, numTenants);
+		numTenantGroups = deterministicRandom()->randomInt(0, maxTenantGroups + 1);
+
 		// See https://github.com/apple/foundationdb/issues/2424
 		if (BUGGIFY) {
 			enableBuggify(true, BuggifyType::Client);
@@ -176,7 +181,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 		}
 
 		maxClearSize = 1 << deterministicRandom()->randomInt(0, 20);
-		conflictRange = KeyRangeRef(LiteralStringRef("\xfe"), LiteralStringRef("\xfe\x00"));
+		conflictRange = KeyRangeRef("\xfe"_sr, "\xfe\x00"_sr);
 		TraceEvent("FuzzApiCorrectnessConfiguration")
 		    .detail("Nodes", nodes)
 		    .detail("NumTenants", numTenants)
@@ -206,6 +211,14 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	std::string description() const override { return "FuzzApiCorrectness"; }
 
 	static TenantName getTenant(int num) { return TenantNameRef(format("tenant_%d", num)); }
+	Optional<TenantGroupName> getTenantGroup(int num) {
+		int groupNum = num % (numTenantGroups + 1);
+		if (groupNum == numTenantGroups - 1) {
+			return Optional<TenantGroupName>();
+		} else {
+			return TenantGroupNameRef(format("tenantgroup_%d", groupNum));
+		}
+	}
 	bool canUseTenant(Optional<TenantName> tenant) { return !tenant.present() || createdTenants.count(tenant.get()); }
 
 	Future<Void> setup(Database const& cx) override {
@@ -226,7 +239,10 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 
 			// The last tenant will not be created
 			if (i < self->numTenants) {
-				tenantFutures.push_back(::success(TenantAPI::createTenant(cx.getReference(), tenantName)));
+				TenantMapEntry entry;
+				entry.tenantGroup = self->getTenantGroup(i);
+				entry.encrypted = SERVER_KNOBS->ENABLE_ENCRYPTION;
+				tenantFutures.push_back(::success(TenantAPI::createTenant(cx.getReference(), tenantName, entry)));
 				self->createdTenants.insert(tenantName);
 			}
 		}
@@ -721,18 +737,18 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 				    error_code_special_keys_no_module_found,
 				    ExceptionContract::possibleIf(specialKeys.contains(key) && !workload->specialKeysRelaxed)),
 				// Read this particular special key may throw timed_out
-				std::make_pair(error_code_timed_out,
-				               ExceptionContract::possibleIf(key == LiteralStringRef("\xff\xff/status/json"))),
+				std::make_pair(error_code_timed_out, ExceptionContract::possibleIf(key == "\xff\xff/status/json"_sr)),
 				// Read this particular special key may throw special_keys_api_failure
 				std::make_pair(
 				    error_code_special_keys_api_failure,
 				    ExceptionContract::possibleIf(
-				        key ==
-				        LiteralStringRef("auto_coordinators")
-				            .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin))),
+				        key == "auto_coordinators"_sr.withPrefix(
+				                   SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin))),
 				std::make_pair(error_code_tenant_not_found,
 				               ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
 				std::make_pair(error_code_invalid_option,
+				               ExceptionContract::possibleIf(tr->getTenant().present() && specialKeys.contains(key))),
+				std::make_pair(error_code_illegal_tenant_access,
 				               ExceptionContract::possibleIf(tr->getTenant().present() && specialKeys.contains(key)))
 			};
 		}
@@ -813,6 +829,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 				std::make_pair(error_code_tenant_not_found,
 				               ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
 				std::make_pair(error_code_invalid_option,
+				               ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange)),
+				std::make_pair(error_code_illegal_tenant_access,
 				               ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange))
 			};
 		}
@@ -859,7 +877,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 				std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible),
 				std::make_pair(error_code_tenant_not_found,
 				               ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
-				std::make_pair(error_code_invalid_option,
+				std::make_pair(error_code_illegal_tenant_access,
 				               ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange))
 			};
 		}
@@ -899,12 +917,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 
 			bool isSpecialKeyRange = specialKeys.contains(key1) && specialKeys.begin <= key2 && key2 <= specialKeys.end;
 			// Read this particular special key may throw special_keys_api_failure
-			Key autoCoordinatorSpecialKey =
-			    LiteralStringRef("auto_coordinators")
-			        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
+			Key autoCoordinatorSpecialKey = "auto_coordinators"_sr.withPrefix(
+			    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
 			KeyRangeRef actorLineageRange = SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ACTORLINEAGE);
 			// Read this particular special key may throw timed_out
-			Key statusJsonSpecialKey = LiteralStringRef("\xff\xff/status/json");
+			Key statusJsonSpecialKey = "\xff\xff/status/json"_sr;
 
 			contract = {
 				std::make_pair(error_code_inverted_range, ExceptionContract::requiredIf(key1 > key2)),
@@ -929,6 +946,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 				std::make_pair(error_code_tenant_not_found,
 				               ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
 				std::make_pair(error_code_invalid_option,
+				               ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange)),
+				std::make_pair(error_code_illegal_tenant_access,
 				               ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange))
 			};
 		}
@@ -956,11 +975,10 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 			limits = makeRangeLimits();
 
 			bool isSpecialKeyRange = specialKeys.contains(key1) && specialKeys.begin <= key2 && key2 <= specialKeys.end;
-			Key autoCoordinatorSpecialKey =
-			    LiteralStringRef("auto_coordinators")
-			        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
+			Key autoCoordinatorSpecialKey = "auto_coordinators"_sr.withPrefix(
+			    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
 			KeyRangeRef actorLineageRange = SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ACTORLINEAGE);
-			Key statusJsonSpecialKey = LiteralStringRef("\xff\xff/status/json");
+			Key statusJsonSpecialKey = "\xff\xff/status/json"_sr;
 
 			contract = {
 				std::make_pair(error_code_inverted_range, ExceptionContract::requiredIf(key1 > key2)),
@@ -986,6 +1004,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 				std::make_pair(error_code_tenant_not_found,
 				               ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
 				std::make_pair(error_code_invalid_option,
+				               ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange)),
+				std::make_pair(error_code_illegal_tenant_access,
 				               ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange))
 			};
 		}
diff --git a/fdbserver/workloads/GetMappedRange.actor.cpp b/fdbserver/workloads/GetMappedRange.actor.cpp
index 3ae5ecdefe..250f26eb08 100644
--- a/fdbserver/workloads/GetMappedRange.actor.cpp
+++ b/fdbserver/workloads/GetMappedRange.actor.cpp
@@ -83,15 +83,11 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	static Value dataOfRecord(int i) { return Key(format("data-of-record-%08d", i)); }
 	static Value dataOfRecord(int i, int split) { return Key(format("data-of-record-%08d-split-%08d", i, split)); }
 
-	static Key indexEntryKey(int i) {
-		return Tuple().append(prefix).append(INDEX).append(indexKey(i)).append(primaryKey(i)).pack();
-	}
-	static Key recordKey(int i) { return Tuple().append(prefix).append(RECORD).append(primaryKey(i)).pack(); }
-	static Key recordKey(int i, int split) {
-		return Tuple().append(prefix).append(RECORD).append(primaryKey(i)).append(split).pack();
-	}
-	static Value recordValue(int i) { return Tuple().append(dataOfRecord(i)).pack(); }
-	static Value recordValue(int i, int split) { return Tuple().append(dataOfRecord(i, split)).pack(); }
+	static Key indexEntryKey(int i) { return Tuple::makeTuple(prefix, INDEX, indexKey(i), primaryKey(i)).pack(); }
+	static Key recordKey(int i) { return Tuple::makeTuple(prefix, RECORD, primaryKey(i)).pack(); }
+	static Key recordKey(int i, int split) { return Tuple::makeTuple(prefix, RECORD, primaryKey(i), split).pack(); }
+	static Value recordValue(int i) { return Tuple::makeTuple(dataOfRecord(i)).pack(); }
+	static Value recordValue(int i, int split) { return Tuple::makeTuple(dataOfRecord(i, split)).pack(); }
 
 	ACTOR Future<Void> fillInRecords(Database cx, int n, GetMappedRangeWorkload* self) {
 		state Transaction tr(cx);
@@ -270,9 +266,9 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	                                   GetMappedRangeWorkload* self,
 	                                   int matchIndex,
 	                                   bool allMissing = false) {
-		Key beginTuple = Tuple().append(prefix).append(INDEX).append(indexKey(beginId)).getDataAsStandalone();
+		Key beginTuple = Tuple::makeTuple(prefix, INDEX, indexKey(beginId)).getDataAsStandalone();
 		state KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
-		Key endTuple = Tuple().append(prefix).append(INDEX).append(indexKey(endId)).getDataAsStandalone();
+		Key endTuple = Tuple::makeTuple(prefix, INDEX, indexKey(endId)).getDataAsStandalone();
 		state KeySelector endSelector = KeySelector(firstGreaterOrEqual(endTuple));
 		state int limit = 100;
 		state int expectedBeginId = beginId;
@@ -322,9 +318,9 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	                                                   Reference<TransactionWrapper>& tr,
 	                                                   GetMappedRangeWorkload* self) {
 		Key mapper = getMapper(self, false);
-		Key beginTuple = Tuple().append(prefix).append(INDEX).append(indexKey(beginId)).getDataAsStandalone();
+		Key beginTuple = Tuple::makeTuple(prefix, INDEX, indexKey(beginId)).getDataAsStandalone();
 		KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
-		Key endTuple = Tuple().append(prefix).append(INDEX).append(indexKey(endId)).getDataAsStandalone();
+		Key endTuple = Tuple::makeTuple(prefix, INDEX, indexKey(endId)).getDataAsStandalone();
 		KeySelector endSelector = KeySelector(firstGreaterOrEqual(endTuple));
 		return tr->getMappedRange(beginSelector,
 		                          endSelector,
diff --git a/fdbserver/workloads/GetRangeStream.actor.cpp b/fdbserver/workloads/GetRangeStream.actor.cpp
index 8d476069f1..68ec509de7 100644
--- a/fdbserver/workloads/GetRangeStream.actor.cpp
+++ b/fdbserver/workloads/GetRangeStream.actor.cpp
@@ -32,10 +32,10 @@ struct GetRangeStream : TestWorkload {
 	bool printKVPairs;
 
 	GetRangeStream(WorkloadContext const& wcx) : TestWorkload(wcx), bytesRead("BytesRead") {
-		useGetRange = getOption(options, LiteralStringRef("useGetRange"), false);
-		begin = getOption(options, LiteralStringRef("begin"), normalKeys.begin);
-		end = getOption(options, LiteralStringRef("end"), normalKeys.end);
-		printKVPairs = getOption(options, LiteralStringRef("printKVPairs"), false);
+		useGetRange = getOption(options, "useGetRange"_sr, false);
+		begin = getOption(options, "begin"_sr, normalKeys.begin);
+		end = getOption(options, "end"_sr, normalKeys.end);
+		printKVPairs = getOption(options, "printKVPairs"_sr, false);
 	}
 
 	std::string description() const override { return "GetRangeStreamWorkload"; }
diff --git a/fdbserver/workloads/GlobalTagThrottling.actor.cpp b/fdbserver/workloads/GlobalTagThrottling.actor.cpp
index 41ea8a630e..07191f3171 100644
--- a/fdbserver/workloads/GlobalTagThrottling.actor.cpp
+++ b/fdbserver/workloads/GlobalTagThrottling.actor.cpp
@@ -25,10 +25,8 @@
 
 class GlobalTagThrottlingWorkload : public TestWorkload {
 	TransactionTag transactionTag;
-	double reservedReadQuota{ 0.0 };
-	double totalReadQuota{ 0.0 };
-	double reservedWriteQuota{ 0.0 };
-	double totalWriteQuota{ 0.0 };
+	double reservedQuota{ 0.0 };
+	double totalQuota{ 0.0 };
 
 	ACTOR static Future<Void> setup(GlobalTagThrottlingWorkload* self, Database cx) {
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
@@ -37,19 +35,13 @@ class GlobalTagThrottlingWorkload : public TestWorkload {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				TraceEvent("GlobalTagThrottlingWorkload_SettingTagQuota")
 				    .detail("Tag", self->transactionTag)
-				    .detail("ReservedReadQuota", self->reservedReadQuota)
-				    .detail("TotalReadQuota", self->totalReadQuota)
-				    .detail("ReservedWriteQuota", self->reservedWriteQuota)
-				    .detail("TotalWriteQuota", self->totalWriteQuota);
-				ThrottleApi::setTagQuota(tr,
-				                         self->transactionTag,
-				                         self->reservedReadQuota,
-				                         self->totalReadQuota,
-				                         self->reservedWriteQuota,
-				                         self->totalWriteQuota);
+				    .detail("ReservedQuota", self->reservedQuota)
+				    .detail("TotalQuota", self->totalQuota);
+				ThrottleApi::setTagQuota(tr, self->transactionTag, self->reservedQuota, self->totalQuota);
 				wait(tr->commit());
 				return Void();
 			} catch (Error& e) {
+				TraceEvent("GlobalTagThrottlingWorkload_SetupError").error(e);
 				wait(tr->onError(e));
 			}
 		};
@@ -58,10 +50,8 @@ class GlobalTagThrottlingWorkload : public TestWorkload {
 public:
 	explicit GlobalTagThrottlingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		transactionTag = getOption(options, "transactionTag"_sr, "sampleTag"_sr);
-		reservedReadQuota = getOption(options, "reservedReadQuota"_sr, 0.0);
-		totalReadQuota = getOption(options, "totalReadQuota"_sr, 0.0);
-		reservedWriteQuota = getOption(options, "reservedWriteQuota"_sr, 0.0);
-		totalWriteQuota = getOption(options, "totalWriteQuota"_sr, 0.0);
+		reservedQuota = getOption(options, "reservedQuota"_sr, 0.0);
+		totalQuota = getOption(options, "totalQuota"_sr, 0.0);
 	}
 
 	std::string description() const override { return "GlobalTagThrottling"; }
diff --git a/fdbserver/workloads/HealthMetricsApi.actor.cpp b/fdbserver/workloads/HealthMetricsApi.actor.cpp
index 29fb0a0b64..988c3970c7 100644
--- a/fdbserver/workloads/HealthMetricsApi.actor.cpp
+++ b/fdbserver/workloads/HealthMetricsApi.actor.cpp
@@ -49,10 +49,10 @@ struct HealthMetricsApiWorkload : TestWorkload {
 	static constexpr const char* NAME = "HealthMetricsApi";
 
 	HealthMetricsApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 120.0);
-		healthMetricsCheckInterval = getOption(options, LiteralStringRef("healthMetricsCheckInterval"), 1.0);
-		sendDetailedHealthMetrics = getOption(options, LiteralStringRef("sendDetailedHealthMetrics"), true);
-		maxAllowedStaleness = getOption(options, LiteralStringRef("maxAllowedStaleness"), 60.0);
+		testDuration = getOption(options, "testDuration"_sr, 120.0);
+		healthMetricsCheckInterval = getOption(options, "healthMetricsCheckInterval"_sr, 1.0);
+		sendDetailedHealthMetrics = getOption(options, "sendDetailedHealthMetrics"_sr, true);
+		maxAllowedStaleness = getOption(options, "maxAllowedStaleness"_sr, 60.0);
 	}
 
 	std::string description() const override { return HealthMetricsApiWorkload::NAME; }
diff --git a/fdbserver/workloads/HighContentionPrefixAllocatorWorkload.actor.cpp b/fdbserver/workloads/HighContentionPrefixAllocatorWorkload.actor.cpp
index c48e2ebc02..b5becdabd7 100644
--- a/fdbserver/workloads/HighContentionPrefixAllocatorWorkload.actor.cpp
+++ b/fdbserver/workloads/HighContentionPrefixAllocatorWorkload.actor.cpp
@@ -38,9 +38,9 @@ struct HighContentionPrefixAllocatorWorkload : TestWorkload {
 
 	HighContentionPrefixAllocatorWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), allocatorSubspace("test_subspace"_sr), allocator(allocatorSubspace) {
-		numRounds = getOption(options, LiteralStringRef("numRounds"), 500);
-		maxTransactionsPerRound = getOption(options, LiteralStringRef("maxTransactionsPerRound"), 20);
-		maxAllocationsPerTransaction = getOption(options, LiteralStringRef("maxAllocationsPerTransaction"), 20);
+		numRounds = getOption(options, "numRounds"_sr, 500);
+		maxTransactionsPerRound = getOption(options, "maxTransactionsPerRound"_sr, 20);
+		maxAllocationsPerTransaction = getOption(options, "maxAllocationsPerTransaction"_sr, 20);
 	}
 
 	std::string description() const override { return HighContentionPrefixAllocatorWorkload::NAME; }
diff --git a/fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp b/fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp
new file mode 100644
index 0000000000..0e2500ece8
--- /dev/null
+++ b/fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp
@@ -0,0 +1,137 @@
+/*
+ * IDDTxnProcessorApiCorrectness.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbserver/DDSharedContext.h"
+#include "fdbserver/DDTxnProcessor.h"
+#include "fdbserver/MoveKeys.actor.h"
+#include "fdbclient/StorageServerInterface.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+bool compareShardInfo(const DDShardInfo& a, const DDShardInfo& other) {
+	std::cout << a.key.toHexString() << " | " << other.key.toHexString() << "\n";
+	std::cout << a.hasDest << " | " << other.hasDest << "\n";
+	std::cout << describe(a.primarySrc) << " | " << describe(other.primarySrc) << "\n";
+	std::cout << describe(a.primaryDest) << " | " << describe(other.primaryDest) << "\n";
+	std::cout << describe(a.remoteSrc) << " | " << describe(other.remoteSrc) << "\n";
+	std::cout << describe(a.remoteDest) << " | " << describe(other.remoteDest) << "\n";
+
+	// Mock DD just care about the server<->key mapping in DDShardInfo
+	return a.key == other.key && a.hasDest == other.hasDest && a.primaryDest == other.primaryDest &&
+	       a.primarySrc == other.primarySrc && a.remoteSrc == other.remoteSrc && a.remoteDest == other.remoteDest;
+}
+
+void verifyInitDataEqual(Reference<InitialDataDistribution> real, Reference<InitialDataDistribution> mock) {
+	// Mock DD just care about the team list and server<->key mapping are consistent with the real cluster
+	ASSERT(std::equal(
+	    real->shards.begin(), real->shards.end(), mock->shards.begin(), mock->shards.end(), compareShardInfo));
+	std::cout << describe(real->primaryTeams) << " | " << describe(mock->primaryTeams) << "\n";
+	ASSERT(real->primaryTeams == mock->primaryTeams);
+	ASSERT(real->remoteTeams == mock->remoteTeams);
+	ASSERT_EQ(real->shards.size(), mock->shards.size());
+}
+
+// Verify that all IDDTxnProcessor API implementations has consistent result
+struct IDDTxnProcessorApiWorkload : TestWorkload {
+	static const char* desc;
+	bool enabled;
+	double testDuration;
+	DDSharedContext ddContext;
+
+	std::shared_ptr<IDDTxnProcessor> real;
+	std::shared_ptr<MockGlobalState> mgs;
+	std::shared_ptr<DDMockTxnProcessor> mock;
+
+	Reference<InitialDataDistribution> realInitDD;
+
+	IDDTxnProcessorApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ddContext(UID()) {
+		enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+	}
+
+	std::string description() const override { return desc; }
+	Future<Void> setup(Database const& cx) override { return enabled ? _setup(cx, this) : Void(); }
+	Future<Void> start(Database const& cx) override { return enabled ? _start(cx, this) : Void(); }
+
+	ACTOR Future<Void> _setup(Database cx, IDDTxnProcessorApiWorkload* self) {
+		self->real = std::make_shared<DDTxnProcessor>(cx);
+		// Get the database configuration so as to use proper team size
+		wait(store(self->ddContext.configuration, self->real->getDatabaseConfiguration()));
+		ASSERT(self->ddContext.configuration.storageTeamSize > 0);
+		// FIXME: add support for generating random teams across DCs
+		ASSERT_EQ(self->ddContext.usableRegions(), 1);
+
+		loop {
+			wait(store(self->ddContext.lock, ::readMoveKeysLock(cx)));
+			// read real InitialDataDistribution
+			try {
+				wait(store(self->realInitDD,
+				           self->real->getInitialDataDistribution(
+				               self->ddContext.id(), self->ddContext.lock, {}, self->ddContext.ddEnabledState.get())));
+				std::cout << "Finish read real InitialDataDistribution: server size "
+				          << self->realInitDD->allServers.size() << ", shard size: " << self->realInitDD->shards.size()
+				          << std::endl;
+				break;
+			} catch (Error& e) {
+				if (e.code() != error_code_movekeys_conflict)
+					throw;
+			}
+		}
+		return Void();
+	}
+
+	ACTOR Future<Void> _start(Database cx, IDDTxnProcessorApiWorkload* self) {
+		state int oldMode = wait(setDDMode(cx, 0));
+		TraceEvent("RMKStartModeSetting").log();
+		self->mgs = std::make_shared<MockGlobalState>();
+		self->mgs->configuration = self->ddContext.configuration;
+		self->mock = std::make_shared<DDMockTxnProcessor>(self->mgs);
+		self->mock->setupMockGlobalState(self->realInitDD);
+
+		Reference<InitialDataDistribution> mockInitData =
+		    self->mock
+		        ->getInitialDataDistribution(
+		            self->ddContext.id(), self->ddContext.lock, {}, self->ddContext.ddEnabledState.get())
+		        .get();
+
+		verifyInitDataEqual(self->realInitDD, mockInitData);
+
+		// wait(timeout(reportErrors(self->worker(cx, self), "IDDTxnProcessorApiWorkload"), self->testDuration,
+		// Void()));
+
+		// Always set the DD mode back, even if we die with an error
+		TraceEvent("RMKDoneMoving").log();
+		wait(success(setDDMode(cx, oldMode)));
+		TraceEvent("RMKDoneModeSetting").log();
+		return Void();
+	}
+
+	// ACTOR Future<Void> worker(Database cx, IDDTxnProcessorApiWorkload* self) { return Void(); }
+
+	Future<bool> check(Database const& cx) override {
+		return tag(delay(testDuration / 2), true);
+	} // Give the database time to recover from our damage
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+const char* IDDTxnProcessorApiWorkload::desc = "IDDTxnProcessorApiCorrectness";
+
+WorkloadFactory<IDDTxnProcessorApiWorkload> IDDTxnProcessorApiWorkload(IDDTxnProcessorApiWorkload::desc);
\ No newline at end of file
diff --git a/fdbserver/workloads/Increment.actor.cpp b/fdbserver/workloads/Increment.actor.cpp
index ee65c19a62..0bd1504e51 100644
--- a/fdbserver/workloads/Increment.actor.cpp
+++ b/fdbserver/workloads/Increment.actor.cpp
@@ -35,12 +35,11 @@ struct Increment : TestWorkload {
 	Increment(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), tooOldRetries("Retries.too_old"),
 	    commitFailedRetries("Retries.commit_failed"), totalLatency("Latency") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 5000.0);
-		actorCount = getOption(options, LiteralStringRef("actorsPerClient"), transactionsPerSecond / 5);
-		nodeCount = getOption(options, LiteralStringRef("nodeCount"), transactionsPerSecond * clientCount);
-		minExpectedTransactionsPerSecond =
-		    transactionsPerSecond * getOption(options, LiteralStringRef("expectedRate"), 0.7);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0);
+		actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5);
+		nodeCount = getOption(options, "nodeCount"_sr, transactionsPerSecond * clientCount);
+		minExpectedTransactionsPerSecond = transactionsPerSecond * getOption(options, "expectedRate"_sr, 0.7);
 	}
 
 	std::string description() const override { return "IncrementWorkload"; }
@@ -84,11 +83,11 @@ struct Increment : TestWorkload {
 				while (true) {
 					try {
 						tr.atomicOp(intToTestKey(deterministicRandom()->randomInt(0, self->nodeCount / 2)),
-						            LiteralStringRef("\x01"),
+						            "\x01"_sr,
 						            MutationRef::AddValue);
 						tr.atomicOp(
 						    intToTestKey(deterministicRandom()->randomInt(self->nodeCount / 2, self->nodeCount)),
-						    LiteralStringRef("\x01"),
+						    "\x01"_sr,
 						    MutationRef::AddValue);
 						wait(tr.commit());
 						break;
@@ -110,7 +109,7 @@ struct Increment : TestWorkload {
 		}
 	}
 	bool incrementCheckData(const VectorRef<KeyValueRef>& data, Version v, Increment* self) {
-		TEST(self->transactions.getValue()); // incrementCheckData transaction has value
+		CODE_PROBE(self->transactions.getValue(), "incrementCheckData transaction has value");
 		if (self->transactions.getValue() && data.size() == 0) {
 			TraceEvent(SevError, "TestFailure")
 			    .detail("Reason", "No successful increments")
diff --git a/fdbserver/workloads/IncrementalBackup.actor.cpp b/fdbserver/workloads/IncrementalBackup.actor.cpp
index e40133ffd0..a93778a8f4 100644
--- a/fdbserver/workloads/IncrementalBackup.actor.cpp
+++ b/fdbserver/workloads/IncrementalBackup.actor.cpp
@@ -25,6 +25,7 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/Arena.h"
 #include "flow/serialize.h"
@@ -44,15 +45,15 @@ struct IncrementalBackupWorkload : TestWorkload {
 	bool clearBackupAgentKeys;
 
 	IncrementalBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		backupDir = getOption(options, LiteralStringRef("backupDir"), LiteralStringRef("file://simfdb/backups/"));
-		tag = getOption(options, LiteralStringRef("tag"), LiteralStringRef("default"));
-		submitOnly = getOption(options, LiteralStringRef("submitOnly"), false);
-		restoreOnly = getOption(options, LiteralStringRef("restoreOnly"), false);
-		waitForBackup = getOption(options, LiteralStringRef("waitForBackup"), false);
-		waitRetries = getOption(options, LiteralStringRef("waitRetries"), -1);
-		stopBackup = getOption(options, LiteralStringRef("stopBackup"), false);
-		checkBeginVersion = getOption(options, LiteralStringRef("checkBeginVersion"), false);
-		clearBackupAgentKeys = getOption(options, LiteralStringRef("clearBackupAgentKeys"), false);
+		backupDir = getOption(options, "backupDir"_sr, "file://simfdb/backups/"_sr);
+		tag = getOption(options, "tag"_sr, "default"_sr);
+		submitOnly = getOption(options, "submitOnly"_sr, false);
+		restoreOnly = getOption(options, "restoreOnly"_sr, false);
+		waitForBackup = getOption(options, "waitForBackup"_sr, false);
+		waitRetries = getOption(options, "waitRetries"_sr, -1);
+		stopBackup = getOption(options, "stopBackup"_sr, false);
+		checkBeginVersion = getOption(options, "checkBeginVersion"_sr, false);
+		clearBackupAgentKeys = getOption(options, "clearBackupAgentKeys"_sr, false);
 	}
 
 	std::string description() const override { return "IncrementalBackup"; }
@@ -145,9 +146,10 @@ struct IncrementalBackupWorkload : TestWorkload {
 	}
 
 	ACTOR static Future<Void> _start(Database cx, IncrementalBackupWorkload* self) {
+		state Standalone<VectorRef<KeyRangeRef>> backupRanges;
+		addDefaultBackupRanges(backupRanges);
+
 		if (self->submitOnly) {
-			Standalone<VectorRef<KeyRangeRef>> backupRanges;
-			backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
 			TraceEvent("IBackupSubmitAttempt").log();
 			try {
 				wait(self->backupAgent.submitBackup(cx,
@@ -157,6 +159,7 @@ struct IncrementalBackupWorkload : TestWorkload {
 				                                    1e8,
 				                                    self->tag.toString(),
 				                                    backupRanges,
+				                                    false,
 				                                    StopWhenDone::False,
 				                                    UsePartitionedLog::False,
 				                                    IncrementalBackupOnly::True));
@@ -231,10 +234,10 @@ struct IncrementalBackupWorkload : TestWorkload {
 			                                       Key(self->tag.toString()),
 			                                       backupURL,
 			                                       {},
+			                                       backupRanges,
 			                                       WaitForComplete::True,
 			                                       invalidVersion,
 			                                       Verbose::True,
-			                                       normalKeys,
 			                                       Key(),
 			                                       Key(),
 			                                       LockDB::True,
diff --git a/fdbserver/workloads/IndexScan.actor.cpp b/fdbserver/workloads/IndexScan.actor.cpp
index ca844c980c..2e39470c52 100644
--- a/fdbserver/workloads/IndexScan.actor.cpp
+++ b/fdbserver/workloads/IndexScan.actor.cpp
@@ -34,11 +34,11 @@ struct IndexScanWorkload : KVWorkload {
 
 	IndexScanWorkload(WorkloadContext const& wcx)
 	  : KVWorkload(wcx), rowsRead(0), chunks(0), failedTransactions(0), scans(0) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		bytesPerRead = getOption(options, LiteralStringRef("bytesPerRead"), 80000);
-		transactionDuration = getOption(options, LiteralStringRef("transactionDuration"), 1.0);
-		singleProcess = getOption(options, LiteralStringRef("singleProcess"), true);
-		readYourWrites = getOption(options, LiteralStringRef("readYourWrites"), true);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		bytesPerRead = getOption(options, "bytesPerRead"_sr, 80000);
+		transactionDuration = getOption(options, "transactionDuration"_sr, 1.0);
+		singleProcess = getOption(options, "singleProcess"_sr, true);
+		readYourWrites = getOption(options, "readYourWrites"_sr, true);
 	}
 
 	std::string description() const override { return "SimpleRead"; }
diff --git a/fdbserver/workloads/Inventory.actor.cpp b/fdbserver/workloads/Inventory.actor.cpp
index c5f8074625..03106cc805 100644
--- a/fdbserver/workloads/Inventory.actor.cpp
+++ b/fdbserver/workloads/Inventory.actor.cpp
@@ -38,12 +38,12 @@ struct InventoryTestWorkload : TestWorkload {
 
 	InventoryTestWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), totalLatency("Latency") {
-		actorCount = getOption(options, LiteralStringRef("actorCount"), 500);
-		nProducts = getOption(options, LiteralStringRef("nProducts"), 100000);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 10000);
-		fractionWriteTransactions = getOption(options, LiteralStringRef("fractionWriteTransactions"), 0.01);
-		productsPerWrite = getOption(options, LiteralStringRef("productsPerWrite"), 2);
+		actorCount = getOption(options, "actorCount"_sr, 500);
+		nProducts = getOption(options, "nProducts"_sr, 100000);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 10000);
+		fractionWriteTransactions = getOption(options, "fractionWriteTransactions"_sr, 0.01);
+		productsPerWrite = getOption(options, "productsPerWrite"_sr, 2);
 	}
 
 	std::string description() const override { return "InventoryTest"; }
diff --git a/fdbserver/workloads/KVStoreTest.actor.cpp b/fdbserver/workloads/KVStoreTest.actor.cpp
index afa9c3dbd6..4b8cd62e58 100644
--- a/fdbserver/workloads/KVStoreTest.actor.cpp
+++ b/fdbserver/workloads/KVStoreTest.actor.cpp
@@ -210,19 +210,19 @@ struct KVStoreTestWorkload : TestWorkload {
 	KVStoreTestWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), reads("Reads"), sets("Sets"), commits("Commits"), setupTook(0) {
 		enabled = !clientId; // only do this on the "first" client
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		operationsPerSecond = getOption(options, LiteralStringRef("operationsPerSecond"), 100e3);
-		commitFraction = getOption(options, LiteralStringRef("commitFraction"), .001);
-		setFraction = getOption(options, LiteralStringRef("setFraction"), .1);
-		nodeCount = getOption(options, LiteralStringRef("nodeCount"), 100000);
-		keyBytes = getOption(options, LiteralStringRef("keyBytes"), 8);
-		valueBytes = getOption(options, LiteralStringRef("valueBytes"), 8);
-		doSetup = getOption(options, LiteralStringRef("setup"), false);
-		doClear = getOption(options, LiteralStringRef("clear"), false);
-		doCount = getOption(options, LiteralStringRef("count"), false);
-		filename = getOption(options, LiteralStringRef("filename"), Value()).toString();
-		saturation = getOption(options, LiteralStringRef("saturation"), false);
-		storeType = getOption(options, LiteralStringRef("storeType"), LiteralStringRef("ssd")).toString();
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		operationsPerSecond = getOption(options, "operationsPerSecond"_sr, 100e3);
+		commitFraction = getOption(options, "commitFraction"_sr, .001);
+		setFraction = getOption(options, "setFraction"_sr, .1);
+		nodeCount = getOption(options, "nodeCount"_sr, 100000);
+		keyBytes = getOption(options, "keyBytes"_sr, 8);
+		valueBytes = getOption(options, "valueBytes"_sr, 8);
+		doSetup = getOption(options, "setup"_sr, false);
+		doClear = getOption(options, "clear"_sr, false);
+		doCount = getOption(options, "count"_sr, false);
+		filename = getOption(options, "filename"_sr, Value()).toString();
+		saturation = getOption(options, "saturation"_sr, false);
+		storeType = getOption(options, "storeType"_sr, "ssd"_sr).toString();
 	}
 	std::string description() const override { return "KVStoreTest"; }
 	Future<Void> setup(Database const& cx) override { return Void(); }
@@ -271,7 +271,7 @@ ACTOR Future<Void> testKVStoreMain(KVStoreTestWorkload* workload, KVTest* ptest)
 		state Key k;
 		state double cst = timer();
 		while (true) {
-			RangeResult kv = wait(test.store->readRange(KeyRangeRef(k, LiteralStringRef("\xff\xff\xff\xff")), 1000));
+			RangeResult kv = wait(test.store->readRange(KeyRangeRef(k, "\xff\xff\xff\xff"_sr), 1000));
 			count += kv.size();
 			if (kv.size() < 1000)
 				break;
diff --git a/fdbserver/workloads/KillRegion.actor.cpp b/fdbserver/workloads/KillRegion.actor.cpp
index 08b3f62946..5580bb8398 100644
--- a/fdbserver/workloads/KillRegion.actor.cpp
+++ b/fdbserver/workloads/KillRegion.actor.cpp
@@ -35,11 +35,14 @@ struct KillRegionWorkload : TestWorkload {
 	KillRegionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled =
 		    !clientId && g_network->isSimulated(); // only do this on the "first" client, and only when in simulation
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		g_simulator.usableRegions = 1;
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		g_simulator->usableRegions = 1;
 	}
 
 	std::string description() const override { return "KillRegionWorkload"; }
+
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
+
 	Future<Void> setup(Database const& cx) override {
 		if (enabled) {
 			return _setup(this, cx);
@@ -57,9 +60,9 @@ struct KillRegionWorkload : TestWorkload {
 
 	ACTOR static Future<Void> _setup(KillRegionWorkload* self, Database cx) {
 		TraceEvent("ForceRecovery_DisablePrimaryBegin").log();
-		wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator.disablePrimary, true)));
+		wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator->disablePrimary, true)));
 		TraceEvent("ForceRecovery_WaitForRemote").log();
-		wait(waitForPrimaryDC(cx, LiteralStringRef("1")));
+		wait(waitForPrimaryDC(cx, "1"_sr));
 		TraceEvent("ForceRecovery_DisablePrimaryComplete").log();
 		return Void();
 	}
@@ -75,33 +78,33 @@ struct KillRegionWorkload : TestWorkload {
 		ASSERT(g_network->isSimulated());
 		if (deterministicRandom()->random01() < 0.5) {
 			TraceEvent("ForceRecovery_DisableRemoteBegin").log();
-			wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator.disableRemote, true)));
+			wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator->disableRemote, true)));
 			TraceEvent("ForceRecovery_WaitForPrimary").log();
-			wait(waitForPrimaryDC(cx, LiteralStringRef("0")));
+			wait(waitForPrimaryDC(cx, "0"_sr));
 			TraceEvent("ForceRecovery_DisableRemoteComplete").log();
-			wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator.originalRegions, true)));
+			wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator->originalRegions, true)));
 		}
 		TraceEvent("ForceRecovery_Wait").log();
 		wait(delay(deterministicRandom()->random01() * self->testDuration));
 
 		// FIXME: killDataCenter breaks simulation if forceKill=false, since some processes can survive and
 		// partially complete a recovery
-		g_simulator.killDataCenter(LiteralStringRef("0"),
-		                           deterministicRandom()->random01() < 0.5 ? ISimulator::KillInstantly
-		                                                                   : ISimulator::RebootAndDelete,
-		                           true);
-		g_simulator.killDataCenter(LiteralStringRef("2"),
-		                           deterministicRandom()->random01() < 0.5 ? ISimulator::KillInstantly
-		                                                                   : ISimulator::RebootAndDelete,
-		                           true);
-		g_simulator.killDataCenter(LiteralStringRef("4"),
-		                           deterministicRandom()->random01() < 0.5 ? ISimulator::KillInstantly
-		                                                                   : ISimulator::RebootAndDelete,
-		                           true);
+		g_simulator->killDataCenter("0"_sr,
+		                            deterministicRandom()->random01() < 0.5 ? ISimulator::KillInstantly
+		                                                                    : ISimulator::RebootAndDelete,
+		                            true);
+		g_simulator->killDataCenter("2"_sr,
+		                            deterministicRandom()->random01() < 0.5 ? ISimulator::KillInstantly
+		                                                                    : ISimulator::RebootAndDelete,
+		                            true);
+		g_simulator->killDataCenter("4"_sr,
+		                            deterministicRandom()->random01() < 0.5 ? ISimulator::KillInstantly
+		                                                                    : ISimulator::RebootAndDelete,
+		                            true);
 
 		TraceEvent("ForceRecovery_Begin").log();
 
-		wait(forceRecovery(cx->getConnectionRecord(), LiteralStringRef("1")));
+		wait(forceRecovery(cx->getConnectionRecord(), "1"_sr));
 
 		TraceEvent("ForceRecovery_UsableRegions").log();
 
@@ -116,7 +119,7 @@ struct KillRegionWorkload : TestWorkload {
 			loop {
 				// only needed if force recovery was unnecessary and we killed the secondary
 				wait(success(ManagementAPI::changeConfig(
-				    cx.getReference(), g_simulator.disablePrimary + " repopulate_anti_quorum=1", true)));
+				    cx.getReference(), g_simulator->disablePrimary + " repopulate_anti_quorum=1", true)));
 				choose {
 					when(wait(waitForStorageRecovered(self))) { break; }
 					when(wait(delay(300.0))) {}
diff --git a/fdbserver/workloads/LocalRatekeeper.actor.cpp b/fdbserver/workloads/LocalRatekeeper.actor.cpp
index 97f8af04bc..ef54b1e116 100644
--- a/fdbserver/workloads/LocalRatekeeper.actor.cpp
+++ b/fdbserver/workloads/LocalRatekeeper.actor.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "fdbclient/FDBTypes.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include <fdbserver/Knobs.h>
 #include <flow/actorcompiler.h>
@@ -50,10 +51,9 @@ struct LocalRatekeeperWorkload : TestWorkload {
 	bool testFailed = false;
 
 	LocalRatekeeperWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		startAfter = getOption(options, LiteralStringRef("startAfter"), startAfter);
-		blockWritesFor = getOption(options,
-		                           LiteralStringRef("blockWritesFor"),
-		                           double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_HARD_MAX) / double(1e6));
+		startAfter = getOption(options, "startAfter"_sr, startAfter);
+		blockWritesFor = getOption(
+		    options, "blockWritesFor"_sr, double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_HARD_MAX) / double(1e6));
 	}
 	std::string description() const override { return "LocalRatekeeperWorkload"; }
 
@@ -82,14 +82,23 @@ struct LocalRatekeeperWorkload : TestWorkload {
 				    .detail("Actual", metrics.localRateLimit);
 			}
 			tr.reset();
-			Version readVersion = wait(tr.getReadVersion());
+			state Version readVersion = invalidVersion;
+			loop {
+				try {
+					Version v = wait(tr.getReadVersion());
+					readVersion = v;
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
 			requests.clear();
 			// we send 100 requests to this storage node and count how many of those get rejected
 			for (int i = 0; i < 100; ++i) {
 				GetValueRequest req;
 				req.version = readVersion;
 				// we don't care about the value
-				req.key = LiteralStringRef("/lkfs");
+				req.key = "/lkfs"_sr;
 				requests.emplace_back(brokenPromiseToNever(ssi.getValue.getReply(req)));
 			}
 			wait(waitForAllReady(requests));
@@ -118,7 +127,7 @@ struct LocalRatekeeperWorkload : TestWorkload {
 	ACTOR static Future<Void> _start(LocalRatekeeperWorkload* self, Database cx) {
 		wait(delay(self->startAfter));
 		state StorageServerInterface ssi = wait(getRandomStorage(cx));
-		g_simulator.disableFor(format("%s/updateStorage", ssi.id().toString().c_str()), now() + self->blockWritesFor);
+		g_simulator->disableFor(format("%s/updateStorage", ssi.id().toString().c_str()), now() + self->blockWritesFor);
 		state Future<Void> done = delay(self->blockWritesFor);
 		// not much will happen until the storage goes over the soft limit
 		wait(delay(double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_SOFT_MAX / 1e6)));
diff --git a/fdbserver/workloads/LockDatabase.actor.cpp b/fdbserver/workloads/LockDatabase.actor.cpp
index 85cfdc6837..81f46f8c32 100644
--- a/fdbserver/workloads/LockDatabase.actor.cpp
+++ b/fdbserver/workloads/LockDatabase.actor.cpp
@@ -31,9 +31,9 @@ struct LockDatabaseWorkload : TestWorkload {
 	bool onlyCheckLocked;
 
 	LockDatabaseWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ok(true) {
-		lockAfter = getOption(options, LiteralStringRef("lockAfter"), 0.0);
-		unlockAfter = getOption(options, LiteralStringRef("unlockAfter"), 10.0);
-		onlyCheckLocked = getOption(options, LiteralStringRef("onlyCheckLocked"), false);
+		lockAfter = getOption(options, "lockAfter"_sr, 0.0);
+		unlockAfter = getOption(options, "unlockAfter"_sr, 10.0);
+		onlyCheckLocked = getOption(options, "onlyCheckLocked"_sr, false);
 		ASSERT(unlockAfter > lockAfter);
 	}
 
@@ -113,7 +113,7 @@ struct LockDatabaseWorkload : TestWorkload {
 				self->ok = false;
 				return Void();
 			} catch (Error& e) {
-				TEST(e.code() == error_code_database_locked); // Database confirmed locked
+				CODE_PROBE(e.code() == error_code_database_locked, "Database confirmed locked");
 				wait(tr.onError(e));
 			}
 		}
diff --git a/fdbserver/workloads/LockDatabaseFrequently.actor.cpp b/fdbserver/workloads/LockDatabaseFrequently.actor.cpp
index bfab394031..3e4d6a6c04 100644
--- a/fdbserver/workloads/LockDatabaseFrequently.actor.cpp
+++ b/fdbserver/workloads/LockDatabaseFrequently.actor.cpp
@@ -30,8 +30,8 @@ struct LockDatabaseFrequentlyWorkload : TestWorkload {
 	PerfIntCounter lockCount{ "LockCount" };
 
 	LockDatabaseFrequentlyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		delayBetweenLocks = getOption(options, LiteralStringRef("delayBetweenLocks"), 0.1);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 60);
+		delayBetweenLocks = getOption(options, "delayBetweenLocks"_sr, 0.1);
+		testDuration = getOption(options, "testDuration"_sr, 60);
 	}
 
 	std::string description() const override { return "LockDatabaseFrequently"; }
diff --git a/fdbserver/workloads/LogMetrics.actor.cpp b/fdbserver/workloads/LogMetrics.actor.cpp
index 1ebb9c8e98..5950af5af6 100644
--- a/fdbserver/workloads/LogMetrics.actor.cpp
+++ b/fdbserver/workloads/LogMetrics.actor.cpp
@@ -35,10 +35,10 @@ struct LogMetricsWorkload : TestWorkload {
 	double logAt, logDuration, logsPerSecond;
 
 	LogMetricsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		logAt = getOption(options, LiteralStringRef("logAt"), 0.0);
-		logDuration = getOption(options, LiteralStringRef("logDuration"), 30.0);
-		logsPerSecond = getOption(options, LiteralStringRef("logsPerSecond"), 20);
-		dataFolder = getOption(options, LiteralStringRef("dataFolder"), LiteralStringRef("")).toString();
+		logAt = getOption(options, "logAt"_sr, 0.0);
+		logDuration = getOption(options, "logDuration"_sr, 30.0);
+		logsPerSecond = getOption(options, "logsPerSecond"_sr, 20);
+		dataFolder = getOption(options, "dataFolder"_sr, ""_sr).toString();
 	}
 
 	std::string description() const override { return "LogMetricsWorkload"; }
diff --git a/fdbserver/workloads/LowLatency.actor.cpp b/fdbserver/workloads/LowLatency.actor.cpp
index dc7dacc7b5..46bd365c95 100644
--- a/fdbserver/workloads/LowLatency.actor.cpp
+++ b/fdbserver/workloads/LowLatency.actor.cpp
@@ -39,12 +39,12 @@ struct LowLatencyWorkload : TestWorkload {
 
 	LowLatencyWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), operations("Operations"), retries("Retries"), ok(true) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 600.0);
-		maxGRVLatency = getOption(options, LiteralStringRef("maxGRVLatency"), 20.0);
-		maxCommitLatency = getOption(options, LiteralStringRef("maxCommitLatency"), 30.0);
-		checkDelay = getOption(options, LiteralStringRef("checkDelay"), 1.0);
-		testWrites = getOption(options, LiteralStringRef("testWrites"), true);
-		testKey = getOption(options, LiteralStringRef("testKey"), LiteralStringRef("testKey"));
+		testDuration = getOption(options, "testDuration"_sr, 600.0);
+		maxGRVLatency = getOption(options, "maxGRVLatency"_sr, 20.0);
+		maxCommitLatency = getOption(options, "maxCommitLatency"_sr, 30.0);
+		checkDelay = getOption(options, "checkDelay"_sr, 1.0);
+		testWrites = getOption(options, "testWrites"_sr, true);
+		testKey = getOption(options, "testKey"_sr, "testKey"_sr);
 	}
 
 	std::string description() const override { return "LowLatency"; }
@@ -81,7 +81,7 @@ struct LowLatencyWorkload : TestWorkload {
 						tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 						tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 						if (doCommit) {
-							tr.set(self->testKey, LiteralStringRef(""));
+							tr.set(self->testKey, ""_sr);
 							wait(tr.commit());
 						} else {
 							wait(success(tr.getReadVersion()));
diff --git a/fdbserver/workloads/MachineAttrition.actor.cpp b/fdbserver/workloads/MachineAttrition.actor.cpp
index e2a1f5492c..2ff53660d9 100644
--- a/fdbserver/workloads/MachineAttrition.actor.cpp
+++ b/fdbserver/workloads/MachineAttrition.actor.cpp
@@ -27,6 +27,7 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "flow/FaultInjection.h"
+#include "flow/DeterministicRandom.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 static std::set<int> const& normalAttritionErrors() {
@@ -60,55 +61,93 @@ ACTOR Future<bool> ignoreSSFailuresForDuration(Database cx, double duration) {
 	}
 }
 
-struct MachineAttritionWorkload : TestWorkload {
+struct MachineAttritionWorkload : FailureInjectionWorkload {
 	bool enabled;
-	int machinesToKill, machinesToLeave, workersToKill, workersToLeave;
-	double testDuration, suspendDuration, liveDuration;
-	bool reboot;
-	bool killDc;
-	bool killMachine;
-	bool killDatahall;
-	bool killProcess;
-	bool killZone;
-	bool killSelf;
+	int machinesToKill = 2, machinesToLeave = 1, workersToKill = 2, workersToLeave = 1;
+	double testDuration = 10.0, suspendDuration = 1.0, liveDuration = 5.0;
+	bool iterate = false;
+	bool reboot = false;
+	bool killDc = false;
+	bool killMachine = false;
+	bool killDatahall = false;
+	bool killProcess = false;
+	bool killZone = false;
+	bool killSelf = false;
 	std::vector<std::string> targetIds;
-	bool replacement;
-	bool waitForVersion;
-	bool allowFaultInjection;
-	Future<bool> ignoreSSFailures;
+	bool replacement = false;
+	bool waitForVersion = false;
+	bool allowFaultInjection = true;
+	Future<bool> ignoreSSFailures = true;
+	double maxRunDuration = 60.0, backoff = 1.5;
 
 	// This is set in setup from the list of workers when the cluster is started
 	std::vector<LocalityData> machines;
 
-	MachineAttritionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	MachineAttritionWorkload(WorkloadContext const& wcx, NoOptions) : FailureInjectionWorkload(wcx) {
+		enabled = !clientId && g_network->isSimulated() && faultInjectionActivated;
+		suspendDuration = 10.0;
+		iterate = true;
+	}
+
+	MachineAttritionWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		// only do this on the "first" client, and only when in simulation and only when fault injection is enabled
 		enabled = !clientId && g_network->isSimulated() && faultInjectionActivated;
-		machinesToKill = getOption(options, LiteralStringRef("machinesToKill"), 2);
-		machinesToLeave = getOption(options, LiteralStringRef("machinesToLeave"), 1);
-		workersToKill = getOption(options, LiteralStringRef("workersToKill"), 2);
-		workersToLeave = getOption(options, LiteralStringRef("workersToLeave"), 1);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		suspendDuration = getOption(options, LiteralStringRef("suspendDuration"), 1.0);
-		liveDuration = getOption(options, LiteralStringRef("liveDuration"), 5.0);
-		reboot = getOption(options, LiteralStringRef("reboot"), false);
-		killDc = getOption(
-		    options, LiteralStringRef("killDc"), g_network->isSimulated() && deterministicRandom()->random01() < 0.25);
-		killMachine = getOption(options, LiteralStringRef("killMachine"), false);
-		killDatahall = getOption(options, LiteralStringRef("killDatahall"), false);
-		killProcess = getOption(options, LiteralStringRef("killProcess"), false);
-		killZone = getOption(options, LiteralStringRef("killZone"), false);
-		killSelf = getOption(options, LiteralStringRef("killSelf"), false);
-		targetIds = getOption(options, LiteralStringRef("targetIds"), std::vector<std::string>());
-		replacement =
-		    getOption(options, LiteralStringRef("replacement"), reboot && deterministicRandom()->random01() < 0.5);
-		waitForVersion = getOption(options, LiteralStringRef("waitForVersion"), false);
-		allowFaultInjection = getOption(options, LiteralStringRef("allowFaultInjection"), true);
-		ignoreSSFailures = true;
+		machinesToKill = getOption(options, "machinesToKill"_sr, machinesToKill);
+		machinesToLeave = getOption(options, "machinesToLeave"_sr, machinesToLeave);
+		workersToKill = getOption(options, "workersToKill"_sr, workersToKill);
+		workersToLeave = getOption(options, "workersToLeave"_sr, workersToLeave);
+		testDuration = getOption(options, "testDuration"_sr, testDuration);
+		suspendDuration = getOption(options, "suspendDuration"_sr, suspendDuration);
+		liveDuration = getOption(options, "liveDuration"_sr, liveDuration);
+		reboot = getOption(options, "reboot"_sr, reboot);
+		killDc = getOption(options, "killDc"_sr, g_network->isSimulated() && deterministicRandom()->random01() < 0.25);
+		killMachine = getOption(options, "killMachine"_sr, killMachine);
+		killDatahall = getOption(options, "killDatahall"_sr, killDatahall);
+		killProcess = getOption(options, "killProcess"_sr, killProcess);
+		killZone = getOption(options, "killZone"_sr, killZone);
+		killSelf = getOption(options, "killSelf"_sr, killSelf);
+		targetIds = getOption(options, "targetIds"_sr, std::vector<std::string>());
+		replacement = getOption(options, "replacement"_sr, reboot && deterministicRandom()->random01() < 0.5);
+		waitForVersion = getOption(options, "waitForVersion"_sr, waitForVersion);
+		allowFaultInjection = getOption(options, "allowFaultInjection"_sr, allowFaultInjection);
+	}
+
+	bool shouldInject(DeterministicRandom& random,
+	                  const WorkloadRequest& work,
+	                  const unsigned alreadyAdded) const override {
+		return work.useDatabase && random.random01() < 1.0 / (2.0 + alreadyAdded);
+	}
+
+	void initializeForInjection(DeterministicRandom& random) {
+		reboot = random.random01() < 0.25;
+		replacement = random.random01() < 0.25;
+		allowFaultInjection = random.random01() < 0.5;
+		suspendDuration = 10.0 * random.random01();
+		if (g_network->isSimulated()) {
+			std::set<Optional<StringRef>> dataCenters;
+			std::set<Optional<StringRef>> dataHalls;
+			std::set<Optional<StringRef>> zones;
+			for (auto process : g_simulator->getAllProcesses()) {
+				dataCenters.emplace(process->locality.dcId().castTo<StringRef>());
+				dataHalls.emplace(process->locality.dataHallId().castTo<StringRef>());
+				zones.emplace(process->locality.zoneId().castTo<StringRef>());
+			}
+			killDc = dataCenters.size() > 0 && random.random01() > (dataHalls.size() < 0 ? 0.1 : 0.25);
+			killDatahall = dataHalls.size() > 0 && killDc && random.random01() < 0.5;
+			killZone = zones.size() > 0 && random.random01() < 0.2;
+		}
+		TraceEvent("AddingFailureInjection")
+		    .detail("Reboot", reboot)
+		    .detail("Replacement", replacement)
+		    .detail("AllowFaultInjection", allowFaultInjection)
+		    .detail("KillDC", killDc)
+		    .detail("KillDataHall", killDatahall)
+		    .detail("KillZone", killZone);
 	}
 
 	static std::vector<ISimulator::ProcessInfo*> getServers() {
 		std::vector<ISimulator::ProcessInfo*> machines;
-		std::vector<ISimulator::ProcessInfo*> all = g_simulator.getAllProcesses();
+		std::vector<ISimulator::ProcessInfo*> all = g_simulator->getAllProcesses();
 		for (int i = 0; i < all.size(); i++)
 			if (!all[i]->failed && all[i]->name == std::string("Server") &&
 			    all[i]->startingClass != ProcessClass::TesterClass)
@@ -198,7 +237,8 @@ struct MachineAttritionWorkload : TestWorkload {
 		}
 		deterministicRandom()->randomShuffle(workers);
 		wait(delay(self->liveDuration));
-		// if a specific kill is requested, it must be accompanied by a set of target IDs otherwise no kills will occur
+		// if a specific kill is requested, it must be accompanied by a set of target IDs otherwise no kills will
+		// occur
 		if (self->killDc) {
 			TraceEvent("Assassination").detail("TargetDataCenterIds", describe(self->targetIds));
 			sendRebootRequests(workers,
@@ -275,140 +315,150 @@ struct MachineAttritionWorkload : TestWorkload {
 	ACTOR static Future<Void> machineKillWorker(MachineAttritionWorkload* self, double meanDelay, Database cx) {
 		ASSERT(g_network->isSimulated());
 		state double delayBeforeKill;
+		state double suspendDuration = self->suspendDuration;
+		state double startTime = now();
 
-		if (self->killDc) {
-			delayBeforeKill = deterministicRandom()->random01() * meanDelay;
-			wait(delay(delayBeforeKill));
-
-			// decide on a machine to kill
-			ASSERT(self->machines.size());
-			Optional<Standalone<StringRef>> target = self->machines.back().dcId();
-
-			ISimulator::KillType kt = ISimulator::Reboot;
-			if (!self->reboot) {
-				int killType = deterministicRandom()->randomInt(0, 3); // FIXME: enable disk stalls
-				if (killType == 0)
-					kt = ISimulator::KillInstantly;
-				else if (killType == 1)
-					kt = ISimulator::InjectFaults;
-				else if (killType == 2)
-					kt = ISimulator::RebootAndDelete;
-				else
-					kt = ISimulator::FailDisk;
-			}
-			TraceEvent("Assassination")
-			    .detail("TargetDatacenter", target)
-			    .detail("Reboot", self->reboot)
-			    .detail("KillType", kt);
-
-			g_simulator.killDataCenter(target, kt);
-		} else if (self->killDatahall) {
-			delayBeforeKill = deterministicRandom()->random01() * meanDelay;
-			wait(delay(delayBeforeKill));
-
-			// It only makes sense to kill a single data hall.
-			ASSERT(self->targetIds.size() == 1);
-			auto target = self->targetIds.front();
-
-			auto kt = ISimulator::KillInstantly;
-			TraceEvent("Assassination").detail("TargetDataHall", target).detail("KillType", kt);
-
-			g_simulator.killDataHall(target, kt);
-		} else {
-			state int killedMachines = 0;
-			while (killedMachines < self->machinesToKill && self->machines.size() > self->machinesToLeave) {
-				TraceEvent("WorkerKillBegin")
-				    .detail("KilledMachines", killedMachines)
-				    .detail("MachinesToKill", self->machinesToKill)
-				    .detail("MachinesToLeave", self->machinesToLeave)
-				    .detail("Machines", self->machines.size());
-				TEST(true); // Killing a machine
-
+		loop {
+			if (self->killDc) {
 				delayBeforeKill = deterministicRandom()->random01() * meanDelay;
 				wait(delay(delayBeforeKill));
-				TraceEvent("WorkerKillAfterDelay").log();
-
-				if (self->waitForVersion) {
-					state Transaction tr(cx);
-					loop {
-						try {
-							tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-							tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-							wait(success(tr.getReadVersion()));
-							break;
-						} catch (Error& e) {
-							wait(tr.onError(e));
-						}
-					}
-				}
 
 				// decide on a machine to kill
-				state LocalityData targetMachine = self->machines.back();
-				if (BUGGIFY_WITH_PROB(0.01)) {
-					TEST(true); // Marked a zone for maintenance before killing it
-					wait(success(
-					    setHealthyZone(cx, targetMachine.zoneId().get(), deterministicRandom()->random01() * 20)));
-				} else if (BUGGIFY_WITH_PROB(0.005)) {
-					TEST(true); // Disable DD for all storage server failures
-					self->ignoreSSFailures =
-					    uncancellable(ignoreSSFailuresForDuration(cx, deterministicRandom()->random01() * 5));
-				}
+				ASSERT(self->machines.size());
+				Optional<Standalone<StringRef>> target = self->machines.back().dcId();
 
+				ISimulator::KillType kt = ISimulator::Reboot;
+				if (!self->reboot) {
+					int killType = deterministicRandom()->randomInt(0, 3); // FIXME: enable disk stalls
+					if (killType == 0)
+						kt = ISimulator::KillInstantly;
+					else if (killType == 1)
+						kt = ISimulator::InjectFaults;
+					else if (killType == 2)
+						kt = ISimulator::RebootAndDelete;
+					else
+						kt = ISimulator::FailDisk;
+				}
 				TraceEvent("Assassination")
-				    .detail("TargetMachine", targetMachine.toString())
-				    .detail("ZoneId", targetMachine.zoneId())
+				    .detail("TargetDatacenter", target)
 				    .detail("Reboot", self->reboot)
-				    .detail("KilledMachines", killedMachines)
-				    .detail("MachinesToKill", self->machinesToKill)
-				    .detail("MachinesToLeave", self->machinesToLeave)
-				    .detail("Machines", self->machines.size())
-				    .detail("Replace", self->replacement);
-
-				if (self->reboot) {
-					if (deterministicRandom()->random01() > 0.5) {
-						g_simulator.rebootProcess(targetMachine.zoneId(), deterministicRandom()->random01() > 0.5);
-					} else {
-						g_simulator.killZone(targetMachine.zoneId(), ISimulator::Reboot);
-					}
-				} else {
-					auto randomDouble = deterministicRandom()->random01();
-					TraceEvent("WorkerKill")
-					    .detail("MachineCount", self->machines.size())
-					    .detail("RandomValue", randomDouble);
-					if (randomDouble < 0.33) {
-						TraceEvent("RebootAndDelete").detail("TargetMachine", targetMachine.toString());
-						g_simulator.killZone(targetMachine.zoneId(), ISimulator::RebootAndDelete);
-					} else {
-						auto kt = ISimulator::KillInstantly;
-						if (self->allowFaultInjection) {
-							if (randomDouble < 0.50) {
-								kt = ISimulator::InjectFaults;
-							}
-							// FIXME: enable disk stalls
-							/*
-							if( randomDouble < 0.56 ) {
-							    kt = ISimulator::InjectFaults;
-							} else if( randomDouble < 0.66 ) {
-							    kt = ISimulator::FailDisk;
-							}
-							*/
-						}
-						g_simulator.killZone(targetMachine.zoneId(), kt);
-					}
-				}
-
-				killedMachines++;
-				if (self->replacement) {
-					// Replace by reshuffling, since we always pick from the back.
-					deterministicRandom()->randomShuffle(self->machines);
-				} else {
-					self->machines.pop_back();
-				}
-
-				wait(delay(meanDelay - delayBeforeKill) && success(self->ignoreSSFailures));
+				    .detail("KillType", kt);
 
+				g_simulator->killDataCenter(target, kt);
+			} else if (self->killDatahall) {
 				delayBeforeKill = deterministicRandom()->random01() * meanDelay;
-				TraceEvent("WorkerKillAfterMeanDelay").detail("DelayBeforeKill", delayBeforeKill);
+				wait(delay(delayBeforeKill));
+
+				// It only makes sense to kill a single data hall.
+				ASSERT(self->targetIds.size() == 1);
+				auto target = self->targetIds.front();
+
+				auto kt = ISimulator::KillInstantly;
+				TraceEvent("Assassination").detail("TargetDataHall", target).detail("KillType", kt);
+
+				g_simulator->killDataHall(target, kt);
+			} else {
+				state int killedMachines = 0;
+				while (killedMachines < self->machinesToKill && self->machines.size() > self->machinesToLeave) {
+					TraceEvent("WorkerKillBegin")
+					    .detail("KilledMachines", killedMachines)
+					    .detail("MachinesToKill", self->machinesToKill)
+					    .detail("MachinesToLeave", self->machinesToLeave)
+					    .detail("Machines", self->machines.size());
+					CODE_PROBE(true, "Killing a machine");
+
+					delayBeforeKill = deterministicRandom()->random01() * meanDelay;
+					wait(delay(delayBeforeKill));
+					TraceEvent("WorkerKillAfterDelay").log();
+
+					if (self->waitForVersion) {
+						state Transaction tr(cx);
+						loop {
+							try {
+								tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+								tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+								wait(success(tr.getReadVersion()));
+								break;
+							} catch (Error& e) {
+								wait(tr.onError(e));
+							}
+						}
+					}
+
+					// decide on a machine to kill
+					state LocalityData targetMachine = self->machines.back();
+					if (BUGGIFY_WITH_PROB(0.01)) {
+						CODE_PROBE(true, "Marked a zone for maintenance before killing it");
+						wait(success(
+						    setHealthyZone(cx, targetMachine.zoneId().get(), deterministicRandom()->random01() * 20)));
+					} else if (BUGGIFY_WITH_PROB(0.005)) {
+						CODE_PROBE(true, "Disable DD for all storage server failures");
+						self->ignoreSSFailures =
+						    uncancellable(ignoreSSFailuresForDuration(cx, deterministicRandom()->random01() * 5));
+					}
+
+					TraceEvent("Assassination")
+					    .detail("TargetMachine", targetMachine.toString())
+					    .detail("ZoneId", targetMachine.zoneId())
+					    .detail("Reboot", self->reboot)
+					    .detail("KilledMachines", killedMachines)
+					    .detail("MachinesToKill", self->machinesToKill)
+					    .detail("MachinesToLeave", self->machinesToLeave)
+					    .detail("Machines", self->machines.size())
+					    .detail("Replace", self->replacement);
+
+					if (self->reboot) {
+						if (deterministicRandom()->random01() > 0.5) {
+							g_simulator->rebootProcess(targetMachine.zoneId(), deterministicRandom()->random01() > 0.5);
+						} else {
+							g_simulator->killZone(targetMachine.zoneId(), ISimulator::Reboot);
+						}
+					} else {
+						auto randomDouble = deterministicRandom()->random01();
+						TraceEvent("WorkerKill")
+						    .detail("MachineCount", self->machines.size())
+						    .detail("RandomValue", randomDouble);
+						if (randomDouble < 0.33) {
+							TraceEvent("RebootAndDelete").detail("TargetMachine", targetMachine.toString());
+							g_simulator->killZone(targetMachine.zoneId(), ISimulator::RebootAndDelete);
+						} else {
+							auto kt = ISimulator::KillInstantly;
+							if (self->allowFaultInjection) {
+								if (randomDouble < 0.50) {
+									kt = ISimulator::InjectFaults;
+								}
+								// FIXME: enable disk stalls
+								/*
+								if( randomDouble < 0.56 ) {
+								    kt = ISimulator::InjectFaults;
+								} else if( randomDouble < 0.66 ) {
+								    kt = ISimulator::FailDisk;
+								}
+								*/
+							}
+							g_simulator->killZone(targetMachine.zoneId(), kt);
+						}
+					}
+
+					killedMachines++;
+					if (self->replacement) {
+						// Replace by reshuffling, since we always pick from the back.
+						deterministicRandom()->randomShuffle(self->machines);
+					} else {
+						self->machines.pop_back();
+					}
+
+					wait(delay(meanDelay - delayBeforeKill) && success(self->ignoreSSFailures));
+
+					delayBeforeKill = deterministicRandom()->random01() * meanDelay;
+					TraceEvent("WorkerKillAfterMeanDelay").detail("DelayBeforeKill", delayBeforeKill);
+				}
+			}
+			if (!self->iterate || now() - startTime > self->maxRunDuration) {
+				break;
+			} else {
+				wait(delay(suspendDuration));
+				suspendDuration *= self->backoff;
 			}
 		}
 
@@ -419,3 +469,5 @@ struct MachineAttritionWorkload : TestWorkload {
 };
 
 WorkloadFactory<MachineAttritionWorkload> MachineAttritionWorkloadFactory("Attrition");
+// TODO: Enable MachineAttritionWorkload injection once this is bug-free
+// FailureInjectorFactory<MachineAttritionWorkload> MachineAttritionFailureWorkloadFactory;
diff --git a/fdbserver/workloads/Mako.actor.cpp b/fdbserver/workloads/Mako.actor.cpp
index d6597674c4..140d7fda45 100644
--- a/fdbserver/workloads/Mako.actor.cpp
+++ b/fdbserver/workloads/Mako.actor.cpp
@@ -76,39 +76,39 @@ struct MakoWorkload : TestWorkload {
 	    commits("Commits"), totalOps("Operations") {
 		// init parameters from test file
 		// Number of rows populated
-		rowCount = getOption(options, LiteralStringRef("rows"), (uint64_t)10000);
+		rowCount = getOption(options, "rows"_sr, (uint64_t)10000);
 		// Test duration in seconds
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 30.0);
-		warmingDelay = getOption(options, LiteralStringRef("warmingDelay"), 0.0);
-		maxInsertRate = getOption(options, LiteralStringRef("maxInsertRate"), 1e12);
+		testDuration = getOption(options, "testDuration"_sr, 30.0);
+		warmingDelay = getOption(options, "warmingDelay"_sr, 0.0);
+		maxInsertRate = getOption(options, "maxInsertRate"_sr, 1e12);
 		// Flag to control whether to populate data into database
-		populateData = getOption(options, LiteralStringRef("populateData"), true);
+		populateData = getOption(options, "populateData"_sr, true);
 		// Flag to control whether to run benchmark
-		runBenchmark = getOption(options, LiteralStringRef("runBenchmark"), true);
+		runBenchmark = getOption(options, "runBenchmark"_sr, true);
 		// Flag to control whether to clean data in the database
-		preserveData = getOption(options, LiteralStringRef("preserveData"), true);
+		preserveData = getOption(options, "preserveData"_sr, true);
 		// If true, force commit for read-only transactions
-		commitGet = getOption(options, LiteralStringRef("commitGet"), false);
+		commitGet = getOption(options, "commitGet"_sr, false);
 		// If true, log latency for set, clear and clearrange
-		latencyForLocalOperation = getOption(options, LiteralStringRef("latencyForLocalOperation"), false);
+		latencyForLocalOperation = getOption(options, "latencyForLocalOperation"_sr, false);
 		// Target total transaction-per-second (TPS) of all clients
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 100000.0) / clientCount;
-		actorCountPerClient = getOption(options, LiteralStringRef("actorCountPerClient"), 16);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 100000.0) / clientCount;
+		actorCountPerClient = getOption(options, "actorCountPerClient"_sr, 16);
 		// Sampling rate (1 sample / <sampleSize> ops) for latency stats
-		sampleSize = getOption(options, LiteralStringRef("sampleSize"), rowCount / 100);
+		sampleSize = getOption(options, "sampleSize"_sr, rowCount / 100);
 		// If true, record latency metrics per periodicLoggingInterval; For details, see tracePeriodically()
-		enableLogging = getOption(options, LiteralStringRef("enableLogging"), false);
-		periodicLoggingInterval = getOption(options, LiteralStringRef("periodicLoggingInterval"), 5.0);
+		enableLogging = getOption(options, "enableLogging"_sr, false);
+		periodicLoggingInterval = getOption(options, "periodicLoggingInterval"_sr, 5.0);
 		// All the generated keys will start with the specified prefix
-		keyPrefix = getOption(options, LiteralStringRef("keyPrefix"), LiteralStringRef("mako")).toString();
+		keyPrefix = getOption(options, "keyPrefix"_sr, "mako"_sr).toString();
 		KEYPREFIXLEN = keyPrefix.size();
 		// If true, the workload will picking up keys which are zipfian distributed
-		zipf = getOption(options, LiteralStringRef("zipf"), false);
-		zipfConstant = getOption(options, LiteralStringRef("zipfConstant"), 0.99);
+		zipf = getOption(options, "zipf"_sr, false);
+		zipfConstant = getOption(options, "zipfConstant"_sr, 0.99);
 		// Specified length of keys and length range of values
-		keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 16);
-		maxValueBytes = getOption(options, LiteralStringRef("valueBytes"), 16);
-		minValueBytes = getOption(options, LiteralStringRef("minValueBytes"), maxValueBytes);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
+		maxValueBytes = getOption(options, "valueBytes"_sr, 16);
+		minValueBytes = getOption(options, "minValueBytes"_sr, maxValueBytes);
 		ASSERT(minValueBytes <= maxValueBytes);
 		// The inserted key is formatted as: fixed prefix('mako') + sequential number + padding('x')
 		// assume we want to insert 10000 rows with keyBytes set to 16,
@@ -135,8 +135,7 @@ struct MakoWorkload : TestWorkload {
 		// scr – SET & CLEAR RANGE
 		// grv – GetReadVersion()
 		// Every transaction is committed unless it contains only GET / GET RANGE operations.
-		operationsSpec =
-		    getOption(options, LiteralStringRef("operations"), LiteralStringRef("g100")).contents().toString();
+		operationsSpec = getOption(options, "operations"_sr, "g100"_sr).contents().toString();
 		//  parse the sequence and extract operations to be executed
 		parseOperationsSpec();
 		for (int i = 0; i < MAX_OP; ++i) {
@@ -149,11 +148,11 @@ struct MakoWorkload : TestWorkload {
 			zipfian_generator3(0, (int)rowCount - 1, zipfConstant);
 		}
 		// Added for checksum verification
-		csSize = getOption(options, LiteralStringRef("csSize"), rowCount / 100);
+		csSize = getOption(options, "csSize"_sr, rowCount / 100);
 		ASSERT(csSize <= rowCount);
-		csCount = getOption(options, LiteralStringRef("csCount"), 0);
+		csCount = getOption(options, "csCount"_sr, 0);
 		checksumVerification = (csCount != 0);
-		doChecksumVerificationOnly = getOption(options, LiteralStringRef("doChecksumVerificationOnly"), false);
+		doChecksumVerificationOnly = getOption(options, "doChecksumVerificationOnly"_sr, false);
 		if (doChecksumVerificationOnly)
 			ASSERT(checksumVerification); // csCount should be non-zero when you do checksum verification only
 		if (csCount) {
diff --git a/fdbserver/workloads/MemoryKeyValueStore.cpp b/fdbserver/workloads/MemoryKeyValueStore.cpp
index 8ac088f210..b59ec9ced7 100644
--- a/fdbserver/workloads/MemoryKeyValueStore.cpp
+++ b/fdbserver/workloads/MemoryKeyValueStore.cpp
@@ -126,12 +126,12 @@ uint64_t MemoryKeyValueStore::size() const {
 
 // The first key in the database; returned by key selectors that choose a key off the front
 Key MemoryKeyValueStore::startKey() const {
-	return LiteralStringRef("");
+	return ""_sr;
 }
 
 // The last key in the database; returned by key selectors that choose a key off the back
 Key MemoryKeyValueStore::endKey() const {
-	return LiteralStringRef("\xff");
+	return "\xff"_sr;
 }
 
 // Debugging function that prints all key-value pairs
diff --git a/fdbserver/workloads/MemoryLifetime.actor.cpp b/fdbserver/workloads/MemoryLifetime.actor.cpp
index 3b0ca269a5..ffa80a8dd6 100644
--- a/fdbserver/workloads/MemoryLifetime.actor.cpp
+++ b/fdbserver/workloads/MemoryLifetime.actor.cpp
@@ -34,7 +34,7 @@ struct MemoryLifetime : KVWorkload {
 	std::string valueString;
 
 	MemoryLifetime(WorkloadContext const& wcx) : KVWorkload(wcx) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0);
+		testDuration = getOption(options, "testDuration"_sr, 60.0);
 		valueString = std::string(maxValueBytes, '.');
 	}
 
diff --git a/fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp b/fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp
new file mode 100644
index 0000000000..11b19c7538
--- /dev/null
+++ b/fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp
@@ -0,0 +1,931 @@
+/*
+ * MetaclusterManagementWorkload.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdint>
+#include <limits>
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/Metacluster.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/RunTransaction.actor.h"
+#include "fdbclient/TenantManagement.actor.h"
+#include "fdbclient/ThreadSafeTransaction.h"
+#include "fdbrpc/simulator.h"
+#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/Knobs.h"
+#include "flow/BooleanParam.h"
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/flow.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+FDB_DEFINE_BOOLEAN_PARAM(AllowPartialMetaclusterOperations);
+
+struct MetaclusterManagementWorkload : TestWorkload {
+
+	struct DataClusterData {
+		Database db;
+		bool registered = false;
+		int tenantGroupCapacity = 0;
+
+		std::set<TenantName> tenants;
+		std::set<TenantGroupName> tenantGroups;
+		std::set<TenantName> ungroupedTenants;
+
+		DataClusterData() {}
+		DataClusterData(Database db) : db(db) {}
+	};
+
+	struct TenantData {
+		ClusterName cluster;
+		Optional<TenantGroupName> tenantGroup;
+
+		TenantData() {}
+		TenantData(ClusterName cluster, Optional<TenantGroupName> tenantGroup)
+		  : cluster(cluster), tenantGroup(tenantGroup) {}
+	};
+
+	struct TenantGroupData {
+		ClusterName cluster;
+		std::set<TenantName> tenants;
+
+		TenantGroupData() {}
+		TenantGroupData(ClusterName cluster) : cluster(cluster) {}
+	};
+
+	Reference<IDatabase> managementDb;
+	std::map<ClusterName, DataClusterData> dataDbs;
+	std::map<TenantGroupName, TenantGroupData> tenantGroups;
+	std::set<TenantName> ungroupedTenants;
+	std::vector<ClusterName> dataDbIndex;
+
+	int64_t totalTenantGroupCapacity = 0;
+	std::map<TenantName, TenantData> createdTenants;
+
+	int maxTenants;
+	int maxTenantGroups;
+	double testDuration;
+
+	MetaclusterManagementWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		maxTenants = std::min<int>(1e8 - 1, getOption(options, "maxTenants"_sr, 1000));
+		maxTenantGroups = std::min<int>(2 * maxTenants, getOption(options, "maxTenantGroups"_sr, 20));
+		testDuration = getOption(options, "testDuration"_sr, 120.0);
+	}
+
+	std::string description() const override { return "MetaclusterManagement"; }
+
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
+		out.insert("MachineAttritionWorkload");
+	}
+
+	Future<Void> setup(Database const& cx) override {
+		if (clientId == 0) {
+			if (g_network->isSimulated() && BUGGIFY) {
+				IKnobCollection::getMutableGlobalKnobCollection().setKnob(
+				    "max_tenants_per_cluster", KnobValueRef::create(int{ deterministicRandom()->randomInt(20, 100) }));
+			}
+			return _setup(cx, this);
+		} else {
+			return Void();
+		}
+	}
+	ACTOR static Future<Void> _setup(Database cx, MetaclusterManagementWorkload* self) {
+		Reference<IDatabase> threadSafeHandle =
+		    wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
+
+		MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
+		self->managementDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
+
+		ASSERT(g_simulator->extraDatabases.size() > 0);
+		for (auto connectionString : g_simulator->extraDatabases) {
+			ClusterConnectionString ccs(connectionString);
+			self->dataDbIndex.push_back(ClusterName(format("cluster_%08d", self->dataDbs.size())));
+			self->dataDbs[self->dataDbIndex.back()] =
+			    DataClusterData(Database::createSimulatedExtraDatabase(connectionString, cx->defaultTenant));
+		}
+
+		wait(success(MetaclusterAPI::createMetacluster(cx.getReference(), "management_cluster"_sr)));
+		return Void();
+	}
+
+	ClusterName chooseClusterName() { return dataDbIndex[deterministicRandom()->randomInt(0, dataDbIndex.size())]; }
+
+	TenantName chooseTenantName() {
+		TenantName tenant(format("tenant%08d", deterministicRandom()->randomInt(0, maxTenants)));
+		return tenant;
+	}
+
+	Optional<TenantGroupName> chooseTenantGroup() {
+		Optional<TenantGroupName> tenantGroup;
+		if (deterministicRandom()->coinflip()) {
+			tenantGroup =
+			    TenantGroupNameRef(format("tenantgroup%08d", deterministicRandom()->randomInt(0, maxTenantGroups)));
+		}
+
+		return tenantGroup;
+	}
+
+	ACTOR static Future<Void> registerCluster(MetaclusterManagementWorkload* self) {
+		state ClusterName clusterName = self->chooseClusterName();
+		state DataClusterData* dataDb = &self->dataDbs[clusterName];
+		state bool retried = false;
+
+		try {
+			state DataClusterEntry entry;
+			entry.capacity.numTenantGroups = deterministicRandom()->randomInt(0, 4);
+
+			loop {
+				try {
+					Future<Void> registerFuture =
+					    MetaclusterAPI::registerCluster(self->managementDb,
+					                                    clusterName,
+					                                    dataDb->db->getConnectionRecord()->getConnectionString(),
+					                                    entry);
+
+					Optional<Void> result = wait(timeout(registerFuture, deterministicRandom()->randomInt(1, 30)));
+					if (result.present()) {
+						break;
+					} else {
+						retried = true;
+					}
+				} catch (Error& e) {
+					if (e.code() == error_code_cluster_already_exists && retried && !dataDb->registered) {
+						Optional<DataClusterMetadata> clusterMetadata =
+						    wait(MetaclusterAPI::tryGetCluster(self->managementDb, clusterName));
+						ASSERT(clusterMetadata.present());
+						break;
+					} else {
+						throw;
+					}
+				}
+			}
+
+			ASSERT(!dataDb->registered);
+
+			dataDb->tenantGroupCapacity = entry.capacity.numTenantGroups;
+			self->totalTenantGroupCapacity += entry.capacity.numTenantGroups;
+			dataDb->registered = true;
+
+			// Get a version to know that the cluster has recovered
+			wait(success(runTransaction(dataDb->db.getReference(),
+			                            [](Reference<ReadYourWritesTransaction> tr) { return tr->getReadVersion(); })));
+		} catch (Error& e) {
+			if (e.code() == error_code_cluster_already_exists) {
+				ASSERT(dataDb->registered);
+				return Void();
+			}
+
+			TraceEvent(SevError, "RegisterClusterFailure").error(e).detail("ClusterName", clusterName);
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> removeCluster(MetaclusterManagementWorkload* self) {
+		state ClusterName clusterName = self->chooseClusterName();
+		state DataClusterData* dataDb = &self->dataDbs[clusterName];
+		state bool retried = false;
+
+		try {
+			loop {
+				// TODO: check force removal
+				Future<Void> removeFuture = MetaclusterAPI::removeCluster(self->managementDb, clusterName, false);
+				try {
+					Optional<Void> result = wait(timeout(removeFuture, deterministicRandom()->randomInt(1, 30)));
+					if (result.present()) {
+						break;
+					} else {
+						retried = true;
+					}
+				} catch (Error& e) {
+					if (e.code() == error_code_cluster_not_found && retried && dataDb->registered) {
+						Optional<DataClusterMetadata> clusterMetadata =
+						    wait(MetaclusterAPI::tryGetCluster(self->managementDb, clusterName));
+
+						ASSERT(!clusterMetadata.present());
+						break;
+					} else {
+						throw;
+					}
+				}
+			}
+
+			ASSERT(dataDb->registered);
+			ASSERT(dataDb->tenants.empty());
+
+			self->totalTenantGroupCapacity -= dataDb->tenantGroupCapacity;
+			dataDb->tenantGroupCapacity = 0;
+			dataDb->registered = false;
+
+			// Get a version to know that the cluster has recovered
+			wait(success(runTransaction(dataDb->db.getReference(),
+			                            [](Reference<ReadYourWritesTransaction> tr) { return tr->getReadVersion(); })));
+		} catch (Error& e) {
+			if (e.code() == error_code_cluster_not_found) {
+				ASSERT(!dataDb->registered);
+				return Void();
+			} else if (e.code() == error_code_cluster_not_empty) {
+				ASSERT(!dataDb->tenants.empty());
+				return Void();
+			}
+
+			TraceEvent(SevError, "RemoveClusterFailure").error(e).detail("ClusterName", clusterName);
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> listClusters(MetaclusterManagementWorkload* self) {
+		state ClusterName clusterName1 = self->chooseClusterName();
+		state ClusterName clusterName2 = self->chooseClusterName();
+		state int limit = deterministicRandom()->randomInt(1, self->dataDbs.size() + 1);
+
+		try {
+			std::map<ClusterName, DataClusterMetadata> clusterList =
+			    wait(MetaclusterAPI::listClusters(self->managementDb, clusterName1, clusterName2, limit));
+
+			ASSERT(clusterName1 <= clusterName2);
+
+			auto resultItr = clusterList.begin();
+
+			int count = 0;
+			for (auto localItr = self->dataDbs.find(clusterName1);
+			     localItr != self->dataDbs.find(clusterName2) && count < limit;
+			     ++localItr) {
+				if (localItr->second.registered) {
+					ASSERT(resultItr != clusterList.end());
+					ASSERT(resultItr->first == localItr->first);
+					ASSERT(resultItr->second.connectionString ==
+					       localItr->second.db->getConnectionRecord()->getConnectionString());
+					++resultItr;
+					++count;
+				}
+			}
+
+			ASSERT(resultItr == clusterList.end());
+		} catch (Error& e) {
+			if (e.code() == error_code_inverted_range) {
+				ASSERT(clusterName1 > clusterName2);
+				return Void();
+			}
+			TraceEvent(SevError, "ListClustersFailure")
+			    .error(e)
+			    .detail("BeginClusterName", clusterName1)
+			    .detail("EndClusterName", clusterName2)
+			    .detail("Limit", limit);
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> getCluster(MetaclusterManagementWorkload* self) {
+		state ClusterName clusterName = self->chooseClusterName();
+		state DataClusterData* dataDb = &self->dataDbs[clusterName];
+
+		try {
+			DataClusterMetadata clusterMetadata = wait(MetaclusterAPI::getCluster(self->managementDb, clusterName));
+			ASSERT(dataDb->registered);
+			ASSERT(dataDb->db->getConnectionRecord()->getConnectionString() == clusterMetadata.connectionString);
+		} catch (Error& e) {
+			if (e.code() == error_code_cluster_not_found) {
+				ASSERT(!dataDb->registered);
+				return Void();
+			}
+			TraceEvent(SevError, "GetClusterFailure").error(e).detail("ClusterName", clusterName);
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Optional<DataClusterEntry>> configureImpl(MetaclusterManagementWorkload* self,
+	                                                              ClusterName clusterName,
+	                                                              DataClusterData* dataDb,
+	                                                              Optional<int64_t> numTenantGroups,
+	                                                              Optional<ClusterConnectionString> connectionString) {
+		state Reference<ITransaction> tr = self->managementDb->createTransaction();
+		loop {
+			try {
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				Optional<DataClusterMetadata> clusterMetadata =
+				    wait(MetaclusterAPI::tryGetClusterTransaction(tr, clusterName));
+				state Optional<DataClusterEntry> entry;
+
+				if (clusterMetadata.present()) {
+					if (numTenantGroups.present()) {
+						entry = clusterMetadata.get().entry;
+						entry.get().capacity.numTenantGroups = numTenantGroups.get();
+					}
+					MetaclusterAPI::updateClusterMetadata(
+					    tr, clusterName, clusterMetadata.get(), connectionString, entry);
+
+					wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
+				}
+
+				return entry;
+			} catch (Error& e) {
+				wait(safeThreadFutureToFuture(tr->onError(e)));
+			}
+		}
+	}
+
+	ACTOR static Future<Void> configureCluster(MetaclusterManagementWorkload* self) {
+		state ClusterName clusterName = self->chooseClusterName();
+		state DataClusterData* dataDb = &self->dataDbs[clusterName];
+		state Optional<DataClusterEntry> updatedEntry;
+
+		state Optional<int64_t> newNumTenantGroups;
+		state Optional<ClusterConnectionString> connectionString;
+		if (deterministicRandom()->coinflip()) {
+			newNumTenantGroups = deterministicRandom()->randomInt(0, 4);
+		}
+		if (deterministicRandom()->coinflip()) {
+			connectionString = dataDb->db->getConnectionRecord()->getConnectionString();
+		}
+
+		try {
+			loop {
+				Optional<Optional<DataClusterEntry>> result =
+				    wait(timeout(configureImpl(self, clusterName, dataDb, newNumTenantGroups, connectionString),
+				                 deterministicRandom()->randomInt(1, 30)));
+				if (result.present()) {
+					updatedEntry = result.get();
+					break;
+				}
+			}
+
+			if (updatedEntry.present()) {
+				int allocatedGroups = dataDb->tenantGroups.size() + dataDb->ungroupedTenants.size();
+				int64_t tenantGroupDelta =
+				    std::max<int64_t>(updatedEntry.get().capacity.numTenantGroups, allocatedGroups) -
+				    std::max<int64_t>(dataDb->tenantGroupCapacity, allocatedGroups);
+
+				self->totalTenantGroupCapacity += tenantGroupDelta;
+				dataDb->tenantGroupCapacity = updatedEntry.get().capacity.numTenantGroups;
+			}
+		} catch (Error& e) {
+			TraceEvent(SevError, "ConfigureClusterFailure").error(e).detail("ClusterName", clusterName);
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> createTenant(MetaclusterManagementWorkload* self) {
+		state TenantName tenant = self->chooseTenantName();
+		state Optional<TenantGroupName> tenantGroup = self->chooseTenantGroup();
+
+		auto itr = self->createdTenants.find(tenant);
+		state bool exists = itr != self->createdTenants.end();
+		state bool tenantGroupExists = tenantGroup.present() && self->tenantGroups.count(tenantGroup.get());
+		state bool hasCapacity = tenantGroupExists || self->ungroupedTenants.size() + self->tenantGroups.size() <
+		                                                  self->totalTenantGroupCapacity;
+		state bool retried = false;
+		state bool preferAssignedCluster = deterministicRandom()->coinflip();
+		// Choose between two preferred clusters because if we get a partial completion and
+		// retry, we want the operation to eventually succeed instead of having a chance of
+		// never re-visiting the original preferred cluster.
+		state std::pair<ClusterName, ClusterName> preferredClusters;
+		state Optional<ClusterName> originalPreferredCluster;
+		if (preferAssignedCluster) {
+			preferredClusters.first = self->chooseClusterName();
+			preferredClusters.second = self->chooseClusterName();
+		}
+
+		state TenantMapEntry tenantMapEntry;
+		tenantMapEntry.tenantGroup = tenantGroup;
+
+		try {
+			loop {
+				try {
+					if (preferAssignedCluster && (!retried || deterministicRandom()->coinflip())) {
+						tenantMapEntry.assignedCluster =
+						    deterministicRandom()->coinflip() ? preferredClusters.first : preferredClusters.second;
+						if (!originalPreferredCluster.present()) {
+							originalPreferredCluster = tenantMapEntry.assignedCluster.get();
+						}
+					}
+					Future<Void> createFuture =
+					    MetaclusterAPI::createTenant(self->managementDb, tenant, tenantMapEntry);
+					Optional<Void> result = wait(timeout(createFuture, deterministicRandom()->randomInt(1, 30)));
+					if (result.present()) {
+						break;
+					} else {
+						retried = true;
+					}
+				} catch (Error& e) {
+					if (e.code() == error_code_tenant_already_exists && retried && !exists) {
+						Optional<TenantMapEntry> entry = wait(MetaclusterAPI::tryGetTenant(self->managementDb, tenant));
+						ASSERT(entry.present());
+						tenantMapEntry = entry.get();
+						break;
+					} else if (preferAssignedCluster && retried &&
+					           originalPreferredCluster.get() != tenantMapEntry.assignedCluster.get() &&
+					           (e.code() == error_code_cluster_no_capacity ||
+					            e.code() == error_code_cluster_not_found ||
+					            e.code() == error_code_invalid_tenant_configuration)) {
+						// When picking a different assigned cluster, it is possible to leave the
+						// tenant creation in a partially completed state, which we want to avoid.
+						// Continue retrying if the new preferred cluster throws errors rather than
+						// exiting immediately so we can allow the operation to finish.
+						continue;
+					} else {
+						throw;
+					}
+				}
+			}
+
+			TenantMapEntry entry = wait(MetaclusterAPI::getTenant(self->managementDb, tenant));
+
+			ASSERT(!exists);
+			ASSERT(hasCapacity);
+			ASSERT(entry.assignedCluster.present());
+			ASSERT(entry.tenantGroup == tenantGroup);
+
+			if (tenantGroup.present()) {
+				auto tenantGroupData =
+				    self->tenantGroups.try_emplace(tenantGroup.get(), entry.assignedCluster.get()).first;
+				ASSERT(tenantGroupData->second.cluster == entry.assignedCluster.get());
+				tenantGroupData->second.tenants.insert(tenant);
+			} else {
+				self->ungroupedTenants.insert(tenant);
+			}
+
+			auto assignedCluster = self->dataDbs.find(entry.assignedCluster.get());
+			ASSERT(!preferAssignedCluster || tenantMapEntry.assignedCluster.get() == assignedCluster->first);
+			ASSERT(assignedCluster != self->dataDbs.end());
+			ASSERT(assignedCluster->second.tenants.insert(tenant).second);
+
+			if (tenantGroup.present()) {
+				assignedCluster->second.tenantGroups.insert(tenantGroup.get());
+			} else {
+				assignedCluster->second.ungroupedTenants.insert(tenant);
+			}
+
+			ASSERT(tenantGroupExists ||
+			       assignedCluster->second.tenantGroupCapacity >=
+			           assignedCluster->second.tenantGroups.size() + assignedCluster->second.ungroupedTenants.size());
+
+			self->createdTenants[tenant] = TenantData(entry.assignedCluster.get(), tenantGroup);
+		} catch (Error& e) {
+			if (e.code() == error_code_tenant_already_exists) {
+				ASSERT(exists);
+				return Void();
+			} else if (e.code() == error_code_metacluster_no_capacity) {
+				ASSERT(!hasCapacity && !exists);
+				return Void();
+			} else if (e.code() == error_code_cluster_no_capacity) {
+				ASSERT(preferAssignedCluster);
+				return Void();
+			} else if (e.code() == error_code_cluster_not_found) {
+				ASSERT(preferAssignedCluster);
+				return Void();
+			} else if (e.code() == error_code_invalid_tenant_configuration) {
+				ASSERT(tenantGroup.present());
+				ASSERT(tenantMapEntry.assignedCluster.present());
+				auto itr = self->tenantGroups.find(tenantGroup.get());
+				ASSERT(itr->second.cluster != tenantMapEntry.assignedCluster.get());
+				return Void();
+			}
+
+			TraceEvent(SevError, "CreateTenantFailure").error(e).detail("TenantName", tenant);
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> deleteTenant(MetaclusterManagementWorkload* self) {
+		state TenantName tenant = self->chooseTenantName();
+
+		auto itr = self->createdTenants.find(tenant);
+		state bool exists = itr != self->createdTenants.end();
+		state bool retried = false;
+
+		try {
+			loop {
+				try {
+					Future<Void> deleteFuture = MetaclusterAPI::deleteTenant(self->managementDb, tenant);
+					Optional<Void> result = wait(timeout(deleteFuture, deterministicRandom()->randomInt(1, 30)));
+
+					if (result.present()) {
+						break;
+					} else {
+						retried = true;
+					}
+				} catch (Error& e) {
+					if (e.code() == error_code_tenant_not_found && retried && exists) {
+						Optional<TenantMapEntry> entry = wait(MetaclusterAPI::tryGetTenant(self->managementDb, tenant));
+						ASSERT(!entry.present());
+						break;
+					} else {
+						throw;
+					}
+				}
+			}
+
+			ASSERT(exists);
+			auto tenantData = self->createdTenants.find(tenant);
+			ASSERT(tenantData != self->createdTenants.end());
+
+			bool erasedTenantGroup = false;
+			if (tenantData->second.tenantGroup.present()) {
+				auto tenantGroupData = self->tenantGroups.find(tenantData->second.tenantGroup.get());
+				ASSERT(tenantGroupData != self->tenantGroups.end());
+				tenantGroupData->second.tenants.erase(tenant);
+				if (tenantGroupData->second.tenants.empty()) {
+					erasedTenantGroup = true;
+					self->tenantGroups.erase(tenantData->second.tenantGroup.get());
+				}
+			} else {
+				self->ungroupedTenants.erase(tenant);
+			}
+
+			auto& dataDb = self->dataDbs[tenantData->second.cluster];
+			ASSERT(dataDb.registered);
+			auto tenantItr = dataDb.tenants.find(tenant);
+			ASSERT(tenantItr != dataDb.tenants.end());
+
+			bool reducedAllocatedCount = false;
+			if (erasedTenantGroup) {
+				reducedAllocatedCount = true;
+				dataDb.tenantGroups.erase(tenantData->second.tenantGroup.get());
+			} else if (!tenantData->second.tenantGroup.present()) {
+				reducedAllocatedCount = true;
+				dataDb.ungroupedTenants.erase(tenant);
+			}
+
+			if (reducedAllocatedCount &&
+			    dataDb.ungroupedTenants.size() + dataDb.tenantGroups.size() >= dataDb.tenantGroupCapacity) {
+				--self->totalTenantGroupCapacity;
+			}
+			dataDb.tenants.erase(tenantItr);
+			self->createdTenants.erase(tenant);
+		} catch (Error& e) {
+			if (e.code() == error_code_tenant_not_found) {
+				ASSERT(!exists);
+				return Void();
+			}
+
+			TraceEvent(SevError, "DeleteTenantFailure").error(e).detail("TenantName", tenant);
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> configureTenant(MetaclusterManagementWorkload* self) {
+		state TenantName tenant = self->chooseTenantName();
+		state Optional<TenantGroupName> newTenantGroup = self->chooseTenantGroup();
+
+		auto itr = self->createdTenants.find(tenant);
+		state bool exists = itr != self->createdTenants.end();
+		state bool tenantGroupExists =
+		    newTenantGroup.present() && self->tenantGroups.find(newTenantGroup.get()) != self->tenantGroups.end();
+
+		state bool hasCapacity = false;
+		if (exists) {
+			auto& dataDb = self->dataDbs[itr->second.cluster];
+			hasCapacity = dataDb.ungroupedTenants.size() + dataDb.tenantGroups.size() < dataDb.tenantGroupCapacity;
+		}
+
+		state std::map<Standalone<StringRef>, Optional<Value>> configurationParameters = { { "tenant_group"_sr,
+			                                                                                 newTenantGroup } };
+
+		try {
+			loop {
+				Future<Void> configureFuture =
+				    MetaclusterAPI::configureTenant(self->managementDb, tenant, configurationParameters);
+				Optional<Void> result = wait(timeout(configureFuture, deterministicRandom()->randomInt(1, 30)));
+
+				if (result.present()) {
+					break;
+				}
+			}
+
+			ASSERT(exists);
+			auto tenantData = self->createdTenants.find(tenant);
+			ASSERT(tenantData != self->createdTenants.end());
+
+			auto& dataDb = self->dataDbs[tenantData->second.cluster];
+			ASSERT(dataDb.registered);
+
+			bool allocationRemoved = false;
+			bool allocationAdded = false;
+			if (tenantData->second.tenantGroup != newTenantGroup) {
+				if (tenantData->second.tenantGroup.present()) {
+					auto& tenantGroupData = self->tenantGroups[tenantData->second.tenantGroup.get()];
+					tenantGroupData.tenants.erase(tenant);
+					if (tenantGroupData.tenants.empty()) {
+						allocationRemoved = true;
+						self->tenantGroups.erase(tenantData->second.tenantGroup.get());
+						dataDb.tenantGroups.erase(tenantData->second.tenantGroup.get());
+					}
+				} else {
+					allocationRemoved = true;
+					self->ungroupedTenants.erase(tenant);
+					dataDb.ungroupedTenants.erase(tenant);
+				}
+
+				if (newTenantGroup.present()) {
+					auto [tenantGroupData, inserted] = self->tenantGroups.try_emplace(
+					    newTenantGroup.get(), TenantGroupData(tenantData->second.cluster));
+					tenantGroupData->second.tenants.insert(tenant);
+					if (inserted) {
+						allocationAdded = true;
+						dataDb.tenantGroups.insert(newTenantGroup.get());
+					}
+				} else {
+					allocationAdded = true;
+					self->ungroupedTenants.insert(tenant);
+					dataDb.ungroupedTenants.insert(tenant);
+				}
+
+				tenantData->second.tenantGroup = newTenantGroup;
+
+				if (allocationAdded && !allocationRemoved) {
+					ASSERT(hasCapacity);
+				} else if (allocationRemoved && !allocationAdded &&
+				           dataDb.ungroupedTenants.size() + dataDb.tenantGroups.size() >= dataDb.tenantGroupCapacity) {
+					--self->totalTenantGroupCapacity;
+				}
+			}
+		} catch (Error& e) {
+			if (e.code() == error_code_tenant_not_found) {
+				ASSERT(!exists);
+				return Void();
+			} else if (e.code() == error_code_cluster_no_capacity) {
+				ASSERT(exists && !hasCapacity);
+				return Void();
+			} else if (e.code() == error_code_invalid_tenant_configuration) {
+				ASSERT(exists && tenantGroupExists &&
+				       self->createdTenants[tenant].cluster != self->tenantGroups[newTenantGroup.get()].cluster);
+				return Void();
+			}
+
+			TraceEvent(SevError, "ConfigureTenantFailure")
+			    .error(e)
+			    .detail("TenantName", tenant)
+			    .detail("TenantGroup", newTenantGroup);
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> renameTenant(MetaclusterManagementWorkload* self) {
+		state TenantName tenant = self->chooseTenantName();
+		state TenantName newTenantName = self->chooseTenantName();
+
+		auto itr = self->createdTenants.find(tenant);
+		state bool exists = itr != self->createdTenants.end();
+
+		itr = self->createdTenants.find(newTenantName);
+		state bool newTenantExists = itr != self->createdTenants.end();
+
+		try {
+			state bool retried = false;
+			loop {
+				try {
+					Future<Void> renameFuture = MetaclusterAPI::renameTenant(self->managementDb, tenant, newTenantName);
+					Optional<Void> result = wait(timeout(renameFuture, deterministicRandom()->randomInt(1, 30)));
+
+					if (result.present()) {
+						break;
+					}
+
+					retried = true;
+				} catch (Error& e) {
+					// If we retry the rename after it had succeeded, we will get an error that we should ignore
+					if (e.code() == error_code_tenant_not_found && exists && !newTenantExists && retried) {
+						break;
+					}
+					throw e;
+				}
+			}
+
+			ASSERT(exists);
+			ASSERT(!newTenantExists);
+
+			Optional<TenantMapEntry> oldEntry = wait(MetaclusterAPI::tryGetTenant(self->managementDb, tenant));
+			ASSERT(!oldEntry.present());
+
+			TenantMapEntry newEntry = wait(MetaclusterAPI::getTenant(self->managementDb, newTenantName));
+
+			auto tenantData = self->createdTenants.find(tenant);
+			ASSERT(tenantData != self->createdTenants.end());
+			ASSERT(tenantData->second.tenantGroup == newEntry.tenantGroup);
+			ASSERT(newEntry.assignedCluster.present() && tenantData->second.cluster == newEntry.assignedCluster.get());
+
+			self->createdTenants[newTenantName] = tenantData->second;
+			self->createdTenants.erase(tenantData);
+
+			auto& dataDb = self->dataDbs[newEntry.assignedCluster.get()];
+			ASSERT(dataDb.registered);
+
+			dataDb.tenants.erase(tenant);
+			dataDb.tenants.insert(newTenantName);
+
+			if (newEntry.tenantGroup.present()) {
+				auto& tenantGroup = self->tenantGroups[newEntry.tenantGroup.get()];
+				tenantGroup.tenants.erase(tenant);
+				tenantGroup.tenants.insert(newTenantName);
+			} else {
+				dataDb.ungroupedTenants.erase(tenant);
+				dataDb.ungroupedTenants.insert(newTenantName);
+				self->ungroupedTenants.erase(tenant);
+				self->ungroupedTenants.insert(newTenantName);
+			}
+		} catch (Error& e) {
+			if (e.code() == error_code_tenant_not_found) {
+				ASSERT(!exists);
+				return Void();
+			} else if (e.code() == error_code_tenant_already_exists) {
+				ASSERT(newTenantExists);
+				return Void();
+			}
+
+			TraceEvent(SevError, "RenameTenantFailure")
+			    .error(e)
+			    .detail("OldTenantName", tenant)
+			    .detail("NewTenantName", newTenantName);
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	Future<Void> start(Database const& cx) override {
+		if (clientId == 0) {
+			return _start(cx, this);
+		} else {
+			return Void();
+		}
+	}
+	ACTOR static Future<Void> _start(Database cx, MetaclusterManagementWorkload* self) {
+		state double start = now();
+
+		// Run a random sequence of operations for the duration of the test
+		while (now() < start + self->testDuration) {
+			state int operation = deterministicRandom()->randomInt(0, 9);
+			if (operation == 0) {
+				wait(registerCluster(self));
+			} else if (operation == 1) {
+				wait(removeCluster(self));
+			} else if (operation == 2) {
+				wait(listClusters(self));
+			} else if (operation == 3) {
+				wait(getCluster(self));
+			} else if (operation == 4) {
+				wait(configureCluster(self));
+			} else if (operation == 5) {
+				wait(createTenant(self));
+			} else if (operation == 6) {
+				wait(deleteTenant(self));
+			} else if (operation == 7) {
+				wait(configureTenant(self));
+			} else if (operation == 8) {
+				wait(renameTenant(self));
+			}
+		}
+
+		return Void();
+	}
+
+	// Checks that the data cluster state matches our local state
+	ACTOR static Future<Void> checkDataCluster(MetaclusterManagementWorkload* self,
+	                                           ClusterName clusterName,
+	                                           DataClusterData clusterData) {
+		state Optional<MetaclusterRegistrationEntry> metaclusterRegistration;
+		state std::vector<std::pair<TenantName, TenantMapEntry>> tenants;
+		state Reference<ReadYourWritesTransaction> tr = clusterData.db->createTransaction();
+
+		loop {
+			try {
+				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				wait(
+				    store(metaclusterRegistration,
+				          MetaclusterMetadata::metaclusterRegistration().get(clusterData.db.getReference())) &&
+				    store(tenants,
+				          TenantAPI::listTenantsTransaction(tr, ""_sr, "\xff\xff"_sr, clusterData.tenants.size() + 1)));
+				break;
+			} catch (Error& e) {
+				wait(safeThreadFutureToFuture(tr->onError(e)));
+			}
+		}
+
+		if (clusterData.registered) {
+			ASSERT(metaclusterRegistration.present() &&
+			       metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_DATA);
+		} else {
+			ASSERT(!metaclusterRegistration.present());
+		}
+
+		ASSERT(tenants.size() == clusterData.tenants.size());
+		for (auto [tenantName, tenantEntry] : tenants) {
+			ASSERT(clusterData.tenants.count(tenantName));
+			ASSERT(self->createdTenants[tenantName].cluster == clusterName);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> decommissionMetacluster(MetaclusterManagementWorkload* self) {
+		state Reference<ITransaction> tr = self->managementDb->createTransaction();
+
+		state bool deleteTenants = deterministicRandom()->coinflip();
+
+		if (deleteTenants) {
+			state std::vector<std::pair<TenantName, TenantMapEntry>> tenants =
+			    wait(MetaclusterAPI::listTenants(self->managementDb, ""_sr, "\xff\xff"_sr, 10e6));
+
+			state std::vector<Future<Void>> deleteTenantFutures;
+			for (auto [tenantName, tenantMapEntry] : tenants) {
+				deleteTenantFutures.push_back(MetaclusterAPI::deleteTenant(self->managementDb, tenantName));
+			}
+
+			wait(waitForAll(deleteTenantFutures));
+		}
+
+		state std::map<ClusterName, DataClusterMetadata> dataClusters = wait(
+		    MetaclusterAPI::listClusters(self->managementDb, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS));
+
+		std::vector<Future<Void>> removeClusterFutures;
+		for (auto [clusterName, clusterMetadata] : dataClusters) {
+			removeClusterFutures.push_back(
+			    MetaclusterAPI::removeCluster(self->managementDb, clusterName, !deleteTenants));
+		}
+
+		wait(waitForAll(removeClusterFutures));
+		wait(MetaclusterAPI::decommissionMetacluster(self->managementDb));
+
+		Optional<MetaclusterRegistrationEntry> entry =
+		    wait(MetaclusterMetadata::metaclusterRegistration().get(self->managementDb));
+		ASSERT(!entry.present());
+
+		return Void();
+	}
+
+	Future<bool> check(Database const& cx) override {
+		if (clientId == 0) {
+			return _check(this);
+		} else {
+			return true;
+		}
+	}
+	ACTOR static Future<bool> _check(MetaclusterManagementWorkload* self) {
+		// The metacluster consistency check runs the tenant consistency check for each cluster
+		state MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
+		    self->managementDb, AllowPartialMetaclusterOperations::False);
+		wait(metaclusterConsistencyCheck.run());
+
+		std::map<ClusterName, DataClusterMetadata> dataClusters = wait(MetaclusterAPI::listClusters(
+		    self->managementDb, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS + 1));
+
+		std::vector<Future<Void>> dataClusterChecks;
+		for (auto [clusterName, dataClusterData] : self->dataDbs) {
+			auto dataClusterItr = dataClusters.find(clusterName);
+			if (dataClusterData.registered) {
+				ASSERT(dataClusterItr != dataClusters.end());
+				ASSERT(dataClusterItr->second.entry.capacity.numTenantGroups == dataClusterData.tenantGroupCapacity);
+			} else {
+				ASSERT(dataClusterItr == dataClusters.end());
+			}
+
+			dataClusterChecks.push_back(checkDataCluster(self, clusterName, dataClusterData));
+		}
+		wait(waitForAll(dataClusterChecks));
+
+		wait(decommissionMetacluster(self));
+
+		return true;
+	}
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<MetaclusterManagementWorkload> MetaclusterManagementWorkloadFactory("MetaclusterManagement");
diff --git a/fdbserver/workloads/MetricLogging.actor.cpp b/fdbserver/workloads/MetricLogging.actor.cpp
index 866ba522c7..92e0ccbae8 100644
--- a/fdbserver/workloads/MetricLogging.actor.cpp
+++ b/fdbserver/workloads/MetricLogging.actor.cpp
@@ -36,17 +36,17 @@ struct MetricLoggingWorkload : TestWorkload {
 	std::vector<Int64MetricHandle> int64Metrics;
 
 	MetricLoggingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), changes("Changes") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		actorCount = getOption(options, LiteralStringRef("actorCount"), 1);
-		metricCount = getOption(options, LiteralStringRef("metricCount"), 1);
-		testBool = getOption(options, LiteralStringRef("testBool"), true);
-		enabled = getOption(options, LiteralStringRef("enabled"), true);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		actorCount = getOption(options, "actorCount"_sr, 1);
+		metricCount = getOption(options, "metricCount"_sr, 1);
+		testBool = getOption(options, "testBool"_sr, true);
+		enabled = getOption(options, "enabled"_sr, true);
 
 		for (int i = 0; i < metricCount; i++) {
 			if (testBool) {
-				boolMetrics.push_back(BoolMetricHandle(LiteralStringRef("TestBool"), format("%d", i)));
+				boolMetrics.push_back(BoolMetricHandle("TestBool"_sr, format("%d", i)));
 			} else {
-				int64Metrics.push_back(Int64MetricHandle(LiteralStringRef("TestInt"), format("%d", i)));
+				int64Metrics.push_back(Int64MetricHandle("TestInt"_sr, format("%d", i)));
 			}
 		}
 	}
diff --git a/fdbserver/workloads/MiniCycle.actor.cpp b/fdbserver/workloads/MiniCycle.actor.cpp
index 5b9b48ab2c..f427dbaae5 100644
--- a/fdbserver/workloads/MiniCycle.actor.cpp
+++ b/fdbserver/workloads/MiniCycle.actor.cpp
@@ -47,7 +47,7 @@ struct MiniCycleWorkload : TestWorkload {
 		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0) / clientCount;
 		actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5);
 		nodeCount = getOption(options, "nodeCount"_sr, transactionsPerSecond * clientCount);
-		keyPrefix = unprintable(getOption(options, "keyPrefix"_sr, LiteralStringRef("")).toString());
+		keyPrefix = unprintable(getOption(options, "keyPrefix"_sr, ""_sr).toString());
 		traceParentProbability = getOption(options, "traceParentProbability "_sr, 0.01);
 		minExpectedTransactionsPerSecond = transactionsPerSecond * getOption(options, "expectedRate"_sr, 0.7);
 	}
diff --git a/fdbserver/workloads/Performance.actor.cpp b/fdbserver/workloads/Performance.actor.cpp
index 0cfc16497a..0119d3bae4 100644
--- a/fdbserver/workloads/Performance.actor.cpp
+++ b/fdbserver/workloads/Performance.actor.cpp
@@ -34,7 +34,7 @@ struct PerformanceWorkload : TestWorkload {
 	PerfMetric maxAchievedTPS;
 
 	PerformanceWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		probeWorkload = getOption(options, LiteralStringRef("probeWorkload"), LiteralStringRef("ReadWrite"));
+		probeWorkload = getOption(options, "probeWorkload"_sr, "ReadWrite"_sr);
 
 		// "Consume" all options and save for later tests
 		for (int i = 0; i < options.size(); i++) {
@@ -44,7 +44,7 @@ struct PerformanceWorkload : TestWorkload {
 				       i,
 				       printable(options[i].key).c_str(),
 				       printable(options[i].value).c_str());
-				options[i].value = LiteralStringRef("");
+				options[i].value = ""_sr;
 			}
 		}
 		printf("saved %d options\n", savedOptions.size());
@@ -78,10 +78,9 @@ struct PerformanceWorkload : TestWorkload {
 	Standalone<VectorRef<VectorRef<KeyValueRef>>> getOpts(double transactionsPerSecond) {
 		Standalone<VectorRef<KeyValueRef>> options;
 		Standalone<VectorRef<VectorRef<KeyValueRef>>> opts;
-		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("testName"), probeWorkload));
-		options.push_back_deep(
-		    options.arena(),
-		    KeyValueRef(LiteralStringRef("transactionsPerSecond"), format("%f", transactionsPerSecond)));
+		options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, probeWorkload));
+		options.push_back_deep(options.arena(),
+		                       KeyValueRef("transactionsPerSecond"_sr, format("%f", transactionsPerSecond)));
 		for (int i = 0; i < savedOptions.size(); i++) {
 			options.push_back_deep(options.arena(), savedOptions[i]);
 			printf("option [%d]: '%s'='%s'\n",
@@ -134,7 +133,7 @@ struct PerformanceWorkload : TestWorkload {
 		std::vector<TesterInterface> testers = wait(self->getTesters(self));
 		self->testers = testers;
 
-		TestSpec spec(LiteralStringRef("PerformanceSetup"), false, false);
+		TestSpec spec("PerformanceSetup"_sr, false, false);
 		spec.options = options;
 		spec.phases = TestWorkload::SETUP;
 		DistributedTestResults results = wait(runWorkload(cx, testers, spec, Optional<TenantName>()));
@@ -169,7 +168,7 @@ struct PerformanceWorkload : TestWorkload {
 			}
 			state DistributedTestResults results;
 			try {
-				TestSpec spec(LiteralStringRef("PerformanceRun"), false, false);
+				TestSpec spec("PerformanceRun"_sr, false, false);
 				spec.phases = TestWorkload::EXECUTION | TestWorkload::METRICS;
 				spec.options = options;
 				DistributedTestResults r = wait(runWorkload(cx, self->testers, spec, Optional<TenantName>()));
diff --git a/fdbserver/workloads/PhysicalShardMove.actor.cpp b/fdbserver/workloads/PhysicalShardMove.actor.cpp
index 8b05ab9e9f..c86e6fa6f0 100644
--- a/fdbserver/workloads/PhysicalShardMove.actor.cpp
+++ b/fdbserver/workloads/PhysicalShardMove.actor.cpp
@@ -1,5 +1,5 @@
 /*
- *PhysicalShardMove.actor.cpp
+ * PhysicalShardMove.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -43,11 +43,14 @@ std::string printValue(const ErrorOr<Optional<Value>>& value) {
 }
 } // namespace
 
-struct SSCheckpointWorkload : TestWorkload {
+struct PhysicalShardMoveWorkLoad : TestWorkload {
+	FlowLock startMoveKeysParallelismLock;
+	FlowLock finishMoveKeysParallelismLock;
+	FlowLock cleanUpDataMoveParallelismLock;
 	const bool enabled;
 	bool pass;
 
-	SSCheckpointWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(!clientId), pass(true) {}
+	PhysicalShardMoveWorkLoad(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(!clientId), pass(true) {}
 
 	void validationFailed(ErrorOr<Optional<Value>> expectedValue, ErrorOr<Optional<Value>> actualValue) {
 		TraceEvent(SevError, "TestFailed")
@@ -56,7 +59,7 @@ struct SSCheckpointWorkload : TestWorkload {
 		pass = false;
 	}
 
-	std::string description() const override { return "SSCheckpoint"; }
+	std::string description() const override { return "PhysicalShardMove"; }
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
@@ -67,133 +70,170 @@ struct SSCheckpointWorkload : TestWorkload {
 		return _start(this, cx);
 	}
 
-	ACTOR Future<Void> _start(SSCheckpointWorkload* self, Database cx) {
-		state Key key = "TestKey"_sr;
-		state Key endKey = "TestKey0"_sr;
-		state Value oldValue = "TestValue"_sr;
-		state KeyRange testRange = KeyRangeRef(key, endKey);
-		state std::vector<CheckpointMetaData> records;
-		state RangeResult res;
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("MoveKeysWorkload"); }
 
+	ACTOR Future<Void> _start(PhysicalShardMoveWorkLoad* self, Database cx) {
 		int ignore = wait(setDDMode(cx, 0));
-		state Version version = wait(self->writeAndVerify(self, cx, key, oldValue));
+		state std::vector<UID> teamA;
+		state std::map<Key, Value> kvs({ { "TestKeyA"_sr, "TestValueA"_sr },
+		                                 { "TestKeyB"_sr, "TestValueB"_sr },
+		                                 { "TestKeyC"_sr, "TestValueC"_sr },
+		                                 { "TestKeyD"_sr, "TestValueD"_sr },
+		                                 { "TestKeyE"_sr, "TestValueE"_sr },
+		                                 { "TestKeyF"_sr, "TestValueF"_sr } });
 
-		// Create checkpoint.
-		state Transaction tr(cx);
-		state CheckpointFormat format = deterministicRandom()->coinflip() ? RocksDBColumnFamily : RocksDB;
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				wait(createCheckpoint(&tr, testRange, format));
-				wait(tr.commit());
-				version = tr.getCommittedVersion();
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
+		Version _ = wait(self->populateData(self, cx, &kvs));
 
-		TraceEvent("TestCheckpointCreated").detail("Range", testRange).detail("Version", version);
+		TraceEvent("TestValueWritten").log();
 
-		// Fetch checkpoint meta data.
-		loop {
-			try {
-				std::vector<CheckpointMetaData> _records = wait(getCheckpointMetaData(cx, testRange, version, format));
-				records = std::move(_records);
-				break;
-			} catch (Error& e) {
-				TraceEvent("TestFetchCheckpointMetadataError")
-				    .errorUnsuppressed(e)
-				    .detail("Range", testRange)
-				    .detail("Version", version);
-
-				// The checkpoint was just created, we don't expect this error.
-				ASSERT(e.code() != error_code_checkpoint_not_found);
-			}
-		}
-
-		TraceEvent("TestCheckpointFetched")
-		    .detail("Range", testRange)
-		    .detail("Version", version)
-		    .detail("Checkpoints", describe(records));
-
-		state std::string pwd = platform::getWorkingDirectory();
-		state std::string folder = pwd + "/checkpoints";
-		platform::eraseDirectoryRecursive(folder);
-		ASSERT(platform::createDirectory(folder));
-
-		// Fetch checkpoint.
-		state std::vector<CheckpointMetaData> fetchedCheckpoints;
-		state int i = 0;
-		for (; i < records.size(); ++i) {
-			loop {
-				TraceEvent("TestFetchingCheckpoint").detail("Checkpoint", records[i].toString());
-				try {
-					state CheckpointMetaData record = wait(fetchCheckpoint(cx, records[0], folder));
-					fetchedCheckpoints.push_back(record);
-					TraceEvent("TestCheckpointFetched").detail("Checkpoint", record.toString());
-					break;
-				} catch (Error& e) {
-					TraceEvent("TestFetchCheckpointError")
-					    .errorUnsuppressed(e)
-					    .detail("Checkpoint", records[i].toString());
-					wait(delay(1));
-				}
-			}
-		}
-
-		state std::string rocksDBTestDir = "rocksdb-kvstore-test-db";
-		platform::eraseDirectoryRecursive(rocksDBTestDir);
-
-		// Restore KVS.
-		state IKeyValueStore* kvStore = keyValueStoreRocksDB(
-		    rocksDBTestDir, deterministicRandom()->randomUniqueID(), KeyValueStoreType::SSD_ROCKSDB_V1);
-		wait(kvStore->init());
-		try {
-			wait(kvStore->restore(fetchedCheckpoints));
-		} catch (Error& e) {
-			TraceEvent(SevError, "TestRestoreCheckpointError")
-			    .errorUnsuppressed(e)
-			    .detail("Checkpoint", describe(records));
-		}
-
-		// Compare the keyrange between the original database and the one restored from checkpoint.
-		// For now, it should have been a single key.
-		tr.reset();
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				RangeResult _res = wait(tr.getRange(KeyRangeRef(key, endKey), CLIENT_KNOBS->TOO_MANY));
-				res = std::move(_res);
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-
-		RangeResult kvRange = wait(kvStore->readRange(testRange));
-		ASSERT(res.size() == kvRange.size());
-		for (int i = 0; i < res.size(); ++i) {
-			ASSERT(res[i] == kvRange[i]);
-		}
-
-		// have this line in a block so we don't break OPEN_FOR_IDE
+		state std::unordered_set<UID> excludes;
+		state std::unordered_set<UID> includes;
+		state int teamSize = 1;
 		{
-			int ignore = wait(setDDMode(cx, 1));
-			(void)ignore;
+			std::vector<UID> _teamA = wait(self->moveShard(self,
+			                                               cx,
+			                                               deterministicRandom()->randomUniqueID(),
+			                                               KeyRangeRef("TestKeyA"_sr, "TestKeyF"_sr),
+			                                               teamSize,
+			                                               includes,
+			                                               excludes));
+			teamA = std::move(_teamA);
+		}
+		excludes.insert(teamA.begin(), teamA.end());
+
+		state uint64_t sh0 = deterministicRandom()->randomUInt64();
+		state uint64_t sh1 = deterministicRandom()->randomUInt64();
+		state uint64_t sh2 = deterministicRandom()->randomUInt64();
+
+		// Move range [TestKeyA, TestKeyB) to sh0.
+		{
+			std::vector<UID> _teamA = wait(self->moveShard(self,
+			                                               cx,
+			                                               UID(sh0, deterministicRandom()->randomUInt64()),
+			                                               KeyRangeRef("TestKeyA"_sr, "TestKeyB"_sr),
+			                                               teamSize,
+			                                               includes,
+			                                               excludes));
+			teamA = std::move(_teamA);
+		}
+
+		// Move range [TestKeyB, TestKeyC) to sh1, on the same server.
+		includes.insert(teamA.begin(), teamA.end());
+		state std::vector<UID> teamB = wait(self->moveShard(self,
+		                                                    cx,
+		                                                    UID(sh1, deterministicRandom()->randomUInt64()),
+		                                                    KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr),
+		                                                    teamSize,
+		                                                    includes,
+		                                                    excludes));
+		ASSERT(std::equal(teamA.begin(), teamA.end(), teamB.begin()));
+
+		state int teamIdx = 0;
+		for (teamIdx = 0; teamIdx < teamA.size(); ++teamIdx) {
+			std::vector<StorageServerShard> shards =
+			    wait(self->getStorageServerShards(cx, teamA[teamIdx], KeyRangeRef("TestKeyA"_sr, "TestKeyC"_sr)));
+			ASSERT(shards.size() == 2);
+			ASSERT(shards[0].desiredId == sh0);
+			ASSERT(shards[1].desiredId == sh1);
+			TraceEvent("TestStorageServerShards", teamA[teamIdx]).detail("Shards", describe(shards));
+		}
+
+		state std::vector<UID> teamC = wait(self->moveShard(self,
+		                                                    cx,
+		                                                    UID(sh2, deterministicRandom()->randomUInt64()),
+		                                                    KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr),
+		                                                    teamSize,
+		                                                    includes,
+		                                                    excludes));
+		ASSERT(std::equal(teamA.begin(), teamA.end(), teamC.begin()));
+
+		for (teamIdx = 0; teamIdx < teamA.size(); ++teamIdx) {
+			std::vector<StorageServerShard> shards =
+			    wait(self->getStorageServerShards(cx, teamA[teamIdx], KeyRangeRef("TestKeyA"_sr, "TestKeyC"_sr)));
+			ASSERT(shards.size() == 2);
+			ASSERT(shards[0].desiredId == sh0);
+			ASSERT(shards[1].id == sh1);
+			ASSERT(shards[1].desiredId == sh2);
+			TraceEvent("TestStorageServerShards", teamA[teamIdx]).detail("Shards", describe(shards));
+		}
+
+		wait(self->validateData(self, cx, KeyRangeRef("TestKeyA"_sr, "TestKeyF"_sr), &kvs));
+		TraceEvent("TestValueVerified").log();
+
+		{
+			int _ = wait(setDDMode(cx, 1));
+			(void)_;
 		}
 		return Void();
 	}
 
-	ACTOR Future<Void> readAndVerify(SSCheckpointWorkload* self,
+	ACTOR Future<Version> populateData(PhysicalShardMoveWorkLoad* self, Database cx, std::map<Key, Value>* kvs) {
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		state Version version;
+		state UID debugID;
+		loop {
+			debugID = deterministicRandom()->randomUniqueID();
+			try {
+				tr->debugTransaction(debugID);
+				for (const auto& [key, value] : *kvs) {
+					tr->set(key, value);
+				}
+				wait(tr->commit());
+				version = tr->getCommittedVersion();
+				break;
+			} catch (Error& e) {
+				TraceEvent("TestCommitError").errorUnsuppressed(e);
+				wait(tr->onError(e));
+			}
+		}
+
+		TraceEvent("PopulateTestDataDone")
+		    .detail("CommitVersion", tr->getCommittedVersion())
+		    .detail("DebugID", debugID);
+
+		return version;
+	}
+
+	ACTOR Future<Void> validateData(PhysicalShardMoveWorkLoad* self,
+	                                Database cx,
+	                                KeyRange range,
+	                                std::map<Key, Value>* kvs) {
+		state Transaction tr(cx);
+		state UID debugID;
+		loop {
+			debugID = deterministicRandom()->randomUniqueID();
+			try {
+				tr.debugTransaction(debugID);
+				RangeResult res = wait(tr.getRange(range, CLIENT_KNOBS->TOO_MANY));
+				ASSERT(!res.more && res.size() < CLIENT_KNOBS->TOO_MANY);
+
+				for (const auto& kv : res) {
+					ASSERT((*kvs)[kv.key] == kv.value);
+				}
+				break;
+			} catch (Error& e) {
+				TraceEvent("TestCommitError").errorUnsuppressed(e);
+				wait(tr.onError(e));
+			}
+		}
+
+		TraceEvent("ValidateTestDataDone").detail("DebugID", debugID);
+
+		return Void();
+	}
+
+	ACTOR Future<Void> readAndVerify(PhysicalShardMoveWorkLoad* self,
 	                                 Database cx,
 	                                 Key key,
 	                                 ErrorOr<Optional<Value>> expectedValue) {
 		state Transaction tr(cx);
+		state Version readVersion;
+		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
 		loop {
 			try {
+				Version _readVersion = wait(tr.getReadVersion());
+				readVersion = readVersion;
 				state Optional<Value> res = wait(timeoutError(tr.get(key), 30.0));
 				const bool equal = !expectedValue.isError() && res == expectedValue.get();
 				if (!equal) {
@@ -201,6 +241,7 @@ struct SSCheckpointWorkload : TestWorkload {
 				}
 				break;
 			} catch (Error& e) {
+				TraceEvent("TestReadError").errorUnsuppressed(e);
 				if (expectedValue.isError() && expectedValue.getError().code() == e.code()) {
 					break;
 				}
@@ -208,35 +249,148 @@ struct SSCheckpointWorkload : TestWorkload {
 			}
 		}
 
+		TraceEvent("TestReadSuccess").detail("Version", readVersion);
+
 		return Void();
 	}
 
-	ACTOR Future<Version> writeAndVerify(SSCheckpointWorkload* self, Database cx, Key key, Optional<Value> value) {
-		state Transaction tr(cx);
+	ACTOR Future<Version> writeAndVerify(PhysicalShardMoveWorkLoad* self, Database cx, Key key, Optional<Value> value) {
+		// state Transaction tr(cx);
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
 		state Version version;
+		state UID debugID;
 		loop {
+			debugID = deterministicRandom()->randomUniqueID();
 			try {
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->debugTransaction(debugID);
 				if (value.present()) {
-					tr.set(key, value.get());
+					tr->set(key, value.get());
+					tr->set("Test?"_sr, value.get());
+					tr->set(key, value.get());
 				} else {
-					tr.clear(key);
+					tr->clear(key);
 				}
-				wait(timeoutError(tr.commit(), 30.0));
-				version = tr.getCommittedVersion();
+				wait(timeoutError(tr->commit(), 30.0));
+				version = tr->getCommittedVersion();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				TraceEvent("TestCommitError").errorUnsuppressed(e);
+				wait(tr->onError(e));
 			}
 		}
 
+		TraceEvent("TestCommitSuccess").detail("CommitVersion", tr->getCommittedVersion()).detail("DebugID", debugID);
+
 		wait(self->readAndVerify(self, cx, key, value));
 
 		return version;
 	}
 
+	// Move keys to a random selected team consisting of a single SS, after disabling DD, so that keys won't be
+	// kept in the new team until DD is enabled.
+	// Returns the address of the single SS of the new team.
+	ACTOR Future<std::vector<UID>> moveShard(PhysicalShardMoveWorkLoad* self,
+	                                         Database cx,
+	                                         UID dataMoveId,
+	                                         KeyRange keys,
+	                                         int teamSize,
+	                                         std::unordered_set<UID> includes,
+	                                         std::unordered_set<UID> excludes) {
+		// Disable DD to avoid DD undoing of our move.
+		int ignore = wait(setDDMode(cx, 0));
+
+		// Pick a random SS as the dest, keys will reside on a single server after the move.
+		std::vector<StorageServerInterface> interfs = wait(getStorageServers(cx));
+		ASSERT(interfs.size() > teamSize - includes.size());
+		while (includes.size() < teamSize) {
+			const auto& interf = interfs[deterministicRandom()->randomInt(0, interfs.size())];
+			if (excludes.count(interf.uniqueID) == 0 && includes.count(interf.uniqueID) == 0) {
+				includes.insert(interf.uniqueID);
+			}
+		}
+
+		state std::vector<UID> dests(includes.begin(), includes.end());
+		state UID owner = deterministicRandom()->randomUniqueID();
+		// state Key ownerKey = "\xff/moveKeysLock/Owner"_sr;
+		state DDEnabledState ddEnabledState;
+
+		state Transaction tr(cx);
+
+		loop {
+			try {
+				TraceEvent("TestMoveShard").detail("Range", keys.toString());
+				state MoveKeysLock moveKeysLock = wait(takeMoveKeysLock(cx, owner));
+
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				state RangeResult dataMoves = wait(tr.getRange(dataMoveKeys, CLIENT_KNOBS->TOO_MANY));
+				Version readVersion = wait(tr.getReadVersion());
+				TraceEvent("TestMoveShardReadDataMoves")
+				    .detail("DataMoves", dataMoves.size())
+				    .detail("ReadVersion", readVersion);
+				state int i = 0;
+				for (; i < dataMoves.size(); ++i) {
+					UID dataMoveId = decodeDataMoveKey(dataMoves[i].key);
+					state DataMoveMetaData dataMove = decodeDataMoveValue(dataMoves[i].value);
+					ASSERT(dataMoveId == dataMove.id);
+					TraceEvent("TestCancelDataMoveBegin").detail("DataMove", dataMove.toString());
+					wait(cleanUpDataMove(cx,
+					                     dataMoveId,
+					                     moveKeysLock,
+					                     &self->cleanUpDataMoveParallelismLock,
+					                     dataMove.range,
+					                     &ddEnabledState));
+					TraceEvent("TestCancelDataMoveEnd").detail("DataMove", dataMove.toString());
+				}
+
+				TraceEvent("TestMoveShardStartMoveKeys").detail("DataMove", dataMoveId);
+				wait(moveKeys(cx,
+				              MoveKeysParams{ dataMoveId,
+				                              keys,
+				                              dests,
+				                              dests,
+				                              moveKeysLock,
+				                              Promise<Void>(),
+				                              &self->startMoveKeysParallelismLock,
+				                              &self->finishMoveKeysParallelismLock,
+				                              false,
+				                              deterministicRandom()->randomUniqueID(), // for logging only
+				                              &ddEnabledState }));
+				break;
+			} catch (Error& e) {
+				if (e.code() == error_code_movekeys_conflict) {
+					// Conflict on moveKeysLocks with the current running DD is expected, just retry.
+					tr.reset();
+				} else {
+					wait(tr.onError(e));
+				}
+			}
+		}
+
+		TraceEvent("TestMoveShardComplete").detail("Range", keys.toString()).detail("NewTeam", describe(dests));
+
+		return dests;
+	}
+
+	ACTOR Future<std::vector<StorageServerShard>> getStorageServerShards(Database cx, UID ssId, KeyRange range) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				Optional<Value> serverListValue = wait(tr.get(serverListKeyFor(ssId)));
+				ASSERT(serverListValue.present());
+				state StorageServerInterface ssi = decodeServerListValue(serverListValue.get());
+				GetShardStateRequest req(range, GetShardStateRequest::READABLE, true);
+				GetShardStateReply rep = wait(ssi.getShardState.getReply(req, TaskPriority::DefaultEndpoint));
+				return rep.shards;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
 	Future<bool> check(Database const& cx) override { return pass; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
-WorkloadFactory<SSCheckpointWorkload> SSCheckpointWorkloadFactory("SSCheckpointWorkload");
\ No newline at end of file
+WorkloadFactory<PhysicalShardMoveWorkLoad> PhysicalShardMoveWorkLoadFactory("PhysicalShardMove");
\ No newline at end of file
diff --git a/fdbserver/workloads/Ping.actor.cpp b/fdbserver/workloads/Ping.actor.cpp
index f939d2525b..c8c67a275f 100644
--- a/fdbserver/workloads/Ping.actor.cpp
+++ b/fdbserver/workloads/Ping.actor.cpp
@@ -53,20 +53,20 @@ struct PingWorkload : TestWorkload {
 	PingWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), messages("Messages"), totalMessageLatency("TotalLatency"),
 	    maxMessageLatency("Max Latency (ms)") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		operationsPerSecond = getOption(options, LiteralStringRef("operationsPerSecondPerClient"), 50.0);
-		usePayload = getOption(options, LiteralStringRef("usePayload"), false);
-		logging = getOption(options, LiteralStringRef("logging"), false);
-		pingWorkers = getOption(options, LiteralStringRef("pingWorkers"), false);
-		registerInterface = getOption(options, LiteralStringRef("registerInterface"), true);
-		broadcastTest = getOption(options, LiteralStringRef("broadcastTest"), false);
-		parallelBroadcast = getOption(options, LiteralStringRef("parallelBroadcast"), false);
-		workerBroadcast = getOption(options, LiteralStringRef("workerBroadcast"), false);
-		int payloadSize = getOption(options, LiteralStringRef("payloadSizeOut"), 1024);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		operationsPerSecond = getOption(options, "operationsPerSecondPerClient"_sr, 50.0);
+		usePayload = getOption(options, "usePayload"_sr, false);
+		logging = getOption(options, "logging"_sr, false);
+		pingWorkers = getOption(options, "pingWorkers"_sr, false);
+		registerInterface = getOption(options, "registerInterface"_sr, true);
+		broadcastTest = getOption(options, "broadcastTest"_sr, false);
+		parallelBroadcast = getOption(options, "parallelBroadcast"_sr, false);
+		workerBroadcast = getOption(options, "workerBroadcast"_sr, false);
+		int payloadSize = getOption(options, "payloadSizeOut"_sr, 1024);
 		payloadOut = std::string(payloadSize, '.');
-		payloadSize = getOption(options, LiteralStringRef("payloadSizeBack"), 1024);
+		payloadSize = getOption(options, "payloadSizeBack"_sr, 1024);
 		payloadBack = std::string(payloadSize, '.');
-		actorCount = getOption(options, LiteralStringRef("actorCount"), 1);
+		actorCount = getOption(options, "actorCount"_sr, 1);
 	}
 
 	std::string description() const override { return "PingWorkload"; }
@@ -155,7 +155,7 @@ struct PingWorkload : TestWorkload {
 
 			LoadedPingRequest req;
 			req.id = deterministicRandom()->randomUniqueID();
-			req.payload = self->usePayload ? self->payloadOut : LiteralStringRef("");
+			req.payload = self->usePayload ? self->payloadOut : ""_sr;
 			req.loadReply = self->usePayload;
 			LoadedReply rep = wait(peer.getReply(req));
 
@@ -275,7 +275,7 @@ struct PingWorkload : TestWorkload {
 
 	// 	LoadedReply rep;
 	// 	rep.id = req.id;
-	// 	rep.payload = req.loadReply ? self->payloadBack : LiteralStringRef("");
+	// 	rep.payload = req.loadReply ? self->payloadBack : ""_sr;
 	// 	req.reply.send( rep );
 
 	// 	return Void();
@@ -296,7 +296,7 @@ struct PingWorkload : TestWorkload {
 
 			LoadedReply rep;
 			rep.id = req.id;
-			rep.payload = req.loadReply ? self->payloadBack : LiteralStringRef("");
+			rep.payload = req.loadReply ? self->payloadBack : ""_sr;
 			req.reply.send(rep);
 
 			// addActor.send( self->packetPonger( self, req ) );
diff --git a/fdbserver/workloads/ProtocolVersion.actor.cpp b/fdbserver/workloads/ProtocolVersion.actor.cpp
index 45b14d1a98..037b9be1be 100644
--- a/fdbserver/workloads/ProtocolVersion.actor.cpp
+++ b/fdbserver/workloads/ProtocolVersion.actor.cpp
@@ -29,10 +29,10 @@ struct ProtocolVersionWorkload : TestWorkload {
 	Future<Void> start(Database const& cx) override { return _start(this, cx); }
 
 	ACTOR Future<Void> _start(ProtocolVersionWorkload* self, Database cx) {
-		state std::vector<ISimulator::ProcessInfo*> allProcesses = g_pSimulator->getAllProcesses();
+		state std::vector<ISimulator::ProcessInfo*> allProcesses = g_simulator->getAllProcesses();
 		state std::vector<ISimulator::ProcessInfo*>::iterator diffVersionProcess =
 		    find_if(allProcesses.begin(), allProcesses.end(), [](const ISimulator::ProcessInfo* p) {
-			    return p->protocolVersion != currentProtocolVersion;
+			    return p->protocolVersion != currentProtocolVersion();
 		    });
 
 		ASSERT(diffVersionProcess != allProcesses.end());
diff --git a/fdbserver/workloads/PubSubMultiples.actor.cpp b/fdbserver/workloads/PubSubMultiples.actor.cpp
index a3c27ae512..270a9af347 100644
--- a/fdbserver/workloads/PubSubMultiples.actor.cpp
+++ b/fdbserver/workloads/PubSubMultiples.actor.cpp
@@ -32,10 +32,10 @@ struct PubSubMultiplesWorkload : TestWorkload {
 	PerfIntCounter messages;
 
 	PubSubMultiplesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), messages("Messages") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		messagesPerSecond = getOption(options, LiteralStringRef("messagesPerSecond"), 500.0) / clientCount;
-		actorCount = getOption(options, LiteralStringRef("actorsPerClient"), 20);
-		inboxesPerActor = getOption(options, LiteralStringRef("inboxesPerActor"), 20);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		messagesPerSecond = getOption(options, "messagesPerSecond"_sr, 500.0) / clientCount;
+		actorCount = getOption(options, "actorsPerClient"_sr, 20);
+		inboxesPerActor = getOption(options, "inboxesPerActor"_sr, 20);
 	}
 
 	std::string description() const override { return "PubSubMultiplesWorkload"; }
diff --git a/fdbserver/workloads/QueuePush.actor.cpp b/fdbserver/workloads/QueuePush.actor.cpp
index bc9c192c6d..3eccbb5422 100644
--- a/fdbserver/workloads/QueuePush.actor.cpp
+++ b/fdbserver/workloads/QueuePush.actor.cpp
@@ -40,16 +40,16 @@ struct QueuePushWorkload : TestWorkload {
 
 	QueuePushWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), commitLatencies(2000), GRVLatencies(2000) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		actorCount = getOption(options, LiteralStringRef("actorCount"), 50);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		actorCount = getOption(options, "actorCount"_sr, 50);
 
-		valueBytes = getOption(options, LiteralStringRef("valueBytes"), 96);
+		valueBytes = getOption(options, "valueBytes"_sr, 96);
 		valueString = std::string(valueBytes, 'x');
 
-		forward = getOption(options, LiteralStringRef("forward"), true);
+		forward = getOption(options, "forward"_sr, true);
 
-		endingKey = LiteralStringRef("9999999900000001");
-		startingKey = LiteralStringRef("0000000000000001");
+		endingKey = "9999999900000001"_sr;
+		startingKey = "0000000000000001"_sr;
 	}
 
 	std::string description() const override { return "QueuePush"; }
diff --git a/fdbserver/workloads/RYWDisable.actor.cpp b/fdbserver/workloads/RYWDisable.actor.cpp
index 2586118cb6..b1b3ca7684 100644
--- a/fdbserver/workloads/RYWDisable.actor.cpp
+++ b/fdbserver/workloads/RYWDisable.actor.cpp
@@ -31,9 +31,9 @@ struct RYWDisableWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 
 	RYWDisableWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 600.0);
-		nodes = getOption(options, LiteralStringRef("nodes"), 100);
-		keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 16);
+		testDuration = getOption(options, "testDuration"_sr, 600.0);
+		nodes = getOption(options, "nodes"_sr, 100);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
 	}
 
 	std::string description() const override { return "RYWDisable"; }
diff --git a/fdbserver/workloads/RYWPerformance.actor.cpp b/fdbserver/workloads/RYWPerformance.actor.cpp
index 22f9cffb7c..584e07f84c 100644
--- a/fdbserver/workloads/RYWPerformance.actor.cpp
+++ b/fdbserver/workloads/RYWPerformance.actor.cpp
@@ -28,9 +28,9 @@
 struct RYWPerformanceWorkload : TestWorkload {
 	int keyBytes, nodes, ranges;
 	RYWPerformanceWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		nodes = getOption(options, LiteralStringRef("nodes"), 10000);
-		ranges = getOption(options, LiteralStringRef("ranges"), 10);
-		keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 16);
+		nodes = getOption(options, "nodes"_sr, 10000);
+		ranges = getOption(options, "ranges"_sr, 10);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
 	}
 
 	std::string description() const override { return "RYWPerformance"; }
@@ -47,7 +47,7 @@ struct RYWPerformanceWorkload : TestWorkload {
 		loop {
 			try {
 				for (int i = 0; i < self->nodes; i++)
-					tr.set(self->keyForIndex(i), LiteralStringRef("bar"));
+					tr.set(self->keyForIndex(i), "bar"_sr);
 
 				wait(tr.commit());
 				break;
@@ -69,7 +69,7 @@ struct RYWPerformanceWorkload : TestWorkload {
 		state int i;
 		if (type == 0) {
 			for (i = 0; i < self->nodes; i++) {
-				tr->set(self->keyForIndex(i), LiteralStringRef("foo"));
+				tr->set(self->keyForIndex(i), "foo"_sr);
 			}
 		} else if (type == 1) {
 			std::vector<Future<Optional<Value>>> gets;
@@ -84,7 +84,7 @@ struct RYWPerformanceWorkload : TestWorkload {
 			}
 			wait(waitForAll(gets));
 			for (i = 0; i < self->nodes; i++) {
-				tr->set(self->keyForIndex(i), LiteralStringRef("foo"));
+				tr->set(self->keyForIndex(i), "foo"_sr);
 			}
 		} else if (type == 3) {
 			std::vector<Future<Optional<Value>>> gets;
@@ -93,19 +93,19 @@ struct RYWPerformanceWorkload : TestWorkload {
 			}
 			wait(waitForAll(gets));
 			for (i = 1; i < self->nodes; i += 2) {
-				tr->set(self->keyForIndex(i), LiteralStringRef("foo"));
+				tr->set(self->keyForIndex(i), "foo"_sr);
 			}
 		} else if (type == 4) {
 			wait(success(tr->getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
 		} else if (type == 5) {
 			wait(success(tr->getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
 			for (i = 0; i < self->nodes; i++) {
-				tr->set(self->keyForIndex(i), LiteralStringRef("foo"));
+				tr->set(self->keyForIndex(i), "foo"_sr);
 			}
 		} else if (type == 6) {
 			wait(success(tr->getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
 			for (i = 0; i < self->nodes; i += 2) {
-				tr->set(self->keyForIndex(i), LiteralStringRef("foo"));
+				tr->set(self->keyForIndex(i), "foo"_sr);
 			}
 		} else if (type == 7) {
 			wait(success(tr->getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
@@ -130,7 +130,7 @@ struct RYWPerformanceWorkload : TestWorkload {
 			}
 			wait(waitForAll(gets));
 			for (i = 0; i < self->nodes; i++) {
-				tr->set(self->keyForIndex(i), LiteralStringRef("foo"));
+				tr->set(self->keyForIndex(i), "foo"_sr);
 			}
 		} else if (type == 11) {
 			std::vector<Future<RangeResult>> gets;
@@ -139,7 +139,7 @@ struct RYWPerformanceWorkload : TestWorkload {
 			}
 			wait(waitForAll(gets));
 			for (i = 0; i < self->nodes; i += 2) {
-				tr->set(self->keyForIndex(i), LiteralStringRef("foo"));
+				tr->set(self->keyForIndex(i), "foo"_sr);
 			}
 		} else if (type == 12) {
 			std::vector<Future<RangeResult>> gets;
diff --git a/fdbserver/workloads/RandomClogging.actor.cpp b/fdbserver/workloads/RandomClogging.actor.cpp
index 7afa89e93b..0cf6796fea 100644
--- a/fdbserver/workloads/RandomClogging.actor.cpp
+++ b/fdbserver/workloads/RandomClogging.actor.cpp
@@ -18,55 +18,88 @@
  * limitations under the License.
  */
 
+#include "flow/DeterministicRandom.h"
+#include "fdbrpc/simulator.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
-#include "fdbrpc/simulator.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-struct RandomCloggingWorkload : TestWorkload {
+struct RandomCloggingWorkload : FailureInjectionWorkload {
 	bool enabled;
-	double testDuration;
-	double scale, clogginess;
-	int swizzleClog;
+	double testDuration = 10.0;
+	double scale = 1.0, clogginess = 1.0;
+	int swizzleClog = 0;
+	bool iterate = false;
+	double maxRunDuration = 60.0, backoff = 1.5, suspend = 10.0;
 
-	RandomCloggingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	RandomCloggingWorkload(WorkloadContext const& wcx, NoOptions) : FailureInjectionWorkload(wcx) {}
+
+	RandomCloggingWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		scale = getOption(options, LiteralStringRef("scale"), 1.0);
-		clogginess = getOption(options, LiteralStringRef("clogginess"), 1.0);
-		swizzleClog = getOption(options, LiteralStringRef("swizzle"), 0);
+		testDuration = getOption(options, "testDuration"_sr, testDuration);
+		scale = getOption(options, "scale"_sr, scale);
+		clogginess = getOption(options, "clogginess"_sr, clogginess);
+		swizzleClog = getOption(options, "swizzle"_sr, swizzleClog);
+	}
+
+	bool shouldInject(DeterministicRandom& random,
+	                  const WorkloadRequest& work,
+	                  const unsigned alreadyAdded) const override {
+		return work.useDatabase && 0.25 / (1 + alreadyAdded) > random.random01();
+	}
+
+	void initFailureInjectionMode(DeterministicRandom& random) override {
+		enabled = this->clientId == 0;
+		scale = std::max(random.random01(), 0.1);
+		clogginess = std::max(random.random01(), 0.1);
+		swizzleClog = random.random01() < 0.3;
+		iterate = random.random01() < 0.5;
 	}
 
 	std::string description() const override {
-		if (&g_simulator == g_network)
+		if (g_simulator == g_network)
 			return "RandomClogging";
 		else
 			return "NoRC";
 	}
 	Future<Void> setup(Database const& cx) override { return Void(); }
 	Future<Void> start(Database const& cx) override {
-		if (&g_simulator == g_network && enabled)
-			return timeout(
-			    reportErrors(swizzleClog ? swizzleClogClient(this) : clogClient(this), "RandomCloggingError"),
-			    testDuration,
-			    Void());
-		else
-			return Void();
+		if (g_network->isSimulated() && enabled) {
+			return _start(this);
+		}
+		return Void();
 	}
 	Future<bool> check(Database const& cx) override { return true; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
+	ACTOR static Future<Void> _start(RandomCloggingWorkload* self) {
+		state Future<Void> done = delay(self->maxRunDuration);
+		loop {
+			wait(done ||
+			     timeout(reportErrors(self->swizzleClog ? self->swizzleClogClient(self) : self->clogClient(self),
+			                          "RandomCloggingError"),
+			             self->testDuration,
+			             Void()));
+			if (!done.isReady() && self->iterate) {
+				wait(delay(self->suspend));
+				self->suspend *= self->backoff;
+			} else {
+				return Void();
+			}
+		}
+	}
+
 	ACTOR void doClog(ISimulator::ProcessInfo* machine, double t, double delay = 0.0) {
 		wait(::delay(delay));
-		g_simulator.clogInterface(machine->address.ip, t);
+		g_simulator->clogInterface(machine->address.ip, t);
 	}
 
 	void clogRandomPair(double t) {
-		auto m1 = deterministicRandom()->randomChoice(g_simulator.getAllProcesses());
-		auto m2 = deterministicRandom()->randomChoice(g_simulator.getAllProcesses());
+		auto m1 = deterministicRandom()->randomChoice(g_simulator->getAllProcesses());
+		auto m2 = deterministicRandom()->randomChoice(g_simulator->getAllProcesses());
 		if (m1->address.ip != m2->address.ip)
-			g_simulator.clogPair(m1->address.ip, m2->address.ip, t);
+			g_simulator->clogPair(m1->address.ip, m2->address.ip, t);
 	}
 
 	ACTOR Future<Void> clogClient(RandomCloggingWorkload* self) {
@@ -74,7 +107,7 @@ struct RandomCloggingWorkload : TestWorkload {
 		state double workloadEnd = now() + self->testDuration;
 		loop {
 			wait(poisson(&lastTime, self->scale / self->clogginess));
-			auto machine = deterministicRandom()->randomChoice(g_simulator.getAllProcesses());
+			auto machine = deterministicRandom()->randomChoice(g_simulator->getAllProcesses());
 			double t = self->scale * 10.0 * exp(-10.0 * deterministicRandom()->random01());
 			t = std::max(0.0, std::min(t, workloadEnd - now()));
 			self->doClog(machine, t);
@@ -97,9 +130,9 @@ struct RandomCloggingWorkload : TestWorkload {
 			//  then unclog in a different order over the course of t seconds
 			std::vector<ISimulator::ProcessInfo*> swizzled;
 			std::vector<double> starts, ends;
-			for (int m = 0; m < g_simulator.getAllProcesses().size(); m++)
+			for (int m = 0; m < g_simulator->getAllProcesses().size(); m++)
 				if (deterministicRandom()->random01() < 0.5) {
-					swizzled.push_back(g_simulator.getAllProcesses()[m]);
+					swizzled.push_back(g_simulator->getAllProcesses()[m]);
 					starts.push_back(deterministicRandom()->random01() * t / 2);
 					ends.push_back(deterministicRandom()->random01() * t / 2 + t / 2);
 				}
@@ -114,3 +147,4 @@ struct RandomCloggingWorkload : TestWorkload {
 };
 
 WorkloadFactory<RandomCloggingWorkload> RandomCloggingWorkloadFactory("RandomClogging");
+FailureInjectorFactory<RandomCloggingWorkload> RandomCloggingFailureInjectionFactory;
diff --git a/fdbserver/workloads/RandomMoveKeys.actor.cpp b/fdbserver/workloads/RandomMoveKeys.actor.cpp
index 6ec85f8381..05b4ffd5c6 100644
--- a/fdbserver/workloads/RandomMoveKeys.actor.cpp
+++ b/fdbserver/workloads/RandomMoveKeys.actor.cpp
@@ -27,31 +27,43 @@
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/ServerDBInfo.h"
 #include "fdbserver/QuietDatabase.h"
+#include "flow/DeterministicRandom.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-struct MoveKeysWorkload : TestWorkload {
+struct MoveKeysWorkload : FailureInjectionWorkload {
 	bool enabled;
-	double testDuration, meanDelay;
-	double maxKeyspace;
+	double testDuration = 10.0, meanDelay = 0.05;
+	double maxKeyspace = 0.1;
 	DatabaseConfiguration configuration;
 
-	MoveKeysWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	MoveKeysWorkload(WorkloadContext const& wcx, NoOptions) : FailureInjectionWorkload(wcx) {
 		enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
-		meanDelay = getOption(options, LiteralStringRef("meanDelay"), 0.05);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		maxKeyspace = getOption(options, LiteralStringRef("maxKeyspace"), 0.1);
+	}
+
+	MoveKeysWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
+		enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
+		meanDelay = getOption(options, "meanDelay"_sr, meanDelay);
+		testDuration = getOption(options, "testDuration"_sr, testDuration);
+		maxKeyspace = getOption(options, "maxKeyspace"_sr, maxKeyspace);
 	}
 
 	std::string description() const override { return "MoveKeysWorkload"; }
 	Future<Void> setup(Database const& cx) override { return Void(); }
 	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 
+	bool shouldInject(DeterministicRandom& random,
+	                  const WorkloadRequest& work,
+	                  const unsigned alreadyAdded) const override {
+		return alreadyAdded < 1 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
+	}
+
 	ACTOR Future<Void> _start(Database cx, MoveKeysWorkload* self) {
 		if (self->enabled) {
 			// Get the database configuration so as to use proper team size
 			state Transaction tr(cx);
 			loop {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 				try {
 					RangeResult res = wait(tr.getRange(configKeys, 1000));
 					ASSERT(res.size() < 1000);
@@ -143,18 +155,18 @@ struct MoveKeysWorkload : TestWorkload {
 			state Promise<Void> signal;
 			state DDEnabledState ddEnabledState;
 			wait(moveKeys(cx,
-			              deterministicRandom()->randomUniqueID(),
-			              keys,
-			              destinationTeamIDs,
-			              destinationTeamIDs,
-			              lock,
-			              signal,
-			              &fl1,
-			              &fl2,
-			              false,
-			              relocateShardInterval.pairID,
-			              &ddEnabledState,
-			              CancelConflictingDataMoves::True));
+			              MoveKeysParams{ deterministicRandom()->randomUniqueID(),
+			                              keys,
+			                              destinationTeamIDs,
+			                              destinationTeamIDs,
+			                              lock,
+			                              signal,
+			                              &fl1,
+			                              &fl2,
+			                              false,
+			                              relocateShardInterval.pairID,
+			                              &ddEnabledState,
+			                              CancelConflictingDataMoves::True }));
 			TraceEvent(relocateShardInterval.end()).detail("Result", "Success");
 			return Void();
 		} catch (Error& e) {
@@ -181,7 +193,7 @@ struct MoveKeysWorkload : TestWorkload {
 	ACTOR Future<Void> forceMasterFailure(Database cx, MoveKeysWorkload* self) {
 		ASSERT(g_network->isSimulated());
 		loop {
-			if (g_simulator.killZone(self->dbInfo->get().master.locality.zoneId(), ISimulator::Reboot, true))
+			if (g_simulator->killZone(self->dbInfo->get().master.locality.zoneId(), ISimulator::Reboot, true))
 				return Void();
 			wait(delay(1.0));
 		}
@@ -232,3 +244,4 @@ struct MoveKeysWorkload : TestWorkload {
 };
 
 WorkloadFactory<MoveKeysWorkload> MoveKeysWorkloadFactory("RandomMoveKeys");
+FailureInjectorFactory<MoveKeysWorkload> MoveKeysFailureInjectionFactory;
diff --git a/fdbserver/workloads/RandomSelector.actor.cpp b/fdbserver/workloads/RandomSelector.actor.cpp
index 1573e6f848..d5d8e23f16 100644
--- a/fdbserver/workloads/RandomSelector.actor.cpp
+++ b/fdbserver/workloads/RandomSelector.actor.cpp
@@ -36,13 +36,13 @@ struct RandomSelectorWorkload : TestWorkload {
 	RandomSelectorWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries") {
 
-		minOperationsPerTransaction = getOption(options, LiteralStringRef("minOperationsPerTransaction"), 10);
-		maxOperationsPerTransaction = getOption(options, LiteralStringRef("minOperationsPerTransaction"), 50);
-		maxKeySpace = getOption(options, LiteralStringRef("maxKeySpace"), 20);
-		maxOffset = getOption(options, LiteralStringRef("maxOffset"), 5);
-		minInitialAmount = getOption(options, LiteralStringRef("minInitialAmount"), 5);
-		maxInitialAmount = getOption(options, LiteralStringRef("maxInitialAmount"), 10);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
+		minOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 10);
+		maxOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 50);
+		maxKeySpace = getOption(options, "maxKeySpace"_sr, 20);
+		maxOffset = getOption(options, "maxOffset"_sr, 5);
+		minInitialAmount = getOption(options, "minInitialAmount"_sr, 5);
+		maxInitialAmount = getOption(options, "maxInitialAmount"_sr, 10);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		fail = false;
 	}
 
diff --git a/fdbserver/workloads/ReadAfterWrite.actor.cpp b/fdbserver/workloads/ReadAfterWrite.actor.cpp
index ecbd97410a..96a95db040 100644
--- a/fdbserver/workloads/ReadAfterWrite.actor.cpp
+++ b/fdbserver/workloads/ReadAfterWrite.actor.cpp
@@ -53,7 +53,7 @@ struct ReadAfterWriteWorkload : KVWorkload {
 	ContinuousSample<double> propagationLatency;
 
 	ReadAfterWriteWorkload(WorkloadContext const& wcx) : KVWorkload(wcx), propagationLatency(SAMPLE_SIZE) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
 	}
 
 	std::string description() const override { return "ReadAfterWriteWorkload"; }
diff --git a/fdbserver/workloads/ReadHotDetection.actor.cpp b/fdbserver/workloads/ReadHotDetection.actor.cpp
index 251c1f90f4..81be6220d6 100644
--- a/fdbserver/workloads/ReadHotDetection.actor.cpp
+++ b/fdbserver/workloads/ReadHotDetection.actor.cpp
@@ -37,10 +37,10 @@ struct ReadHotDetectionWorkload : TestWorkload {
 	bool passed;
 
 	ReadHotDetectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 120.0);
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 1000.0) / clientCount;
-		actorCount = getOption(options, LiteralStringRef("actorsPerClient"), transactionsPerSecond / 5);
-		keyCount = getOption(options, LiteralStringRef("keyCount"), 100);
+		testDuration = getOption(options, "testDuration"_sr, 120.0);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 1000.0) / clientCount;
+		actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5);
+		keyCount = getOption(options, "keyCount"_sr, 100);
 		readKey = StringRef(format("testkey%08x", deterministicRandom()->randomInt(0, keyCount)));
 	}
 
@@ -89,7 +89,7 @@ struct ReadHotDetectionWorkload : TestWorkload {
 				wait(tr.onError(e));
 			}
 		}
-		self->wholeRange = KeyRangeRef(LiteralStringRef(""), LiteralStringRef("\xff"));
+		self->wholeRange = KeyRangeRef(""_sr, "\xff"_sr);
 		// TraceEvent("RHDLog").detail("Phase", "DoneSetup");
 		return Void();
 	}
diff --git a/fdbserver/workloads/ReadWrite.actor.cpp b/fdbserver/workloads/ReadWrite.actor.cpp
index ef77eaae76..d7a44cb6e1 100644
--- a/fdbserver/workloads/ReadWrite.actor.cpp
+++ b/fdbserver/workloads/ReadWrite.actor.cpp
@@ -386,23 +386,21 @@ struct ReadWriteWorkload : ReadWriteCommon {
 
 	ReadWriteWorkload(WorkloadContext const& wcx)
 	  : ReadWriteCommon(wcx), dependentReads(false), adjacentReads(false), adjacentWrites(false) {
-		extraReadConflictRangesPerTransaction =
-		    getOption(options, LiteralStringRef("extraReadConflictRangesPerTransaction"), 0);
-		extraWriteConflictRangesPerTransaction =
-		    getOption(options, LiteralStringRef("extraWriteConflictRangesPerTransaction"), 0);
-		dependentReads = getOption(options, LiteralStringRef("dependentReads"), false);
-		inconsistentReads = getOption(options, LiteralStringRef("inconsistentReads"), false);
-		adjacentReads = getOption(options, LiteralStringRef("adjacentReads"), false);
-		adjacentWrites = getOption(options, LiteralStringRef("adjacentWrites"), false);
-		rampUpLoad = getOption(options, LiteralStringRef("rampUpLoad"), false);
-		rampSweepCount = getOption(options, LiteralStringRef("rampSweepCount"), 1);
-		rangeReads = getOption(options, LiteralStringRef("rangeReads"), false);
-		rampTransactionType = getOption(options, LiteralStringRef("rampTransactionType"), false);
-		rampUpConcurrency = getOption(options, LiteralStringRef("rampUpConcurrency"), false);
-		batchPriority = getOption(options, LiteralStringRef("batchPriority"), false);
-		descriptionString = getOption(options, LiteralStringRef("description"), LiteralStringRef("ReadWrite"));
-		if (hasOption(options, LiteralStringRef("transactionTag"))) {
-			transactionTag = getOption(options, LiteralStringRef("transactionTag"), ""_sr);
+		extraReadConflictRangesPerTransaction = getOption(options, "extraReadConflictRangesPerTransaction"_sr, 0);
+		extraWriteConflictRangesPerTransaction = getOption(options, "extraWriteConflictRangesPerTransaction"_sr, 0);
+		dependentReads = getOption(options, "dependentReads"_sr, false);
+		inconsistentReads = getOption(options, "inconsistentReads"_sr, false);
+		adjacentReads = getOption(options, "adjacentReads"_sr, false);
+		adjacentWrites = getOption(options, "adjacentWrites"_sr, false);
+		rampUpLoad = getOption(options, "rampUpLoad"_sr, false);
+		rampSweepCount = getOption(options, "rampSweepCount"_sr, 1);
+		rangeReads = getOption(options, "rangeReads"_sr, false);
+		rampTransactionType = getOption(options, "rampTransactionType"_sr, false);
+		rampUpConcurrency = getOption(options, "rampUpConcurrency"_sr, false);
+		batchPriority = getOption(options, "batchPriority"_sr, false);
+		descriptionString = getOption(options, "description"_sr, "ReadWrite"_sr);
+		if (hasOption(options, "transactionTag"_sr)) {
+			transactionTag = getOption(options, "transactionTag"_sr, ""_sr);
 		}
 
 		if (rampUpConcurrency)
@@ -411,8 +409,8 @@ struct ReadWriteWorkload : ReadWriteCommon {
 		{
 			// with P(hotTrafficFraction) an access is directed to one of a fraction
 			//   of hot keys, else it is directed to a disjoint set of cold keys
-			hotKeyFraction = getOption(options, LiteralStringRef("hotKeyFraction"), 0.0);
-			double hotTrafficFraction = getOption(options, LiteralStringRef("hotTrafficFraction"), 0.0);
+			hotKeyFraction = getOption(options, "hotKeyFraction"_sr, 0.0);
+			double hotTrafficFraction = getOption(options, "hotTrafficFraction"_sr, 0.0);
 			ASSERT(hotKeyFraction >= 0 && hotTrafficFraction <= 1);
 			ASSERT(hotKeyFraction <= hotTrafficFraction); // hot keys should be actually hot!
 			// p(Cold key) = (1-FHP) * (1-hkf)
@@ -690,6 +688,10 @@ struct ReadWriteWorkload : ReadWriteCommon {
 
 						break;
 					} catch (Error& e) {
+						if (e.code() == error_code_tag_throttled) {
+							++self->transactionsTagThrottled;
+						}
+
 						self->transactionFailureMetric->errorCode = e.code();
 						self->transactionFailureMetric->log();
 
diff --git a/fdbserver/workloads/RemoveServersSafely.actor.cpp b/fdbserver/workloads/RemoveServersSafely.actor.cpp
index 6be4a58cee..6fc3fab69d 100644
--- a/fdbserver/workloads/RemoveServersSafely.actor.cpp
+++ b/fdbserver/workloads/RemoveServersSafely.actor.cpp
@@ -44,17 +44,17 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 	RemoveServersSafelyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled =
 		    !clientId && g_network->isSimulated(); // only do this on the "first" client, and only when in simulation
-		minMachinesToKill = getOption(options, LiteralStringRef("minMachinesToKill"), 1);
-		maxMachinesToKill = getOption(options, LiteralStringRef("maxMachinesToKill"), 10);
+		minMachinesToKill = getOption(options, "minMachinesToKill"_sr, 1);
+		maxMachinesToKill = getOption(options, "maxMachinesToKill"_sr, 10);
 		maxMachinesToKill = std::max(minMachinesToKill, maxMachinesToKill);
-		maxSafetyCheckRetries = getOption(options, LiteralStringRef("maxSafetyCheckRetries"), 50);
-		minDelay = getOption(options, LiteralStringRef("minDelay"), 0.0);
-		maxDelay = getOption(options, LiteralStringRef("maxDelay"), 60.0);
-		kill1Timeout = getOption(options, LiteralStringRef("kill1Timeout"), 60.0);
-		kill2Timeout = getOption(options, LiteralStringRef("kill2Timeout"), 6000.0);
+		maxSafetyCheckRetries = getOption(options, "maxSafetyCheckRetries"_sr, 50);
+		minDelay = getOption(options, "minDelay"_sr, 0.0);
+		maxDelay = getOption(options, "maxDelay"_sr, 60.0);
+		kill1Timeout = getOption(options, "kill1Timeout"_sr, 60.0);
+		kill2Timeout = getOption(options, "kill2Timeout"_sr, 6000.0);
 		killProcesses = deterministicRandom()->random01() < 0.5;
 		if (g_network->isSimulated()) {
-			g_simulator.allowLogSetKills = false;
+			g_simulator->allowLogSetKills = false;
 		}
 	}
 
@@ -80,7 +80,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("Zoneid", it->locality.zoneId().get().toString())
 			    .detail("MachineId", it->locality.machineId().get().toString());
 
-			if (g_simulator.protectedAddresses.count(it->address) == 0)
+			if (g_simulator->protectedAddresses.count(it->address) == 0)
 				processAddrs.push_back(pAddr);
 			machineProcesses[machineIp].insert(pAddr);
 
@@ -127,13 +127,13 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		for (AddressExclusion ex : toKill1) {
 			AddressExclusion machineIp(ex.ip);
 			ASSERT(machine_ids.count(machineIp));
-			g_simulator.disableSwapToMachine(machine_ids[machineIp]);
+			g_simulator->disableSwapToMachine(machine_ids[machineIp]);
 		}
 
 		for (AddressExclusion ex : toKill2) {
 			AddressExclusion machineIp(ex.ip);
 			ASSERT(machine_ids.count(machineIp));
-			g_simulator.disableSwapToMachine(machine_ids[machineIp]);
+			g_simulator->disableSwapToMachine(machine_ids[machineIp]);
 		}
 
 		return Void();
@@ -178,7 +178,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			}
 		}
 		// Get the list of processes matching network address
-		for (auto processInfo : g_simulator.getAllProcesses()) {
+		for (auto processInfo : g_simulator->getAllProcesses()) {
 			auto processNet = AddressExclusion(processInfo->address.ip, processInfo->address.port);
 			if (processAddrs.find(processNet) != processAddrs.end()) {
 				processes.push_back(processInfo);
@@ -189,7 +189,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				    .detail("Failed", processInfo->failed)
 				    .detail("Excluded", processInfo->excluded)
 				    .detail("Rebooting", processInfo->rebooting)
-				    .detail("Protected", g_simulator.protectedAddresses.count(processInfo->address));
+				    .detail("Protected", g_simulator->protectedAddresses.count(processInfo->address));
 			} else {
 				TraceEvent("RemoveAndKill", functionId)
 				    .detail("Step", "ProcessNotToKill")
@@ -198,7 +198,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				    .detail("Failed", processInfo->failed)
 				    .detail("Excluded", processInfo->excluded)
 				    .detail("Rebooting", processInfo->rebooting)
-				    .detail("Protected", g_simulator.protectedAddresses.count(processInfo->address));
+				    .detail("Protected", g_simulator->protectedAddresses.count(processInfo->address));
 			}
 		}
 		TraceEvent("RemoveAndKill", functionId)
@@ -223,9 +223,9 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("AddrTotal", procAddrs.size())
 		    .detail("ProcTotal", procArray.size())
 		    .detail("Addresses", describe(procAddrs))
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 		for (auto& procAddr : procAddrs) {
-			g_simulator.excludeAddress(NetworkAddress(procAddr.ip, procAddr.port, true, false));
+			g_simulator->excludeAddress(NetworkAddress(procAddr.ip, procAddr.port, true, false));
 		}
 		for (auto& procRecord : procArray) {
 			procRecord->excluded = true;
@@ -235,7 +235,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("Process", describe(*procRecord))
 			    .detail("Failed", procRecord->failed)
 			    .detail("Rebooting", procRecord->rebooting)
-			    .detail("ClusterAvailable", g_simulator.isAvailable());
+			    .detail("ClusterAvailable", g_simulator->isAvailable());
 		}
 		return procArray;
 	}
@@ -250,9 +250,9 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("AddrTotal", procAddrs.size())
 		    .detail("ProcTotal", procArray.size())
 		    .detail("Addresses", describe(procAddrs))
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 		for (auto& procAddr : procAddrs) {
-			g_simulator.includeAddress(NetworkAddress(procAddr.ip, procAddr.port, true, false));
+			g_simulator->includeAddress(NetworkAddress(procAddr.ip, procAddr.port, true, false));
 		}
 		for (auto& procRecord : procArray) {
 			// Only change the exclusion member, if not failed since it will require a reboot to revive it
@@ -264,7 +264,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("Process", describe(*procRecord))
 			    .detail("Failed", procRecord->failed)
 			    .detail("Rebooting", procRecord->rebooting)
-			    .detail("ClusterAvailable", g_simulator.isAvailable());
+			    .detail("ClusterAvailable", g_simulator->isAvailable());
 		}
 		return procArray;
 	}
@@ -307,7 +307,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 
 			// Check if we can kill the added process
 			bCanKillProcess =
-			    g_simulator.canKillProcesses(processesLeft, processesDead, ISimulator::KillInstantly, nullptr);
+			    g_simulator->canKillProcesses(processesLeft, processesDead, ISimulator::KillInstantly, nullptr);
 
 			// Remove the added processes
 			processesLeft.resize(processesLeft.size() - killProcArray.size());
@@ -350,7 +350,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("Step", "exclude list first")
 		    .detail("ToKill", describe(toKill1))
 		    .detail("KillTotal", toKill1.size())
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 
 		// toKill1 may kill too many servers to make cluster unavailable.
 		// Get the processes in toKill1 that are safe to kill
@@ -361,7 +361,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("Step", "exclude list first")
 		    .detail("ToKillModified", describe(toKill1))
 		    .detail("KillTotalModified", toKill1.size())
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 
 		self->excludeAddresses(toKill1);
 
@@ -374,7 +374,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("KillTotal", toKill1.size())
 		    .detail("Processes", killProcArray.size())
 		    .detail("ToKill1", describe(toKill1))
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 
 		// Include the servers, if unable to exclude
 		// Reinclude when buggify is on to increase the surface area of the next set of excludes
@@ -385,7 +385,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("Step", "include all first")
 			    .detail("KillTotal", toKill1.size())
 			    .detail("ToKill", describe(toKill1))
-			    .detail("ClusterAvailable", g_simulator.isAvailable());
+			    .detail("ClusterAvailable", g_simulator->isAvailable());
 			wait(includeServers(cx, std::vector<AddressExclusion>(1)));
 			wait(includeLocalities(cx, std::vector<std::string>(), failed, true));
 			wait(includeLocalities(cx, std::vector<std::string>(), !failed, true));
@@ -403,7 +403,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("Step", "exclude list second")
 		    .detail("KillTotal", toKill2.size())
 		    .detail("ToKill", describe(toKill2))
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 		self->excludeAddresses(toKill2);
 
 		// The second set of machines is selected so that we can always make progress without it, even after the
@@ -413,7 +413,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("ToKill2", describe(toKill2))
 		    .detail("KillTotal", toKill2.size())
 		    .detail("Processes", killProcArray.size())
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 		wait(reportErrors(timeoutError(removeAndKill(self, cx, toKill2, bClearedFirst ? &toKill1 : nullptr, true),
 		                               self->kill2Timeout),
 		                  "RemoveServersSafelyError",
@@ -423,14 +423,14 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("Step", "excluded second list")
 		    .detail("KillTotal", toKill2.size())
 		    .detail("ToKill", describe(toKill2))
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 
 		// Get the updated list of processes which may have changed due to reboots, deletes, etc
 		TraceEvent("RemoveAndKill")
 		    .detail("Step", "include all second")
 		    .detail("KillTotal", toKill2.size())
 		    .detail("ToKill", describe(toKill2))
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 		wait(includeServers(cx, std::vector<AddressExclusion>(1)));
 		wait(includeLocalities(cx, std::vector<std::string>(), failed, true));
 		wait(includeLocalities(cx, std::vector<std::string>(), !failed, true));
@@ -454,31 +454,31 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("Step", removeViaClear ? "ClearProcesses" : "IgnoreProcesses")
 			    .detail("Addresses", describe(killAddrs))
 			    .detail("Processes", killProcArray.size())
-			    .detail("ClusterAvailable", g_simulator.isAvailable())
+			    .detail("ClusterAvailable", g_simulator->isAvailable())
 			    .detail("RemoveViaClear", removeViaClear);
 			for (auto& killProcess : killProcArray) {
-				if (g_simulator.protectedAddresses.count(killProcess->address))
+				if (g_simulator->protectedAddresses.count(killProcess->address))
 					TraceEvent("RemoveAndKill", functionId)
 					    .detail("Step", "NoKill Process")
 					    .detail("Process", describe(*killProcess))
 					    .detail("Failed", killProcess->failed)
 					    .detail("Rebooting", killProcess->rebooting)
-					    .detail("ClusterAvailable", g_simulator.isAvailable())
-					    .detail("Protected", g_simulator.protectedAddresses.count(killProcess->address));
+					    .detail("ClusterAvailable", g_simulator->isAvailable())
+					    .detail("Protected", g_simulator->protectedAddresses.count(killProcess->address));
 				else if (removeViaClear) {
-					g_simulator.rebootProcess(killProcess, ISimulator::RebootProcessAndDelete);
+					g_simulator->rebootProcess(killProcess, ISimulator::RebootProcessAndDelete);
 					TraceEvent("RemoveAndKill", functionId)
 					    .detail("Step", "Clear Process")
 					    .detail("Process", describe(*killProcess))
 					    .detail("Failed", killProcess->failed)
 					    .detail("Rebooting", killProcess->rebooting)
-					    .detail("ClusterAvailable", g_simulator.isAvailable())
-					    .detail("Protected", g_simulator.protectedAddresses.count(killProcess->address));
+					    .detail("ClusterAvailable", g_simulator->isAvailable())
+					    .detail("Protected", g_simulator->protectedAddresses.count(killProcess->address));
 				}
 				/*
 				                else {
-				                    g_simulator.killProcess( killProcess, ISimulator::KillInstantly );
-				                    TraceEvent("RemoveAndKill", functionId).detail("Step", "Kill Process").detail("Process", describe(*killProcess)).detail("Failed", killProcess->failed).detail("Rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address));
+				                    g_simulator->killProcess( killProcess, ISimulator::KillInstantly );
+				                    TraceEvent("RemoveAndKill", functionId).detail("Step", "Kill Process").detail("Process", describe(*killProcess)).detail("Failed", killProcess->failed).detail("Rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator->isAvailable()).detail("Protected", g_simulator->protectedAddresses.count(killProcess->address));
 				                }
 				*/
 			}
@@ -493,15 +493,15 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("Addresses", describe(killAddrs))
 			    .detail("Processes", killProcArray.size())
 			    .detail("Zones", zoneIds.size())
-			    .detail("ClusterAvailable", g_simulator.isAvailable());
+			    .detail("ClusterAvailable", g_simulator->isAvailable());
 			for (auto& zoneId : zoneIds) {
-				killedMachine = g_simulator.killZone(
+				killedMachine = g_simulator->killZone(
 				    zoneId, removeViaClear ? ISimulator::RebootAndDelete : ISimulator::KillInstantly);
 				TraceEvent(killedMachine ? SevInfo : SevWarn, "RemoveAndKill")
 				    .detail("Step", removeViaClear ? "Clear Machine" : "Kill Machine")
 				    .detail("ZoneId", zoneId)
 				    .detail(removeViaClear ? "Cleared" : "Killed", killedMachine)
-				    .detail("ClusterAvailable", g_simulator.isAvailable());
+				    .detail("ClusterAvailable", g_simulator->isAvailable());
 			}
 		}
 
@@ -520,7 +520,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		// First clear the exclusion list and exclude the given list
 		TraceEvent("RemoveAndKill", functionId)
 		    .detail("Step", "Including all")
-		    .detail("ClusterAvailable", g_simulator.isAvailable())
+		    .detail("ClusterAvailable", g_simulator->isAvailable())
 		    .detail("MarkExcludeAsFailed", markExcludeAsFailed);
 		state bool failed = true;
 		wait(includeServers(cx, std::vector<AddressExclusion>(1)));
@@ -528,7 +528,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		wait(includeLocalities(cx, std::vector<std::string>(), !failed, true));
 		TraceEvent("RemoveAndKill", functionId)
 		    .detail("Step", "Included all")
-		    .detail("ClusterAvailable", g_simulator.isAvailable())
+		    .detail("ClusterAvailable", g_simulator->isAvailable())
 		    .detail("MarkExcludeAsFailed", markExcludeAsFailed);
 		// Reinclude the addresses that were excluded, if present
 		if (pIncAddrs) {
@@ -629,7 +629,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("ToKill", describe(toKill))
 		    .detail("Addresses", describe(toKillArray))
 		    .detail("FailedAddresses", describe(toKillMarkFailedArray))
-		    .detail("ClusterAvailable", g_simulator.isAvailable())
+		    .detail("ClusterAvailable", g_simulator->isAvailable())
 		    .detail("MarkExcludeAsFailed", markExcludeAsFailed);
 
 		state bool excludeLocalitiesInsteadOfServers = deterministicRandom()->coinflip();
@@ -682,14 +682,14 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				TraceEvent("RemoveAndKill", functionId)
 				    .detail("Step", "Wait For Server Exclusion")
 				    .detail("Addresses", describe(toKill))
-				    .detail("ClusterAvailable", g_simulator.isAvailable());
+				    .detail("ClusterAvailable", g_simulator->isAvailable());
 				wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)));
 			}
 
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "coordinators auto")
-			    .detail("DesiredCoordinators", g_simulator.desiredCoordinators)
-			    .detail("ClusterAvailable", g_simulator.isAvailable());
+			    .detail("DesiredCoordinators", g_simulator->desiredCoordinators)
+			    .detail("ClusterAvailable", g_simulator->isAvailable());
 
 			// Setup the coordinators BEFORE the exclusion
 			// Otherwise, we may end up with NotEnoughMachinesForCoordinators
@@ -697,7 +697,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			state int nQuorum;
 			while (true) {
 				cycle++;
-				nQuorum = ((g_simulator.desiredCoordinators + 1) / 2) * 2 - 1;
+				nQuorum = ((g_simulator->desiredCoordinators + 1) / 2) * 2 - 1;
 				CoordinatorsResult result = wait(changeQuorum(cx, autoQuorumChange(nQuorum)));
 				TraceEvent(result == CoordinatorsResult::SUCCESS || result == CoordinatorsResult::SAME_NETWORK_ADDRESSES
 				               ? SevInfo
@@ -707,7 +707,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				    .detail("Result", (int)result)
 				    .detail("Attempt", cycle)
 				    .detail("Quorum", nQuorum)
-				    .detail("DesiredCoordinators", g_simulator.desiredCoordinators);
+				    .detail("DesiredCoordinators", g_simulator->desiredCoordinators);
 				if (result == CoordinatorsResult::SUCCESS || result == CoordinatorsResult::SAME_NETWORK_ADDRESSES)
 					break;
 			}
@@ -716,19 +716,19 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		} else {
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "nothing to clear")
-			    .detail("ClusterAvailable", g_simulator.isAvailable());
+			    .detail("ClusterAvailable", g_simulator->isAvailable());
 		}
 
 		TraceEvent("RemoveAndKill", functionId)
 		    .detail("Step", "done")
-		    .detail("ClusterAvailable", g_simulator.isAvailable());
+		    .detail("ClusterAvailable", g_simulator->isAvailable());
 
 		return Void();
 	}
 
 	static std::vector<ISimulator::ProcessInfo*> getServers() {
 		std::vector<ISimulator::ProcessInfo*> machines;
-		std::vector<ISimulator::ProcessInfo*> all = g_simulator.getAllProcesses();
+		std::vector<ISimulator::ProcessInfo*> all = g_simulator->getAllProcesses();
 		for (int i = 0; i < all.size(); i++) {
 			if (all[i]->name == std::string("Server") && all[i]->isAvailableClass()) {
 				machines.push_back(all[i]);
@@ -793,7 +793,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			addressToIndexMap[workers[i].address] = i;
 		}
 
-		std::vector<ISimulator::ProcessInfo*> processes = g_simulator.getAllProcesses();
+		std::vector<ISimulator::ProcessInfo*> processes = g_simulator->getAllProcesses();
 		for (auto process : processes) {
 			if (addressToIndexMap.find(process->address) != addressToIndexMap.end()) {
 				if (workers[addressToIndexMap[process->address]].locality.processId().present()) {
diff --git a/fdbserver/workloads/ReportConflictingKeys.actor.cpp b/fdbserver/workloads/ReportConflictingKeys.actor.cpp
index 33ac14584a..14a5e69a34 100644
--- a/fdbserver/workloads/ReportConflictingKeys.actor.cpp
+++ b/fdbserver/workloads/ReportConflictingKeys.actor.cpp
@@ -40,23 +40,21 @@ struct ReportConflictingKeysWorkload : TestWorkload {
 	ReportConflictingKeysWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), invalidReports("InvalidReports"), commits("commits"), conflicts("Conflicts"),
 	    xacts("Transactions") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		// transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 5000.0) / clientCount;
-		actorCount = getOption(options, LiteralStringRef("actorsPerClient"), 1);
-		keyPrefix = unprintable(
-		    getOption(options, LiteralStringRef("keyPrefix"), LiteralStringRef("ReportConflictingKeysWorkload"))
-		        .toString());
-		keyBytes = getOption(options, LiteralStringRef("keyBytes"), 64);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		// transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0) / clientCount;
+		actorCount = getOption(options, "actorsPerClient"_sr, 1);
+		keyPrefix = unprintable(getOption(options, "keyPrefix"_sr, "ReportConflictingKeysWorkload"_sr).toString());
+		keyBytes = getOption(options, "keyBytes"_sr, 64);
 
-		readConflictRangeCount = getOption(options, LiteralStringRef("readConflictRangeCountPerTx"), 1);
-		writeConflictRangeCount = getOption(options, LiteralStringRef("writeConflictRangeCountPerTx"), 1);
+		readConflictRangeCount = getOption(options, "readConflictRangeCountPerTx"_sr, 1);
+		writeConflictRangeCount = getOption(options, "writeConflictRangeCountPerTx"_sr, 1);
 		ASSERT(readConflictRangeCount >= 1 && writeConflictRangeCount >= 1);
 		// modeled by geometric distribution: (1 - prob) / prob = mean - 1, where we add at least one conflictRange to
 		// each tx
 		addReadConflictRangeProb = (readConflictRangeCount - 1.0) / readConflictRangeCount;
 		addWriteConflictRangeProb = (writeConflictRangeCount - 1.0) / writeConflictRangeCount;
 		ASSERT(keyPrefix.size() + 8 <= keyBytes); // make sure the string format is valid
-		nodeCount = getOption(options, LiteralStringRef("nodeCount"), 100);
+		nodeCount = getOption(options, "nodeCount"_sr, 100);
 	}
 
 	std::string description() const override { return "ReportConflictingKeysWorkload"; }
@@ -187,14 +185,20 @@ struct ReportConflictingKeysWorkload : TestWorkload {
 				// check API correctness
 				if (foundConflict) {
 					// \xff\xff/transaction/conflicting_keys is always initialized to false, skip it here
-					state KeyRange ckr =
-					    KeyRangeRef(keyAfter(LiteralStringRef("").withPrefix(conflictingKeysRange.begin)),
-					                LiteralStringRef("\xff\xff").withPrefix(conflictingKeysRange.begin));
+					state KeyRange ckr = KeyRangeRef(keyAfter(""_sr.withPrefix(conflictingKeysRange.begin)),
+					                                 "\xff\xff"_sr.withPrefix(conflictingKeysRange.begin));
 					// The getRange here using the special key prefix "\xff\xff/transaction/conflicting_keys/" happens
 					// locally Thus, the error handling is not needed here
-					Future<RangeResult> conflictingKeyRangesFuture = tr2->getRange(ckr, CLIENT_KNOBS->TOO_MANY);
+					state Future<RangeResult> conflictingKeyRangesFuture = tr2->getRange(ckr, CLIENT_KNOBS->TOO_MANY);
 					ASSERT(conflictingKeyRangesFuture.isReady());
 
+					wait(validateSpecialSubrangeRead(tr2.getPtr(),
+					                                 firstGreaterOrEqual(ckr.begin),
+					                                 firstGreaterOrEqual(ckr.end),
+					                                 GetRangeLimits(),
+					                                 Reverse::False,
+					                                 conflictingKeyRangesFuture.get()));
+
 					tr2 = makeReference<ReadYourWritesTransaction>(cx);
 
 					const RangeResult conflictingKeyRanges = conflictingKeyRangesFuture.get();
diff --git a/fdbserver/workloads/RestoreBackup.actor.cpp b/fdbserver/workloads/RestoreBackup.actor.cpp
index c08fc7de70..c2f30e490c 100644
--- a/fdbserver/workloads/RestoreBackup.actor.cpp
+++ b/fdbserver/workloads/RestoreBackup.actor.cpp
@@ -20,6 +20,7 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/SystemData.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
@@ -37,10 +38,10 @@ struct RestoreBackupWorkload final : TestWorkload {
 	StopWhenDone stopWhenDone{ false };
 
 	RestoreBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		backupDir = getOption(options, LiteralStringRef("backupDir"), LiteralStringRef("file://simfdb/backups/"));
-		tag = getOption(options, LiteralStringRef("tag"), LiteralStringRef("default"));
-		delayFor = getOption(options, LiteralStringRef("delayFor"), 10.0);
-		stopWhenDone.set(getOption(options, LiteralStringRef("stopWhenDone"), false));
+		backupDir = getOption(options, "backupDir"_sr, "file://simfdb/backups/"_sr);
+		tag = getOption(options, "tag"_sr, "default"_sr);
+		delayFor = getOption(options, "delayFor"_sr, 10.0);
+		stopWhenDone.set(getOption(options, "stopWhenDone"_sr, false));
 	}
 
 	static constexpr const char* DESCRIPTION = "RestoreBackup";
@@ -93,11 +94,13 @@ struct RestoreBackupWorkload final : TestWorkload {
 	}
 
 	ACTOR static Future<Void> clearDatabase(Database cx) {
-		// TODO: Batch to avoid large clear ranges?
 		state Transaction tr(cx);
 		loop {
 			try {
 				tr.clear(normalKeys);
+				for (auto& r : getSystemBackupRanges()) {
+					tr.clear(r);
+				}
 				wait(tr.commit());
 				return Void();
 			} catch (Error& e) {
diff --git a/fdbserver/workloads/RestoreFromBlob.actor.cpp b/fdbserver/workloads/RestoreFromBlob.actor.cpp
index 9d072bb731..ad5a66262a 100644
--- a/fdbserver/workloads/RestoreFromBlob.actor.cpp
+++ b/fdbserver/workloads/RestoreFromBlob.actor.cpp
@@ -35,16 +35,13 @@ struct RestoreFromBlobWorkload : TestWorkload {
 	static constexpr const char* DESCRIPTION = "RestoreFromBlob";
 
 	RestoreFromBlobWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 10.0);
-		backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
-		auto backupURLString =
-		    getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000")).toString();
-		auto accessKeyEnvVar =
-		    getOption(options, LiteralStringRef("accessKeyVar"), LiteralStringRef("BLOB_ACCESS_KEY")).toString();
-		auto secretKeyEnvVar =
-		    getOption(options, LiteralStringRef("secretKeyVar"), LiteralStringRef("BLOB_SECRET_KEY")).toString();
-		bool provideKeys = getOption(options, LiteralStringRef("provideKeys"), false);
-		waitForComplete.set(getOption(options, LiteralStringRef("waitForComplete"), true));
+		restoreAfter = getOption(options, "restoreAfter"_sr, 10.0);
+		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
+		auto backupURLString = getOption(options, "backupURL"_sr, "http://0.0.0.0:10000"_sr).toString();
+		auto accessKeyEnvVar = getOption(options, "accessKeyVar"_sr, "BLOB_ACCESS_KEY"_sr).toString();
+		auto secretKeyEnvVar = getOption(options, "secretKeyVar"_sr, "BLOB_SECRET_KEY"_sr).toString();
+		bool provideKeys = getOption(options, "provideKeys"_sr, false);
+		waitForComplete.set(getOption(options, "waitForComplete"_sr, true));
 		if (provideKeys) {
 			updateBackupURL(backupURLString, accessKeyEnvVar, "<access_key>", secretKeyEnvVar, "<secret_key>");
 		}
@@ -58,7 +55,8 @@ struct RestoreFromBlobWorkload : TestWorkload {
 	ACTOR static Future<Void> _start(Database cx, RestoreFromBlobWorkload* self) {
 		state FileBackupAgent backupAgent;
 		state Standalone<VectorRef<KeyRangeRef>> restoreRanges;
-		restoreRanges.push_back_deep(restoreRanges.arena(), normalKeys);
+
+		addDefaultBackupRanges(restoreRanges);
 
 		wait(delay(self->restoreAfter));
 		Version v = wait(
diff --git a/fdbserver/workloads/Rollback.actor.cpp b/fdbserver/workloads/Rollback.actor.cpp
index 3c3ea8efa2..a2ff27cafe 100644
--- a/fdbserver/workloads/Rollback.actor.cpp
+++ b/fdbserver/workloads/Rollback.actor.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "flow/DeterministicRandom.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
@@ -31,23 +32,31 @@
 // The workload first clogs network link between the chosen proxy and all tLogs but the unclogTlog;
 // While the network is still clogged, the workload kills the proxy and clogs the unclogged tlog's interface.
 // Note: The clogged network link's latency will become "clogDuration".
-struct RollbackWorkload : TestWorkload {
-	bool enableFailures, multiple, enabled;
-	double meanDelay, clogDuration, testDuration;
+struct RollbackWorkload : FailureInjectionWorkload {
+	bool enableFailures = false, multiple = true, enabled;
+	double meanDelay = 20.0, clogDuration = clogDuration = 3.0, testDuration = 10.0;
 
-	RollbackWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	RollbackWorkload(WorkloadContext const& wcx, NoOptions) : FailureInjectionWorkload(wcx) {}
+
+	RollbackWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
-		meanDelay = getOption(options, LiteralStringRef("meanDelay"), 20.0); // Only matters if multiple==true
-		clogDuration = getOption(options, LiteralStringRef("clogDuration"), 3.0);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		enableFailures = getOption(options, LiteralStringRef("enableFailures"), false);
-		multiple = getOption(options, LiteralStringRef("multiple"), true);
+		meanDelay = getOption(options, "meanDelay"_sr, meanDelay); // Only matters if multiple==true
+		clogDuration = getOption(options, "clogDuration"_sr, clogDuration);
+		testDuration = getOption(options, "testDuration"_sr, testDuration);
+		enableFailures = getOption(options, "enableFailures"_sr, enableFailures);
+		multiple = getOption(options, "multiple"_sr, multiple);
+	}
+
+	void initFailureInjectionMode(DeterministicRandom& random) override {
+		enabled = clientId == 0;
+		multiple = random.coinflip();
+		enableFailures = random.random01() < 0.2;
 	}
 
 	std::string description() const override { return "RollbackWorkload"; }
 	Future<Void> setup(Database const& cx) override { return Void(); }
 	Future<Void> start(Database const& cx) override {
-		if (&g_simulator == g_network && enabled)
+		if (g_simulator == g_network && enabled)
 			return timeout(reportErrors(rollbackFailureWorker(cx, this, meanDelay), "RollbackFailureWorkerError"),
 			               testDuration,
 			               Void());
@@ -83,8 +92,8 @@ struct RollbackWorkload : TestWorkload {
 
 		for (int t = 0; t < tlogs.size(); t++) {
 			if (t != utIndex) {
-				g_simulator.clogPair(proxy.address().ip, tlogs[t].address().ip, self->clogDuration);
-				// g_simulator.clogInterface( g_simulator.getProcess( system.tlogs[t].commit.getEndpoint() ),
+				g_simulator->clogPair(proxy.address().ip, tlogs[t].address().ip, self->clogDuration);
+				// g_simulator->clogInterface( g_simulator->getProcess( system.tlogs[t].commit.getEndpoint() ),
 				// self->clogDuration, ClogAll );
 			}
 		}
@@ -93,13 +102,14 @@ struct RollbackWorkload : TestWorkload {
 		wait(delay(self->clogDuration / 3));
 		system = self->dbInfo->get();
 
-		// Kill the proxy and clog the unclogged tlog
 		if (self->enableFailures) {
-			g_simulator.killProcess(g_simulator.getProcessByAddress(proxy.address()), ISimulator::KillInstantly);
-			g_simulator.clogInterface(uncloggedTLog.ip, self->clogDuration, ClogAll);
+			// Reboot the proxy and clog the unclogged tlog.
+			g_simulator->rebootProcess(g_simulator->getProcessByAddress(proxy.address()), ISimulator::Reboot);
+			g_simulator->clogInterface(uncloggedTLog.ip, self->clogDuration, ClogAll);
 		} else {
-			g_simulator.clogInterface(proxy.address().ip, self->clogDuration, ClogAll);
-			g_simulator.clogInterface(uncloggedTLog.ip, self->clogDuration, ClogAll);
+			// Alternatively, if we're not injecting machine failures, clog the proxy and the unclogged tlog.
+			g_simulator->clogInterface(proxy.address().ip, self->clogDuration, ClogAll);
+			g_simulator->clogInterface(uncloggedTLog.ip, self->clogDuration, ClogAll);
 		}
 		return Void();
 	}
@@ -122,3 +132,4 @@ struct RollbackWorkload : TestWorkload {
 };
 
 WorkloadFactory<RollbackWorkload> RollbackWorkloadFactory("Rollback");
+FailureInjectorFactory<RollbackWorkload> RollbackFailureInjectorFactory;
diff --git a/fdbserver/workloads/RyowCorrectness.actor.cpp b/fdbserver/workloads/RyowCorrectness.actor.cpp
index dfaa5bfc8c..07290cf9df 100644
--- a/fdbserver/workloads/RyowCorrectness.actor.cpp
+++ b/fdbserver/workloads/RyowCorrectness.actor.cpp
@@ -29,7 +29,7 @@
 #define TRACE_TRANSACTION 0
 
 #if TRACE_TRANSACTION
-StringRef debugKey = LiteralStringRef("0000000000uldlamzpspf");
+StringRef debugKey = "0000000000uldlamzpspf"_sr;
 #endif
 
 // A struct representing an operation to be performed on a transaction
@@ -62,8 +62,8 @@ struct RyowCorrectnessWorkload : ApiWorkload {
 	int opsPerTransaction;
 
 	RyowCorrectnessWorkload(WorkloadContext const& wcx) : ApiWorkload(wcx, 1) {
-		duration = getOption(options, LiteralStringRef("duration"), 60);
-		opsPerTransaction = getOption(options, LiteralStringRef("opsPerTransaction"), 50);
+		duration = getOption(options, "duration"_sr, 60);
+		opsPerTransaction = getOption(options, "opsPerTransaction"_sr, 50);
 	}
 
 	std::string description() const override { return "RyowCorrectness"; }
@@ -142,7 +142,7 @@ struct RyowCorrectnessWorkload : ApiWorkload {
 	void pushKVPair(std::vector<RangeResult>& results, Key const& key, Optional<Value> const& value) {
 		RangeResult result;
 		if (!value.present())
-			result.push_back_deep(result.arena(), KeyValueRef(key, LiteralStringRef("VALUE_NOT_PRESENT")));
+			result.push_back_deep(result.arena(), KeyValueRef(key, "VALUE_NOT_PRESENT"_sr));
 		else
 			result.push_back_deep(result.arena(), KeyValueRef(key, value.get()));
 
diff --git a/fdbserver/workloads/SaveAndKill.actor.cpp b/fdbserver/workloads/SaveAndKill.actor.cpp
index c9187ce6ff..069b6877ac 100644
--- a/fdbserver/workloads/SaveAndKill.actor.cpp
+++ b/fdbserver/workloads/SaveAndKill.actor.cpp
@@ -45,8 +45,9 @@ struct SaveAndKillWorkload : TestWorkload {
 	}
 
 	std::string description() const override { return "SaveAndKillWorkload"; }
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
 	Future<Void> setup(Database const& cx) override {
-		g_simulator.disableSwapsToAll();
+		g_simulator->disableSwapsToAll();
 		return Void();
 	}
 	Future<Void> start(Database const& cx) override { return _start(this); }
@@ -60,19 +61,22 @@ struct SaveAndKillWorkload : TestWorkload {
 		ini.LoadFile(self->restartInfo.c_str());
 
 		ini.SetValue("RESTORE", "isRestoring", format("%d", self->isRestoring).c_str());
-		ini.SetValue("META", "processesPerMachine", format("%d", g_simulator.processesPerMachine).c_str());
-		ini.SetValue("META", "listenersPerProcess", format("%d", g_simulator.listenersPerProcess).c_str());
-		ini.SetValue("META", "desiredCoordinators", format("%d", g_simulator.desiredCoordinators).c_str());
-		ini.SetValue("META", "connectionString", g_simulator.connectionString.c_str());
-		ini.SetValue("META", "testerCount", format("%d", g_simulator.testerCount).c_str());
-		ini.SetValue("META", "tssMode", format("%d", g_simulator.tssMode).c_str());
+		ini.SetValue("META", "processesPerMachine", format("%d", g_simulator->processesPerMachine).c_str());
+		ini.SetValue("META", "listenersPerProcess", format("%d", g_simulator->listenersPerProcess).c_str());
+		ini.SetValue("META", "desiredCoordinators", format("%d", g_simulator->desiredCoordinators).c_str());
+		ini.SetValue("META", "connectionString", g_simulator->connectionString.c_str());
+		ini.SetValue("META", "testerCount", format("%d", g_simulator->testerCount).c_str());
+		ini.SetValue("META", "tssMode", format("%d", g_simulator->tssMode).c_str());
 		ini.SetValue("META", "mockDNS", INetworkConnections::net()->convertMockDNSToString().c_str());
 
 		ini.SetBoolValue("META", "enableEncryption", SERVER_KNOBS->ENABLE_ENCRYPTION);
 		ini.SetBoolValue("META", "enableTLogEncryption", SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION);
+		ini.SetBoolValue("META", "enableStorageServerEncryption", SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION);
+		ini.SetBoolValue("META", "enableBlobGranuleEncryption", SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION);
 
-		std::vector<ISimulator::ProcessInfo*> processes = g_simulator.getAllProcesses();
-		std::map<NetworkAddress, ISimulator::ProcessInfo*> rebootingProcesses = g_simulator.currentlyRebootingProcesses;
+		std::vector<ISimulator::ProcessInfo*> processes = g_simulator->getAllProcesses();
+		std::map<NetworkAddress, ISimulator::ProcessInfo*> rebootingProcesses =
+		    g_simulator->currentlyRebootingProcesses;
 		std::map<std::string, ISimulator::ProcessInfo*> allProcessesMap;
 		for (const auto& [_, process] : rebootingProcesses) {
 			if (allProcessesMap.find(process->dataFolder) == allProcessesMap.end() && !process->isSpawnedKVProcess()) {
@@ -137,14 +141,14 @@ struct SaveAndKillWorkload : TestWorkload {
 		ini.SaveFile(self->restartInfo.c_str());
 
 		for (auto process = allProcessesMap.begin(); process != allProcessesMap.end(); process++) {
-			g_simulator.killProcess(process->second, ISimulator::Reboot);
+			g_simulator->killProcess(process->second, ISimulator::Reboot);
 		}
 
 		for (i = 0; i < 100; i++) {
 			wait(delay(0.0));
 		}
 
-		g_simulator.stop();
+		g_simulator->stop();
 
 		return Void();
 	}
diff --git a/fdbserver/workloads/SelectorCorrectness.actor.cpp b/fdbserver/workloads/SelectorCorrectness.actor.cpp
index 5cb94e8f7d..7391850be4 100644
--- a/fdbserver/workloads/SelectorCorrectness.actor.cpp
+++ b/fdbserver/workloads/SelectorCorrectness.actor.cpp
@@ -35,12 +35,12 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 	SelectorCorrectnessWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries") {
 
-		minOperationsPerTransaction = getOption(options, LiteralStringRef("minOperationsPerTransaction"), 10);
-		maxOperationsPerTransaction = getOption(options, LiteralStringRef("minOperationsPerTransaction"), 50);
-		maxKeySpace = getOption(options, LiteralStringRef("maxKeySpace"), 10);
-		maxOffset = getOption(options, LiteralStringRef("maxOffset"), 20);
-		testReadYourWrites = getOption(options, LiteralStringRef("testReadYourWrites"), true);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
+		minOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 10);
+		maxOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 50);
+		maxKeySpace = getOption(options, "maxKeySpace"_sr, 10);
+		maxOffset = getOption(options, "maxOffset"_sr, 20);
+		testReadYourWrites = getOption(options, "testReadYourWrites"_sr, true);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
 	}
 
 	std::string description() const override { return "SelectorCorrectness"; }
diff --git a/fdbserver/workloads/Serializability.actor.cpp b/fdbserver/workloads/Serializability.actor.cpp
index a191ae01ed..2ae286b54a 100644
--- a/fdbserver/workloads/Serializability.actor.cpp
+++ b/fdbserver/workloads/Serializability.actor.cpp
@@ -65,9 +65,9 @@ struct SerializabilityWorkload : TestWorkload {
 	};
 
 	SerializabilityWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), success(true) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 30.0);
-		numOps = getOption(options, LiteralStringRef("numOps"), 21);
-		nodes = getOption(options, LiteralStringRef("nodes"), 1000);
+		testDuration = getOption(options, "testDuration"_sr, 30.0);
+		numOps = getOption(options, "numOps"_sr, 21);
+		nodes = getOption(options, "nodes"_sr, 1000);
 
 		adjacentKeys = false; // deterministicRandom()->random01() < 0.5;
 		valueSizeRange = std::make_pair(0, 100);
diff --git a/fdbserver/workloads/Sideband.actor.cpp b/fdbserver/workloads/Sideband.actor.cpp
index 15a8f09122..d992972d43 100644
--- a/fdbserver/workloads/Sideband.actor.cpp
+++ b/fdbserver/workloads/Sideband.actor.cpp
@@ -59,8 +59,8 @@ struct SidebandWorkload : TestWorkload {
 	SidebandWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), messages("Messages"), consistencyErrors("Causal Consistency Errors"),
 	    keysUnexpectedlyPresent("KeysUnexpectedlyPresent") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		operationsPerSecond = getOption(options, LiteralStringRef("operationsPerSecond"), 50.0);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		operationsPerSecond = getOption(options, "operationsPerSecond"_sr, 50.0);
 	}
 
 	std::string description() const override { return "SidebandWorkload"; }
@@ -151,7 +151,7 @@ struct SidebandWorkload : TestWorkload {
 						++self->keysUnexpectedlyPresent;
 						break;
 					}
-					tr.set(messageKey, LiteralStringRef("deadbeef"));
+					tr.set(messageKey, "deadbeef"_sr);
 					wait(tr.commit());
 					commitVersion = tr.getCommittedVersion();
 					break;
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.actor.cpp
index 8a7070d34c..a518385507 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.actor.cpp
@@ -44,8 +44,8 @@ struct SidebandSingleWorkload : TestWorkload {
 	SidebandSingleWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), messages("Messages"), consistencyErrors("Causal Consistency Errors"),
 	    keysUnexpectedlyPresent("KeysUnexpectedlyPresent") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		operationsPerSecond = getOption(options, LiteralStringRef("operationsPerSecond"), 50.0);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		operationsPerSecond = getOption(options, "operationsPerSecond"_sr, 50.0);
 	}
 
 	std::string description() const override { return "SidebandSingleWorkload"; }
@@ -94,7 +94,7 @@ struct SidebandSingleWorkload : TestWorkload {
 			// first set, this is the "old" value, always retry
 			loop {
 				try {
-					tr0.set(messageKey, LiteralStringRef("oldbeef"));
+					tr0.set(messageKey, "oldbeef"_sr);
 					wait(tr0.commit());
 					break;
 				} catch (Error& e) {
@@ -104,7 +104,7 @@ struct SidebandSingleWorkload : TestWorkload {
 			// second set, the checker should see this, no retries on unknown result
 			loop {
 				try {
-					tr.set(messageKey, LiteralStringRef("deadbeef"));
+					tr.set(messageKey, "deadbeef"_sr);
 					wait(tr.commit());
 					commitVersion = tr.getCommittedVersion();
 					break;
@@ -128,6 +128,11 @@ struct SidebandSingleWorkload : TestWorkload {
 	}
 
 	ACTOR Future<Void> checker(SidebandSingleWorkload* self, Database cx) {
+		// Required for GRV Cache to work in simulation.
+		// Normally, MVC would set the shared state and it is verified upon setting the
+		// transaction option for GRV Cache.
+		// In simulation, explicitly initialize it when used for tests.
+		cx->initSharedState();
 		loop {
 			// Pair represents <Key, commitVersion>
 			state std::pair<uint64_t, Version> message = waitNext(self->interf.getFuture());
@@ -144,7 +149,7 @@ struct SidebandSingleWorkload : TestWorkload {
 						    .detail("LocalReadVersion",
 						            tr.getReadVersion().get()); // will assert that ReadVersion is set
 						++self->consistencyErrors;
-					} else if (val.get() != LiteralStringRef("deadbeef")) {
+					} else if (val.get() != "deadbeef"_sr) {
 						// If we read something NOT "deadbeef" and there was no commit_unknown_result,
 						// the cache somehow read a stale version of our key
 						if (message.second != invalidVersion) {
diff --git a/fdbserver/workloads/SimpleAtomicAdd.actor.cpp b/fdbserver/workloads/SimpleAtomicAdd.actor.cpp
index 503bcba2a3..b65710baf6 100644
--- a/fdbserver/workloads/SimpleAtomicAdd.actor.cpp
+++ b/fdbserver/workloads/SimpleAtomicAdd.actor.cpp
@@ -35,12 +35,12 @@ struct SimpleAtomicAddWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 
 	SimpleAtomicAddWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		addValue = getOption(options, LiteralStringRef("addValue"), 1);
-		iterations = getOption(options, LiteralStringRef("iterations"), 100);
-		initialize = getOption(options, LiteralStringRef("initialize"), false);
-		initialValue = getOption(options, LiteralStringRef("initialValue"), 0);
-		sumKey = getOption(options, LiteralStringRef("sumKey"), LiteralStringRef("sumKey"));
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		addValue = getOption(options, "addValue"_sr, 1);
+		iterations = getOption(options, "iterations"_sr, 100);
+		initialize = getOption(options, "initialize"_sr, false);
+		initialValue = getOption(options, "initialValue"_sr, 0);
+		sumKey = getOption(options, "sumKey"_sr, "sumKey"_sr);
 	}
 
 	std::string description() const override { return "SimpleAtomicAdd"; }
diff --git a/fdbserver/workloads/SkewedReadWrite.actor.cpp b/fdbserver/workloads/SkewedReadWrite.actor.cpp
index 78576f957f..f537a0a1e5 100644
--- a/fdbserver/workloads/SkewedReadWrite.actor.cpp
+++ b/fdbserver/workloads/SkewedReadWrite.actor.cpp
@@ -49,7 +49,7 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 	int hotServerCount = 0, currentHotRound = -1;
 
 	SkewedReadWriteWorkload(WorkloadContext const& wcx) : ReadWriteCommon(wcx) {
-		descriptionString = getOption(options, LiteralStringRef("description"), LiteralStringRef("SkewedReadWrite"));
+		descriptionString = getOption(options, "description"_sr, "SkewedReadWrite"_sr);
 		hotServerFraction = getOption(options, "hotServerFraction"_sr, 0.2);
 		hotServerShardFraction = getOption(options, "hotServerShardFraction"_sr, 1.0);
 		hotReadWriteServerOverlap = getOption(options, "hotReadWriteServerOverlap"_sr, 0.0);
@@ -215,7 +215,8 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 			self->startReadWriteClients(cx, clients);
 			wait(timeout(waitForAll(clients), self->testDuration / self->skewRound, Void()));
 			clients.clear();
-			wait(delay(5.0) >> updateServerShards(cx, self));
+			wait(delay(5.0));
+			wait(updateServerShards(cx, self));
 		}
 
 		return Void();
@@ -368,7 +369,12 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 WorkloadFactory<SkewedReadWriteWorkload> SkewedReadWriteWorkloadFactory("SkewedReadWrite");
 
 TEST_CASE("/KVWorkload/methods/ParseKeyForIndex") {
-	auto wk = SkewedReadWriteWorkload(WorkloadContext());
+	WorkloadContext wcx;
+	wcx.clientId = 1;
+	wcx.clientCount = 1;
+	wcx.sharedRandomNumber = 1;
+
+	auto wk = SkewedReadWriteWorkload(wcx);
 	for (int i = 0; i < 1000; ++i) {
 		auto idx = deterministicRandom()->randomInt64(0, wk.nodeCount);
 		Key k = wk.keyForIndex(idx);
diff --git a/fdbserver/workloads/SlowTaskWorkload.actor.cpp b/fdbserver/workloads/SlowTaskWorkload.actor.cpp
index a1e33251ae..f03debca86 100644
--- a/fdbserver/workloads/SlowTaskWorkload.actor.cpp
+++ b/fdbserver/workloads/SlowTaskWorkload.actor.cpp
@@ -22,6 +22,7 @@
 
 #include "fmt/format.h"
 #include "fdbserver/workloads/workloads.actor.h"
+#include "flow/SignalSafeUnwind.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // Stress test the slow task profiler or flow profiler
@@ -42,22 +43,24 @@ struct SlowTaskWorkload : TestWorkload {
 
 	ACTOR static Future<Void> go() {
 		wait(delay(1));
+		int64_t phc = dl_iterate_phdr_calls;
 		int64_t startProfilesDeferred = getNumProfilesDeferred();
 		int64_t startProfilesOverflowed = getNumProfilesOverflowed();
 		int64_t startProfilesCaptured = getNumProfilesCaptured();
 		int64_t exc = 0;
-		fprintf(stdout, "Slow task starting\n");
+		fprintf(stderr, "Slow task starting\n");
 		for (int i = 0; i < 10; i++) {
-			fprintf(stdout, "  %d\n", i);
+			fprintf(stderr, "  %d\n", i);
 			double end = timer() + 1;
 			while (timer() < end) {
 				do_slow_exception_thing(&exc);
 			}
 		}
-		fmt::print(stdout,
-		           "Slow task complete: {0} exceptions; {1} profiles deferred, {2} profiles overflowed, {3} profiles "
-		           "captured\n",
+		fmt::print(stderr,
+		           "Slow task complete: {0} exceptions; {1} calls to dl_iterate_phdr, {2}"
+		           " profiles deferred, {3} profiles overflowed, {4} profiles captured\n",
 		           exc,
+		           dl_iterate_phdr_calls - phc,
 		           getNumProfilesDeferred() - startProfilesDeferred,
 		           getNumProfilesOverflowed() - startProfilesOverflowed,
 		           getNumProfilesCaptured() - startProfilesCaptured);
diff --git a/fdbserver/workloads/SnapTest.actor.cpp b/fdbserver/workloads/SnapTest.actor.cpp
index 6f28a034fd..e2eb69e8be 100644
--- a/fdbserver/workloads/SnapTest.actor.cpp
+++ b/fdbserver/workloads/SnapTest.actor.cpp
@@ -114,14 +114,13 @@ public: // ctor & dtor
 		std::string workloadName = "SnapTest";
 		maxRetryCntToRetrieveMessage = 10;
 
-		numSnaps = getOption(options, LiteralStringRef("numSnaps"), 0);
-		maxSnapDelay = getOption(options, LiteralStringRef("maxSnapDelay"), 25.0);
-		testID = getOption(options, LiteralStringRef("testID"), 0);
-		restartInfoLocation =
-		    getOption(options, LiteralStringRef("restartInfoLocation"), LiteralStringRef("simfdb/restartInfo.ini"))
-		        .toString();
+		numSnaps = getOption(options, "numSnaps"_sr, 0);
+		maxSnapDelay = getOption(options, "maxSnapDelay"_sr, 25.0);
+		testID = getOption(options, "testID"_sr, 0);
+		restartInfoLocation = getOption(options, "restartInfoLocation"_sr, "simfdb/restartInfo.ini"_sr).toString();
 		skipCheck = false;
-		retryLimit = getOption(options, LiteralStringRef("retryLimit"), 5);
+		retryLimit = getOption(options, "retryLimit"_sr, 5);
+		g_simulator->allowLogSetKills = false;
 	}
 
 public: // workload functions
@@ -148,7 +147,7 @@ public: // workload functions
 		loop {
 			try {
 				Standalone<StringRef> keyStr =
-				    LiteralStringRef("\xff/SnapTestFailStatus/").withSuffix(StringRef(self->snapUID.toString()));
+				    "\xff/SnapTestFailStatus/"_sr.withSuffix(StringRef(self->snapUID.toString()));
 				TraceEvent("TestKeyStr").detail("Value", keyStr);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				Optional<Value> val = wait(tr.get(keyStr));
@@ -232,7 +231,7 @@ public: // workload functions
 			loop {
 				self->snapUID = deterministicRandom()->randomUniqueID();
 				try {
-					StringRef snapCmdRef = LiteralStringRef("/bin/snap_create.sh");
+					StringRef snapCmdRef = "/bin/snap_create.sh"_sr;
 					Future<Void> status = snapCreate(cx, snapCmdRef, self->snapUID);
 					wait(status);
 					break;
@@ -290,7 +289,7 @@ public: // workload functions
 					}
 
 					for (int i = 0; i < kvRange.size(); i++) {
-						if (kvRange[i].key.startsWith(LiteralStringRef("snapKey"))) {
+						if (kvRange[i].key.startsWith("snapKey"_sr)) {
 							std::string tmp1 = kvRange[i].key.substr(7).toString();
 							int64_t id = strtol(tmp1.c_str(), nullptr, 0);
 							if (id % 2 != 0) {
@@ -321,7 +320,7 @@ public: // workload functions
 			loop {
 				self->snapUID = deterministicRandom()->randomUniqueID();
 				try {
-					StringRef snapCmdRef = LiteralStringRef("/bin/snap_create1.sh");
+					StringRef snapCmdRef = "/bin/snap_create1.sh"_sr;
 					Future<Void> status = snapCreate(cx, snapCmdRef, self->snapUID);
 					wait(status);
 					break;
diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
index f2bbbde625..5c7f269088 100644
--- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
@@ -28,6 +28,7 @@
 #include "fdbclient/Schemas.h"
 #include "fdbclient/SpecialKeySpace.actor.h"
 #include "fdbserver/Knobs.h"
+#include "fdbclient/TenantManagement.actor.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/IRandom.h"
@@ -39,22 +40,24 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 	double testDuration, absoluteRandomProb, transactionsPerSecond;
 	PerfIntCounter wrongResults, keysCount;
 	Reference<ReadYourWritesTransaction> ryw; // used to store all populated data
-	std::vector<std::shared_ptr<SKSCTestImpl>> impls;
+	std::vector<std::shared_ptr<SKSCTestRWImpl>> rwImpls;
+	std::vector<std::shared_ptr<SKSCTestAsyncReadImpl>> asyncReadImpls;
 	Standalone<VectorRef<KeyRangeRef>> keys;
+	Standalone<VectorRef<KeyRangeRef>> rwKeys;
 
 	SpecialKeySpaceCorrectnessWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), wrongResults("Wrong Results"), keysCount("Number of generated keys") {
-		minKeysPerRange = getOption(options, LiteralStringRef("minKeysPerRange"), 1);
-		maxKeysPerRange = getOption(options, LiteralStringRef("maxKeysPerRange"), 100);
-		rangeCount = getOption(options, LiteralStringRef("rangeCount"), 10);
-		keyBytes = getOption(options, LiteralStringRef("keyBytes"), 16);
-		valBytes = getOption(options, LiteralStringRef("valueBytes"), 16);
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 100.0);
-		actorCount = getOption(options, LiteralStringRef("actorCount"), 1);
-		absoluteRandomProb = getOption(options, LiteralStringRef("absoluteRandomProb"), 0.5);
+		minKeysPerRange = getOption(options, "minKeysPerRange"_sr, 1);
+		maxKeysPerRange = getOption(options, "maxKeysPerRange"_sr, 100);
+		rangeCount = getOption(options, "rangeCount"_sr, 10);
+		keyBytes = getOption(options, "keyBytes"_sr, 16);
+		valBytes = getOption(options, "valueBytes"_sr, 16);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 100.0);
+		actorCount = getOption(options, "actorCount"_sr, 1);
+		absoluteRandomProb = getOption(options, "absoluteRandomProb"_sr, 0.5);
 		// Controls the relative size of read/write conflict ranges and the number of random getranges
-		conflictRangeSizeFactor = getOption(options, LiteralStringRef("conflictRangeSizeFactor"), 10);
+		conflictRangeSizeFactor = getOption(options, "conflictRangeSizeFactor"_sr, 10);
 		ASSERT(conflictRangeSizeFactor >= 1);
 	}
 
@@ -63,9 +66,9 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 	Future<bool> check(Database const& cx) override { return wrongResults.getValue() == 0; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
-
 	// disable the default timeout setting
 	double getCheckTimeout() const override { return std::numeric_limits<double>::max(); }
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("MoveKeysWorkload"); }
 
 	Future<Void> _setup(Database cx, SpecialKeySpaceCorrectnessWorkload* self) {
 		cx->specialKeySpace = std::make_unique<SpecialKeySpace>();
@@ -81,12 +84,20 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			Key startKey(baseKey + "/");
 			Key endKey(baseKey + "/\xff");
 			self->keys.push_back_deep(self->keys.arena(), KeyRangeRef(startKey, endKey));
-			self->impls.push_back(std::make_shared<SKSCTestImpl>(KeyRangeRef(startKey, endKey)));
-			// Although there are already ranges registered, the testing range will replace them
-			cx->specialKeySpace->registerKeyRange(SpecialKeySpace::MODULE::TESTONLY,
-			                                      SpecialKeySpace::IMPLTYPE::READWRITE,
-			                                      self->keys.back(),
-			                                      self->impls.back().get());
+			if (deterministicRandom()->random01() < 0.2) {
+				self->asyncReadImpls.push_back(std::make_shared<SKSCTestAsyncReadImpl>(KeyRangeRef(startKey, endKey)));
+				cx->specialKeySpace->registerKeyRange(SpecialKeySpace::MODULE::TESTONLY,
+				                                      SpecialKeySpace::IMPLTYPE::READONLY,
+				                                      self->keys.back(),
+				                                      self->asyncReadImpls.back().get());
+			} else {
+				self->rwImpls.push_back(std::make_shared<SKSCTestRWImpl>(KeyRangeRef(startKey, endKey)));
+				// Although there are already ranges registered, the testing range will replace them
+				cx->specialKeySpace->registerKeyRange(SpecialKeySpace::MODULE::TESTONLY,
+				                                      SpecialKeySpace::IMPLTYPE::READWRITE,
+				                                      self->keys.back(),
+				                                      self->rwImpls.back().get());
+			}
 			// generate keys in each key range
 			int keysInRange = deterministicRandom()->randomInt(self->minKeysPerRange, self->maxKeysPerRange + 1);
 			self->keysCount += keysInRange;
@@ -118,8 +129,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				// This test is not valid for API versions smaller than 630
 				return;
 			}
-			f = success(ryw.get(LiteralStringRef("\xff\xff/status/json")));
-			TEST(!f.isReady()); // status json not ready
+			f = success(ryw.get("\xff\xff/status/json"_sr));
+			CODE_PROBE(!f.isReady(), "status json not ready");
 		}
 		ASSERT(f.isError());
 		ASSERT(f.getError().code() == error_code_transaction_cancelled);
@@ -154,7 +165,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			}
 
 			// check ryw result consistency
-			KeyRange rkr = self->randomKeyRange();
+			KeyRange rkr = self->randomRWKeyRange();
 			KeyRef rkey1 = rkr.begin;
 			KeyRef rkey2 = rkr.end;
 			// randomly set/clear two keys or clear a key range
@@ -233,13 +244,13 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				    .detail("TestValue", printable(res2[i].value));
 				return false;
 			}
-			TEST(true); // Special key space keys equal
+			CODE_PROBE(true, "Special key space keys equal");
 		}
 		return true;
 	}
 
-	KeyRange randomKeyRange() {
-		Key prefix = keys[deterministicRandom()->randomInt(0, rangeCount)].begin;
+	KeyRange randomRWKeyRange() {
+		Key prefix = rwImpls[deterministicRandom()->randomInt(0, rwImpls.size())]->getKeyRange().begin;
 		Key rkey1 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, keyBytes)))
 		                .withPrefix(prefix);
 		Key rkey2 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, keyBytes)))
@@ -278,24 +289,152 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 
 	GetRangeLimits randomLimits() {
 		// TODO : fix knobs for row_unlimited
-		int rowLimits = deterministicRandom()->randomInt(1, keysCount.getValue() + 1);
+		int rowLimits = deterministicRandom()->randomInt(0, keysCount.getValue() + 1);
 		// The largest key's bytes is longest prefix bytes + 1(for '/') + generated key bytes
 		// 8 here refers to bytes of KeyValueRef
 		int byteLimits = deterministicRandom()->randomInt(
 		    1, keysCount.getValue() * (keyBytes + (rangeCount + 1) + valBytes + 8) + 1);
 
-		return GetRangeLimits(rowLimits, byteLimits);
+		auto limit = GetRangeLimits(rowLimits, byteLimits);
+		// minRows is always initilized to 1
+		if (limit.rows == 0)
+			limit.minRows = 0;
+		return limit;
 	}
 
 	ACTOR Future<Void> testSpecialKeySpaceErrors(Database cx_, SpecialKeySpaceCorrectnessWorkload* self) {
-		Database cx = cx_->clone();
+		state Database cx = cx_->clone();
+		try {
+			wait(success(TenantAPI::createTenant(cx.getReference(), TenantName("foo"_sr))));
+		} catch (Error& e) {
+			ASSERT(e.code() == error_code_tenant_already_exists || e.code() == error_code_actor_cancelled);
+		}
 		state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
+		state Reference<ReadYourWritesTransaction> tenantTx =
+		    makeReference<ReadYourWritesTransaction>(cx, TenantName("foo"_sr));
+		// Use new transactions that may use default tenant rather than re-use tx
+		// This is because tx will reject raw access for later tests if default tenant is set
+		state Reference<ReadYourWritesTransaction> defaultTx1 = makeReference<ReadYourWritesTransaction>(cx);
+		state Reference<ReadYourWritesTransaction> defaultTx2 = makeReference<ReadYourWritesTransaction>(cx);
+		state bool disableRyw = deterministicRandom()->coinflip();
+		// tenant transaction accessing modules that do not support tenants
+		// tenant getRange
+		try {
+			wait(success(tenantTx->getRange(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT),
+			                                CLIENT_KNOBS->TOO_MANY)));
+			ASSERT(false);
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled)
+				throw;
+			ASSERT(e.code() == error_code_illegal_tenant_access);
+			tenantTx->reset();
+		}
+		// tenant set + commit
+		try {
+			tenantTx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			tenantTx->set(SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck"), ValueRef());
+			wait(tenantTx->commit());
+			ASSERT(false);
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled)
+				throw;
+			ASSERT(e.code() == error_code_illegal_tenant_access);
+			tenantTx->reset();
+		}
+		// tenant clear
+		try {
+			tenantTx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			tenantTx->clear(SpecialKeySpace::getManagementApiCommandRange("exclude"));
+			ASSERT(false);
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled)
+				throw;
+			ASSERT(e.code() == error_code_illegal_tenant_access);
+			tenantTx->reset();
+		}
+		// tenant check that conflict ranges stay the same after commit
+		// and depending on if RYW is disabled
+		{
+			state RangeResult readresult1;
+			state RangeResult readresult2;
+			state RangeResult writeResult1;
+			state RangeResult writeResult2;
+			try {
+				if (disableRyw) {
+					defaultTx1->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
+				}
+				defaultTx1->addReadConflictRange(singleKeyRange("testKeylll"_sr));
+				defaultTx1->addWriteConflictRange(singleKeyRange("testKeylll"_sr));
+				wait(store(readresult1, defaultTx1->getRange(readConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY)));
+				wait(store(writeResult1, defaultTx1->getRange(writeConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY)));
+				wait(defaultTx1->commit());
+				CODE_PROBE(true, "conflict range tenant commit succeeded");
+			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled)
+					throw;
+				CODE_PROBE(true, "conflict range tenant commit error thrown");
+			}
+			wait(store(readresult2, defaultTx1->getRange(readConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY)));
+			wait(store(writeResult2, defaultTx1->getRange(writeConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY)));
+			ASSERT(readresult1 == readresult2);
+			ASSERT(writeResult1 == writeResult2);
+			defaultTx1->reset();
+		}
+		// proper conflict ranges
+		loop {
+			try {
+				if (disableRyw) {
+					defaultTx1->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
+					defaultTx2->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
+				}
+				defaultTx1->setOption(FDBTransactionOptions::REPORT_CONFLICTING_KEYS);
+				defaultTx2->setOption(FDBTransactionOptions::REPORT_CONFLICTING_KEYS);
+				wait(success(defaultTx1->getReadVersion()));
+				wait(success(defaultTx2->getReadVersion()));
+				defaultTx1->addReadConflictRange(singleKeyRange("foo"_sr));
+				defaultTx1->addWriteConflictRange(singleKeyRange("foo"_sr));
+				defaultTx2->addWriteConflictRange(singleKeyRange("foo"_sr));
+				wait(defaultTx2->commit());
+				try {
+					wait(defaultTx1->commit());
+					ASSERT(false);
+				} catch (Error& e) {
+					state Error err = e;
+					if (err.code() != error_code_not_committed) {
+						wait(defaultTx1->onError(err));
+						wait(defaultTx2->onError(err));
+						continue;
+					}
+					// Read conflict ranges of defaultTx1 and check for "foo" with no tenant prefix
+					state RangeResult readConflictRange =
+					    wait(defaultTx1->getRange(readConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY));
+					state RangeResult writeConflictRange =
+					    wait(defaultTx1->getRange(writeConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY));
+					state RangeResult conflictKeys =
+					    wait(defaultTx1->getRange(conflictingKeysRange, CLIENT_KNOBS->TOO_MANY));
+
+					// size is 2 because singleKeyRange includes the key after
+					ASSERT(readConflictRange.size() == 2 &&
+					       readConflictRange.begin()->key == readConflictRangeKeysRange.begin.withSuffix("foo"_sr));
+					ASSERT(writeConflictRange.size() == 2 &&
+					       writeConflictRange.begin()->key == writeConflictRangeKeysRange.begin.withSuffix("foo"_sr));
+					ASSERT(conflictKeys.size() == 2 &&
+					       conflictKeys.begin()->key == conflictingKeysRange.begin.withSuffix("foo"_sr));
+					defaultTx1->reset();
+					defaultTx2->reset();
+					break;
+				}
+			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled)
+					throw;
+				wait(defaultTx2->onError(e));
+			}
+		}
 		// begin key outside module range
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(
-			    KeyRangeRef(LiteralStringRef("\xff\xff/transactio"), LiteralStringRef("\xff\xff/transaction0")),
-			    CLIENT_KNOBS->TOO_MANY)));
+			wait(success(tx->getRange(KeyRangeRef("\xff\xff/transactio"_sr, "\xff\xff/transaction0"_sr),
+			                          CLIENT_KNOBS->TOO_MANY)));
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -306,9 +445,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// end key outside module range
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(
-			    KeyRangeRef(LiteralStringRef("\xff\xff/transaction/"), LiteralStringRef("\xff\xff/transaction1")),
-			    CLIENT_KNOBS->TOO_MANY)));
+			wait(success(tx->getRange(KeyRangeRef("\xff\xff/transaction/"_sr, "\xff\xff/transaction1"_sr),
+			                          CLIENT_KNOBS->TOO_MANY)));
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -319,9 +457,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// both begin and end outside module range
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(
-			    KeyRangeRef(LiteralStringRef("\xff\xff/transaction"), LiteralStringRef("\xff\xff/transaction1")),
-			    CLIENT_KNOBS->TOO_MANY)));
+			wait(success(tx->getRange(KeyRangeRef("\xff\xff/transaction"_sr, "\xff\xff/transaction1"_sr),
+			                          CLIENT_KNOBS->TOO_MANY)));
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -332,10 +469,9 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// legal range read using the module range
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(
-			    KeyRangeRef(LiteralStringRef("\xff\xff/transaction/"), LiteralStringRef("\xff\xff/transaction0")),
-			    CLIENT_KNOBS->TOO_MANY)));
-			TEST(true); // read transaction special keyrange
+			wait(success(tx->getRange(KeyRangeRef("\xff\xff/transaction/"_sr, "\xff\xff/transaction0"_sr),
+			                          CLIENT_KNOBS->TOO_MANY)));
+			CODE_PROBE(true, "read transaction special keyrange");
 			tx->reset();
 		} catch (Error& e) {
 			throw;
@@ -344,8 +480,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 			tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED);
-			const KeyRef startKey = LiteralStringRef("\xff\xff/transactio");
-			const KeyRef endKey = LiteralStringRef("\xff\xff/transaction1");
+			const KeyRef startKey = "\xff\xff/transactio"_sr;
+			const KeyRef endKey = "\xff\xff/transaction1"_sr;
 			RangeResult result =
 			    wait(tx->getRange(KeyRangeRef(startKey, endKey), GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
 			// The whole transaction module should be empty
@@ -357,11 +493,11 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// end keySelector inside module range, *** a tricky corner case ***
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			tx->addReadConflictRange(singleKeyRange(LiteralStringRef("testKey")));
+			tx->addReadConflictRange(singleKeyRange("testKey"_sr));
 			KeySelector begin = KeySelectorRef(readConflictRangeKeysRange.begin, false, 1);
-			KeySelector end = KeySelectorRef(LiteralStringRef("\xff\xff/transaction0"), false, 0);
+			KeySelector end = KeySelectorRef("\xff\xff/transaction0"_sr, false, 0);
 			wait(success(tx->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY))));
-			TEST(true); // end key selector inside module range
+			CODE_PROBE(true, "end key selector inside module range");
 			tx->reset();
 		} catch (Error& e) {
 			throw;
@@ -369,9 +505,9 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// No module found error case with keys
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/A_no_module_related_prefix"),
-			                                      LiteralStringRef("\xff\xff/I_am_also_not_in_any_module")),
-			                          CLIENT_KNOBS->TOO_MANY)));
+			wait(success(tx->getRange(
+			    KeyRangeRef("\xff\xff/A_no_module_related_prefix"_sr, "\xff\xff/I_am_also_not_in_any_module"_sr),
+			    CLIENT_KNOBS->TOO_MANY)));
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -382,8 +518,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// No module found error with KeySelectors, *** a tricky corner case ***
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			KeySelector begin = KeySelectorRef(LiteralStringRef("\xff\xff/zzz_i_am_not_a_module"), false, 1);
-			KeySelector end = KeySelectorRef(LiteralStringRef("\xff\xff/zzz_to_be_the_final_one"), false, 2);
+			KeySelector begin = KeySelectorRef("\xff\xff/zzz_i_am_not_a_module"_sr, false, 1);
+			KeySelector end = KeySelectorRef("\xff\xff/zzz_to_be_the_final_one"_sr, false, 2);
 			wait(success(tx->getRange(begin, end, CLIENT_KNOBS->TOO_MANY)));
 			ASSERT(false);
 		} catch (Error& e) {
@@ -395,7 +531,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// begin and end keySelectors clamp up to the boundary of the module
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			const KeyRef key = LiteralStringRef("\xff\xff/cluster_file_path");
+			const KeyRef key = "\xff\xff/cluster_file_path"_sr;
 			KeySelector begin = KeySelectorRef(key, false, 0);
 			KeySelector end = KeySelectorRef(keyAfter(key), false, 2);
 			RangeResult result = wait(tx->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
@@ -406,8 +542,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		}
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			tx->addReadConflictRange(singleKeyRange(LiteralStringRef("readKey")));
-			const KeyRef key = LiteralStringRef("\xff\xff/transaction/a_to_be_the_first");
+			tx->addReadConflictRange(singleKeyRange("readKey"_sr));
+			const KeyRef key = "\xff\xff/transaction/a_to_be_the_first"_sr;
 			KeySelector begin = KeySelectorRef(key, false, 0);
 			KeySelector end = KeySelectorRef(key, false, 2);
 			RangeResult result = wait(tx->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
@@ -420,7 +556,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// Writes are disabled by default
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			tx->set(LiteralStringRef("\xff\xff/I_am_not_a_range_can_be_written"), ValueRef());
+			tx->set("\xff\xff/I_am_not_a_range_can_be_written"_sr, ValueRef());
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
 				throw;
@@ -431,7 +567,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 			tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-			tx->set(LiteralStringRef("\xff\xff/I_am_not_a_range_can_be_written"), ValueRef());
+			tx->set("\xff\xff/I_am_not_a_range_can_be_written"_sr, ValueRef());
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -455,8 +591,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// base key of the end key selector not in (\xff\xff, \xff\xff\xff), throw key_outside_legal_range()
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			const KeySelector startKeySelector = KeySelectorRef(LiteralStringRef("\xff\xff/test"), true, -200);
-			const KeySelector endKeySelector = KeySelectorRef(LiteralStringRef("test"), true, -10);
+			const KeySelector startKeySelector = KeySelectorRef("\xff\xff/test"_sr, true, -200);
+			const KeySelector endKeySelector = KeySelectorRef("test"_sr, true, -10);
 			RangeResult result =
 			    wait(tx->getRange(startKeySelector, endKeySelector, GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
 			ASSERT(false);
@@ -469,9 +605,9 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// test case when registered range is the same as the underlying module
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			state RangeResult result = wait(tx->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
-			                                                         LiteralStringRef("\xff\xff/worker_interfaces0")),
-			                                             CLIENT_KNOBS->TOO_MANY));
+			state RangeResult result =
+			    wait(tx->getRange(KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr),
+			                      CLIENT_KNOBS->TOO_MANY));
 			// Note: there's possibility we get zero workers
 			if (result.size()) {
 				state KeyValueRef entry = deterministicRandom()->randomChoice(result);
@@ -489,8 +625,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 
 	ACTOR static Future<Void> testConflictRanges(Database cx_, bool read, SpecialKeySpaceCorrectnessWorkload* self) {
 		state StringRef prefix = read ? readConflictRangeKeysRange.begin : writeConflictRangeKeysRange.begin;
-		TEST(read); // test read conflict range special key implementation
-		TEST(!read); // test write conflict range special key implementation
+		CODE_PROBE(read, "test read conflict range special key implementation");
+		CODE_PROBE(!read, "test write conflict range special key implementation");
 		// Get a default special key range instance
 		Database cx = cx_->clone();
 		state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
@@ -534,8 +670,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				tx->addWriteConflictRange(range);
 			}
 			// TODO test that fails if we don't wait on tx->pendingReads()
-			referenceTx->set(range.begin, LiteralStringRef("1"));
-			referenceTx->set(range.end, LiteralStringRef("0"));
+			referenceTx->set(range.begin, "1"_sr);
+			referenceTx->set(range.end, "0"_sr);
 		}
 		if (!read && deterministicRandom()->coinflip()) {
 			try {
@@ -545,10 +681,10 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					throw;
 				return Void();
 			}
-			TEST(true); // Read write conflict range of committed transaction
+			CODE_PROBE(true, "Read write conflict range of committed transaction");
 		}
 		try {
-			wait(success(tx->get(LiteralStringRef("\xff\xff/1314109/i_hope_this_isn't_registered"))));
+			wait(success(tx->get("\xff\xff/1314109/i_hope_this_isn't_registered"_sr)));
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -656,7 +792,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				    ValueRef());
 			}
 			RangeResult result = wait(tx->getRange(
-			    KeyRangeRef(LiteralStringRef("options/"), LiteralStringRef("options0"))
+			    KeyRangeRef("options/"_sr, "options0"_sr)
 			        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin),
 			    CLIENT_KNOBS->TOO_MANY));
 			ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY);
@@ -668,8 +804,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 			tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-			tx->set(LiteralStringRef("Invalid_Network_Address")
-			            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("exclude")),
+			tx->set("Invalid_Network_Address"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("exclude")),
 			        ValueRef());
 			wait(tx->commit());
 			ASSERT(false);
@@ -699,7 +834,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				// test getRange
 				state RangeResult result = wait(tx->getRange(
-				    KeyRangeRef(LiteralStringRef("process/class_type/"), LiteralStringRef("process/class_type0"))
+				    KeyRangeRef("process/class_type/"_sr, "process/class_type0"_sr)
 				        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin),
 				    CLIENT_KNOBS->TOO_MANY));
 				ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY);
@@ -727,10 +862,10 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					state Key addr =
 					    Key("process/class_type/" + formatIpPort(worker.address.ip, worker.address.port))
 					        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin);
-					tx->set(addr, LiteralStringRef("InvalidProcessType"));
+					tx->set(addr, "InvalidProcessType"_sr);
 					// test ryw
 					Optional<Value> processType = wait(tx->get(addr));
-					ASSERT(processType.present() && processType.get() == LiteralStringRef("InvalidProcessType"));
+					ASSERT(processType.present() && processType.get() == "InvalidProcessType"_sr);
 					// test ryw disabled
 					tx->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
 					Optional<Value> originalProcessType = wait(tx->get(addr));
@@ -769,7 +904,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				// test getRange
 				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 				state RangeResult class_source_result = wait(tx->getRange(
-				    KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0"))
+				    KeyRangeRef("process/class_source/"_sr, "process/class_source0"_sr)
 				        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin),
 				    CLIENT_KNOBS->TOO_MANY));
 				ASSERT(!class_source_result.more && class_source_result.size() < CLIENT_KNOBS->TOO_MANY);
@@ -787,7 +922,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 							if (kv.key == addr) {
 								ASSERT(kv.value.toString() == worker.processClass.sourceString());
 								// Default source string is command_line
-								ASSERT(kv.value == LiteralStringRef("command_line"));
+								ASSERT(kv.value == "command_line"_sr);
 								found = true;
 								break;
 							}
@@ -815,7 +950,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					    .detail("ClassSource", class_source.present() ? class_source.get().toString() : "__Nothing");
 					// Very rarely, we get an empty worker list, thus no class_source data
 					if (class_source.present())
-						ASSERT(class_source.get() == LiteralStringRef("set_class"));
+						ASSERT(class_source.get() == "set_class"_sr);
 					tx->reset();
 				} else {
 					// If no worker process returned, skip the test
@@ -863,31 +998,34 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		}
 		TraceEvent(SevDebug, "DatabaseLocked").log();
 		// if database locked, fdb read should get database_locked error
-		try {
-			tx->reset();
-			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			RangeResult res = wait(tx->getRange(normalKeys, 1));
-		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled)
-				throw;
-			ASSERT(e.code() == error_code_database_locked);
+		tx->reset();
+		loop {
+			try {
+				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
+				RangeResult res = wait(tx->getRange(normalKeys, 1));
+			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled)
+					throw;
+				if (e.code() == error_code_grv_proxy_memory_limit_exceeded ||
+				    e.code() == error_code_batch_transaction_throttled) {
+					wait(tx->onError(e));
+				} else {
+					ASSERT(e.code() == error_code_database_locked);
+					break;
+				}
+			}
 		}
 		// make sure we unlock the database
 		// unlock is idempotent, thus we can commit many times until successful
+		tx->reset();
 		loop {
 			try {
-				tx->reset();
 				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 				tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				// unlock the database
 				tx->clear(SpecialKeySpace::getManagementApiCommandPrefix("lock"));
 				wait(tx->commit());
 				TraceEvent(SevDebug, "DatabaseUnlocked").log();
-				tx->reset();
-				// read should be successful
-				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-				RangeResult res = wait(tx->getRange(normalKeys, 1));
-				tx->reset();
 				break;
 			} catch (Error& e) {
 				TraceEvent(SevDebug, "DatabaseUnlockFailure").error(e);
@@ -895,9 +1033,23 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				wait(tx->onError(e));
 			}
 		}
+
+		tx->reset();
+		loop {
+			try {
+				// read should be successful
+				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
+				RangeResult res = wait(tx->getRange(normalKeys, 1));
+				break;
+			} catch (Error& e) {
+				wait(tx->onError(e));
+			}
+		}
+
 		// test consistencycheck which only used by ConsistencyCheck Workload
 		// Note: we have exclusive ownership of fdbShouldConsistencyCheckBeSuspended,
 		// no existing workloads can modify the key
+		tx->reset();
 		{
 			try {
 				tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@@ -949,9 +1101,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				Optional<Value> res = wait(tx->get(coordinatorsKey));
 				ASSERT(res.present()); // Otherwise, database is in a bad state
 				state ClusterConnectionString cs(res.get().toString());
-				Optional<Value> coordinator_processes_key =
-				    wait(tx->get(LiteralStringRef("processes")
-				                     .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators"))));
+				Optional<Value> coordinator_processes_key = wait(
+				    tx->get("processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators"))));
 				ASSERT(coordinator_processes_key.present());
 				state std::vector<std::string> process_addresses;
 				boost::split(
@@ -977,7 +1128,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			state std::vector<std::string> old_coordinators_processes;
 			state bool possible_to_add_coordinator;
 			state KeyRange coordinators_key_range =
-			    KeyRangeRef(LiteralStringRef("process/"), LiteralStringRef("process0"))
+			    KeyRangeRef("process/"_sr, "process0"_sr)
 			        .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators"));
 			loop {
 				try {
@@ -990,9 +1141,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					                              ? deterministicRandom()->randomAlphaNumeric(8)
 					                              : ccStr.clusterKeyName().toString();
 					// get current coordinators
-					Optional<Value> processes_key =
-					    wait(tx->get(LiteralStringRef("processes")
-					                     .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators"))));
+					Optional<Value> processes_key = wait(tx->get(
+					    "processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators"))));
 					ASSERT(processes_key.present());
 					boost::split(
 					    old_coordinators_processes, processes_key.get().toString(), [](char c) { return c == ','; });
@@ -1033,12 +1183,12 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 						for (const auto& address : old_coordinators_processes) {
 							new_processes_key += "," + address;
 						}
-						tx->set(LiteralStringRef("processes")
-						            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
-						        Value(new_processes_key));
+						tx->set(
+						    "processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
+						    Value(new_processes_key));
 						// update cluster description
-						tx->set(LiteralStringRef("cluster_description")
-						            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
+						tx->set("cluster_description"_sr.withPrefix(
+						            SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
 						        Value(new_cluster_description));
 						wait(tx->commit());
 						ASSERT(false);
@@ -1105,9 +1255,9 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 							new_processes_key += new_processes_key.size() ? "," : "";
 							new_processes_key += address;
 						}
-						tx->set(LiteralStringRef("processes")
-						            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
-						        Value(new_processes_key));
+						tx->set(
+						    "processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
+						    Value(new_processes_key));
 						wait(tx->commit());
 						ASSERT(false);
 					} catch (Error& e) {
@@ -1185,8 +1335,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				                                      CLIENT_KNOBS->TOO_MANY));
 				// By default, data_distribution/mode := "-1"
 				ASSERT(!ddKVs.more && ddKVs.size() == 1);
-				ASSERT(ddKVs[0].key == LiteralStringRef("mode").withPrefix(
-				                           SpecialKeySpace::getManagementApiCommandPrefix("datadistribution")));
+				ASSERT(ddKVs[0].key ==
+				       "mode"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("datadistribution")));
 				ASSERT(ddKVs[0].value == Value(boost::lexical_cast<std::string>(-1)));
 				tx->reset();
 				break;
@@ -1286,8 +1436,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 					KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");
-					tx->set(LiteralStringRef("mode").withPrefix(ddPrefix), LiteralStringRef("0"));
-					tx->set(LiteralStringRef("rebalance_ignored").withPrefix(ddPrefix),
+					tx->set("mode"_sr.withPrefix(ddPrefix), "0"_sr);
+					tx->set("rebalance_ignored"_sr.withPrefix(ddPrefix),
 					        BinaryWriter::toValue(ddIgnoreValue, Unversioned()));
 					wait(tx->commit());
 					tx->reset();
@@ -1329,8 +1479,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					tx->clear(ignoreSSFailuresZoneString.withPrefix(
 					    SpecialKeySpace::getManagementApiCommandPrefix("maintenance")));
 					KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");
-					tx->clear(LiteralStringRef("mode").withPrefix(ddPrefix));
-					tx->clear(LiteralStringRef("rebalance_ignored").withPrefix(ddPrefix));
+					tx->clear("mode"_sr.withPrefix(ddPrefix));
+					tx->clear("rebalance_ignored"_sr.withPrefix(ddPrefix));
 					wait(tx->commit());
 					tx->reset();
 					break;
@@ -1372,14 +1522,14 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					Version readVersion = wait(tr1->getReadVersion());
 					tr2->setVersion(readVersion);
 					KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");
-					tr1->set(LiteralStringRef("mode").withPrefix(ddPrefix), LiteralStringRef("1"));
+					tr1->set("mode"_sr.withPrefix(ddPrefix), "1"_sr);
 					wait(tr1->commit());
 					// randomly read the moveKeysLockOwnerKey/moveKeysLockWriteKey
 					// both of them should be grabbed when changing dd mode
 					wait(success(
 					    tr2->get(deterministicRandom()->coinflip() ? moveKeysLockOwnerKey : moveKeysLockWriteKey)));
 					// tr2 shoulde never succeed, just write to a key to make it not a read-only transaction
-					tr2->set(LiteralStringRef("unused_key"), LiteralStringRef(""));
+					tr2->set("unused_key"_sr, ""_sr);
 					wait(tr2->commit());
 					ASSERT(false); // commit should always fail due to conflict
 				} catch (Error& e) {
diff --git a/fdbserver/workloads/StatusWorkload.actor.cpp b/fdbserver/workloads/StatusWorkload.actor.cpp
index 29279fc036..52fbb43860 100644
--- a/fdbserver/workloads/StatusWorkload.actor.cpp
+++ b/fdbserver/workloads/StatusWorkload.actor.cpp
@@ -39,11 +39,10 @@ struct StatusWorkload : TestWorkload {
 	StatusWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), requests("Status requests issued"), replies("Status replies received"),
 	    errors("Status Errors"), totalSize("Status reply size sum") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		requestsPerSecond = getOption(options, LiteralStringRef("requestsPerSecond"), 0.5);
-		enableLatencyBands =
-		    getOption(options, LiteralStringRef("enableLatencyBands"), deterministicRandom()->random01() < 0.5);
-		auto statusSchemaStr = getOption(options, LiteralStringRef("schema"), JSONSchemas::statusSchema);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		requestsPerSecond = getOption(options, "requestsPerSecond"_sr, 0.5);
+		enableLatencyBands = getOption(options, "enableLatencyBands"_sr, deterministicRandom()->random01() < 0.5);
+		auto statusSchemaStr = getOption(options, "schema"_sr, JSONSchemas::statusSchema);
 		if (statusSchemaStr.size()) {
 			json_spirit::mValue schema = readJSONStrictly(statusSchemaStr.toString());
 			parsedSchema = schema.get_obj();
diff --git a/fdbserver/workloads/StorageQuota.actor.cpp b/fdbserver/workloads/StorageQuota.actor.cpp
new file mode 100644
index 0000000000..d3ab1ec826
--- /dev/null
+++ b/fdbserver/workloads/StorageQuota.actor.cpp
@@ -0,0 +1,78 @@
+/*
+ * StorageQuota.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+
+#include "flow/Trace.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct StorageQuotaWorkload : TestWorkload {
+	StorageQuotaWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+
+	std::string description() const override { return "StorageQuotaWorkload"; }
+	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> start(Database const& cx) override { return (clientId == 0) ? _start(cx) : Void(); }
+	Future<bool> check(Database const& cx) override { return true; }
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	ACTOR Future<Void> _start(Database cx) {
+		wait(setStorageQuotaHelper(cx, "name1"_sr, 100));
+		wait(setStorageQuotaHelper(cx, "name2"_sr, 200));
+		wait(setStorageQuotaHelper(cx, "name1"_sr, 300));
+
+		state Optional<uint64_t> quota1 = wait(getStorageQuotaHelper(cx, "name1"_sr));
+		ASSERT(quota1.present() && quota1.get() == 300);
+		state Optional<uint64_t> quota2 = wait(getStorageQuotaHelper(cx, "name2"_sr));
+		ASSERT(quota2.present() && quota2.get() == 200);
+		state Optional<uint64_t> quota3 = wait(getStorageQuotaHelper(cx, "name3"_sr));
+		ASSERT(!quota3.present());
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> setStorageQuotaHelper(Database cx, StringRef tenantName, uint64_t quota) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				setStorageQuota(tr, tenantName, quota);
+				wait(tr.commit());
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR static Future<Optional<uint64_t>> getStorageQuotaHelper(Database cx, StringRef tenantName) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				state Optional<uint64_t> quota = wait(getStorageQuota(&tr, tenantName));
+				wait(tr.commit());
+				return quota;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+};
+
+WorkloadFactory<StorageQuotaWorkload> StorageQuotaWorkloadFactory("StorageQuota");
diff --git a/fdbserver/workloads/StorageServerCheckpointRestoreTest.actor.cpp b/fdbserver/workloads/StorageServerCheckpointRestoreTest.actor.cpp
new file mode 100644
index 0000000000..691e929aaa
--- /dev/null
+++ b/fdbserver/workloads/StorageServerCheckpointRestoreTest.actor.cpp
@@ -0,0 +1,251 @@
+/*
+ *PhysicalShardMove.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbrpc/simulator.h"
+#include "fdbserver/IKeyValueStore.h"
+#include "fdbserver/ServerCheckpoint.actor.h"
+#include "fdbserver/MoveKeys.actor.h"
+#include "fdbserver/QuietDatabase.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/flow.h"
+#include <cstdint>
+#include <limits>
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+std::string printValue(const ErrorOr<Optional<Value>>& value) {
+	if (value.isError()) {
+		return value.getError().name();
+	}
+	return value.get().present() ? value.get().get().toString() : "Value Not Found.";
+}
+} // namespace
+
+struct SSCheckpointRestoreWorkload : TestWorkload {
+	const bool enabled;
+	bool pass;
+
+	SSCheckpointRestoreWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(!clientId), pass(true) {}
+
+	void validationFailed(ErrorOr<Optional<Value>> expectedValue, ErrorOr<Optional<Value>> actualValue) {
+		TraceEvent(SevError, "TestFailed")
+		    .detail("ExpectedValue", printValue(expectedValue))
+		    .detail("ActualValue", printValue(actualValue));
+		pass = false;
+	}
+
+	std::string description() const override { return "SSCheckpoint"; }
+
+	Future<Void> setup(Database const& cx) override { return Void(); }
+
+	Future<Void> start(Database const& cx) override {
+		if (!enabled) {
+			return Void();
+		}
+		return _start(this, cx);
+	}
+
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("MoveKeysWorkload"); }
+
+	ACTOR Future<Void> _start(SSCheckpointRestoreWorkload* self, Database cx) {
+		state Key key = "TestKey"_sr;
+		state Key endKey = "TestKey0"_sr;
+		state Value oldValue = "TestValue"_sr;
+		state KeyRange testRange = KeyRangeRef(key, endKey);
+		state std::vector<CheckpointMetaData> records;
+
+		int ignore = wait(setDDMode(cx, 0));
+		state Version version = wait(self->writeAndVerify(self, cx, key, oldValue));
+
+		// Create checkpoint.
+		state Transaction tr(cx);
+		state CheckpointFormat format = deterministicRandom()->coinflip() ? RocksDBColumnFamily : RocksDB;
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				wait(createCheckpoint(&tr, testRange, format));
+				wait(tr.commit());
+				version = tr.getCommittedVersion();
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		TraceEvent("TestCheckpointCreated").detail("Range", testRange).detail("Version", version);
+
+		// Fetch checkpoint meta data.
+		loop {
+			records.clear();
+			try {
+				std::vector<CheckpointMetaData> _records = wait(getCheckpointMetaData(cx, testRange, version, format));
+				records = std::move(_records);
+				break;
+			} catch (Error& e) {
+				TraceEvent("TestFetchCheckpointMetadataError")
+				    .errorUnsuppressed(e)
+				    .detail("Range", testRange)
+				    .detail("Version", version);
+
+				// The checkpoint was just created, we don't expect this error.
+				ASSERT(e.code() != error_code_checkpoint_not_found);
+			}
+		}
+
+		TraceEvent("TestCheckpointFetched")
+		    .detail("Range", testRange)
+		    .detail("Version", version)
+		    .detail("Checkpoints", describe(records));
+
+		state std::string pwd = platform::getWorkingDirectory();
+		state std::string folder = pwd + "/checkpoints";
+		platform::eraseDirectoryRecursive(folder);
+		ASSERT(platform::createDirectory(folder));
+
+		// Fetch checkpoint.
+		state std::vector<CheckpointMetaData> fetchedCheckpoints;
+		state int i = 0;
+		for (; i < records.size(); ++i) {
+			loop {
+				TraceEvent("TestFetchingCheckpoint").detail("Checkpoint", records[i].toString());
+				try {
+					state CheckpointMetaData record = wait(fetchCheckpoint(cx, records[0], folder));
+					fetchedCheckpoints.push_back(record);
+					TraceEvent("TestCheckpointFetched").detail("Checkpoint", record.toString());
+					break;
+				} catch (Error& e) {
+					TraceEvent("TestFetchCheckpointError")
+					    .errorUnsuppressed(e)
+					    .detail("Checkpoint", records[i].toString());
+					wait(delay(1));
+				}
+			}
+		}
+
+		state std::string rocksDBTestDir = "rocksdb-kvstore-test-db";
+		platform::eraseDirectoryRecursive(rocksDBTestDir);
+
+		// Restore KVS.
+		state IKeyValueStore* kvStore = keyValueStoreRocksDB(
+		    rocksDBTestDir, deterministicRandom()->randomUniqueID(), KeyValueStoreType::SSD_ROCKSDB_V1);
+		wait(kvStore->init());
+		try {
+			wait(kvStore->restore(fetchedCheckpoints));
+		} catch (Error& e) {
+			TraceEvent(SevError, "TestRestoreCheckpointError")
+			    .errorUnsuppressed(e)
+			    .detail("Checkpoint", describe(records));
+		}
+
+		// Compare the keyrange between the original database and the one restored from checkpoint.
+		// For now, it should have been a single key.
+		tr.reset();
+		state RangeResult res;
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				RangeResult _res = wait(tr.getRange(KeyRangeRef(key, endKey), CLIENT_KNOBS->TOO_MANY));
+				res = std::move(_res);
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		RangeResult kvRange = wait(kvStore->readRange(testRange));
+		ASSERT(res.size() == kvRange.size());
+		for (int i = 0; i < res.size(); ++i) {
+			ASSERT(res[i] == kvRange[i]);
+		}
+
+		Future<Void> close = kvStore->onClosed();
+		kvStore->dispose();
+		wait(close);
+
+		{
+			int ignore = wait(setDDMode(cx, 1));
+			(void)ignore;
+		}
+		return Void();
+	}
+
+	ACTOR Future<Void> readAndVerify(SSCheckpointRestoreWorkload* self,
+	                                 Database cx,
+	                                 Key key,
+	                                 ErrorOr<Optional<Value>> expectedValue) {
+		state Transaction tr(cx);
+
+		loop {
+			try {
+				state Optional<Value> res = wait(timeoutError(tr.get(key), 30.0));
+				const bool equal = !expectedValue.isError() && res == expectedValue.get();
+				if (!equal) {
+					self->validationFailed(expectedValue, ErrorOr<Optional<Value>>(res));
+				}
+				break;
+			} catch (Error& e) {
+				if (expectedValue.isError() && expectedValue.getError().code() == e.code()) {
+					break;
+				}
+				wait(tr.onError(e));
+			}
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<Version> writeAndVerify(SSCheckpointRestoreWorkload* self,
+	                                     Database cx,
+	                                     Key key,
+	                                     Optional<Value> value) {
+		state Transaction tr(cx);
+		state Version version;
+		loop {
+			try {
+				if (value.present()) {
+					tr.set(key, value.get());
+				} else {
+					tr.clear(key);
+				}
+				wait(timeoutError(tr.commit(), 30.0));
+				version = tr.getCommittedVersion();
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		wait(self->readAndVerify(self, cx, key, value));
+
+		return version;
+	}
+
+	Future<bool> check(Database const& cx) override { return pass; }
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<SSCheckpointRestoreWorkload> SSCheckpointRestoreWorkloadFactory("SSCheckpointRestoreWorkload");
\ No newline at end of file
diff --git a/fdbserver/workloads/Storefront.actor.cpp b/fdbserver/workloads/Storefront.actor.cpp
index cd036074c5..e1c2b72baa 100644
--- a/fdbserver/workloads/Storefront.actor.cpp
+++ b/fdbserver/workloads/Storefront.actor.cpp
@@ -43,15 +43,12 @@ struct StorefrontWorkload : TestWorkload {
 	StorefrontWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"),
 	    spuriousCommitFailures("Spurious Commit Failures"), totalLatency("Total Latency") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 1000.0);
-		actorCount =
-		    getOption(options, LiteralStringRef("actorsPerClient"), std::max((int)(transactionsPerSecond / 100), 1));
-		maxOrderSize = getOption(options, LiteralStringRef("maxOrderSize"), 20);
-		itemCount =
-		    getOption(options, LiteralStringRef("itemCount"), transactionsPerSecond * clientCount * maxOrderSize);
-		minExpectedTransactionsPerSecond =
-		    transactionsPerSecond * getOption(options, LiteralStringRef("expectedRate"), 0.9);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 1000.0);
+		actorCount = getOption(options, "actorsPerClient"_sr, std::max((int)(transactionsPerSecond / 100), 1));
+		maxOrderSize = getOption(options, "maxOrderSize"_sr, 20);
+		itemCount = getOption(options, "itemCount"_sr, transactionsPerSecond * clientCount * maxOrderSize);
+		minExpectedTransactionsPerSecond = transactionsPerSecond * getOption(options, "expectedRate"_sr, 0.9);
 	}
 
 	std::string description() const override { return "StorefrontWorkload"; }
@@ -202,8 +199,7 @@ struct StorefrontWorkload : TestWorkload {
 
 	ACTOR Future<bool> tableBalancer(Database cx, StorefrontWorkload* self, KeyRangeRef keyRange) {
 		state std::vector<Future<std::vector<int>>> accumulators;
-		accumulators.push_back(
-		    self->orderAccumulator(cx, self, KeyRangeRef(LiteralStringRef("/orders/f"), LiteralStringRef("/orders0"))));
+		accumulators.push_back(self->orderAccumulator(cx, self, KeyRangeRef("/orders/f"_sr, "/orders0"_sr)));
 		for (int c = 0; c < 15; c++)
 			accumulators.push_back(self->orderAccumulator(
 			    cx, self, KeyRangeRef(Key(format("/orders/%x", c)), Key(format("/orders/%x", c + 1)))));
diff --git a/fdbserver/workloads/StreamingRead.actor.cpp b/fdbserver/workloads/StreamingRead.actor.cpp
index 71b3df8bbb..891c835d2e 100644
--- a/fdbserver/workloads/StreamingRead.actor.cpp
+++ b/fdbserver/workloads/StreamingRead.actor.cpp
@@ -40,17 +40,17 @@ struct StreamingReadWorkload : TestWorkload {
 	StreamingReadWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), readKeys("Keys Read"), readValueBytes("Value Bytes Read"),
 	    latencies(2000) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		actorCount = getOption(options, LiteralStringRef("actorCount"), 20);
-		readsPerTransaction = getOption(options, LiteralStringRef("readsPerTransaction"), 10);
-		rangesPerTransaction = getOption(options, LiteralStringRef("rangesPerTransaction"), 1);
-		nodeCount = getOption(options, LiteralStringRef("nodeCount"), 100000);
-		keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 16);
-		valueBytes = std::max(getOption(options, LiteralStringRef("valueBytes"), 96), 16);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		actorCount = getOption(options, "actorCount"_sr, 20);
+		readsPerTransaction = getOption(options, "readsPerTransaction"_sr, 10);
+		rangesPerTransaction = getOption(options, "rangesPerTransaction"_sr, 1);
+		nodeCount = getOption(options, "nodeCount"_sr, 100000);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
+		valueBytes = std::max(getOption(options, "valueBytes"_sr, 96), 16);
 		std::string valueFormat = "%016llx" + std::string(valueBytes - 16, '.');
-		warmingDelay = getOption(options, LiteralStringRef("warmingDelay"), 0.0);
+		warmingDelay = getOption(options, "warmingDelay"_sr, 0.0);
 		constantValue = Value(format(valueFormat.c_str(), 42));
-		readSequentially = getOption(options, LiteralStringRef("readSequentially"), false);
+		readSequentially = getOption(options, "readSequentially"_sr, false);
 	}
 
 	std::string description() const override { return "StreamingRead"; }
diff --git a/fdbserver/workloads/SubmitBackup.actor.cpp b/fdbserver/workloads/SubmitBackup.actor.cpp
index aa4dd13d9b..0e48ff4e53 100644
--- a/fdbserver/workloads/SubmitBackup.actor.cpp
+++ b/fdbserver/workloads/SubmitBackup.actor.cpp
@@ -23,6 +23,7 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -39,13 +40,13 @@ struct SubmitBackupWorkload final : TestWorkload {
 	IncrementalBackupOnly incremental{ false };
 
 	SubmitBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		backupDir = getOption(options, LiteralStringRef("backupDir"), LiteralStringRef("file://simfdb/backups/"));
-		tag = getOption(options, LiteralStringRef("tag"), LiteralStringRef("default"));
-		delayFor = getOption(options, LiteralStringRef("delayFor"), 10.0);
-		initSnapshotInterval = getOption(options, LiteralStringRef("initSnapshotInterval"), 0);
-		snapshotInterval = getOption(options, LiteralStringRef("snapshotInterval"), 1e8);
-		stopWhenDone.set(getOption(options, LiteralStringRef("stopWhenDone"), true));
-		incremental.set(getOption(options, LiteralStringRef("incremental"), false));
+		backupDir = getOption(options, "backupDir"_sr, "file://simfdb/backups/"_sr);
+		tag = getOption(options, "tag"_sr, "default"_sr);
+		delayFor = getOption(options, "delayFor"_sr, 10.0);
+		initSnapshotInterval = getOption(options, "initSnapshotInterval"_sr, 0);
+		snapshotInterval = getOption(options, "snapshotInterval"_sr, 1e8);
+		stopWhenDone.set(getOption(options, "stopWhenDone"_sr, true));
+		incremental.set(getOption(options, "incremental"_sr, false));
 	}
 
 	static constexpr const char* DESCRIPTION = "SubmitBackup";
@@ -53,7 +54,7 @@ struct SubmitBackupWorkload final : TestWorkload {
 	ACTOR static Future<Void> _start(SubmitBackupWorkload* self, Database cx) {
 		wait(delay(self->delayFor));
 		Standalone<VectorRef<KeyRangeRef>> backupRanges;
-		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
+		addDefaultBackupRanges(backupRanges);
 		try {
 			wait(self->backupAgent.submitBackup(cx,
 			                                    self->backupDir,
@@ -62,6 +63,7 @@ struct SubmitBackupWorkload final : TestWorkload {
 			                                    self->snapshotInterval,
 			                                    self->tag.toString(),
 			                                    backupRanges,
+			                                    false,
 			                                    self->stopWhenDone,
 			                                    UsePartitionedLog::False,
 			                                    self->incremental));
diff --git a/fdbserver/workloads/SuspendProcesses.actor.cpp b/fdbserver/workloads/SuspendProcesses.actor.cpp
index ce1a5c05e2..7b24b11eeb 100644
--- a/fdbserver/workloads/SuspendProcesses.actor.cpp
+++ b/fdbserver/workloads/SuspendProcesses.actor.cpp
@@ -34,10 +34,9 @@ struct SuspendProcessesWorkload : TestWorkload {
 	double waitTimeDuration;
 
 	SuspendProcessesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		prefixSuspendProcesses =
-		    getOption(options, LiteralStringRef("prefixesSuspendProcesses"), std::vector<std::string>());
-		waitTimeDuration = getOption(options, LiteralStringRef("waitTimeDuration"), 0);
-		suspendTimeDuration = getOption(options, LiteralStringRef("suspendTimeDuration"), 0);
+		prefixSuspendProcesses = getOption(options, "prefixesSuspendProcesses"_sr, std::vector<std::string>());
+		waitTimeDuration = getOption(options, "waitTimeDuration"_sr, 0);
+		suspendTimeDuration = getOption(options, "suspendTimeDuration"_sr, 0);
 	}
 
 	std::string description() const override { return "SuspendProcesses"; }
@@ -51,16 +50,14 @@ struct SuspendProcessesWorkload : TestWorkload {
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				RangeResult kvs = wait(tr.getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
-				                                               LiteralStringRef("\xff\xff/worker_interfaces0")),
-				                                   CLIENT_KNOBS->TOO_MANY));
+				RangeResult kvs =
+				    wait(tr.getRange(KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr),
+				                     CLIENT_KNOBS->TOO_MANY));
 				ASSERT(!kvs.more);
 				std::vector<Standalone<StringRef>> suspendProcessInterfaces;
 				for (auto it : kvs) {
-					auto ip_port =
-					    (it.key.endsWith(LiteralStringRef(":tls")) ? it.key.removeSuffix(LiteralStringRef(":tls"))
-					                                               : it.key)
-					        .removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
+					auto ip_port = (it.key.endsWith(":tls"_sr) ? it.key.removeSuffix(":tls"_sr) : it.key)
+					                   .removePrefix("\xff\xff/worker_interfaces/"_sr);
 					for (auto& killProcess : self->prefixSuspendProcesses) {
 						if (boost::starts_with(ip_port.toString().c_str(), killProcess.c_str())) {
 							suspendProcessInterfaces.push_back(it.value);
diff --git a/fdbserver/workloads/TagThrottleApi.actor.cpp b/fdbserver/workloads/TagThrottleApi.actor.cpp
index b24ead205f..b1b06d2f8f 100644
--- a/fdbserver/workloads/TagThrottleApi.actor.cpp
+++ b/fdbserver/workloads/TagThrottleApi.actor.cpp
@@ -33,7 +33,7 @@ struct TagThrottleApiWorkload : TestWorkload {
 	constexpr static const char* NAME = "TagThrottleApi";
 
 	TagThrottleApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		autoThrottleEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
 	}
 
diff --git a/fdbserver/workloads/TargetedKill.actor.cpp b/fdbserver/workloads/TargetedKill.actor.cpp
index f9c39e499a..2ba29b814f 100644
--- a/fdbserver/workloads/TargetedKill.actor.cpp
+++ b/fdbserver/workloads/TargetedKill.actor.cpp
@@ -39,12 +39,12 @@ struct TargetedKillWorkload : TestWorkload {
 
 	TargetedKillWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
-		killAt = getOption(options, LiteralStringRef("killAt"), 5.0);
-		reboot = getOption(options, LiteralStringRef("reboot"), false);
-		suspendDuration = getOption(options, LiteralStringRef("suspendDuration"), 1.0);
-		machineToKill = getOption(options, LiteralStringRef("machineToKill"), LiteralStringRef("master")).toString();
-		killAllMachineProcesses = getOption(options, LiteralStringRef("killWholeMachine"), false);
-		numKillStorages = getOption(options, LiteralStringRef("numKillStorages"), 1);
+		killAt = getOption(options, "killAt"_sr, 5.0);
+		reboot = getOption(options, "reboot"_sr, false);
+		suspendDuration = getOption(options, "suspendDuration"_sr, 1.0);
+		machineToKill = getOption(options, "machineToKill"_sr, "master"_sr).toString();
+		killAllMachineProcesses = getOption(options, "killWholeMachine"_sr, false);
+		numKillStorages = getOption(options, "numKillStorages"_sr, 1);
 	}
 
 	std::string description() const override { return "TargetedKillWorkload"; }
@@ -61,8 +61,8 @@ struct TargetedKillWorkload : TestWorkload {
 	                          NetworkAddress address,
 	                          Database cx,
 	                          TargetedKillWorkload* self) {
-		if (&g_simulator == g_network) {
-			g_simulator.killInterface(address, ISimulator::KillInstantly);
+		if (g_simulator == g_network) {
+			g_simulator->killInterface(address, ISimulator::KillInstantly);
 			return Void();
 		}
 
diff --git a/fdbserver/workloads/TaskBucketCorrectness.actor.cpp b/fdbserver/workloads/TaskBucketCorrectness.actor.cpp
index 7a14a29652..1df20d1ca0 100644
--- a/fdbserver/workloads/TaskBucketCorrectness.actor.cpp
+++ b/fdbserver/workloads/TaskBucketCorrectness.actor.cpp
@@ -71,7 +71,7 @@ struct SayHelloTaskFunc : TaskFuncBase {
 
 		state Key key = StringRef("Hello_" + deterministicRandom()->randomUniqueID().toString());
 		state Key value;
-		auto itor = task->params.find(LiteralStringRef("name"));
+		auto itor = task->params.find("name"_sr);
 		if (itor != task->params.end()) {
 			value = itor->value;
 			TraceEvent("TaskBucketCorrectnessSayHello").detail("SayHelloTaskFunc", printable(itor->value));
@@ -79,11 +79,11 @@ struct SayHelloTaskFunc : TaskFuncBase {
 			ASSERT(false);
 		}
 
-		if (!task->params[LiteralStringRef("chained")].compare(LiteralStringRef("false"))) {
+		if (!task->params["chained"_sr].compare("false"_sr)) {
 			wait(done->set(tr, taskBucket));
 		} else {
-			int subtaskCount = atoi(task->params[LiteralStringRef("subtaskCount")].toString().c_str());
-			int currTaskNumber = atoi(value.removePrefix(LiteralStringRef("task_")).toString().c_str());
+			int subtaskCount = atoi(task->params["subtaskCount"_sr].toString().c_str());
+			int currTaskNumber = atoi(value.removePrefix("task_"_sr).toString().c_str());
 			TraceEvent("TaskBucketCorrectnessSayHello")
 			    .detail("SubtaskCount", subtaskCount)
 			    .detail("CurrTaskNumber", currTaskNumber);
@@ -94,9 +94,9 @@ struct SayHelloTaskFunc : TaskFuncBase {
 				                                    SayHelloTaskFunc::version,
 				                                    StringRef(),
 				                                    deterministicRandom()->randomInt(0, 2));
-				new_task->params[LiteralStringRef("name")] = StringRef(format("task_%d", currTaskNumber + 1));
-				new_task->params[LiteralStringRef("chained")] = task->params[LiteralStringRef("chained")];
-				new_task->params[LiteralStringRef("subtaskCount")] = task->params[LiteralStringRef("subtaskCount")];
+				new_task->params["name"_sr] = StringRef(format("task_%d", currTaskNumber + 1));
+				new_task->params["chained"_sr] = task->params["chained"_sr];
+				new_task->params["subtaskCount"_sr] = task->params["subtaskCount"_sr];
 				Reference<TaskFuture> taskDone = futureBucket->future(tr);
 				new_task->params[Task::reservedTaskParamKeyDone] = taskDone->key;
 				taskBucket->addTask(tr, new_task);
@@ -112,7 +112,7 @@ struct SayHelloTaskFunc : TaskFuncBase {
 		return Void();
 	}
 };
-StringRef SayHelloTaskFunc::name = LiteralStringRef("SayHello");
+StringRef SayHelloTaskFunc::name = "SayHello"_sr;
 REGISTER_TASKFUNC(SayHelloTaskFunc);
 
 struct SayHelloToEveryoneTaskFunc : TaskFuncBase {
@@ -141,15 +141,15 @@ struct SayHelloToEveryoneTaskFunc : TaskFuncBase {
 		state std::vector<Reference<TaskFuture>> vectorFuture;
 
 		int subtaskCount = 1;
-		if (!task->params[LiteralStringRef("chained")].compare(LiteralStringRef("false"))) {
-			subtaskCount = atoi(task->params[LiteralStringRef("subtaskCount")].toString().c_str());
+		if (!task->params["chained"_sr].compare("false"_sr)) {
+			subtaskCount = atoi(task->params["subtaskCount"_sr].toString().c_str());
 		}
 		for (int i = 0; i < subtaskCount; ++i) {
 			auto new_task = makeReference<Task>(
 			    SayHelloTaskFunc::name, SayHelloTaskFunc::version, StringRef(), deterministicRandom()->randomInt(0, 2));
-			new_task->params[LiteralStringRef("name")] = StringRef(format("task_%d", i));
-			new_task->params[LiteralStringRef("chained")] = task->params[LiteralStringRef("chained")];
-			new_task->params[LiteralStringRef("subtaskCount")] = task->params[LiteralStringRef("subtaskCount")];
+			new_task->params["name"_sr] = StringRef(format("task_%d", i));
+			new_task->params["chained"_sr] = task->params["chained"_sr];
+			new_task->params["subtaskCount"_sr] = task->params["subtaskCount"_sr];
 			Reference<TaskFuture> taskDone = futureBucket->future(tr);
 			new_task->params[Task::reservedTaskParamKeyDone] = taskDone->key;
 			taskBucket->addTask(tr, new_task);
@@ -160,14 +160,14 @@ struct SayHelloToEveryoneTaskFunc : TaskFuncBase {
 		wait(taskBucket->finish(tr, task));
 
 		Key key = StringRef("Hello_" + deterministicRandom()->randomUniqueID().toString());
-		Value value = LiteralStringRef("Hello, Everyone!");
+		Value value = "Hello, Everyone!"_sr;
 		TraceEvent("TaskBucketCorrectnessSayHello").detail("SayHelloToEveryoneTaskFunc", printable(value));
 		tr->set(key, value);
 
 		return Void();
 	}
 };
-StringRef SayHelloToEveryoneTaskFunc::name = LiteralStringRef("SayHelloToEveryone");
+StringRef SayHelloToEveryoneTaskFunc::name = "SayHelloToEveryone"_sr;
 REGISTER_TASKFUNC(SayHelloToEveryoneTaskFunc);
 
 struct SaidHelloTaskFunc : TaskFuncBase {
@@ -195,14 +195,14 @@ struct SaidHelloTaskFunc : TaskFuncBase {
 		wait(taskBucket->finish(tr, task));
 
 		Key key = StringRef("Hello_" + deterministicRandom()->randomUniqueID().toString());
-		Value value = LiteralStringRef("Said hello to everyone!");
+		Value value = "Said hello to everyone!"_sr;
 		TraceEvent("TaskBucketCorrectnessSayHello").detail("SaidHelloTaskFunc", printable(value));
 		tr->set(key, value);
 
 		return Void();
 	}
 };
-StringRef SaidHelloTaskFunc::name = LiteralStringRef("SaidHello");
+StringRef SaidHelloTaskFunc::name = "SaidHello"_sr;
 REGISTER_TASKFUNC(SaidHelloTaskFunc);
 
 // A workload which test the correctness of TaskBucket
@@ -211,8 +211,8 @@ struct TaskBucketCorrectnessWorkload : TestWorkload {
 	int subtaskCount;
 
 	TaskBucketCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		chained = getOption(options, LiteralStringRef("chained"), false);
-		subtaskCount = getOption(options, LiteralStringRef("subtaskCount"), 20);
+		chained = getOption(options, "chained"_sr, false);
+		subtaskCount = getOption(options, "subtaskCount"_sr, 20);
 	}
 
 	std::string description() const override { return "TaskBucketCorrectness"; }
@@ -228,11 +228,11 @@ struct TaskBucketCorrectnessWorkload : TestWorkload {
 	                                Reference<FutureBucket> futureBucket,
 	                                bool chained,
 	                                int subtaskCount) {
-		state Key addedInitKey(LiteralStringRef("addedInitTasks"));
+		state Key addedInitKey("addedInitTasks"_sr);
 		Optional<Standalone<StringRef>> res = wait(tr->get(addedInitKey));
 		if (res.present())
 			return Void();
-		tr->set(addedInitKey, LiteralStringRef("true"));
+		tr->set(addedInitKey, "true"_sr);
 
 		Reference<TaskFuture> allDone = futureBucket->future(tr);
 		auto task = makeReference<Task>(SayHelloToEveryoneTaskFunc::name,
@@ -240,8 +240,8 @@ struct TaskBucketCorrectnessWorkload : TestWorkload {
 		                                allDone->key,
 		                                deterministicRandom()->randomInt(0, 2));
 
-		task->params[LiteralStringRef("chained")] = chained ? LiteralStringRef("true") : LiteralStringRef("false");
-		task->params[LiteralStringRef("subtaskCount")] = StringRef(format("%d", subtaskCount));
+		task->params["chained"_sr] = chained ? "true"_sr : "false"_sr;
+		task->params["subtaskCount"_sr] = StringRef(format("%d", subtaskCount));
 		taskBucket->addTask(tr, task);
 		auto taskDone = makeReference<Task>(
 		    SaidHelloTaskFunc::name, SaidHelloTaskFunc::version, StringRef(), deterministicRandom()->randomInt(0, 2));
@@ -251,9 +251,9 @@ struct TaskBucketCorrectnessWorkload : TestWorkload {
 
 	ACTOR Future<Void> _start(Database cx, TaskBucketCorrectnessWorkload* self) {
 		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state Subspace taskSubspace(LiteralStringRef("backup-agent"));
-		state Reference<TaskBucket> taskBucket(new TaskBucket(taskSubspace.get(LiteralStringRef("tasks"))));
-		state Reference<FutureBucket> futureBucket(new FutureBucket(taskSubspace.get(LiteralStringRef("futures"))));
+		state Subspace taskSubspace("backup-agent"_sr);
+		state Reference<TaskBucket> taskBucket(new TaskBucket(taskSubspace.get("tasks"_sr)));
+		state Reference<FutureBucket> futureBucket(new FutureBucket(taskSubspace.get("futures"_sr)));
 
 		try {
 			if (self->clientId == 0) {
@@ -319,8 +319,7 @@ struct TaskBucketCorrectnessWorkload : TestWorkload {
 			data.insert(format("task_%d", i));
 		}
 
-		RangeResult values = wait(tr->getRange(
-		    KeyRangeRef(LiteralStringRef("Hello_\x00"), LiteralStringRef("Hello_\xff")), CLIENT_KNOBS->TOO_MANY));
+		RangeResult values = wait(tr->getRange(KeyRangeRef("Hello_\x00"_sr, "Hello_\xff"_sr), CLIENT_KNOBS->TOO_MANY));
 		if (values.size() != data.size()) {
 			TraceEvent(SevError, "CheckSayHello")
 			    .detail("CountNotMatchIs", values.size())
@@ -355,56 +354,53 @@ TEST_CASE("/fdbclient/TaskBucket/Subspace") {
 	print_subspace_key(subspace_test, 0);
 	ASSERT(subspace_test.key().toString() == "");
 
-	Subspace subspace_test1(LiteralStringRef("abc"));
+	Subspace subspace_test1("abc"_sr);
 	print_subspace_key(subspace_test1, 1);
-	ASSERT(subspace_test1.key() == LiteralStringRef("abc"));
+	ASSERT(subspace_test1.key() == "abc"_sr);
 
-	Tuple t;
-	t.append(LiteralStringRef("user"));
+	Tuple t = Tuple::makeTuple("user"_sr);
 	Subspace subspace_test2(t);
 	print_subspace_key(subspace_test2, 2);
-	ASSERT(subspace_test2.key() == LiteralStringRef("\x01user\x00"));
+	ASSERT(subspace_test2.key() == "\x01user\x00"_sr);
 
-	Subspace subspace_test3(t, LiteralStringRef("abc"));
+	Subspace subspace_test3(t, "abc"_sr);
 	print_subspace_key(subspace_test3, 3);
-	ASSERT(subspace_test3.key() == LiteralStringRef("abc\x01user\x00"));
+	ASSERT(subspace_test3.key() == "abc\x01user\x00"_sr);
 
-	Tuple t1;
-	t1.append(1);
+	Tuple t1 = Tuple::makeTuple(1);
 	Subspace subspace_test4(t1);
 	print_subspace_key(subspace_test4, 4);
-	ASSERT(subspace_test4.key() == LiteralStringRef("\x15\x01"));
+	ASSERT(subspace_test4.key() == "\x15\x01"_sr);
 
 	t.append(123);
-	Subspace subspace_test5(t, LiteralStringRef("abc"));
+	Subspace subspace_test5(t, "abc"_sr);
 	print_subspace_key(subspace_test5, 5);
-	ASSERT(subspace_test5.key() == LiteralStringRef("abc\x01user\x00\x15\x7b"));
+	ASSERT(subspace_test5.key() == "abc\x01user\x00\x15\x7b"_sr);
 
 	// Subspace pack
 	printf("%d==========%s===%d\n", 6, printable(subspace_test5.pack(t)).c_str(), subspace_test5.pack(t).size());
-	ASSERT(subspace_test5.pack(t) == LiteralStringRef("abc\x01user\x00\x15\x7b\x01user\x00\x15\x7b"));
+	ASSERT(subspace_test5.pack(t) == "abc\x01user\x00\x15\x7b\x01user\x00\x15\x7b"_sr);
 
 	printf("%d==========%s===%d\n", 7, printable(subspace_test5.pack(t1)).c_str(), subspace_test5.pack(t1).size());
-	ASSERT(subspace_test5.pack(t1) == LiteralStringRef("abc\x01user\x00\x15\x7b\x15\x01"));
+	ASSERT(subspace_test5.pack(t1) == "abc\x01user\x00\x15\x7b\x15\x01"_sr);
 
 	// Subspace getItem
 	Subspace subspace_test6(t);
-	Subspace subspace_test7 = subspace_test6.get(LiteralStringRef("subitem"));
+	Subspace subspace_test7 = subspace_test6.get("subitem"_sr);
 	print_subspace_key(subspace_test7, 8);
-	ASSERT(subspace_test7.key() == LiteralStringRef("\x01user\x00\x15\x7b\x01subitem\x00"));
+	ASSERT(subspace_test7.key() == "\x01user\x00\x15\x7b\x01subitem\x00"_sr);
 
 	// Subspace unpack
 	Tuple t2 = subspace_test6.unpack(subspace_test7.key());
 	Subspace subspace_test8(t2);
 	print_subspace_key(subspace_test8, 9);
-	ASSERT(subspace_test8.key() == LiteralStringRef("\x01subitem\x00"));
+	ASSERT(subspace_test8.key() == "\x01subitem\x00"_sr);
 
 	// pack
-	Tuple t3;
-	t3.append(StringRef());
+	Tuple t3 = Tuple::makeTuple(""_sr);
 	printf("%d==========%s===%d\n", 10, printable(subspace_test5.pack(t3)).c_str(), subspace_test5.pack(t3).size());
 	ASSERT(subspace_test5.pack(t3) == subspace_test5.pack(StringRef()));
-	ASSERT(subspace_test5.pack(t3) == LiteralStringRef("abc\x01user\x00\x15\x7b\x01\x00"));
+	ASSERT(subspace_test5.pack(t3) == "abc\x01user\x00\x15\x7b\x01\x00"_sr);
 
 	printf("%d==========%s===%d\n",
 	       11,
@@ -417,14 +413,14 @@ TEST_CASE("/fdbclient/TaskBucket/Subspace") {
 	       subspace_test5.range(t3).end.size());
 	ASSERT(subspace_test5.range(t3).end == subspace_test5.get(StringRef()).range().end);
 
-	StringRef def = LiteralStringRef("def");
-	StringRef ghi = LiteralStringRef("ghi");
+	StringRef def = "def"_sr;
+	StringRef ghi = "ghi"_sr;
 	t3.append(def);
 	t3.append(ghi);
 	printf("%d==========%s===%d\n", 13, printable(subspace_test5.pack(t3)).c_str(), subspace_test5.pack(t3).size());
 	ASSERT(subspace_test5.pack(t3) == subspace_test5.get(StringRef()).get(def).pack(ghi));
-	ASSERT(subspace_test5.pack(t3) == LiteralStringRef("abc\x01user\x00\x15\x7b\x01\x00\x01"
-	                                                   "def\x00\x01ghi\x00"));
+	ASSERT(subspace_test5.pack(t3) == "abc\x01user\x00\x15\x7b\x01\x00\x01"
+	                                  "def\x00\x01ghi\x00"_sr);
 
 	printf("%d==========%s===%d\n",
 	       14,
diff --git a/fdbserver/workloads/TenantEntryCacheWorkload.actor.cpp b/fdbserver/workloads/TenantEntryCacheWorkload.actor.cpp
new file mode 100644
index 0000000000..07b716073b
--- /dev/null
+++ b/fdbserver/workloads/TenantEntryCacheWorkload.actor.cpp
@@ -0,0 +1,324 @@
+/*
+ * TenantEntryCacheWorkload.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/TenantManagement.actor.h"
+
+#include "fdbclient/Knobs.h"
+#include "fdbclient/TenantEntryCache.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+TenantEntryCachePayload<int64_t> createPayload(const TenantName& name, const TenantMapEntry& entry) {
+	TenantEntryCachePayload<int64_t> payload;
+	payload.name = name;
+	payload.entry = entry;
+	payload.payload = entry.id;
+
+	return payload;
+}
+} // namespace
+
+struct TenantEntryCacheWorkload : TestWorkload {
+	const TenantName tenantNamePrefix = "tenant_entry_cache_workload_"_sr;
+	TenantName localTenantNamePrefix;
+	int maxTenants;
+	int clientId;
+
+	TenantEntryCacheWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		clientId = wcx.clientId;
+		maxTenants = std::max(3, std::min<int>(1e8 - 1, getOption(options, "maxTenants"_sr, 1000)));
+		localTenantNamePrefix = format("%stenant_%d_", tenantNamePrefix.toString().c_str(), clientId);
+	}
+	~TenantEntryCacheWorkload() {}
+
+	static void compareTenants(Optional<TenantEntryCachePayload<int64_t>> left, TenantMapEntry& right) {
+		ASSERT(left.present());
+		ASSERT_EQ(left.get().entry.id, right.id);
+		ASSERT_EQ(left.get().entry.prefix.compare(right.prefix), 0);
+		ASSERT_EQ(left.get().payload, right.id);
+	}
+
+	ACTOR static Future<Void> compareContents(std::vector<std::pair<TenantName, TenantMapEntry>>* tenants,
+	                                          Reference<TenantEntryCache<int64_t>> cache) {
+		state int i;
+		for (i = 0; i < tenants->size(); i++) {
+			if (deterministicRandom()->coinflip()) {
+				Optional<TenantEntryCachePayload<int64_t>> e = wait(cache->getById(tenants->at(i).second.id));
+				compareTenants(e, tenants->at(i).second);
+			} else {
+				Optional<TenantEntryCachePayload<int64_t>> e = wait(cache->getByName(tenants->at(i).first));
+				compareTenants(e, tenants->at(i).second);
+			}
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> testTenantNotFound(Database cx) {
+		state Reference<TenantEntryCache<int64_t>> cache = makeReference<TenantEntryCache<int64_t>>(cx, createPayload);
+		TraceEvent("TenantNotFoundStart");
+
+		wait(cache->init());
+		// Ensure associated counter values gets updated
+		ASSERT_EQ(cache->numRefreshByInit(), 1);
+
+		state TenantMapEntry dummy(std::numeric_limits<int64_t>::max(), TenantState::READY, true /* encrypted */);
+		Optional<TenantEntryCachePayload<int64_t>> value = wait(cache->getById(dummy.id));
+		ASSERT(!value.present());
+
+		Optional<TenantEntryCachePayload<int64_t>> value1 = wait(cache->getByPrefix(dummy.prefix));
+		ASSERT(!value1.present());
+
+		// Ensure associated counter values gets updated
+		ASSERT_EQ(cache->numRefreshByMisses(), 2);
+
+		TraceEvent("TenantNotFoundEnd");
+		return Void();
+	}
+
+	ACTOR static Future<Void> testCreateTenantsAndLookup(
+	    Database cx,
+	    TenantEntryCacheWorkload* self,
+	    std::vector<std::pair<TenantName, TenantMapEntry>>* tenantList) {
+		state Reference<TenantEntryCache<int64_t>> cache = makeReference<TenantEntryCache<int64_t>>(cx, createPayload);
+		state int nTenants = deterministicRandom()->randomInt(5, self->maxTenants);
+
+		TraceEvent("CreateTenantsAndLookupStart");
+
+		wait(cache->init());
+		// Ensure associated counter values gets updated
+		ASSERT_EQ(cache->numRefreshByInit(), 1);
+		ASSERT_GE(cache->numCacheRefreshes(), 1);
+
+		tenantList->clear();
+		state int i = 0;
+		state std::unordered_set<TenantName> tenantNames;
+		while (i < nTenants) {
+			state TenantName name(format("%s%08d",
+			                             self->localTenantNamePrefix.toString().c_str(),
+			                             deterministicRandom()->randomInt(0, self->maxTenants)));
+			if (tenantNames.find(name) != tenantNames.end()) {
+				continue;
+			}
+
+			Optional<TenantMapEntry> entry = wait(TenantAPI::createTenant(cx.getReference(), StringRef(name)));
+			ASSERT(entry.present());
+			tenantList->emplace_back(std::make_pair(name, entry.get()));
+			tenantNames.emplace(name);
+			i++;
+		}
+
+		wait(compareContents(tenantList, cache));
+
+		TraceEvent("CreateTenantsAndLookupEnd");
+		return Void();
+	}
+
+	ACTOR static Future<Void> testTenantInsert(Database cx,
+	                                           TenantEntryCacheWorkload* self,
+	                                           std::vector<std::pair<TenantName, TenantMapEntry>>* tenantList) {
+		state Reference<TenantEntryCache<int64_t>> cache = makeReference<TenantEntryCache<int64_t>>(cx, createPayload);
+
+		ASSERT(!tenantList->empty() && tenantList->size() >= 2);
+
+		TraceEvent("TestTenantInsertStart");
+
+		wait(cache->init());
+		// Ensure associated counter values gets updated
+		ASSERT_EQ(cache->numRefreshByInit(), 1);
+		ASSERT_GE(cache->numCacheRefreshes(), 1);
+
+		state std::pair<TenantName, TenantMapEntry> p = tenantList->at(0);
+		state Optional<TenantEntryCachePayload<int64_t>> entry;
+
+		// Tenant rename
+		p.first = TenantName(format("%s%08d",
+		                            self->localTenantNamePrefix.toString().c_str(),
+		                            deterministicRandom()->randomInt(self->maxTenants + 100, self->maxTenants + 200)));
+		cache->put(p);
+		Optional<TenantEntryCachePayload<int64_t>> e = wait(cache->getByName(p.first));
+		entry = e;
+		compareTenants(entry, p.second);
+
+		// Tenant delete & recreate
+		p.second.id = p.second.id + deterministicRandom()->randomInt(self->maxTenants + 500, self->maxTenants + 700);
+		cache->put(p);
+		Optional<TenantEntryCachePayload<int64_t>> e1 = wait(cache->getById(p.second.id));
+		entry = e1;
+		compareTenants(entry, p.second);
+		ASSERT_EQ(p.first.contents().toString().compare(entry.get().name.contents().toString()), 0);
+
+		// Delete a tenant and rename an existing TenantEntry to reuse the name of deleted tenant
+		state std::pair<TenantName, TenantMapEntry> p1 = tenantList->back();
+		tenantList->pop_back();
+		wait(TenantAPI::deleteTenant(cx.getReference(), p1.first));
+		cache->put(std::make_pair(p1.first, p.second));
+		Optional<TenantEntryCachePayload<int64_t>> e2 = wait(cache->getById(p.second.id));
+		entry = e2;
+		compareTenants(entry, p.second);
+		ASSERT_EQ(p1.first.contents().toString().compare(entry.get().name.contents().toString()), 0);
+
+		TraceEvent("TestTenantInsertEnd");
+		return Void();
+	}
+
+	ACTOR static Future<Void> testCacheReload(Database cx,
+	                                          std::vector<std::pair<TenantName, TenantMapEntry>>* tenantList) {
+		state Reference<TenantEntryCache<int64_t>> cache = makeReference<TenantEntryCache<int64_t>>(cx, createPayload);
+
+		ASSERT(!tenantList->empty());
+
+		TraceEvent("CacheReloadStart");
+
+		wait(cache->init());
+		// Ensure associated counter values gets updated
+		ASSERT_EQ(cache->numRefreshByInit(), 1);
+		ASSERT_GE(cache->numCacheRefreshes(), 1);
+
+		wait(compareContents(tenantList, cache));
+
+		TraceEvent("CacheReloadEnd");
+		return Void();
+	}
+
+	ACTOR static Future<Void> testTenantCacheDefaultFunc(Database cx) {
+		wait(delay(0.0));
+		return Void();
+	}
+
+	ACTOR static Future<Void> testCacheRefresh(Database cx) {
+		state Reference<TenantEntryCache<int64_t>> cache = makeReference<TenantEntryCache<int64_t>>(cx, createPayload);
+
+		TraceEvent("TestCacheRefreshStart");
+
+		wait(cache->init());
+		// Ensure associated counter values gets updated
+		ASSERT_EQ(cache->numRefreshByInit(), 1);
+		ASSERT_GE(cache->numCacheRefreshes(), 1);
+
+		// Fault injection may cause delays in cluster recovery/availability, spin a loop to let cache refresh to
+		// trigger with a max wait of 5 mins; timed_out error is thrown if cache refresh isn't triggered.
+
+		state int64_t startTime = now();
+		state int64_t waitUntill = startTime + 300; // 5 mins max wait
+		loop {
+			// InitRefresh + multiple timer based invocations (at least 2 invocations of cache->refresh())
+			if (cache->numCacheRefreshes() >= 2) {
+				break;
+			}
+
+			if (now() > waitUntill) {
+				throw timed_out();
+			}
+
+			TraceEvent("TestCacheRefreshWait").detail("Elapsed", now() - startTime);
+			wait(delay(CLIENT_KNOBS->TENANT_ENTRY_CACHE_LIST_REFRESH_INTERVAL));
+		}
+
+		TraceEvent("TestCacheRefreshEnd").detail("ElapsedTotal", now() - startTime);
+		return Void();
+	}
+
+	ACTOR static Future<Void> tenantEntryRemove(Database cx,
+	                                            std::vector<std::pair<TenantName, TenantMapEntry>>* tenantList) {
+		state Reference<TenantEntryCache<int64_t>> cache = makeReference<TenantEntryCache<int64_t>>(
+		    cx, deterministicRandom()->randomUniqueID(), createPayload, TenantEntryCacheRefreshMode::NONE);
+
+		wait(cache->init());
+
+		ASSERT(!tenantList->empty());
+
+		// Remove an entry from the cache
+		state int idx = deterministicRandom()->randomInt(0, tenantList->size());
+		Optional<TenantEntryCachePayload<int64_t>> entry = wait(cache->getByName(tenantList->at(idx).first));
+		ASSERT(entry.present());
+
+		TraceEvent("TestTenantEntryRemoveStart")
+		    .detail("Id", tenantList->at(idx).second.id)
+		    .detail("Name", tenantList->at(idx).first)
+		    .detail("Prefix", tenantList->at(idx).second.prefix);
+
+		wait(TenantAPI::deleteTenant(cx.getReference(), tenantList->at(idx).first));
+
+		if (deterministicRandom()->coinflip()) {
+			wait(cache->removeEntryById(tenantList->at(idx).second.id));
+		} else if (deterministicRandom()->coinflip()) {
+			wait(cache->removeEntryByPrefix(tenantList->at(idx).second.prefix));
+		} else {
+			wait(cache->removeEntryByName(tenantList->at(idx).first));
+		}
+
+		state Optional<TenantEntryCachePayload<int64_t>> e = wait(cache->getById(tenantList->at(idx).second.id));
+		ASSERT(!e.present());
+		state Optional<TenantEntryCachePayload<int64_t>> e1 =
+		    wait(cache->getByPrefix(tenantList->at(idx).second.prefix));
+		ASSERT(!e1.present());
+		state Optional<TenantEntryCachePayload<int64_t>> e2 = wait(cache->getByName(tenantList->at(idx).first));
+		ASSERT(!e2.present());
+
+		// Ensure remove-entry is an idempotent operation
+		cache->removeEntryByName(tenantList->at(idx).first);
+		Optional<TenantEntryCachePayload<int64_t>> e3 = wait(cache->getById(tenantList->at(idx).second.id));
+		ASSERT(!e3.present());
+
+		return Void();
+	}
+
+	Future<Void> setup(Database const& cx) override {
+		if (clientId == 0 && g_network->isSimulated() && BUGGIFY) {
+			IKnobCollection::getMutableGlobalKnobCollection().setKnob("tenant_entry_cache_list_refresh_interval",
+			                                                          KnobValueRef::create(int{ 2 }));
+		}
+
+		return Void();
+	}
+
+	Future<Void> start(Database const& cx) override {
+		if (clientId == 0) {
+			return _start(cx, this);
+		}
+		return Void();
+	}
+
+	ACTOR Future<Void> _start(Database cx, TenantEntryCacheWorkload* self) {
+		state std::vector<std::pair<TenantName, TenantMapEntry>> tenantList;
+
+		wait(testTenantNotFound(cx));
+		wait(testCreateTenantsAndLookup(cx, self, &tenantList));
+		wait(testTenantInsert(cx, self, &tenantList));
+		wait(tenantEntryRemove(cx, &tenantList));
+		wait(testTenantCacheDefaultFunc(cx));
+		wait(testCacheRefresh(cx));
+
+		return Void();
+	}
+
+	std::string description() const override { return "TenantEntryCache"; }
+	Future<bool> check(Database const& cx) override { return true; }
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<TenantEntryCacheWorkload> TenantEntryCacheWorkloadFactory("TenantEntryCache");
\ No newline at end of file
diff --git a/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp b/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp
new file mode 100644
index 0000000000..d52ba98fc2
--- /dev/null
+++ b/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp
@@ -0,0 +1,347 @@
+/*
+ * TenantManagementConcurrencyWorkload.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdint>
+#include <limits>
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/TenantManagement.actor.h"
+#include "fdbclient/ThreadSafeTransaction.h"
+#include "fdbrpc/simulator.h"
+#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
+#include "fdbserver/workloads/TenantConsistency.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/Knobs.h"
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/flow.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct TenantManagementConcurrencyWorkload : TestWorkload {
+	const TenantName tenantNamePrefix = "tenant_management_concurrency_workload_"_sr;
+	const Key testParametersKey = "test_parameters"_sr;
+
+	int maxTenants;
+	int maxTenantGroups;
+	double testDuration;
+	bool useMetacluster;
+
+	Reference<IDatabase> mvDb;
+	Database dataDb;
+
+	TenantManagementConcurrencyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		maxTenants = std::min<int>(1e8 - 1, getOption(options, "maxTenants"_sr, 100));
+		maxTenantGroups = std::min<int>(2 * maxTenants, getOption(options, "maxTenantGroups"_sr, 20));
+		testDuration = getOption(options, "testDuration"_sr, 120.0);
+
+		if (clientId == 0) {
+			useMetacluster = deterministicRandom()->coinflip();
+		} else {
+			// Other clients read the metacluster state from the database
+			useMetacluster = false;
+		}
+	}
+
+	std::string description() const override { return "TenantManagementConcurrency"; }
+
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
+		out.insert("MachineAttritionWorkload");
+	}
+
+	struct TestParameters {
+		constexpr static FileIdentifier file_identifier = 14350843;
+
+		bool useMetacluster = false;
+
+		TestParameters() {}
+		TestParameters(bool useMetacluster) : useMetacluster(useMetacluster) {}
+
+		template <class Ar>
+		void serialize(Ar& ar) {
+			serializer(ar, useMetacluster);
+		}
+
+		Value encode() const { return ObjectWriter::toValue(*this, Unversioned()); }
+		static TestParameters decode(ValueRef const& value) {
+			return ObjectReader::fromStringRef<TestParameters>(value, Unversioned());
+		}
+	};
+
+	Future<Void> setup(Database const& cx) override {
+		if (clientId == 0 && g_network->isSimulated() && BUGGIFY) {
+			IKnobCollection::getMutableGlobalKnobCollection().setKnob(
+			    "max_tenants_per_cluster", KnobValueRef::create(int{ deterministicRandom()->randomInt(20, 100) }));
+		}
+
+		return _setup(cx, this);
+	}
+	ACTOR static Future<Void> _setup(Database cx, TenantManagementConcurrencyWorkload* self) {
+		Reference<IDatabase> threadSafeHandle =
+		    wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
+
+		MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
+		self->mvDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
+
+		if (self->useMetacluster && self->clientId == 0) {
+			wait(success(MetaclusterAPI::createMetacluster(cx.getReference(), "management_cluster"_sr)));
+
+			DataClusterEntry entry;
+			entry.capacity.numTenantGroups = 1e9;
+			wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, g_simulator->extraDatabases[0], entry));
+		}
+
+		state Transaction tr(cx);
+		if (self->clientId == 0) {
+			// Send test parameters to the other clients
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::RAW_ACCESS);
+					tr.set(self->testParametersKey, TestParameters(self->useMetacluster).encode());
+					wait(tr.commit());
+					break;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+		} else {
+			// Read the tenant subspace chosen and saved by client 0
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::RAW_ACCESS);
+					Optional<Value> val = wait(tr.get(self->testParametersKey));
+					if (val.present()) {
+						TestParameters params = TestParameters::decode(val.get());
+						self->useMetacluster = params.useMetacluster;
+						break;
+					}
+
+					wait(delay(1.0));
+					tr.reset();
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+		}
+
+		if (self->useMetacluster) {
+			ASSERT(g_simulator->extraDatabases.size() == 1);
+			self->dataDb = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], cx->defaultTenant);
+		} else {
+			self->dataDb = cx;
+		}
+
+		return Void();
+	}
+
+	TenantName chooseTenantName() {
+		TenantName tenant(
+		    format("%s%08d", tenantNamePrefix.toString().c_str(), deterministicRandom()->randomInt(0, maxTenants)));
+
+		return tenant;
+	}
+
+	Optional<TenantGroupName> chooseTenantGroup() {
+		Optional<TenantGroupName> tenantGroup;
+		if (deterministicRandom()->coinflip()) {
+			tenantGroup =
+			    TenantGroupNameRef(format("tenantgroup%08d", deterministicRandom()->randomInt(0, maxTenantGroups)));
+		}
+
+		return tenantGroup;
+	}
+
+	ACTOR static Future<Void> createTenant(TenantManagementConcurrencyWorkload* self) {
+		state TenantName tenant = self->chooseTenantName();
+		state TenantMapEntry entry;
+		entry.tenantGroup = self->chooseTenantGroup();
+
+		try {
+			loop {
+				Future<Void> createFuture =
+				    self->useMetacluster ? MetaclusterAPI::createTenant(self->mvDb, tenant, entry)
+				                         : success(TenantAPI::createTenant(self->dataDb.getReference(), tenant, entry));
+				Optional<Void> result = wait(timeout(createFuture, 30));
+				if (result.present()) {
+					break;
+				}
+			}
+
+			return Void();
+		} catch (Error& e) {
+			if (e.code() == error_code_tenant_removed) {
+				ASSERT(self->useMetacluster);
+			} else if (e.code() != error_code_tenant_already_exists && e.code() != error_code_cluster_no_capacity) {
+				TraceEvent(SevError, "CreateTenantFailure").error(e).detail("TenantName", tenant);
+				ASSERT(false);
+			}
+
+			return Void();
+		}
+	}
+
+	ACTOR static Future<Void> deleteTenant(TenantManagementConcurrencyWorkload* self) {
+		state TenantName tenant = self->chooseTenantName();
+
+		try {
+			loop {
+				Future<Void> deleteFuture = self->useMetacluster
+				                                ? MetaclusterAPI::deleteTenant(self->mvDb, tenant)
+				                                : TenantAPI::deleteTenant(self->dataDb.getReference(), tenant);
+				Optional<Void> result = wait(timeout(deleteFuture, 30));
+
+				if (result.present()) {
+					break;
+				}
+			}
+
+			return Void();
+		} catch (Error& e) {
+			if (e.code() != error_code_tenant_not_found) {
+				TraceEvent(SevError, "DeleteTenantFailure").error(e).detail("TenantName", tenant);
+			}
+			return Void();
+		}
+	}
+
+	ACTOR static Future<Void> configureImpl(TenantManagementConcurrencyWorkload* self,
+	                                        TenantName tenant,
+	                                        std::map<Standalone<StringRef>, Optional<Value>> configParams) {
+		if (self->useMetacluster) {
+			wait(MetaclusterAPI::configureTenant(self->mvDb, tenant, configParams));
+		} else {
+			state Reference<ReadYourWritesTransaction> tr = self->dataDb->createTransaction();
+			loop {
+				try {
+					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					TenantMapEntry entry = wait(TenantAPI::getTenantTransaction(tr, tenant));
+					TenantMapEntry updatedEntry = entry;
+					for (auto param : configParams) {
+						updatedEntry.configure(param.first, param.second);
+					}
+					wait(TenantAPI::configureTenantTransaction(tr, tenant, entry, updatedEntry));
+					wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
+					break;
+				} catch (Error& e) {
+					wait(tr->onError(e));
+				}
+			}
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> configureTenant(TenantManagementConcurrencyWorkload* self) {
+		state TenantName tenant = self->chooseTenantName();
+		state std::map<Standalone<StringRef>, Optional<Value>> configParams;
+		configParams["tenant_group"_sr] = self->chooseTenantGroup();
+
+		try {
+			loop {
+				Optional<Void> result = wait(timeout(configureImpl(self, tenant, configParams), 30));
+
+				if (result.present()) {
+					break;
+				}
+			}
+
+			return Void();
+		} catch (Error& e) {
+			if (e.code() != error_code_tenant_not_found && e.code() != error_code_invalid_tenant_state) {
+				TraceEvent(SevError, "ConfigureTenantFailure").error(e).detail("TenantName", tenant);
+			}
+			return Void();
+		}
+	}
+
+	ACTOR static Future<Void> renameTenant(TenantManagementConcurrencyWorkload* self) {
+		state TenantName oldTenant = self->chooseTenantName();
+		state TenantName newTenant = self->chooseTenantName();
+
+		try {
+			loop {
+				Future<Void> renameFuture =
+				    self->useMetacluster ? MetaclusterAPI::renameTenant(self->mvDb, oldTenant, newTenant)
+				                         : TenantAPI::renameTenant(self->dataDb.getReference(), oldTenant, newTenant);
+				Optional<Void> result = wait(timeout(renameFuture, 30));
+
+				if (result.present()) {
+					break;
+				}
+			}
+
+			return Void();
+		} catch (Error& e) {
+			if (e.code() == error_code_invalid_tenant_state || e.code() == error_code_tenant_removed ||
+			    e.code() == error_code_cluster_no_capacity) {
+				ASSERT(self->useMetacluster);
+			} else if (e.code() != error_code_tenant_not_found && e.code() != error_code_tenant_already_exists) {
+				TraceEvent(SevError, "RenameTenantFailure")
+				    .error(e)
+				    .detail("OldTenant", oldTenant)
+				    .detail("NewTenant", newTenant);
+			}
+			return Void();
+		}
+	}
+
+	Future<Void> start(Database const& cx) override { return _start(cx, this); }
+	ACTOR static Future<Void> _start(Database cx, TenantManagementConcurrencyWorkload* self) {
+		state double start = now();
+
+		// Run a random sequence of tenant management operations for the duration of the test
+		while (now() < start + self->testDuration) {
+			state int operation = deterministicRandom()->randomInt(0, 4);
+			if (operation == 0) {
+				wait(createTenant(self));
+			} else if (operation == 1) {
+				wait(deleteTenant(self));
+			} else if (operation == 2) {
+				wait(configureTenant(self));
+			} else if (operation == 3) {
+				wait(renameTenant(self));
+			}
+		}
+
+		return Void();
+	}
+
+	Future<bool> check(Database const& cx) override { return _check(cx, this); }
+	ACTOR static Future<bool> _check(Database cx, TenantManagementConcurrencyWorkload* self) {
+		if (self->useMetacluster) {
+			// The metacluster consistency check runs the tenant consistency check for each cluster
+			state MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
+			    self->mvDb, AllowPartialMetaclusterOperations::True);
+			wait(metaclusterConsistencyCheck.run());
+		} else {
+			state TenantConsistencyCheck<DatabaseContext> tenantConsistencyCheck(self->dataDb.getReference());
+			wait(tenantConsistencyCheck.run());
+		}
+
+		return true;
+	}
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<TenantManagementConcurrencyWorkload> TenantManagementConcurrencyWorkloadFactory(
+    "TenantManagementConcurrency");
diff --git a/fdbserver/workloads/TenantManagementWorkload.actor.cpp b/fdbserver/workloads/TenantManagementWorkload.actor.cpp
index bf454b8afb..b6a29cfd96 100644
--- a/fdbserver/workloads/TenantManagementWorkload.actor.cpp
+++ b/fdbserver/workloads/TenantManagementWorkload.actor.cpp
@@ -20,83 +20,177 @@
 
 #include <cstdint>
 #include <limits>
+#include "fdbclient/ClientBooleanParams.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/MetaclusterManagement.actor.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/RunTransaction.actor.h"
 #include "fdbclient/TenantManagement.actor.h"
 #include "fdbclient/TenantSpecialKeys.actor.h"
-#include "fdbclient/libb64/decode.h"
+#include "fdbclient/ThreadSafeTransaction.h"
 #include "fdbrpc/simulator.h"
+#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
+#include "fdbserver/workloads/TenantConsistency.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/Knobs.h"
+#include "flow/ApiVersion.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
+#include "flow/ThreadHelper.actor.h"
 #include "flow/flow.h"
+#include "libb64/decode.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct TenantManagementWorkload : TestWorkload {
-	struct TenantState {
+	struct TenantData {
 		int64_t id;
+		Optional<TenantGroupName> tenantGroup;
 		bool empty;
+		bool encrypted;
 
-		TenantState() : id(-1), empty(true) {}
-		TenantState(int64_t id, bool empty) : id(id), empty(empty) {}
+		TenantData() : id(-1), empty(true) {}
+		TenantData(int64_t id, Optional<TenantGroupName> tenantGroup, bool empty, bool encrypted)
+		  : id(id), tenantGroup(tenantGroup), empty(empty), encrypted(encrypted) {}
 	};
 
-	std::map<TenantName, TenantState> createdTenants;
+	struct TenantGroupData {
+		int64_t tenantCount = 0;
+	};
+
+	std::map<TenantName, TenantData> createdTenants;
+	std::map<TenantGroupName, TenantGroupData> createdTenantGroups;
 	int64_t maxId = -1;
-	Key tenantSubspace;
 
 	const Key keyName = "key"_sr;
-	const Key tenantSubspaceKey = "tenant_subspace"_sr;
+	const Key testParametersKey = "test_parameters"_sr;
 	const Value noTenantValue = "no_tenant"_sr;
 	const TenantName tenantNamePrefix = "tenant_management_workload_"_sr;
+	const ClusterName dataClusterName = "cluster1"_sr;
 	TenantName localTenantNamePrefix;
+	TenantName localTenantGroupNamePrefix;
 
 	const Key specialKeysTenantMapPrefix = SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT)
-	                                           .begin.withSuffix(TenantRangeImpl<true>::submoduleRange.begin)
-	                                           .withSuffix(TenantRangeImpl<true>::mapSubRange.begin);
+	                                           .begin.withSuffix(TenantRangeImpl::submoduleRange.begin)
+	                                           .withSuffix(TenantRangeImpl::mapSubRange.begin);
+	const Key specialKeysTenantConfigPrefix = SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT)
+	                                              .begin.withSuffix(TenantRangeImpl::submoduleRange.begin)
+	                                              .withSuffix(TenantRangeImpl::configureSubRange.begin);
+	const Key specialKeysTenantRenamePrefix = SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT)
+	                                              .begin.withSuffix(TenantRangeImpl::submoduleRange.begin)
+	                                              .withSuffix(TenantRangeImpl::renameSubRange.begin);
 
 	int maxTenants;
+	int maxTenantGroups;
 	double testDuration;
+	bool useMetacluster;
+	bool singleClient;
 
-	enum class OperationType { SPECIAL_KEYS, MANAGEMENT_DATABASE, MANAGEMENT_TRANSACTION };
+	Version oldestDeletionVersion = 0;
+	Version newestDeletionVersion = 0;
 
-	static OperationType randomOperationType() {
-		int randomNum = deterministicRandom()->randomInt(0, 3);
-		if (randomNum == 0) {
-			return OperationType::SPECIAL_KEYS;
-		} else if (randomNum == 1) {
-			return OperationType::MANAGEMENT_DATABASE;
+	Reference<IDatabase> mvDb;
+	Database dataDb;
+
+	// This test exercises multiple different ways to work with tenants
+	enum class OperationType {
+		// Use the special key-space APIs
+		SPECIAL_KEYS,
+		// Use the ManagementAPI functions that take a Database object and implement a retry loop
+		MANAGEMENT_DATABASE,
+		// Use the ManagementAPI functions that take a Transaction object
+		MANAGEMENT_TRANSACTION,
+		// Use the Metacluster API, if applicable. Note: not all APIs have a metacluster variant,
+		// and if there isn't one this will choose one of the other options.
+		METACLUSTER
+	};
+
+	OperationType randomOperationType() {
+		double metaclusterProb = useMetacluster ? 0.9 : 0.1;
+
+		if (deterministicRandom()->random01() < metaclusterProb) {
+			return OperationType::METACLUSTER;
 		} else {
-			return OperationType::MANAGEMENT_TRANSACTION;
+			return (OperationType)deterministicRandom()->randomInt(0, 3);
 		}
 	}
 
 	TenantManagementWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		maxTenants = std::min<int>(1e8 - 1, getOption(options, "maxTenants"_sr, 1000));
-		testDuration = getOption(options, "testDuration"_sr, 60.0);
+		maxTenantGroups = std::min<int>(2 * maxTenants, getOption(options, "maxTenantGroups"_sr, 20));
+		testDuration = getOption(options, "testDuration"_sr, 120.0);
+		singleClient = getOption(options, "singleClient"_sr, false);
 
 		localTenantNamePrefix = format("%stenant_%d_", tenantNamePrefix.toString().c_str(), clientId);
+		localTenantGroupNamePrefix = format("%stenantgroup_%d_", tenantNamePrefix.toString().c_str(), clientId);
+
+		bool defaultUseMetacluster = false;
+		if (clientId == 0 && g_network->isSimulated() && !g_simulator->extraDatabases.empty()) {
+			defaultUseMetacluster = deterministicRandom()->coinflip();
+		}
+
+		useMetacluster = getOption(options, "useMetacluster"_sr, defaultUseMetacluster);
 	}
 
 	std::string description() const override { return "TenantManagement"; }
 
-	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
+	struct TestParameters {
+		constexpr static FileIdentifier file_identifier = 1527576;
+
+		bool useMetacluster = false;
+
+		TestParameters() {}
+		TestParameters(bool useMetacluster) : useMetacluster(useMetacluster) {}
+
+		template <class Ar>
+		void serialize(Ar& ar) {
+			serializer(ar, useMetacluster);
+		}
+
+		Value encode() const { return ObjectWriter::toValue(*this, Unversioned()); }
+		static TestParameters decode(ValueRef const& value) {
+			return ObjectReader::fromStringRef<TestParameters>(value, Unversioned());
+		}
+	};
+
+	Future<Void> setup(Database const& cx) override {
+		if (clientId == 0 && g_network->isSimulated() && BUGGIFY) {
+			IKnobCollection::getMutableGlobalKnobCollection().setKnob(
+			    "max_tenants_per_cluster", KnobValueRef::create(int{ deterministicRandom()->randomInt(20, 100) }));
+		}
+
+		if (clientId == 0 || !singleClient) {
+			return _setup(cx, this);
+		} else {
+			return Void();
+		}
+	}
+
 	ACTOR Future<Void> _setup(Database cx, TenantManagementWorkload* self) {
+		Reference<IDatabase> threadSafeHandle =
+		    wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
+
+		MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
+		self->mvDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
+
+		if (self->useMetacluster && self->clientId == 0) {
+			wait(success(MetaclusterAPI::createMetacluster(cx.getReference(), "management_cluster"_sr)));
+
+			DataClusterEntry entry;
+			entry.capacity.numTenantGroups = 1e9;
+			wait(MetaclusterAPI::registerCluster(
+			    self->mvDb, self->dataClusterName, g_simulator->extraDatabases[0], entry));
+		}
+
 		state Transaction tr(cx);
 		if (self->clientId == 0) {
-			self->tenantSubspace = makeString(deterministicRandom()->randomInt(0, 10));
-			loop {
-				generateRandomData(mutateString(self->tenantSubspace), self->tenantSubspace.size());
-				if (!self->tenantSubspace.startsWith(systemKeys.begin)) {
-					break;
-				}
-			}
+			// Communicates test parameters to all other clients by storing it in a key
 			loop {
 				try {
 					tr.setOption(FDBTransactionOptions::RAW_ACCESS);
-					tr.set(self->keyName, self->noTenantValue);
-					tr.set(self->tenantSubspaceKey, self->tenantSubspace);
-					tr.set(tenantDataPrefixKey, self->tenantSubspace);
+					tr.set(self->testParametersKey, TestParameters(self->useMetacluster).encode());
 					wait(tr.commit());
 					break;
 				} catch (Error& e) {
@@ -104,22 +198,47 @@ struct TenantManagementWorkload : TestWorkload {
 				}
 			}
 		} else {
+			// Read the parameters chosen and saved by client 0
 			loop {
 				try {
 					tr.setOption(FDBTransactionOptions::RAW_ACCESS);
-					Optional<Value> val = wait(tr.get(self->tenantSubspaceKey));
+					Optional<Value> val = wait(tr.get(self->testParametersKey));
 					if (val.present()) {
-						self->tenantSubspace = val.get();
+						TestParameters params = TestParameters::decode(val.get());
+						self->useMetacluster = params.useMetacluster;
 						break;
 					}
 
 					wait(delay(1.0));
+					tr.reset();
 				} catch (Error& e) {
 					wait(tr.onError(e));
 				}
 			}
 		}
 
+		if (self->useMetacluster) {
+			ASSERT(g_simulator->extraDatabases.size() == 1);
+			self->dataDb = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], cx->defaultTenant);
+		} else {
+			self->dataDb = cx;
+		}
+
+		if (self->clientId == 0) {
+			// Set a key outside of all tenants to make sure that our tenants aren't writing to the regular key-space
+			state Transaction dataTr(self->dataDb);
+			loop {
+				try {
+					dataTr.setOption(FDBTransactionOptions::RAW_ACCESS);
+					dataTr.set(self->keyName, self->noTenantValue);
+					wait(dataTr.commit());
+					break;
+				} catch (Error& e) {
+					wait(dataTr.onError(e));
+				}
+			}
+		}
+
 		return Void();
 	}
 
@@ -133,8 +252,75 @@ struct TenantManagementWorkload : TestWorkload {
 		return tenant;
 	}
 
-	ACTOR Future<Void> createTenant(Database cx, TenantManagementWorkload* self) {
-		state OperationType operationType = TenantManagementWorkload::randomOperationType();
+	Optional<TenantGroupName> chooseTenantGroup(bool allowSystemTenantGroup, bool allowEmptyGroup = true) {
+		Optional<TenantGroupName> tenantGroup;
+		if (!allowEmptyGroup || deterministicRandom()->coinflip()) {
+			tenantGroup = TenantGroupNameRef(format("%s%08d",
+			                                        localTenantGroupNamePrefix.toString().c_str(),
+			                                        deterministicRandom()->randomInt(0, maxTenantGroups)));
+			if (allowSystemTenantGroup && deterministicRandom()->random01() < 0.02) {
+				tenantGroup = tenantGroup.get().withPrefix("\xff"_sr);
+			}
+		}
+
+		return tenantGroup;
+	}
+
+	Future<Optional<TenantMapEntry>> tryGetTenant(TenantName tenantName, OperationType operationType) {
+		if (operationType == OperationType::METACLUSTER) {
+			return MetaclusterAPI::tryGetTenant(mvDb, tenantName);
+		} else {
+			return TenantAPI::tryGetTenant(dataDb.getReference(), tenantName);
+		}
+	}
+
+	// Creates tenant(s) using the specified operation type
+	ACTOR static Future<Void> createTenantImpl(Reference<ReadYourWritesTransaction> tr,
+	                                           std::map<TenantName, TenantMapEntry> tenantsToCreate,
+	                                           OperationType operationType,
+	                                           TenantManagementWorkload* self) {
+		if (operationType == OperationType::SPECIAL_KEYS) {
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			for (auto [tenant, entry] : tenantsToCreate) {
+				tr->set(self->specialKeysTenantMapPrefix.withSuffix(tenant), ""_sr);
+				if (entry.tenantGroup.present()) {
+					tr->set(self->specialKeysTenantConfigPrefix.withSuffix(
+					            Tuple::makeTuple(tenant, "tenant_group"_sr).pack()),
+					        entry.tenantGroup.get());
+				}
+			}
+			wait(tr->commit());
+		} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
+			ASSERT(tenantsToCreate.size() == 1);
+			wait(success(TenantAPI::createTenant(
+			    self->dataDb.getReference(), tenantsToCreate.begin()->first, tenantsToCreate.begin()->second)));
+		} else if (operationType == OperationType::MANAGEMENT_TRANSACTION) {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			int64_t _nextId = wait(TenantAPI::getNextTenantId(tr));
+			int64_t nextId = _nextId;
+
+			std::vector<Future<Void>> createFutures;
+			for (auto [tenant, entry] : tenantsToCreate) {
+				entry.setId(nextId++);
+				createFutures.push_back(success(TenantAPI::createTenantTransaction(tr, tenant, entry)));
+			}
+			TenantMetadata::lastTenantId().set(tr, nextId - 1);
+			wait(waitForAll(createFutures));
+			wait(tr->commit());
+		} else {
+			ASSERT(tenantsToCreate.size() == 1);
+			if (deterministicRandom()->coinflip()) {
+				tenantsToCreate.begin()->second.assignedCluster = self->dataClusterName;
+			}
+			wait(MetaclusterAPI::createTenant(
+			    self->mvDb, tenantsToCreate.begin()->first, tenantsToCreate.begin()->second));
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> createTenant(TenantManagementWorkload* self) {
+		state OperationType operationType = self->randomOperationType();
 		int numTenants = 1;
 
 		// For transaction-based operations, test creating multiple tenants in the same transaction
@@ -142,72 +328,171 @@ struct TenantManagementWorkload : TestWorkload {
 			numTenants = deterministicRandom()->randomInt(1, 5);
 		}
 
+		// Tracks whether any tenant exists in the database or not. This variable is updated if we have to retry
+		// the creation.
 		state bool alreadyExists = false;
+
+		// True if any tenant name starts with \xff
 		state bool hasSystemTenant = false;
 
-		state std::set<TenantName> tenantsToCreate;
+		// True if any tenant group name starts with \xff
+		state bool hasSystemTenantGroup = false;
+
+		state int newTenants = 0;
+		state std::map<TenantName, TenantMapEntry> tenantsToCreate;
 		for (int i = 0; i < numTenants; ++i) {
 			TenantName tenant = self->chooseTenantName(true);
-			tenantsToCreate.insert(tenant);
+			while (tenantsToCreate.count(tenant)) {
+				tenant = self->chooseTenantName(true);
+			}
 
-			alreadyExists = alreadyExists || self->createdTenants.count(tenant);
+			TenantMapEntry entry;
+			entry.tenantGroup = self->chooseTenantGroup(true);
+			if (operationType == OperationType::SPECIAL_KEYS) {
+				entry.encrypted = SERVER_KNOBS->ENABLE_ENCRYPTION;
+			} else {
+				entry.encrypted = deterministicRandom()->coinflip();
+			}
+
+			if (self->createdTenants.count(tenant)) {
+				alreadyExists = true;
+			} else if (!tenantsToCreate.count(tenant)) {
+				++newTenants;
+			}
+
+			tenantsToCreate[tenant] = entry;
 			hasSystemTenant = hasSystemTenant || tenant.startsWith("\xff"_sr);
+			hasSystemTenantGroup = hasSystemTenantGroup || entry.tenantGroup.orDefault(""_sr).startsWith("\xff"_sr);
 		}
 
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		// If any tenant existed at the start of this function, then we expect the creation to fail or be a no-op,
+		// depending on the type of create operation being executed
+		state bool existedAtStart = alreadyExists;
+
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dataDb);
+		state int64_t minTenantCount = std::numeric_limits<int64_t>::max();
+		state int64_t finalTenantCount = 0;
 
 		loop {
 			try {
-				if (operationType == OperationType::SPECIAL_KEYS) {
-					tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-					for (auto tenant : tenantsToCreate) {
-						tr->set(self->specialKeysTenantMapPrefix.withSuffix(tenant), ""_sr);
+				// First, attempt to create the tenants
+				state bool retried = false;
+				loop {
+					if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
+					    operationType == OperationType::SPECIAL_KEYS) {
+						tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+						wait(store(finalTenantCount, TenantMetadata::tenantCount().getD(tr, Snapshot::False, 0)));
+						minTenantCount = std::min(finalTenantCount, minTenantCount);
 					}
-					wait(tr->commit());
-				} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
-					ASSERT(tenantsToCreate.size() == 1);
-					wait(success(TenantAPI::createTenant(cx.getReference(), *tenantsToCreate.begin())));
-				} else {
-					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-					int64_t _nextId = wait(TenantAPI::getNextTenantId(tr));
-					int64_t nextId = _nextId;
+					try {
+						Optional<Void> result = wait(timeout(createTenantImpl(tr, tenantsToCreate, operationType, self),
+						                                     deterministicRandom()->randomInt(1, 30)));
 
-					std::vector<Future<Void>> createFutures;
-					for (auto tenant : tenantsToCreate) {
-						createFutures.push_back(success(TenantAPI::createTenantTransaction(tr, tenant, nextId++)));
+						if (result.present()) {
+							// Make sure that we had capacity to create the tenants. This cannot be validated for
+							// database operations because we cannot determine the tenant count in the same transaction
+							// that the tenant is created
+							if (operationType == OperationType::SPECIAL_KEYS ||
+							    operationType == OperationType::MANAGEMENT_TRANSACTION) {
+								ASSERT(minTenantCount + newTenants <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
+							} else {
+								// Database operations shouldn't get here if the tenant already exists
+								ASSERT(!alreadyExists);
+							}
+
+							break;
+						}
+
+						retried = true;
+						tr->reset();
+					} catch (Error& e) {
+						// If we retried the creation after our initial attempt succeeded, then we proceed with the rest
+						// of the creation steps normally. Otherwise, the creation happened elsewhere and we failed
+						// here, so we can rethrow the error.
+						if (e.code() == error_code_tenant_already_exists && !existedAtStart) {
+							ASSERT(operationType == OperationType::METACLUSTER ||
+							       operationType == OperationType::MANAGEMENT_DATABASE);
+							ASSERT(retried);
+							break;
+						} else {
+							throw;
+						}
+					}
+
+					// Check the state of the first created tenant
+					Optional<TenantMapEntry> resultEntry =
+					    wait(self->tryGetTenant(tenantsToCreate.begin()->first, operationType));
+
+					if (resultEntry.present()) {
+						if (resultEntry.get().tenantState == TenantState::READY) {
+							// The tenant now exists, so we will retry and expect the creation to react accordingly
+							alreadyExists = true;
+						} else {
+							// Only a metacluster tenant creation can end up in a partially created state
+							// We should be able to retry and pick up where we left off
+							ASSERT(operationType == OperationType::METACLUSTER);
+							ASSERT(resultEntry.get().tenantState == TenantState::REGISTERING);
+						}
 					}
-					tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(nextId - 1));
-					wait(waitForAll(createFutures));
-					wait(tr->commit());
 				}
 
-				if (operationType == OperationType::MANAGEMENT_DATABASE) {
-					ASSERT(!alreadyExists);
+				// Check that using the wrong creation type fails depending on whether we are using a metacluster
+				ASSERT(self->useMetacluster == (operationType == OperationType::METACLUSTER));
+
+				// Database-based creation modes will fail if the tenant already existed
+				if (operationType == OperationType::MANAGEMENT_DATABASE ||
+				    operationType == OperationType::METACLUSTER) {
+					ASSERT(!existedAtStart);
 				}
 
+				// It is not legal to create a tenant or tenant group starting with \xff
 				ASSERT(!hasSystemTenant);
+				ASSERT(!hasSystemTenantGroup);
 
-				state std::set<TenantName>::iterator tenantItr;
+				state std::map<TenantName, TenantMapEntry>::iterator tenantItr;
 				for (tenantItr = tenantsToCreate.begin(); tenantItr != tenantsToCreate.end(); ++tenantItr) {
-					if (self->createdTenants.count(*tenantItr)) {
+					// Ignore any tenants that already existed
+					if (self->createdTenants.count(tenantItr->first)) {
 						continue;
 					}
 
-					state Optional<TenantMapEntry> entry = wait(TenantAPI::tryGetTenant(cx.getReference(), *tenantItr));
+					// Read the created tenant object and verify that its state is correct
+					state Optional<TenantMapEntry> entry = wait(self->tryGetTenant(tenantItr->first, operationType));
+
 					ASSERT(entry.present());
 					ASSERT(entry.get().id > self->maxId);
-					ASSERT(entry.get().prefix.startsWith(self->tenantSubspace));
+					ASSERT(entry.get().tenantGroup == tenantItr->second.tenantGroup);
+					ASSERT(entry.get().tenantState == TenantState::READY);
 
+					if (self->useMetacluster) {
+						// In a metacluster, we should also see that the tenant was created on the data cluster
+						Optional<TenantMapEntry> dataEntry =
+						    wait(TenantAPI::tryGetTenant(self->dataDb.getReference(), tenantItr->first));
+						ASSERT(dataEntry.present());
+						ASSERT(dataEntry.get().id == entry.get().id);
+						ASSERT(dataEntry.get().tenantGroup == entry.get().tenantGroup);
+						ASSERT(dataEntry.get().tenantState == TenantState::READY);
+					}
+
+					// Update our local tenant state to include the newly created one
 					self->maxId = entry.get().id;
-					self->createdTenants[*tenantItr] = TenantState(entry.get().id, true);
+					self->createdTenants[tenantItr->first] =
+					    TenantData(entry.get().id, tenantItr->second.tenantGroup, true, tenantItr->second.encrypted);
 
+					// If this tenant has a tenant group, create or update the entry for it
+					if (tenantItr->second.tenantGroup.present()) {
+						self->createdTenantGroups[tenantItr->second.tenantGroup.get()].tenantCount++;
+					}
+
+					// Randomly decide to insert a key into the tenant
 					state bool insertData = deterministicRandom()->random01() < 0.5;
 					if (insertData) {
-						state Transaction insertTr(cx, *tenantItr);
+						state Transaction insertTr(self->dataDb, tenantItr->first);
 						loop {
 							try {
-								insertTr.set(self->keyName, *tenantItr);
+								// The value stored in the key will be the name of the tenant
+								insertTr.set(self->keyName, tenantItr->first);
 								wait(insertTr.commit());
 								break;
 							} catch (Error& e) {
@@ -215,15 +500,17 @@ struct TenantManagementWorkload : TestWorkload {
 							}
 						}
 
-						self->createdTenants[*tenantItr].empty = false;
+						self->createdTenants[tenantItr->first].empty = false;
 
-						state Transaction checkTr(cx);
+						// Make sure that the key inserted correctly concatenates the tenant prefix with the
+						// relative key
+						state Transaction checkTr(self->dataDb);
 						loop {
 							try {
 								checkTr.setOption(FDBTransactionOptions::RAW_ACCESS);
 								Optional<Value> val = wait(checkTr.get(self->keyName.withPrefix(entry.get().prefix)));
 								ASSERT(val.present());
-								ASSERT(val.get() == *tenantItr);
+								ASSERT(val.get() == tenantItr->first);
 								break;
 							} catch (Error& e) {
 								wait(checkTr.onError(e));
@@ -231,28 +518,51 @@ struct TenantManagementWorkload : TestWorkload {
 						}
 					}
 
-					wait(self->checkTenant(cx, self, *tenantItr, self->createdTenants[*tenantItr]));
+					// Perform some final tenant validation
+					wait(checkTenantContents(self, tenantItr->first, self->createdTenants[tenantItr->first]));
 				}
+
 				return Void();
 			} catch (Error& e) {
 				if (e.code() == error_code_invalid_tenant_name) {
 					ASSERT(hasSystemTenant);
 					return Void();
-				} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
+				} else if (e.code() == error_code_invalid_tenant_group_name) {
+					ASSERT(hasSystemTenantGroup);
+					return Void();
+				} else if (e.code() == error_code_invalid_metacluster_operation) {
+					ASSERT(operationType == OperationType::METACLUSTER != self->useMetacluster);
+					return Void();
+				} else if (e.code() == error_code_cluster_no_capacity) {
+					// Confirm that we overshot our capacity. This check cannot be done for database operations
+					// because we cannot transactionally get the tenant count with the creation.
+					if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
+					    operationType == OperationType::SPECIAL_KEYS) {
+						ASSERT(finalTenantCount + newTenants > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
+					}
+					return Void();
+				}
+
+				// Database-based operations should not need to be retried
+				else if (operationType == OperationType::MANAGEMENT_DATABASE ||
+				         operationType == OperationType::METACLUSTER) {
 					if (e.code() == error_code_tenant_already_exists) {
-						ASSERT(alreadyExists && operationType == OperationType::MANAGEMENT_DATABASE);
+						ASSERT(existedAtStart);
 					} else {
 						ASSERT(tenantsToCreate.size() == 1);
 						TraceEvent(SevError, "CreateTenantFailure")
 						    .error(e)
-						    .detail("TenantName", *tenantsToCreate.begin());
+						    .detail("TenantName", tenantsToCreate.begin()->first);
 					}
 					return Void();
-				} else {
+				}
+
+				// Transaction-based operations should be retried
+				else {
 					try {
 						wait(tr->onError(e));
 					} catch (Error& e) {
-						for (auto tenant : tenantsToCreate) {
+						for (auto [tenant, _] : tenantsToCreate) {
 							TraceEvent(SevError, "CreateTenantFailure").error(e).detail("TenantName", tenant);
 						}
 						return Void();
@@ -262,16 +572,71 @@ struct TenantManagementWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<Void> deleteTenant(Database cx, TenantManagementWorkload* self) {
-		state TenantName beginTenant = self->chooseTenantName(true);
-		state OperationType operationType = TenantManagementWorkload::randomOperationType();
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+	// Deletes the tenant or tenant range using the specified operation type
+	ACTOR static Future<Void> deleteTenantImpl(Reference<ReadYourWritesTransaction> tr,
+	                                           TenantName beginTenant,
+	                                           Optional<TenantName> endTenant,
+	                                           std::vector<TenantName> tenants,
+	                                           OperationType operationType,
+	                                           TenantManagementWorkload* self) {
+		state int tenantIndex;
+		if (operationType == OperationType::SPECIAL_KEYS) {
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			Key key = self->specialKeysTenantMapPrefix.withSuffix(beginTenant);
+			if (endTenant.present()) {
+				tr->clear(KeyRangeRef(key, self->specialKeysTenantMapPrefix.withSuffix(endTenant.get())));
+			} else {
+				tr->clear(key);
+			}
+			wait(tr->commit());
+		} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
+			ASSERT(!endTenant.present() && tenants.size() == 1);
+			wait(TenantAPI::deleteTenant(self->dataDb.getReference(), beginTenant));
+		} else if (operationType == OperationType::MANAGEMENT_TRANSACTION) {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			std::vector<Future<Void>> deleteFutures;
+			for (tenantIndex = 0; tenantIndex != tenants.size(); ++tenantIndex) {
+				deleteFutures.push_back(TenantAPI::deleteTenantTransaction(tr, tenants[tenantIndex]));
+			}
 
-		state Optional<TenantName> endTenant = operationType != OperationType::MANAGEMENT_DATABASE &&
-		                                               !beginTenant.startsWith("\xff"_sr) &&
-		                                               deterministicRandom()->random01() < 0.2
-		                                           ? Optional<TenantName>(self->chooseTenantName(false))
-		                                           : Optional<TenantName>();
+			wait(waitForAll(deleteFutures));
+			wait(tr->commit());
+		} else {
+			ASSERT(!endTenant.present() && tenants.size() == 1);
+			wait(MetaclusterAPI::deleteTenant(self->mvDb, beginTenant));
+		}
+
+		return Void();
+	}
+
+	// Returns GRV and eats GRV errors
+	ACTOR static Future<Version> getReadVersion(Reference<ReadYourWritesTransaction> tr) {
+		loop {
+			try {
+				Version version = wait(tr->getReadVersion());
+				return version;
+			} catch (Error& e) {
+				if (e.code() == error_code_grv_proxy_memory_limit_exceeded ||
+				    e.code() == error_code_batch_transaction_throttled) {
+					wait(tr->onError(e));
+				} else {
+					throw;
+				}
+			}
+		}
+	}
+
+	ACTOR static Future<Void> deleteTenant(TenantManagementWorkload* self) {
+		state TenantName beginTenant = self->chooseTenantName(true);
+		state OperationType operationType = self->randomOperationType();
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dataDb);
+
+		// For transaction-based deletion, we randomly allow the deletion of a range of tenants
+		state Optional<TenantName> endTenant =
+		    operationType != OperationType::MANAGEMENT_DATABASE && operationType != OperationType::METACLUSTER &&
+		            !beginTenant.startsWith("\xff"_sr) && deterministicRandom()->random01() < 0.2
+		        ? Optional<TenantName>(self->chooseTenantName(false))
+		        : Optional<TenantName>();
 
 		if (endTenant.present() && endTenant < beginTenant) {
 			TenantName temp = beginTenant;
@@ -280,9 +645,21 @@ struct TenantManagementWorkload : TestWorkload {
 		}
 
 		auto itr = self->createdTenants.find(beginTenant);
+
+		// True if the beginTenant should exist and be deletable. This is updated if a deletion fails and gets
+		// retried.
 		state bool alreadyExists = itr != self->createdTenants.end();
+
+		// True if the beginTenant existed at the start of this function
+		state bool existedAtStart = alreadyExists;
+
+		// True if all of the tenants in the range are empty and can be deleted
 		state bool isEmpty = true;
 
+		// True if we expect that some tenant will be deleted
+		state bool anyExists = alreadyExists;
+
+		// Collect a list of all tenants that we expect should be deleted by this operation
 		state std::vector<TenantName> tenants;
 		if (!endTenant.present()) {
 			tenants.push_back(beginTenant);
@@ -291,15 +668,18 @@ struct TenantManagementWorkload : TestWorkload {
 			     itr != self->createdTenants.end() && itr->first < endTenant.get();
 			     ++itr) {
 				tenants.push_back(itr->first);
+				anyExists = true;
 			}
 		}
 
+		// Check whether each tenant is empty.
 		state int tenantIndex;
 		try {
 			if (alreadyExists || endTenant.present()) {
 				for (tenantIndex = 0; tenantIndex < tenants.size(); ++tenantIndex) {
+					// For most tenants, we will delete the contents and make them empty
 					if (deterministicRandom()->random01() < 0.9) {
-						state Transaction clearTr(cx, tenants[tenantIndex]);
+						state Transaction clearTr(self->dataDb, tenants[tenantIndex]);
 						loop {
 							try {
 								clearTr.clear(self->keyName);
@@ -312,7 +692,9 @@ struct TenantManagementWorkload : TestWorkload {
 								wait(clearTr.onError(e));
 							}
 						}
-					} else {
+					}
+					// Otherwise, we will just report the current emptiness of the tenant
+					else {
 						auto itr = self->createdTenants.find(tenants[tenantIndex]);
 						ASSERT(itr != self->createdTenants.end());
 						isEmpty = isEmpty && itr->second.empty;
@@ -329,38 +711,110 @@ struct TenantManagementWorkload : TestWorkload {
 
 		loop {
 			try {
-				if (operationType == OperationType::SPECIAL_KEYS) {
-					tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-					Key key = self->specialKeysTenantMapPrefix.withSuffix(beginTenant);
-					if (endTenant.present()) {
-						tr->clear(KeyRangeRef(key, self->specialKeysTenantMapPrefix.withSuffix(endTenant.get())));
-					} else {
-						tr->clear(key);
-					}
-					wait(tr->commit());
-				} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
-					ASSERT(tenants.size() == 1);
-					for (tenantIndex = 0; tenantIndex != tenants.size(); ++tenantIndex) {
-						wait(TenantAPI::deleteTenant(cx.getReference(), tenants[tenantIndex]));
-					}
-				} else {
-					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					std::vector<Future<Void>> deleteFutures;
-					for (tenantIndex = 0; tenantIndex != tenants.size(); ++tenantIndex) {
-						deleteFutures.push_back(TenantAPI::deleteTenantTransaction(tr, tenants[tenantIndex]));
+				// Attempt to delete the tenant(s)
+				state bool retried = false;
+				loop {
+					try {
+						state Version beforeVersion = wait(self->getReadVersion(tr));
+						Optional<Void> result =
+						    wait(timeout(deleteTenantImpl(tr, beginTenant, endTenant, tenants, operationType, self),
+						                 deterministicRandom()->randomInt(1, 30)));
+
+						if (result.present()) {
+							if (anyExists) {
+								if (self->oldestDeletionVersion == 0 && !tenants.empty()) {
+									tr->reset();
+									Version afterVersion = wait(self->getReadVersion(tr));
+									self->oldestDeletionVersion = afterVersion;
+								}
+								self->newestDeletionVersion = beforeVersion;
+							}
+
+							// Database operations shouldn't get here if the tenant didn't exist
+							ASSERT(operationType == OperationType::SPECIAL_KEYS ||
+							       operationType == OperationType::MANAGEMENT_TRANSACTION || alreadyExists);
+							break;
+						}
+
+						retried = true;
+						tr->reset();
+					} catch (Error& e) {
+						// If we retried the deletion after our initial attempt succeeded, then we proceed with the
+						// rest of the deletion steps normally. Otherwise, the deletion happened elsewhere and we
+						// failed here, so we can rethrow the error.
+						if (e.code() == error_code_tenant_not_found && existedAtStart) {
+							ASSERT(operationType == OperationType::METACLUSTER ||
+							       operationType == OperationType::MANAGEMENT_DATABASE);
+							ASSERT(retried);
+							break;
+						} else if (e.code() == error_code_grv_proxy_memory_limit_exceeded ||
+						           e.code() == error_code_batch_transaction_throttled) {
+							// GRV proxy returns an error
+							wait(tr->onError(e));
+							continue;
+						} else {
+							throw;
+						}
 					}
 
-					wait(waitForAll(deleteFutures));
-					wait(tr->commit());
+					if (!tenants.empty()) {
+						// Check the state of the first deleted tenant
+						Optional<TenantMapEntry> resultEntry =
+						    wait(self->tryGetTenant(*tenants.begin(), operationType));
+
+						if (!resultEntry.present()) {
+							alreadyExists = false;
+						} else if (resultEntry.get().tenantState == TenantState::REMOVING) {
+							ASSERT(operationType == OperationType::METACLUSTER);
+						} else {
+							ASSERT(resultEntry.get().tenantState == TenantState::READY);
+						}
+					}
 				}
 
-				if (!alreadyExists && !endTenant.present() && operationType != OperationType::MANAGEMENT_DATABASE) {
+				// The management transaction operation is a no-op if there are no tenants to delete in a range
+				// delete
+				if (tenants.size() == 0 && operationType == OperationType::MANAGEMENT_TRANSACTION) {
 					return Void();
 				}
 
-				ASSERT(alreadyExists || endTenant.present());
+				// The special keys operation is a no-op if the begin and end tenant are equal (i.e. the range is
+				// empty)
+				if (endTenant.present() && beginTenant == endTenant.get() &&
+				    operationType == OperationType::SPECIAL_KEYS) {
+					return Void();
+				}
+
+				// Check that using the wrong deletion type fails depending on whether we are using a metacluster
+				ASSERT(self->useMetacluster == (operationType == OperationType::METACLUSTER));
+
+				// Transaction-based operations do not fail if the tenant isn't present. If we attempted to delete a
+				// single tenant that didn't exist, we can just return.
+				if (!existedAtStart && !endTenant.present() &&
+				    (operationType == OperationType::MANAGEMENT_TRANSACTION ||
+				     operationType == OperationType::SPECIAL_KEYS)) {
+					return Void();
+				}
+
+				ASSERT(existedAtStart || endTenant.present());
+
+				// Deletion should not succeed if any tenant in the range wasn't empty
 				ASSERT(isEmpty);
+
+				// Update our local state to remove the deleted tenants
 				for (auto tenant : tenants) {
+					auto itr = self->createdTenants.find(tenant);
+					ASSERT(itr != self->createdTenants.end());
+
+					// If the tenant group has no tenants remaining, stop tracking it
+					if (itr->second.tenantGroup.present()) {
+						auto tenantGroupItr = self->createdTenantGroups.find(itr->second.tenantGroup.get());
+						ASSERT(tenantGroupItr != self->createdTenantGroups.end());
+						if (--tenantGroupItr->second.tenantCount == 0) {
+							self->createdTenantGroups.erase(tenantGroupItr);
+						}
+					}
+
 					self->createdTenants.erase(tenant);
 				}
 				return Void();
@@ -368,9 +822,16 @@ struct TenantManagementWorkload : TestWorkload {
 				if (e.code() == error_code_tenant_not_empty) {
 					ASSERT(!isEmpty);
 					return Void();
-				} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
+				} else if (e.code() == error_code_invalid_metacluster_operation) {
+					ASSERT(operationType == OperationType::METACLUSTER != self->useMetacluster);
+					return Void();
+				}
+
+				// Database-based operations do not need to be retried
+				else if (operationType == OperationType::MANAGEMENT_DATABASE ||
+				         operationType == OperationType::METACLUSTER) {
 					if (e.code() == error_code_tenant_not_found) {
-						ASSERT(!alreadyExists && !endTenant.present());
+						ASSERT(!existedAtStart && !endTenant.present());
 					} else {
 						TraceEvent(SevError, "DeleteTenantFailure")
 						    .error(e)
@@ -378,7 +839,10 @@ struct TenantManagementWorkload : TestWorkload {
 						    .detail("EndTenant", endTenant);
 					}
 					return Void();
-				} else {
+				}
+
+				// Transaction-based operations should be retried
+				else {
 					try {
 						wait(tr->onError(e));
 					} catch (Error& e) {
@@ -393,17 +857,24 @@ struct TenantManagementWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<Void> checkTenant(Database cx,
-	                               TenantManagementWorkload* self,
-	                               TenantName tenant,
-	                               TenantState tenantState) {
-		state Transaction tr(cx, tenant);
+	// Performs some validation on a tenant's contents
+	ACTOR static Future<Void> checkTenantContents(TenantManagementWorkload* self,
+	                                              TenantName tenant,
+	                                              TenantData tenantData) {
+		state Transaction tr(self->dataDb, tenant);
 		loop {
 			try {
+				// We only every store a single key in each tenant. Therefore we expect a range read of the entire
+				// tenant to return either 0 or 1 keys, depending on whether that key has been set.
 				state RangeResult result = wait(tr.getRange(KeyRangeRef(""_sr, "\xff"_sr), 2));
-				if (tenantState.empty) {
+
+				// An empty tenant should have no data
+				if (tenantData.empty) {
 					ASSERT(result.size() == 0);
-				} else {
+				}
+				// A non-empty tenant should have our single key. The value of that key should be the name of the
+				// tenant.
+				else {
 					ASSERT(result.size() == 1);
 					ASSERT(result[0].key == self->keyName);
 					ASSERT(result[0].value == tenant);
@@ -417,69 +888,113 @@ struct TenantManagementWorkload : TestWorkload {
 		return Void();
 	}
 
+	// Convert the JSON document returned by the special-key space when reading tenant metadata
+	// into a TenantMapEntry
 	static TenantMapEntry jsonToTenantMapEntry(ValueRef tenantJson) {
 		json_spirit::mValue jsonObject;
 		json_spirit::read_string(tenantJson.toString(), jsonObject);
 		JSONDoc jsonDoc(jsonObject);
 
 		int64_t id;
+
 		std::string prefix;
 		std::string base64Prefix;
 		std::string printablePrefix;
+		std::string tenantStateStr;
+		std::string base64TenantGroup;
+		std::string printableTenantGroup;
+		bool encrypted;
+		std::string assignedClusterStr;
+
 		jsonDoc.get("id", id);
 		jsonDoc.get("prefix.base64", base64Prefix);
 		jsonDoc.get("prefix.printable", printablePrefix);
+		jsonDoc.get("prefix.encrypted", encrypted);
 
 		prefix = base64::decoder::from_string(base64Prefix);
 		ASSERT(prefix == unprintable(printablePrefix));
 
-		Key prefixKey = KeyRef(prefix);
-		TenantMapEntry entry(id, prefixKey.substr(0, prefixKey.size() - 8));
+		jsonDoc.get("tenant_state", tenantStateStr);
 
-		ASSERT(entry.prefix == prefixKey);
+		Optional<TenantGroupName> tenantGroup;
+		if (jsonDoc.tryGet("tenant_group.base64", base64TenantGroup)) {
+			jsonDoc.get("tenant_group.printable", printableTenantGroup);
+			std::string tenantGroupStr = base64::decoder::from_string(base64TenantGroup);
+			ASSERT(tenantGroupStr == unprintable(printableTenantGroup));
+			tenantGroup = TenantGroupNameRef(tenantGroupStr);
+		}
+
+		Optional<ClusterName> assignedCluster;
+		if (jsonDoc.tryGet("assigned_cluster", assignedClusterStr)) {
+			assignedCluster = ClusterNameRef(assignedClusterStr);
+		}
+
+		TenantMapEntry entry(id, TenantMapEntry::stringToTenantState(tenantStateStr), tenantGroup, encrypted);
+		ASSERT(entry.prefix == prefix);
 		return entry;
 	}
 
-	ACTOR Future<Void> getTenant(Database cx, TenantManagementWorkload* self) {
+	// Gets the metadata for a tenant using the specified operation type
+	ACTOR static Future<TenantMapEntry> getTenantImpl(Reference<ReadYourWritesTransaction> tr,
+	                                                  TenantName tenant,
+	                                                  OperationType operationType,
+	                                                  TenantManagementWorkload* self) {
+		state TenantMapEntry entry;
+		if (operationType == OperationType::SPECIAL_KEYS) {
+			Key key = self->specialKeysTenantMapPrefix.withSuffix(tenant);
+			Optional<Value> value = wait(tr->get(key));
+			if (!value.present()) {
+				throw tenant_not_found();
+			}
+			entry = TenantManagementWorkload::jsonToTenantMapEntry(value.get());
+		} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
+			wait(store(entry, TenantAPI::getTenant(self->dataDb.getReference(), tenant)));
+		} else if (operationType == OperationType::MANAGEMENT_TRANSACTION) {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			wait(store(entry, TenantAPI::getTenantTransaction(tr, tenant)));
+		} else {
+			wait(store(entry, MetaclusterAPI::getTenant(self->mvDb, tenant)));
+		}
+
+		return entry;
+	}
+
+	ACTOR static Future<Void> getTenant(TenantManagementWorkload* self) {
 		state TenantName tenant = self->chooseTenantName(true);
+		state OperationType operationType = self->randomOperationType();
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dataDb);
+
+		// True if the tenant should should exist and return a result
 		auto itr = self->createdTenants.find(tenant);
-		state bool alreadyExists = itr != self->createdTenants.end();
-		state TenantState tenantState = itr->second;
-		state OperationType operationType = TenantManagementWorkload::randomOperationType();
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		state bool alreadyExists = itr != self->createdTenants.end() &&
+		                           !(operationType == OperationType::METACLUSTER && !self->useMetacluster);
+		state TenantData tenantData = alreadyExists ? itr->second : TenantData();
 
 		loop {
 			try {
-				state TenantMapEntry entry;
-				if (operationType == OperationType::SPECIAL_KEYS) {
-					Key key = self->specialKeysTenantMapPrefix.withSuffix(tenant);
-					Optional<Value> value = wait(tr->get(key));
-					if (!value.present()) {
-						throw tenant_not_found();
-					}
-					entry = TenantManagementWorkload::jsonToTenantMapEntry(value.get());
-				} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
-					TenantMapEntry _entry = wait(TenantAPI::getTenant(cx.getReference(), tenant));
-					entry = _entry;
-				} else {
-					tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-					TenantMapEntry _entry = wait(TenantAPI::getTenantTransaction(tr, tenant));
-					entry = _entry;
-				}
+				// Get the tenant metadata and check that it matches our local state
+				state TenantMapEntry entry = wait(getTenantImpl(tr, tenant, operationType, self));
 				ASSERT(alreadyExists);
-				ASSERT(entry.id == tenantState.id);
-				wait(self->checkTenant(cx, self, tenant, tenantState));
+				ASSERT(entry.id == tenantData.id);
+				ASSERT(entry.tenantGroup == tenantData.tenantGroup);
+				wait(checkTenantContents(self, tenant, tenantData));
 				return Void();
 			} catch (Error& e) {
-				state bool retry = true;
+				state bool retry = false;
 				state Error error = e;
 
 				if (e.code() == error_code_tenant_not_found) {
 					ASSERT(!alreadyExists);
 					return Void();
-				} else if (operationType != OperationType::MANAGEMENT_DATABASE) {
+				}
+
+				// Transaction-based operations should retry
+				else if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
+				         operationType == OperationType::SPECIAL_KEYS) {
 					try {
+						retry = true;
 						wait(tr->onError(e));
+						retry = true;
 					} catch (Error& e) {
 						error = e;
 						retry = false;
@@ -494,12 +1009,42 @@ struct TenantManagementWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<Void> listTenants(Database cx, TenantManagementWorkload* self) {
+	// Gets a list of tenants using the specified operation type
+	ACTOR static Future<std::vector<std::pair<TenantName, TenantMapEntry>>> listTenantsImpl(
+	    Reference<ReadYourWritesTransaction> tr,
+	    TenantName beginTenant,
+	    TenantName endTenant,
+	    int limit,
+	    OperationType operationType,
+	    TenantManagementWorkload* self) {
+		state std::vector<std::pair<TenantName, TenantMapEntry>> tenants;
+
+		if (operationType == OperationType::SPECIAL_KEYS) {
+			KeyRange range = KeyRangeRef(beginTenant, endTenant).withPrefix(self->specialKeysTenantMapPrefix);
+			RangeResult results = wait(tr->getRange(range, limit));
+			for (auto result : results) {
+				tenants.push_back(std::make_pair(result.key.removePrefix(self->specialKeysTenantMapPrefix),
+				                                 TenantManagementWorkload::jsonToTenantMapEntry(result.value)));
+			}
+		} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
+			wait(store(tenants, TenantAPI::listTenants(self->dataDb.getReference(), beginTenant, endTenant, limit)));
+		} else if (operationType == OperationType::MANAGEMENT_TRANSACTION) {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			wait(store(tenants, TenantAPI::listTenantsTransaction(tr, beginTenant, endTenant, limit)));
+		} else {
+			wait(store(tenants, MetaclusterAPI::listTenants(self->mvDb, beginTenant, endTenant, limit)));
+		}
+
+		return tenants;
+	}
+
+	ACTOR static Future<Void> listTenants(TenantManagementWorkload* self) {
 		state TenantName beginTenant = self->chooseTenantName(false);
 		state TenantName endTenant = self->chooseTenantName(false);
-		state int limit = std::min(CLIENT_KNOBS->TOO_MANY, deterministicRandom()->randomInt(1, self->maxTenants * 2));
-		state OperationType operationType = TenantManagementWorkload::randomOperationType();
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		state int limit = std::min(CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1,
+		                           deterministicRandom()->randomInt(1, self->maxTenants * 2));
+		state OperationType operationType = self->randomOperationType();
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dataDb);
 
 		if (beginTenant > endTenant) {
 			std::swap(beginTenant, endTenant);
@@ -507,27 +1052,20 @@ struct TenantManagementWorkload : TestWorkload {
 
 		loop {
 			try {
-				state std::map<TenantName, TenantMapEntry> tenants;
-				if (operationType == OperationType::SPECIAL_KEYS) {
-					KeyRange range = KeyRangeRef(beginTenant, endTenant).withPrefix(self->specialKeysTenantMapPrefix);
-					RangeResult results = wait(tr->getRange(range, limit));
-					for (auto result : results) {
-						tenants[result.key.removePrefix(self->specialKeysTenantMapPrefix)] =
-						    TenantManagementWorkload::jsonToTenantMapEntry(result.value);
-					}
-				} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
-					std::map<TenantName, TenantMapEntry> _tenants =
-					    wait(TenantAPI::listTenants(cx.getReference(), beginTenant, endTenant, limit));
-					tenants = _tenants;
-				} else {
-					tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-					std::map<TenantName, TenantMapEntry> _tenants =
-					    wait(TenantAPI::listTenantsTransaction(tr, beginTenant, endTenant, limit));
-					tenants = _tenants;
+				// Attempt to read the chosen list of tenants
+				state std::vector<std::pair<TenantName, TenantMapEntry>> tenants =
+				    wait(listTenantsImpl(tr, beginTenant, endTenant, limit, operationType, self));
+
+				// Attempting to read the list of tenants using the metacluster API in a non-metacluster should
+				// return nothing in this test
+				if (operationType == OperationType::METACLUSTER && !self->useMetacluster) {
+					ASSERT(tenants.size() == 0);
+					return Void();
 				}
 
 				ASSERT(tenants.size() <= limit);
 
+				// Compare the resulting tenant list to the list we expected to get
 				auto localItr = self->createdTenants.lower_bound(beginTenant);
 				auto tenantMapItr = tenants.begin();
 				for (; tenantMapItr != tenants.end(); ++tenantMapItr, ++localItr) {
@@ -535,19 +1073,19 @@ struct TenantManagementWorkload : TestWorkload {
 					ASSERT(localItr->first == tenantMapItr->first);
 				}
 
-				if (!(tenants.size() == limit || localItr == self->createdTenants.end())) {
-					for (auto tenant : self->createdTenants) {
-						TraceEvent("ExistingTenant").detail("Tenant", tenant.first);
-					}
-				}
+				// Make sure the list terminated at the right spot
 				ASSERT(tenants.size() == limit || localItr == self->createdTenants.end() ||
 				       localItr->first >= endTenant);
 				return Void();
 			} catch (Error& e) {
-				state bool retry = true;
+				state bool retry = false;
 				state Error error = e;
-				if (operationType != OperationType::MANAGEMENT_DATABASE) {
+
+				// Transaction-based operations need to be retried
+				if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
+				    operationType == OperationType::SPECIAL_KEYS) {
 					try {
+						retry = true;
 						wait(tr->onError(e));
 					} catch (Error& e) {
 						error = e;
@@ -567,20 +1105,114 @@ struct TenantManagementWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<Void> renameTenant(Database cx, TenantManagementWorkload* self) {
-		// Currently only supporting MANAGEMENT_DATABASE op, so numTenants should always be 1
-		// state OperationType operationType = TenantManagementWorkload::randomOperationType();
-		int numTenants = 1;
+	// Helper function that checks tenant keyspace and updates internal Tenant Map after a rename operation
+	ACTOR static Future<Void> verifyTenantRename(TenantManagementWorkload* self,
+	                                             TenantName oldTenantName,
+	                                             TenantName newTenantName) {
+		state Optional<TenantMapEntry> oldTenantEntry =
+		    wait(TenantAPI::tryGetTenant(self->dataDb.getReference(), oldTenantName));
+		state Optional<TenantMapEntry> newTenantEntry =
+		    wait(TenantAPI::tryGetTenant(self->dataDb.getReference(), newTenantName));
+		ASSERT(!oldTenantEntry.present());
+		ASSERT(newTenantEntry.present());
+		TenantData tData = self->createdTenants[oldTenantName];
+		self->createdTenants[newTenantName] = tData;
+		self->createdTenants.erase(oldTenantName);
+		state Transaction insertTr(self->dataDb, newTenantName);
+		if (!tData.empty) {
+			loop {
+				try {
+					insertTr.set(self->keyName, newTenantName);
+					wait(insertTr.commit());
+					break;
+				} catch (Error& e) {
+					wait(insertTr.onError(e));
+				}
+			}
+		}
+		return Void();
+	}
 
-		state std::vector<TenantName> oldTenantNames;
-		state std::vector<TenantName> newTenantNames;
+	ACTOR static Future<Void> verifyTenantRenames(TenantManagementWorkload* self,
+	                                              std::map<TenantName, TenantName> tenantRenames) {
+		state std::map<TenantName, TenantName> tenantRenamesCopy = tenantRenames;
+		state std::map<TenantName, TenantName>::iterator iter = tenantRenamesCopy.begin();
+		for (; iter != tenantRenamesCopy.end(); ++iter) {
+			wait(verifyTenantRename(self, iter->first, iter->second));
+			wait(checkTenantContents(self, iter->second, self->createdTenants[iter->second]));
+		}
+		return Void();
+	}
+
+	ACTOR static Future<Void> renameTenantImpl(Reference<ReadYourWritesTransaction> tr,
+	                                           OperationType operationType,
+	                                           std::map<TenantName, TenantName> tenantRenames,
+	                                           bool tenantNotFound,
+	                                           bool tenantExists,
+	                                           bool tenantOverlap,
+	                                           TenantManagementWorkload* self) {
+		if (operationType == OperationType::SPECIAL_KEYS) {
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			for (auto& iter : tenantRenames) {
+				tr->set(self->specialKeysTenantRenamePrefix.withSuffix(iter.first), iter.second);
+			}
+			wait(tr->commit());
+		} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
+			ASSERT(tenantRenames.size() == 1);
+			auto iter = tenantRenames.begin();
+			wait(TenantAPI::renameTenant(self->dataDb.getReference(), iter->first, iter->second));
+			ASSERT(!tenantNotFound && !tenantExists);
+		} else if (operationType == OperationType::MANAGEMENT_TRANSACTION) {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			std::vector<Future<Void>> renameFutures;
+			for (auto& iter : tenantRenames) {
+				renameFutures.push_back(success(TenantAPI::renameTenantTransaction(tr, iter.first, iter.second)));
+			}
+			wait(waitForAll(renameFutures));
+			wait(tr->commit());
+			ASSERT(!tenantNotFound && !tenantExists);
+		} else { // operationType == OperationType::METACLUSTER
+			ASSERT(tenantRenames.size() == 1);
+			auto iter = tenantRenames.begin();
+			wait(MetaclusterAPI::renameTenant(self->mvDb, iter->first, iter->second));
+		}
+		return Void();
+	}
+
+	ACTOR static Future<Void> renameTenant(TenantManagementWorkload* self) {
+		state OperationType operationType = self->randomOperationType();
+		state int numTenants = 1;
+		state Reference<ReadYourWritesTransaction> tr = self->dataDb->createTransaction();
+
+		if (operationType == OperationType::SPECIAL_KEYS || operationType == OperationType::MANAGEMENT_TRANSACTION) {
+			numTenants = deterministicRandom()->randomInt(1, 5);
+		}
+
+		state std::map<TenantName, TenantName> tenantRenames;
+		state std::set<TenantName> allTenantNames;
+
+		// Tenant Error flags
 		state bool tenantExists = false;
 		state bool tenantNotFound = false;
+		state bool tenantOverlap = false;
+		state bool unknownResult = false;
+
 		for (int i = 0; i < numTenants; ++i) {
 			TenantName oldTenant = self->chooseTenantName(false);
 			TenantName newTenant = self->chooseTenantName(false);
-			newTenantNames.push_back(newTenant);
-			oldTenantNames.push_back(oldTenant);
+			bool checkOverlap =
+			    oldTenant == newTenant || allTenantNames.count(oldTenant) || allTenantNames.count(newTenant);
+			// renameTenantTransaction does not handle rename collisions:
+			// reject the rename here if it has overlap and we are doing a transaction operation
+			// and then pick another combination
+			if (checkOverlap && operationType == OperationType::MANAGEMENT_TRANSACTION) {
+				--i;
+				continue;
+			}
+			tenantOverlap = tenantOverlap || checkOverlap;
+			tenantRenames[oldTenant] = newTenant;
+			allTenantNames.insert(oldTenant);
+			allTenantNames.insert(newTenant);
 			if (!self->createdTenants.count(oldTenant)) {
 				tenantNotFound = true;
 			}
@@ -591,88 +1223,536 @@ struct TenantManagementWorkload : TestWorkload {
 
 		loop {
 			try {
-				ASSERT(oldTenantNames.size() == 1);
-				state int tenantIndex = 0;
-				for (; tenantIndex != oldTenantNames.size(); ++tenantIndex) {
-					state TenantName oldTenantName = oldTenantNames[tenantIndex];
-					state TenantName newTenantName = newTenantNames[tenantIndex];
-					// Perform rename, then check against the DB for the new results
-					wait(TenantAPI::renameTenant(cx.getReference(), oldTenantName, newTenantName));
-					ASSERT(!tenantNotFound && !tenantExists);
-					state Optional<TenantMapEntry> oldTenantEntry =
-					    wait(TenantAPI::tryGetTenant(cx.getReference(), oldTenantName));
-					state Optional<TenantMapEntry> newTenantEntry =
-					    wait(TenantAPI::tryGetTenant(cx.getReference(), newTenantName));
-					ASSERT(!oldTenantEntry.present());
-					ASSERT(newTenantEntry.present());
-
-					// Update Internal Tenant Map and check for correctness
-					TenantState tState = self->createdTenants[oldTenantName];
-					self->createdTenants[newTenantName] = tState;
-					self->createdTenants.erase(oldTenantName);
-					if (!tState.empty) {
-						state Transaction insertTr(cx, newTenantName);
-						loop {
-							try {
-								insertTr.set(self->keyName, newTenantName);
-								wait(insertTr.commit());
-								break;
-							} catch (Error& e) {
-								wait(insertTr.onError(e));
-							}
-						}
-					}
-					wait(self->checkTenant(cx, self, newTenantName, self->createdTenants[newTenantName]));
-				}
+				wait(renameTenantImpl(
+				    tr, operationType, tenantRenames, tenantNotFound, tenantExists, tenantOverlap, self));
+				wait(verifyTenantRenames(self, tenantRenames));
+				// Check that using the wrong rename API fails depending on whether we are using a metacluster
+				ASSERT(self->useMetacluster == (operationType == OperationType::METACLUSTER));
 				return Void();
 			} catch (Error& e) {
-				ASSERT(oldTenantNames.size() == 1);
 				if (e.code() == error_code_tenant_not_found) {
-					TraceEvent("RenameTenantOldTenantNotFound")
-					    .detail("OldTenantName", oldTenantNames[0])
-					    .detail("NewTenantName", newTenantNames[0]);
-					ASSERT(tenantNotFound);
+					if (unknownResult) {
+						wait(verifyTenantRenames(self, tenantRenames));
+					} else {
+						ASSERT(tenantNotFound);
+					}
+					return Void();
 				} else if (e.code() == error_code_tenant_already_exists) {
-					TraceEvent("RenameTenantNewTenantAlreadyExists")
-					    .detail("OldTenantName", oldTenantNames[0])
-					    .detail("NewTenantName", newTenantNames[0]);
-					ASSERT(tenantExists);
+					if (unknownResult) {
+						wait(verifyTenantRenames(self, tenantRenames));
+					} else {
+						ASSERT(tenantExists);
+					}
+					return Void();
+				} else if (e.code() == error_code_special_keys_api_failure) {
+					ASSERT(tenantOverlap);
+					return Void();
+				} else if (e.code() == error_code_invalid_metacluster_operation) {
+					ASSERT(operationType == OperationType::METACLUSTER != self->useMetacluster);
+					return Void();
+				} else if (e.code() == error_code_cluster_no_capacity) {
+					// This error should only occur on metacluster due to the multi-stage process.
+					// Having temporary tenants may exceed capacity, so we disallow the rename.
+					ASSERT(operationType == OperationType::METACLUSTER);
+					return Void();
 				} else {
-					TraceEvent(SevError, "RenameTenantFailure")
-					    .error(e)
-					    .detail("OldTenantName", oldTenantNames[0])
-					    .detail("NewTenantName", newTenantNames[0]);
+					try {
+						// In the case of commit_unknown_result, assume we continue retrying
+						// until it's successful. Next loop around may throw error because it's
+						// already been moved, so account for that and update internal map as needed.
+						if (e.code() == error_code_commit_unknown_result) {
+							ASSERT(operationType != OperationType::MANAGEMENT_DATABASE &&
+							       operationType != OperationType::METACLUSTER);
+							unknownResult = true;
+						}
+						wait(tr->onError(e));
+					} catch (Error& e) {
+						TraceEvent(SevError, "RenameTenantFailure")
+						    .error(e)
+						    .detail("TenantRenames", describe(tenantRenames));
+						return Void();
+					}
 				}
-				return Void();
 			}
 		}
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
+	// Changes the configuration of a tenant
+	ACTOR static Future<Void> configureTenantImpl(Reference<ReadYourWritesTransaction> tr,
+	                                              TenantName tenant,
+	                                              std::map<Standalone<StringRef>, Optional<Value>> configParameters,
+	                                              OperationType operationType,
+	                                              bool specialKeysUseInvalidTuple,
+	                                              TenantManagementWorkload* self) {
+		if (operationType == OperationType::SPECIAL_KEYS) {
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			for (auto const& [config, value] : configParameters) {
+				Tuple t;
+				if (specialKeysUseInvalidTuple) {
+					// Wrong number of items
+					if (deterministicRandom()->coinflip()) {
+						int numItems = deterministicRandom()->randomInt(0, 3);
+						if (numItems > 0) {
+							t.append(tenant);
+						}
+						if (numItems > 1) {
+							t.append(config).append(""_sr);
+						}
+					}
+					// Wrong data types
+					else {
+						if (deterministicRandom()->coinflip()) {
+							t.append(0).append(config);
+						} else {
+							t.append(tenant).append(0);
+						}
+					}
+				} else {
+					t.append(tenant).append(config);
+				}
+				if (value.present()) {
+					tr->set(self->specialKeysTenantConfigPrefix.withSuffix(t.pack()), value.get());
+				} else {
+					tr->clear(self->specialKeysTenantConfigPrefix.withSuffix(t.pack()));
+				}
+			}
+
+			wait(tr->commit());
+			ASSERT(!specialKeysUseInvalidTuple);
+		} else if (operationType == OperationType::METACLUSTER) {
+			wait(MetaclusterAPI::configureTenant(self->mvDb, tenant, configParameters));
+		} else {
+			// We don't have a transaction or database variant of this function
+			ASSERT(false);
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> configureTenant(TenantManagementWorkload* self) {
+		state OperationType operationType =
+		    deterministicRandom()->coinflip() ? OperationType::SPECIAL_KEYS : OperationType::METACLUSTER;
+
+		state TenantName tenant = self->chooseTenantName(true);
+		auto itr = self->createdTenants.find(tenant);
+		state bool exists = itr != self->createdTenants.end();
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dataDb);
+
+		state std::map<Standalone<StringRef>, Optional<Value>> configuration;
+		state Optional<TenantGroupName> newTenantGroup;
+
+		// If true, the options generated may include an unknown option
+		state bool hasInvalidOption = deterministicRandom()->random01() < 0.1;
+
+		// True if any tenant group name starts with \xff
+		state bool hasSystemTenantGroup = false;
+
+		state bool specialKeysUseInvalidTuple =
+		    operationType == OperationType::SPECIAL_KEYS && deterministicRandom()->random01() < 0.1;
+
+		// Generate a tenant group. Sometimes do this at the same time that we include an invalid option to ensure
+		// that the configure function still fails
+		if (!hasInvalidOption || deterministicRandom()->coinflip()) {
+			newTenantGroup = self->chooseTenantGroup(true);
+			hasSystemTenantGroup = hasSystemTenantGroup || newTenantGroup.orDefault(""_sr).startsWith("\xff"_sr);
+			configuration["tenant_group"_sr] = newTenantGroup;
+		}
+		if (hasInvalidOption) {
+			configuration["invalid_option"_sr] = ""_sr;
+		}
+
+		loop {
+			try {
+				wait(configureTenantImpl(tr, tenant, configuration, operationType, specialKeysUseInvalidTuple, self));
+
+				ASSERT(exists);
+				ASSERT(!hasInvalidOption);
+				ASSERT(!hasSystemTenantGroup);
+				ASSERT(!specialKeysUseInvalidTuple);
+
+				auto itr = self->createdTenants.find(tenant);
+				if (itr->second.tenantGroup.present()) {
+					auto tenantGroupItr = self->createdTenantGroups.find(itr->second.tenantGroup.get());
+					ASSERT(tenantGroupItr != self->createdTenantGroups.end());
+					if (--tenantGroupItr->second.tenantCount == 0) {
+						self->createdTenantGroups.erase(tenantGroupItr);
+					}
+				}
+				if (newTenantGroup.present()) {
+					self->createdTenantGroups[newTenantGroup.get()].tenantCount++;
+				}
+				itr->second.tenantGroup = newTenantGroup;
+				return Void();
+			} catch (Error& e) {
+				state Error error = e;
+				if (e.code() == error_code_tenant_not_found) {
+					ASSERT(!exists);
+					return Void();
+				} else if (e.code() == error_code_special_keys_api_failure) {
+					ASSERT(hasInvalidOption || specialKeysUseInvalidTuple);
+					return Void();
+				} else if (e.code() == error_code_invalid_tenant_configuration) {
+					ASSERT(hasInvalidOption);
+					return Void();
+				} else if (e.code() == error_code_invalid_metacluster_operation) {
+					ASSERT(operationType == OperationType::METACLUSTER != self->useMetacluster);
+					return Void();
+				} else if (e.code() == error_code_invalid_tenant_group_name) {
+					ASSERT(hasSystemTenantGroup);
+					return Void();
+				}
+
+				try {
+					wait(tr->onError(e));
+				} catch (Error&) {
+					TraceEvent(SevError, "ConfigureTenantFailure").error(error).detail("TenantName", tenant);
+					return Void();
+				}
+			}
+		}
+	}
+
+	// Gets the metadata for a tenant group using the specified operation type
+	ACTOR static Future<Optional<TenantGroupEntry>> getTenantGroupImpl(Reference<ReadYourWritesTransaction> tr,
+	                                                                   TenantGroupName tenant,
+	                                                                   OperationType operationType,
+	                                                                   TenantManagementWorkload* self) {
+		state Optional<TenantGroupEntry> entry;
+		if (operationType == OperationType::MANAGEMENT_DATABASE) {
+			wait(store(entry, TenantAPI::tryGetTenantGroup(self->dataDb.getReference(), tenant)));
+		} else if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
+		           operationType == OperationType::SPECIAL_KEYS) {
+			// There is no special-keys interface for reading tenant groups currently, so read them
+			// using the TenantAPI.
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			wait(store(entry, TenantAPI::tryGetTenantGroupTransaction(tr, tenant)));
+		} else {
+			wait(store(entry, MetaclusterAPI::tryGetTenantGroup(self->mvDb, tenant)));
+		}
+
+		return entry;
+	}
+
+	ACTOR static Future<Void> getTenantGroup(TenantManagementWorkload* self) {
+		state TenantGroupName tenantGroup = self->chooseTenantGroup(true, false).get();
+		state OperationType operationType = self->randomOperationType();
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dataDb);
+
+		// True if the tenant group should should exist and return a result
+		auto itr = self->createdTenantGroups.find(tenantGroup);
+		state bool alreadyExists = itr != self->createdTenantGroups.end() &&
+		                           !(operationType == OperationType::METACLUSTER && !self->useMetacluster);
+
+		loop {
+			try {
+				// Get the tenant group metadata and check that it matches our local state
+				state Optional<TenantGroupEntry> entry = wait(getTenantGroupImpl(tr, tenantGroup, operationType, self));
+				ASSERT(alreadyExists == entry.present());
+				if (entry.present()) {
+					ASSERT(entry.get().assignedCluster.present() == (operationType == OperationType::METACLUSTER));
+				}
+				return Void();
+			} catch (Error& e) {
+				state bool retry = false;
+				state Error error = e;
+
+				// Transaction-based operations should retry
+				if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
+				    operationType == OperationType::SPECIAL_KEYS) {
+					try {
+						wait(tr->onError(e));
+						retry = true;
+					} catch (Error& e) {
+						error = e;
+						retry = false;
+					}
+				}
+
+				if (!retry) {
+					TraceEvent(SevError, "GetTenantGroupFailure").error(error).detail("TenantGroupName", tenantGroup);
+					return Void();
+				}
+			}
+		}
+	}
+
+	// Gets a list of tenant groups using the specified operation type
+	ACTOR static Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroupsImpl(
+	    Reference<ReadYourWritesTransaction> tr,
+	    TenantGroupName beginTenantGroup,
+	    TenantGroupName endTenantGroup,
+	    int limit,
+	    OperationType operationType,
+	    TenantManagementWorkload* self) {
+		state std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups;
+
+		if (operationType == OperationType::MANAGEMENT_DATABASE) {
+			wait(store(
+			    tenantGroups,
+			    TenantAPI::listTenantGroups(self->dataDb.getReference(), beginTenantGroup, endTenantGroup, limit)));
+		} else if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
+		           operationType == OperationType::SPECIAL_KEYS) {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			wait(store(tenantGroups,
+			           TenantAPI::listTenantGroupsTransaction(tr, beginTenantGroup, endTenantGroup, limit)));
+		} else {
+			wait(store(tenantGroups,
+			           MetaclusterAPI::listTenantGroups(self->mvDb, beginTenantGroup, endTenantGroup, limit)));
+		}
+
+		return tenantGroups;
+	}
+
+	ACTOR static Future<Void> listTenantGroups(TenantManagementWorkload* self) {
+		state TenantGroupName beginTenantGroup = self->chooseTenantGroup(false, false).get();
+		state TenantGroupName endTenantGroup = self->chooseTenantGroup(false, false).get();
+		state int limit = std::min(CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1,
+		                           deterministicRandom()->randomInt(1, self->maxTenants * 2));
+		state OperationType operationType = self->randomOperationType();
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dataDb);
+
+		if (beginTenantGroup > endTenantGroup) {
+			std::swap(beginTenantGroup, endTenantGroup);
+		}
+
+		loop {
+			try {
+				// Attempt to read the chosen list of tenant groups
+				state std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups =
+				    wait(listTenantGroupsImpl(tr, beginTenantGroup, endTenantGroup, limit, operationType, self));
+
+				// Attempting to read the list of tenant groups using the metacluster API in a non-metacluster should
+				// return nothing in this test
+				if (operationType == OperationType::METACLUSTER && !self->useMetacluster) {
+					ASSERT(tenantGroups.size() == 0);
+					return Void();
+				}
+
+				ASSERT(tenantGroups.size() <= limit);
+
+				// Compare the resulting tenant list to the list we expected to get
+				auto localItr = self->createdTenantGroups.lower_bound(beginTenantGroup);
+				auto tenantMapItr = tenantGroups.begin();
+				for (; tenantMapItr != tenantGroups.end(); ++tenantMapItr, ++localItr) {
+					ASSERT(localItr != self->createdTenantGroups.end());
+					ASSERT(localItr->first == tenantMapItr->first);
+				}
+
+				// Make sure the list terminated at the right spot
+				ASSERT(tenantGroups.size() == limit || localItr == self->createdTenantGroups.end() ||
+				       localItr->first >= endTenantGroup);
+				return Void();
+			} catch (Error& e) {
+				state bool retry = false;
+				state Error error = e;
+
+				// Transaction-based operations need to be retried
+				if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
+				    operationType == OperationType::SPECIAL_KEYS) {
+					try {
+						retry = true;
+						wait(tr->onError(e));
+					} catch (Error& e) {
+						error = e;
+						retry = false;
+					}
+				}
+
+				if (!retry) {
+					TraceEvent(SevError, "ListTenantGroupFailure")
+					    .error(error)
+					    .detail("BeginTenant", beginTenantGroup)
+					    .detail("EndTenant", endTenantGroup);
+
+					return Void();
+				}
+			}
+		}
+	}
+
+	Future<Void> start(Database const& cx) override {
+		if (clientId == 0 || !singleClient) {
+			return _start(cx, this);
+		} else {
+			return Void();
+		}
+	}
+
 	ACTOR Future<Void> _start(Database cx, TenantManagementWorkload* self) {
 		state double start = now();
+
+		// Run a random sequence of tenant management operations for the duration of the test
 		while (now() < start + self->testDuration) {
-			state int operation = deterministicRandom()->randomInt(0, 5);
+			state int operation = deterministicRandom()->randomInt(0, 8);
 			if (operation == 0) {
-				wait(self->createTenant(cx, self));
+				wait(createTenant(self));
 			} else if (operation == 1) {
-				wait(self->deleteTenant(cx, self));
+				wait(deleteTenant(self));
 			} else if (operation == 2) {
-				wait(self->getTenant(cx, self));
+				wait(getTenant(self));
 			} else if (operation == 3) {
-				wait(self->listTenants(cx, self));
-			} else {
-				wait(self->renameTenant(cx, self));
+				wait(listTenants(self));
+			} else if (operation == 4) {
+				wait(renameTenant(self));
+			} else if (operation == 5) {
+				wait(configureTenant(self));
+			} else if (operation == 6) {
+				wait(getTenantGroup(self));
+			} else if (operation == 7) {
+				wait(listTenantGroups(self));
 			}
 		}
 
 		return Void();
 	}
 
-	Future<bool> check(Database const& cx) override { return _check(cx, this); }
-	ACTOR Future<bool> _check(Database cx, TenantManagementWorkload* self) {
-		state Transaction tr(cx);
+	// Verify that the set of tenants in the database matches our local state
+	ACTOR static Future<Void> compareTenants(TenantManagementWorkload* self) {
+		state std::map<TenantName, TenantData>::iterator localItr = self->createdTenants.begin();
+		state std::vector<Future<Void>> checkTenants;
+		state TenantName beginTenant = ""_sr.withPrefix(self->localTenantNamePrefix);
+		state TenantName endTenant = "\xff\xff"_sr.withPrefix(self->localTenantNamePrefix);
 
+		loop {
+			// Read the tenant list from the data cluster.
+			state std::vector<std::pair<TenantName, TenantMapEntry>> dataClusterTenants =
+			    wait(TenantAPI::listTenants(self->dataDb.getReference(), beginTenant, endTenant, 1000));
+
+			auto dataItr = dataClusterTenants.begin();
+
+			TenantNameRef lastTenant;
+			while (dataItr != dataClusterTenants.end()) {
+				ASSERT(localItr != self->createdTenants.end());
+				ASSERT(dataItr->first == localItr->first);
+				ASSERT(dataItr->second.tenantGroup == localItr->second.tenantGroup);
+				ASSERT(dataItr->second.encrypted == localItr->second.encrypted);
+
+				checkTenants.push_back(checkTenantContents(self, dataItr->first, localItr->second));
+				lastTenant = dataItr->first;
+
+				++localItr;
+				++dataItr;
+			}
+
+			if (dataClusterTenants.size() < 1000) {
+				break;
+			} else {
+				beginTenant = keyAfter(lastTenant);
+			}
+		}
+
+		ASSERT(localItr == self->createdTenants.end());
+		wait(waitForAll(checkTenants));
+		return Void();
+	}
+
+	// Check that the given tenant group has the expected number of tenants
+	ACTOR template <class DB>
+	static Future<Void> checkTenantGroupTenantCount(Reference<DB> db, TenantGroupName tenantGroup, int expectedCount) {
+		TenantGroupName const& tenantGroupRef = tenantGroup;
+		int const& expectedCountRef = expectedCount;
+
+		KeyBackedSet<Tuple>::RangeResultType tenants =
+		    wait(runTransaction(db, [tenantGroupRef, expectedCountRef](Reference<typename DB::TransactionT> tr) {
+			    tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			    return TenantMetadata::tenantGroupTenantIndex().getRange(tr,
+			                                                             Tuple::makeTuple(tenantGroupRef),
+			                                                             Tuple::makeTuple(keyAfter(tenantGroupRef)),
+			                                                             expectedCountRef + 1);
+		    }));
+
+		ASSERT(tenants.results.size() == expectedCount && !tenants.more);
+		return Void();
+	}
+
+	// Verify that the set of tenants in the database matches our local state
+	ACTOR static Future<Void> compareTenantGroups(TenantManagementWorkload* self) {
+		state std::map<TenantName, TenantGroupData>::iterator localItr = self->createdTenantGroups.begin();
+		state TenantName beginTenantGroup = ""_sr.withPrefix(self->localTenantGroupNamePrefix);
+		state TenantName endTenantGroup = "\xff\xff"_sr.withPrefix(self->localTenantGroupNamePrefix);
+		state std::vector<Future<Void>> checkTenantGroups;
+
+		loop {
+			// Read the tenant group list from the data cluster.
+			state KeyBackedRangeResult<std::pair<TenantGroupName, TenantGroupEntry>> dataClusterTenantGroups;
+			TenantName const& beginTenantGroupRef = beginTenantGroup;
+			TenantName const& endTenantGroupRef = endTenantGroup;
+			wait(
+			    store(dataClusterTenantGroups,
+			          runTransaction(self->dataDb.getReference(),
+			                         [beginTenantGroupRef, endTenantGroupRef](Reference<ReadYourWritesTransaction> tr) {
+				                         tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				                         return TenantMetadata::tenantGroupMap().getRange(
+				                             tr, beginTenantGroupRef, endTenantGroupRef, 1000);
+			                         })));
+
+			auto dataItr = dataClusterTenantGroups.results.begin();
+
+			TenantGroupNameRef lastTenantGroup;
+			while (dataItr != dataClusterTenantGroups.results.end()) {
+				ASSERT(localItr != self->createdTenantGroups.end());
+				ASSERT(dataItr->first == localItr->first);
+				ASSERT(!dataItr->second.assignedCluster.present());
+				lastTenantGroup = dataItr->first;
+
+				checkTenantGroups.push_back(checkTenantGroupTenantCount(
+				    self->dataDb.getReference(), dataItr->first, localItr->second.tenantCount));
+
+				++localItr;
+				++dataItr;
+			}
+
+			if (!dataClusterTenantGroups.more) {
+				break;
+			} else {
+				beginTenantGroup = keyAfter(lastTenantGroup);
+			}
+		}
+
+		ASSERT(localItr == self->createdTenantGroups.end());
+		return Void();
+	}
+
+	// Check that the tenant tombstones are properly cleaned up
+	ACTOR static Future<Void> checkTombstoneCleanup(TenantManagementWorkload* self) {
+		state Reference<ReadYourWritesTransaction> tr = self->dataDb->createTransaction();
+		loop {
+			try {
+				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+
+				Optional<TenantTombstoneCleanupData> tombstoneCleanupData =
+				    wait(TenantMetadata::tombstoneCleanupData().get(tr));
+
+				if (self->oldestDeletionVersion != 0) {
+					ASSERT(tombstoneCleanupData.present());
+					if (self->newestDeletionVersion - self->oldestDeletionVersion >
+					    CLIENT_KNOBS->TENANT_TOMBSTONE_CLEANUP_INTERVAL * CLIENT_KNOBS->VERSIONS_PER_SECOND) {
+						ASSERT(tombstoneCleanupData.get().tombstonesErasedThrough >= 0);
+					}
+				}
+				break;
+			} catch (Error& e) {
+				wait(tr->onError(e));
+			}
+		}
+
+		return Void();
+	}
+
+	Future<bool> check(Database const& cx) override {
+		if (clientId == 0 || !singleClient) {
+			return _check(cx, this);
+		} else {
+			return true;
+		}
+	}
+
+	ACTOR static Future<bool> _check(Database cx, TenantManagementWorkload* self) {
+		state Transaction tr(self->dataDb);
+
+		// Check that the key we set outside of the tenant is present and has the correct value
+		// This is the same key we set inside some of our tenants, so this checks that no tenant
+		// writes accidentally happened in the raw key-space
 		loop {
 			try {
 				tr.setOption(FDBTransactionOptions::RAW_ACCESS);
@@ -684,34 +1764,19 @@ struct TenantManagementWorkload : TestWorkload {
 			}
 		}
 
-		state std::map<TenantName, TenantState>::iterator itr = self->createdTenants.begin();
-		state std::vector<Future<Void>> checkTenants;
-		state TenantName beginTenant = ""_sr.withPrefix(self->localTenantNamePrefix);
-		state TenantName endTenant = "\xff\xff"_sr.withPrefix(self->localTenantNamePrefix);
+		wait(compareTenants(self) && compareTenantGroups(self));
 
-		loop {
-			std::map<TenantName, TenantMapEntry> tenants =
-			    wait(TenantAPI::listTenants(cx.getReference(), beginTenant, endTenant, 1000));
-
-			TenantNameRef lastTenant;
-			for (auto tenant : tenants) {
-				ASSERT(itr != self->createdTenants.end());
-				ASSERT(tenant.first == itr->first);
-				checkTenants.push_back(self->checkTenant(cx, self, tenant.first, itr->second));
-				lastTenant = tenant.first;
-				++itr;
-			}
-
-			if (tenants.size() < 1000) {
-				break;
-			} else {
-				beginTenant = keyAfter(lastTenant);
-			}
+		if (self->useMetacluster) {
+			// The metacluster consistency check runs the tenant consistency check for each cluster
+			state MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
+			    self->mvDb, AllowPartialMetaclusterOperations::False);
+			wait(metaclusterConsistencyCheck.run());
+			wait(checkTombstoneCleanup(self));
+		} else {
+			state TenantConsistencyCheck<DatabaseContext> tenantConsistencyCheck(self->dataDb.getReference());
+			wait(tenantConsistencyCheck.run());
 		}
 
-		ASSERT(itr == self->createdTenants.end());
-		wait(waitForAll(checkTenants));
-
 		return true;
 	}
 
diff --git a/fdbserver/workloads/ThreadSafety.actor.cpp b/fdbserver/workloads/ThreadSafety.actor.cpp
index b4a8f1e3e8..c3a6103996 100644
--- a/fdbserver/workloads/ThreadSafety.actor.cpp
+++ b/fdbserver/workloads/ThreadSafety.actor.cpp
@@ -123,9 +123,9 @@ struct ThreadSafetyWorkload : TestWorkload {
 
 	ThreadSafetyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), stopped(false) {
 
-		threadsPerClient = getOption(options, LiteralStringRef("threadsPerClient"), 3);
-		threadDuration = getOption(options, LiteralStringRef("threadDuration"), 60.0);
-		numKeys = getOption(options, LiteralStringRef("numKeys"), 100);
+		threadsPerClient = getOption(options, "threadsPerClient"_sr, 3);
+		threadDuration = getOption(options, "threadDuration"_sr, 60.0);
+		numKeys = getOption(options, "numKeys"_sr, 100);
 
 		commitBarrier.setNumRequired(threadsPerClient);
 
@@ -149,7 +149,7 @@ struct ThreadSafetyWorkload : TestWorkload {
 		self->db = dbRef;
 
 		if (deterministicRandom()->coinflip()) {
-			MultiVersionApi::api->selectApiVersion(cx->apiVersion);
+			MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
 			self->db = MultiVersionDatabase::debugCreateFromExistingDatabase(dbRef);
 		}
 
diff --git a/fdbserver/workloads/Throttling.actor.cpp b/fdbserver/workloads/Throttling.actor.cpp
index fc906c8e55..afcff1ea39 100644
--- a/fdbserver/workloads/Throttling.actor.cpp
+++ b/fdbserver/workloads/Throttling.actor.cpp
@@ -83,12 +83,12 @@ struct ThrottlingWorkload : KVWorkload {
 	static constexpr const char* NAME = "Throttling";
 
 	ThrottlingWorkload(WorkloadContext const& wcx) : KVWorkload(wcx), transactionsCommitted(0) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0);
-		actorsPerClient = getOption(options, LiteralStringRef("actorsPerClient"), 10);
-		writesPerTransaction = getOption(options, LiteralStringRef("writesPerTransaction"), 10);
-		readsPerTransaction = getOption(options, LiteralStringRef("readsPerTransaction"), 10);
-		throttlingMultiplier = getOption(options, LiteralStringRef("throttlingMultiplier"), 0.5);
-		int maxBurst = getOption(options, LiteralStringRef("maxBurst"), 1000);
+		testDuration = getOption(options, "testDuration"_sr, 60.0);
+		actorsPerClient = getOption(options, "actorsPerClient"_sr, 10);
+		writesPerTransaction = getOption(options, "writesPerTransaction"_sr, 10);
+		readsPerTransaction = getOption(options, "readsPerTransaction"_sr, 10);
+		throttlingMultiplier = getOption(options, "throttlingMultiplier"_sr, 0.5);
+		int maxBurst = getOption(options, "maxBurst"_sr, 1000);
 		tokenBucket.maxBurst = maxBurst;
 	}
 
@@ -131,14 +131,14 @@ struct ThrottlingWorkload : KVWorkload {
 		state json_spirit::mValue logSchema = readJSONStrictly(JSONSchemas::logHealthSchema.toString()).get_obj();
 		loop {
 			try {
-				RangeResult result = wait(
-				    tr.getRange(prefixRange(LiteralStringRef("\xff\xff/metrics/health/")), CLIENT_KNOBS->TOO_MANY));
+				RangeResult result =
+				    wait(tr.getRange(prefixRange("\xff\xff/metrics/health/"_sr), CLIENT_KNOBS->TOO_MANY));
 				ASSERT(!result.more);
 				for (const auto& [k, v] : result) {
-					ASSERT(k.startsWith(LiteralStringRef("\xff\xff/metrics/health/")));
+					ASSERT(k.startsWith("\xff\xff/metrics/health/"_sr));
 					auto valueObj = readJSONStrictly(v.toString()).get_obj();
-					if (k.removePrefix(LiteralStringRef("\xff\xff/metrics/health/")) == LiteralStringRef("aggregate")) {
-						TEST(true); // Test aggregate health metrics schema
+					if (k.removePrefix("\xff\xff/metrics/health/"_sr) == "aggregate"_sr) {
+						CODE_PROBE(true, "Test aggregate health metrics schema");
 						std::string errorStr;
 						if (!schemaMatch(aggregateSchema, valueObj, errorStr, SevError, true)) {
 							TraceEvent(SevError, "AggregateHealthSchemaValidationFailed")
@@ -148,10 +148,9 @@ struct ThrottlingWorkload : KVWorkload {
 						}
 						auto tpsLimit = valueObj.at("tps_limit").get_real();
 						self->tokenBucket.transactionRate = tpsLimit * self->throttlingMultiplier / self->clientCount;
-					} else if (k.removePrefix(LiteralStringRef("\xff\xff/metrics/health/"))
-					               .startsWith(LiteralStringRef("storage/"))) {
-						TEST(true); // Test storage health metrics schema
-						UID::fromString(k.removePrefix(LiteralStringRef("\xff\xff/metrics/health/storage/"))
+					} else if (k.removePrefix("\xff\xff/metrics/health/"_sr).startsWith("storage/"_sr)) {
+						CODE_PROBE(true, "Test storage health metrics schema");
+						UID::fromString(k.removePrefix("\xff\xff/metrics/health/storage/"_sr)
 						                    .toString()); // Will throw if it's not a valid uid
 						std::string errorStr;
 						if (!schemaMatch(storageSchema, valueObj, errorStr, SevError, true)) {
@@ -160,10 +159,9 @@ struct ThrottlingWorkload : KVWorkload {
 							    .detail("JSON", json_spirit::write_string(json_spirit::mValue(v.toString())));
 							self->correctSpecialKeys = false;
 						}
-					} else if (k.removePrefix(LiteralStringRef("\xff\xff/metrics/health/"))
-					               .startsWith(LiteralStringRef("log/"))) {
-						TEST(true); // Test log health metrics schema
-						UID::fromString(k.removePrefix(LiteralStringRef("\xff\xff/metrics/health/log/"))
+					} else if (k.removePrefix("\xff\xff/metrics/health/"_sr).startsWith("log/"_sr)) {
+						CODE_PROBE(true, "Test log health metrics schema");
+						UID::fromString(k.removePrefix("\xff\xff/metrics/health/log/"_sr)
 						                    .toString()); // Will throw if it's not a valid uid
 						std::string errorStr;
 						if (!schemaMatch(logSchema, valueObj, errorStr, SevError, true)) {
diff --git a/fdbserver/workloads/Throughput.actor.cpp b/fdbserver/workloads/Throughput.actor.cpp
index 099154a171..a8e1d8106b 100644
--- a/fdbserver/workloads/Throughput.actor.cpp
+++ b/fdbserver/workloads/Throughput.actor.cpp
@@ -310,55 +310,52 @@ struct ThroughputWorkload : TestWorkload {
 		auto multi = makeReference<MeasureMulti>();
 		measurer = multi;
 
-		targetLatency = getOption(options, LiteralStringRef("targetLatency"), 0.05);
+		targetLatency = getOption(options, "targetLatency"_sr, 0.05);
 
-		int keyCount = getOption(options, LiteralStringRef("nodeCount"), (uint64_t)100000);
-		int keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 16);
-		int maxValueBytes = getOption(options, LiteralStringRef("valueBytes"), 100);
-		int minValueBytes = getOption(options, LiteralStringRef("minValueBytes"), maxValueBytes);
-		double sweepDuration = getOption(options, LiteralStringRef("sweepDuration"), 0);
-		double sweepDelay = getOption(options, LiteralStringRef("sweepDelay"), 0);
+		int keyCount = getOption(options, "nodeCount"_sr, (uint64_t)100000);
+		int keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
+		int maxValueBytes = getOption(options, "valueBytes"_sr, 100);
+		int minValueBytes = getOption(options, "minValueBytes"_sr, maxValueBytes);
+		double sweepDuration = getOption(options, "sweepDuration"_sr, 0);
+		double sweepDelay = getOption(options, "sweepDelay"_sr, 0);
 
-		auto AType =
-		    Reference<ITransactor>(new RWTransactor(getOption(options, LiteralStringRef("readsPerTransactionA"), 10),
-		                                            getOption(options, LiteralStringRef("writesPerTransactionA"), 0),
-		                                            keyCount,
-		                                            keyBytes,
-		                                            minValueBytes,
-		                                            maxValueBytes));
-		auto BType =
-		    Reference<ITransactor>(new RWTransactor(getOption(options, LiteralStringRef("readsPerTransactionB"), 5),
-		                                            getOption(options, LiteralStringRef("writesPerTransactionB"), 5),
-		                                            keyCount,
-		                                            keyBytes,
-		                                            minValueBytes,
-		                                            maxValueBytes));
+		auto AType = Reference<ITransactor>(new RWTransactor(getOption(options, "readsPerTransactionA"_sr, 10),
+		                                                     getOption(options, "writesPerTransactionA"_sr, 0),
+		                                                     keyCount,
+		                                                     keyBytes,
+		                                                     minValueBytes,
+		                                                     maxValueBytes));
+		auto BType = Reference<ITransactor>(new RWTransactor(getOption(options, "readsPerTransactionB"_sr, 5),
+		                                                     getOption(options, "writesPerTransactionB"_sr, 5),
+		                                                     keyCount,
+		                                                     keyBytes,
+		                                                     minValueBytes,
+		                                                     maxValueBytes));
 
 		if (sweepDuration > 0) {
 			op = Reference<ITransactor>(new SweepTransactor(sweepDuration, sweepDelay, AType, BType));
 		} else {
-			op = Reference<ITransactor>(
-			    new ABTransactor(getOption(options, LiteralStringRef("alpha"), 0.1), AType, BType));
+			op = Reference<ITransactor>(new ABTransactor(getOption(options, "alpha"_sr, 0.1), AType, BType));
 		}
 
-		double measureDelay = getOption(options, LiteralStringRef("measureDelay"), 50.0);
-		double measureDuration = getOption(options, LiteralStringRef("measureDuration"), 10.0);
+		double measureDelay = getOption(options, "measureDelay"_sr, 50.0);
+		double measureDuration = getOption(options, "measureDuration"_sr, 10.0);
 		multi->ms.push_back(Reference<IMeasurer>(new MeasureSinglePeriod(measureDelay, measureDuration)));
 
-		double measurePeriod = getOption(options, LiteralStringRef("measurePeriod"), 0.0);
+		double measurePeriod = getOption(options, "measurePeriod"_sr, 0.0);
 		std::vector<std::string> periodicMetrics =
-		    getOption(options, LiteralStringRef("measurePeriodicMetrics"), std::vector<std::string>());
+		    getOption(options, "measurePeriodicMetrics"_sr, std::vector<std::string>());
 		if (measurePeriod) {
 			ASSERT(periodicMetrics.size() != 0);
 			multi->ms.push_back(Reference<IMeasurer>(new MeasurePeriodically(
 			    measurePeriod, std::set<std::string>(periodicMetrics.begin(), periodicMetrics.end()))));
 		}
 
-		Pgain = getOption(options, LiteralStringRef("ProportionalGain"), 0.1);
-		Igain = getOption(options, LiteralStringRef("IntegralGain"), 0.005);
+		Pgain = getOption(options, "ProportionalGain"_sr, 0.1);
+		Igain = getOption(options, "IntegralGain"_sr, 0.005);
 
 		testDuration = measureDelay + measureDuration;
-		// testDuration = getOption( options, LiteralStringRef("testDuration"), measureDelay + measureDuration );
+		// testDuration = getOption( options, "testDuration"_sr, measureDelay + measureDuration );
 	}
 
 	std::string description() const override { return "Throughput"; }
diff --git a/fdbserver/workloads/TimeKeeperCorrectness.actor.cpp b/fdbserver/workloads/TimeKeeperCorrectness.actor.cpp
index bfbbd5fe1c..acaa025d05 100644
--- a/fdbserver/workloads/TimeKeeperCorrectness.actor.cpp
+++ b/fdbserver/workloads/TimeKeeperCorrectness.actor.cpp
@@ -29,7 +29,7 @@ struct TimeKeeperCorrectnessWorkload : TestWorkload {
 	std::map<int64_t, Version> inMemTimeKeeper;
 
 	TimeKeeperCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 20.0);
+		testDuration = getOption(options, "testDuration"_sr, 20.0);
 	}
 
 	std::string description() const override { return "TimeKeeperCorrectness"; }
@@ -79,23 +79,23 @@ struct TimeKeeperCorrectnessWorkload : TestWorkload {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				std::vector<std::pair<int64_t, Version>> allItems =
+				KeyBackedRangeResult<std::pair<int64_t, Version>> allItems =
 				    wait(dbTimeKeeper.getRange(tr, 0, Optional<int64_t>(), self->inMemTimeKeeper.size() + 2));
 
-				if (allItems.size() > SERVER_KNOBS->TIME_KEEPER_MAX_ENTRIES + 1) {
+				if (allItems.results.size() > SERVER_KNOBS->TIME_KEEPER_MAX_ENTRIES + 1) {
 					TraceEvent(SevError, "TKCorrectness_TooManyEntries")
 					    .detail("Expected", SERVER_KNOBS->TIME_KEEPER_MAX_ENTRIES + 1)
-					    .detail("Found", allItems.size());
+					    .detail("Found", allItems.results.size());
 					return false;
 				}
 
-				if (allItems.size() < self->testDuration / SERVER_KNOBS->TIME_KEEPER_DELAY) {
+				if (allItems.results.size() < self->testDuration / SERVER_KNOBS->TIME_KEEPER_DELAY) {
 					TraceEvent(SevWarnAlways, "TKCorrectness_TooFewEntries")
 					    .detail("Expected", self->testDuration / SERVER_KNOBS->TIME_KEEPER_DELAY)
-					    .detail("Found", allItems.size());
+					    .detail("Found", allItems.results.size());
 				}
 
-				for (auto item : allItems) {
+				for (auto item : allItems.results) {
 					auto it = self->inMemTimeKeeper.lower_bound(item.first);
 					if (it == self->inMemTimeKeeper.end()) {
 						continue;
diff --git a/fdbserver/workloads/TriggerRecovery.actor.cpp b/fdbserver/workloads/TriggerRecovery.actor.cpp
index ee5fc2fd9a..14d739a833 100644
--- a/fdbserver/workloads/TriggerRecovery.actor.cpp
+++ b/fdbserver/workloads/TriggerRecovery.actor.cpp
@@ -35,10 +35,10 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 	Optional<int32_t> currentNumOfResolvers;
 
 	TriggerRecoveryLoopWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		startTime = getOption(options, LiteralStringRef("startTime"), 0.0);
-		numRecoveries = getOption(options, LiteralStringRef("numRecoveries"), deterministicRandom()->randomInt(1, 10));
-		delayBetweenRecoveries = getOption(options, LiteralStringRef("delayBetweenRecoveries"), 0.0);
-		killAllProportion = getOption(options, LiteralStringRef("killAllProportion"), 0.1);
+		startTime = getOption(options, "startTime"_sr, 0.0);
+		numRecoveries = getOption(options, "numRecoveries"_sr, deterministicRandom()->randomInt(1, 10));
+		delayBetweenRecoveries = getOption(options, "delayBetweenRecoveries"_sr, 0.0);
+		killAllProportion = getOption(options, "killAllProportion"_sr, 0.1);
 		ASSERT((numRecoveries > 0) && (startTime >= 0) && (delayBetweenRecoveries >= 0));
 		TraceEvent(SevInfo, "TriggerRecoveryLoopSetup")
 		    .detail("StartTime", startTime)
@@ -112,16 +112,14 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				RangeResult kvs = wait(tr.getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
-				                                               LiteralStringRef("\xff\xff/worker_interfaces0")),
-				                                   CLIENT_KNOBS->TOO_MANY));
+				RangeResult kvs =
+				    wait(tr.getRange(KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr),
+				                     CLIENT_KNOBS->TOO_MANY));
 				ASSERT(!kvs.more);
 				std::map<Key, Value> address_interface;
 				for (auto it : kvs) {
-					auto ip_port =
-					    (it.key.endsWith(LiteralStringRef(":tls")) ? it.key.removeSuffix(LiteralStringRef(":tls"))
-					                                               : it.key)
-					        .removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
+					auto ip_port = (it.key.endsWith(":tls"_sr) ? it.key.removeSuffix(":tls"_sr) : it.key)
+					                   .removePrefix("\xff\xff/worker_interfaces/"_sr);
 					address_interface[ip_port] = it.value;
 				}
 				for (auto it : address_interface) {
@@ -129,7 +127,7 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 						BinaryReader::fromStringRef<ClientWorkerInterface>(it.second, IncludeVersion())
 						    .reboot.send(RebootRequest());
 					else
-						tr.set(LiteralStringRef("\xff\xff/reboot_worker"), it.second);
+						tr.set("\xff\xff/reboot_worker"_sr, it.second);
 				}
 				TraceEvent(SevInfo, "TriggerRecoveryLoop_AttempedKillAll").log();
 				return Void();
diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp
index 83488f07a4..8bafb1a262 100644
--- a/fdbserver/workloads/UnitTests.actor.cpp
+++ b/fdbserver/workloads/UnitTests.actor.cpp
@@ -42,10 +42,12 @@ void forceLinkRESTClientTests();
 void forceLinkRESTUtilsTests();
 void forceLinkRESTKmsConnectorTest();
 void forceLinkCompressionUtilsTest();
+void forceLinkAtomicTests();
 
 struct UnitTestWorkload : TestWorkload {
 	bool enabled;
 	std::string testPattern;
+	Optional<std::string> testsIgnored;
 	int testRunLimit;
 	UnitTestParameters testParams;
 	bool cleanupAfterTests;
@@ -59,6 +61,9 @@ struct UnitTestWorkload : TestWorkload {
 	    totalSimTime("Total flow time (s)") {
 		enabled = !clientId; // only do this on the "first" client
 		testPattern = getOption(options, "testsMatching"_sr, Value()).toString();
+		if (hasOption(options, "testsIgnored"_sr)) {
+			testsIgnored = getOption(options, "testsIgnored"_sr, Value()).toString();
+		}
 		testRunLimit = getOption(options, "maxTestCases"_sr, -1);
 		if (g_network->isSimulated()) {
 			testParams.setDataDir(getOption(options, "dataDir"_sr, "simfdb/unittests/"_sr).toString());
@@ -81,7 +86,7 @@ struct UnitTestWorkload : TestWorkload {
 		forceLinkMemcpyTests();
 		forceLinkMemcpyPerfTests();
 		forceLinkStreamCipherTests();
-		void forceLinkBlobCipherTests();
+		forceLinkBlobCipherTests();
 		forceLinkParallelStreamTests();
 		forceLinkSimExternalConnectionTests();
 		forceLinkMutationLogReaderTests();
@@ -94,6 +99,7 @@ struct UnitTestWorkload : TestWorkload {
 		forceLinkRESTUtilsTests();
 		forceLinkRESTKmsConnectorTest();
 		forceLinkCompressionUtilsTest();
+		forceLinkAtomicTests();
 	}
 
 	std::string description() const override { return "UnitTests"; }
@@ -115,20 +121,31 @@ struct UnitTestWorkload : TestWorkload {
 		m.push_back(totalSimTime.getMetric());
 	}
 
+	bool testMatched(std::string const& testName) const {
+		return StringRef(testName).startsWith(testPattern) &&
+		       (!testsIgnored.present() || !StringRef(testName).startsWith(testsIgnored.get()));
+	}
+
 	ACTOR static Future<Void> runUnitTests(UnitTestWorkload* self) {
 		state std::vector<UnitTest*> tests;
 
 		for (auto test = g_unittests.tests; test != nullptr; test = test->next) {
-			if (StringRef(test->name).startsWith(self->testPattern)) {
+			if (self->testMatched(test->name)) {
 				++self->testsAvailable;
 				tests.push_back(test);
 			}
 		}
 
+		std::sort(tests.begin(), tests.end(), [](auto lhs, auto rhs) {
+			return std::string_view(lhs->name) < std::string_view(rhs->name);
+		});
+
 		fprintf(stdout, "Found %zu tests\n", tests.size());
 
 		if (tests.size() == 0) {
-			TraceEvent(SevError, "NoMatchingUnitTests").detail("TestPattern", self->testPattern);
+			TraceEvent(SevError, "NoMatchingUnitTests")
+			    .detail("TestPattern", self->testPattern)
+			    .detail("TestsIgnored", self->testsIgnored);
 			++self->testsFailed;
 			return Void();
 		}
@@ -142,6 +159,11 @@ struct UnitTestWorkload : TestWorkload {
 			state UnitTest* test = *t;
 			printf("Testing %s\n", test->name);
 
+			TraceEvent(SevInfo, "RunningUnitTest")
+			    .detail("Name", test->name)
+			    .detail("File", test->file)
+			    .detail("Line", test->line);
+
 			state Error result = success();
 			state double start_now = now();
 			state double start_timer = timer();
diff --git a/fdbserver/workloads/Unreadable.actor.cpp b/fdbserver/workloads/Unreadable.actor.cpp
index ee9dce883c..10781db717 100644
--- a/fdbserver/workloads/Unreadable.actor.cpp
+++ b/fdbserver/workloads/Unreadable.actor.cpp
@@ -32,8 +32,8 @@ struct UnreadableWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 
 	UnreadableWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 600.0);
-		nodeCount = getOption(options, LiteralStringRef("nodeCount"), (uint64_t)100000);
+		testDuration = getOption(options, "testDuration"_sr, 600.0);
+		nodeCount = getOption(options, "nodeCount"_sr, (uint64_t)100000);
 	}
 
 	std::string description() const override { return "Unreadable"; }
diff --git a/fdbserver/workloads/VersionStamp.actor.cpp b/fdbserver/workloads/VersionStamp.actor.cpp
index 89ee83c254..1278b8ccb9 100644
--- a/fdbserver/workloads/VersionStamp.actor.cpp
+++ b/fdbserver/workloads/VersionStamp.actor.cpp
@@ -25,6 +25,7 @@
 #include "fdbserver/workloads/BulkSetup.actor.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbserver/workloads/workloads.actor.h"
+#include "flow/ApiVersion.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct VersionStampWorkload : TestWorkload {
@@ -45,16 +46,16 @@ struct VersionStampWorkload : TestWorkload {
 	bool allowMetadataVersionKey;
 
 	VersionStampWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0);
-		transactionsPerSecond = getOption(options, LiteralStringRef("transactionsPerSecond"), 5000.0);
-		nodeCount = getOption(options, LiteralStringRef("nodeCount"), (uint64_t)10000);
-		keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 4);
-		failIfDataLost = getOption(options, LiteralStringRef("failIfDataLost"), true);
-		const Key prefix = getOption(options, LiteralStringRef("prefix"), LiteralStringRef("VS_"));
-		vsKeyPrefix = LiteralStringRef("K_").withPrefix(prefix);
-		vsValuePrefix = LiteralStringRef("V_").withPrefix(prefix);
-		validateExtraDB = getOption(options, LiteralStringRef("validateExtraDB"), false);
-		soleOwnerOfMetadataVersionKey = getOption(options, LiteralStringRef("soleOwnerOfMetadataVersionKey"), false);
+		testDuration = getOption(options, "testDuration"_sr, 60.0);
+		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0);
+		nodeCount = getOption(options, "nodeCount"_sr, (uint64_t)10000);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 4);
+		failIfDataLost = getOption(options, "failIfDataLost"_sr, true);
+		const Key prefix = getOption(options, "prefix"_sr, "VS_"_sr);
+		vsKeyPrefix = "K_"_sr.withPrefix(prefix);
+		vsValuePrefix = "V_"_sr.withPrefix(prefix);
+		validateExtraDB = getOption(options, "validateExtraDB"_sr, false);
+		soleOwnerOfMetadataVersionKey = getOption(options, "soleOwnerOfMetadataVersionKey"_sr, false);
 	}
 
 	std::string description() const override { return "VersionStamp"; }
@@ -72,13 +73,13 @@ struct VersionStampWorkload : TestWorkload {
 		} else if (choice < 0.3) {
 			apiVersion = 520;
 		} else {
-			apiVersion = Database::API_VERSION_LATEST;
+			apiVersion = ApiVersion::LATEST_VERSION;
 		}
 		TraceEvent("VersionStampApiVersion").detail("ApiVersion", apiVersion);
 
-		allowMetadataVersionKey = apiVersion >= 610 || apiVersion == Database::API_VERSION_LATEST;
+		allowMetadataVersionKey = apiVersion >= 610 || apiVersion == ApiVersion::LATEST_VERSION;
 
-		cx->apiVersion = apiVersion;
+		cx->apiVersion = ApiVersion(apiVersion);
 		if (clientId == 0)
 			return _start(cx, this, 1 / transactionsPerSecond);
 		return Void();
@@ -155,8 +156,8 @@ struct VersionStampWorkload : TestWorkload {
 
 	ACTOR Future<bool> _check(Database cx, VersionStampWorkload* self) {
 		if (self->validateExtraDB) {
-			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-			cx = Database::createDatabase(extraFile, -1);
+			ASSERT(g_simulator->extraDatabases.size() == 1);
+			cx = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], cx->defaultTenant);
 		}
 		state ReadYourWritesTransaction tr(cx);
 		// We specifically wish to grab the smalles read version that we can get and maintain it, to
@@ -205,7 +206,7 @@ struct VersionStampWorkload : TestWorkload {
 				if (self->failIfDataLost) {
 					ASSERT(result.size() == self->key_commit.size());
 				} else {
-					TEST(result.size() > 0); // Not all data should always be lost.
+					CODE_PROBE(result.size() > 0, "Not all data should always be lost.");
 				}
 
 				//TraceEvent("VST_Check0").detail("Size", result.size()).detail("NodeCount", self->nodeCount).detail("KeyCommit", self->key_commit.size()).detail("ReadVersion", readVersion);
@@ -260,7 +261,7 @@ struct VersionStampWorkload : TestWorkload {
 				if (self->failIfDataLost) {
 					ASSERT(result.size() == self->versionStampKey_commit.size());
 				} else {
-					TEST(result.size() > 0); // Not all data should always be lost (2)
+					CODE_PROBE(result.size() > 0, "Not all data should always be lost (2)");
 				}
 
 				//TraceEvent("VST_Check1").detail("Size", result.size()).detail("VsKeyCommitSize", self->versionStampKey_commit.size());
@@ -312,9 +313,9 @@ struct VersionStampWorkload : TestWorkload {
 		state double lastTime = now();
 		state Database extraDB;
 
-		if (g_simulator.extraDB != nullptr) {
-			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-			extraDB = Database::createDatabase(extraFile, -1);
+		if (!g_simulator->extraDatabases.empty()) {
+			ASSERT(g_simulator->extraDatabases.size() == 1);
+			extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], cx->defaultTenant);
 		}
 
 		state Future<Void> metadataWatch = Void();
@@ -342,7 +343,7 @@ struct VersionStampWorkload : TestWorkload {
 			} else if (oldVSFormat) {
 				versionStampValue = value;
 			} else {
-				versionStampValue = value.withSuffix(LiteralStringRef("\x00\x00\x00\x00"));
+				versionStampValue = value.withSuffix("\x00\x00\x00\x00"_sr);
 			}
 
 			state bool ryw = deterministicRandom()->coinflip();
@@ -380,7 +381,7 @@ struct VersionStampWorkload : TestWorkload {
 
 				} catch (Error& e) {
 					err = e;
-					if (err.code() == error_code_database_locked && g_simulator.extraDB != nullptr) {
+					if (err.code() == error_code_database_locked && !g_simulator->extraDatabases.empty()) {
 						//TraceEvent("VST_CommitDatabaseLocked");
 						cx_is_primary = !cx_is_primary;
 						tr = ReadYourWritesTransaction(cx_is_primary ? cx : extraDB);
diff --git a/fdbserver/workloads/WatchAndWait.actor.cpp b/fdbserver/workloads/WatchAndWait.actor.cpp
index 0d227923d3..a8017949a9 100644
--- a/fdbserver/workloads/WatchAndWait.actor.cpp
+++ b/fdbserver/workloads/WatchAndWait.actor.cpp
@@ -36,12 +36,12 @@ struct WatchAndWaitWorkload : TestWorkload {
 	PerfIntCounter triggers, retries;
 
 	WatchAndWaitWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), triggers("Triggers"), retries("Retries") {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 600.0);
-		watchCount = getOption(options, LiteralStringRef("watchCount"), (uint64_t)10000);
-		nodeCount = getOption(options, LiteralStringRef("nodeCount"), (uint64_t)100000);
-		nodePrefix = getOption(options, LiteralStringRef("nodePrefix"), (int64_t)-1);
-		keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 4);
-		triggerWatches = getOption(options, LiteralStringRef("triggerWatches"), false);
+		testDuration = getOption(options, "testDuration"_sr, 600.0);
+		watchCount = getOption(options, "watchCount"_sr, (uint64_t)10000);
+		nodeCount = getOption(options, "nodeCount"_sr, (uint64_t)100000);
+		nodePrefix = getOption(options, "nodePrefix"_sr, (int64_t)-1);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 4);
+		triggerWatches = getOption(options, "triggerWatches"_sr, false);
 
 		if (watchCount > nodeCount) {
 			watchCount = nodeCount;
diff --git a/fdbserver/workloads/Watches.actor.cpp b/fdbserver/workloads/Watches.actor.cpp
index 47a030b061..37f3e624d2 100644
--- a/fdbserver/workloads/Watches.actor.cpp
+++ b/fdbserver/workloads/Watches.actor.cpp
@@ -36,10 +36,10 @@ struct WatchesWorkload : TestWorkload {
 	std::vector<int> nodeOrder;
 
 	WatchesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), cycles("Cycles"), cycleLatencies(sampleSize) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 600.0);
-		nodes = getOption(options, LiteralStringRef("nodeCount"), 100);
-		extraPerNode = getOption(options, LiteralStringRef("extraPerNode"), 1000);
-		keyBytes = std::max(getOption(options, LiteralStringRef("keyBytes"), 16), 16);
+		testDuration = getOption(options, "testDuration"_sr, 600.0);
+		nodes = getOption(options, "nodeCount"_sr, 100);
+		extraPerNode = getOption(options, "extraPerNode"_sr, 1000);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
 
 		for (int i = 0; i < nodes + 1; i++)
 			nodeOrder.push_back(i);
diff --git a/fdbserver/workloads/WatchesSameKeyCorrectness.actor.cpp b/fdbserver/workloads/WatchesSameKeyCorrectness.actor.cpp
index 7976f28757..ec728e2cf5 100644
--- a/fdbserver/workloads/WatchesSameKeyCorrectness.actor.cpp
+++ b/fdbserver/workloads/WatchesSameKeyCorrectness.actor.cpp
@@ -31,17 +31,17 @@ struct WatchesSameKeyWorkload : TestWorkload {
 	std::vector<Future<Void>> cases;
 
 	WatchesSameKeyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		numWatches = getOption(options, LiteralStringRef("numWatches"), 3);
+		numWatches = getOption(options, "numWatches"_sr, 3);
 	}
 
 	std::string description() const override { return "WatchesSameKeyCorrectness"; }
 
 	Future<Void> setup(Database const& cx) override {
-		cases.push_back(case1(cx, LiteralStringRef("foo1"), this));
-		cases.push_back(case2(cx, LiteralStringRef("foo2"), this));
-		cases.push_back(case3(cx, LiteralStringRef("foo3"), this));
-		cases.push_back(case4(cx, LiteralStringRef("foo4"), this));
-		cases.push_back(case5(cx, LiteralStringRef("foo5"), this));
+		cases.push_back(case1(cx, "foo1"_sr, this));
+		cases.push_back(case2(cx, "foo2"_sr, this));
+		cases.push_back(case3(cx, "foo3"_sr, this));
+		cases.push_back(case4(cx, "foo4"_sr, this));
+		cases.push_back(case5(cx, "foo5"_sr, this));
 		return Void();
 	}
 
diff --git a/fdbserver/workloads/WriteBandwidth.actor.cpp b/fdbserver/workloads/WriteBandwidth.actor.cpp
index 516f153761..f77ce0584e 100644
--- a/fdbserver/workloads/WriteBandwidth.actor.cpp
+++ b/fdbserver/workloads/WriteBandwidth.actor.cpp
@@ -40,12 +40,12 @@ struct WriteBandwidthWorkload : KVWorkload {
 	WriteBandwidthWorkload(WorkloadContext const& wcx)
 	  : KVWorkload(wcx), loadTime(0.0), transactions("Transactions"), retries("Retries"), commitLatencies(2000),
 	    GRVLatencies(2000) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
-		keysPerTransaction = getOption(options, LiteralStringRef("keysPerTransaction"), 100);
+		testDuration = getOption(options, "testDuration"_sr, 10.0);
+		keysPerTransaction = getOption(options, "keysPerTransaction"_sr, 100);
 		valueString = std::string(maxValueBytes, '.');
 
-		warmingDelay = getOption(options, LiteralStringRef("warmingDelay"), 0.0);
-		maxInsertRate = getOption(options, LiteralStringRef("maxInsertRate"), 1e12);
+		warmingDelay = getOption(options, "warmingDelay"_sr, 0.0);
+		maxInsertRate = getOption(options, "maxInsertRate"_sr, 1e12);
 	}
 
 	std::string description() const override { return "WriteBandwidth"; }
diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.actor.cpp
index c7253748da..2793d439b0 100644
--- a/fdbserver/workloads/WriteDuringRead.actor.cpp
+++ b/fdbserver/workloads/WriteDuringRead.actor.cpp
@@ -27,6 +27,7 @@
 #include "flow/ActorCollection.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbclient/Atomic.h"
+#include "flow/ApiVersion.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct WriteDuringReadWorkload : TestWorkload {
@@ -57,15 +58,14 @@ struct WriteDuringReadWorkload : TestWorkload {
 
 	WriteDuringReadWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), success(true) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0);
-		slowModeStart = getOption(options, LiteralStringRef("slowModeStart"), 1000.0);
-		numOps = getOption(options, LiteralStringRef("numOps"), 21);
-		rarelyCommit = getOption(options, LiteralStringRef("rarelyCommit"), false);
-		maximumTotalData = getOption(options, LiteralStringRef("maximumTotalData"), 3e6);
-		maximumDataWritten =
-		    getOption(options, LiteralStringRef("maximumDataWritten"), std::numeric_limits<int64_t>::max());
-		minNode = getOption(options, LiteralStringRef("minNode"), 0);
-		useSystemKeys = getOption(options, LiteralStringRef("useSystemKeys"), deterministicRandom()->random01() < 0.5);
+		testDuration = getOption(options, "testDuration"_sr, 60.0);
+		slowModeStart = getOption(options, "slowModeStart"_sr, 1000.0);
+		numOps = getOption(options, "numOps"_sr, 21);
+		rarelyCommit = getOption(options, "rarelyCommit"_sr, false);
+		maximumTotalData = getOption(options, "maximumTotalData"_sr, 3e6);
+		maximumDataWritten = getOption(options, "maximumDataWritten"_sr, std::numeric_limits<int64_t>::max());
+		minNode = getOption(options, "minNode"_sr, 0);
+		useSystemKeys = getOption(options, "useSystemKeys"_sr, deterministicRandom()->random01() < 0.5);
 		adjacentKeys = deterministicRandom()->random01() < 0.5;
 		initialKeyDensity = deterministicRandom()->random01(); // This fraction of keys are present before the first
 		                                                       // transaction (and after an unknown result)
@@ -85,13 +85,13 @@ struct WriteDuringReadWorkload : TestWorkload {
 		minNode = std::max(minNode, nodes - newNodes);
 		nodes = newNodes;
 
-		TEST(adjacentKeys &&
-		     (nodes + minNode) > CLIENT_KNOBS->KEY_SIZE_LIMIT); // WriteDuringReadWorkload testing large keys
+		CODE_PROBE(adjacentKeys && (nodes + minNode) > CLIENT_KNOBS->KEY_SIZE_LIMIT,
+		           "WriteDuringReadWorkload testing large keys");
 
-		useExtraDB = g_simulator.extraDB != nullptr;
+		useExtraDB = g_network->isSimulated() && !g_simulator->extraDatabases.empty();
 		if (useExtraDB) {
-			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
-			extraDB = Database::createDatabase(extraFile, -1);
+			ASSERT(g_simulator->extraDatabases.size() == 1);
+			extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], wcx.defaultTenant);
 			useSystemKeys = false;
 		}
 
@@ -102,7 +102,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 		}
 
 		maxClearSize = 1 << deterministicRandom()->randomInt(0, 20);
-		conflictRange = KeyRangeRef(LiteralStringRef("\xfe"), LiteralStringRef("\xfe\x00"));
+		conflictRange = KeyRangeRef("\xfe"_sr, "\xfe\x00"_sr);
 		if (clientId == 0)
 			TraceEvent("RYWConfiguration")
 			    .detail("Nodes", nodes)
@@ -680,7 +680,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 
 			loop {
 				wait(delay(now() - startTime > self->slowModeStart ||
-				                   (g_network->isSimulated() && g_simulator.speedUpSimulation)
+				                   (g_network->isSimulated() && g_simulator->speedUpSimulation)
 				               ? 1.0
 				               : 0.1));
 				try {
diff --git a/fdbserver/workloads/WriteTagThrottling.actor.cpp b/fdbserver/workloads/WriteTagThrottling.actor.cpp
index ad639b2a96..b2e5f66a43 100644
--- a/fdbserver/workloads/WriteTagThrottling.actor.cpp
+++ b/fdbserver/workloads/WriteTagThrottling.actor.cpp
@@ -66,23 +66,23 @@ struct WriteTagThrottlingWorkload : KVWorkload {
 	WriteTagThrottlingWorkload(WorkloadContext const& wcx)
 	  : KVWorkload(wcx), badActorReadLatency(SAMPLE_SIZE), goodActorReadLatency(SAMPLE_SIZE),
 	    badActorCommitLatency(SAMPLE_SIZE), goodActorCommitLatency(SAMPLE_SIZE) {
-		testDuration = getOption(options, LiteralStringRef("testDuration"), 120.0);
-		badOpRate = getOption(options, LiteralStringRef("badOpRate"), 0.9);
-		numWritePerTr = getOption(options, LiteralStringRef("numWritePerTr"), 1);
-		numReadPerTr = getOption(options, LiteralStringRef("numReadPerTr"), 1);
-		numClearPerTr = getOption(options, LiteralStringRef("numClearPerTr"), 1);
-		hotRangeRate = getOption(options, LiteralStringRef("hotRangeRate"), 0.1);
-		populateData = getOption(options, LiteralStringRef("populateData"), true);
+		testDuration = getOption(options, "testDuration"_sr, 120.0);
+		badOpRate = getOption(options, "badOpRate"_sr, 0.9);
+		numWritePerTr = getOption(options, "numWritePerTr"_sr, 1);
+		numReadPerTr = getOption(options, "numReadPerTr"_sr, 1);
+		numClearPerTr = getOption(options, "numClearPerTr"_sr, 1);
+		hotRangeRate = getOption(options, "hotRangeRate"_sr, 0.1);
+		populateData = getOption(options, "populateData"_sr, true);
 
-		writeThrottle = getOption(options, LiteralStringRef("writeThrottle"), false);
-		badActorPerClient = getOption(options, LiteralStringRef("badActorPerClient"), 1);
-		goodActorPerClient = getOption(options, LiteralStringRef("goodActorPerClient"), 1);
+		writeThrottle = getOption(options, "writeThrottle"_sr, false);
+		badActorPerClient = getOption(options, "badActorPerClient"_sr, 1);
+		goodActorPerClient = getOption(options, "goodActorPerClient"_sr, 1);
 		actorCount = goodActorPerClient + badActorPerClient;
 
 		keyCount = getOption(options,
-		                     LiteralStringRef("keyCount"),
+		                     "keyCount"_sr,
 		                     std::max(3000, clientCount * actorCount * 3)); // enough keys to avoid too many conflicts
-		trInterval = actorCount * 1.0 / getOption(options, LiteralStringRef("trPerSecond"), 1000);
+		trInterval = actorCount * 1.0 / getOption(options, "trPerSecond"_sr, 1000);
 		if (badActorPerClient > 0) {
 			rangeEachBadActor = keyCount / (clientCount * badActorPerClient);
 		}
diff --git a/fdbservice/CMakeLists.txt b/fdbservice/CMakeLists.txt
index 865f5155d6..3bfcbf0314 100644
--- a/fdbservice/CMakeLists.txt
+++ b/fdbservice/CMakeLists.txt
@@ -3,4 +3,4 @@ add_executable(fdbmonitor ${FDBSERVICE_SRCS})
 
 get_target_property(fdbclient_target_includes fdbclient INCLUDE_DIRECTORIES)
 target_link_libraries(fdbmonitor PUBLIC SimpleOpt)
-target_include_directories(fdbmonitor PUBLIC "${fdbclient_target_includes}")
\ No newline at end of file
+target_include_directories(fdbmonitor PUBLIC "${fdbclient_target_includes}")
diff --git a/fdbservice/ServiceBase.cpp b/fdbservice/ServiceBase.cpp
index 8e54cdc3d6..36122183c7 100644
--- a/fdbservice/ServiceBase.cpp
+++ b/fdbservice/ServiceBase.cpp
@@ -357,4 +357,4 @@ void CServiceBase::WriteErrorLogEntry(const char* function, int error) {
 	WriteEventLogEntry(message, EVENTLOG_ERROR_TYPE);
 }
 
-#pragma endregion
\ No newline at end of file
+#pragma endregion
diff --git a/flow/ApiVersion.h.cmake b/flow/ApiVersion.h.cmake
new file mode 100644
index 0000000000..780f694907
--- /dev/null
+++ b/flow/ApiVersion.h.cmake
@@ -0,0 +1,75 @@
+/*
+ * ApiVersion.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FLOW_CODE_API_VERSION_H
+#define FLOW_CODE_API_VERSION_H
+
+#pragma once
+#include "flow/Trace.h"
+#include <cstdint>
+
+constexpr int noBackwardsCompatibility = 13;
+
+// The first check second expression version doesn't need to change because it's just for earlier API versions.
+#define API_VERSION_FEATURE(v, x)                                                                                      \
+	static_assert(v <= @FDB_AV_LATEST_VERSION@, "Feature API version too large");                                   \
+	struct x {                                                                                                         \
+		static constexpr uint64_t apiVersion = v;                                                                      \
+	};                                                                                                                 \
+	constexpr bool has##x() const { return this->version() >= x ::apiVersion; }                                        \
+	static constexpr ApiVersion with##x() { return ApiVersion(x ::apiVersion); }
+
+class ApiVersion {
+	int _version;
+
+public:
+	// Statics.
+	constexpr static int LATEST_VERSION = @FDB_AV_LATEST_VERSION@;
+
+	constexpr explicit ApiVersion(int version) : _version(version) {}
+	constexpr ApiVersion() : _version(0) {}
+
+	constexpr bool isValid() const {
+		return version() > noBackwardsCompatibility && version() <= LATEST_VERSION;
+	}
+
+	constexpr int version() const { return _version; }
+
+	// comparison operators
+	constexpr bool operator==(const ApiVersion other) const { return version() == other.version(); }
+	constexpr bool operator!=(const ApiVersion other) const { return version() != other.version(); }
+	constexpr bool operator<=(const ApiVersion other) const { return version() <= other.version(); }
+	constexpr bool operator>=(const ApiVersion other) const { return version() >= other.version(); }
+	constexpr bool operator<(const ApiVersion other) const { return version() < other.version(); }
+	constexpr bool operator>(const ApiVersion other) const { return version() > other.version(); }
+
+public: // introduced features
+    API_VERSION_FEATURE(@FDB_AV_SNAPSHOT_RYW@, SnapshotRYW);
+    API_VERSION_FEATURE(@FDB_AV_INLINE_UPDATE_DATABASE@, InlineUpdateDatabase);
+    API_VERSION_FEATURE(@FDB_AV_PERSISTENT_OPTIONS@, PersistentOptions);
+    API_VERSION_FEATURE(@FDB_AV_TRACE_FILE_IDENTIFIER@, TraceFileIdentifier);
+    API_VERSION_FEATURE(@FDB_AV_CLUSTER_SHARED_STATE_MAP@, ClusterSharedStateMap);
+    API_VERSION_FEATURE(@FDB_AV_BLOB_RANGE_API@, BlobRangeApi);
+    API_VERSION_FEATURE(@FDB_AV_CREATE_DB_FROM_CONN_STRING@, CreateDBFromConnString);
+    API_VERSION_FEATURE(@FDB_AV_FUTURE_GET_BOOL@, FutureGetBool);
+    API_VERSION_FEATURE(@FDB_AV_FUTURE_PROTOCOL_VERSION_API@, FutureProtocolVersionApi);
+};
+
+#endif // FLOW_CODE_API_VERSION_H
diff --git a/flow/ApiVersions.cmake b/flow/ApiVersions.cmake
new file mode 100644
index 0000000000..ad7ef59c35
--- /dev/null
+++ b/flow/ApiVersions.cmake
@@ -0,0 +1,13 @@
+# API Versions
+set(FDB_AV_LATEST_VERSION                   "720")
+
+# Features
+set(FDB_AV_SNAPSHOT_RYW                     "300")
+set(FDB_AV_INLINE_UPDATE_DATABASE           "610")
+set(FDB_AV_PERSISTENT_OPTIONS               "610")
+set(FDB_AV_TRACE_FILE_IDENTIFIER            "630")
+set(FDB_AV_CLUSTER_SHARED_STATE_MAP         "710")
+set(FDB_AV_BLOB_RANGE_API                   "720")
+set(FDB_AV_CREATE_DB_FROM_CONN_STRING       "720")
+set(FDB_AV_FUTURE_GET_BOOL                  "720")
+set(FDB_AV_FUTURE_PROTOCOL_VERSION_API      "720")
\ No newline at end of file
diff --git a/flow/Arena.cpp b/flow/Arena.cpp
index 07b7740a49..dcdaa39ee2 100644
--- a/flow/Arena.cpp
+++ b/flow/Arena.cpp
@@ -774,3 +774,45 @@ TEST_CASE("/flow/Arena/Size") {
 
 	return Void();
 }
+
+TEST_CASE("flow/StringRef/eat") {
+	StringRef str = "test/case"_sr;
+	StringRef first = str.eat("/");
+	ASSERT(first == "test"_sr);
+	ASSERT(str == "case"_sr);
+
+	str = "test/case"_sr;
+	first = str.eat("/"_sr);
+	ASSERT(first == "test"_sr);
+	ASSERT(str == "case"_sr);
+
+	str = "testcase"_sr;
+	first = str.eat("/"_sr);
+	ASSERT(first == "testcase"_sr);
+	ASSERT(str == ""_sr);
+
+	str = "testcase/"_sr;
+	first = str.eat("/"_sr);
+	ASSERT(first == "testcase"_sr);
+	ASSERT(str == ""_sr);
+
+	str = "test/case/extra"_sr;
+	first = str.eat("/"_sr);
+	ASSERT(first == "test"_sr);
+	ASSERT(str == "case/extra"_sr);
+
+	bool hasSep;
+	str = "test/case"_sr;
+	first = str.eat("/"_sr, &hasSep);
+	ASSERT(hasSep);
+	ASSERT(first == "test"_sr);
+	ASSERT(str == "case"_sr);
+
+	str = "testcase"_sr;
+	first = str.eat("/", &hasSep);
+	ASSERT(!hasSep);
+	ASSERT(first == "testcase"_sr);
+	ASSERT(str == ""_sr);
+
+	return Void();
+}
\ No newline at end of file
diff --git a/flow/BlobCipher.cpp b/flow/BlobCipher.cpp
deleted file mode 100644
index 9c5cdcc37e..0000000000
--- a/flow/BlobCipher.cpp
+++ /dev/null
@@ -1,1233 +0,0 @@
-/*
- * BlobCipher.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "flow/BlobCipher.h"
-
-#include "flow/EncryptUtils.h"
-#include "flow/Knobs.h"
-#include "flow/Error.h"
-#include "flow/FastRef.h"
-#include "flow/IRandom.h"
-#include "flow/ITrace.h"
-#include "flow/network.h"
-#include "flow/Trace.h"
-#include "flow/UnitTest.h"
-
-#include <cstring>
-#include <memory>
-#include <string>
-#include <utility>
-
-namespace {
-bool isEncryptHeaderAuthTokenModeValid(const EncryptAuthTokenMode mode) {
-	return mode >= ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE && mode < ENCRYPT_HEADER_AUTH_TOKEN_LAST;
-}
-} // namespace
-
-// BlobCipherKey class methods
-
-BlobCipherKey::BlobCipherKey(const EncryptCipherDomainId& domainId,
-                             const EncryptCipherBaseKeyId& baseCiphId,
-                             const uint8_t* baseCiph,
-                             int baseCiphLen) {
-	EncryptCipherRandomSalt salt;
-	if (g_network->isSimulated()) {
-		salt = deterministicRandom()->randomUInt64();
-	} else {
-		salt = nondeterministicRandom()->randomUInt64();
-	}
-	initKey(domainId, baseCiph, baseCiphLen, baseCiphId, salt);
-}
-
-BlobCipherKey::BlobCipherKey(const EncryptCipherDomainId& domainId,
-                             const EncryptCipherBaseKeyId& baseCiphId,
-                             const uint8_t* baseCiph,
-                             int baseCiphLen,
-                             const EncryptCipherRandomSalt& salt) {
-	initKey(domainId, baseCiph, baseCiphLen, baseCiphId, salt);
-}
-
-void BlobCipherKey::initKey(const EncryptCipherDomainId& domainId,
-                            const uint8_t* baseCiph,
-                            int baseCiphLen,
-                            const EncryptCipherBaseKeyId& baseCiphId,
-                            const EncryptCipherRandomSalt& salt) {
-	// Set the base encryption key properties
-	baseCipher = std::make_unique<uint8_t[]>(AES_256_KEY_LENGTH);
-	memset(baseCipher.get(), 0, AES_256_KEY_LENGTH);
-	memcpy(baseCipher.get(), baseCiph, std::min<int>(baseCiphLen, AES_256_KEY_LENGTH));
-	baseCipherLen = baseCiphLen;
-	baseCipherId = baseCiphId;
-	// Set the encryption domain for the base encryption key
-	encryptDomainId = domainId;
-	randomSalt = salt;
-	// derive the encryption key
-	cipher = std::make_unique<uint8_t[]>(AES_256_KEY_LENGTH);
-	memset(cipher.get(), 0, AES_256_KEY_LENGTH);
-	applyHmacSha256Derivation();
-	// update the key creation time
-	creationTime = now();
-
-	TraceEvent("BlobCipherKey")
-	    .detail("DomainId", domainId)
-	    .detail("BaseCipherId", baseCipherId)
-	    .detail("BaseCipherLen", baseCipherLen)
-	    .detail("RandomSalt", randomSalt)
-	    .detail("CreationTime", creationTime);
-}
-
-void BlobCipherKey::applyHmacSha256Derivation() {
-	Arena arena;
-	uint8_t buf[baseCipherLen + sizeof(EncryptCipherRandomSalt)];
-	memcpy(&buf[0], baseCipher.get(), baseCipherLen);
-	memcpy(&buf[0] + baseCipherLen, &randomSalt, sizeof(EncryptCipherRandomSalt));
-	HmacSha256DigestGen hmacGen(baseCipher.get(), baseCipherLen);
-	StringRef digest = hmacGen.digest(&buf[0], baseCipherLen + sizeof(EncryptCipherRandomSalt), arena);
-	std::copy(digest.begin(), digest.end(), cipher.get());
-	if (digest.size() < AES_256_KEY_LENGTH) {
-		memcpy(cipher.get() + digest.size(), buf, AES_256_KEY_LENGTH - digest.size());
-	}
-}
-
-void BlobCipherKey::reset() {
-	memset(baseCipher.get(), 0, baseCipherLen);
-	memset(cipher.get(), 0, AES_256_KEY_LENGTH);
-}
-
-// BlobKeyIdCache class methods
-
-BlobCipherKeyIdCache::BlobCipherKeyIdCache()
-  : domainId(ENCRYPT_INVALID_DOMAIN_ID), latestBaseCipherKeyId(), latestRandomSalt() {}
-
-BlobCipherKeyIdCache::BlobCipherKeyIdCache(EncryptCipherDomainId dId)
-  : domainId(dId), latestBaseCipherKeyId(), latestRandomSalt() {
-	TraceEvent("Init_BlobCipherKeyIdCache").detail("DomainId", domainId);
-}
-
-BlobCipherKeyIdCacheKey BlobCipherKeyIdCache::getCacheKey(const EncryptCipherBaseKeyId& baseCipherKeyId,
-                                                          const EncryptCipherRandomSalt& salt) {
-	if (baseCipherKeyId == ENCRYPT_INVALID_CIPHER_KEY_ID || salt == ENCRYPT_INVALID_RANDOM_SALT) {
-		throw encrypt_invalid_id();
-	}
-	return std::make_pair(baseCipherKeyId, salt);
-}
-
-Reference<BlobCipherKey> BlobCipherKeyIdCache::getLatestCipherKey() {
-	if (!latestBaseCipherKeyId.present()) {
-		return Reference<BlobCipherKey>();
-	}
-	ASSERT_NE(latestBaseCipherKeyId.get(), ENCRYPT_INVALID_CIPHER_KEY_ID);
-	ASSERT(latestRandomSalt.present());
-	ASSERT_NE(latestRandomSalt.get(), ENCRYPT_INVALID_RANDOM_SALT);
-
-	return getCipherByBaseCipherId(latestBaseCipherKeyId.get(), latestRandomSalt.get());
-}
-
-Reference<BlobCipherKey> BlobCipherKeyIdCache::getCipherByBaseCipherId(const EncryptCipherBaseKeyId& baseCipherKeyId,
-                                                                       const EncryptCipherRandomSalt& salt) {
-	BlobCipherKeyIdCacheMapCItr itr = keyIdCache.find(getCacheKey(baseCipherKeyId, salt));
-	if (itr == keyIdCache.end()) {
-		return Reference<BlobCipherKey>();
-	}
-	return itr->second;
-}
-
-Reference<BlobCipherKey> BlobCipherKeyIdCache::insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId,
-                                                                   const uint8_t* baseCipher,
-                                                                   int baseCipherLen) {
-	ASSERT_GT(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID);
-
-	// BaseCipherKeys are immutable, given the routine invocation updates 'latestCipher',
-	// ensure no key-tampering is done
-	Reference<BlobCipherKey> latestCipherKey = getLatestCipherKey();
-	if (latestCipherKey.isValid() && latestCipherKey->getBaseCipherId() == baseCipherId) {
-		if (memcmp(latestCipherKey->rawBaseCipher(), baseCipher, baseCipherLen) == 0) {
-			TraceEvent("InsertBaseCipherKey_AlreadyPresent")
-			    .detail("BaseCipherKeyId", baseCipherId)
-			    .detail("DomainId", domainId);
-			// Key is already present; nothing more to do.
-			return latestCipherKey;
-		} else {
-			TraceEvent("InsertBaseCipherKey_UpdateCipher")
-			    .detail("BaseCipherKeyId", baseCipherId)
-			    .detail("DomainId", domainId);
-			throw encrypt_update_cipher();
-		}
-	}
-
-	Reference<BlobCipherKey> cipherKey =
-	    makeReference<BlobCipherKey>(domainId, baseCipherId, baseCipher, baseCipherLen);
-	BlobCipherKeyIdCacheKey cacheKey = getCacheKey(cipherKey->getBaseCipherId(), cipherKey->getSalt());
-	keyIdCache.emplace(cacheKey, cipherKey);
-
-	// Update the latest BaseCipherKeyId for the given encryption domain
-	latestBaseCipherKeyId = baseCipherId;
-	latestRandomSalt = cipherKey->getSalt();
-
-	return cipherKey;
-}
-
-Reference<BlobCipherKey> BlobCipherKeyIdCache::insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId,
-                                                                   const uint8_t* baseCipher,
-                                                                   int baseCipherLen,
-                                                                   const EncryptCipherRandomSalt& salt) {
-	ASSERT_NE(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID);
-	ASSERT_NE(salt, ENCRYPT_INVALID_RANDOM_SALT);
-
-	BlobCipherKeyIdCacheKey cacheKey = getCacheKey(baseCipherId, salt);
-
-	// BaseCipherKeys are immutable, ensure that cached value doesn't get updated.
-	BlobCipherKeyIdCacheMapCItr itr = keyIdCache.find(cacheKey);
-	if (itr != keyIdCache.end()) {
-		if (memcmp(itr->second->rawBaseCipher(), baseCipher, baseCipherLen) == 0) {
-			TraceEvent("InsertBaseCipherKey_AlreadyPresent")
-			    .detail("BaseCipherKeyId", baseCipherId)
-			    .detail("DomainId", domainId);
-			// Key is already present; nothing more to do.
-			return itr->second;
-		} else {
-			TraceEvent("InsertBaseCipherKey_UpdateCipher")
-			    .detail("BaseCipherKeyId", baseCipherId)
-			    .detail("DomainId", domainId);
-			throw encrypt_update_cipher();
-		}
-	}
-
-	Reference<BlobCipherKey> cipherKey =
-	    makeReference<BlobCipherKey>(domainId, baseCipherId, baseCipher, baseCipherLen, salt);
-	keyIdCache.emplace(cacheKey, cipherKey);
-	return cipherKey;
-}
-
-void BlobCipherKeyIdCache::cleanup() {
-	for (auto& keyItr : keyIdCache) {
-		keyItr.second->reset();
-	}
-
-	keyIdCache.clear();
-}
-
-std::vector<Reference<BlobCipherKey>> BlobCipherKeyIdCache::getAllCipherKeys() {
-	std::vector<Reference<BlobCipherKey>> cipherKeys;
-	for (auto& keyItr : keyIdCache) {
-		cipherKeys.push_back(keyItr.second);
-	}
-	return cipherKeys;
-}
-
-// BlobCipherKeyCache class methods
-
-Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipherDomainId& domainId,
-                                                             const EncryptCipherBaseKeyId& baseCipherId,
-                                                             const uint8_t* baseCipher,
-                                                             int baseCipherLen) {
-	if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID) {
-		throw encrypt_invalid_id();
-	}
-
-	try {
-		auto domainItr = domainCacheMap.find(domainId);
-		if (domainItr == domainCacheMap.end()) {
-			// Add mapping to track new encryption domain
-			Reference<BlobCipherKeyIdCache> keyIdCache = makeReference<BlobCipherKeyIdCache>(domainId);
-			Reference<BlobCipherKey> cipherKey =
-			    keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen);
-			domainCacheMap.emplace(domainId, keyIdCache);
-			return cipherKey;
-		} else {
-			// Track new baseCipher keys
-			Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
-			return keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen);
-		}
-
-		TraceEvent("InsertCipherKey").detail("DomainId", domainId).detail("BaseCipherKeyId", baseCipherId);
-	} catch (Error& e) {
-		TraceEvent("InsertCipherKey_Failed").detail("BaseCipherKeyId", baseCipherId).detail("DomainId", domainId);
-		throw;
-	}
-}
-
-Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipherDomainId& domainId,
-                                                             const EncryptCipherBaseKeyId& baseCipherId,
-                                                             const uint8_t* baseCipher,
-                                                             int baseCipherLen,
-                                                             const EncryptCipherRandomSalt& salt) {
-	if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID ||
-	    salt == ENCRYPT_INVALID_RANDOM_SALT) {
-		throw encrypt_invalid_id();
-	}
-
-	Reference<BlobCipherKey> cipherKey;
-	try {
-		auto domainItr = domainCacheMap.find(domainId);
-		if (domainItr == domainCacheMap.end()) {
-			// Add mapping to track new encryption domain
-			Reference<BlobCipherKeyIdCache> keyIdCache = makeReference<BlobCipherKeyIdCache>(domainId);
-			cipherKey = keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen, salt);
-			domainCacheMap.emplace(domainId, keyIdCache);
-		} else {
-			// Track new baseCipher keys
-			Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
-			cipherKey = keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen, salt);
-		}
-
-		TraceEvent("InsertCipherKey")
-		    .detail("DomainId", domainId)
-		    .detail("BaseCipherKeyId", baseCipherId)
-		    .detail("Salt", salt);
-	} catch (Error& e) {
-		TraceEvent("InsertCipherKey_Failed")
-		    .detail("BaseCipherKeyId", baseCipherId)
-		    .detail("DomainId", domainId)
-		    .detail("Salt", salt);
-		throw;
-	}
-
-	return cipherKey;
-}
-
-Reference<BlobCipherKey> BlobCipherKeyCache::getLatestCipherKey(const EncryptCipherDomainId& domainId) {
-	if (domainId == ENCRYPT_INVALID_DOMAIN_ID) {
-		TraceEvent("GetLatestCipherKey_InvalidID").detail("DomainId", domainId);
-		throw encrypt_invalid_id();
-	}
-	auto domainItr = domainCacheMap.find(domainId);
-	if (domainItr == domainCacheMap.end()) {
-		TraceEvent("GetLatestCipherKey_DomainNotFound").detail("DomainId", domainId);
-		return Reference<BlobCipherKey>();
-	}
-
-	Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
-	Reference<BlobCipherKey> cipherKey = keyIdCache->getLatestCipherKey();
-	if (cipherKey.isValid() && (now() - cipherKey->getCreationTime()) > FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL) {
-		TraceEvent("GetLatestCipherKey_ExpiredTTL")
-		    .detail("DomainId", domainId)
-		    .detail("BaseCipherId", cipherKey->getBaseCipherId());
-		return Reference<BlobCipherKey>();
-	}
-
-	return cipherKey;
-}
-
-Reference<BlobCipherKey> BlobCipherKeyCache::getCipherKey(const EncryptCipherDomainId& domainId,
-                                                          const EncryptCipherBaseKeyId& baseCipherId,
-                                                          const EncryptCipherRandomSalt& salt) {
-	auto domainItr = domainCacheMap.find(domainId);
-	if (domainItr == domainCacheMap.end()) {
-		return Reference<BlobCipherKey>();
-	}
-
-	Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
-	return keyIdCache->getCipherByBaseCipherId(baseCipherId, salt);
-}
-
-void BlobCipherKeyCache::resetEncryptDomainId(const EncryptCipherDomainId domainId) {
-	auto domainItr = domainCacheMap.find(domainId);
-	if (domainItr == domainCacheMap.end()) {
-		return;
-	}
-
-	Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
-	keyIdCache->cleanup();
-	TraceEvent("ResetEncryptDomainId").detail("DomainId", domainId);
-}
-
-void BlobCipherKeyCache::cleanup() noexcept {
-	Reference<BlobCipherKeyCache> instance = BlobCipherKeyCache::getInstance();
-	for (auto& domainItr : instance->domainCacheMap) {
-		Reference<BlobCipherKeyIdCache> keyIdCache = domainItr.second;
-		keyIdCache->cleanup();
-		TraceEvent("BlobCipherKeyCache_Cleanup").detail("DomainId", domainItr.first);
-	}
-
-	instance->domainCacheMap.clear();
-}
-
-std::vector<Reference<BlobCipherKey>> BlobCipherKeyCache::getAllCiphers(const EncryptCipherDomainId& domainId) {
-	auto domainItr = domainCacheMap.find(domainId);
-	if (domainItr == domainCacheMap.end()) {
-		return {};
-	}
-
-	Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
-	return keyIdCache->getAllCipherKeys();
-}
-
-// EncryptBlobCipherAes265Ctr class methods
-
-EncryptBlobCipherAes265Ctr::EncryptBlobCipherAes265Ctr(Reference<BlobCipherKey> tCipherKey,
-                                                       Reference<BlobCipherKey> hCipherKey,
-                                                       const uint8_t* cipherIV,
-                                                       const int ivLen,
-                                                       const EncryptAuthTokenMode mode)
-  : ctx(EVP_CIPHER_CTX_new()), textCipherKey(tCipherKey), headerCipherKey(hCipherKey), authTokenMode(mode) {
-	ASSERT(isEncryptHeaderAuthTokenModeValid(mode));
-	ASSERT_EQ(ivLen, AES_256_IV_LENGTH);
-	memcpy(&iv[0], cipherIV, ivLen);
-	init();
-}
-
-EncryptBlobCipherAes265Ctr::EncryptBlobCipherAes265Ctr(Reference<BlobCipherKey> tCipherKey,
-                                                       Reference<BlobCipherKey> hCipherKey,
-                                                       const EncryptAuthTokenMode mode)
-  : ctx(EVP_CIPHER_CTX_new()), textCipherKey(tCipherKey), headerCipherKey(hCipherKey), authTokenMode(mode) {
-	ASSERT(isEncryptHeaderAuthTokenModeValid(mode));
-	generateRandomData(iv, AES_256_IV_LENGTH);
-	init();
-}
-
-void EncryptBlobCipherAes265Ctr::init() {
-	if (ctx == nullptr) {
-		throw encrypt_ops_error();
-	}
-	if (EVP_EncryptInit_ex(ctx, EVP_aes_256_ctr(), nullptr, nullptr, nullptr) != 1) {
-		throw encrypt_ops_error();
-	}
-	if (EVP_EncryptInit_ex(ctx, nullptr, nullptr, textCipherKey.getPtr()->data(), iv) != 1) {
-		throw encrypt_ops_error();
-	}
-}
-
-Reference<EncryptBuf> EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plaintext,
-                                                          const int plaintextLen,
-                                                          BlobCipherEncryptHeader* header,
-                                                          Arena& arena) {
-	TEST(true); // Encrypting data with BlobCipher
-
-	memset(reinterpret_cast<uint8_t*>(header), 0, sizeof(BlobCipherEncryptHeader));
-
-	// Alloc buffer computation accounts for 'header authentication' generation scheme. If single-auth-token needs
-	// to be generated, allocate buffer sufficient to append header to the cipherText optimizing memcpy cost.
-
-	const int allocSize = authTokenMode == ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE
-	                          ? plaintextLen + AES_BLOCK_SIZE + sizeof(BlobCipherEncryptHeader)
-	                          : plaintextLen + AES_BLOCK_SIZE;
-	Reference<EncryptBuf> encryptBuf = makeReference<EncryptBuf>(allocSize, arena);
-	uint8_t* ciphertext = encryptBuf->begin();
-	int bytes{ 0 };
-	if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
-		TraceEvent("Encrypt_UpdateFailed")
-		    .detail("BaseCipherId", textCipherKey->getBaseCipherId())
-		    .detail("EncryptDomainId", textCipherKey->getDomainId());
-		throw encrypt_ops_error();
-	}
-
-	int finalBytes{ 0 };
-	if (EVP_EncryptFinal_ex(ctx, ciphertext + bytes, &finalBytes) != 1) {
-		TraceEvent("Encrypt_FinalFailed")
-		    .detail("BaseCipherId", textCipherKey->getBaseCipherId())
-		    .detail("EncryptDomainId", textCipherKey->getDomainId());
-		throw encrypt_ops_error();
-	}
-
-	if ((bytes + finalBytes) != plaintextLen) {
-		TraceEvent("Encrypt_UnexpectedCipherLen")
-		    .detail("PlaintextLen", plaintextLen)
-		    .detail("EncryptedBufLen", bytes + finalBytes);
-		throw encrypt_ops_error();
-	}
-
-	// Populate encryption header flags details
-	header->flags.size = sizeof(BlobCipherEncryptHeader);
-	header->flags.headerVersion = EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION;
-	header->flags.encryptMode = ENCRYPT_CIPHER_MODE_AES_256_CTR;
-	header->flags.authTokenMode = authTokenMode;
-
-	// Populate cipherText encryption-key details
-	header->cipherTextDetails.baseCipherId = textCipherKey->getBaseCipherId();
-	header->cipherTextDetails.encryptDomainId = textCipherKey->getDomainId();
-	header->cipherTextDetails.salt = textCipherKey->getSalt();
-	memcpy(&header->iv[0], &iv[0], AES_256_IV_LENGTH);
-
-	if (authTokenMode == ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
-		// No header 'authToken' generation needed.
-	} else {
-		// Populate header encryption-key details
-		header->cipherHeaderDetails.encryptDomainId = headerCipherKey->getDomainId();
-		header->cipherHeaderDetails.baseCipherId = headerCipherKey->getBaseCipherId();
-		header->cipherHeaderDetails.salt = headerCipherKey->getSalt();
-
-		// Populate header authToken details
-		if (header->flags.authTokenMode == ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE) {
-			ASSERT_GE(allocSize, (bytes + finalBytes + sizeof(BlobCipherEncryptHeader)));
-			ASSERT_GE(encryptBuf->getLogicalSize(), (bytes + finalBytes + sizeof(BlobCipherEncryptHeader)));
-
-			memcpy(&ciphertext[bytes + finalBytes],
-			       reinterpret_cast<const uint8_t*>(header),
-			       sizeof(BlobCipherEncryptHeader));
-			StringRef authToken = computeAuthToken(ciphertext,
-			                                       bytes + finalBytes + sizeof(BlobCipherEncryptHeader),
-			                                       headerCipherKey->rawCipher(),
-			                                       AES_256_KEY_LENGTH,
-			                                       arena);
-			memcpy(&header->singleAuthToken.authToken[0], authToken.begin(), AUTH_TOKEN_SIZE);
-		} else {
-			ASSERT_EQ(header->flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
-
-			StringRef cipherTextAuthToken =
-			    computeAuthToken(ciphertext,
-			                     bytes + finalBytes,
-			                     reinterpret_cast<const uint8_t*>(&header->cipherTextDetails.salt),
-			                     sizeof(EncryptCipherRandomSalt),
-			                     arena);
-			memcpy(&header->multiAuthTokens.cipherTextAuthToken[0], cipherTextAuthToken.begin(), AUTH_TOKEN_SIZE);
-			StringRef headerAuthToken = computeAuthToken(reinterpret_cast<const uint8_t*>(header),
-			                                             sizeof(BlobCipherEncryptHeader),
-			                                             headerCipherKey->rawCipher(),
-			                                             AES_256_KEY_LENGTH,
-			                                             arena);
-			memcpy(&header->multiAuthTokens.headerAuthToken[0], headerAuthToken.begin(), AUTH_TOKEN_SIZE);
-		}
-	}
-
-	encryptBuf->setLogicalSize(plaintextLen);
-	return encryptBuf;
-}
-
-Standalone<StringRef> EncryptBlobCipherAes265Ctr::encryptBlobGranuleChunk(const uint8_t* plaintext,
-                                                                          const int plaintextLen) {
-	Standalone<StringRef> encrypted = makeString(plaintextLen);
-	uint8_t* ciphertext = mutateString(encrypted);
-	int bytes{ 0 };
-
-	if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
-		TraceEvent("Encrypt_UpdateFailed")
-		    .detail("BaseCipherId", textCipherKey->getBaseCipherId())
-		    .detail("EncryptDomainId", textCipherKey->getDomainId());
-		throw encrypt_ops_error();
-	}
-	int finalBytes{ 0 };
-	if (EVP_EncryptFinal_ex(ctx, ciphertext + bytes, &finalBytes) != 1) {
-		TraceEvent("Encrypt_FinalFailed")
-		    .detail("BaseCipherId", textCipherKey->getBaseCipherId())
-		    .detail("EncryptDomainId", textCipherKey->getDomainId());
-		throw encrypt_ops_error();
-	}
-	if ((bytes + finalBytes) != plaintextLen) {
-		TraceEvent("Encrypt_UnexpectedCipherLen")
-		    .detail("PlaintextLen", plaintextLen)
-		    .detail("EncryptedBufLen", bytes + finalBytes);
-		throw encrypt_ops_error();
-	}
-	return encrypted;
-}
-
-EncryptBlobCipherAes265Ctr::~EncryptBlobCipherAes265Ctr() {
-	if (ctx != nullptr) {
-		EVP_CIPHER_CTX_free(ctx);
-	}
-}
-
-// DecryptBlobCipherAes256Ctr class methods
-
-DecryptBlobCipherAes256Ctr::DecryptBlobCipherAes256Ctr(Reference<BlobCipherKey> tCipherKey,
-                                                       Reference<BlobCipherKey> hCipherKey,
-                                                       const uint8_t* iv)
-  : ctx(EVP_CIPHER_CTX_new()), textCipherKey(tCipherKey), headerCipherKey(hCipherKey),
-    headerAuthTokenValidationDone(false), authTokensValidationDone(false) {
-	if (ctx == nullptr) {
-		throw encrypt_ops_error();
-	}
-	if (!EVP_DecryptInit_ex(ctx, EVP_aes_256_ctr(), nullptr, nullptr, nullptr)) {
-		throw encrypt_ops_error();
-	}
-	if (!EVP_DecryptInit_ex(ctx, nullptr, nullptr, tCipherKey.getPtr()->data(), iv)) {
-		throw encrypt_ops_error();
-	}
-}
-
-void DecryptBlobCipherAes256Ctr::verifyHeaderAuthToken(const BlobCipherEncryptHeader& header, Arena& arena) {
-	if (header.flags.authTokenMode != ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI) {
-		// NoneAuthToken mode; no authToken is generated; nothing to do
-		// SingleAuthToken mode; verification will happen as part of decryption.
-		return;
-	}
-
-	ASSERT_EQ(header.flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
-
-	BlobCipherEncryptHeader headerCopy;
-	memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-	       reinterpret_cast<const uint8_t*>(&header),
-	       sizeof(BlobCipherEncryptHeader));
-	memset(reinterpret_cast<uint8_t*>(&headerCopy.multiAuthTokens.headerAuthToken), 0, AUTH_TOKEN_SIZE);
-	StringRef computedHeaderAuthToken = computeAuthToken(reinterpret_cast<const uint8_t*>(&headerCopy),
-	                                                     sizeof(BlobCipherEncryptHeader),
-	                                                     headerCipherKey->rawCipher(),
-	                                                     AES_256_KEY_LENGTH,
-	                                                     arena);
-	if (memcmp(&header.multiAuthTokens.headerAuthToken[0], computedHeaderAuthToken.begin(), AUTH_TOKEN_SIZE) != 0) {
-		TraceEvent("VerifyEncryptBlobHeader_AuthTokenMismatch")
-		    .detail("HeaderVersion", header.flags.headerVersion)
-		    .detail("HeaderMode", header.flags.encryptMode)
-		    .detail("MultiAuthHeaderAuthToken",
-		            StringRef(arena, &header.multiAuthTokens.headerAuthToken[0], AUTH_TOKEN_SIZE).toString())
-		    .detail("ComputedHeaderAuthToken", computedHeaderAuthToken.toString());
-		throw encrypt_header_authtoken_mismatch();
-	}
-
-	headerAuthTokenValidationDone = true;
-}
-
-void DecryptBlobCipherAes256Ctr::verifyHeaderSingleAuthToken(const uint8_t* ciphertext,
-                                                             const int ciphertextLen,
-                                                             const BlobCipherEncryptHeader& header,
-                                                             uint8_t* buff,
-                                                             Arena& arena) {
-	// Header authToken not set for single auth-token mode.
-	ASSERT(!headerAuthTokenValidationDone);
-
-	// prepare the payload {cipherText + encryptionHeader}
-	memcpy(&buff[0], ciphertext, ciphertextLen);
-	memcpy(&buff[ciphertextLen], reinterpret_cast<const uint8_t*>(&header), sizeof(BlobCipherEncryptHeader));
-	// ensure the 'authToken' is reset before computing the 'authentication token'
-	BlobCipherEncryptHeader* eHeader = (BlobCipherEncryptHeader*)(&buff[ciphertextLen]);
-	memset(reinterpret_cast<uint8_t*>(&eHeader->singleAuthToken), 0, 2 * AUTH_TOKEN_SIZE);
-
-	StringRef computed = computeAuthToken(
-	    buff, ciphertextLen + sizeof(BlobCipherEncryptHeader), headerCipherKey->rawCipher(), AES_256_KEY_LENGTH, arena);
-	if (memcmp(&header.singleAuthToken.authToken[0], computed.begin(), AUTH_TOKEN_SIZE) != 0) {
-		TraceEvent("VerifyEncryptBlobHeader_AuthTokenMismatch")
-		    .detail("HeaderVersion", header.flags.headerVersion)
-		    .detail("HeaderMode", header.flags.encryptMode)
-		    .detail("SingleAuthToken",
-		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_SIZE).toString())
-		    .detail("ComputedSingleAuthToken", computed.toString());
-		throw encrypt_header_authtoken_mismatch();
-	}
-}
-
-void DecryptBlobCipherAes256Ctr::verifyHeaderMultiAuthToken(const uint8_t* ciphertext,
-                                                            const int ciphertextLen,
-                                                            const BlobCipherEncryptHeader& header,
-                                                            uint8_t* buff,
-                                                            Arena& arena) {
-	if (!headerAuthTokenValidationDone) {
-		verifyHeaderAuthToken(header, arena);
-	}
-	StringRef computedCipherTextAuthToken =
-	    computeAuthToken(ciphertext,
-	                     ciphertextLen,
-	                     reinterpret_cast<const uint8_t*>(&header.cipherTextDetails.salt),
-	                     sizeof(EncryptCipherRandomSalt),
-	                     arena);
-	if (memcmp(&header.multiAuthTokens.cipherTextAuthToken[0], computedCipherTextAuthToken.begin(), AUTH_TOKEN_SIZE) !=
-	    0) {
-		TraceEvent("VerifyEncryptBlobHeader_AuthTokenMismatch")
-		    .detail("HeaderVersion", header.flags.headerVersion)
-		    .detail("HeaderMode", header.flags.encryptMode)
-		    .detail("MultiAuthCipherTextAuthToken",
-		            StringRef(arena, &header.multiAuthTokens.cipherTextAuthToken[0], AUTH_TOKEN_SIZE).toString())
-		    .detail("ComputedCipherTextAuthToken", computedCipherTextAuthToken.toString());
-		throw encrypt_header_authtoken_mismatch();
-	}
-}
-
-void DecryptBlobCipherAes256Ctr::verifyAuthTokens(const uint8_t* ciphertext,
-                                                  const int ciphertextLen,
-                                                  const BlobCipherEncryptHeader& header,
-                                                  uint8_t* buff,
-                                                  Arena& arena) {
-	if (header.flags.authTokenMode == ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE) {
-		verifyHeaderSingleAuthToken(ciphertext, ciphertextLen, header, buff, arena);
-	} else {
-		ASSERT_EQ(header.flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
-		verifyHeaderMultiAuthToken(ciphertext, ciphertextLen, header, buff, arena);
-	}
-
-	authTokensValidationDone = true;
-}
-
-void DecryptBlobCipherAes256Ctr::verifyEncryptHeaderMetadata(const BlobCipherEncryptHeader& header) {
-	// validate header flag sanity
-	if (header.flags.headerVersion != EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION ||
-	    header.flags.encryptMode != ENCRYPT_CIPHER_MODE_AES_256_CTR ||
-	    !isEncryptHeaderAuthTokenModeValid((EncryptAuthTokenMode)header.flags.authTokenMode)) {
-		TraceEvent("VerifyEncryptBlobHeader")
-		    .detail("HeaderVersion", header.flags.headerVersion)
-		    .detail("ExpectedVersion", EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION)
-		    .detail("EncryptCipherMode", header.flags.encryptMode)
-		    .detail("ExpectedCipherMode", ENCRYPT_CIPHER_MODE_AES_256_CTR)
-		    .detail("EncryptHeaderAuthTokenMode", header.flags.authTokenMode);
-		throw encrypt_header_metadata_mismatch();
-	}
-}
-
-Reference<EncryptBuf> DecryptBlobCipherAes256Ctr::decrypt(const uint8_t* ciphertext,
-                                                          const int ciphertextLen,
-                                                          const BlobCipherEncryptHeader& header,
-                                                          Arena& arena) {
-	TEST(true); // Decrypting data with BlobCipher
-
-	verifyEncryptHeaderMetadata(header);
-
-	if (header.flags.authTokenMode != ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE && !headerCipherKey.isValid()) {
-		TraceEvent("Decrypt_InvalidHeaderCipherKey").detail("AuthTokenMode", header.flags.authTokenMode);
-		throw encrypt_ops_error();
-	}
-
-	const int allocSize = header.flags.authTokenMode == ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE
-	                          ? ciphertextLen + AES_BLOCK_SIZE + sizeof(BlobCipherEncryptHeader)
-	                          : ciphertextLen + AES_BLOCK_SIZE;
-	Reference<EncryptBuf> decrypted = makeReference<EncryptBuf>(allocSize, arena);
-
-	if (header.flags.authTokenMode != ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
-		verifyAuthTokens(ciphertext, ciphertextLen, header, decrypted->begin(), arena);
-		ASSERT(authTokensValidationDone);
-	}
-
-	uint8_t* plaintext = decrypted->begin();
-	int bytesDecrypted{ 0 };
-	if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
-		TraceEvent("Decrypt_UpdateFailed")
-		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
-		    .detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
-		throw encrypt_ops_error();
-	}
-
-	int finalBlobBytes{ 0 };
-	if (EVP_DecryptFinal_ex(ctx, plaintext + bytesDecrypted, &finalBlobBytes) <= 0) {
-		TraceEvent("Decrypt_FinalFailed")
-		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
-		    .detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
-		throw encrypt_ops_error();
-	}
-
-	if ((bytesDecrypted + finalBlobBytes) != ciphertextLen) {
-		TraceEvent("Encrypt_UnexpectedPlaintextLen")
-		    .detail("CiphertextLen", ciphertextLen)
-		    .detail("DecryptedBufLen", bytesDecrypted + finalBlobBytes);
-		throw encrypt_ops_error();
-	}
-
-	decrypted->setLogicalSize(ciphertextLen);
-	return decrypted;
-}
-
-DecryptBlobCipherAes256Ctr::~DecryptBlobCipherAes256Ctr() {
-	if (ctx != nullptr) {
-		EVP_CIPHER_CTX_free(ctx);
-	}
-}
-
-// HmacSha256DigestGen class methods
-
-HmacSha256DigestGen::HmacSha256DigestGen(const unsigned char* key, size_t len) : ctx(HMAC_CTX_new()) {
-	if (!HMAC_Init_ex(ctx, key, len, EVP_sha256(), nullptr)) {
-		throw encrypt_ops_error();
-	}
-}
-
-HmacSha256DigestGen::~HmacSha256DigestGen() {
-	if (ctx != nullptr) {
-		HMAC_CTX_free(ctx);
-	}
-}
-
-StringRef HmacSha256DigestGen::digest(const unsigned char* data, size_t len, Arena& arena) {
-	TEST(true); // Digest generation
-	unsigned int digestLen = HMAC_size(ctx);
-	auto digest = new (arena) unsigned char[digestLen];
-	if (HMAC_Update(ctx, data, len) != 1) {
-		throw encrypt_ops_error();
-	}
-
-	if (HMAC_Final(ctx, digest, &digestLen) != 1) {
-		throw encrypt_ops_error();
-	}
-
-	return StringRef(arena, digest, digestLen);
-}
-
-StringRef computeAuthToken(const uint8_t* payload,
-                           const int payloadLen,
-                           const uint8_t* key,
-                           const int keyLen,
-                           Arena& arena) {
-	HmacSha256DigestGen hmacGenerator(key, keyLen);
-	StringRef digest = hmacGenerator.digest(payload, payloadLen, arena);
-
-	ASSERT_GE(digest.size(), AUTH_TOKEN_SIZE);
-	return digest;
-}
-
-// Only used to link unit tests
-void forceLinkBlobCipherTests() {}
-
-// Tests cases includes:
-// 1. Populate cache by inserting 'baseCipher' details for new encryptionDomainIds
-// 2. Random lookup for cipherKeys and content validation
-// 3. Inserting of 'identical' cipherKey (already cached) more than once works as desired.
-// 4. Inserting of 'non-identical' cipherKey (already cached) more than once works as desired.
-// 5. Validation encryption ops (correctness):
-//  5.1. Encrypt a buffer followed by decryption of the buffer, validate the contents.
-//  5.2. Simulate anomalies such as: EncryptionHeader corruption, authToken mismatch / encryptionMode mismatch etc.
-// 6. Cache cleanup
-//  6.1  cleanup cipherKeys by given encryptDomainId
-//  6.2. Cleanup all cached cipherKeys
-TEST_CASE("flow/BlobCipher") {
-	TraceEvent("BlobCipherTest_Start").log();
-
-	// Construct a dummy External Key Manager representation and populate with some keys
-	class BaseCipher : public ReferenceCounted<BaseCipher>, NonCopyable {
-	public:
-		EncryptCipherDomainId domainId;
-		int len;
-		EncryptCipherBaseKeyId keyId;
-		std::unique_ptr<uint8_t[]> key;
-		EncryptCipherRandomSalt generatedSalt;
-
-		BaseCipher(const EncryptCipherDomainId& dId, const EncryptCipherBaseKeyId& kId)
-		  : domainId(dId), len(deterministicRandom()->randomInt(AES_256_KEY_LENGTH / 2, AES_256_KEY_LENGTH + 1)),
-		    keyId(kId), key(std::make_unique<uint8_t[]>(len)) {
-			generateRandomData(key.get(), len);
-		}
-	};
-
-	using BaseKeyMap = std::unordered_map<EncryptCipherBaseKeyId, Reference<BaseCipher>>;
-	using DomainKeyMap = std::unordered_map<EncryptCipherDomainId, BaseKeyMap>;
-	DomainKeyMap domainKeyMap;
-	const EncryptCipherDomainId minDomainId = 1;
-	const EncryptCipherDomainId maxDomainId = deterministicRandom()->randomInt(minDomainId, minDomainId + 10) + 5;
-	const EncryptCipherBaseKeyId minBaseCipherKeyId = 100;
-	const EncryptCipherBaseKeyId maxBaseCipherKeyId =
-	    deterministicRandom()->randomInt(minBaseCipherKeyId, minBaseCipherKeyId + 50) + 15;
-	for (int dId = minDomainId; dId <= maxDomainId; dId++) {
-		for (int kId = minBaseCipherKeyId; kId <= maxBaseCipherKeyId; kId++) {
-			domainKeyMap[dId].emplace(kId, makeReference<BaseCipher>(dId, kId));
-		}
-	}
-	ASSERT_EQ(domainKeyMap.size(), maxDomainId);
-
-	Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
-
-	// validate getLatestCipherKey return empty when there's no cipher key
-	TraceEvent("BlobCipherTest_LatestKeyNotExists").log();
-	Reference<BlobCipherKey> latestKeyNonexists =
-	    cipherKeyCache->getLatestCipherKey(deterministicRandom()->randomInt(minDomainId, maxDomainId));
-	ASSERT(!latestKeyNonexists.isValid());
-	try {
-		cipherKeyCache->getLatestCipherKey(ENCRYPT_INVALID_DOMAIN_ID);
-		ASSERT(false); // shouldn't get here
-	} catch (Error& e) {
-		ASSERT_EQ(e.code(), error_code_encrypt_invalid_id);
-	}
-
-	// insert BlobCipher keys into BlobCipherKeyCache map and validate
-	TraceEvent("BlobCipherTest_InsertKeys").log();
-	for (auto& domainItr : domainKeyMap) {
-		for (auto& baseKeyItr : domainItr.second) {
-			Reference<BaseCipher> baseCipher = baseKeyItr.second;
-
-			cipherKeyCache->insertCipherKey(
-			    baseCipher->domainId, baseCipher->keyId, baseCipher->key.get(), baseCipher->len);
-			Reference<BlobCipherKey> fetchedKey = cipherKeyCache->getLatestCipherKey(baseCipher->domainId);
-			baseCipher->generatedSalt = fetchedKey->getSalt();
-		}
-	}
-	// insert EncryptHeader BlobCipher key
-	Reference<BaseCipher> headerBaseCipher = makeReference<BaseCipher>(ENCRYPT_HEADER_DOMAIN_ID, 1);
-	cipherKeyCache->insertCipherKey(
-	    headerBaseCipher->domainId, headerBaseCipher->keyId, headerBaseCipher->key.get(), headerBaseCipher->len);
-
-	TraceEvent("BlobCipherTest_InsertKeysDone").log();
-
-	// validate the cipherKey lookups work as desired
-	for (auto& domainItr : domainKeyMap) {
-		for (auto& baseKeyItr : domainItr.second) {
-			Reference<BaseCipher> baseCipher = baseKeyItr.second;
-			Reference<BlobCipherKey> cipherKey =
-			    cipherKeyCache->getCipherKey(baseCipher->domainId, baseCipher->keyId, baseCipher->generatedSalt);
-			ASSERT(cipherKey.isValid());
-			// validate common cipher properties - domainId, baseCipherId, baseCipherLen, rawBaseCipher
-			ASSERT_EQ(cipherKey->getBaseCipherId(), baseCipher->keyId);
-			ASSERT_EQ(cipherKey->getDomainId(), baseCipher->domainId);
-			ASSERT_EQ(cipherKey->getBaseCipherLen(), baseCipher->len);
-			// ensure that baseCipher matches with the cached information
-			ASSERT_EQ(std::memcmp(cipherKey->rawBaseCipher(), baseCipher->key.get(), cipherKey->getBaseCipherLen()), 0);
-			// validate the encryption derivation
-			ASSERT_NE(std::memcmp(cipherKey->rawCipher(), baseCipher->key.get(), cipherKey->getBaseCipherLen()), 0);
-		}
-	}
-	TraceEvent("BlobCipherTest_LooksupDone").log();
-
-	// Ensure attemtping to insert existing cipherKey (identical) more than once is treated as a NOP
-	try {
-		Reference<BaseCipher> baseCipher = domainKeyMap[minDomainId][minBaseCipherKeyId];
-		cipherKeyCache->insertCipherKey(
-		    baseCipher->domainId, baseCipher->keyId, baseCipher->key.get(), baseCipher->len);
-	} catch (Error& e) {
-		throw;
-	}
-	TraceEvent("BlobCipherTest_ReinsertIdempotentKeyDone").log();
-
-	// Ensure attemtping to insert an existing cipherKey (modified) fails with appropriate error
-	try {
-		Reference<BaseCipher> baseCipher = domainKeyMap[minDomainId][minBaseCipherKeyId];
-		uint8_t rawCipher[baseCipher->len];
-		memcpy(rawCipher, baseCipher->key.get(), baseCipher->len);
-		// modify few bytes in the cipherKey
-		for (int i = 2; i < 5; i++) {
-			rawCipher[i]++;
-		}
-		cipherKeyCache->insertCipherKey(baseCipher->domainId, baseCipher->keyId, &rawCipher[0], baseCipher->len);
-	} catch (Error& e) {
-		if (e.code() != error_code_encrypt_update_cipher) {
-			throw;
-		}
-	}
-	TraceEvent("BlobCipherTest_ReinsertNonIdempotentKeyDone").log();
-
-	// Validate Encryption ops
-	Reference<BlobCipherKey> cipherKey = cipherKeyCache->getLatestCipherKey(minDomainId);
-	Reference<BlobCipherKey> headerCipherKey = cipherKeyCache->getLatestCipherKey(ENCRYPT_HEADER_DOMAIN_ID);
-	const int bufLen = deterministicRandom()->randomInt(786, 2127) + 512;
-	uint8_t orgData[bufLen];
-	generateRandomData(&orgData[0], bufLen);
-
-	Arena arena;
-	uint8_t iv[AES_256_IV_LENGTH];
-	generateRandomData(&iv[0], AES_256_IV_LENGTH);
-
-	BlobCipherEncryptHeader headerCopy;
-	// validate basic encrypt followed by decrypt operation for AUTH_MODE_NONE
-	{
-		TraceEvent("NoneAuthMode_Start").log();
-
-		EncryptBlobCipherAes265Ctr encryptor(
-		    cipherKey, Reference<BlobCipherKey>(), iv, AES_256_IV_LENGTH, ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE);
-		BlobCipherEncryptHeader header;
-		Reference<EncryptBuf> encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-
-		ASSERT_EQ(encrypted->getLogicalSize(), bufLen);
-		ASSERT_NE(memcmp(&orgData[0], encrypted->begin(), bufLen), 0);
-		ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
-		ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
-		ASSERT_EQ(header.flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE);
-
-		TraceEvent("BlobCipherTest_EncryptDone")
-		    .detail("HeaderVersion", header.flags.headerVersion)
-		    .detail("HeaderEncryptMode", header.flags.encryptMode)
-		    .detail("DomainId", header.cipherTextDetails.encryptDomainId)
-		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId);
-
-		Reference<BlobCipherKey> tCipherKeyKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
-		                                                                      header.cipherTextDetails.baseCipherId,
-		                                                                      header.cipherTextDetails.salt);
-		ASSERT(tCipherKeyKey->isEqual(cipherKey));
-		DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, Reference<BlobCipherKey>(), &header.iv[0]);
-		Reference<EncryptBuf> decrypted = decryptor.decrypt(encrypted->begin(), bufLen, header, arena);
-
-		ASSERT_EQ(decrypted->getLogicalSize(), bufLen);
-		ASSERT_EQ(memcmp(decrypted->begin(), &orgData[0], bufLen), 0);
-
-		TraceEvent("BlobCipherTest_DecryptDone").log();
-
-		// induce encryption header corruption - headerVersion corrupted
-		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-		       reinterpret_cast<const uint8_t*>(&header),
-		       sizeof(BlobCipherEncryptHeader));
-		headerCopy.flags.headerVersion += 1;
-		try {
-			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, Reference<BlobCipherKey>(), header.iv);
-			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
-			ASSERT(false); // error expected
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
-				throw;
-			}
-		}
-
-		// induce encryption header corruption - encryptionMode corrupted
-		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-		       reinterpret_cast<const uint8_t*>(&header),
-		       sizeof(BlobCipherEncryptHeader));
-		headerCopy.flags.encryptMode += 1;
-		try {
-			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, Reference<BlobCipherKey>(), header.iv);
-			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
-			ASSERT(false); // error expected
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
-				throw;
-			}
-		}
-
-		// induce encrypted buffer payload corruption
-		try {
-			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-			uint8_t temp[bufLen];
-			memcpy(encrypted->begin(), &temp[0], bufLen);
-			int tIdx = deterministicRandom()->randomInt(0, bufLen - 1);
-			temp[tIdx] += 1;
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, Reference<BlobCipherKey>(), header.iv);
-			decrypted = decryptor.decrypt(&temp[0], bufLen, header, arena);
-		} catch (Error& e) {
-			// No authToken, hence, no corruption detection supported
-			ASSERT(false);
-		}
-
-		TraceEvent("NoneAuthMode_Done").log();
-	}
-
-	// validate basic encrypt followed by decrypt operation for AUTH_TOKEN_MODE_SINGLE
-	{
-		TraceEvent("SingleAuthMode_Start").log();
-
-		EncryptBlobCipherAes265Ctr encryptor(
-		    cipherKey, headerCipherKey, iv, AES_256_IV_LENGTH, ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
-		BlobCipherEncryptHeader header;
-		Reference<EncryptBuf> encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-
-		ASSERT_EQ(encrypted->getLogicalSize(), bufLen);
-		ASSERT_NE(memcmp(&orgData[0], encrypted->begin(), bufLen), 0);
-		ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
-		ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
-		ASSERT_EQ(header.flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
-
-		TraceEvent("BlobCipherTest_EncryptDone")
-		    .detail("HeaderVersion", header.flags.headerVersion)
-		    .detail("HeaderEncryptMode", header.flags.encryptMode)
-		    .detail("DomainId", header.cipherTextDetails.encryptDomainId)
-		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
-		    .detail("HeaderAuthToken",
-		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_SIZE).toString());
-
-		Reference<BlobCipherKey> tCipherKeyKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
-		                                                                      header.cipherTextDetails.baseCipherId,
-		                                                                      header.cipherTextDetails.salt);
-		Reference<BlobCipherKey> hCipherKey = cipherKeyCache->getCipherKey(header.cipherHeaderDetails.encryptDomainId,
-		                                                                   header.cipherHeaderDetails.baseCipherId,
-		                                                                   header.cipherHeaderDetails.salt);
-		ASSERT(tCipherKeyKey->isEqual(cipherKey));
-		DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv);
-		Reference<EncryptBuf> decrypted = decryptor.decrypt(encrypted->begin(), bufLen, header, arena);
-
-		ASSERT_EQ(decrypted->getLogicalSize(), bufLen);
-		ASSERT_EQ(memcmp(decrypted->begin(), &orgData[0], bufLen), 0);
-
-		TraceEvent("BlobCipherTest_DecryptDone").log();
-
-		// induce encryption header corruption - headerVersion corrupted
-		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-		       reinterpret_cast<const uint8_t*>(&header),
-		       sizeof(BlobCipherEncryptHeader));
-		headerCopy.flags.headerVersion += 1;
-		try {
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv);
-			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
-			ASSERT(false); // error expected
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
-				throw;
-			}
-		}
-
-		// induce encryption header corruption - encryptionMode corrupted
-		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-		       reinterpret_cast<const uint8_t*>(&header),
-		       sizeof(BlobCipherEncryptHeader));
-		headerCopy.flags.encryptMode += 1;
-		try {
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv);
-			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
-			ASSERT(false); // error expected
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
-				throw;
-			}
-		}
-
-		// induce encryption header corruption - authToken mismatch
-		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-		       reinterpret_cast<const uint8_t*>(&header),
-		       sizeof(BlobCipherEncryptHeader));
-		int hIdx = deterministicRandom()->randomInt(0, AUTH_TOKEN_SIZE - 1);
-		headerCopy.singleAuthToken.authToken[hIdx] += 1;
-		try {
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv);
-			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
-			ASSERT(false); // error expected
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
-				throw;
-			}
-		}
-
-		// induce encrypted buffer payload corruption
-		try {
-			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-			uint8_t temp[bufLen];
-			memcpy(encrypted->begin(), &temp[0], bufLen);
-			int tIdx = deterministicRandom()->randomInt(0, bufLen - 1);
-			temp[tIdx] += 1;
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, header.iv);
-			decrypted = decryptor.decrypt(&temp[0], bufLen, header, arena);
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
-				throw;
-			}
-		}
-
-		TraceEvent("SingleAuthMode_Done").log();
-	}
-
-	// validate basic encrypt followed by decrypt operation for AUTH_TOKEN_MODE_MULTI
-	{
-		TraceEvent("MultiAuthMode_Start").log();
-
-		EncryptBlobCipherAes265Ctr encryptor(
-		    cipherKey, headerCipherKey, iv, AES_256_IV_LENGTH, ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
-		BlobCipherEncryptHeader header;
-		Reference<EncryptBuf> encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-
-		ASSERT_EQ(encrypted->getLogicalSize(), bufLen);
-		ASSERT_NE(memcmp(&orgData[0], encrypted->begin(), bufLen), 0);
-		ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
-		ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
-		ASSERT_EQ(header.flags.authTokenMode, ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI);
-
-		TraceEvent("BlobCipherTest_EncryptDone")
-		    .detail("HeaderVersion", header.flags.headerVersion)
-		    .detail("HeaderEncryptMode", header.flags.encryptMode)
-		    .detail("DomainId", header.cipherTextDetails.encryptDomainId)
-		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
-		    .detail("HeaderAuthToken",
-		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_SIZE).toString());
-
-		Reference<BlobCipherKey> tCipherKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
-		                                                                   header.cipherTextDetails.baseCipherId,
-		                                                                   header.cipherTextDetails.salt);
-		Reference<BlobCipherKey> hCipherKey = cipherKeyCache->getCipherKey(header.cipherHeaderDetails.encryptDomainId,
-		                                                                   header.cipherHeaderDetails.baseCipherId,
-		                                                                   header.cipherHeaderDetails.salt);
-
-		ASSERT(tCipherKey->isEqual(cipherKey));
-		DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv);
-		Reference<EncryptBuf> decrypted = decryptor.decrypt(encrypted->begin(), bufLen, header, arena);
-
-		ASSERT_EQ(decrypted->getLogicalSize(), bufLen);
-		ASSERT_EQ(memcmp(decrypted->begin(), &orgData[0], bufLen), 0);
-
-		TraceEvent("BlobCipherTest_DecryptDone").log();
-
-		// induce encryption header corruption - headerVersion corrupted
-		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-		       reinterpret_cast<const uint8_t*>(&header),
-		       sizeof(BlobCipherEncryptHeader));
-		headerCopy.flags.headerVersion += 1;
-		try {
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv);
-			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
-			ASSERT(false); // error expected
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
-				throw;
-			}
-		}
-
-		// induce encryption header corruption - encryptionMode corrupted
-		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-		       reinterpret_cast<const uint8_t*>(&header),
-		       sizeof(BlobCipherEncryptHeader));
-		headerCopy.flags.encryptMode += 1;
-		try {
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv);
-			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
-			ASSERT(false); // error expected
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_metadata_mismatch) {
-				throw;
-			}
-		}
-
-		// induce encryption header corruption - cipherText authToken mismatch
-		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-		       reinterpret_cast<const uint8_t*>(&header),
-		       sizeof(BlobCipherEncryptHeader));
-		int hIdx = deterministicRandom()->randomInt(0, AUTH_TOKEN_SIZE - 1);
-		headerCopy.multiAuthTokens.cipherTextAuthToken[hIdx] += 1;
-		try {
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv);
-			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
-			ASSERT(false); // error expected
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
-				throw;
-			}
-		}
-
-		// induce encryption header corruption - header authToken mismatch
-		encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-		memcpy(reinterpret_cast<uint8_t*>(&headerCopy),
-		       reinterpret_cast<const uint8_t*>(&header),
-		       sizeof(BlobCipherEncryptHeader));
-		hIdx = deterministicRandom()->randomInt(0, AUTH_TOKEN_SIZE - 1);
-		headerCopy.multiAuthTokens.headerAuthToken[hIdx] += 1;
-		try {
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv);
-			decrypted = decryptor.decrypt(encrypted->begin(), bufLen, headerCopy, arena);
-			ASSERT(false); // error expected
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
-				throw;
-			}
-		}
-
-		try {
-			encrypted = encryptor.encrypt(&orgData[0], bufLen, &header, arena);
-			uint8_t temp[bufLen];
-			memcpy(encrypted->begin(), &temp[0], bufLen);
-			int tIdx = deterministicRandom()->randomInt(0, bufLen - 1);
-			temp[tIdx] += 1;
-			DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, header.iv);
-			decrypted = decryptor.decrypt(&temp[0], bufLen, header, arena);
-		} catch (Error& e) {
-			if (e.code() != error_code_encrypt_header_authtoken_mismatch) {
-				throw;
-			}
-		}
-
-		TraceEvent("MultiAuthMode_Done").log();
-	}
-
-	// Validate dropping encryptDomainId cached keys
-	const EncryptCipherDomainId candidate = deterministicRandom()->randomInt(minDomainId, maxDomainId);
-	cipherKeyCache->resetEncryptDomainId(candidate);
-	std::vector<Reference<BlobCipherKey>> cachedKeys = cipherKeyCache->getAllCiphers(candidate);
-	ASSERT(cachedKeys.empty());
-
-	// Validate dropping all cached cipherKeys
-	cipherKeyCache->cleanup();
-	for (int dId = minDomainId; dId < maxDomainId; dId++) {
-		std::vector<Reference<BlobCipherKey>> cachedKeys = cipherKeyCache->getAllCiphers(dId);
-		ASSERT(cachedKeys.empty());
-	}
-
-	TraceEvent("BlobCipherTest_Done").log();
-	return Void();
-}
diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt
index 9f0513e7a9..37f1cc4187 100644
--- a/flow/CMakeLists.txt
+++ b/flow/CMakeLists.txt
@@ -7,10 +7,16 @@ list(REMOVE_ITEM FLOW_SRCS TLSTest.cpp)
 list(REMOVE_ITEM FLOW_SRCS MkCertCli.cpp)
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
-  list(APPEND FLOW_SRCS aarch64/memcmp.S aarch64/memcpy.S)
+    list(APPEND FLOW_SRCS aarch64/memcmp.S aarch64/memcpy.S)
 endif()
 
 make_directory(${CMAKE_CURRENT_BINARY_DIR}/include/flow)
+set(FDB_API_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/ApiVersions.cmake" CACHE STRING "Api version cmake file." FORCE)
+include(${FDB_API_VERSION_FILE})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ApiVersion.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/flow/ApiVersion.h)
+set(FDB_PROTOCOL_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/ProtocolVersions.cmake" CACHE STRING "Protocol version cmake file." FORCE)
+include(${FDB_PROTOCOL_VERSION_FILE})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ProtocolVersion.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/flow/ProtocolVersion.h)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/SourceVersion.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/flow/SourceVersion.h)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/flow/config.h)
 
@@ -24,79 +30,81 @@ add_flow_target(STATIC_LIBRARY NAME flow_sampling SRCS ${FLOW_SRCS})
 add_flow_target(LINK_TEST NAME flowlinktest SRCS LinkTest.cpp)
 target_link_libraries(flowlinktest PRIVATE flow stacktrace)
 
-find_package(ZLIB)
-if(ZLIB_FOUND)
-  add_compile_definitions(ZLIB_LIB_SUPPORTED)
-  target_link_libraries(flow PRIVATE ZLIB::ZLIB)
-else()
-  message(STATUS "ZLIB package not found")
+#find_package(ZLIB)
+#if(ZLIB_FOUND)
+#  target_compile_definitions(flow PUBLIC ZLIB_LIB_SUPPORTED)
+#  target_link_libraries(flow PUBLIC ZLIB::ZLIB)
+#else()
+#  message(STATUS "ZLIB package not found")
+#endif()
+
+set(IS_ARM_MAC NO)
+if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
+    set(IS_ARM_MAC YES)
+endif()
+
+# TODO: Limit ZSTD availability to CLANG, for gcc resolve library link ordering
+if(CLANG AND NOT IS_ARM_MAC)
+    include(CompileZstd)
+    compile_zstd()
+    target_link_libraries(flow PUBLIC ZSTD::ZSTD)
+    target_compile_definitions(flow PUBLIC ZSTD_LIB_SUPPORTED)
 endif()
 
 foreach(ft flow flow_sampling flowlinktest)
-  target_include_directories(${ft} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+    target_include_directories(${ft} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 
-  target_link_libraries(${ft} PRIVATE stacktrace)
-  target_link_libraries(${ft} PUBLIC fmt::fmt SimpleOpt crc32)
-  if(UNIX AND NOT APPLE)
-    target_link_libraries(${ft} PRIVATE folly_memcpy)
-    target_compile_definitions(${ft} PRIVATE WITH_FOLLY_MEMCPY)
-  endif()
-
-  if (NOT APPLE AND NOT WIN32)
-    set (FLOW_LIBS ${FLOW_LIBS} rt)
-  elseif(WIN32)
-    target_link_libraries(${ft} PUBLIC winmm.lib)
-    target_link_libraries(${ft} PUBLIC psapi.lib)
-  endif()
-
-  if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
-    set (FLOW_LIBS ${FLOW_LIBS} execinfo devstat)
-    find_library(EIO eio)
-    if(EIO)
-      target_link_libraries(${ft} PUBLIC ${EIO})
+    target_link_libraries(${ft} PRIVATE stacktrace)
+    target_link_libraries(${ft} PUBLIC fmt::fmt SimpleOpt crc32)
+    if(UNIX AND NOT APPLE)
+        target_link_libraries(${ft} PRIVATE folly_memcpy)
+        target_compile_definitions(${ft} PRIVATE WITH_FOLLY_MEMCPY)
     endif()
-  endif()
-  target_link_libraries(${ft} PRIVATE ${FLOW_LIBS})
 
-  if(USE_VALGRIND)
-    target_link_libraries(${ft} PUBLIC Valgrind)
-  endif()
-  target_link_libraries(${ft} PUBLIC OpenSSL::SSL)
-  if(USE_WOLFSSL)
-    target_include_directories(${ft} SYSTEM BEFORE PUBLIC ${WOLFSSL_INCLUDE_DIR}/wolfssl)
-  endif()
-  target_link_libraries(${ft} PUBLIC Threads::Threads ${CMAKE_DL_LIBS})
-  if(USE_SANITIZER)
-    target_link_libraries(${ft} PUBLIC boost_asan)
-  else()
+    if(NOT APPLE AND NOT WIN32)
+        set(FLOW_LIBS ${FLOW_LIBS} rt)
+    elseif(WIN32)
+        target_link_libraries(${ft} PUBLIC winmm.lib)
+        target_link_libraries(${ft} PUBLIC psapi.lib)
+    endif()
+
+    if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+        set(FLOW_LIBS ${FLOW_LIBS} execinfo devstat)
+        find_library(EIO eio)
+        if(EIO)
+            target_link_libraries(${ft} PUBLIC ${EIO})
+        endif()
+    endif()
+    target_link_libraries(${ft} PRIVATE ${FLOW_LIBS})
+
+    if(USE_VALGRIND)
+        target_link_libraries(${ft} PUBLIC Valgrind)
+    endif()
+    target_link_libraries(${ft} PUBLIC OpenSSL::SSL)
+    if(USE_WOLFSSL)
+        target_include_directories(${ft} SYSTEM BEFORE PUBLIC ${WOLFSSL_INCLUDE_DIR}/wolfssl)
+    endif()
+    target_link_libraries(${ft} PUBLIC Threads::Threads ${CMAKE_DL_LIBS})
     target_link_libraries(${ft} PUBLIC boost_target)
-  endif()
-  if(USE_VALGRIND)
-    target_link_libraries(${ft} PUBLIC Valgrind)
-  endif()
+    if(USE_VALGRIND)
+        target_link_libraries(${ft} PUBLIC Valgrind)
+    endif()
 
-  if(APPLE)
-    find_library(IO_KIT IOKit)
-    find_library(CORE_FOUNDATION CoreFoundation)
-    target_link_libraries(${ft} PRIVATE ${IO_KIT} ${CORE_FOUNDATION})
-  endif()
+    if(APPLE)
+        find_library(IO_KIT IOKit)
+        find_library(CORE_FOUNDATION CoreFoundation)
+        target_link_libraries(${ft} PRIVATE ${IO_KIT} ${CORE_FOUNDATION})
+    endif()
 endforeach()
 
 target_compile_definitions(flow_sampling PRIVATE -DENABLE_SAMPLING)
 if(WIN32)
-  add_dependencies(flow_sampling_actors flow_actors)
+    add_dependencies(flow_sampling_actors flow_actors)
 endif()
 
 if(OPEN_FOR_IDE)
-  add_library(mkcert OBJECT MkCertCli.cpp)
-  add_library(mtls_unittest OBJECT TLSTest.cpp)
+    add_library(mkcert OBJECT MkCertCli.cpp)
 else()
-  add_executable(mkcert MkCertCli.cpp)
-  add_executable(mtls_unittest TLSTest.cpp)
+    add_executable(mkcert MkCertCli.cpp)
 endif()
 target_link_libraries(mkcert PUBLIC flow)
-target_link_libraries(mtls_unittest PUBLIC flow)
-if(NOT OPEN_FOR_IDE)
-  add_test(NAME mutual_tls_unittest
-           COMMAND $<TARGET_FILE:mtls_unittest>)
-endif()
diff --git a/flow/CodeProbe.cpp b/flow/CodeProbe.cpp
new file mode 100644
index 0000000000..0a4d391cd9
--- /dev/null
+++ b/flow/CodeProbe.cpp
@@ -0,0 +1,293 @@
+/*
+ * CodeProbe.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flow/CodeProbe.h"
+#include "flow/CodeProbeUtils.h"
+#include "flow/Arena.h"
+#include "flow/network.h"
+
+#include <map>
+#include <fmt/format.h>
+#include <boost/core/demangle.hpp>
+#include <boost/unordered_map.hpp>
+#include <typeinfo>
+
+namespace probe {
+
+namespace {
+
+std::vector<ExecutionContext> fromStrings(std::vector<std::string> const& ctxs) {
+	std::vector<ExecutionContext> res;
+	for (auto const& ctx : ctxs) {
+		std::string c;
+		c.reserve(ctx.size());
+		std::transform(ctx.begin(), ctx.end(), std::back_inserter(c), [](char c) { return std::tolower(c); });
+		if (c == "all") {
+			res.push_back(ExecutionContext::Net2);
+			res.push_back(ExecutionContext::Simulation);
+		} else if (c == "simulation") {
+			res.push_back(ExecutionContext::Simulation);
+		} else if (c == "net2") {
+			res.push_back(ExecutionContext::Net2);
+		} else {
+			throw invalid_option_value();
+		}
+	}
+	std::sort(res.begin(), res.end());
+	res.erase(std::unique(res.begin(), res.end()), res.end());
+	return res;
+}
+
+std::string_view normalizePath(const char* path) {
+	std::string_view srcBase(FDB_SOURCE_DIR);
+	std::string_view binBase(FDB_SOURCE_DIR);
+	std::string_view filename(path);
+	if (srcBase.size() < filename.size() && filename.substr(0, srcBase.size()) == srcBase) {
+		filename.remove_prefix(srcBase.size());
+	} else if (binBase.size() < filename.size() && filename.substr(0, binBase.size()) == binBase) {
+		filename.remove_prefix(binBase.size());
+	}
+	if (filename[0] == '/') {
+		filename.remove_prefix(1);
+	}
+	return filename;
+}
+
+struct CodeProbes {
+	struct Location {
+		std::string_view file;
+		unsigned line;
+		Location(std::string_view file, unsigned line) : file(file), line(line) {}
+		bool operator==(Location const& rhs) const { return line == rhs.line && file == rhs.file; }
+		bool operator!=(Location const& rhs) const { return line != rhs.line && file != rhs.file; }
+		bool operator<(Location const& rhs) const {
+			if (file < rhs.file) {
+				return true;
+			} else if (file == rhs.file) {
+				return line < rhs.line;
+			} else {
+				return false;
+			}
+		}
+		bool operator<=(Location const& rhs) const {
+			if (file < rhs.file) {
+				return true;
+			} else if (file == rhs.file) {
+				return line <= rhs.line;
+			} else {
+				return false;
+			}
+		}
+		bool operator>(Location const& rhs) const { return rhs < *this; }
+		bool operator>=(Location const& rhs) const { return rhs <= *this; }
+	};
+
+	std::multimap<Location, ICodeProbe const*> codeProbes;
+
+	void traceMissedProbes(Optional<ExecutionContext> context) const;
+
+	void add(ICodeProbe const* probe) {
+		Location loc(probe->filename(), probe->line());
+		codeProbes.emplace(loc, probe);
+	}
+
+	static CodeProbes& instance() {
+		static CodeProbes probes;
+		return probes;
+	}
+
+	void verify() const {
+		std::map<std::pair<std::string_view, std::string_view>, ICodeProbe const*> comments;
+		for (auto probe : codeProbes) {
+			auto file = probe.first.file;
+			auto comment = probe.second->comment();
+			auto commentEntry = std::make_pair(file, std::string_view(comment));
+			ASSERT(file == probe.second->filename());
+			auto iter = comments.find(commentEntry);
+			if (iter != comments.end() && probe.second->line() != iter->second->line()) {
+				fmt::print("ERROR ({}:{}): {} isn't unique in file {}. Previously seen here: {}:{}\n",
+				           probe.first.file,
+				           probe.first.line,
+				           iter->first.second,
+				           probe.second->filename(),
+				           iter->second->filename(),
+				           iter->second->line());
+				// ICodeProbe const& fst = *iter->second;
+				// ICodeProbe const& snd = *probe.second;
+				// fmt::print("\t1st Type: {}\n", boost::core::demangle(typeid(fst).name()));
+				// fmt::print("\t2nd Type: {}\n", boost::core::demangle(typeid(snd).name()));
+				// fmt::print("\n");
+				// fmt::print("\t1st Comment: {}\n", fst.comment());
+				// fmt::print("\t2nd Comment: {}\n", snd.comment());
+				// fmt::print("\n");
+				// fmt::print("\t1st CompUnit: {}\n", fst.compilationUnit());
+				// fmt::print("\t2nd CompUnit: {}\n", snd.compilationUnit());
+				// fmt::print("\n");
+			} else {
+				comments.emplace(commentEntry, probe.second);
+			}
+		}
+	}
+
+	void printXML() const {
+		verify();
+		fmt::print("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n");
+		fmt::print("<CoverageTool>\n");
+		if (codeProbes.empty()) {
+			fmt::print("\t<CoverageCases/>\n");
+			fmt::print("\t<Inputs/>\n");
+		} else {
+			std::vector<std::string_view> files;
+			fmt::print("\t<CoverageCases>\n");
+			for (auto probe : codeProbes) {
+				files.push_back(probe.first.file);
+				fmt::print("\t\t<Case File=\"{}\" Line=\"{}\" Comment=\"{}\" Condition=\"{}\"/>\n",
+				           probe.first.file,
+				           probe.first.line,
+				           probe.second->comment(),
+				           probe.second->condition());
+			}
+			fmt::print("\t</CoverageCases>\n");
+			fmt::print("\t<Inputs>\n");
+			for (auto const& f : files) {
+				fmt::print("\t\t<Input>{}</Input>\n", f);
+			}
+			fmt::print("\t</Inputs>\n");
+		}
+		fmt::print("</CoverageTool>\n");
+	}
+
+	void printJSON(std::vector<std::string> const& context = std::vector<std::string>()) const {
+		verify();
+		do {
+			struct foo {};
+			foo f;
+			fmt::print("{}\n", boost::core::demangle(typeid(f).name()));
+		} while (false);
+		do {
+			struct foo {};
+			foo f;
+			fmt::print("{}\n", boost::core::demangle(typeid(f).name()));
+		} while (false);
+		auto contexts = fromStrings(context);
+		const ICodeProbe* prev = nullptr;
+		for (auto probe : codeProbes) {
+			auto p = probe.second;
+			if (!contexts.empty()) {
+				bool print = false;
+				for (auto c : contexts) {
+					print = print || p->expectInContext(c);
+				}
+				if (!print) {
+					continue;
+				}
+			}
+			if (prev == nullptr || *prev != *probe.second) {
+				fmt::print(
+				    "{{ \"File\": \"{}\", \"Line\": {}, \"Comment\": \"{}\", \"Condition\": \"{}\", \"Function\": "
+				    "\"{}\" }}\n",
+				    probe.first.file,
+				    p->line(),
+				    p->comment(),
+				    p->condition(),
+				    p->function());
+			}
+			prev = probe.second;
+		}
+	}
+};
+
+size_t hash_value(CodeProbes::Location const& location) {
+	size_t seed = 0;
+	boost::hash_combine(seed, location.file);
+	boost::hash_combine(seed, location.line);
+	return seed;
+}
+
+void CodeProbes::traceMissedProbes(Optional<ExecutionContext> context) const {
+	boost::unordered_map<Location, bool> locations;
+	for (auto probe : codeProbes) {
+		decltype(locations.begin()) iter;
+		std::tie(iter, std::ignore) = locations.emplace(probe.first, false);
+		iter->second = iter->second || probe.second->wasHit();
+	}
+	for (const auto& [loc, probe] : codeProbes) {
+		auto iter = locations.find(loc);
+		ASSERT(iter != locations.end());
+		if (!iter->second && probe->shouldTrace()) {
+			iter->second = true;
+			probe->trace(false);
+		}
+	}
+}
+
+} // namespace
+
+std::string functionNameFromInnerType(const char* name) {
+	auto res = boost::core::demangle(name);
+	auto pos = res.find_last_of(':');
+	ASSERT(pos != res.npos);
+	return res.substr(0, pos - 1);
+}
+
+void registerProbe(const ICodeProbe& probe) {
+	CodeProbes::instance().add(&probe);
+}
+
+void traceMissedProbes(Optional<ExecutionContext> context) {
+	CodeProbes::instance().traceMissedProbes(context);
+}
+
+ICodeProbe::~ICodeProbe() {}
+
+bool ICodeProbe::operator==(const ICodeProbe& other) const {
+	return filename() == other.filename() && line() == other.line();
+}
+
+bool ICodeProbe::operator!=(const ICodeProbe& other) const {
+	return !(*this == other);
+}
+
+std::string_view ICodeProbe::filename() const {
+	return normalizePath(filePath());
+}
+
+void ICodeProbe::printProbesXML() {
+	CodeProbes::instance().printXML();
+}
+
+void ICodeProbe::printProbesJSON(std::vector<std::string> const& ctxs) {
+	CodeProbes::instance().printJSON(ctxs);
+}
+
+// annotations
+namespace assert {
+
+bool NoSim::operator()(ICodeProbe const* self) const {
+	return !g_network->isSimulated();
+}
+
+bool SimOnly::operator()(ICodeProbe const* self) const {
+	return g_network->isSimulated();
+}
+
+} // namespace assert
+
+} // namespace probe
diff --git a/flow/CompressedInt.actor.cpp b/flow/CompressedInt.actor.cpp
index ef889cf23f..0081f79c1d 100644
--- a/flow/CompressedInt.actor.cpp
+++ b/flow/CompressedInt.actor.cpp
@@ -86,12 +86,12 @@ void testCompressedInt(IntType n, StringRef rep = StringRef()) {
 }
 
 TEST_CASE("/flow/compressed_ints") {
-	testCompressedInt<int>(-2, LiteralStringRef("\x7e"));
-	testCompressedInt<int>(-1, LiteralStringRef("\x7f"));
-	testCompressedInt<int>(0, LiteralStringRef("\x80"));
-	testCompressedInt<int>(1, LiteralStringRef("\x81"));
-	testCompressedInt<int>(2, LiteralStringRef("\x82"));
-	testCompressedInt<int64_t>(0x4000000000000000, LiteralStringRef("\xFF\xC0\x40\x00\x00\x00\x00\x00\x00\x00"));
+	testCompressedInt<int>(-2, "\x7e"_sr);
+	testCompressedInt<int>(-1, "\x7f"_sr);
+	testCompressedInt<int>(0, "\x80"_sr);
+	testCompressedInt<int>(1, "\x81"_sr);
+	testCompressedInt<int>(2, "\x82"_sr);
+	testCompressedInt<int64_t>(0x4000000000000000, "\xFF\xC0\x40\x00\x00\x00\x00\x00\x00\x00"_sr);
 
 	int64_t n = 0;
 	for (int i = 0; i < 10000000; ++i) {
diff --git a/flow/CompressionUtils.cpp b/flow/CompressionUtils.cpp
index 3e7c4dbe94..133fb775a4 100644
--- a/flow/CompressionUtils.cpp
+++ b/flow/CompressionUtils.cpp
@@ -21,6 +21,7 @@
 #include "flow/CompressionUtils.h"
 
 #include "flow/Arena.h"
+#include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/UnitTest.h"
 
@@ -29,9 +30,32 @@
 #include <boost/iostreams/filter/gzip.hpp>
 #endif
 #include <boost/iostreams/filtering_streambuf.hpp>
+#ifdef ZSTD_LIB_SUPPORTED
+#include <boost/iostreams/filter/zstd.hpp>
+#endif
 #include <sstream>
 
+namespace {
+std::unordered_set<CompressionFilter> getSupportedFilters() {
+	std::unordered_set<CompressionFilter> filters;
+
+	filters.insert(CompressionFilter::NONE);
+#ifdef ZLIB_LIB_SUPPORTED
+	filters.insert(CompressionFilter::GZIP);
+#endif
+#ifdef ZSTD_LIB_SUPPORTED
+	filters.insert(CompressionFilter::ZSTD);
+#endif
+	ASSERT_GE(filters.size(), 1);
+	return filters;
+}
+} // namespace
+
+std::unordered_set<CompressionFilter> CompressionUtils::supportedFilters = getSupportedFilters();
+
 StringRef CompressionUtils::compress(const CompressionFilter filter, const StringRef& data, Arena& arena) {
+	checkFilterSupported(filter);
+
 	if (filter == CompressionFilter::NONE) {
 		return StringRef(arena, data);
 	}
@@ -42,11 +66,17 @@ StringRef CompressionUtils::compress(const CompressionFilter filter, const Strin
 		return CompressionUtils::compress(filter, data, bio::gzip::default_compression, arena);
 	}
 #endif
-	throw not_implemented();
+#ifdef ZSTD_LIB_SUPPORTED
+	if (filter == CompressionFilter::ZSTD) {
+		return CompressionUtils::compress(filter, data, bio::zstd::default_compression, arena);
+	}
+#endif
+
+	throw internal_error(); // We should never get here
 }
 
 StringRef CompressionUtils::compress(const CompressionFilter filter, const StringRef& data, int level, Arena& arena) {
-	ASSERT(filter < CompressionFilter::LAST);
+	checkFilterSupported(filter);
 
 	if (filter == CompressionFilter::NONE) {
 		return StringRef(arena, data);
@@ -62,6 +92,12 @@ StringRef CompressionUtils::compress(const CompressionFilter filter, const Strin
 		out.push(bio::gzip_compressor(bio::gzip_params(level)));
 	}
 #endif
+#ifdef ZSTD_LIB_SUPPORTED
+	if (filter == CompressionFilter::ZSTD) {
+		out.push(bio::zstd_compressor(bio::zstd_params(level)));
+	}
+#endif
+
 	out.push(decomStream);
 	bio::copy(out, compStream);
 
@@ -69,7 +105,7 @@ StringRef CompressionUtils::compress(const CompressionFilter filter, const Strin
 }
 
 StringRef CompressionUtils::decompress(const CompressionFilter filter, const StringRef& data, Arena& arena) {
-	ASSERT(filter < CompressionFilter::LAST);
+	checkFilterSupported(filter);
 
 	if (filter == CompressionFilter::NONE) {
 		return StringRef(arena, data);
@@ -85,20 +121,107 @@ StringRef CompressionUtils::decompress(const CompressionFilter filter, const Str
 		out.push(bio::gzip_decompressor());
 	}
 #endif
+#ifdef ZSTD_LIB_SUPPORTED
+	if (filter == CompressionFilter::ZSTD) {
+		out.push(bio::zstd_decompressor());
+	}
+#endif
+
 	out.push(compStream);
 	bio::copy(out, decompStream);
 
 	return StringRef(arena, decompStream.str());
 }
 
+int CompressionUtils::getDefaultCompressionLevel(CompressionFilter filter) {
+	checkFilterSupported(filter);
+
+	if (filter == CompressionFilter::NONE) {
+		return -1;
+	}
+
+#ifdef ZLIB_LIB_SUPPORTED
+	if (filter == CompressionFilter::GZIP) {
+		// opt for high speed compression, larger levels have a high cpu cost and not much compression ratio
+		// improvement, according to benchmarks
+		// return boost::iostream::gzip::default_compression;
+		// return boost::iostream::gzip::best_compression;
+		return boost::iostreams::gzip::best_speed;
+	}
+#endif
+#ifdef ZSTD_LIB_SUPPORTED
+	if (filter == CompressionFilter::ZSTD) {
+		// opt for high speed compression, larger levels have a high cpu cost and not much compression ratio
+		// improvement, according to benchmarks
+		// return boost::iostreams::zstd::default_compression;
+		// return boost::iostreams::zstd::best_compression;
+		return boost::iostreams::zstd::best_speed;
+	}
+#endif
+
+	throw internal_error(); // We should never get here
+}
+
+CompressionFilter CompressionUtils::getRandomFilter() {
+	ASSERT_GE(supportedFilters.size(), 1);
+	std::vector<CompressionFilter> filters;
+	filters.insert(filters.end(), CompressionUtils::supportedFilters.begin(), CompressionUtils::supportedFilters.end());
+
+	ASSERT_GE(filters.size(), 1);
+
+	CompressionFilter res;
+	if (filters.size() == 1) {
+		res = filters[0];
+	} else {
+		int idx = deterministicRandom()->randomInt(0, filters.size());
+		res = filters[idx];
+	}
+
+	ASSERT(supportedFilters.find(res) != supportedFilters.end());
+	return res;
+}
+
 // Only used to link unit tests
 void forceLinkCompressionUtilsTest() {}
 
+namespace {
+void testCompression(CompressionFilter filter) {
+	Arena arena;
+	const int size = deterministicRandom()->randomInt(512, 1024);
+	Standalone<StringRef> uncompressed = makeString(size);
+	deterministicRandom()->randomBytes(mutateString(uncompressed), size);
+
+	Standalone<StringRef> compressed = CompressionUtils::compress(filter, uncompressed, arena);
+	ASSERT_NE(compressed.compare(uncompressed), 0);
+
+	StringRef verify = CompressionUtils::decompress(filter, compressed, arena);
+	ASSERT_EQ(verify.compare(uncompressed), 0);
+}
+
+void testCompression2(CompressionFilter filter) {
+	Arena arena;
+	const int size = deterministicRandom()->randomInt(512, 1024);
+	std::string s(size, 'x');
+	Standalone<StringRef> uncompressed = Standalone<StringRef>(StringRef(s));
+	printf("Size before: %d\n", (int)uncompressed.size());
+
+	Standalone<StringRef> compressed = CompressionUtils::compress(filter, uncompressed, arena);
+	ASSERT_NE(compressed.compare(uncompressed), 0);
+	printf("Size after: %d\n", (int)compressed.size());
+	// Assert compressed size is less than half.
+	ASSERT(compressed.size() * 2 < uncompressed.size());
+
+	StringRef verify = CompressionUtils::decompress(filter, compressed, arena);
+	ASSERT_EQ(verify.compare(uncompressed), 0);
+}
+
+} // namespace
+
 TEST_CASE("/CompressionUtils/noCompression") {
 	Arena arena;
 	const int size = deterministicRandom()->randomInt(512, 1024);
 	Standalone<StringRef> uncompressed = makeString(size);
-	generateRandomData(mutateString(uncompressed), size);
+	deterministicRandom()->randomBytes(mutateString(uncompressed), size);
 
 	Standalone<StringRef> compressed = CompressionUtils::compress(CompressionFilter::NONE, uncompressed, arena);
 	ASSERT_EQ(compressed.compare(uncompressed), 0);
@@ -106,25 +229,38 @@ TEST_CASE("/CompressionUtils/noCompression") {
 	StringRef verify = CompressionUtils::decompress(CompressionFilter::NONE, compressed, arena);
 	ASSERT_EQ(verify.compare(uncompressed), 0);
 
-	TraceEvent("NoCompression_Done").log();
+	TraceEvent("NoCompressionDone");
 
 	return Void();
 }
 
 #ifdef ZLIB_LIB_SUPPORTED
 TEST_CASE("/CompressionUtils/gzipCompression") {
-	Arena arena;
-	const int size = deterministicRandom()->randomInt(512, 1024);
-	Standalone<StringRef> uncompressed = makeString(size);
-	generateRandomData(mutateString(uncompressed), size);
+	testCompression(CompressionFilter::GZIP);
+	TraceEvent("GzipCompressionDone");
 
-	Standalone<StringRef> compressed = CompressionUtils::compress(CompressionFilter::GZIP, uncompressed, arena);
-	ASSERT_NE(compressed.compare(uncompressed), 0);
+	return Void();
+}
 
-	StringRef verify = CompressionUtils::decompress(CompressionFilter::GZIP, compressed, arena);
-	ASSERT_EQ(verify.compare(uncompressed), 0);
-
-	TraceEvent("GzipCompression_Done").log();
+TEST_CASE("/CompressionUtils/gzipCompression2") {
+	testCompression2(CompressionFilter::GZIP);
+	TraceEvent("GzipCompression2Done");
+
+	return Void();
+}
+#endif
+
+#ifdef ZSTD_LIB_SUPPORTED
+TEST_CASE("/CompressionUtils/zstdCompression") {
+	testCompression(CompressionFilter::ZSTD);
+	TraceEvent("ZstdCompressionDone");
+
+	return Void();
+}
+
+TEST_CASE("/CompressionUtils/zstdCompression2") {
+	testCompression2(CompressionFilter::ZSTD);
+	TraceEvent("ZstdCompression2Done");
 
 	return Void();
 }
diff --git a/flow/DeterministicRandom.cpp b/flow/DeterministicRandom.cpp
index 5fff2e0cb5..b55616e16f 100644
--- a/flow/DeterministicRandom.cpp
+++ b/flow/DeterministicRandom.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fmt/format.h"
+#include "flow/Arena.h"
 #include "flow/DeterministicRandom.h"
 
 #include <cstring>
@@ -92,11 +93,12 @@ uint64_t DeterministicRandom::randomUInt64() {
 }
 
 uint32_t DeterministicRandom::randomSkewedUInt32(uint32_t min, uint32_t maxPlusOne) {
-	std::uniform_real_distribution<double> distribution(std::log(min), std::log(maxPlusOne - 1));
-	double logpower = distribution(random);
-	uint32_t loguniform = static_cast<uint32_t>(std::pow(10, logpower));
-	// doubles can be imprecise, so let's make sure we don't violate an edge case.
-	return std::max(std::min(loguniform, maxPlusOne - 1), min);
+	ASSERT(min < maxPlusOne);
+	std::uniform_real_distribution<double> distribution(std::log(std::max<double>(min, 1.0 / M_E)),
+	                                                    std::log(maxPlusOne));
+	double exponent = distribution(random);
+	uint32_t value = static_cast<uint32_t>(std::pow(M_E, exponent));
+	return std::max(std::min(value, maxPlusOne - 1), min);
 }
 
 UID DeterministicRandom::randomUniqueID() {
@@ -124,6 +126,23 @@ std::string DeterministicRandom::randomAlphaNumeric(int length) {
 	return s;
 }
 
+void DeterministicRandom::randomBytes(uint8_t* buf, int length) {
+	constexpr const int unitLen = sizeof(decltype(gen64()));
+	for (int i = 0; i < length; i += unitLen) {
+		auto val = gen64();
+		memcpy(buf + i, &val, std::min(unitLen, length - i));
+	}
+	if (randLog && useRandLog) {
+		constexpr const int cutOff = 32;
+		bool tooLong = length > cutOff;
+		fmt::print(randLog,
+		           "Rbytes[{}] {}{}\n",
+		           length,
+		           StringRef(buf, std::min(cutOff, length)).printable(),
+		           tooLong ? "..." : "");
+	}
+}
+
 uint64_t DeterministicRandom::peek() const {
 	return next;
 }
@@ -134,10 +153,3 @@ void DeterministicRandom::addref() {
 void DeterministicRandom::delref() {
 	ReferenceCounted<DeterministicRandom>::delref();
 }
-
-void generateRandomData(uint8_t* buffer, int length) {
-	for (int i = 0; i < length; i += sizeof(uint32_t)) {
-		uint32_t val = deterministicRandom()->randomUInt32();
-		memcpy(&buffer[i], &val, std::min(length - i, (int)sizeof(uint32_t)));
-	}
-}
diff --git a/flow/EncryptUtils.cpp b/flow/EncryptUtils.cpp
index 12daf881d9..67015293b0 100644
--- a/flow/EncryptUtils.cpp
+++ b/flow/EncryptUtils.cpp
@@ -19,11 +19,17 @@
  */
 
 #include "flow/EncryptUtils.h"
+#include "flow/IRandom.h"
+#include "flow/Knobs.h"
 #include "flow/Trace.h"
 
 #include <boost/algorithm/string.hpp>
 #include <boost/format.hpp>
 
+const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME = "FdbSystemKeyspaceEncryptDomain"_sr;
+const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"_sr;
+const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME = "FdbEncryptHeaderDomain"_sr;
+
 EncryptCipherMode encryptModeFromString(const std::string& modeStr) {
 	if (modeStr == "NONE") {
 		return ENCRYPT_CIPHER_MODE_NONE;
@@ -61,7 +67,78 @@ std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix,
                                         int64_t expAfterTS) {
 	// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
 	// parsing tools
+	std::string dName = domainName.toString();
+	// Underscores are invalid in trace event detail name.
+	boost::replace_all(dName, "_", "-");
 	boost::format fmter("%s.%lld.%s.%llu.%lld.%lld");
-	return boost::str(
-	    boost::format(fmter % prefix % domainId % domainName.toString() % baseCipherId % refAfterTS % expAfterTS));
+	return boost::str(boost::format(fmter % prefix % domainId % dName % baseCipherId % refAfterTS % expAfterTS));
 }
+
+int getEncryptHeaderAuthTokenSize(int algo) {
+	switch (algo) {
+	case ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA:
+		return 32;
+	case ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC:
+		return 16;
+	default:
+		throw not_implemented();
+	}
+}
+
+bool isEncryptHeaderAuthTokenAlgoValid(const EncryptAuthTokenAlgo algo) {
+	return algo >= EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE &&
+	       algo < EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_LAST;
+}
+
+bool isEncryptHeaderAuthTokenModeValid(const EncryptAuthTokenMode mode) {
+	return mode >= EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE &&
+	       mode < EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_LAST;
+}
+
+bool isEncryptHeaderAuthTokenDetailsValid(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) {
+	if (!isEncryptHeaderAuthTokenModeValid(mode) || !isEncryptHeaderAuthTokenAlgoValid(algo) ||
+	    (mode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE &&
+	     algo != EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE) ||
+	    (mode != EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE &&
+	     algo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE)) {
+		return false;
+	}
+	return true;
+}
+
+// Routine enables mapping EncryptHeader authTokenAlgo for a given authTokenMode; rules followed are:
+// 1. AUTH_TOKEN_NONE overrides authTokenAlgo configuration (as expected)
+// 2. AuthToken mode governed by the FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO
+EncryptAuthTokenAlgo getAuthTokenAlgoFromMode(const EncryptAuthTokenMode mode) {
+	EncryptAuthTokenAlgo algo;
+
+	if (mode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
+		// TOKEN_MODE_NONE overrides authTokenAlgo
+		algo = EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE;
+	} else {
+		algo = (EncryptAuthTokenAlgo)FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO;
+		// Ensure cluster authTokenAlgo sanity
+		if (algo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE) {
+			TraceEvent(SevWarn, "AuthTokenAlgoMisconfiguration").detail("Algo", algo).detail("Mode", mode);
+			throw not_implemented();
+		}
+	}
+	ASSERT(isEncryptHeaderAuthTokenDetailsValid(mode, algo));
+	return algo;
+}
+
+EncryptAuthTokenMode getRandomAuthTokenMode() {
+	std::vector<EncryptAuthTokenMode> modes = { EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE,
+		                                        EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE,
+		                                        EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI };
+	int idx = deterministicRandom()->randomInt(0, modes.size());
+	return modes[idx];
+}
+
+EncryptAuthTokenAlgo getRandomAuthTokenAlgo() {
+	EncryptAuthTokenAlgo algo = deterministicRandom()->coinflip()
+	                                ? EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC
+	                                : EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA;
+
+	return algo;
+}
\ No newline at end of file
diff --git a/flow/Error.cpp b/flow/Error.cpp
index 04302d98d6..6ec952286c 100644
--- a/flow/Error.cpp
+++ b/flow/Error.cpp
@@ -199,6 +199,20 @@ void breakpoint_me() {
 	return;
 }
 
+// FIXME: combine with bindings/c/fdb_c.cpp fdb_error_predicate function
+const std::set<int> transactionRetryableErrors = { error_code_not_committed,
+	                                               error_code_transaction_too_old,
+	                                               error_code_future_version,
+	                                               error_code_commit_proxy_memory_limit_exceeded,
+	                                               error_code_grv_proxy_memory_limit_exceeded,
+	                                               error_code_process_behind,
+	                                               error_code_batch_transaction_throttled,
+	                                               error_code_tag_throttled,
+	                                               error_code_unknown_tenant,
+	                                               // maybe committed error
+	                                               error_code_cluster_version_changed,
+	                                               error_code_commit_unknown_result };
+
 TEST_CASE("/flow/AssertTest") {
 	// this is mostly checking bug for bug compatibility with the C integer / sign promotion rules.
 
diff --git a/flow/FastAlloc.cpp b/flow/FastAlloc.cpp
index 5b4d80185f..7ceb539d6b 100644
--- a/flow/FastAlloc.cpp
+++ b/flow/FastAlloc.cpp
@@ -519,7 +519,12 @@ void FastAllocator<Size>::getMagazine() {
 		--g_allocation_tracing_disabled;
 	}
 #endif
-	block = (void**)::allocate(magazine_size * Size, /*allowLargePages*/ false, /*includeGuardPages*/ true);
+#ifdef VALGRIND
+	const bool includeGuardPages = false;
+#else
+	const bool includeGuardPages = true;
+#endif
+	block = (void**)::allocate(magazine_size * Size, /*allowLargePages*/ false, includeGuardPages);
 #endif
 
 	// void** block = new void*[ magazine_size * PSize ];
diff --git a/flow/Histogram.cpp b/flow/Histogram.cpp
index 63662f67f2..19a4946a23 100644
--- a/flow/Histogram.cpp
+++ b/flow/Histogram.cpp
@@ -206,8 +206,7 @@ std::string Histogram::drawHistogram() {
 
 TEST_CASE("/flow/histogram/smoke_test") {
 	{
-		Reference<Histogram> h =
-		    Histogram::getHistogram(LiteralStringRef("smoke_test"), LiteralStringRef("counts"), Histogram::Unit::bytes);
+		Reference<Histogram> h = Histogram::getHistogram("smoke_test"_sr, "counts"_sr, Histogram::Unit::bytes);
 
 		h->sample(0);
 		ASSERT(h->buckets[0] == 1);
@@ -222,15 +221,13 @@ TEST_CASE("/flow/histogram/smoke_test") {
 		ASSERT(h->buckets[0] == 0);
 		h->sample(0);
 		ASSERT(h->buckets[0] == 1);
-		h = Histogram::getHistogram(
-		    LiteralStringRef("smoke_test"), LiteralStringRef("counts2"), Histogram::Unit::bytes);
+		h = Histogram::getHistogram("smoke_test"_sr, "counts2"_sr, Histogram::Unit::bytes);
 
 		// confirm that old h was deallocated.
-		h = Histogram::getHistogram(LiteralStringRef("smoke_test"), LiteralStringRef("counts"), Histogram::Unit::bytes);
+		h = Histogram::getHistogram("smoke_test"_sr, "counts"_sr, Histogram::Unit::bytes);
 		ASSERT(h->buckets[0] == 0);
 
-		h = Histogram::getHistogram(
-		    LiteralStringRef("smoke_test"), LiteralStringRef("times"), Histogram::Unit::microseconds);
+		h = Histogram::getHistogram("smoke_test"_sr, "times"_sr, Histogram::Unit::microseconds);
 
 		h->sampleSeconds(0.000000);
 		h->sampleSeconds(0.0000019);
diff --git a/flow/Hostname.actor.cpp b/flow/Hostname.actor.cpp
index 110061e5c9..84a3cc7897 100644
--- a/flow/Hostname.actor.cpp
+++ b/flow/Hostname.actor.cpp
@@ -43,7 +43,7 @@ ACTOR Future<Optional<NetworkAddress>> resolveImpl(Hostname* self) {
 	try {
 		std::vector<NetworkAddress> addresses =
 		    wait(INetworkConnections::net()->resolveTCPEndpointWithDNSCache(self->host, self->service));
-		NetworkAddress address = addresses[deterministicRandom()->randomInt(0, addresses.size())];
+		NetworkAddress address = INetworkConnections::pickOneAddress(addresses);
 		address.flags = 0; // Reset the parsed address to public
 		address.fromHostname = NetworkAddressFromHostname::True;
 		if (self->isTLS) {
@@ -84,7 +84,7 @@ Optional<NetworkAddress> Hostname::resolveBlocking() {
 	try {
 		std::vector<NetworkAddress> addresses =
 		    INetworkConnections::net()->resolveTCPEndpointBlockingWithDNSCache(host, service);
-		NetworkAddress address = addresses[deterministicRandom()->randomInt(0, addresses.size())];
+		NetworkAddress address = INetworkConnections::pickOneAddress(addresses);
 		address.flags = 0; // Reset the parsed address to public
 		address.fromHostname = NetworkAddressFromHostname::True;
 		if (isTLS) {
diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
index 1d0923bd55..7bf080776b 100644
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -18,9 +18,13 @@
  * limitations under the License.
  */
 
+#include "flow/EncryptUtils.h"
+#include "flow/Error.h"
 #include "flow/flow.h"
 #include "flow/Knobs.h"
 #include "flow/BooleanParam.h"
+#include "flow/UnitTest.h"
+
 #include <cmath>
 #include <cinttypes>
 
@@ -127,6 +131,12 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( NETWORK_TEST_REQUEST_SIZE,                             1 );
 	init( NETWORK_TEST_SCRIPT_MODE,                          false );
 
+	//Authorization
+	init( ALLOW_TOKENLESS_TENANT_ACCESS,                     false );
+	init( PUBLIC_KEY_FILE_MAX_SIZE,                    1024 * 1024 );
+	init( PUBLIC_KEY_FILE_REFRESH_INTERVAL_SECONDS,             30 );
+	init( MAX_CACHED_EXPIRED_TOKENS,                          1024 );
+
 	//AsyncFileCached
 	init( PAGE_CACHE_4K,                                   2LL<<30 );
 	init( PAGE_CACHE_64K,                                200LL<<20 );
@@ -154,7 +164,9 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	//AsyncFileKAIO
 	init( MAX_OUTSTANDING,                                      64 );
 	init( MIN_SUBMIT,                                           10 );
-	init( DISK_METRIC_LOGGING_INTERVAL,                        5.0 );
+	init( SQLITE_DISK_METRIC_LOGGING_INTERVAL,                 5.0 );
+	init( KAIO_LATENCY_LOGGING_INTERVAL,                      30.0 );
+	init( KAIO_LATENCY_SAMPLE_SIZE,                          30000 );
 
 	init( PAGE_WRITE_CHECKSUM_HISTORY,                           0 ); if( randomize && BUGGIFY ) PAGE_WRITE_CHECKSUM_HISTORY = 10000000;
 	init( DISABLE_POSIX_KERNEL_AIO,                              0 );
@@ -168,6 +180,13 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( LOW_PRIORITY_DELAY_COUNT,                              5 );
 	init( LOW_PRIORITY_MAX_DELAY,                              5.0 );
 
+	// HTTP
+	init( HTTP_READ_SIZE,                                 128*1024 );
+	init( HTTP_SEND_SIZE,                                  32*1024 );
+	init( HTTP_VERBOSE_LEVEL,                                    0 );
+	init( HTTP_REQUEST_ID_HEADER,                               "" );
+	init( HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT, false );
+
 	//IAsyncFile
 	init( INCREMENTAL_DELETE_TRUNCATE_AMOUNT,                  5e8 ); //500MB
 	init( INCREMENTAL_DELETE_INTERVAL,                         1.0 ); //every 1 second
@@ -275,10 +294,17 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( HEALTH_MONITOR_CONNECTION_MAX_CLOSED,                  5 );
 
 	// Encryption
-	init( ENCRYPT_CIPHER_KEY_CACHE_TTL, isSimulated ? 120 : 10 * 60 );
-	if ( randomize && BUGGIFY) { ENCRYPT_CIPHER_KEY_CACHE_TTL = deterministicRandom()->randomInt(50, 100); }
+	init( ENCRYPT_CIPHER_KEY_CACHE_TTL, isSimulated ? 5 * 60 : 10 * 60 );
+	if ( randomize && BUGGIFY) { ENCRYPT_CIPHER_KEY_CACHE_TTL = deterministicRandom()->randomInt(2, 10) * 60; }
 	init( ENCRYPT_KEY_REFRESH_INTERVAL,   isSimulated ? 60 : 8 * 60 );
 	if ( randomize && BUGGIFY) { ENCRYPT_KEY_REFRESH_INTERVAL = deterministicRandom()->randomInt(2, 10); }
+	init( TOKEN_CACHE_SIZE,                                    100 );
+	init( ENCRYPT_KEY_CACHE_LOGGING_INTERVAL,                  5.0 );
+	init( ENCRYPT_KEY_CACHE_LOGGING_SAMPLE_SIZE,              1000 );
+	// Refer to EncryptUtil::EncryptAuthTokenAlgo for more details
+	init( ENCRYPT_HEADER_AUTH_TOKEN_ENABLED,                  true ); if ( randomize && BUGGIFY ) { ENCRYPT_HEADER_AUTH_TOKEN_ENABLED = !ENCRYPT_HEADER_AUTH_TOKEN_ENABLED; }
+	init( ENCRYPT_HEADER_AUTH_TOKEN_ALGO,                        1 ); if ( randomize && BUGGIFY ) { ENCRYPT_HEADER_AUTH_TOKEN_ALGO = getRandomAuthTokenAlgo(); }
+
 
 	// REST Client
 	init( RESTCLIENT_MAX_CONNECTIONPOOL_SIZE,                   10 );
@@ -300,22 +326,68 @@ static std::string toLower(std::string const& name) {
 	return lower_name;
 }
 
+// Converts the given string into a double. If any errors are
+// encountered, it throws an invalid_option_value exception.
+static double safe_stod(std::string const& str) {
+	size_t n;
+	double value = std::stod(str, &n);
+	if (n < str.size()) {
+		throw invalid_option_value();
+	}
+	return value;
+}
+
+// Converts the given (possibly hexadecimal) string into an
+// integer. If any errors are encountered, it throws an
+// invalid_option_value exception.
+static int safe_stoi(std::string const& str) {
+	size_t n;
+	int value = std::stoi(str, &n, 0);
+	if (n < str.size()) {
+		throw invalid_option_value();
+	}
+	return value;
+}
+
+// Converts the given (possibly hexadecimal) string into a 64-bit
+// integer. If any errors are encountered, it throws an
+// invalid_option_value exception.
+static int64_t safe_stoi64(std::string const& str) {
+	size_t n;
+	int64_t value = static_cast<int64_t>(std::stoll(str, &n, 0));
+	if (n < str.size()) {
+		throw invalid_option_value();
+	}
+	return value;
+}
+
+// Converts the given string into a bool. "true" and "false" are case
+// insenstively interpreted as true and false. Otherwise, any non-zero
+// integer is true. If any errors are encountered, it throws an
+// invalid_option_value exception.
+static bool safe_stob(std::string const& str) {
+	if (toLower(str) == "true") {
+		return true;
+	} else if (toLower(str) == "false") {
+		return false;
+	} else {
+		return safe_stoi(str) != 0;
+	}
+}
+
+// Parses a string value into the appropriate type based upon the knob
+// name. If any errors are encountered, it throws an
+// invalid_option_value exception.
 ParsedKnobValue Knobs::parseKnobValue(std::string const& knob, std::string const& value) const {
 	try {
 		if (double_knobs.count(knob)) {
-			return std::stod(value);
+			return safe_stod(value);
 		} else if (bool_knobs.count(knob)) {
-			if (toLower(value) == "true") {
-				return true;
-			} else if (toLower(value) == "false") {
-				return false;
-			} else {
-				return (std::stoi(value) != 0);
-			}
+			return safe_stob(value);
 		} else if (int64_knobs.count(knob)) {
-			return static_cast<int64_t>(std::stol(value, nullptr, 0));
+			return safe_stoi64(value);
 		} else if (int_knobs.count(knob)) {
-			return std::stoi(value, nullptr, 0);
+			return safe_stoi(value);
 		} else if (string_knobs.count(knob)) {
 			return value;
 		}
@@ -467,3 +539,23 @@ void Knobs::trace() const {
 		    .detail("Value", *k.second.value)
 		    .detail("Atomic", k.second.atomic);
 }
+
+TEST_CASE("/flow/Knobs/ParseKnobValue") {
+	// Test the safe conversion functions.
+	ASSERT_EQ(safe_stod("4.0"), 4.0);
+
+	ASSERT_EQ(safe_stoi("4"), 4);
+
+	ASSERT_EQ(safe_stoi64("4"), (int64_t)4);
+	try {
+		[[maybe_unused]] int64_t value = safe_stoi64("4GiB");
+		UNREACHABLE();
+	} catch (Error& e) {
+		ASSERT_EQ(e.code(), error_code_invalid_option_value);
+	}
+
+	ASSERT_EQ(safe_stob("true"), true);
+	ASSERT_EQ(safe_stob("false"), false);
+
+	return Void();
+}
diff --git a/flow/MkCert.cpp b/flow/MkCert.cpp
index f609eba54a..b9af4f90c3 100644
--- a/flow/MkCert.cpp
+++ b/flow/MkCert.cpp
@@ -166,13 +166,13 @@ PrivateKey makeEcP256() {
 	return PrivateKey(DerEncoded{}, StringRef(buf, len));
 }
 
-PrivateKey makeRsa2048Bit() {
+PrivateKey makeRsa4096Bit() {
 	auto kctx = AutoCPointer(::EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, nullptr), &::EVP_PKEY_CTX_free);
 	OSSL_ASSERT(kctx);
 	auto key = AutoCPointer(nullptr, &::EVP_PKEY_free);
 	auto keyRaw = std::add_pointer_t<EVP_PKEY>();
 	OSSL_ASSERT(0 < ::EVP_PKEY_keygen_init(kctx));
-	OSSL_ASSERT(0 < ::EVP_PKEY_CTX_set_rsa_keygen_bits(kctx, 2048));
+	OSSL_ASSERT(0 < ::EVP_PKEY_CTX_set_rsa_keygen_bits(kctx, 4096));
 	OSSL_ASSERT(0 < ::EVP_PKEY_keygen(kctx, &keyRaw));
 	OSSL_ASSERT(keyRaw);
 	key.reset(keyRaw);
diff --git a/flow/MkCertCli.cpp b/flow/MkCertCli.cpp
index 7517e2a203..c809bf8eb5 100644
--- a/flow/MkCertCli.cpp
+++ b/flow/MkCertCli.cpp
@@ -49,6 +49,7 @@ enum EMkCertOpt : int {
 	OPT_PRINT_SERVER_CERT,
 	OPT_PRINT_CLIENT_CERT,
 	OPT_PRINT_ARGUMENTS,
+	OPT_ENABLE_TRACE,
 };
 
 CSimpleOpt::SOption gOptions[] = { { OPT_HELP, "--help", SO_NONE },
@@ -68,6 +69,7 @@ CSimpleOpt::SOption gOptions[] = { { OPT_HELP, "--help", SO_NONE },
 	                               { OPT_PRINT_SERVER_CERT, "--print-server-cert", SO_NONE },
 	                               { OPT_PRINT_CLIENT_CERT, "--print-client-cert", SO_NONE },
 	                               { OPT_PRINT_ARGUMENTS, "--print-args", SO_NONE },
+	                               { OPT_ENABLE_TRACE, "--trace", SO_NONE },
 	                               SO_END_OF_OPTIONS };
 
 template <size_t Len>
@@ -191,6 +193,7 @@ int main(int argc, char** argv) {
 	auto printServerCert = false;
 	auto printClientCert = false;
 	auto printArguments = false;
+	auto enableTrace = false;
 	auto args = CSimpleOpt(argc, argv, gOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
 	while (args.Next()) {
 		if (auto err = args.LastError()) {
@@ -266,6 +269,8 @@ int main(int argc, char** argv) {
 			case OPT_PRINT_ARGUMENTS:
 				printArguments = true;
 				break;
+			case OPT_ENABLE_TRACE:
+				enableTrace = true;
 			default:
 				fmt::print(stderr, "ERROR: Unknown option {}\n", args.OptionText());
 				return FDB_EXIT_ERROR;
@@ -277,12 +282,14 @@ int main(int argc, char** argv) {
 		platformInit();
 		Error::init();
 		g_network = newNet2(TLSConfig());
-		openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, ".", "mkcert");
+		if (enableTrace)
+			openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, ".", "mkcert");
 		auto thread = std::thread([]() { g_network->run(); });
-		auto cleanUpGuard = ScopeExit([&thread]() {
+		auto cleanUpGuard = ScopeExit([&thread, enableTrace]() {
 			g_network->stop();
 			thread.join();
-			closeTraceFile();
+			if (enableTrace)
+				closeTraceFile();
 		});
 
 		serverArgs.transformPathToAbs();
diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp
index f3ec1a89a1..94072a0fda 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.actor.cpp
@@ -50,8 +50,11 @@
 #include "flow/ProtocolVersion.h"
 #include "flow/SendBufferIterator.h"
 #include "flow/TLSConfig.actor.h"
+#include "flow/WatchFile.actor.h"
 #include "flow/genericactors.actor.h"
 #include "flow/Util.h"
+#include "flow/UnitTest.h"
+#include "flow/ScopeExit.h"
 
 #ifdef ADDRESS_SANITIZER
 #include <sanitizer/lsan_interface.h>
@@ -87,6 +90,10 @@ void initProfiling() {
 	net2backtraces = new volatile void*[net2backtraces_max];
 	other_backtraces = new volatile void*[net2backtraces_max];
 
+	// According to folk wisdom, calling this once before setting up the signal handler makes
+	// it async signal safe in practice :-/
+	backtrace(const_cast<void**>(other_backtraces), net2backtraces_max);
+
 	sigemptyset(&sigprof_set);
 	sigaddset(&sigprof_set, SIGPROF);
 }
@@ -146,8 +153,8 @@ public:
 	void initMetrics() override;
 
 	// INetworkConnections interface
-	Future<Reference<IConnection>> connect(NetworkAddress toAddr, const std::string& host) override;
-	Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr, const std::string& host) override;
+	Future<Reference<IConnection>> connect(NetworkAddress toAddr, tcp::socket* existingSocket = nullptr) override;
+	Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr) override;
 	Future<Reference<IUDPSocket>> createUDPSocket(NetworkAddress toAddr) override;
 	Future<Reference<IUDPSocket>> createUDPSocket(bool isV6) override;
 	// The mock DNS methods should only be used in simulation.
@@ -214,7 +221,7 @@ public:
 		globals[id] = v;
 	}
 
-	ProtocolVersion protocolVersion() const override { return currentProtocolVersion; }
+	ProtocolVersion protocolVersion() const override { return currentProtocolVersion(); }
 
 	std::vector<flowGlobalType> globals;
 
@@ -236,6 +243,7 @@ public:
 	int sslHandshakerThreadsStarted;
 	int sslPoolHandshakesInProgress;
 	TLSConfig tlsConfig;
+	Reference<TLSPolicy> activeTlsPolicy;
 	Future<Void> backgroundCertRefresh;
 	ETLSInitState tlsInitializedState;
 
@@ -505,9 +513,11 @@ public:
 
 	NetworkAddress getPeerAddress() const override { return peer_address; }
 
+	bool hasTrustedPeer() const override { return true; }
+
 	UID getDebugID() const override { return id; }
 
-	tcp::socket& getSocket() { return socket; }
+	tcp::socket& getSocket() override { return socket; }
 
 private:
 	UID id;
@@ -837,12 +847,17 @@ public:
 	explicit SSLConnection(boost::asio::io_service& io_service,
 	                       Reference<ReferencedObject<boost::asio::ssl::context>> context)
 	  : id(nondeterministicRandom()->randomUniqueID()), socket(io_service), ssl_sock(socket, context->mutate()),
-	    sslContext(context) {}
+	    sslContext(context), has_trusted_peer(false) {}
+
+	explicit SSLConnection(Reference<ReferencedObject<boost::asio::ssl::context>> context, tcp::socket* existingSocket)
+	  : id(nondeterministicRandom()->randomUniqueID()), socket(std::move(*existingSocket)),
+	    ssl_sock(socket, context->mutate()), sslContext(context) {}
 
 	// This is not part of the IConnection interface, because it is wrapped by INetwork::connect()
 	ACTOR static Future<Reference<IConnection>> connect(boost::asio::io_service* ios,
 	                                                    Reference<ReferencedObject<boost::asio::ssl::context>> context,
-	                                                    NetworkAddress addr) {
+	                                                    NetworkAddress addr,
+	                                                    tcp::socket* existingSocket = nullptr) {
 		std::pair<IPAddress, uint16_t> peerIP = std::make_pair(addr.ip, addr.port);
 		auto iter(g_network->networkInfo.serverTLSConnectionThrottler.find(peerIP));
 		if (iter != g_network->networkInfo.serverTLSConnectionThrottler.end()) {
@@ -857,9 +872,15 @@ public:
 			}
 		}
 
+		if (existingSocket != nullptr) {
+			Reference<SSLConnection> self(new SSLConnection(context, existingSocket));
+			self->peer_address = addr;
+			self->init();
+			return self;
+		}
+
 		state Reference<SSLConnection> self(new SSLConnection(*ios, context));
 		self->peer_address = addr;
-
 		try {
 			auto to = tcpEndpoint(self->peer_address);
 			BindPromise p("N2_ConnectError", self->id);
@@ -869,7 +890,7 @@ public:
 			wait(onConnected);
 			self->init();
 			return self;
-		} catch (Error& e) {
+		} catch (Error&) {
 			// Either the connection failed, or was cancelled by the caller
 			self->closeSocket();
 			throw;
@@ -887,6 +908,9 @@ public:
 
 		try {
 			Future<Void> onHandshook;
+			ConfigureSSLStream(N2::g_net2->activeTlsPolicy, self->ssl_sock, [self = self](bool verifyOk) {
+				self->has_trusted_peer = verifyOk;
+			});
 
 			// If the background handshakers are not all busy, use one
 			if (N2::g_net2->sslPoolHandshakesInProgress < N2::g_net2->sslHandshakerThreadsStarted) {
@@ -936,8 +960,12 @@ public:
 		doAcceptHandshake(self, connected);
 		try {
 			choose {
-				when(wait(connected.getFuture())) { return Void(); }
-				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) { throw connection_failed(); }
+				when(wait(connected.getFuture())) {
+					return Void();
+				}
+				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
+					throw connection_failed();
+				}
 			}
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled) {
@@ -962,6 +990,10 @@ public:
 
 		try {
 			Future<Void> onHandshook;
+			ConfigureSSLStream(N2::g_net2->activeTlsPolicy, self->ssl_sock, [self = self](bool verifyOk) {
+				self->has_trusted_peer = verifyOk;
+			});
+
 			// If the background handshakers are not all busy, use one
 			if (N2::g_net2->sslPoolHandshakesInProgress < N2::g_net2->sslHandshakerThreadsStarted) {
 				holder = Hold(&N2::g_net2->sslPoolHandshakesInProgress);
@@ -992,8 +1024,12 @@ public:
 		doConnectHandshake(self, connected);
 		try {
 			choose {
-				when(wait(connected.getFuture())) { return Void(); }
-				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) { throw connection_failed(); }
+				when(wait(connected.getFuture())) {
+					return Void();
+				}
+				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
+					throw connection_failed();
+				}
 			}
 		} catch (Error& e) {
 			// Either the connection failed, or was cancelled by the caller
@@ -1095,9 +1131,11 @@ public:
 
 	NetworkAddress getPeerAddress() const override { return peer_address; }
 
+	bool hasTrustedPeer() const override { return has_trusted_peer; }
+
 	UID getDebugID() const override { return id; }
 
-	tcp::socket& getSocket() { return socket; }
+	tcp::socket& getSocket() override { return socket; }
 
 	ssl_socket& getSSLSocket() { return ssl_sock; }
 
@@ -1107,6 +1145,7 @@ private:
 	ssl_socket ssl_sock;
 	NetworkAddress peer_address;
 	Reference<ReferencedObject<boost::asio::ssl::context>> sslContext;
+	bool has_trusted_peer;
 
 	void init() {
 		// Socket settings that have to be set after connect or accept succeeds
@@ -1152,6 +1191,16 @@ public:
 	            NetworkAddress listenAddress)
 	  : io_service(io_service), listenAddress(listenAddress), acceptor(io_service, tcpEndpoint(listenAddress)),
 	    contextVar(contextVar) {
+		// when port 0 is passed in, a random port will be opened
+		// set listenAddress as the address with the actual port opened instead of port 0
+		if (listenAddress.port == 0) {
+			this->listenAddress = NetworkAddress::parse(acceptor.local_endpoint()
+			                                                .address()
+			                                                .to_string()
+			                                                .append(":")
+			                                                .append(std::to_string(acceptor.local_endpoint().port()))
+			                                                .append(listenAddress.isTLS() ? ":tls" : ""));
+		}
 		platform::setCloseOnExec(acceptor.native_handle());
 	}
 
@@ -1227,45 +1276,11 @@ Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics)
 	updateNow();
 }
 
-ACTOR static Future<Void> watchFileForChanges(std::string filename, AsyncTrigger* fileChanged) {
-	if (filename == "") {
-		return Never();
-	}
-	state bool firstRun = true;
-	state bool statError = false;
-	state std::time_t lastModTime = 0;
-	loop {
-		try {
-			std::time_t modtime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename));
-			if (firstRun) {
-				lastModTime = modtime;
-				firstRun = false;
-			}
-			if (lastModTime != modtime || statError) {
-				lastModTime = modtime;
-				statError = false;
-				fileChanged->trigger();
-			}
-		} catch (Error& e) {
-			if (e.code() == error_code_io_error) {
-				// EACCES, ELOOP, ENOENT all come out as io_error(), but are more of a system
-				// configuration issue than an FDB problem.  If we managed to load valid
-				// certificates, then there's no point in crashing, but we should complain
-				// loudly.  IAsyncFile will log the error, but not necessarily as a warning.
-				TraceEvent(SevWarnAlways, "TLSCertificateRefreshStatError").detail("File", filename);
-				statError = true;
-			} else {
-				throw;
-			}
-		}
-		wait(delay(FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS));
-	}
-}
-
 ACTOR static Future<Void> reloadCertificatesOnChange(
     TLSConfig config,
     std::function<void()> onPolicyFailure,
-    AsyncVar<Reference<ReferencedObject<boost::asio::ssl::context>>>* contextVar) {
+    AsyncVar<Reference<ReferencedObject<boost::asio::ssl::context>>>* contextVar,
+    Reference<TLSPolicy>* policy) {
 	if (FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS <= 0) {
 		return Void();
 	}
@@ -1279,9 +1294,13 @@ ACTOR static Future<Void> reloadCertificatesOnChange(
 	state int mismatches = 0;
 	state AsyncTrigger fileChanged;
 	state std::vector<Future<Void>> lifetimes;
-	lifetimes.push_back(watchFileForChanges(config.getCertificatePathSync(), &fileChanged));
-	lifetimes.push_back(watchFileForChanges(config.getKeyPathSync(), &fileChanged));
-	lifetimes.push_back(watchFileForChanges(config.getCAPathSync(), &fileChanged));
+	const int& intervalSeconds = FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS;
+	lifetimes.push_back(watchFileForChanges(
+	    config.getCertificatePathSync(), &fileChanged, &intervalSeconds, "TLSCertificateRefreshStatError"));
+	lifetimes.push_back(
+	    watchFileForChanges(config.getKeyPathSync(), &fileChanged, &intervalSeconds, "TLSKeyRefreshStatError"));
+	lifetimes.push_back(
+	    watchFileForChanges(config.getCAPathSync(), &fileChanged, &intervalSeconds, "TLSCARefreshStatError"));
 	loop {
 		wait(fileChanged.onTrigger());
 		TraceEvent("TLSCertificateRefreshBegin").log();
@@ -1289,7 +1308,8 @@ ACTOR static Future<Void> reloadCertificatesOnChange(
 		try {
 			LoadedTLSConfig loaded = wait(config.loadAsync());
 			boost::asio::ssl::context context(boost::asio::ssl::context::tls);
-			ConfigureSSLContext(loaded, &context, onPolicyFailure);
+			ConfigureSSLContext(loaded, context);
+			*policy = makeReference<TLSPolicy>(loaded, onPolicyFailure);
 			TraceEvent(SevInfo, "TLSCertificateRefreshSucceeded").log();
 			mismatches = 0;
 			contextVar->set(ReferencedObject<boost::asio::ssl::context>::from(std::move(context)));
@@ -1321,12 +1341,15 @@ void Net2::initTLS(ETLSInitState targetState) {
 			    .detail("KeyPath", tlsConfig.getKeyPathSync())
 			    .detail("HasPassword", !loaded.getPassword().empty())
 			    .detail("VerifyPeers", boost::algorithm::join(loaded.getVerifyPeers(), "|"));
-			ConfigureSSLContext(tlsConfig.loadSync(), &newContext, onPolicyFailure);
+			auto loadedTlsConfig = tlsConfig.loadSync();
+			ConfigureSSLContext(loadedTlsConfig, newContext);
+			activeTlsPolicy = makeReference<TLSPolicy>(loadedTlsConfig, onPolicyFailure);
 			sslContextVar.set(ReferencedObject<boost::asio::ssl::context>::from(std::move(newContext)));
 		} catch (Error& e) {
 			TraceEvent("Net2TLSInitError").error(e);
 		}
-		backgroundCertRefresh = reloadCertificatesOnChange(tlsConfig, onPolicyFailure, &sslContextVar);
+		backgroundCertRefresh =
+		    reloadCertificatesOnChange(tlsConfig, onPolicyFailure, &sslContextVar, &activeTlsPolicy);
 	}
 
 	// If a TLS connection is actually going to be used then start background threads if configured
@@ -1378,29 +1401,29 @@ ACTOR Future<Void> Net2::logTimeOffset() {
 }
 
 void Net2::initMetrics() {
-	bytesReceived.init(LiteralStringRef("Net2.BytesReceived"));
-	countWriteProbes.init(LiteralStringRef("Net2.CountWriteProbes"));
-	countReadProbes.init(LiteralStringRef("Net2.CountReadProbes"));
-	countReads.init(LiteralStringRef("Net2.CountReads"));
-	countWouldBlock.init(LiteralStringRef("Net2.CountWouldBlock"));
-	countWrites.init(LiteralStringRef("Net2.CountWrites"));
-	countRunLoop.init(LiteralStringRef("Net2.CountRunLoop"));
-	countCantSleep.init(LiteralStringRef("Net2.CountCantSleep"));
-	countWontSleep.init(LiteralStringRef("Net2.CountWontSleep"));
-	countTimers.init(LiteralStringRef("Net2.CountTimers"));
-	countTasks.init(LiteralStringRef("Net2.CountTasks"));
-	countYields.init(LiteralStringRef("Net2.CountYields"));
-	countYieldBigStack.init(LiteralStringRef("Net2.CountYieldBigStack"));
-	countYieldCalls.init(LiteralStringRef("Net2.CountYieldCalls"));
-	countASIOEvents.init(LiteralStringRef("Net2.CountASIOEvents"));
-	countYieldCallsTrue.init(LiteralStringRef("Net2.CountYieldCallsTrue"));
-	countRunLoopProfilingSignals.init(LiteralStringRef("Net2.CountRunLoopProfilingSignals"));
-	countTLSPolicyFailures.init(LiteralStringRef("Net2.CountTLSPolicyFailures"));
-	priorityMetric.init(LiteralStringRef("Net2.Priority"));
-	awakeMetric.init(LiteralStringRef("Net2.Awake"));
-	slowTaskMetric.init(LiteralStringRef("Net2.SlowTask"));
-	countLaunchTime.init(LiteralStringRef("Net2.CountLaunchTime"));
-	countReactTime.init(LiteralStringRef("Net2.CountReactTime"));
+	bytesReceived.init("Net2.BytesReceived"_sr);
+	countWriteProbes.init("Net2.CountWriteProbes"_sr);
+	countReadProbes.init("Net2.CountReadProbes"_sr);
+	countReads.init("Net2.CountReads"_sr);
+	countWouldBlock.init("Net2.CountWouldBlock"_sr);
+	countWrites.init("Net2.CountWrites"_sr);
+	countRunLoop.init("Net2.CountRunLoop"_sr);
+	countCantSleep.init("Net2.CountCantSleep"_sr);
+	countWontSleep.init("Net2.CountWontSleep"_sr);
+	countTimers.init("Net2.CountTimers"_sr);
+	countTasks.init("Net2.CountTasks"_sr);
+	countYields.init("Net2.CountYields"_sr);
+	countYieldBigStack.init("Net2.CountYieldBigStack"_sr);
+	countYieldCalls.init("Net2.CountYieldCalls"_sr);
+	countASIOEvents.init("Net2.CountASIOEvents"_sr);
+	countYieldCallsTrue.init("Net2.CountYieldCallsTrue"_sr);
+	countRunLoopProfilingSignals.init("Net2.CountRunLoopProfilingSignals"_sr);
+	countTLSPolicyFailures.init("Net2.CountTLSPolicyFailures"_sr);
+	priorityMetric.init("Net2.Priority"_sr);
+	awakeMetric.init("Net2.Awake"_sr);
+	slowTaskMetric.init("Net2.SlowTask"_sr);
+	countLaunchTime.init("Net2.CountLaunchTime"_sr);
+	countReactTime.init("Net2.CountReactTime"_sr);
 }
 
 bool Net2::checkRunnable() {
@@ -1818,17 +1841,17 @@ THREAD_HANDLE Net2::startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg, in
 	return ::startThread(func, arg, stackSize, name);
 }
 
-Future<Reference<IConnection>> Net2::connect(NetworkAddress toAddr, const std::string& host) {
+Future<Reference<IConnection>> Net2::connect(NetworkAddress toAddr, tcp::socket* existingSocket) {
 	if (toAddr.isTLS()) {
 		initTLS(ETLSInitState::CONNECT);
-		return SSLConnection::connect(&this->reactor.ios, this->sslContextVar.get(), toAddr);
+		return SSLConnection::connect(&this->reactor.ios, this->sslContextVar.get(), toAddr, existingSocket);
 	}
 
 	return Connection::connect(&this->reactor.ios, toAddr);
 }
 
-Future<Reference<IConnection>> Net2::connectExternal(NetworkAddress toAddr, const std::string& host) {
-	return connect(toAddr, host);
+Future<Reference<IConnection>> Net2::connectExternal(NetworkAddress toAddr) {
+	return connect(toAddr);
 }
 
 Future<Reference<IUDPSocket>> Net2::createUDPSocket(NetworkAddress toAddr) {
@@ -2115,7 +2138,7 @@ struct TestGVR {
 };
 
 template <class F>
-void startThreadF(F&& func) {
+THREAD_HANDLE startThreadF(F&& func) {
 	struct Thing {
 		F f;
 		Thing(F&& f) : f(std::move(f)) {}
@@ -2127,71 +2150,153 @@ void startThreadF(F&& func) {
 		}
 	};
 	Thing* t = new Thing(std::move(func));
-	startThread(Thing::start, t);
+	return g_network->startThread(Thing::start, t);
+}
+
+TEST_CASE("flow/Net2/ThreadSafeQueue/Interface") {
+	ThreadSafeQueue<int> tq;
+	ASSERT(!tq.pop().present());
+	ASSERT(tq.canSleep());
+
+	ASSERT(tq.push(1) == true);
+	ASSERT(!tq.canSleep());
+	ASSERT(!tq.canSleep());
+	ASSERT(tq.push(2) == false);
+	ASSERT(tq.push(3) == false);
+
+	ASSERT(tq.pop().get() == 1);
+	ASSERT(tq.pop().get() == 2);
+	ASSERT(tq.push(4) == false);
+	ASSERT(tq.pop().get() == 3);
+	ASSERT(tq.pop().get() == 4);
+	ASSERT(!tq.pop().present());
+	ASSERT(tq.canSleep());
+	return Void();
+}
+
+// A helper struct used by queueing tests which use multiple threads.
+struct QueueTestThreadState {
+	QueueTestThreadState(int threadId, int toProduce) : threadId(threadId), toProduce(toProduce) {}
+	int threadId;
+	THREAD_HANDLE handle;
+	int toProduce;
+	int produced = 0;
+	Promise<Void> doneProducing;
+	int consumed = 0;
+
+	static int valueToThreadId(int value) { return value >> 20; }
+	int elementValue(int index) { return index + (threadId << 20); }
+	int nextProduced() { return elementValue(produced++); }
+	int nextConsumed() { return elementValue(consumed++); }
+	void checkDone() {
+		ASSERT_EQ(produced, toProduce);
+		ASSERT_EQ(consumed, produced);
+	}
+};
+
+TEST_CASE("flow/Net2/ThreadSafeQueue/Threaded") {
+	// Uses ThreadSafeQueue from multiple threads. Verifies that all pushed elements are popped, maintaining the
+	// ordering within a thread.
+	noUnseed = true; // multi-threading inherently non-deterministic
+
+	ThreadSafeQueue<int> queue;
+	state std::vector<QueueTestThreadState> perThread = { QueueTestThreadState(0, 1000000),
+		                                                  QueueTestThreadState(1, 100000),
+		                                                  QueueTestThreadState(2, 1000000) };
+	state std::vector<Future<Void>> doneProducing;
+
+	int total = 0;
+	for (int t = 0; t < perThread.size(); ++t) {
+		auto& s = perThread[t];
+		doneProducing.push_back(s.doneProducing.getFuture());
+		total += s.toProduce;
+		s.handle = startThreadF([&queue, &s]() {
+			printf("Thread%d\n", s.threadId);
+			int nextYield = 0;
+			while (s.produced < s.toProduce) {
+				queue.push(s.nextProduced());
+				if (nextYield-- == 0) {
+					std::this_thread::yield();
+					nextYield = nondeterministicRandom()->randomInt(0, 100);
+				}
+			}
+			printf("T%dDone\n", s.threadId);
+			s.doneProducing.send(Void());
+		});
+	}
+	int consumed = 0;
+	while (consumed < total) {
+		Optional<int> element = queue.pop();
+		if (element.present()) {
+			int v = element.get();
+			auto& s = perThread[QueueTestThreadState::valueToThreadId(v)];
+			++consumed;
+			ASSERT(v == s.nextConsumed());
+		} else {
+			std::this_thread::yield();
+		}
+		if ((consumed & 3) == 0)
+			queue.canSleep();
+	}
+
+	wait(waitForAll(doneProducing));
+
+	// Make sure we continue on the main thread.
+	Promise<Void> signal;
+	state Future<Void> doneConsuming = signal.getFuture();
+	g_network->onMainThread(std::move(signal), TaskPriority::DefaultOnMainThread);
+	wait(doneConsuming);
+
+	for (int t = 0; t < perThread.size(); ++t) {
+		waitThread(perThread[t].handle);
+		perThread[t].checkDone();
+	}
+	return Void();
+}
+
+// NB: This could be a test for any INetwork implementation, but Sim2 doesn't
+// satisfy this requirement yet.
+TEST_CASE("noSim/flow/Net2/onMainThreadFIFO") {
+	// Verifies that signals processed by onMainThread() are executed in order.
+	noUnseed = true; // multi-threading inherently non-deterministic
+
+	state std::vector<QueueTestThreadState> perThread = { QueueTestThreadState(0, 1000000),
+		                                                  QueueTestThreadState(1, 100000),
+		                                                  QueueTestThreadState(2, 1000000) };
+	state std::vector<Future<Void>> doneProducing;
+	for (int t = 0; t < perThread.size(); ++t) {
+		auto& s = perThread[t];
+		doneProducing.push_back(s.doneProducing.getFuture());
+		s.handle = startThreadF([&s]() {
+			int nextYield = 0;
+			while (s.produced < s.toProduce) {
+				if (nextYield-- == 0) {
+					std::this_thread::yield();
+					nextYield = nondeterministicRandom()->randomInt(0, 100);
+				}
+				int v = s.nextProduced();
+				onMainThreadVoid([&s, v]() { ASSERT_EQ(v, s.nextConsumed()); });
+			}
+			s.doneProducing.send(Void());
+		});
+	}
+	wait(waitForAll(doneProducing));
+
+	// Wait for one more onMainThread to wait for all scheduled signals to be executed.
+	Promise<Void> signal;
+	state Future<Void> doneConsuming = signal.getFuture();
+	g_network->onMainThread(std::move(signal), TaskPriority::DefaultOnMainThread);
+	wait(doneConsuming);
+
+	for (int t = 0; t < perThread.size(); ++t) {
+		waitThread(perThread[t].handle);
+		perThread[t].checkDone();
+	}
+	return Void();
 }
 
 void net2_test(){
-	/*printf("ThreadSafeQueue test\n");
-	printf("  Interface: ");
-	ThreadSafeQueue<int> tq;
-	ASSERT( tq.canSleep() == true );
-
-	ASSERT( tq.push( 1 ) == true ) ;
-	ASSERT( tq.push( 2 ) == false );
-	ASSERT( tq.push( 3 ) == false );
-
-	ASSERT( tq.pop().get() == 1 );
-	ASSERT( tq.pop().get() == 2 );
-	ASSERT( tq.push( 4 ) == false );
-	ASSERT( tq.pop().get() == 3 );
-	ASSERT( tq.pop().get() == 4 );
-	ASSERT( !tq.pop().present() );
-	printf("OK\n");
-
-	printf("Threaded: ");
-	Event finished, finished2;
-	int thread1Iterations = 1000000, thread2Iterations = 100000;
-
-	if (thread1Iterations)
-	    startThreadF([&](){
-	        printf("Thread1\n");
-	        for(int i=0; i<thread1Iterations; i++)
-	            tq.push(i);
-	        printf("T1Done\n");
-	        finished.set();
-	    });
-	if (thread2Iterations)
-	    startThreadF([&](){
-	        printf("Thread2\n");
-	        for(int i=0; i<thread2Iterations; i++)
-	            tq.push(i + (1<<20));
-	        printf("T2Done\n");
-	        finished2.set();
-	    });
-	int c = 0, mx[2]={0, 1<<20}, p = 0;
-	while (c < thread1Iterations + thread2Iterations)
-	{
-	    Optional<int> i = tq.pop();
-	    if (i.present()) {
-	        int v = i.get();
-	        ++c;
-	        if (mx[v>>20] != v)
-	            printf("Wrong value dequeued!\n");
-	        ASSERT( mx[v>>20] == v );
-	        mx[v>>20] = v + 1;
-	    } else {
-	        ++p;
-	        _mm_pause();
-	    }
-	    if ((c&3)==0) tq.canSleep();
-	}
-	printf("%d %d %x %x %s\n", c, p, mx[0], mx[1], mx[0]==thread1Iterations && mx[1]==(1<<20)+thread2Iterations ? "OK" :
-	"FAIL");
-
-	finished.block();
-	finished2.block();
-
-
+	/*
 	g_network = newNet2();  // for promise serialization below
 
 	Endpoint destination;
@@ -2213,7 +2318,7 @@ void net2_test(){
 	    reqs.resize(10000);
 	    for(int i=0; i<10000; i++) {
 	        TestGVR &req = reqs[i];
-	        req.key = LiteralStringRef("Foobar");
+	        req.key = "Foobar"_sr;
 
 	        SerializeSource<TestGVR> what(req);
 
diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index 01ee9b2303..f9fb6e258a 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -21,49 +21,58 @@
 #ifdef _WIN32
 // This has to come as the first include on Win32 for rand_s() to be found
 #define _CRT_RAND_S
-#include <stdlib.h>
-#include <math.h> // For _set_FMA3_enable workaround in platformInit
-#endif
+#endif // _WIN32
 
-#include <errno.h>
-#include "fmt/format.h"
 #include "flow/Platform.h"
-#include "flow/Platform.actor.h"
-#include "flow/Arena.h"
-
-#include "flow/StreamCipher.h"
-#include "flow/BlobCipher.h"
-#include "flow/Trace.h"
-#include "flow/Error.h"
-
-#include "flow/Knobs.h"
 
+#include <algorithm>
 #include <iostream>
 #include <fstream>
 #include <sstream>
 #include <cstring>
-#include <algorithm>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+
 #include <boost/format.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/operations.hpp>
+#include <boost/algorithm/string.hpp>
 
-#include <sys/types.h>
-#include <time.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include "flow/UnitTest.h"
+#include "fmt/format.h"
+
+#include "flow/Arena.h"
+#include "flow/Error.h"
 #include "flow/FaultInjection.h"
+#include "flow/Knobs.h"
+#include "flow/Platform.actor.h"
+#include "flow/ScopeExit.h"
+#include "flow/StreamCipher.h"
+#include "flow/Trace.h"
+#include "flow/Trace.h"
+#include "flow/UnitTest.h"
+#include "flow/Util.h"
 
 #ifdef _WIN32
-#include <windows.h>
-#include <winioctl.h>
-#include <io.h>
-#include <psapi.h>
-#include <stdio.h>
+
 #include <conio.h>
 #include <direct.h>
+#include <io.h>
+#include <math.h> // For _set_FMA3_enable workaround in platformInit
 #include <pdh.h>
 #include <pdhmsg.h>
+#include <processenv.h>
+#include <psapi.h>
+#include <stdlib.h>
+#include <windows.h>
+#include <winioctl.h>
+
 #pragma comment(lib, "pdh.lib")
 
 // for SHGetFolderPath
@@ -78,19 +87,18 @@
 #define CANONICAL_PATH_SEPARATOR '/'
 
 #include <dirent.h>
-#include <sys/time.h>
-#include <sys/mman.h>
-#include <unistd.h>
 #include <ftw.h>
 #include <pwd.h>
 #include <sched.h>
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <sys/statvfs.h> /* Needed for disk capacity */
+
 #if !defined(__aarch64__) && !defined(__powerpc64__)
 #include <cpuid.h>
 #endif
 
-/* Needed for disk capacity */
-#include <sys/statvfs.h>
-
 /* getifaddrs */
 #include <sys/socket.h>
 #include <ifaddrs.h>
@@ -111,7 +119,7 @@
 #include <signal.h>
 /* Needed for gnu_dev_{major,minor} */
 #include <sys/sysmacros.h>
-#endif
+#endif // __linux__
 
 #ifdef __FreeBSD__
 /* Needed for processor affinity */
@@ -144,29 +152,31 @@
 #include <devstat.h>
 #include <kvm.h>
 #include <libutil.h>
-#endif
+#endif // __FreeBSD__
 
 #ifdef __APPLE__
-#include <sys/uio.h>
-#include <sys/syslimits.h>
-#include <mach/mach.h>
+/* Needed for cross-platform 'environ' */
+#include <crt_externs.h>
 #include <mach-o/dyld.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/sysctl.h>
-#include <netinet/in.h>
-#include <net/if.h>
+#include <mach/mach.h>
 #include <net/if_dl.h>
+#include <net/if.h>
 #include <net/route.h>
+#include <netinet/in.h>
+#include <sys/mount.h>
+#include <sys/param.h>
+#include <sys/sysctl.h>
+#include <sys/syslimits.h>
+#include <sys/uio.h>
 
 #include <CoreFoundation/CoreFoundation.h>
 #include <IOKit/IOKitLib.h>
 #include <IOKit/storage/IOBlockStorageDriver.h>
 #include <IOKit/storage/IOMedia.h>
 #include <IOKit/IOBSD.h>
-#endif
+#endif // __APPLE_
 
-#endif
+#endif // __unixish__
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -400,25 +410,24 @@ uint64_t getMemoryUsage() {
 #endif
 }
 
-#if defined(__linux__)
+#ifdef __linux__
+namespace linux_os {
+
+namespace {
+
 void getMemoryInfo(std::map<StringRef, int64_t>& request, std::stringstream& memInfoStream) {
 	size_t count = request.size();
 	if (count == 0)
 		return;
 
-	while (count > 0 && !memInfoStream.eof()) {
-		std::string key;
-
-		memInfoStream >> key;
+	keyValueReader<std::string, int64_t>(memInfoStream, [&](const std::string& key, const int64_t& value) -> bool {
 		auto item = request.find(StringRef(key));
-		if (item != request.end()) {
-			int64_t value;
-			memInfoStream >> value;
+		if (item != std::end(request)) {
 			item->second = value;
-			count--;
+			--count;
 		}
-		memInfoStream.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
-	}
+		return count != 0;
+	});
 }
 
 int64_t getLowWatermark(std::stringstream& zoneInfoStream) {
@@ -438,10 +447,8 @@ int64_t getLowWatermark(std::stringstream& zoneInfoStream) {
 
 	return lowWatermark;
 }
-#endif
 
-void getMachineRAMInfo(MachineRAMInfo& memInfo) {
-#if defined(__linux__)
+void getMachineRAMInfoImpl(MachineRAMInfo& memInfo) {
 	std::ifstream zoneInfoFileStream("/proc/zoneinfo", std::ifstream::in);
 	int64_t lowWatermark = 0;
 	if (!zoneInfoFileStream.good()) {
@@ -459,24 +466,22 @@ void getMachineRAMInfo(MachineRAMInfo& memInfo) {
 	}
 
 	std::map<StringRef, int64_t> request = {
-		{ LiteralStringRef("MemTotal:"), 0 },       { LiteralStringRef("MemFree:"), 0 },
-		{ LiteralStringRef("MemAvailable:"), -1 },  { LiteralStringRef("Active(file):"), 0 },
-		{ LiteralStringRef("Inactive(file):"), 0 }, { LiteralStringRef("SwapTotal:"), 0 },
-		{ LiteralStringRef("SwapFree:"), 0 },       { LiteralStringRef("SReclaimable:"), 0 },
+		{ "MemTotal:"_sr, 0 },       { "MemFree:"_sr, 0 },   { "MemAvailable:"_sr, -1 }, { "Active(file):"_sr, 0 },
+		{ "Inactive(file):"_sr, 0 }, { "SwapTotal:"_sr, 0 }, { "SwapFree:"_sr, 0 },      { "SReclaimable:"_sr, 0 },
 	};
 
 	std::stringstream memInfoStream;
 	memInfoStream << fileStream.rdbuf();
 	getMemoryInfo(request, memInfoStream);
 
-	int64_t memFree = request[LiteralStringRef("MemFree:")];
-	int64_t pageCache = request[LiteralStringRef("Active(file):")] + request[LiteralStringRef("Inactive(file):")];
-	int64_t slabReclaimable = request[LiteralStringRef("SReclaimable:")];
-	int64_t usedSwap = request[LiteralStringRef("SwapTotal:")] - request[LiteralStringRef("SwapFree:")];
+	int64_t memFree = request["MemFree:"_sr];
+	int64_t pageCache = request["Active(file):"_sr] + request["Inactive(file):"_sr];
+	int64_t slabReclaimable = request["SReclaimable:"_sr];
+	int64_t usedSwap = request["SwapTotal:"_sr] - request["SwapFree:"_sr];
 
-	memInfo.total = 1024 * request[LiteralStringRef("MemTotal:")];
-	if (request[LiteralStringRef("MemAvailable:")] != -1) {
-		memInfo.available = 1024 * (request[LiteralStringRef("MemAvailable:")] - usedSwap);
+	memInfo.total = 1024 * request["MemTotal:"_sr];
+	if (request["MemAvailable:"_sr] != -1) {
+		memInfo.available = 1024 * (request["MemAvailable:"_sr] - usedSwap);
 	} else {
 		memInfo.available =
 		    1024 * (std::max<int64_t>(0,
@@ -486,6 +491,35 @@ void getMachineRAMInfo(MachineRAMInfo& memInfo) {
 	}
 
 	memInfo.committed = memInfo.total - memInfo.available;
+}
+
+} // anonymous namespace
+
+std::map<std::string, int64_t> reportCGroupCpuStat() {
+	// Default path to the cpu,cpuacct
+	// See manpages for cgroup
+	static const std::string PATH_TO_CPU_CPUACCT = "/sys/fs/cgroup/cpu,cpuacct/cpu.stat";
+
+	std::map<std::string, int64_t> result;
+	std::ifstream ifs(PATH_TO_CPU_CPUACCT);
+	if (!ifs.is_open()) {
+		return result;
+	}
+
+	keyValueReader<std::string, int64_t>(ifs, [&](const std::string& key, const int64_t& value) -> bool {
+		result[key] = value;
+		return true;
+	});
+
+	return result;
+}
+
+} // namespace linux_os
+#endif // #ifdef __linux__
+
+void getMachineRAMInfo(MachineRAMInfo& memInfo) {
+#if defined(__linux__)
+	linux_os::getMachineRAMInfoImpl(memInfo);
 #elif defined(__FreeBSD__)
 	int status;
 
@@ -1933,6 +1967,39 @@ std::string epochsToGMTString(double epochs) {
 	return timeString;
 }
 
+std::vector<std::string> getEnvironmentKnobOptions() {
+	constexpr const size_t ENVKNOB_PREFIX_LEN = sizeof(ENVIRONMENT_KNOB_OPTION_PREFIX) - 1;
+	std::vector<std::string> knobOptions;
+#if defined(_WIN32)
+	auto e = GetEnvironmentStrings();
+	if (e == nullptr)
+		return {};
+	auto cleanup = ScopeExit([e]() { FreeEnvironmentStrings(e); });
+	while (*e) {
+		auto candidate = std::string_view(e);
+		if (boost::starts_with(candidate, ENVIRONMENT_KNOB_OPTION_PREFIX))
+			knobOptions.emplace_back(candidate.substr(ENVKNOB_PREFIX_LEN));
+		e += (candidate.size() + 1);
+	}
+#else
+	char** e = nullptr;
+#ifdef __linux__
+	e = environ;
+#elif defined(__APPLE__)
+	e = *_NSGetEnviron();
+#else
+#error Port me!
+#endif
+	for (; e && *e; e++) {
+		std::string_view envOption(*e);
+		if (boost::starts_with(envOption, ENVIRONMENT_KNOB_OPTION_PREFIX)) {
+			knobOptions.emplace_back(envOption.substr(ENVKNOB_PREFIX_LEN));
+		}
+	}
+#endif
+	return knobOptions;
+}
+
 void setMemoryQuota(size_t limit) {
 	if (limit == 0) {
 		return;
@@ -2451,7 +2518,7 @@ bool createDirectory(std::string const& directory) {
 
 const uint8_t separatorChar = CANONICAL_PATH_SEPARATOR;
 StringRef separator(&separatorChar, 1);
-StringRef dotdot = LiteralStringRef("..");
+StringRef dotdot = ".."_sr;
 
 std::string cleanPath(std::string const& path) {
 	std::vector<StringRef> finalParts;
@@ -3115,19 +3182,19 @@ void outOfMemory() {
 		char* demangled = abi::__cxa_demangle(i->first, nullptr, nullptr, nullptr);
 		if (demangled) {
 			s = demangled;
-			if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::")))
-				s = s.substr(LiteralStringRef("(anonymous namespace)::").size());
+			if (StringRef(s).startsWith("(anonymous namespace)::"_sr))
+				s = s.substr("(anonymous namespace)::"_sr.size());
 			free(demangled);
 		} else
 			s = i->first;
 #else
 		s = i->first;
-		if (StringRef(s).startsWith(LiteralStringRef("class `anonymous namespace'::")))
-			s = s.substr(LiteralStringRef("class `anonymous namespace'::").size());
-		else if (StringRef(s).startsWith(LiteralStringRef("class ")))
-			s = s.substr(LiteralStringRef("class ").size());
-		else if (StringRef(s).startsWith(LiteralStringRef("struct ")))
-			s = s.substr(LiteralStringRef("struct ").size());
+		if (StringRef(s).startsWith("class `anonymous namespace'::"_sr))
+			s = s.substr("class `anonymous namespace'::"_sr.size());
+		else if (StringRef(s).startsWith("class "_sr))
+			s = s.substr("class "_sr.size());
+		else if (StringRef(s).startsWith("struct "_sr))
+			s = s.substr("struct "_sr.size());
 #endif
 		typeNames.emplace_back(s, i->first);
 	}
@@ -3207,10 +3274,10 @@ void outOfMemory() {
 }
 
 // Because the lambda used with nftw below cannot capture
-int __eraseDirectoryRecurseiveCount;
+int __eraseDirectoryRecursiveCount;
 
 int eraseDirectoryRecursive(std::string const& dir) {
-	__eraseDirectoryRecurseiveCount = 0;
+	__eraseDirectoryRecursiveCount = 0;
 #ifdef _WIN32
 	system(("rd /s /q \"" + dir + "\"").c_str());
 #elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)
@@ -3219,7 +3286,7 @@ int eraseDirectoryRecursive(std::string const& dir) {
 	    [](const char* fpath, const struct stat* sb, int typeflag, struct FTW* ftwbuf) -> int {
 		    int r = remove(fpath);
 		    if (r == 0)
-			    ++__eraseDirectoryRecurseiveCount;
+			    ++__eraseDirectoryRecursiveCount;
 		    return r;
 	    },
 	    64,
@@ -3236,7 +3303,7 @@ int eraseDirectoryRecursive(std::string const& dir) {
 #error Port me!
 #endif
 	// INJECT_FAULT( platform_error, "eraseDirectoryRecursive" );
-	return __eraseDirectoryRecurseiveCount;
+	return __eraseDirectoryRecursiveCount;
 }
 
 TmpFile::TmpFile() : filename("") {
@@ -3464,7 +3531,12 @@ void* loadLibrary(const char* lib_path) {
 	void* dlobj = nullptr;
 
 #if defined(__unixish__)
-	dlobj = dlopen(lib_path, RTLD_LAZY | RTLD_LOCAL);
+	dlobj = dlopen(lib_path,
+	               RTLD_LAZY | RTLD_LOCAL
+#ifdef USE_SANITIZER // Keep alive dlopen()-ed libs for symbolized XSAN backtrace
+	                   | RTLD_NODELETE
+#endif
+	);
 	if (dlobj == nullptr) {
 		TraceEvent(SevWarn, "LoadLibraryFailed").detail("Library", lib_path).detail("Error", dlerror());
 	}
@@ -3576,6 +3648,12 @@ void platformInit() {
 #endif
 }
 
+std::vector<std::function<void()>> g_crashHandlerCallbacks;
+
+void registerCrashHandlerCallback(void (*f)()) {
+	g_crashHandlerCallbacks.push_back(f);
+}
+
 // The crashHandler function is registered to handle signals before the process terminates.
 // Basic information about the crash is printed/traced, and stdout and trace events are flushed.
 void crashHandler(int sig) {
@@ -3584,11 +3662,19 @@ void crashHandler(int sig) {
 	//  but the idea is that we're about to crash anyway...
 	std::string backtrace = platform::get_backtrace();
 
-	bool error = (sig != SIGUSR2);
+	bool error = (sig != SIGUSR2 && sig != SIGTERM);
 
 	StreamCipherKey::cleanup();
 	StreamCipher::cleanup();
-	BlobCipherKeyCache::cleanup();
+
+	for (auto& f : g_crashHandlerCallbacks) {
+		f();
+	}
+
+	fprintf(error ? stderr : stdout, "SIGNAL: %s (%d)\n", strsignal(sig), sig);
+	if (error) {
+		fprintf(stderr, "Trace: %s\n", backtrace.c_str());
+	}
 
 	fflush(stdout);
 	{
@@ -3600,8 +3686,9 @@ void crashHandler(int sig) {
 	}
 	flushTraceFileVoid();
 
-	fprintf(stderr, "SIGNAL: %s (%d)\n", strsignal(sig), sig);
-	fprintf(stderr, "Trace: %s\n", backtrace.c_str());
+#ifdef USE_GCOV
+	__gcov_flush();
+#endif
 
 	struct sigaction sa;
 	sa.sa_handler = SIG_DFL;
@@ -3641,6 +3728,8 @@ void registerCrashHandler() {
 	sigaction(SIGSEGV, &action, nullptr);
 	sigaction(SIGBUS, &action, nullptr);
 	sigaction(SIGUSR2, &action, nullptr);
+	sigaction(SIGTERM, &action, nullptr);
+	sigaction(SIGABRT, &action, nullptr);
 #else
 	// No crash handler for other platforms!
 #endif
@@ -3711,7 +3800,7 @@ void profileHandler(int sig) {
 	ps->timestamp = checkThreadTime.is_lock_free() ? checkThreadTime.load() : 0;
 
 	// SOMEDAY: should we limit the maximum number of frames from backtrace beyond just available space?
-	size_t size = platform::raw_backtrace(ps->frames, net2backtraces_max - net2backtraces_offset - 2);
+	size_t size = backtrace(ps->frames, net2backtraces_max - net2backtraces_offset - 2);
 
 	ps->length = size;
 
@@ -3936,21 +4025,19 @@ TEST_CASE("/flow/Platform/getMemoryInfo") {
 	                        "VmallocUsed:      410576 kB\n";
 
 	std::map<StringRef, int64_t> request = {
-		{ LiteralStringRef("MemTotal:"), 0 },     { LiteralStringRef("MemFree:"), 0 },
-		{ LiteralStringRef("MemAvailable:"), 0 }, { LiteralStringRef("Buffers:"), 0 },
-		{ LiteralStringRef("Cached:"), 0 },       { LiteralStringRef("SwapTotal:"), 0 },
-		{ LiteralStringRef("SwapFree:"), 0 },
+		{ "MemTotal:"_sr, 0 }, { "MemFree:"_sr, 0 },   { "MemAvailable:"_sr, 0 }, { "Buffers:"_sr, 0 },
+		{ "Cached:"_sr, 0 },   { "SwapTotal:"_sr, 0 }, { "SwapFree:"_sr, 0 },
 	};
 
 	std::stringstream memInfoStream(memString);
-	getMemoryInfo(request, memInfoStream);
-	ASSERT(request[LiteralStringRef("MemTotal:")] == 24733228);
-	ASSERT(request[LiteralStringRef("MemFree:")] == 2077580);
-	ASSERT(request[LiteralStringRef("MemAvailable:")] == 0);
-	ASSERT(request[LiteralStringRef("Buffers:")] == 266940);
-	ASSERT(request[LiteralStringRef("Cached:")] == 16798292);
-	ASSERT(request[LiteralStringRef("SwapTotal:")] == 25165820);
-	ASSERT(request[LiteralStringRef("SwapFree:")] == 23680228);
+	linux_os::getMemoryInfo(request, memInfoStream);
+	ASSERT(request["MemTotal:"_sr] == 24733228);
+	ASSERT(request["MemFree:"_sr] == 2077580);
+	ASSERT(request["MemAvailable:"_sr] == 0);
+	ASSERT(request["Buffers:"_sr] == 266940);
+	ASSERT(request["Cached:"_sr] == 16798292);
+	ASSERT(request["SwapTotal:"_sr] == 25165820);
+	ASSERT(request["SwapFree:"_sr] == 23680228);
 	for (auto& item : request) {
 		fmt::print("{}:{}\n", item.first.toString().c_str(), item.second);
 	}
@@ -3994,14 +4081,14 @@ TEST_CASE("/flow/Platform/getMemoryInfo") {
 	                         "AnonHugePages:   1275904 kB\n";
 
 	std::stringstream memInfoStream1(memString1);
-	getMemoryInfo(request, memInfoStream1);
-	ASSERT(request[LiteralStringRef("MemTotal:")] == 31856496);
-	ASSERT(request[LiteralStringRef("MemFree:")] == 25492716);
-	ASSERT(request[LiteralStringRef("MemAvailable:")] == 28470756);
-	ASSERT(request[LiteralStringRef("Buffers:")] == 313644);
-	ASSERT(request[LiteralStringRef("Cached:")] == 2956444);
-	ASSERT(request[LiteralStringRef("SwapTotal:")] == 0);
-	ASSERT(request[LiteralStringRef("SwapFree:")] == 0);
+	linux_os::getMemoryInfo(request, memInfoStream1);
+	ASSERT(request["MemTotal:"_sr] == 31856496);
+	ASSERT(request["MemFree:"_sr] == 25492716);
+	ASSERT(request["MemAvailable:"_sr] == 28470756);
+	ASSERT(request["Buffers:"_sr] == 313644);
+	ASSERT(request["Cached:"_sr] == 2956444);
+	ASSERT(request["SwapTotal:"_sr] == 0);
+	ASSERT(request["SwapFree:"_sr] == 0);
 	for (auto& item : request) {
 		fmt::print("{}:{}\n", item.first.toString().c_str(), item.second);
 	}
diff --git a/flow/ProtocolVersion.cpp b/flow/ProtocolVersion.cpp
new file mode 100644
index 0000000000..b70e7b78ec
--- /dev/null
+++ b/flow/ProtocolVersion.cpp
@@ -0,0 +1,37 @@
+/*
+ * ProtocolVersion.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "flow/ProtocolVersion.h"
+
+namespace {
+
+ProtocolVersion g_currentProtocolVersion(defaultProtocolVersionValue);
+
+}
+
+ProtocolVersion currentProtocolVersion() {
+	static ProtocolVersion firstReturnedProtocolVersion = g_currentProtocolVersion;
+	// Make sure the protocol version is not changed, once it is already in use
+	ASSERT(firstReturnedProtocolVersion == g_currentProtocolVersion);
+	return g_currentProtocolVersion;
+}
+
+void useFutureProtocolVersion() {
+	g_currentProtocolVersion = ProtocolVersion(futureProtocolVersionValue);
+}
\ No newline at end of file
diff --git a/flow/include/flow/ProtocolVersion.h b/flow/ProtocolVersion.h.cmake
similarity index 54%
rename from flow/include/flow/ProtocolVersion.h
rename to flow/ProtocolVersion.h.cmake
index 323dcffcf0..e6642b2fa9 100644
--- a/flow/include/flow/ProtocolVersion.h
+++ b/flow/ProtocolVersion.h.cmake
@@ -23,26 +23,23 @@
 #include <cstdint>
 
 // This version impacts both communications and the deserialization of certain database and IKeyValueStore keys.
-//
-// The convention is that 'x' and 'y' should match the major and minor version of the software, and 'z' should be 0.
-// To make a change without a corresponding increase to the x.y version, increment the 'dev' digit.
-//
-// The last 2 bytes (4 digits) of the protocol version do not affect compatibility. These two bytes are not currently
-// used and should not be changed from 0.
-//                                                         xyzdev
-//                                                         vvvv
-constexpr uint64_t currentProtocolVersionValue = 0x0FDB00B072000000LL;
+constexpr uint64_t defaultProtocolVersionValue = @FDB_PV_DEFAULT_VERSION@;
 
 // The first protocol version that cannot be downgraded from. Ordinarily, this will be two release versions larger
 // than the current version, meaning that we only support downgrades between consecutive release versions.
-constexpr uint64_t minInvalidProtocolVersionValue = 0x0FDB00B074000000LL;
+constexpr uint64_t minInvalidProtocolVersionValue = @FDB_PV_MIN_INVALID_VERSION@;
 
 // The lowest protocol version that can be downgraded to.
-constexpr uint64_t minCompatibleProtocolVersionValue = 0x0FDB00B071000000LL;
+constexpr uint64_t minCompatibleProtocolVersionValue = @FDB_PV_MIN_COMPATIBLE_VERSION@;
 
+// The protocol version that will most likely follow the current one
+// Used only for testing upgrades to the future version
+constexpr uint64_t futureProtocolVersionValue = @FDB_PV_FUTURE_VERSION@;
+
+// The first check second expression version doesn't need to change because it's just for earlier protocol versions.
 #define PROTOCOL_VERSION_FEATURE(v, x)                                                                                 \
-	static_assert((v & 0xF0FFFFLL) == 0 || v < 0x0FDB00B071000000LL, "Unexpected feature protocol version");           \
-	static_assert(v <= currentProtocolVersionValue, "Feature protocol version too large");                             \
+	static_assert((v & @FDB_PV_LSB_MASK@) == 0 || v < 0x0FDB00B071000000LL, "Unexpected feature protocol version");             \
+	static_assert(v <= defaultProtocolVersionValue, "Feature protocol version too large");                             \
 	struct x {                                                                                                         \
 		static constexpr uint64_t protocolVersion = v;                                                                 \
 	};                                                                                                                 \
@@ -107,73 +104,76 @@ public: // introduced features
 	// We stopped using the dev version consistently in the past.
 	// To ensure binaries work across patch releases (e.g., 6.2.0 to 6.2.22), we require that the protocol version be
 	// the same for each of them.
-	PROTOCOL_VERSION_FEATURE(0x0FDB00A200090000LL, Watches);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00A2000D0000LL, MovableCoordinatedState);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00A340000000LL, ProcessID);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00A400040000LL, OpenDatabase);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00A446020000LL, Locality);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00A460010000LL, MultiGenerationTLog);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00A460010000LL, SharedMutations);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00A551000000LL, InexpensiveMultiVersionClient);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00A560010000LL, TagLocality);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B060000000LL, Fearless);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B061020000LL, EndpointAddrList);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B061030000LL, IPv6);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B061030000LL, TLogVersion);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B061070000LL, PseudoLocalities);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B061070000LL, ShardedTxsTags);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, TLogQueueEntryRef);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, GenerationRegVal);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, MovableCoordinatedStateV2);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, KeyServerValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, LogsValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ServerTagValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, TagLocalityListValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, DatacenterReplicasValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ProcessClassValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, WorkerListValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, BackupStartValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, LogRangeEncodeValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, HealthyZoneValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, DRBackupRanges);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, RegionConfiguration);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ReplicationPolicy);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, BackupMutations);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ClusterControllerPriorityInfo);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ProcessIDFile);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, CloseUnusedConnection);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, DBCoreState);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, TagThrottleValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, StorageCacheValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreStatusValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreRequestValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreRequestDoneVersionValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreRequestTriggerValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreWorkerInterfaceValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, BackupProgressValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, KeyServerValueV2);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063000000LL, UnifiedTLogSpilling);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, BackupWorker);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, ReportConflictingKeys);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, SmallEndpoints);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, CacheRole);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B070010000LL, StableInterfaces);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, ServerListValue);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, TagThrottleValueReason);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, SpanContext);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, TSS);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, ChangeFeed);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, BlobGranule);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, NetworkAddressHostnameFlag);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageMetadata);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, PerpetualWiggleMetadata);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, Tenants);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageInterfaceReadiness);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, ResolverPrivateMutations);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B072000000LL, OTELSpanContext);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B072000000LL, SWVersionTracking);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B072000000LL, EncryptionAtRest);
-	PROTOCOL_VERSION_FEATURE(0x0FDB00B072000000LL, ShardEncodeLocationMetaData);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_WATCHES@, Watches);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_MOVABLE_COORDINATED_STATE@, MovableCoordinatedState);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_PROCESS_ID@, ProcessID);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_OPEN_DATABASE@, OpenDatabase);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_LOCALITY@, Locality);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_MULTIGENERATION_TLOG@, MultiGenerationTLog);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_SHARED_MUTATIONS@, SharedMutations);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_INEXPENSIVE_MULTIVERSION_CLIENT@, InexpensiveMultiVersionClient);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_TAG_LOCALITY@, TagLocality);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_FEARLESS@, Fearless);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_ENDPOINT_ADDR_LIST@, EndpointAddrList);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_IPV6@, IPv6);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_TLOG_VERSION@, TLogVersion);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_PSEUDO_LOCALITIES@, PseudoLocalities);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_SHARDED_TXS_TAGS@, ShardedTxsTags);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_TLOG_QUEUE_ENTRY_REF@, TLogQueueEntryRef);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_GENERATION_REG_VAL@, GenerationRegVal);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_MOVABLE_COORDINATED_STATE_V2@, MovableCoordinatedStateV2);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_KEY_SERVER_VALUE@, KeyServerValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_LOGS_VALUE@, LogsValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_SERVER_TAG_VALUE@, ServerTagValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_TAG_LOCALITY_LIST_VALUE@, TagLocalityListValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_DATACENTER_REPLICAS_VALUE@, DatacenterReplicasValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_PROCESS_CLASS_VALUE@, ProcessClassValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_WORKER_LIST_VALUE@, WorkerListValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_BACKUP_START_VALUE@, BackupStartValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_LOG_RANGE_ENCODE_VALUE@, LogRangeEncodeValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_HEALTHY_ZONE_VALUE@, HealthyZoneValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_DR_BACKUP_RANGES@, DRBackupRanges);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_REGION_CONFIGURATION@, RegionConfiguration);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_REPLICATION_POLICY@, ReplicationPolicy);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_BACKUP_MUTATIONS@, BackupMutations);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_CLUSTER_CONTROLLER_PRIORITY_INFO@, ClusterControllerPriorityInfo);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_PROCESS_ID_FILE@, ProcessIDFile);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_CLOSE_UNUSED_CONNECTION@, CloseUnusedConnection);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_DB_CORE_STATE@, DBCoreState);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_TAG_THROTTLE_VALUE@, TagThrottleValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_STORAGE_CACHE_VALUE@, StorageCacheValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_RESTORE_STATUS_VALUE@, RestoreStatusValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_RESTORE_REQUEST_VALUE@, RestoreRequestValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_RESTORE_REQUEST_DONE_VERSION_VALUE@, RestoreRequestDoneVersionValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_RESTORE_REQUEST_TRIGGER_VALUE@, RestoreRequestTriggerValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_RESTORE_WORKER_INTERFACE_VALUE@, RestoreWorkerInterfaceValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_BACKUP_PROGRESS_VALUE@, BackupProgressValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_KEY_SERVER_VALUE_V2@, KeyServerValueV2);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_UNIFIED_TLOG_SPILLING@, UnifiedTLogSpilling);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_BACKUP_WORKER@, BackupWorker);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_REPORT_CONFLICTING_KEYS@, ReportConflictingKeys);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_SMALL_ENDPOINTS@, SmallEndpoints);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_CACHE_ROLE@, CacheRole);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_STABLE_INTERFACES@, StableInterfaces);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_SERVER_LIST_VALUE@, ServerListValue);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_TAG_THROTTLE_VALUE_REASON@, TagThrottleValueReason);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_SPAN_CONTEXT@, SpanContext);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_TSS@, TSS);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_CHANGE_FEED@, ChangeFeed);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_BLOB_GRANULE@, BlobGranule);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_NETWORK_ADDRESS_HOSTNAME_FLAG@, NetworkAddressHostnameFlag);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_STORAGE_METADATA@, StorageMetadata);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_PERPETUAL_WIGGLE_METADATA@, PerpetualWiggleMetadata);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_STORAGE_INTERFACE_READINESS@, StorageInterfaceReadiness);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_RESOLVER_PRIVATE_MUTATIONS@, ResolverPrivateMutations);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_OTEL_SPAN_CONTEXT@, OTELSpanContext);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_SW_VERSION_TRACKING@, SWVersionTracking);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_ENCRYPTION_AT_REST@, EncryptionAtRest);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_SHARD_ENCODE_LOCATION_METADATA@, ShardEncodeLocationMetaData);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_TENANTS@, Tenants);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_BLOB_GRANULE_FILE@, BlobGranuleFile);
+	PROTOCOL_VERSION_FEATURE(@FDB_ENCRYPTED_SNAPSHOT_BACKUP_FILE@, EncryptedSnapshotBackupFile);
+	PROTOCOL_VERSION_FEATURE(@FDB_PV_CLUSTER_ID_SPECIAL_KEY@, ClusterIdSpecialKey);
 };
 
 template <>
@@ -183,23 +183,31 @@ struct Traceable<ProtocolVersion> : std::true_type {
 	}
 };
 
-constexpr ProtocolVersion currentProtocolVersion(currentProtocolVersionValue);
+constexpr ProtocolVersion defaultProtocolVersion(defaultProtocolVersionValue);
 constexpr ProtocolVersion minInvalidProtocolVersion(minInvalidProtocolVersionValue);
 constexpr ProtocolVersion minCompatibleProtocolVersion(minCompatibleProtocolVersionValue);
 
+// The protocol version of the process, normally it is equivalent defaultProtocolVersion
+// The currentProtocolVersion can be overridden dynamically for testing upgrades
+// to a future FDB version
+ProtocolVersion currentProtocolVersion();
+
+// Assume the next future protocol version as the current one. Used for testing purposes only
+void useFutureProtocolVersion();
+
 // This assert is intended to help prevent incrementing the leftmost digits accidentally. It will probably need to
 // change when we reach version 10.
-static_assert(currentProtocolVersion.version() < 0x0FDB00B100000000LL, "Unexpected protocol version");
+static_assert(defaultProtocolVersion.version() < @FDB_PV_LEFT_MOST_CHECK@, "Unexpected protocol version");
 
 // The last two bytes of the protocol version are currently masked out in compatibility checks. We do not use them,
 // so prevent them from being inadvertently changed.
 //
 // We also do not modify the protocol version for patch releases, so prevent modifying the patch version digit.
-static_assert((currentProtocolVersion.version() & 0xF0FFFFLL) == 0, "Unexpected protocol version");
+static_assert((defaultProtocolVersion.version() & @FDB_PV_LSB_MASK@) == 0, "Unexpected protocol version");
 
 // Downgrades must support at least one minor version.
 static_assert(minInvalidProtocolVersion.version() >=
-                  (currentProtocolVersion.version() & 0xFFFFFFFFFF000000LL) + 0x0000000002000000,
+                  (defaultProtocolVersion.version() & 0xFFFFFFFFFF000000LL) + 0x0000000002000000,
               "Downgrades must support one minor version");
 
 // The min invalid protocol version should be the smallest possible protocol version associated with a minor release
diff --git a/flow/ProtocolVersions.cmake b/flow/ProtocolVersions.cmake
new file mode 100644
index 0000000000..463466d26e
--- /dev/null
+++ b/flow/ProtocolVersions.cmake
@@ -0,0 +1,92 @@
+# Protocol Versions.
+# This version impacts both communications and the deserialization of certain database and IKeyValueStore keys.
+#
+# The convention is that 'x' and 'y' should match the major and minor version of the software, and 'z' should be 0.
+# To make a change without a corresponding increase to the x.y version, increment the 'dev' digit.
+#
+# The last 2 bytes (4 digits) of the protocol version do not affect compatibility. These two bytes are not currently
+# used and should not be changed from 0.
+#                                                          xyzdev
+#                                                          vvvv
+set(FDB_PV_DEFAULT_VERSION                      "0x0FDB00B072000000LL")
+set(FDB_PV_FUTURE_VERSION                       "0x0FDB00B073000000LL")
+set(FDB_PV_MIN_COMPATIBLE_VERSION               "0x0FDB00B071000000LL")
+set(FDB_PV_MIN_INVALID_VERSION                  "0x0FDB00B074000000LL")
+set(FDB_PV_LEFT_MOST_CHECK                      "0x0FDB00B100000000LL")
+set(FDB_PV_LSB_MASK                             "0xF0FFFFLL")
+
+# The 5th digit from right is dev version, for example, 2 in 0x0FDB00B061020000LL;
+# It was used to identify a protocol change (e.g., interface change) between major/minor versions (say 5.1 and 5.2)
+# We stopped using the dev version consistently in the past.
+# To ensure binaries work across patch releases (e.g., 6.2.0 to 6.2.22), we require that the protocol version be
+# the same for each of them.
+set(FDB_PV_WATCHES                              "0x0FDB00A200090000LL")
+set(FDB_PV_MOVABLE_COORDINATED_STATE            "0x0FDB00A2000D0000LL")
+set(FDB_PV_PROCESS_ID                           "0x0FDB00A340000000LL")
+set(FDB_PV_OPEN_DATABASE                        "0x0FDB00A400040000LL")
+set(FDB_PV_LOCALITY                             "0x0FDB00A446020000LL")
+set(FDB_PV_MULTIGENERATION_TLOG                 "0x0FDB00A460010000LL")
+set(FDB_PV_SHARED_MUTATIONS                     "0x0FDB00A460010000LL")
+set(FDB_PV_INEXPENSIVE_MULTIVERSION_CLIENT      "0x0FDB00A551000000LL")
+set(FDB_PV_TAG_LOCALITY                         "0x0FDB00A560010000LL")
+set(FDB_PV_FEARLESS                             "0x0FDB00B060000000LL")
+set(FDB_PV_ENDPOINT_ADDR_LIST                   "0x0FDB00B061020000LL")
+set(FDB_PV_IPV6                                 "0x0FDB00B061030000LL")
+set(FDB_PV_TLOG_VERSION                         "0x0FDB00B061030000LL")
+set(FDB_PV_PSEUDO_LOCALITIES                    "0x0FDB00B061070000LL")
+set(FDB_PV_SHARDED_TXS_TAGS                     "0x0FDB00B061070000LL")
+set(FDB_PV_TLOG_QUEUE_ENTRY_REF                 "0x0FDB00B062010001LL")
+set(FDB_PV_GENERATION_REG_VAL                   "0x0FDB00B062010001LL")
+set(FDB_PV_MOVABLE_COORDINATED_STATE_V2         "0x0FDB00B062010001LL")
+set(FDB_PV_KEY_SERVER_VALUE                     "0x0FDB00B062010001LL")
+set(FDB_PV_LOGS_VALUE                           "0x0FDB00B062010001LL")
+set(FDB_PV_SERVER_TAG_VALUE                     "0x0FDB00B062010001LL")
+set(FDB_PV_TAG_LOCALITY_LIST_VALUE              "0x0FDB00B062010001LL")
+set(FDB_PV_DATACENTER_REPLICAS_VALUE            "0x0FDB00B062010001LL")
+set(FDB_PV_PROCESS_CLASS_VALUE                  "0x0FDB00B062010001LL")
+set(FDB_PV_WORKER_LIST_VALUE                    "0x0FDB00B062010001LL")
+set(FDB_PV_BACKUP_START_VALUE                   "0x0FDB00B062010001LL")
+set(FDB_PV_LOG_RANGE_ENCODE_VALUE               "0x0FDB00B062010001LL")
+set(FDB_PV_HEALTHY_ZONE_VALUE                   "0x0FDB00B062010001LL")
+set(FDB_PV_DR_BACKUP_RANGES                     "0x0FDB00B062010001LL")
+set(FDB_PV_REGION_CONFIGURATION                 "0x0FDB00B062010001LL")
+set(FDB_PV_REPLICATION_POLICY                   "0x0FDB00B062010001LL")
+set(FDB_PV_BACKUP_MUTATIONS                     "0x0FDB00B062010001LL")
+set(FDB_PV_CLUSTER_CONTROLLER_PRIORITY_INFO     "0x0FDB00B062010001LL")
+set(FDB_PV_PROCESS_ID_FILE                      "0x0FDB00B062010001LL")
+set(FDB_PV_CLOSE_UNUSED_CONNECTION              "0x0FDB00B062010001LL")
+set(FDB_PV_DB_CORE_STATE                        "0x0FDB00B063010000LL")
+set(FDB_PV_TAG_THROTTLE_VALUE                   "0x0FDB00B063010000LL")
+set(FDB_PV_STORAGE_CACHE_VALUE                  "0x0FDB00B063010000LL")
+set(FDB_PV_RESTORE_STATUS_VALUE                 "0x0FDB00B063010000LL")
+set(FDB_PV_RESTORE_REQUEST_VALUE                "0x0FDB00B063010000LL")
+set(FDB_PV_RESTORE_REQUEST_DONE_VERSION_VALUE   "0x0FDB00B063010000LL")
+set(FDB_PV_RESTORE_REQUEST_TRIGGER_VALUE        "0x0FDB00B063010000LL")
+set(FDB_PV_RESTORE_WORKER_INTERFACE_VALUE       "0x0FDB00B063010000LL")
+set(FDB_PV_BACKUP_PROGRESS_VALUE                "0x0FDB00B063010000LL")
+set(FDB_PV_KEY_SERVER_VALUE_V2                  "0x0FDB00B063010000LL")
+set(FDB_PV_UNIFIED_TLOG_SPILLING                "0x0FDB00B063000000LL")
+set(FDB_PV_BACKUP_WORKER                        "0x0FDB00B063010000LL")
+set(FDB_PV_REPORT_CONFLICTING_KEYS              "0x0FDB00B063010000LL")
+set(FDB_PV_SMALL_ENDPOINTS                      "0x0FDB00B063010000LL")
+set(FDB_PV_CACHE_ROLE                           "0x0FDB00B063010000LL")
+set(FDB_PV_STABLE_INTERFACES                    "0x0FDB00B070010000LL")
+set(FDB_PV_SERVER_LIST_VALUE                    "0x0FDB00B070010001LL")
+set(FDB_PV_TAG_THROTTLE_VALUE_REASON            "0x0FDB00B070010001LL")
+set(FDB_PV_SPAN_CONTEXT                         "0x0FDB00B070010001LL")
+set(FDB_PV_TSS                                  "0x0FDB00B070010001LL")
+set(FDB_PV_CHANGE_FEED                          "0x0FDB00B071010000LL")
+set(FDB_PV_BLOB_GRANULE                         "0x0FDB00B071010000LL")
+set(FDB_PV_NETWORK_ADDRESS_HOSTNAME_FLAG        "0x0FDB00B071010000LL")
+set(FDB_PV_STORAGE_METADATA                     "0x0FDB00B071010000LL")
+set(FDB_PV_PERPETUAL_WIGGLE_METADATA            "0x0FDB00B071010000LL")
+set(FDB_PV_STORAGE_INTERFACE_READINESS          "0x0FDB00B071010000LL")
+set(FDB_PV_TENANTS                              "0x0FDB00B071010000LL")
+set(FDB_PV_RESOLVER_PRIVATE_MUTATIONS           "0x0FDB00B072000000LL")
+set(FDB_PV_OTEL_SPAN_CONTEXT                    "0x0FDB00B072000000LL")
+set(FDB_PV_SW_VERSION_TRACKING                  "0x0FDB00B072000000LL")
+set(FDB_PV_ENCRYPTION_AT_REST                   "0x0FDB00B072000000LL")
+set(FDB_PV_SHARD_ENCODE_LOCATION_METADATA       "0x0FDB00B072000000LL")
+set(FDB_PV_BLOB_GRANULE_FILE                    "0x0FDB00B072000000LL")
+set(FDB_ENCRYPTED_SNAPSHOT_BACKUP_FILE          "0x0FDB00B072000000LL")
+set(FDB_PV_CLUSTER_ID_SPECIAL_KEY               "0x0FDB00B072000000LL")
diff --git a/flow/README.md b/flow/README.md
index a69ef6220d..142fe68413 100644
--- a/flow/README.md
+++ b/flow/README.md
@@ -538,7 +538,6 @@ convenience types for their `Standalone` versions (not a complete list):
 |        T         | Standalone<T> alias |
 |:----------------:|:-------------------:|
 | StringRef        | N/A                 |
-| LiteralStringRef | N/A                 |
 | KeyRef           | Key                 |
 | ValueRef         | Value               |
 | KeyValueRef      | KeyValue            |
diff --git a/flow/SignalSafeUnwind.cpp b/flow/SignalSafeUnwind.cpp
new file mode 100644
index 0000000000..a43dd81277
--- /dev/null
+++ b/flow/SignalSafeUnwind.cpp
@@ -0,0 +1,55 @@
+/*
+ * SignalSafeUnwind.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flow/SignalSafeUnwind.h"
+
+int64_t dl_iterate_phdr_calls = 0;
+
+#if defined(__linux__) && !defined(USE_SANITIZER)
+
+#include <link.h>
+#include <mutex>
+
+static int (*chain_dl_iterate_phdr)(int (*callback)(struct dl_phdr_info* info, size_t size, void* data),
+                                    void* data) = nullptr;
+
+static void initChain() {
+	static std::once_flag flag;
+
+	// Ensure that chain_dl_iterate_phdr points to the "real" function that we are overriding
+	std::call_once(flag, []() { *(void**)&chain_dl_iterate_phdr = dlsym(RTLD_NEXT, "dl_iterate_phdr"); });
+
+	if (!chain_dl_iterate_phdr) {
+		criticalError(FDB_EXIT_ERROR, "SignalSafeUnwindError", "Unable to find dl_iterate_phdr symbol");
+	}
+}
+
+// This overrides the function in libc!
+extern "C" int dl_iterate_phdr(int (*callback)(struct dl_phdr_info* info, size_t size, void* data), void* data) {
+	interlockedIncrement64(&dl_iterate_phdr_calls);
+
+	initChain();
+
+	setProfilingEnabled(0);
+	int result = chain_dl_iterate_phdr(callback, data);
+	setProfilingEnabled(1);
+	return result;
+}
+#endif
\ No newline at end of file
diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp
index cf5f8d7b2f..8cd4f95c55 100644
--- a/flow/StreamCipher.cpp
+++ b/flow/StreamCipher.cpp
@@ -113,16 +113,6 @@ void StreamCipher::cleanup() noexcept {
 	}
 }
 
-void applyHmacKeyDerivationFunc(StreamCipherKey* cipherKey, HmacSha256StreamCipher* hmacGenerator, Arena& arena) {
-	uint8_t buf[cipherKey->size() + sizeof(uint64_t)];
-	memcpy(&buf[0], cipherKey->data(), cipherKey->size());
-	uint64_t seed = deterministicRandom()->randomUInt64();
-	memcpy(&buf[0] + cipherKey->size(), &seed, sizeof(uint64_t));
-	StringRef digest = hmacGenerator->digest(&buf[0], cipherKey->size() + sizeof(uint64_t), arena);
-	std::copy(digest.begin(), digest.end(), &buf[0]);
-	cipherKey->initializeKey(&buf[0], cipherKey->size());
-}
-
 EncryptionStreamCipher::EncryptionStreamCipher(const StreamCipherKey* key, const StreamCipher::IV& iv)
   : cipher(StreamCipher(key->size())) {
 	EVP_EncryptInit_ex(cipher.getCtx(), EVP_aes_256_gcm(), nullptr, nullptr, nullptr);
@@ -131,7 +121,7 @@ EncryptionStreamCipher::EncryptionStreamCipher(const StreamCipherKey* key, const
 }
 
 StringRef EncryptionStreamCipher::encrypt(unsigned char const* plaintext, int len, Arena& arena) {
-	TEST(true); // Encrypting data with StreamCipher
+	CODE_PROBE(true, "Encrypting data with StreamCipher");
 	auto ciphertext = new (arena) unsigned char[len + AES_BLOCK_SIZE];
 	int bytes{ 0 };
 	EVP_EncryptUpdate(cipher.getCtx(), ciphertext, &bytes, plaintext, len);
@@ -153,7 +143,7 @@ DecryptionStreamCipher::DecryptionStreamCipher(const StreamCipherKey* key, const
 }
 
 StringRef DecryptionStreamCipher::decrypt(unsigned char const* ciphertext, int len, Arena& arena) {
-	TEST(true); // Decrypting data with StreamCipher
+	CODE_PROBE(true, "Decrypting data with StreamCipher");
 	auto plaintext = new (arena) unsigned char[len];
 	int bytesDecrypted{ 0 };
 	EVP_DecryptUpdate(cipher.getCtx(), plaintext, &bytesDecrypted, ciphertext, len);
@@ -173,15 +163,6 @@ HmacSha256StreamCipher::HmacSha256StreamCipher() : cipher(EVP_MAX_KEY_LENGTH) {
 	HMAC_Init_ex(cipher.getHmacCtx(), NULL, 0, EVP_sha256(), nullptr);
 }
 
-StringRef HmacSha256StreamCipher::digest(unsigned char const* data, int len, Arena& arena) {
-	TEST(true); // Digest using StreamCipher
-	unsigned int digestLen = HMAC_size(cipher.getHmacCtx());
-	auto digest = new (arena) unsigned char[digestLen];
-	HMAC_Update(cipher.getHmacCtx(), data, len);
-	HMAC_Final(cipher.getHmacCtx(), digest, &digestLen);
-	return StringRef(digest, digestLen);
-}
-
 StringRef HmacSha256StreamCipher::finish(Arena& arena) {
 	unsigned int digestLen = HMAC_size(cipher.getHmacCtx());
 	auto digest = new (arena) unsigned char[digestLen];
@@ -199,11 +180,11 @@ TEST_CASE("flow/StreamCipher") {
 	StreamCipherKey const* key = StreamCipherKey::getGlobalCipherKey();
 
 	StreamCipher::IV iv;
-	generateRandomData(iv.data(), iv.size());
+	deterministicRandom()->randomBytes(iv.data(), iv.size());
 
 	Arena arena;
 	std::vector<unsigned char> plaintext(deterministicRandom()->randomInt(0, 10001));
-	generateRandomData(&plaintext.front(), plaintext.size());
+	deterministicRandom()->randomBytes(&plaintext.front(), plaintext.size());
 	std::vector<unsigned char> ciphertext(plaintext.size() + AES_BLOCK_SIZE);
 	std::vector<unsigned char> decryptedtext(plaintext.size() + AES_BLOCK_SIZE);
 
diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp
index 6ff7decbd6..5a75b37f7f 100644
--- a/flow/SystemMonitor.cpp
+++ b/flow/SystemMonitor.cpp
@@ -18,6 +18,8 @@
  * limitations under the License.
  */
 
+#include <fstream>
+
 #include "flow/flow.h"
 #include "flow/Histogram.h"
 #include "flow/Platform.h"
@@ -67,6 +69,35 @@ SystemStatistics getSystemStatistics() {
 	    .detail("ApproximateUnusedMemory" #size, FastAllocator<size>::getApproximateMemoryUnused())                    \
 	    .detail("ActiveThreads" #size, FastAllocator<size>::getActiveThreads())
 
+namespace {
+
+#ifdef __linux__
+// Converts cgroup key, e.g. nr_periods, to NrPeriods
+std::string capitalizeCgroupKey(const std::string& key) {
+	bool wordStart = true;
+	std::string result;
+	result.reserve(key.size());
+
+	for (const char ch : key) {
+		if (std::isalnum(ch)) {
+			if (wordStart) {
+				result.push_back(std::toupper(ch));
+				wordStart = false;
+			} else {
+				result.push_back(ch);
+			}
+		} else {
+			// Skip non-alnum characters
+			wordStart = true;
+		}
+	}
+
+	return result;
+}
+#endif // __linux__
+
+} // anonymous namespace
+
 SystemStatistics customSystemMonitor(std::string const& eventName, StatisticsState* statState, bool machineMetrics) {
 	const IPAddress ipAddr = machineState.ip.present() ? machineState.ip.get() : IPAddress();
 	SystemStatistics currentStats = getSystemStatistics(
@@ -284,8 +315,8 @@ SystemStatistics customSystemMonitor(std::string const& eventName, StatisticsSta
 		}
 
 		if (machineMetrics) {
-			TraceEvent("MachineMetrics")
-			    .detail("Elapsed", currentStats.elapsed)
+			auto traceEvent = TraceEvent("MachineMetrics");
+			traceEvent.detail("Elapsed", currentStats.elapsed)
 			    .detail("MbpsSent", currentStats.machineMegabitsSent / currentStats.elapsed)
 			    .detail("MbpsReceived", currentStats.machineMegabitsReceived / currentStats.elapsed)
 			    .detail("OutSegs", currentStats.machineOutSegs)
@@ -298,6 +329,11 @@ SystemStatistics customSystemMonitor(std::string const& eventName, StatisticsSta
 			    .detail("ZoneID", machineState.zoneId)
 			    .detail("MachineID", machineState.machineId)
 			    .trackLatest("MachineMetrics");
+#ifdef __linux__
+			for (const auto& [k, v] : linux_os::reportCGroupCpuStat()) {
+				traceEvent.detail(capitalizeCgroupKey(k).c_str(), v);
+			}
+#endif // __linux__
 		}
 	}
 
@@ -315,19 +351,19 @@ SystemStatistics customSystemMonitor(std::string const& eventName, StatisticsSta
 				char* demangled = abi::__cxa_demangle(i->first, nullptr, nullptr, nullptr);
 				if (demangled) {
 					s = demangled;
-					if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::")))
-						s = s.substr(LiteralStringRef("(anonymous namespace)::").size());
+					if (StringRef(s).startsWith("(anonymous namespace)::"_sr))
+						s = s.substr("(anonymous namespace)::"_sr.size());
 					free(demangled);
 				} else
 					s = i->first;
 #else
 				s = i->first;
-				if (StringRef(s).startsWith(LiteralStringRef("class `anonymous namespace'::")))
-					s = s.substr(LiteralStringRef("class `anonymous namespace'::").size());
-				else if (StringRef(s).startsWith(LiteralStringRef("class ")))
-					s = s.substr(LiteralStringRef("class ").size());
-				else if (StringRef(s).startsWith(LiteralStringRef("struct ")))
-					s = s.substr(LiteralStringRef("struct ").size());
+				if (StringRef(s).startsWith("class `anonymous namespace'::"_sr))
+					s = s.substr("class `anonymous namespace'::"_sr.size());
+				else if (StringRef(s).startsWith("class "_sr))
+					s = s.substr("class "_sr.size());
+				else if (StringRef(s).startsWith("struct "_sr))
+					s = s.substr("struct "_sr.size());
 #endif
 				typeNames.emplace_back(s, i->first);
 			}
diff --git a/flow/TDMetric.cpp b/flow/TDMetric.cpp
index 0d66e286f0..016205524a 100644
--- a/flow/TDMetric.cpp
+++ b/flow/TDMetric.cpp
@@ -21,15 +21,15 @@
 #include "flow/TDMetric.actor.h"
 #include "flow/flow.h"
 
-const StringRef BaseEventMetric::metricType = LiteralStringRef("Event");
+const StringRef BaseEventMetric::metricType = "Event"_sr;
 template <>
-const StringRef Int64Metric::metricType = LiteralStringRef("Int64");
+const StringRef Int64Metric::metricType = "Int64"_sr;
 template <>
-const StringRef DoubleMetric::metricType = LiteralStringRef("Double");
+const StringRef DoubleMetric::metricType = "Double"_sr;
 template <>
-const StringRef BoolMetric::metricType = LiteralStringRef("Bool");
+const StringRef BoolMetric::metricType = "Bool"_sr;
 template <>
-const StringRef StringMetric::metricType = LiteralStringRef("String");
+const StringRef StringMetric::metricType = "String"_sr;
 
 std::string reduceFilename(std::string const& filename) {
 	std::string r = filename;
@@ -60,29 +60,29 @@ std::string reduceFilename(std::string const& filename) {
 }
 
 void MetricKeyRef::writeField(BinaryWriter& wr) const {
-	wr.serializeBytes(LiteralStringRef("\x01"));
+	wr.serializeBytes("\x01"_sr);
 	wr.serializeBytes(fieldName);
-	wr.serializeBytes(LiteralStringRef("\x00\x01"));
+	wr.serializeBytes("\x00\x01"_sr);
 	wr.serializeBytes(fieldType);
-	wr.serializeBytes(LiteralStringRef("\x00"));
+	wr.serializeBytes("\x00"_sr);
 }
 
 void MetricKeyRef::writeMetricName(BinaryWriter& wr) const {
-	wr.serializeBytes(LiteralStringRef("\x01"));
+	wr.serializeBytes("\x01"_sr);
 	wr.serializeBytes(name.name);
-	wr.serializeBytes(LiteralStringRef("\x00\x01"));
+	wr.serializeBytes("\x00\x01"_sr);
 	wr.serializeBytes(name.type);
-	wr.serializeBytes(LiteralStringRef("\x00\x01"));
+	wr.serializeBytes("\x00\x01"_sr);
 	wr.serializeBytes(address);
-	wr.serializeBytes(LiteralStringRef("\x00\x01"));
+	wr.serializeBytes("\x00\x01"_sr);
 	wr.serializeBytes(name.id);
-	wr.serializeBytes(LiteralStringRef("\x00"));
+	wr.serializeBytes("\x00"_sr);
 }
 
 const Standalone<StringRef> MetricKeyRef::packLatestKey() const {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(prefix);
-	wr.serializeBytes(LiteralStringRef("\x01TDMetricsLastValue\x00"));
+	wr.serializeBytes("\x01TDMetricsLastValue\x00"_sr);
 	writeMetricName(wr);
 	return wr.toValue();
 }
@@ -91,9 +91,9 @@ const Standalone<StringRef> MetricKeyRef::packDataKey(int64_t time) const {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(prefix);
 	if (isField())
-		wr.serializeBytes(LiteralStringRef("\x01TDFieldData\x00"));
+		wr.serializeBytes("\x01TDFieldData\x00"_sr);
 	else
-		wr.serializeBytes(LiteralStringRef("\x01TDMetricData\x00"));
+		wr.serializeBytes("\x01TDMetricData\x00"_sr);
 	writeMetricName(wr);
 	if (isField())
 		writeField(wr);
@@ -107,15 +107,15 @@ const Standalone<StringRef> MetricKeyRef::packFieldRegKey() const {
 	ASSERT(isField());
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(prefix);
-	wr.serializeBytes(LiteralStringRef("\x01TDFields\x00\x01"));
+	wr.serializeBytes("\x01TDFields\x00\x01"_sr);
 	wr.serializeBytes(name.name);
-	wr.serializeBytes(LiteralStringRef("\x00\x01"));
+	wr.serializeBytes("\x00\x01"_sr);
 	wr.serializeBytes(name.type);
-	wr.serializeBytes(LiteralStringRef("\x00\x01"));
+	wr.serializeBytes("\x00\x01"_sr);
 	wr.serializeBytes(fieldName);
-	wr.serializeBytes(LiteralStringRef("\x00\x01"));
+	wr.serializeBytes("\x00\x01"_sr);
 	wr.serializeBytes(fieldType);
-	wr.serializeBytes(LiteralStringRef("\x00"));
+	wr.serializeBytes("\x00"_sr);
 	return wr.toValue();
 }
 
@@ -134,7 +134,7 @@ bool TDMetricCollection::canLog(int level) const {
 void TDMetricCollection::checkRoll(uint64_t t, int64_t usedBytes) {
 	currentTimeBytes += usedBytes;
 	if (currentTimeBytes > 1e6) {
-		TEST(true); // metrics were rolled
+		CODE_PROBE(true, "metrics were rolled");
 		currentTimeBytes = 0;
 		rollTimes.push_back(t);
 		for (auto& it : metricMap)
diff --git a/flow/TLSConfig.actor.cpp b/flow/TLSConfig.actor.cpp
index 83ce48f526..c7e891284c 100644
--- a/flow/TLSConfig.actor.cpp
+++ b/flow/TLSConfig.actor.cpp
@@ -81,7 +81,7 @@ void LoadedTLSConfig::print(FILE* fp) {
 	int num_certs = 0;
 	boost::asio::ssl::context context(boost::asio::ssl::context::tls);
 	try {
-		ConfigureSSLContext(*this, &context);
+		ConfigureSSLContext(*this, context);
 	} catch (Error& e) {
 		fprintf(fp, "There was an error in loading the certificate chain.\n");
 		throw;
@@ -109,51 +109,58 @@ void LoadedTLSConfig::print(FILE* fp) {
 	X509_STORE_CTX_free(store_ctx);
 }
 
-void ConfigureSSLContext(const LoadedTLSConfig& loaded,
-                         boost::asio::ssl::context* context,
-                         std::function<void()> onPolicyFailure) {
+void ConfigureSSLContext(const LoadedTLSConfig& loaded, boost::asio::ssl::context& context) {
 	try {
-		context->set_options(boost::asio::ssl::context::default_workarounds);
-		context->set_verify_mode(boost::asio::ssl::context::verify_peer |
-		                         boost::asio::ssl::verify_fail_if_no_peer_cert);
+		context.set_options(boost::asio::ssl::context::default_workarounds);
+		auto verifyFailIfNoPeerCert = boost::asio::ssl::verify_fail_if_no_peer_cert;
+		// Servers get to accept connections without peer certs as "untrusted" clients
+		if (loaded.getEndpointType() == TLSEndpointType::SERVER)
+			verifyFailIfNoPeerCert = 0;
+		context.set_verify_mode(boost::asio::ssl::context::verify_peer | verifyFailIfNoPeerCert);
 
-		if (loaded.isTLSEnabled()) {
-			auto tlsPolicy = makeReference<TLSPolicy>(loaded.getEndpointType());
-			tlsPolicy->set_verify_peers({ loaded.getVerifyPeers() });
-
-			context->set_verify_callback(
-			    [policy = tlsPolicy, onPolicyFailure](bool preverified, boost::asio::ssl::verify_context& ctx) {
-				    bool success = policy->verify_peer(preverified, ctx.native_handle());
-				    if (!success) {
-					    onPolicyFailure();
-				    }
-				    return success;
-			    });
-		} else {
-			// Insecurely always except if TLS is not enabled.
-			context->set_verify_callback([](bool, boost::asio::ssl::verify_context&) { return true; });
-		}
-
-		context->set_password_callback([password = loaded.getPassword()](
-		                                   size_t, boost::asio::ssl::context::password_purpose) { return password; });
+		context.set_password_callback([password = loaded.getPassword()](
+		                                  size_t, boost::asio::ssl::context::password_purpose) { return password; });
 
 		const std::string& CABytes = loaded.getCABytes();
 		if (CABytes.size()) {
-			context->add_certificate_authority(boost::asio::buffer(CABytes.data(), CABytes.size()));
+			context.add_certificate_authority(boost::asio::buffer(CABytes.data(), CABytes.size()));
 		}
 
 		const std::string& keyBytes = loaded.getKeyBytes();
 		if (keyBytes.size()) {
-			context->use_private_key(boost::asio::buffer(keyBytes.data(), keyBytes.size()),
-			                         boost::asio::ssl::context::pem);
+			context.use_private_key(boost::asio::buffer(keyBytes.data(), keyBytes.size()),
+			                        boost::asio::ssl::context::pem);
 		}
 
 		const std::string& certBytes = loaded.getCertificateBytes();
 		if (certBytes.size()) {
-			context->use_certificate_chain(boost::asio::buffer(certBytes.data(), certBytes.size()));
+			context.use_certificate_chain(boost::asio::buffer(certBytes.data(), certBytes.size()));
 		}
 	} catch (boost::system::system_error& e) {
-		TraceEvent("TLSConfigureError")
+		TraceEvent("TLSContextConfigureError")
+		    .detail("What", e.what())
+		    .detail("Value", e.code().value())
+		    .detail("WhichMeans", TLSPolicy::ErrorString(e.code()));
+		throw tls_error();
+	}
+}
+
+void ConfigureSSLStream(Reference<TLSPolicy> policy,
+                        boost::asio::ssl::stream<boost::asio::ip::tcp::socket&>& stream,
+                        std::function<void(bool)> callback) {
+	try {
+		stream.set_verify_callback([policy, callback](bool preverified, boost::asio::ssl::verify_context& ctx) {
+			bool success = policy->verify_peer(preverified, ctx.native_handle());
+			if (!success) {
+				if (policy->on_failure)
+					policy->on_failure();
+			}
+			if (callback)
+				callback(success);
+			return success;
+		});
+	} catch (boost::system::system_error& e) {
+		TraceEvent("TLSStreamConfigureError")
 		    .detail("What", e.what())
 		    .detail("Value", e.code().value())
 		    .detail("WhichMeans", TLSPolicy::ErrorString(e.code()));
@@ -171,11 +178,7 @@ std::string TLSConfig::getCertificatePathSync() const {
 		return envCertPath;
 	}
 
-	const char* defaultCertFileName = "fdb.pem";
-	if (fileExists(defaultCertFileName)) {
-		return defaultCertFileName;
-	}
-
+	const char* defaultCertFileName = "cert.pem";
 	if (fileExists(joinPath(platform::getDefaultConfigPath(), defaultCertFileName))) {
 		return joinPath(platform::getDefaultConfigPath(), defaultCertFileName);
 	}
@@ -193,13 +196,9 @@ std::string TLSConfig::getKeyPathSync() const {
 		return envKeyPath;
 	}
 
-	const char* defaultCertFileName = "fdb.pem";
-	if (fileExists(defaultCertFileName)) {
-		return defaultCertFileName;
-	}
-
-	if (fileExists(joinPath(platform::getDefaultConfigPath(), defaultCertFileName))) {
-		return joinPath(platform::getDefaultConfigPath(), defaultCertFileName);
+	const char* defaultKeyFileName = "key.pem";
+	if (fileExists(joinPath(platform::getDefaultConfigPath(), defaultKeyFileName))) {
+		return joinPath(platform::getDefaultConfigPath(), defaultKeyFileName);
 	}
 
 	return std::string();
@@ -261,6 +260,11 @@ LoadedTLSConfig TLSConfig::loadSync() const {
 	return loaded;
 }
 
+TLSPolicy::TLSPolicy(const LoadedTLSConfig& loaded, std::function<void()> on_failure)
+  : rules(), on_failure(std::move(on_failure)), is_client(loaded.getEndpointType() == TLSEndpointType::CLIENT) {
+	set_verify_peers(loaded.getVerifyPeers());
+}
+
 // And now do the same thing, but async...
 
 ACTOR static Future<Void> readEntireFile(std::string filename, std::string* destination) {
diff --git a/flow/TLSTest.cpp b/flow/TLSTest.cpp
index edf78128e5..ebbbdacd3a 100644
--- a/flow/TLSTest.cpp
+++ b/flow/TLSTest.cpp
@@ -20,7 +20,6 @@
 
 #include <boost/asio.hpp>
 #include <boost/asio/ssl.hpp>
-#include <boost/bind.hpp>
 #include <fmt/format.h>
 #include <algorithm>
 #include <iostream>
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index bf4d5b62f6..4717247d16 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -37,6 +37,7 @@
 #include "flow/EventTypes.actor.h"
 #include "flow/TDMetric.actor.h"
 #include "flow/MetricSample.h"
+#include "flow/network.h"
 
 #ifdef _WIN32
 #include <windows.h>
@@ -162,11 +163,11 @@ public:
 	bool logTraceEventMetrics;
 
 	void initMetrics() {
-		SevErrorNames.init(LiteralStringRef("TraceEvents.SevError"));
-		SevWarnAlwaysNames.init(LiteralStringRef("TraceEvents.SevWarnAlways"));
-		SevWarnNames.init(LiteralStringRef("TraceEvents.SevWarn"));
-		SevInfoNames.init(LiteralStringRef("TraceEvents.SevInfo"));
-		SevDebugNames.init(LiteralStringRef("TraceEvents.SevDebug"));
+		SevErrorNames.init("TraceEvents.SevError"_sr);
+		SevWarnAlwaysNames.init("TraceEvents.SevWarnAlways"_sr);
+		SevWarnNames.init("TraceEvents.SevWarn"_sr);
+		SevInfoNames.init("TraceEvents.SevInfo"_sr);
+		SevDebugNames.init("TraceEvents.SevDebug"_sr);
 		logTraceEventMetrics = true;
 	}
 
@@ -568,6 +569,16 @@ public:
 		universalFields[name] = value;
 	}
 
+	void setLocalAddress(const NetworkAddress& addr) {
+		MutexHolder holder(mutex);
+		this->localAddress = addr;
+	}
+
+	void disposeWriter() {
+		writer->addref();
+		writer.clear();
+	}
+
 	Future<Void> pingWriterThread() {
 		auto ping = new WriterThread::Ping;
 		auto f = ping->ack.getFuture();
@@ -589,7 +600,7 @@ public:
 NetworkAddress getAddressIndex() {
 	// ahm
 	//	if( g_network->isSimulated() )
-	//		return g_simulator.getCurrentProcess()->address;
+	//		return g_simulator->getCurrentProcess()->address;
 	//	else
 	return g_network->getLocalAddress();
 }
@@ -813,6 +824,18 @@ void addUniversalTraceField(const std::string& name, const std::string& value) {
 	g_traceLog.addUniversalTraceField(name, value);
 }
 
+void setTraceLocalAddress(const NetworkAddress& addr) {
+	g_traceLog.setLocalAddress(addr);
+}
+
+void disposeTraceFileWriter() {
+	g_traceLog.disposeWriter();
+}
+
+std::string getTraceFormatExtension() {
+	return std::string(g_traceLog.formatter->getExtension());
+}
+
 BaseTraceEvent::BaseTraceEvent() : initialized(true), enabled(false), logged(true) {}
 BaseTraceEvent::BaseTraceEvent(Severity severity, const char* type, UID id)
   : initialized(false), enabled(g_network == nullptr || FLOW_KNOBS->MIN_TRACE_SEVERITY <= severity), logged(false),
@@ -1286,6 +1309,10 @@ thread_local bool BaseTraceEvent::networkThread = false;
 void BaseTraceEvent::setNetworkThread() {
 	if (!networkThread) {
 		if (FLOW_KNOBS->ALLOCATION_TRACING_ENABLED) {
+			// Ensure that threadId is initialized before we enable allocation tracing, otherwise it would
+			// be initialized either by first normal usage of TraceEvent or by allocation tracing which is
+			// non-deterministic, and we could run into https://github.com/apple/foundationdb/issues/7872.
+			getTraceThreadId();
 			--g_allocation_tracing_disabled;
 		}
 
@@ -1337,7 +1364,9 @@ std::string BaseTraceEvent::printRealTime(double time) {
 }
 
 TraceInterval& TraceInterval::begin() {
-	pairID = nondeterministicRandom()->randomUniqueID();
+	if (!pairID.isValid()) {
+		pairID = nondeterministicRandom()->randomUniqueID();
+	}
 	count = 0;
 	return *this;
 }
diff --git a/flow/actorcompiler/ActorCompiler.cs b/flow/actorcompiler/ActorCompiler.cs
index 70465321f6..3847c6a977 100644
--- a/flow/actorcompiler/ActorCompiler.cs
+++ b/flow/actorcompiler/ActorCompiler.cs
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -251,16 +251,16 @@ namespace actorcompiler
             lines = 0;
 
             writer.WriteLine(memberIndentStr + "template<> struct Descriptor<struct {0}> {{", descr.name);
-            writer.WriteLine(memberIndentStr + "\tstatic StringRef typeName() {{ return LiteralStringRef(\"{0}\"); }}", descr.name);
+            writer.WriteLine(memberIndentStr + "\tstatic StringRef typeName() {{ return \"{0}\"_sr; }}", descr.name);
             writer.WriteLine(memberIndentStr + "\ttypedef {0} type;", descr.name);
             lines += 3;
 
             foreach (var dec in descr.body)
             {
                 writer.WriteLine(memberIndentStr + "\tstruct {0}Descriptor {{", dec.name);
-                writer.WriteLine(memberIndentStr + "\t\tstatic StringRef name() {{ return LiteralStringRef(\"{0}\"); }}", dec.name);
-                writer.WriteLine(memberIndentStr + "\t\tstatic StringRef typeName() {{ return LiteralStringRef(\"{0}\"); }}", dec.type);
-                writer.WriteLine(memberIndentStr + "\t\tstatic StringRef comment() {{ return LiteralStringRef(\"{0}\"); }}", dec.comment);
+                writer.WriteLine(memberIndentStr + "\t\tstatic StringRef name() {{ return \"{0}\"_sr; }}", dec.name);
+                writer.WriteLine(memberIndentStr + "\t\tstatic StringRef typeName() {{ return \"{0}\"_sr; }}", dec.type);
+                writer.WriteLine(memberIndentStr + "\t\tstatic StringRef comment() {{ return \"{0}\"_sr; }}", dec.comment);
                 writer.WriteLine(memberIndentStr + "\t\ttypedef {0} type;", dec.type);
                 writer.WriteLine(memberIndentStr + "\t\tstatic inline type get({0}& from);", descr.name);
                 writer.WriteLine(memberIndentStr + "\t};");
@@ -1302,7 +1302,7 @@ namespace actorcompiler
             constructor.WriteLine("this->lineage.setActorName(\"{0}\");", actor.name);
             constructor.WriteLine("LineageScope _(&this->lineage);");
             constructor.WriteLine("#endif");
-            // constructor.WriteLine("getCurrentLineage()->modify(&StackLineage::actorName) = LiteralStringRef(\"{0}\");", actor.name);
+            // constructor.WriteLine("getCurrentLineage()->modify(&StackLineage::actorName) = \"{0}\"_sr;", actor.name);
             constructor.WriteLine("this->{0};", body.call());
             ProbeExit(constructor, actor.name);
             WriteFunction(writer, constructor, constructor.BodyText);
diff --git a/flow/actorcompiler/ParseTree.cs b/flow/actorcompiler/ParseTree.cs
index 8e94773bc7..689e982d11 100644
--- a/flow/actorcompiler/ParseTree.cs
+++ b/flow/actorcompiler/ParseTree.cs
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/flow/actorcompiler/Program.cs b/flow/actorcompiler/Program.cs
index e3a491c0e2..cdb8ee8d6e 100644
--- a/flow/actorcompiler/Program.cs
+++ b/flow/actorcompiler/Program.cs
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/flow/config.h.cmake b/flow/config.h.cmake
index 864d7c68fb..cad9c61186 100644
--- a/flow/config.h.cmake
+++ b/flow/config.h.cmake
@@ -5,6 +5,8 @@
 # define FDB_CLEAN_BUILD
 #endif // FDB_RELEASE
 #cmakedefine OPEN_FOR_IDE
+#define FDB_SOURCE_DIR "${CMAKE_SOURCE_DIR}"
+#define FDB_BINARY_DIR "${CMAKE_BINARY_DIR}"
 #ifdef WIN32
 # define _WIN32_WINNT ${WINDOWS_TARGET}
 # define WINVER ${WINDOWS_TARGET}
diff --git a/flow/flat_buffers.cpp b/flow/flat_buffers.cpp
index ea1c1c2dce..bf7948e024 100644
--- a/flow/flat_buffers.cpp
+++ b/flow/flat_buffers.cpp
@@ -473,13 +473,13 @@ TEST_CASE("/flow/FlatBuffers/VectorRef") {
 				vec.push_back(arena, str);
 			}
 			ObjectWriter writer(Unversioned());
-			writer.serialize(FileIdentifierFor<decltype(vec)>::value, arena, vec);
+			writer.serialize(FileIdentifierFor<decltype(vec)>::value, vec);
 			serializedVector = StringRef(readerArena, writer.toStringRef());
 		}
 		ArenaObjectReader reader(readerArena, serializedVector, Unversioned());
-		// The VectorRef and Arena arguments are intentionally in a different order from the serialize call above.
+		// The Arena argument is intentionally missing from the serialize call above.
 		// Arenas need to get serialized after any Ref types whose memory they own. In order for schema evolution to be
-		// possible, it needs to be okay to reorder an Arena so that it appears after a newly added Ref type. For this
+		// possible, it needs to be okay to move/add an Arena so that it appears after a newly added Ref type. For this
 		// reason, Arenas are ignored by the wire protocol entirely. We test that behavior here.
 		reader.deserialize(FileIdentifierFor<decltype(outVec)>::value, outVec, vecArena);
 	}
diff --git a/flow/flow.cpp b/flow/flow.cpp
index e758ffbfd3..0da59e6d76 100644
--- a/flow/flow.cpp
+++ b/flow/flow.cpp
@@ -20,6 +20,7 @@
 
 #include "flow/flow.h"
 #include "flow/DeterministicRandom.h"
+#include "flow/Error.h"
 #include "flow/UnitTest.h"
 #include "flow/rte_memcpy.h"
 #ifdef WITH_FOLLY_MEMCPY
@@ -27,6 +28,8 @@
 #endif
 #include <stdarg.h>
 #include <cinttypes>
+#include <openssl/err.h>
+#include <openssl/rand.h>
 
 std::atomic<bool> startSampling = false;
 LineageReference rootLineage;
@@ -261,7 +264,7 @@ int vsformat(std::string& outputString, const char* form, va_list args) {
 		return -1;
 	}
 
-	TEST(true); // large format result
+	CODE_PROBE(true, "large format result");
 
 	outputString.resize(size + 1);
 	size = vsnprintf(&outputString[0], outputString.size(), form, args);
@@ -374,6 +377,77 @@ void enableBuggify(bool enabled, BuggifyType type) {
 	buggifyActivated[int(type)] = enabled;
 }
 
+// Make OpenSSL use DeterministicRandom as RNG source such that simulation runs stay deterministic w/ e.g. signature ops
+void bindDeterministicRandomToOpenssl() {
+	// TODO: implement ifdef branch for 3.x using provider API
+#ifndef OPENSSL_IS_BORINGSSL
+	static const RAND_METHOD method = {
+		// replacement for RAND_seed(), which reseeds OpenSSL RNG
+		[](const void*, int) -> int { return 1; },
+		// replacement for RAND_bytes(), which fills given buffer with random byte sequence
+		[](unsigned char* buf, int length) -> int {
+		    if (g_network)
+			    ASSERT_ABORT(g_network->isSimulated());
+		    deterministicRandom()->randomBytes(buf, length);
+		    return 1;
+		},
+		// replacement for RAND_cleanup(), a no-op for simulation
+		[]() -> void {},
+		// replacement for RAND_add(), which reseeds OpenSSL RNG with randomness hint
+		[](const void*, int, double) -> int { return 1; },
+		// replacement for default pseudobytes getter (same as RAND_bytes by default)
+		[](unsigned char* buf, int length) -> int {
+		    if (g_network)
+			    ASSERT_ABORT(g_network->isSimulated());
+		    deterministicRandom()->randomBytes(buf, length);
+		    return 1;
+		},
+		// status function for PRNG readiness check
+		[]() -> int { return 1; },
+	};
+
+	if (1 != ::RAND_set_rand_method(&method)) {
+		auto ec = ::ERR_get_error();
+		char msg[256]{
+			0,
+		};
+		if (ec) {
+			::ERR_error_string_n(ec, msg, sizeof(msg));
+		}
+		fprintf(stderr,
+		        "ERROR: Failed to bind DeterministicRandom to OpenSSL RNG\n"
+		        "       OpenSSL error message: '%s'\n",
+		        msg);
+		throw internal_error();
+	} else {
+		printf("DeterministicRandom successfully bound to OpenSSL RNG\n");
+	}
+#else // OPENSSL_IS_BORINGSSL
+	static const RAND_METHOD method = {
+		[](const void*, int) -> void {},
+		[](unsigned char* buf, unsigned long length) -> int {
+		    if (g_network)
+			    ASSERT_ABORT(g_network->isSimulated());
+		    ASSERT(length <= std::numeric_limits<int>::max());
+		    deterministicRandom()->randomBytes(buf, length);
+		    return 1;
+		},
+		[]() -> void {},
+		[](const void*, int, double) -> void {},
+		[](unsigned char* buf, unsigned long length) -> int {
+		    if (g_network)
+			    ASSERT_ABORT(g_network->isSimulated());
+		    ASSERT(length <= std::numeric_limits<int>::max());
+		    deterministicRandom()->randomBytes(buf, length);
+		    return 1;
+		},
+		[]() -> int { return 1; },
+	};
+	::RAND_set_rand_method(&method);
+	printf("DeterministicRandom successfully bound to OpenSSL RNG\n");
+#endif // OPENSSL_IS_BORINGSSL
+}
+
 namespace {
 // Simple message for flatbuffers unittests
 struct Int {
@@ -451,7 +525,7 @@ TEST_CASE("/flow/FlatBuffers/Standalone") {
 		ASSERT(in == out);
 	}
 	{
-		StringRef in = LiteralStringRef("foobar");
+		StringRef in = "foobar"_sr;
 		Standalone<StringRef> out;
 		ObjectWriter writer(Unversioned());
 		writer.serialize(in);
diff --git a/flow/genericactors.actor.cpp b/flow/genericactors.actor.cpp
index 523e5d8027..91fc86dbdc 100644
--- a/flow/genericactors.actor.cpp
+++ b/flow/genericactors.actor.cpp
@@ -68,6 +68,24 @@ ACTOR Future<Void> timeoutWarningCollector(FutureStream<Void> input, double logD
 	}
 }
 
+ACTOR Future<Void> waitForMost(std::vector<Future<ErrorOr<Void>>> futures,
+                               int faultTolerance,
+                               Error e,
+                               double waitMultiplierForSlowFutures) {
+	state std::vector<Future<bool>> successFutures;
+	state double startTime = now();
+	successFutures.reserve(futures.size());
+	for (const auto& future : futures) {
+		successFutures.push_back(fmap([](auto const& result) { return result.present(); }, future));
+	}
+	bool success = wait(quorumEqualsTrue(successFutures, successFutures.size() - faultTolerance));
+	if (!success) {
+		throw e;
+	}
+	wait(delay((now() - startTime) * waitMultiplierForSlowFutures) || waitForAll(successFutures));
+	return Void();
+}
+
 ACTOR Future<bool> quorumEqualsTrue(std::vector<Future<bool>> futures, int required) {
 	state std::vector<Future<Void>> true_futures;
 	state std::vector<Future<Void>> false_futures;
@@ -168,6 +186,14 @@ ACTOR Future<Void> testSubscriber(Reference<IAsyncListener<int>> output, Optiona
 	}
 }
 
+static Future<ErrorOr<Void>> goodTestFuture(double duration) {
+	return tag(delay(duration), ErrorOr<Void>(Void()));
+}
+
+static Future<ErrorOr<Void>> badTestFuture(double duration, Error e) {
+	return tag(delay(duration), ErrorOr<Void>(e));
+}
+
 } // namespace
 
 TEST_CASE("/flow/genericactors/AsyncListener") {
@@ -181,6 +207,39 @@ TEST_CASE("/flow/genericactors/AsyncListener") {
 	return Void();
 }
 
+TEST_CASE("/flow/genericactors/WaitForMost") {
+	state std::vector<Future<ErrorOr<Void>>> futures;
+	{
+		futures = { goodTestFuture(1), goodTestFuture(2), goodTestFuture(3) };
+		wait(waitForMost(futures, 1, operation_failed(), 0.0)); // Don't wait for slowest future
+		ASSERT(!futures[2].isReady());
+	}
+	{
+		futures = { goodTestFuture(1), goodTestFuture(2), goodTestFuture(3) };
+		wait(waitForMost(futures, 0, operation_failed(), 0.0)); // Wait for all futures
+		ASSERT(futures[2].isReady());
+	}
+	{
+		futures = { goodTestFuture(1), goodTestFuture(2), goodTestFuture(3) };
+		wait(waitForMost(futures, 1, operation_failed(), 1.0)); // Wait for slowest future
+		ASSERT(futures[2].isReady());
+	}
+	{
+		futures = { goodTestFuture(1), goodTestFuture(2), badTestFuture(1, success()) };
+		wait(waitForMost(futures, 1, operation_failed(), 1.0)); // Error ignored
+	}
+	{
+		futures = { goodTestFuture(1), goodTestFuture(2), badTestFuture(1, success()) };
+		try {
+			wait(waitForMost(futures, 0, operation_failed(), 1.0));
+			ASSERT(false);
+		} catch (Error& e) {
+			ASSERT_EQ(e.code(), error_code_operation_failed);
+		}
+	}
+	return Void();
+}
+
 #if false
 TEST_CASE("/flow/genericactors/generic/storeTuple") {
 	state std::vector<UID> resA;
diff --git a/flow/include/flow/ActorCollection.h b/flow/include/flow/ActorCollection.h
index ae4567f178..c1fbae1129 100644
--- a/flow/include/flow/ActorCollection.h
+++ b/flow/include/flow/ActorCollection.h
@@ -67,7 +67,7 @@ class ActorCollection : NonCopyable {
 	Future<Void> m_out;
 
 public:
-	explicit ActorCollection(bool returnWhenEmptied) {
+	explicit ActorCollection(bool returnWhenEmptied = false) {
 		m_out = actorCollection(m_add.getFuture(), nullptr, nullptr, nullptr, nullptr, returnWhenEmptied);
 	}
 
diff --git a/flow/include/flow/Arena.h b/flow/include/flow/Arena.h
index bfef0b4a6b..a6defdadf1 100644
--- a/flow/include/flow/Arena.h
+++ b/flow/include/flow/Arena.h
@@ -34,6 +34,7 @@
 #include <algorithm>
 #include <boost/functional/hash.hpp>
 #include <stdint.h>
+#include <string_view>
 #include <string>
 #include <cstring>
 #include <limits>
@@ -41,6 +42,8 @@
 #include <set>
 #include <type_traits>
 #include <sstream>
+#include <string_view>
+#include <fmt/format.h>
 
 // TrackIt is a zero-size class for tracking constructions, destructions, and assignments of instances
 // of a class.  Just inherit TrackIt<T> from T to enable tracking of construction and destruction of
@@ -97,6 +100,7 @@ FDB_DECLARE_BOOLEAN_PARAM(FastInaccurateEstimate);
 // memory is freed by deleting the entire Arena at once. See flow/README.md for details on using Arenas.
 class Arena {
 public:
+	constexpr static auto fb_must_appear_last = true;
 	Arena();
 	explicit Arena(size_t reservedSize);
 	//~Arena();
@@ -362,6 +366,8 @@ class Standalone : private Arena, public T {
 public:
 	using RefType = T;
 
+	constexpr static auto fb_must_appear_last = false;
+
 	// T must have no destructor
 	Arena& arena() { return *(Arena*)this; }
 	const Arena& arena() const { return *(const Arena*)this; }
@@ -530,7 +536,9 @@ public:
 		return substr(0, size() - s.size());
 	}
 
-	std::string toString() const { return std::string((const char*)data, length); }
+	std::string toString() const { return std::string(reinterpret_cast<const char*>(data), length); }
+
+	std::string_view toStringView() const { return std::string_view(reinterpret_cast<const char*>(data), length); }
 
 	static bool isPrintable(char c) { return c > 32 && c < 127; }
 	inline std::string printable() const;
@@ -587,14 +595,20 @@ public:
 	}
 
 	// Removes bytes from begin up to and including the sep string, returns StringRef of the part before sep
-	StringRef eat(StringRef sep) {
+	StringRef eat(StringRef sep, bool* foundSep = nullptr) {
 		for (int i = 0, iend = size() - sep.size(); i <= iend; ++i) {
 			if (sep.compare(substr(i, sep.size())) == 0) {
+				if (foundSep) {
+					*foundSep = true;
+				}
 				StringRef token = substr(0, i);
 				*this = substr(i + sep.size());
 				return token;
 			}
 		}
+		if (foundSep) {
+			*foundSep = false;
+		}
 		return eat();
 	}
 	StringRef eat() {
@@ -602,7 +616,9 @@ public:
 		*this = StringRef();
 		return r;
 	}
-	StringRef eat(const char* sep) { return eat(StringRef((const uint8_t*)sep, (int)strlen(sep))); }
+	StringRef eat(const char* sep, bool* foundSep = nullptr) {
+		return eat(StringRef((const uint8_t*)sep, (int)strlen(sep)), foundSep);
+	}
 	// Return StringRef of bytes from begin() up to but not including the first byte matching any byte in sep,
 	// and remove that sequence (including the sep byte) from *this
 	// Returns and removes all bytes from *this if no bytes within sep were found
@@ -715,15 +731,17 @@ struct Traceable<Standalone<T>> : std::conditional<Traceable<T>::value, std::tru
 	static std::string toString(const Standalone<T>& value) { return Traceable<T>::toString(value); }
 };
 
-namespace literal_string_ref {
-template <class T, int Size>
-StringRef LiteralStringRefHelper(const char* str) {
-	static_assert(std::is_same_v<T, const char(&)[Size]> || std::is_same_v<T, const char[Size]>,
-	              "Argument to LiteralStringRef must be a literal string");
-	return StringRef(reinterpret_cast<const uint8_t*>(str), Size - 1);
-}
-} // namespace literal_string_ref
-#define LiteralStringRef(str) literal_string_ref::LiteralStringRefHelper<decltype(str), sizeof(str)>(str)
+#define __FILE__sr StringRef(reinterpret_cast<const uint8_t*>(__FILE__), sizeof(__FILE__) - 1)
+#define __FUNCTION__sr StringRef(reinterpret_cast<const uint8_t*>(__FUNCTION__), sizeof(__FUNCTION__) - 1)
+
+template <>
+struct fmt::formatter<StringRef> : formatter<std::string_view> {
+	template <typename FormatContext>
+	auto format(const StringRef& str, FormatContext& ctx) -> decltype(ctx.out()) {
+		std::string_view view(reinterpret_cast<const char*>(str.begin()), str.size());
+		return formatter<string_view>::format(view, ctx);
+	}
+};
 
 inline StringRef operator"" _sr(const char* str, size_t size) {
 	return StringRef(reinterpret_cast<const uint8_t*>(str), size);
@@ -805,6 +823,10 @@ inline bool operator==(const StringRef& lhs, const StringRef& rhs) {
 	ASSERT(lhs.size() >= 0);
 	return lhs.size() == rhs.size() && memcmp(lhs.begin(), rhs.begin(), static_cast<unsigned int>(lhs.size())) == 0;
 }
+template <int N>
+inline bool operator==(const StringRef& lhs, const char (&rhs)[N]) {
+	return lhs == StringRef(reinterpret_cast<const uint8_t*>(rhs), N - 1);
+}
 inline bool operator<(const StringRef& lhs, const StringRef& rhs) {
 	if (std::min(lhs.size(), rhs.size()) > 0) {
 		int c = memcmp(lhs.begin(), rhs.begin(), std::min(lhs.size(), rhs.size()));
@@ -831,6 +853,40 @@ inline bool operator>=(const StringRef& lhs, const StringRef& rhs) {
 	return !(lhs < rhs);
 }
 
+typedef uint64_t Word;
+// Get the number of prefix bytes that are the same between a and b, up to their common length of cl
+static inline int commonPrefixLength(uint8_t const* ap, uint8_t const* bp, int cl) {
+	int i = 0;
+	const int wordEnd = cl - sizeof(Word) + 1;
+
+	for (; i < wordEnd; i += sizeof(Word)) {
+		Word a = *(Word*)ap;
+		Word b = *(Word*)bp;
+		if (a != b) {
+			return i + ctzll(a ^ b) / 8;
+		}
+		ap += sizeof(Word);
+		bp += sizeof(Word);
+	}
+
+	for (; i < cl; i++) {
+		if (*ap != *bp) {
+			return i;
+		}
+		++ap;
+		++bp;
+	}
+	return cl;
+}
+
+static inline int commonPrefixLength(const StringRef& a, const StringRef& b) {
+	return commonPrefixLength(a.begin(), b.begin(), std::min(a.size(), b.size()));
+}
+
+static inline int commonPrefixLength(const StringRef& a, const StringRef& b, int skipLen) {
+	return commonPrefixLength(a.begin() + skipLen, b.begin() + skipLen, std::min(a.size(), b.size()) - skipLen);
+}
+
 // This trait is used by VectorRef to determine if deep copy constructor should recursively
 // call deep copies of each element.
 //
diff --git a/flow/include/flow/CodeProbe.h b/flow/include/flow/CodeProbe.h
new file mode 100644
index 0000000000..b39e4221da
--- /dev/null
+++ b/flow/include/flow/CodeProbe.h
@@ -0,0 +1,320 @@
+/*
+ * CodeProbe.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FLOW_TESTPROBE_H_
+#define FLOW_TESTPROBE_H_
+
+#include "flow/Knobs.h"
+#include "flow/Trace.h"
+
+namespace probe {
+
+struct ICodeProbe;
+
+enum class AnnotationType { Decoration, Assertion, Context, Functional };
+enum class ExecutionContext { Simulation, Net2 };
+
+namespace context {
+struct Net2 {
+	constexpr static AnnotationType type = AnnotationType::Context;
+	constexpr static ExecutionContext value = ExecutionContext::Net2;
+};
+struct Sim2 {
+	constexpr static AnnotationType type = AnnotationType::Context;
+	constexpr static ExecutionContext value = ExecutionContext::Simulation;
+};
+
+constexpr Net2 net2;
+constexpr Sim2 sim2;
+
+template <class Left, class Right>
+struct OrContext {
+	typename std::remove_const<Left>::type left;
+	typename std::remove_const<Right>::type right;
+	constexpr OrContext(Left left, Right right) : left(left), right(right) {}
+	constexpr bool operator()(ExecutionContext context) const { return left(context) || right(context); }
+};
+
+template <class Left, class Right>
+constexpr std::enable_if_t<Left::type == AnnotationType::Context && Right::type == AnnotationType::Context,
+                           OrContext<Left, Right>>
+operator|(Left const& lhs, Right const& rhs) {
+	return OrContext<Left, Right>(lhs, rhs);
+}
+
+} // namespace context
+
+namespace assert {
+struct NoSim {
+	constexpr static AnnotationType type = AnnotationType::Assertion;
+	bool operator()(ICodeProbe const* self) const;
+};
+struct SimOnly {
+	constexpr static AnnotationType type = AnnotationType::Assertion;
+	bool operator()(ICodeProbe const* self) const;
+};
+
+template <class Left, class Right>
+struct AssertOr {
+	typename std::remove_const<Left>::type left;
+	typename std::remove_const<Right>::type right;
+	constexpr AssertOr() {}
+	constexpr bool operator()(ICodeProbe* self) const { return left(self) || right(self); }
+};
+template <class Left, class Right>
+struct AssertAnd {
+	typename std::remove_const<Left>::type left;
+	typename std::remove_const<Right>::type right;
+	constexpr AssertAnd() {}
+	constexpr bool operator()(ICodeProbe* self) const { return left(self) && right(self); }
+};
+template <class T>
+struct AssertNot {
+	typename std::remove_const<T>::type other;
+	constexpr bool operator()(ICodeProbe* self) const { return !other(self); }
+};
+
+template <class Left, class Right>
+constexpr std::enable_if_t<Left::type == AnnotationType::Assertion && Right::type == AnnotationType::Assertion,
+                           AssertOr<Left, Right>>
+operator||(Left const& lhs, Right const& rhs) {
+	return AssertOr<Left, Right>();
+}
+template <class Left, class Right>
+constexpr std::enable_if_t<Left::type == AnnotationType::Assertion && Right::type == AnnotationType::Assertion,
+                           AssertAnd<Left, Right>>
+operator&&(Left const& lhs, Right const& rhs) {
+	return AssertAnd<Left, Right>();
+}
+
+template <class T>
+constexpr std::enable_if_t<T::type == AnnotationType::Assertion, AssertNot<T>> operator!(T const&) {
+	return AssertNot<T>();
+}
+
+constexpr SimOnly simOnly;
+constexpr auto noSim = !simOnly;
+
+} // namespace assert
+
+namespace func {
+
+struct Deduplicate {
+	constexpr static AnnotationType type = AnnotationType::Functional;
+};
+
+constexpr Deduplicate deduplicate;
+
+} // namespace func
+
+template <class... Args>
+struct CodeProbeAnnotations;
+
+template <>
+struct CodeProbeAnnotations<> {
+	static constexpr bool providesContext = false;
+	void hit(ICodeProbe* self) {}
+	void trace(const ICodeProbe*, BaseTraceEvent&, bool) const {}
+	constexpr bool expectContext(ExecutionContext context, bool prevHadSomeContext = false) const {
+		return !prevHadSomeContext;
+	}
+	constexpr bool shouldTrace(ICodeProbe const*) const { return true; }
+	constexpr bool deduplicate() const { return false; }
+};
+
+template <class Head, class... Tail>
+struct CodeProbeAnnotations<Head, Tail...> {
+	using HeadType = typename std::remove_const<Head>::type;
+	using ChildType = CodeProbeAnnotations<Tail...>;
+
+	static constexpr bool providesContext = HeadType::type == AnnotationType::Context || ChildType::providesContext;
+	static_assert(HeadType::type != AnnotationType::Context || !ChildType::providesContext,
+	              "Only one context annotation can be used");
+
+	HeadType head;
+	ChildType tail;
+
+	void hit(ICodeProbe* self) {
+		if constexpr (Head::type == AnnotationType::Assertion) {
+			ASSERT(head(self));
+		}
+		tail.hit(self);
+	}
+
+	bool shouldTrace(ICodeProbe const* self) const {
+		if constexpr (Head::type == AnnotationType::Assertion) {
+			if (!head(self)) {
+				return false;
+			}
+		}
+		return tail.shouldTrace(self);
+	}
+
+	void trace(const ICodeProbe* self, BaseTraceEvent& evt, bool condition) const {
+		if constexpr (Head::type == AnnotationType::Decoration) {
+			head.trace(self, evt, condition);
+		}
+		tail.trace(self, evt, condition);
+	}
+	// This should behave like the following:
+	// 1. If no context is passed in the code probe, we expect to see this in every context
+	// 2. Otherwise we will return true iff the execution context we're looking for has been passed to the probe
+	constexpr bool expectContext(ExecutionContext context, bool prevHadSomeContext = false) const {
+		if constexpr (HeadType::type == AnnotationType::Context) {
+			if (HeadType::value == context) {
+				return true;
+			} else {
+				return tail.expectContext(context, true);
+			}
+		} else {
+			return tail.expectContext(context, prevHadSomeContext);
+		}
+	}
+
+	constexpr bool deduplicate() const {
+		if constexpr (std::is_same_v<HeadType, func::Deduplicate>) {
+			return true;
+		} else {
+			return tail.deduplicate();
+		}
+	}
+};
+
+struct ICodeProbe {
+	virtual ~ICodeProbe();
+
+	bool operator==(ICodeProbe const& other) const;
+	bool operator!=(ICodeProbe const& other) const;
+
+	std::string_view filename() const;
+	virtual const char* filePath() const = 0;
+	virtual unsigned line() const = 0;
+	virtual const char* comment() const = 0;
+	virtual const char* condition() const = 0;
+	virtual const char* compilationUnit() const = 0;
+	virtual void trace(bool) const = 0;
+	virtual bool shouldTrace() const = 0;
+	virtual bool wasHit() const = 0;
+	virtual unsigned hitCount() const = 0;
+	virtual bool expectInContext(ExecutionContext context) const = 0;
+	virtual std::string function() const = 0;
+	virtual bool deduplicate() const = 0;
+
+	static void printProbesXML();
+	static void printProbesJSON(std::vector<std::string> const& ctxs = std::vector<std::string>());
+};
+
+void registerProbe(ICodeProbe const& probe);
+std::string functionNameFromInnerType(const char* name);
+
+template <class FileName, class Condition, class Comment, class CompUnit, unsigned Line, class Annotations>
+struct CodeProbeImpl : ICodeProbe {
+	static CodeProbeImpl* instancePtr() { return &_instance; }
+	static CodeProbeImpl& instance() { return _instance; }
+	void hit() {
+		if (_hitCount++ == 0) {
+			trace(true);
+		}
+		annotations.hit(this);
+	}
+
+	void trace(bool condition) const override {
+		TraceEvent evt(intToSeverity(FLOW_KNOBS->CODE_COV_TRACE_EVENT_SEVERITY), "CodeCoverage");
+		evt.detail("File", filename())
+		    .detail("Line", Line)
+		    .detail("Condition", Condition::value())
+		    .detail("Covered", condition)
+		    .detail("Comment", Comment::value());
+		annotations.trace(this, evt, condition);
+	}
+
+	bool shouldTrace() const override { return annotations.shouldTrace(this); }
+
+	bool wasHit() const override { return _hitCount > 0; }
+	unsigned hitCount() const override { return _hitCount; }
+
+	const char* filePath() const override { return FileName::value(); }
+	unsigned line() const override { return Line; }
+	const char* comment() const override { return Comment::value(); }
+	const char* condition() const override { return Condition::value(); }
+	const char* compilationUnit() const override { return CompUnit::value(); }
+	bool expectInContext(ExecutionContext context) const override { return annotations.expectContext(context); }
+	std::string function() const override { return functionNameFromInnerType(typeid(FileName).name()); }
+	bool deduplicate() const override { return annotations.deduplicate(); }
+
+private:
+	CodeProbeImpl() { registerProbe(*this); }
+	inline static CodeProbeImpl _instance;
+	unsigned _hitCount = 0;
+	Annotations annotations;
+};
+
+template <class FileName, class Condition, class Comment, class CompUnit, unsigned Line, class... Annotations>
+CodeProbeImpl<FileName, Condition, Comment, CompUnit, Line, CodeProbeAnnotations<Annotations...>>& probeInstance(
+    Annotations&... annotations) {
+	return CodeProbeImpl<FileName, Condition, Comment, CompUnit, Line, CodeProbeAnnotations<Annotations...>>::
+	    instance();
+}
+
+} // namespace probe
+
+#ifdef COMPILATION_UNIT
+#define CODE_PROBE_QUOTE(x) #x
+#define CODE_PROBE_EXPAND_AND_QUOTE(x) CODE_PROBE_QUOTE(x)
+#define CODE_PROBE_COMPILATION_UNIT CODE_PROBE_EXPAND_AND_QUOTE(COMPILATION_UNIT)
+#else
+#define CODE_PROBE_COMPILATION_UNIT "COMPILATION_UNIT not set"
+#endif
+
+#define _CODE_PROBE_IMPL(file, line, condition, comment, compUnit, fileType, condType, commentType, compUnitType, ...) \
+	struct fileType {                                                                                                  \
+		constexpr static const char* value() { return file; }                                                          \
+	};                                                                                                                 \
+	struct condType {                                                                                                  \
+		constexpr static const char* value() { return #condition; }                                                    \
+	};                                                                                                                 \
+	struct commentType {                                                                                               \
+		constexpr static const char* value() { return comment; }                                                       \
+	};                                                                                                                 \
+	struct compUnitType {                                                                                              \
+		constexpr static const char* value() { return compUnit; }                                                      \
+	};                                                                                                                 \
+	if (condition) {                                                                                                   \
+		probe::probeInstance<fileType, condType, commentType, compUnitType, line>(__VA_ARGS__).hit();                  \
+	}
+
+#define _CODE_PROBE_T2(type, counter) type##counter
+#define _CODE_PROBE_T(type, counter) _CODE_PROBE_T2(type, counter)
+
+#define CODE_PROBE(condition, comment, ...)                                                                            \
+	do {                                                                                                               \
+		_CODE_PROBE_IMPL(__FILE__,                                                                                     \
+		                 __LINE__,                                                                                     \
+		                 condition,                                                                                    \
+		                 comment,                                                                                      \
+		                 CODE_PROBE_COMPILATION_UNIT,                                                                  \
+		                 _CODE_PROBE_T(FileType, __LINE__),                                                            \
+		                 _CODE_PROBE_T(CondType, __LINE__),                                                            \
+		                 _CODE_PROBE_T(CommentType, __LINE__),                                                         \
+		                 _CODE_PROBE_T(CompilationUnitType, __LINE__),                                                 \
+		                 __VA_ARGS__)                                                                                  \
+	} while (false)
+
+#endif // FLOW_TESTPROBE_H_
diff --git a/fdbclient/TenantSpecialKeys.cpp b/flow/include/flow/CodeProbeUtils.h
similarity index 58%
rename from fdbclient/TenantSpecialKeys.cpp
rename to flow/include/flow/CodeProbeUtils.h
index 05d51b7e93..07b04dcd98 100644
--- a/fdbclient/TenantSpecialKeys.cpp
+++ b/flow/include/flow/CodeProbeUtils.h
@@ -1,5 +1,5 @@
 /*
- * TenantSpecialKeys.cpp
+ * CodeProbeUtils.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,16 +18,15 @@
  * limitations under the License.
  */
 
-#include "fdbclient/TenantSpecialKeys.actor.h"
+#ifndef FLOW_CODE_PROBE_UTILS_H
+#define FLOW_CODE_PROBE_UTILS_H
+#include "flow/CodeProbe.h"
+#include "flow/Arena.h"
 
-template <>
-const KeyRangeRef TenantRangeImpl<true>::submoduleRange = KeyRangeRef("tenant/"_sr, "tenant0"_sr);
+namespace probe {
 
-template <>
-const KeyRangeRef TenantRangeImpl<true>::mapSubRange = KeyRangeRef("map/"_sr, "map0"_sr);
+void traceMissedProbes(Optional<ExecutionContext> context);
 
-template <>
-const KeyRangeRef TenantRangeImpl<false>::submoduleRange = KeyRangeRef(""_sr, "\xff"_sr);
+}
 
-template <>
-const KeyRangeRef TenantRangeImpl<false>::mapSubRange = KeyRangeRef("tenant_map/"_sr, "tenant_map0"_sr);
+#endif // FLOW_CODE_PROBE_UTILS_H
diff --git a/flow/include/flow/CompressionUtils.h b/flow/include/flow/CompressionUtils.h
index a83898d726..d7b3c5c047 100644
--- a/flow/include/flow/CompressionUtils.h
+++ b/flow/include/flow/CompressionUtils.h
@@ -24,11 +24,12 @@
 
 #include "flow/Arena.h"
 
+#include <unordered_set>
+
 enum class CompressionFilter {
 	NONE,
-#ifdef ZLIB_LIB_SUPPORTED
 	GZIP,
-#endif
+	ZSTD,
 	LAST // Always the last member
 };
 
@@ -37,16 +38,17 @@ struct CompressionUtils {
 	static StringRef compress(const CompressionFilter filter, const StringRef& data, int level, Arena& arena);
 	static StringRef decompress(const CompressionFilter filter, const StringRef& data, Arena& arena);
 
+	static int getDefaultCompressionLevel(CompressionFilter filter);
+	static CompressionFilter getRandomFilter();
+
 	static CompressionFilter fromFilterString(const std::string& filter) {
 		if (filter == "NONE") {
 			return CompressionFilter::NONE;
-		}
-#ifdef ZLIB_LIB_SUPPORTED
-		else if (filter == "GZIP") {
+		} else if (filter == "GZIP") {
 			return CompressionFilter::GZIP;
-		}
-#endif
-		else {
+		} else if (filter == "ZSTD") {
+			return CompressionFilter::ZSTD;
+		} else {
 			throw not_implemented();
 		}
 	}
@@ -54,16 +56,22 @@ struct CompressionUtils {
 	static std::string toString(const CompressionFilter filter) {
 		if (filter == CompressionFilter::NONE) {
 			return "NONE";
-		}
-#ifdef ZLIB_LIB_SUPPORTED
-		else if (filter == CompressionFilter::GZIP) {
+		} else if (filter == CompressionFilter::GZIP) {
 			return "GZP";
-		}
-#endif
-		else {
+		} else if (filter == CompressionFilter::ZSTD) {
+			return "ZSTD";
+		} else {
 			throw not_implemented();
 		}
 	}
+
+	static void checkFilterSupported(const CompressionFilter filter) {
+		if (CompressionUtils::supportedFilters.find(filter) == CompressionUtils::supportedFilters.end()) {
+			throw not_implemented();
+		}
+	}
+
+	static std::unordered_set<CompressionFilter> supportedFilters;
 };
 
 #endif // FLOW_COMPRRESSION_UTILS_H
diff --git a/flow/include/flow/DebugTrace.h b/flow/include/flow/DebugTrace.h
index f97ff508b1..bcc2670eae 100644
--- a/flow/include/flow/DebugTrace.h
+++ b/flow/include/flow/DebugTrace.h
@@ -26,4 +26,7 @@
 constexpr bool debugLogTraces = false;
 #define DebugLogTraceEvent(...) DebugTraceEvent(debugLogTraces, __VA_ARGS__)
 
+constexpr bool debugRelocationTraces = false;
+#define DebugRelocationTraceEvent(...) DebugTraceEvent(debugRelocationTraces, __VA_ARGS__)
+
 #endif // FOUNDATIONDB_DEBUGTRACE_H
diff --git a/flow/include/flow/DeterministicRandom.h b/flow/include/flow/DeterministicRandom.h
index 3f5115c371..235a910f6c 100644
--- a/flow/include/flow/DeterministicRandom.h
+++ b/flow/include/flow/DeterministicRandom.h
@@ -49,6 +49,7 @@ public:
 	UID randomUniqueID() override;
 	char randomAlphaNumeric() override;
 	std::string randomAlphaNumeric(int length) override;
+	void randomBytes(uint8_t* buf, int length) override;
 	uint64_t peek() const override;
 	void addref() override;
 	void delref() override;
diff --git a/flow/include/flow/EncryptUtils.h b/flow/include/flow/EncryptUtils.h
index 0587da9a81..ec5b00b231 100644
--- a/flow/include/flow/EncryptUtils.h
+++ b/flow/include/flow/EncryptUtils.h
@@ -29,22 +29,29 @@
 #include <string>
 #include <string_view>
 
-#define ENCRYPT_INVALID_DOMAIN_ID -1
-#define ENCRYPT_INVALID_CIPHER_KEY_ID 0
-#define ENCRYPT_INVALID_RANDOM_SALT 0
-
-#define AUTH_TOKEN_SIZE 32
-
-#define SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID -2
-#define ENCRYPT_HEADER_DOMAIN_ID -3
-
-const std::string FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain";
+constexpr const int AUTH_TOKEN_HMAC_SHA_SIZE = 32;
+constexpr const int AUTH_TOKEN_AES_CMAC_SIZE = 16;
+constexpr const int AUTH_TOKEN_MAX_SIZE = AUTH_TOKEN_HMAC_SHA_SIZE;
 
 using EncryptCipherDomainId = int64_t;
-using EncryptCipherDomainName = StringRef;
+using EncryptCipherDomainNameRef = StringRef;
+using EncryptCipherDomainName = Standalone<EncryptCipherDomainNameRef>;
 using EncryptCipherBaseKeyId = uint64_t;
 using EncryptCipherRandomSalt = uint64_t;
 
+constexpr const EncryptCipherDomainId INVALID_ENCRYPT_DOMAIN_ID = -1;
+constexpr const EncryptCipherDomainId SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID = -2;
+constexpr const EncryptCipherDomainId ENCRYPT_HEADER_DOMAIN_ID = -3;
+constexpr const EncryptCipherDomainId FDB_DEFAULT_ENCRYPT_DOMAIN_ID = -4;
+
+constexpr const EncryptCipherBaseKeyId INVALID_ENCRYPT_CIPHER_KEY_ID = 0;
+
+constexpr const EncryptCipherRandomSalt INVALID_ENCRYPT_RANDOM_SALT = 0;
+
+extern const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
+extern const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
+extern const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME;
+
 typedef enum {
 	ENCRYPT_CIPHER_MODE_NONE = 0,
 	ENCRYPT_CIPHER_MODE_AES_256_CTR = 1,
@@ -73,6 +80,23 @@ typedef enum {
 static_assert(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_LAST <= std::numeric_limits<uint8_t>::max(),
               "EncryptHeaderAuthToken value overflow");
 
+typedef enum {
+	ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE = 0,
+	ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA = 1,
+	ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC = 2,
+	ENCRYPT_HEADER_AUTH_TOKEN_ALGO_LAST = 3 // Always the last element
+} EncryptAuthTokenAlgo;
+
+static_assert(EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_LAST <= std::numeric_limits<uint8_t>::max(),
+              "EncryptHeaerAuthTokenAlgo value overflow");
+
+bool isEncryptHeaderAuthTokenModeValid(const EncryptAuthTokenMode mode);
+bool isEncryptHeaderAuthTokenAlgoValid(const EncryptAuthTokenAlgo algo);
+bool isEncryptHeaderAuthTokenDetailsValid(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo);
+EncryptAuthTokenAlgo getAuthTokenAlgoFromMode(const EncryptAuthTokenMode mode);
+EncryptAuthTokenMode getRandomAuthTokenMode();
+EncryptAuthTokenAlgo getRandomAuthTokenAlgo();
+
 constexpr std::string_view ENCRYPT_DBG_TRACE_CACHED_PREFIX = "Chd";
 constexpr std::string_view ENCRYPT_DBG_TRACE_QUERY_PREFIX = "Qry";
 constexpr std::string_view ENCRYPT_DBG_TRACE_INSERT_PREFIX = "Ins";
@@ -91,4 +115,6 @@ std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix,
                                         int64_t refAfterTS,
                                         int64_t expAfterTS);
 
+int getEncryptHeaderAuthTokenSize(int algo);
+
 #endif
diff --git a/flow/include/flow/Error.h b/flow/include/flow/Error.h
index 1ebb8620ef..0e18b90bcc 100644
--- a/flow/include/flow/Error.h
+++ b/flow/include/flow/Error.h
@@ -79,6 +79,7 @@ private:
 };
 
 Error systemErrorCodeToError();
+extern const std::set<int> transactionRetryableErrors;
 
 #undef ERROR
 #define ERROR(name, number, description)                                                                               \
diff --git a/flow/include/flow/EventTypes.actor.h b/flow/include/flow/EventTypes.actor.h
index 95abffff0f..ce6bee51ab 100644
--- a/flow/include/flow/EventTypes.actor.h
+++ b/flow/include/flow/EventTypes.actor.h
@@ -22,11 +22,11 @@
 
 // When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
 // version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_EVENTTYPES_ACTOR_G_H)
-#define FDBCLIENT_EVENTTYPES_ACTOR_G_H
+#if defined(NO_INTELLISENSE) && !defined(FLOW_EVENTTYPES_ACTOR_G_H)
+#define FLOW_EVENTTYPES_ACTOR_G_H
 #include "flow/EventTypes.actor.g.h"
-#elif !defined(FDBCLIENT_EVENTTYPES_ACTOR_H)
-#define FDBCLIENT_EVENTTYPESS_ACTOR_H
+#elif !defined(FLOW_EVENTTYPES_ACTOR_H)
+#define FLOW_EVENTTYPES_ACTOR_H
 
 #include "flow/flow.h"
 #include "flow/TDMetric.actor.h"
diff --git a/flow/include/flow/FastRef.h b/flow/include/flow/FastRef.h
index 3c65cc6002..2e49b34de2 100644
--- a/flow/include/flow/FastRef.h
+++ b/flow/include/flow/FastRef.h
@@ -24,6 +24,7 @@
 
 #include <atomic>
 #include <cstdint>
+#include <utility>
 
 // The thread safety this class provides is that it's safe to call addref and
 // delref on the same object concurrently in different threads. Subclass does
diff --git a/flow/include/flow/IRandom.h b/flow/include/flow/IRandom.h
index 1430fb018d..73d6b7f675 100644
--- a/flow/include/flow/IRandom.h
+++ b/flow/include/flow/IRandom.h
@@ -143,6 +143,7 @@ public:
 	virtual UID randomUniqueID() = 0;
 	virtual char randomAlphaNumeric() = 0;
 	virtual std::string randomAlphaNumeric(int length) = 0;
+	virtual void randomBytes(uint8_t* buf, int length) = 0;
 	virtual uint32_t randomSkewedUInt32(uint32_t min, uint32_t maxPlusOne) = 0;
 	virtual uint64_t peek() const = 0; // returns something that is probably different for different random states.
 	                                   // Deterministic (and idempotent) for a deterministic generator.
@@ -173,6 +174,19 @@ public:
 	}
 
 	bool coinflip() { return (this->random01() < 0.5); }
+
+	// Picks a number between 2^minExp and 2^maxExp, but uniformly distributed over exponential buckets 2^n - 2^n+1
+	// For example, randomExp(0, 4) would have a 25% chance of returning 1, a 25% chance of returning 2-3, a 25% chance
+	// of returning 4-7, and a 25% chance of returning 8-15
+	// Similar in Expected Value to doing 1 << randomInt(minExp, maxExp+1), except numbers returned aren't just powers
+	// of 2
+	int randomExp(int minExp, int maxExp) {
+		if (minExp == maxExp) { // N=2, case
+			return 1 << minExp;
+		}
+		int val = 1 << this->randomInt(minExp, maxExp);
+		return this->randomInt(val, val * 2);
+	}
 };
 
 extern FILE* randLog;
@@ -196,7 +210,4 @@ Reference<IRandom> nondeterministicRandom();
 // WARNING: This is not thread safe and must not be called from any other thread than the network thread!
 Reference<IRandom> debugRandom();
 
-// Populates a buffer with a random sequence of bytes
-void generateRandomData(uint8_t* buffer, int length);
-
 #endif
diff --git a/flow/include/flow/IRateControl.h b/flow/include/flow/IRateControl.h
index 28a2408537..bd2c0b8004 100644
--- a/flow/include/flow/IRateControl.h
+++ b/flow/include/flow/IRateControl.h
@@ -53,7 +53,13 @@ public:
 		// Replenish budget based on time since last update
 		double ts = now();
 		// returnUnused happens to do exactly what we want here
-		returnUnused((ts - m_last_update) / m_seconds * m_limit);
+		auto unused = double(m_limit) * (ts - m_last_update) / m_seconds;
+		if (unused >= double(std::numeric_limits<int>::max())) {
+			// prevent int overflow
+			m_budget = m_limit;
+		} else {
+			returnUnused(int(unused));
+		}
 		m_last_update = ts;
 		m_budget -= n;
 		// If budget is still >= 0 then it's safe to use the allowance right now.
diff --git a/flow/include/flow/Knobs.h b/flow/include/flow/Knobs.h
index 9cb074f5df..688d00703e 100644
--- a/flow/include/flow/Knobs.h
+++ b/flow/include/flow/Knobs.h
@@ -195,6 +195,12 @@ public:
 	int NETWORK_TEST_REQUEST_SIZE;
 	bool NETWORK_TEST_SCRIPT_MODE;
 
+	// Authorization
+	bool ALLOW_TOKENLESS_TENANT_ACCESS;
+	int PUBLIC_KEY_FILE_MAX_SIZE;
+	int PUBLIC_KEY_FILE_REFRESH_INTERVAL_SECONDS;
+	int MAX_CACHED_EXPIRED_TOKENS;
+
 	// AsyncFileCached
 	int64_t PAGE_CACHE_4K;
 	int64_t PAGE_CACHE_64K;
@@ -221,7 +227,9 @@ public:
 	// AsyncFileKAIO
 	int MAX_OUTSTANDING;
 	int MIN_SUBMIT;
-	double DISK_METRIC_LOGGING_INTERVAL;
+	double SQLITE_DISK_METRIC_LOGGING_INTERVAL;
+	double KAIO_LATENCY_LOGGING_INTERVAL;
+	int KAIO_LATENCY_SAMPLE_SIZE;
 
 	int PAGE_WRITE_CHECKSUM_HISTORY;
 	int DISABLE_POSIX_KERNEL_AIO;
@@ -235,6 +243,13 @@ public:
 	int LOW_PRIORITY_DELAY_COUNT;
 	double LOW_PRIORITY_MAX_DELAY;
 
+	// HTTP
+	int HTTP_READ_SIZE;
+	int HTTP_SEND_SIZE;
+	int HTTP_VERBOSE_LEVEL;
+	std::string HTTP_REQUEST_ID_HEADER;
+	bool HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT; // skip verify md5 checksum for 206 response
+
 	// IAsyncFile
 	int64_t INCREMENTAL_DELETE_TRUNCATE_AMOUNT;
 	double INCREMENTAL_DELETE_INTERVAL;
@@ -346,6 +361,13 @@ public:
 	// Encryption
 	int64_t ENCRYPT_CIPHER_KEY_CACHE_TTL;
 	int64_t ENCRYPT_KEY_REFRESH_INTERVAL;
+	double ENCRYPT_KEY_CACHE_LOGGING_INTERVAL;
+	double ENCRYPT_KEY_CACHE_LOGGING_SAMPLE_SIZE;
+	bool ENCRYPT_HEADER_AUTH_TOKEN_ENABLED;
+	int ENCRYPT_HEADER_AUTH_TOKEN_ALGO;
+
+	// Authorization
+	int TOKEN_CACHE_SIZE;
 
 	// RESTClient
 	int RESTCLIENT_MAX_CONNECTIONPOOL_SIZE;
diff --git a/flow/include/flow/MkCert.h b/flow/include/flow/MkCert.h
index 70b091fe12..3d7fd5cd19 100644
--- a/flow/include/flow/MkCert.h
+++ b/flow/include/flow/MkCert.h
@@ -39,7 +39,7 @@ void printPrivateKey(FILE* out, StringRef privateKeyPem);
 
 PrivateKey makeEcP256();
 
-PrivateKey makeRsa2048Bit();
+PrivateKey makeRsa4096Bit();
 
 struct Asn1EntryRef {
 	// field must match one of ASN.1 object short/long names: e.g. "C", "countryName", "CN", "commonName",
diff --git a/flow/include/flow/Net2Packet.h b/flow/include/flow/Net2Packet.h
index 5031a924b8..d9c10a3bcb 100644
--- a/flow/include/flow/Net2Packet.h
+++ b/flow/include/flow/Net2Packet.h
@@ -44,9 +44,8 @@ class UnsentPacketQueue : NonCopyable {
 public:
 	UnsentPacketQueue()
 	  : unsent_first(0), unsent_last(0),
-	    sendQueueLatencyHistogram(Histogram::getHistogram(LiteralStringRef("UnsentPacketQueue"),
-	                                                      LiteralStringRef("QueueWait"),
-	                                                      Histogram::Unit::microseconds)) {}
+	    sendQueueLatencyHistogram(
+	        Histogram::getHistogram("UnsentPacketQueue"_sr, "QueueWait"_sr, Histogram::Unit::microseconds)) {}
 
 	~UnsentPacketQueue() {
 		discardAll();
diff --git a/flow/include/flow/ObjectSerializer.h b/flow/include/flow/ObjectSerializer.h
index bd4cfe33e0..ab3a54a4b6 100644
--- a/flow/include/flow/ObjectSerializer.h
+++ b/flow/include/flow/ObjectSerializer.h
@@ -25,8 +25,15 @@
 #include "flow/ProtocolVersion.h"
 
 #include <unordered_map>
+#include <any>
 
-using ContextVariableMap = std::unordered_map<std::string_view, void*>;
+using ContextVariableMap = std::unordered_map<std::string_view, std::any>;
+
+template <class T>
+struct HasVariableMap_t : std::false_type {};
+
+template <class T>
+constexpr bool HasVariableMap = HasVariableMap_t<T>::value;
 
 template <class Ar>
 struct LoadContext {
@@ -53,6 +60,11 @@ struct LoadContext {
 	void addArena(Arena& arena) { arena = ar->arena(); }
 
 	LoadContext& context() { return *this; }
+
+	template <class Archiver = Ar>
+	std::enable_if_t<HasVariableMap<Archiver>, std::any&> variable(std::string_view name) {
+		return ar->variable(name);
+	}
 };
 
 template <class Ar, class Allocator>
@@ -90,7 +102,7 @@ public:
 			// Some file identifiers are changed in 7.0, so file identifier mismatches
 			// are expected during a downgrade from 7.0 to 6.3
 			bool expectMismatch = mProtocolVersion.get() >= ProtocolVersion(0x0FDB00B070000000LL) &&
-			                      currentProtocolVersion < ProtocolVersion(0x0FDB00B070000000LL);
+			                      currentProtocolVersion() < ProtocolVersion(0x0FDB00B070000000LL);
 			{
 				TraceEvent te(expectMismatch ? SevInfo : SevError, "MismatchedFileIdentifier");
 				if (expectMismatch) {
@@ -110,23 +122,9 @@ public:
 		deserialize(FileIdentifierFor<Item>::value, item);
 	}
 
-	template <class T>
-	bool variable(std::string_view name, T* val) {
-		auto p = variables->insert(std::make_pair(name, val));
-		return p.second;
-	}
+	std::any& variable(std::string_view name) { return variables->at(name); }
 
-	template <class T>
-	T& variable(std::string_view name) {
-		auto res = variables->at(name);
-		return *reinterpret_cast<T*>(res);
-	}
-
-	template <class T>
-	T const& variable(std::string_view name) const {
-		auto res = variables->at(name);
-		return *reinterpret_cast<T*>(res);
-	}
+	std::any const& variable(std::string_view name) const { return variables->at(name); }
 };
 
 class ObjectReader : public _ObjectReader<ObjectReader> {
@@ -267,6 +265,11 @@ private:
 	int size = 0;
 };
 
+template <>
+struct HasVariableMap_t<ObjectReader> : std::true_type {};
+template <>
+struct HasVariableMap_t<ArenaObjectReader> : std::true_type {};
+
 // this special case is needed - the code expects
 // Standalone<T> and T to be equivalent for serialization
 namespace detail {
diff --git a/flow/include/flow/ObjectSerializerTraits.h b/flow/include/flow/ObjectSerializerTraits.h
index d396bd5808..964c56e516 100644
--- a/flow/include/flow/ObjectSerializerTraits.h
+++ b/flow/include/flow/ObjectSerializerTraits.h
@@ -37,11 +37,6 @@ struct is_fb_function_t<T, typename std::enable_if<T::is_fb_visitor>::type> : st
 template <class T>
 constexpr bool is_fb_function = is_fb_function_t<T>::value;
 
-template <class Visitor, class... Items>
-typename std::enable_if<is_fb_function<Visitor>, void>::type serializer(Visitor& visitor, Items&... items) {
-	visitor(items...);
-}
-
 template <class... Ts>
 struct pack {};
 
@@ -61,6 +56,40 @@ struct index_impl<0, pack<T, Ts...>> {
 template <int i, class Pack>
 using index_t = typename index_impl<i, Pack>::type;
 
+template <class T, typename = void>
+struct fb_must_appear_last_t : std::false_type {};
+
+template <class T>
+struct fb_must_appear_last_t<T, typename std::enable_if<T::fb_must_appear_last>::type>
+  : std::conditional_t<T::fb_must_appear_last, std::true_type, std::false_type> {};
+
+template <class T>
+constexpr bool fb_must_appear_last = fb_must_appear_last_t<T>::value;
+
+template <class Item, class... Items>
+constexpr bool fb_appears_last_property_helper(pack<Item, Items...>) {
+	if constexpr (sizeof...(Items) == 0) {
+		return true;
+	} else {
+		return !fb_must_appear_last<Item> && fb_appears_last_property_helper(pack<Items...>{});
+	}
+}
+template <class... Items>
+constexpr bool fb_appears_last_property(pack<Items...>) {
+	if constexpr (sizeof...(Items) == 0) {
+		return true;
+	} else {
+		return fb_appears_last_property_helper(pack<Items...>{});
+	}
+}
+
+template <class Visitor, class... Items>
+typename std::enable_if<is_fb_function<Visitor>, void>::type serializer(Visitor& visitor, Items&... items) {
+	static_assert(fb_appears_last_property(pack<Items...>{}),
+	              "An argument to a serializer call that must appear last (Arena?) does not appear last");
+	visitor(items...);
+}
+
 template <class T, typename = void>
 struct scalar_traits : std::false_type {
 	constexpr static size_t size = 0;
diff --git a/flow/include/flow/OwningResource.h b/flow/include/flow/OwningResource.h
new file mode 100644
index 0000000000..9635129733
--- /dev/null
+++ b/flow/include/flow/OwningResource.h
@@ -0,0 +1,155 @@
+/*
+ * OwningResource.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FLOW_OWNING_REOSURCE_H
+#define FLOW_OWNING_REOSURCE_H
+
+#include "flow/FastRef.h"
+
+// Consider the following situation:
+//
+//   1. An ACTOR A0 allocates an object O
+//   2. A0 spawns another ACTOR A1, which depends on O
+//   3. A0 triggers A1 and then terminates, destroying O
+//   4. Since A1 is triggered by A0 while not knowing A0 is terminated and O is released, it would cause a SEGV error.
+//
+// In this header file, two classes
+//
+//   * ResourceOwningRef
+//   * ResourceWeakRef
+//
+// are provided. The ResourceOwningRef is the reference that "holds" the resource, When it is destructed, the resource
+// is also released; while ResourceWeakRef is the reference that "weakly holds" the resource. Before each access, it is
+// the user's responsibility to verify if the resource is still available, via the available() method.
+//
+// With the two classes, the issue above can be solved by:
+//
+//    1. A0 allocates the object O via ResourceOwningRef
+//    2. A0 forwards O to A1, via ResourceWeakRef
+//    3. Every time A1 accesses O, it will verify if the resource is still available.
+//    4. When A0 terminates, O is released and all ResourceWeakRef available() call will report the resource is not
+//       available anymore, preventing the SEGV error being raised.
+
+namespace details {
+
+// The class holding the pointer to the resource.
+// SOMEDAY: Think using std::unique_ptr
+template <typename T>
+struct Resource : public ReferenceCounted<Resource<T>>, NonCopyable {
+	T* resource;
+
+	Resource(T* resource_) : resource(resource_) {}
+	~Resource() { delete resource; }
+
+	void reset(T* resource_) {
+		delete resource;
+		resource = resource_;
+	}
+};
+
+template <typename T>
+class ResourceRef {
+protected:
+	Reference<Resource<T>> resourceRef;
+
+	ResourceRef(const Reference<Resource<T>>& ref) : resourceRef(ref) {}
+	ResourceRef(Reference<Resource<T>>&& ref) : resourceRef(std::move(ref)) {}
+	ResourceRef& operator=(const Reference<Resource<T>>& ref) {
+		resourceRef = ref.resourceRef;
+		return *this;
+	}
+	ResourceRef& operator=(Reference<Resource<T>>&& ref) {
+		resourceRef = std::move(ref);
+		return *this;
+	}
+
+	virtual ~ResourceRef() {}
+
+public:
+	// Retrieves the resource as a pointer
+	T* operator->() const noexcept { return resourceRef->resource; }
+
+	// Retrieves the resource as a reference
+	T& operator*() const {
+		if (!available()) {
+			throw internal_error();
+		} else {
+			return *(resourceRef->resource);
+		}
+	}
+
+	// Returns true if the resource is available, i.e. not nullptr
+	bool available() const noexcept { return resourceRef->resource != nullptr; }
+};
+
+} // namespace details
+
+// The class that holds a Reference to the details::Resource which holds the real object. If the instance is destroyed,
+// the object is destroyed, too.
+template <typename T>
+class ResourceOwningRef : public details::ResourceRef<T>, NonCopyable {
+	template <typename U>
+	friend class ResourceWeakRef;
+
+	template <typename U>
+	friend class ActorWeakSelfRef;
+
+public:
+	ResourceOwningRef(T* resource) : details::ResourceRef<T>(makeReference<details::Resource<T>>(resource)) {}
+	virtual ~ResourceOwningRef() { details::ResourceRef<T>::resourceRef->reset(nullptr); }
+};
+
+// The class that weakly holds a Reference to the details::Resource. Destroying the reference will have no impact to the
+// real object. On the other hand, each time accessing the object requires a verification that the object is still alive
+template <typename T>
+class ResourceWeakRef : public details::ResourceRef<T> {
+public:
+	ResourceWeakRef(const ResourceOwningRef<T>& ref) : details::ResourceRef<T>(ref.resourceRef) {}
+	ResourceWeakRef(const ResourceWeakRef& ref) : details::ResourceRef<T>(ref.resourceRef) {}
+};
+
+// A unique reference that takes the ownership of the self object. The self object is widely used as the "global"
+// context of each role.
+template <typename T>
+using ActorOwningSelfRef = ResourceOwningRef<T>;
+
+// A wrapper of ResourceWeakRef, used to forward the widely used `self` pointer from the core ACTOR to other ACTORs. It
+// will check the resource before returning it. If the resource is not available, an operation_cancelled error will be
+// thrown to terminate the current ACTOR.
+template <typename T>
+class ActorWeakSelfRef : public ResourceWeakRef<T> {
+public:
+	ActorWeakSelfRef(const ResourceOwningRef<T>& ref) : ResourceWeakRef<T>(ref) {}
+	ActorWeakSelfRef(const ResourceWeakRef<T>& ref) : ResourceWeakRef<T>(ref) {}
+	ActorWeakSelfRef(const ActorWeakSelfRef<T>& ref)
+	  : ResourceWeakRef<T>(static_cast<const ResourceWeakRef<T>&>(ref)) {}
+
+	// Retrieves the resource as a pointer, throws operation_cancelled if the resource is not available
+	T* operator->() const {
+		if (!ResourceWeakRef<T>::available())
+			throw operation_cancelled();
+		return ResourceWeakRef<T>::resourceRef->resource;
+	}
+
+	// Gets the reference to the resource, Throws operation_cancelled if the resource is not available
+	T& operator*() const { return *(this->operator->()); }
+};
+
+#endif // FLOW_OWNING_REOSURCE_H
\ No newline at end of file
diff --git a/flow/include/flow/Platform.h b/flow/include/flow/Platform.h
index 17dfff2650..d43949a8ed 100644
--- a/flow/include/flow/Platform.h
+++ b/flow/include/flow/Platform.h
@@ -143,6 +143,7 @@ inline static T& makeDependent(T& value) {
 	return value;
 }
 
+#include <map>
 #include <string>
 #include <vector>
 
@@ -244,6 +245,16 @@ double getProcessorTimeThread();
 
 double getProcessorTimeProcess();
 
+#ifdef __linux__
+namespace linux_os {
+
+// Collects the /sys/fs/cgroup/cpu,cpuacct/cpu.stat information and returns the content
+// For more information about cpu,cpuacct, check manpages for cgroup
+std::map<std::string, int64_t> reportCGroupCpuStat();
+
+} // namespace linux_os
+#endif // __linux__
+
 uint64_t getMemoryUsage();
 
 uint64_t getResidentMemoryUsage();
@@ -282,6 +293,10 @@ void getLocalTime(const time_t* timep, struct tm* result);
 // get GMT time string from an epoch seconds double
 std::string epochsToGMTString(double epochs);
 
+#define ENVIRONMENT_KNOB_OPTION_PREFIX "FDB_KNOB_"
+// returns list of environment variables with prefix ENVIRONMENT_KNOB_OPTION_PREFIX
+std::vector<std::string> getEnvironmentKnobOptions();
+
 void setMemoryQuota(size_t limit);
 
 void* allocate(size_t length, bool allowLargePages, bool includeGuardPages);
@@ -312,11 +327,12 @@ void renameFile(std::string const& fromPath, std::string const& toPath);
 void atomicReplace(std::string const& path, std::string const& content, bool textmode = true);
 
 // Read a file into memory
+// This requires the file to be seekable
 std::string readFileBytes(std::string const& filename, int maxSize);
 
 // Read a file into memory supplied by the caller
 // If 'len' is greater than file size, then read the filesize bytes.
-void readFileBytes(std::string const& filename, uint8_t* buff, int64_t len);
+size_t readFileBytes(std::string const& filename, uint8_t* buff, int64_t len);
 
 // Write data buffer into file
 void writeFileBytes(std::string const& filename, const char* data, size_t count);
@@ -662,7 +678,7 @@ inline static void flushOutputStreams() {
 #error Missing symbol export
 #endif
 
-#define crashAndDie() (*(volatile int*)0 = 0)
+#define crashAndDie() std::abort()
 
 #ifdef _WIN32
 #define strcasecmp stricmp
@@ -768,7 +784,7 @@ int64_t getNumProfilesDeferred();
 int64_t getNumProfilesOverflowed();
 int64_t getNumProfilesCaptured();
 
-#else
+#else // __cplusplus
 #define EXTERNC
 #endif // __cplusplus
 
@@ -790,7 +806,14 @@ EXTERNC void flushAndExit(int exitCode);
 // Initilization code that's run at the beginning of every entry point (except fdbmonitor)
 void platformInit();
 
+// Register a callback which will run as part of the crash handler. Use in conjunction with registerCrashHandler.
+// The callback being added should be simple and unlikely to fail, otherwise it will fail the crash handler,
+// preventing necessary logging being printed. Also, the crash handler may not be comprehensive in handling all
+// failure cases.
+void registerCrashHandlerCallback(void (*f)());
+
 void registerCrashHandler();
+
 void setupRunLoopProfiler();
 EXTERNC void setProfilingEnabled(int enabled);
 
diff --git a/flow/include/flow/SignalSafeUnwind.h b/flow/include/flow/SignalSafeUnwind.h
new file mode 100644
index 0000000000..2d17b6637a
--- /dev/null
+++ b/flow/include/flow/SignalSafeUnwind.h
@@ -0,0 +1,30 @@
+/*
+ * SignalSafeUnwind.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FLOW_SIGNAL_SAFE_UNWIND
+#define FLOW_SIGNAL_SAFE_UNWIND
+#pragma once
+
+#include "flow/Platform.h"
+
+// This can be used by tests to measure the number of calls to dl_iterate_phdr intercepted
+extern int64_t dl_iterate_phdr_calls;
+
+#endif
\ No newline at end of file
diff --git a/flow/include/flow/StreamCipher.h b/flow/include/flow/StreamCipher.h
index cc7df84e47..6a64593fb6 100644
--- a/flow/include/flow/StreamCipher.h
+++ b/flow/include/flow/StreamCipher.h
@@ -53,7 +53,7 @@ public:
 	int size() const { return keySize; }
 	uint8_t* data() const { return arr.get(); }
 	void initializeKey(uint8_t* data, int len);
-	void initializeRandomTestKey() { generateRandomData(arr.get(), keySize); }
+	void initializeRandomTestKey() { deterministicRandom()->randomBytes(arr.get(), keySize); }
 	void reset() { memset(arr.get(), 0, keySize); }
 
 	static bool isGlobalKeyPresent();
@@ -104,8 +104,5 @@ class HmacSha256StreamCipher final : NonCopyable, public ReferenceCounted<HmacSh
 
 public:
 	HmacSha256StreamCipher();
-	StringRef digest(unsigned char const* data, int len, Arena&);
 	StringRef finish(Arena&);
 };
-
-void applyHmacKeyDerivationFunc(StreamCipherKey* cipherKey, HmacSha256StreamCipher* hmacGenerator, Arena& arena);
diff --git a/flow/include/flow/SystemMonitor.h b/flow/include/flow/SystemMonitor.h
index 528b659807..d6ba0d540a 100644
--- a/flow/include/flow/SystemMonitor.h
+++ b/flow/include/flow/SystemMonitor.h
@@ -92,53 +92,46 @@ struct NetworkData {
 	double countReactTime;
 
 	void init() {
-		bytesSent = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.BytesSent"));
-		countPacketsReceived = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountPacketsReceived"));
-		countPacketsGenerated = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountPacketsGenerated"));
-		bytesReceived = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.BytesReceived"));
-		countWriteProbes = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountWriteProbes"));
-		countReadProbes = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountReadProbes"));
-		countReads = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountReads"));
-		countWouldBlock = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountWouldBlock"));
-		countWrites = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountWrites"));
-		countRunLoop = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountRunLoop"));
-		countCantSleep = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountCantSleep"));
-		countWontSleep = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountWontSleep"));
-		countTimers = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountTimers"));
-		countTasks = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountTasks"));
-		countYields = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountYields"));
-		countYieldBigStack = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountYieldBigStack"));
-		countYieldCalls = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountYieldCalls"));
-		countASIOEvents = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountASIOEvents"));
-		countYieldCallsTrue = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountYieldCallsTrue"));
-		countRunLoopProfilingSignals =
-		    Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountRunLoopProfilingSignals"));
-		countConnEstablished = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountConnEstablished"));
-		countConnClosedWithError = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountConnClosedWithError"));
-		countConnClosedWithoutError =
-		    Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountConnClosedWithoutError"));
-		countTLSPolicyFailures = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountTLSPolicyFailures"));
-		countLaunchTime = DoubleMetric::getValueOrDefault(LiteralStringRef("Net2.CountLaunchTime"));
-		countReactTime = DoubleMetric::getValueOrDefault(LiteralStringRef("Net2.CountReactTime"));
-		countFileLogicalWrites = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountLogicalWrites"));
-		countFileLogicalReads = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountLogicalReads"));
-		countAIOSubmit = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountAIOSubmit"));
-		countAIOCollect = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountAIOCollect"));
-		countFileCacheWrites = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountCacheWrites"));
-		countFileCacheReads = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountCacheReads"));
-		countFileCacheWritesBlocked =
-		    Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountCacheWritesBlocked"));
-		countFileCacheReadsBlocked =
-		    Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountCacheReadsBlocked"));
-		countFileCachePageReadsMerged =
-		    Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountCachePageReadsMerged"));
-		countFileCacheFinds = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountCacheFinds"));
-		countFileCacheReadBytes = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountCacheReadBytes"));
-		countFilePageCacheHits = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountCachePageReadsHit"));
-		countFilePageCacheMisses =
-		    Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountCachePageReadsMissed"));
-		countFilePageCacheEvictions =
-		    Int64Metric::getValueOrDefault(LiteralStringRef("EvictablePageCache.CacheEvictions"));
+		bytesSent = Int64Metric::getValueOrDefault("Net2.BytesSent"_sr);
+		countPacketsReceived = Int64Metric::getValueOrDefault("Net2.CountPacketsReceived"_sr);
+		countPacketsGenerated = Int64Metric::getValueOrDefault("Net2.CountPacketsGenerated"_sr);
+		bytesReceived = Int64Metric::getValueOrDefault("Net2.BytesReceived"_sr);
+		countWriteProbes = Int64Metric::getValueOrDefault("Net2.CountWriteProbes"_sr);
+		countReadProbes = Int64Metric::getValueOrDefault("Net2.CountReadProbes"_sr);
+		countReads = Int64Metric::getValueOrDefault("Net2.CountReads"_sr);
+		countWouldBlock = Int64Metric::getValueOrDefault("Net2.CountWouldBlock"_sr);
+		countWrites = Int64Metric::getValueOrDefault("Net2.CountWrites"_sr);
+		countRunLoop = Int64Metric::getValueOrDefault("Net2.CountRunLoop"_sr);
+		countCantSleep = Int64Metric::getValueOrDefault("Net2.CountCantSleep"_sr);
+		countWontSleep = Int64Metric::getValueOrDefault("Net2.CountWontSleep"_sr);
+		countTimers = Int64Metric::getValueOrDefault("Net2.CountTimers"_sr);
+		countTasks = Int64Metric::getValueOrDefault("Net2.CountTasks"_sr);
+		countYields = Int64Metric::getValueOrDefault("Net2.CountYields"_sr);
+		countYieldBigStack = Int64Metric::getValueOrDefault("Net2.CountYieldBigStack"_sr);
+		countYieldCalls = Int64Metric::getValueOrDefault("Net2.CountYieldCalls"_sr);
+		countASIOEvents = Int64Metric::getValueOrDefault("Net2.CountASIOEvents"_sr);
+		countYieldCallsTrue = Int64Metric::getValueOrDefault("Net2.CountYieldCallsTrue"_sr);
+		countRunLoopProfilingSignals = Int64Metric::getValueOrDefault("Net2.CountRunLoopProfilingSignals"_sr);
+		countConnEstablished = Int64Metric::getValueOrDefault("Net2.CountConnEstablished"_sr);
+		countConnClosedWithError = Int64Metric::getValueOrDefault("Net2.CountConnClosedWithError"_sr);
+		countConnClosedWithoutError = Int64Metric::getValueOrDefault("Net2.CountConnClosedWithoutError"_sr);
+		countTLSPolicyFailures = Int64Metric::getValueOrDefault("Net2.CountTLSPolicyFailures"_sr);
+		countLaunchTime = DoubleMetric::getValueOrDefault("Net2.CountLaunchTime"_sr);
+		countReactTime = DoubleMetric::getValueOrDefault("Net2.CountReactTime"_sr);
+		countFileLogicalWrites = Int64Metric::getValueOrDefault("AsyncFile.CountLogicalWrites"_sr);
+		countFileLogicalReads = Int64Metric::getValueOrDefault("AsyncFile.CountLogicalReads"_sr);
+		countAIOSubmit = Int64Metric::getValueOrDefault("AsyncFile.CountAIOSubmit"_sr);
+		countAIOCollect = Int64Metric::getValueOrDefault("AsyncFile.CountAIOCollect"_sr);
+		countFileCacheWrites = Int64Metric::getValueOrDefault("AsyncFile.CountCacheWrites"_sr);
+		countFileCacheReads = Int64Metric::getValueOrDefault("AsyncFile.CountCacheReads"_sr);
+		countFileCacheWritesBlocked = Int64Metric::getValueOrDefault("AsyncFile.CountCacheWritesBlocked"_sr);
+		countFileCacheReadsBlocked = Int64Metric::getValueOrDefault("AsyncFile.CountCacheReadsBlocked"_sr);
+		countFileCachePageReadsMerged = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsMerged"_sr);
+		countFileCacheFinds = Int64Metric::getValueOrDefault("AsyncFile.CountCacheFinds"_sr);
+		countFileCacheReadBytes = Int64Metric::getValueOrDefault("AsyncFile.CountCacheReadBytes"_sr);
+		countFilePageCacheHits = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsHit"_sr);
+		countFilePageCacheMisses = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsMissed"_sr);
+		countFilePageCacheEvictions = Int64Metric::getValueOrDefault("EvictablePageCache.CacheEvictions"_sr);
 	}
 };
 
diff --git a/flow/include/flow/TDMetric.actor.h b/flow/include/flow/TDMetric.actor.h
index f100da0007..5adceede15 100644
--- a/flow/include/flow/TDMetric.actor.h
+++ b/flow/include/flow/TDMetric.actor.h
@@ -171,12 +171,12 @@ inline StringRef metricTypeName() {
 #define MAKE_TYPENAME(T, S)                                                                                            \
 	template <>                                                                                                        \
 	inline StringRef metricTypeName<T>() {                                                                             \
-		return LiteralStringRef(S);                                                                                    \
+		return S;                                                                                                      \
 	}
-MAKE_TYPENAME(bool, "Bool")
-MAKE_TYPENAME(int64_t, "Int64")
-MAKE_TYPENAME(double, "Double")
-MAKE_TYPENAME(Standalone<StringRef>, "String")
+MAKE_TYPENAME(bool, "Bool"_sr)
+MAKE_TYPENAME(int64_t, "Int64"_sr)
+MAKE_TYPENAME(double, "Double"_sr)
+MAKE_TYPENAME(Standalone<StringRef>, "String"_sr)
 #undef MAKE_TYPENAME
 
 struct BaseMetric;
@@ -358,7 +358,7 @@ struct Descriptor {
 	using fields = std::tuple<>;
 	typedef make_index_sequence_impl<0, index_sequence<>, std::tuple_size<fields>::value>::type field_indexes;
 
-	static StringRef typeName() { return LiteralStringRef(""); }
+	static StringRef typeName() { return ""_sr; }
 #endif
 };
 
@@ -430,9 +430,7 @@ struct FieldValueBlockEncoding<double> {
 
 template <>
 struct FieldValueBlockEncoding<bool> {
-	inline void write(BinaryWriter& w, bool v) {
-		w.serializeBytes(v ? LiteralStringRef("\x01") : LiteralStringRef("\x00"));
-	}
+	inline void write(BinaryWriter& w, bool v) { w.serializeBytes(v ? "\x01"_sr : "\x00"_sr); }
 	bool read(BinaryReader& r) {
 		uint8_t* v = (uint8_t*)r.readBytes(sizeof(uint8_t));
 		return *v != 0;
@@ -716,7 +714,7 @@ struct MakeEventField {
 };
 
 struct TimeDescriptor {
-	static StringRef name() { return LiteralStringRef("Time"); }
+	static StringRef name() { return "Time"_sr; }
 };
 
 struct BaseMetric {
diff --git a/flow/include/flow/TLSConfig.actor.h b/flow/include/flow/TLSConfig.actor.h
index 248f9aa3cd..5963fb57e9 100644
--- a/flow/include/flow/TLSConfig.actor.h
+++ b/flow/include/flow/TLSConfig.actor.h
@@ -33,6 +33,8 @@
 #include <string>
 #include <vector>
 #include <boost/system/system_error.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/ssl.hpp>
 #include "flow/FastRef.h"
 #include "flow/Knobs.h"
 #include "flow/flow.h"
@@ -180,8 +182,7 @@ public:
 
 	// Return the explicitly set path.
 	// If one was not set, return the path from the environment.
-	// (Cert and Key only) If neither exist, check for fdb.pem in cwd
-	// (Cert and Key only) If fdb.pem doesn't exist, check for it in default config dir
+	// (Cert and Key only) If cert.pem or key.pem don't exist, check for them in default config dir
 	// Otherwise return the empty string.
 	// Theoretically, fileExists() can block, so these functions are labelled as synchronous
 	// TODO: make an easy to use Future<bool> fileExists, and port lots of code over to it.
@@ -201,21 +202,23 @@ private:
 	TLSEndpointType endpointType = TLSEndpointType::UNSET;
 };
 
-namespace boost {
-namespace asio {
-namespace ssl {
-struct context;
-}
-} // namespace asio
-} // namespace boost
-void ConfigureSSLContext(
-    const LoadedTLSConfig& loaded,
-    boost::asio::ssl::context* context,
-    std::function<void()> onPolicyFailure = []() {});
+class TLSPolicy;
+
+void ConfigureSSLContext(const LoadedTLSConfig& loaded, boost::asio::ssl::context& context);
+
+// Set up SSL for stream object based on policy.
+// Optionally arm a callback that gets called with verify-outcome of each cert in peer certificate chain:
+// e.g. for peer with a valid, trusted length-3 certificate chain (root CA, intermediate CA, and server certs),
+// callback(true) will be called 3 times.
+void ConfigureSSLStream(Reference<TLSPolicy> policy,
+                        boost::asio::ssl::stream<boost::asio::ip::tcp::socket&>& stream,
+                        std::function<void(bool)> callback);
 
 class TLSPolicy : ReferenceCounted<TLSPolicy> {
+	void set_verify_peers(std::vector<std::string> verify_peers);
+
 public:
-	TLSPolicy(TLSEndpointType client) : is_client(client == TLSEndpointType::CLIENT) {}
+	TLSPolicy(const LoadedTLSConfig& loaded, std::function<void()> on_failure);
 	virtual ~TLSPolicy();
 
 	virtual void addref() { ReferenceCounted<TLSPolicy>::addref(); }
@@ -223,7 +226,6 @@ public:
 
 	static std::string ErrorString(boost::system::error_code e);
 
-	void set_verify_peers(std::vector<std::string> verify_peers);
 	bool verify_peer(bool preverified, X509_STORE_CTX* store_ctx);
 
 	std::string toString() const;
@@ -242,6 +244,7 @@ public:
 	};
 
 	std::vector<Rule> rules;
+	std::function<void()> on_failure;
 	bool is_client;
 };
 
diff --git a/flow/include/flow/ThreadHelper.actor.h b/flow/include/flow/ThreadHelper.actor.h
index 8b3a1be341..ea074f7bc7 100644
--- a/flow/include/flow/ThreadHelper.actor.h
+++ b/flow/include/flow/ThreadHelper.actor.h
@@ -98,8 +98,9 @@ struct ThreadCallback {
 	virtual void error(const Error&, int& userParam) = 0;
 	virtual ThreadCallback* addCallback(ThreadCallback* cb);
 
-	virtual void clearCallback(ThreadCallback* cb) {
+	virtual bool clearCallback(ThreadCallback* cb) {
 		// If this is the only registered callback this will be called with (possibly) arbitrary pointers
+		return false;
 	}
 
 	// Note that when a ThreadCallback is destroyed it must have no MultiCallbackHolders, but this can't be
@@ -187,10 +188,10 @@ public:
 		return (ThreadCallback*)this;
 	}
 
-	void clearCallback(ThreadCallback* callback) override {
+	bool clearCallback(ThreadCallback* callback) override {
 		MultiCallbackHolder* h = callback->getHolder(this);
 		if (h == nullptr) {
-			return;
+			return false;
 		}
 
 		UNSTOPPABLE_ASSERT(h->index < callbacks.size() && h->index >= 0);
@@ -204,6 +205,7 @@ public:
 
 		callbacks.pop_back();
 		callback->destroyHolder(h);
+		return true;
 	}
 
 	bool canFire(int notMadeActive) const override { return true; }
@@ -424,14 +426,16 @@ public:
 
 		// Only clear the callback if it belongs to the caller, because
 		// another actor could be waiting on it now!
-		if (callback == cb)
+		bool cleared = false;
+		if (callback == cb) {
 			callback = nullptr;
-		else if (callback != nullptr)
-			callback->clearCallback(cb);
+			cleared = true;
+		} else if (callback != nullptr) {
+			cleared = callback->clearCallback(cb);
+		}
 
 		this->mutex.leave();
-
-		return true;
+		return cleared;
 	}
 
 	void setCancel(Future<Void>&& cf) { cancelFuture = std::move(cf); }
diff --git a/flow/include/flow/Trace.h b/flow/include/flow/Trace.h
index f153f144de..6e3b01ce63 100644
--- a/flow/include/flow/Trace.h
+++ b/flow/include/flow/Trace.h
@@ -256,6 +256,7 @@ FORMAT_TRACEABLE(long int, "%ld");
 FORMAT_TRACEABLE(unsigned long int, "%lu");
 FORMAT_TRACEABLE(long long int, "%lld");
 FORMAT_TRACEABLE(unsigned long long int, "%llu");
+FORMAT_TRACEABLE(float, "%g");
 FORMAT_TRACEABLE(double, "%g");
 FORMAT_TRACEABLE(void*, "%p");
 FORMAT_TRACEABLE(volatile long, "%ld");
@@ -513,7 +514,7 @@ public:
 
 	void log();
 
-	void disable() { enabled = false; } // Disables the trace event so it doesn't get
+	void disable() { enabled = false; } // Disables the trace event so it doesn't get logged
 
 	virtual ~BaseTraceEvent(); // Actually logs the event
 
@@ -583,7 +584,7 @@ private:
 class StringRef;
 
 struct TraceInterval {
-	TraceInterval(const char* type) : type(type), count(-1), severity(SevInfo) {}
+	TraceInterval(const char* type, UID id = UID()) : type(type), pairID(id), count(-1), severity(SevInfo) {}
 
 	TraceInterval& begin();
 	TraceInterval& end() { return *this; }
@@ -661,6 +662,9 @@ void removeTraceRole(std::string const& role);
 void retrieveTraceLogIssues(std::set<std::string>& out);
 void setTraceLogGroup(const std::string& role);
 void addUniversalTraceField(std::string const& name, std::string const& value);
+void setTraceLocalAddress(const NetworkAddress& addr);
+void disposeTraceFileWriter();
+std::string getTraceFormatExtension();
 uint64_t getTraceThreadId();
 
 template <class T>
diff --git a/flow/include/flow/Util.h b/flow/include/flow/Util.h
index 727065a4cf..2dfd0b1540 100644
--- a/flow/include/flow/Util.h
+++ b/flow/include/flow/Util.h
@@ -20,9 +20,48 @@
 
 #ifndef _FLOW_UTIL_H_
 #define _FLOW_UTIL_H_
+
 #pragma once
 
 #include <algorithm>
+#include <functional>
+#include <iosfwd>
+
+// Read key/value pairs from stream. The stream is constituted by lines of text.
+// Each line contains a pair of key/value, separated by space/tab. e.g.
+//
+// Key1   Value1      tailing characters
+// Key2   Value2      tailing characters
+//
+// The tailing characters will be ignored.
+//
+// K and V should have
+//
+//   std::istream& operator>>(std::istream&, K&);
+//
+// implemented.
+template <typename K, typename V>
+void keyValueReader(std::istream& stream, std::function<bool(const K&, const V&)> consumer) {
+	std::stringstream lineParser;
+	std::string line;
+	K key;
+	V value;
+	while (std::getline(stream, line)) {
+		lineParser.clear();
+		lineParser.str(std::move(line));
+		try {
+			lineParser >> key >> value;
+		} catch (std::ios_base::failure&) {
+			continue;
+		}
+		if (lineParser.fail() || lineParser.bad()) {
+			continue;
+		}
+		if (!consumer(key, value)) {
+			break;
+		}
+	}
+}
 
 template <typename C>
 void swapAndPop(C* container, int index) {
diff --git a/flow/include/flow/WatchFile.actor.h b/flow/include/flow/WatchFile.actor.h
new file mode 100644
index 0000000000..bad2c58e14
--- /dev/null
+++ b/flow/include/flow/WatchFile.actor.h
@@ -0,0 +1,77 @@
+/*
+ * WatchFile.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+// When actually compiled (NO_INTELLISENSE), include the generated
+// version of this file.  In intellisense use the source version.
+#if defined(NO_INTELLISENSE) && !defined(FLOW_WATCH_FILE_ACTOR_G_H)
+#define FLOW_WATCH_FILE_ACTOR_G_H
+#include "flow/WatchFile.actor.g.h"
+#elif !defined(FLOW_WATCH_FILE_ACTOR_H)
+#define FLOW_WATCH_FILE_ACTOR_H
+
+#include <ctime>
+#include <string>
+#include "flow/IAsyncFile.h"
+#include "flow/genericactors.actor.h"
+#include "flow/actorcompiler.h"
+
+ACTOR static Future<Void> watchFileForChanges(std::string filename,
+                                              AsyncTrigger* fileChanged,
+                                              const int* intervalSeconds,
+                                              const char* errorType) {
+	if (filename == "") {
+		return Never();
+	}
+	state bool firstRun = true;
+	state bool statError = false;
+	state std::time_t lastModTime = 0;
+	loop {
+		try {
+			std::time_t modtime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename));
+			if (firstRun) {
+				lastModTime = modtime;
+				firstRun = false;
+			}
+			if (lastModTime != modtime || statError) {
+				lastModTime = modtime;
+				statError = false;
+				fileChanged->trigger();
+			}
+		} catch (Error& e) {
+			if (e.code() == error_code_io_error) {
+				// EACCES, ELOOP, ENOENT all come out as io_error(), but are more of a system
+				// configuration issue than an FDB problem.  If we managed to load valid
+				// certificates, then there's no point in crashing, but we should complain
+				// loudly.  IAsyncFile will log the error, but not necessarily as a warning.
+				TraceEvent(SevWarnAlways, errorType).detail("File", filename);
+				statError = true;
+			} else {
+				throw;
+			}
+		}
+		wait(delay(*intervalSeconds));
+	}
+}
+
+#include "flow/unactorcompiler.h"
+
+#endif // FLOW_WATCH_FILE_ACTOR_H
diff --git a/flow/include/flow/error_definitions.h b/flow/include/flow/error_definitions.h
index 519a0cb8b0..e967caead3 100755
--- a/flow/include/flow/error_definitions.h
+++ b/flow/include/flow/error_definitions.h
@@ -65,7 +65,7 @@ ERROR( database_locked, 1038, "Database is locked" )
 ERROR( cluster_version_changed, 1039, "The protocol version of the cluster has changed" )
 ERROR( external_client_already_loaded, 1040, "External client has already been loaded" )
 ERROR( lookup_failed, 1041, "DNS lookup failed" )
-ERROR( proxy_memory_limit_exceeded, 1042, "CommitProxy commit memory limit exceeded" )
+ERROR( commit_proxy_memory_limit_exceeded, 1042, "CommitProxy commit memory limit exceeded" )
 ERROR( shutdown_in_progress, 1043, "Operation no longer supported due to shutdown" )
 ERROR( serialization_failed, 1044, "Failed to deserialize an object" )
 ERROR( connection_unreferenced, 1048, "No peer references for connection" )
@@ -97,6 +97,10 @@ ERROR( unexpected_encoding_type, 1073, "Page content decoding failed" )
 ERROR( encryption_key_not_found, 1074, "Encryption key not found" )
 ERROR( data_move_cancelled, 1075, "Data move was cancelled" )
 ERROR( data_move_dest_team_not_found, 1076, "Dest team was not found for data move" )
+ERROR( blob_worker_full, 1077, "Blob worker cannot take on more granule assignments" )
+ERROR( grv_proxy_memory_limit_exceeded, 1078, "GetReadVersion proxy memory limit exceeded" )
+ERROR( blob_granule_request_failed, 1079, "BlobGranule request failed" )
+ERROR( storage_too_many_feed_streams, 1080, "Too many feed streams to a single storage server" )
 
 ERROR( broken_promise, 1100, "Broken promise" )
 ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" )
@@ -123,7 +127,7 @@ ERROR( dd_tracker_cancelled, 1215, "The data distribution tracker has been cance
 ERROR( failed_to_progress, 1216, "Process has failed to make sufficient progress" )
 ERROR( invalid_cluster_id, 1217, "Attempted to join cluster with a different cluster ID" )
 ERROR( restart_cluster_controller, 1218, "Restart cluster controller process" )
-ERROR( please_reboot_remote_kv_store, 1219, "Need to reboot the storage engine process as it died abnormally")
+ERROR( please_reboot_kv_store, 1219, "Need to reboot the storage engine")
 ERROR( incompatible_software_version, 1220, "Current software does not support database format" )
 
 // 15xx Platform errors
@@ -150,6 +154,7 @@ ERROR( http_bad_request_id, 1525, "HTTP response contained an unexpected X-Reque
 ERROR( rest_invalid_uri, 1526, "Invalid REST URI")
 ERROR( rest_invalid_rest_client_knob, 1527, "Invalid RESTClient knob")
 ERROR( rest_connectpool_key_not_found, 1528, "ConnectKey not found in connection pool")
+ERROR( lock_file_failure, 1529, "Unable to lock the file")
 
 
 // 2xxx Attempt (presumably by a _client_) to do something illegal.  If an error is known to
@@ -228,11 +233,27 @@ ERROR( tenant_name_required, 2130, "Tenant name must be specified to access data
 ERROR( tenant_not_found, 2131, "Tenant does not exist" )
 ERROR( tenant_already_exists, 2132, "A tenant with the given name already exists" )
 ERROR( tenant_not_empty, 2133, "Cannot delete a non-empty tenant" )
-ERROR( invalid_tenant_name, 2134, "Tenant name cannot begin with \\xff");
-ERROR( tenant_prefix_allocator_conflict, 2135, "The database already has keys stored at the prefix allocated for the tenant");
-ERROR( tenants_disabled, 2136, "Tenants have been disabled in the cluster");
-ERROR( unknown_tenant, 2137, "Tenant is not available from this server")
-ERROR( illegal_tenant_access, 2138, "Illegal tenant access")
+ERROR( invalid_tenant_name, 2134, "Tenant name cannot begin with \\xff" )
+ERROR( tenant_prefix_allocator_conflict, 2135, "The database already has keys stored at the prefix allocated for the tenant" )
+ERROR( tenants_disabled, 2136, "Tenants have been disabled in the cluster" )
+ERROR( unknown_tenant, 2137, "Tenant is not available from this server" )
+ERROR( illegal_tenant_access, 2138, "Illegal tenant access" )
+ERROR( invalid_tenant_group_name, 2139, "Tenant group name cannot begin with \\xff" )
+ERROR( invalid_tenant_configuration, 2140, "Tenant configuration is invalid" )
+ERROR( cluster_no_capacity, 2141, "Cluster does not have capacity to perform the specified operation" )
+ERROR( tenant_removed, 2142, "The tenant was removed" )
+ERROR( invalid_tenant_state, 2143, "Operation cannot be applied to tenant in its current state" )
+
+ERROR( invalid_cluster_name, 2160, "Data cluster name cannot begin with \\xff" )
+ERROR( invalid_metacluster_operation, 2161, "Metacluster operation performed on non-metacluster" )
+ERROR( cluster_already_exists, 2162, "A data cluster with the given name already exists" )
+ERROR( cluster_not_found, 2163, "Data cluster does not exist" )
+ERROR( cluster_not_empty, 2164, "Cluster must be empty" )
+ERROR( cluster_already_registered, 2165, "Data cluster is already registered with a metacluster" )
+ERROR( metacluster_no_capacity, 2166, "Metacluster does not have capacity to create new tenants" )
+ERROR( management_cluster_invalid_access, 2167, "Standard transactions cannot be run against the management cluster" )
+ERROR( tenant_creation_permanently_failed, 2168, "The tenant creation did not complete in a timely manner and has permanently failed" )
+ERROR( cluster_removed, 2169, "The cluster is being removed from the metacluster" )
 
 // 2200 - errors from bindings and official APIs
 ERROR( api_version_unset, 2200, "API version is not set" )
@@ -296,28 +317,28 @@ ERROR( json_malformed, 2401, "JSON string was malformed")
 ERROR( json_eof_expected, 2402, "JSON string did not terminate where expected")
 
 // 2500 - disk snapshot based backup errors
-ERROR( snap_disable_tlog_pop_failed,  2500, "Failed to disable tlog pops")
-ERROR( snap_storage_failed,  2501, "Failed to snapshot storage nodes")
-ERROR( snap_tlog_failed,  2502, "Failed to snapshot TLog nodes")
-ERROR( snap_coord_failed,  2503, "Failed to snapshot coordinator nodes")
-ERROR( snap_enable_tlog_pop_failed,  2504, "Failed to enable tlog pops")
-ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted")
-ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered")
-ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured")
-ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported")
-ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string")
+ERROR( snap_disable_tlog_pop_failed,  2500, "Failed to disable tlog pops" )
+ERROR( snap_storage_failed,  2501, "Failed to snapshot storage nodes" )
+ERROR( snap_tlog_failed,  2502, "Failed to snapshot TLog nodes" )
+ERROR( snap_coord_failed,  2503, "Failed to snapshot coordinator nodes" )
+ERROR( snap_enable_tlog_pop_failed,  2504, "Failed to enable tlog pops" )
+ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted" )
+ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered" )
+ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured" )
+ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported" )
+ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string" )
 
 // 27XX - Encryption operations errors
-ERROR( encrypt_ops_error, 2700, "Encryption operation error")
-ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch")
-ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing")
-ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired")
-ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch")
-ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key")
-ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details")
-ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed")
-ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc.")
-ERROR( encrypt_unsupported, 2709, "Encryption not supported")
+ERROR( encrypt_ops_error, 2700, "Encryption operation error" )
+ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch" )
+ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing" )
+ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired" )
+ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch" )
+ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key" )
+ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details" )
+ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed" )
+ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc." )
+ERROR( encrypt_unsupported, 2709, "Encryption not supported" )
 
 // 4xxx Internal errors (those that should be generated only by bugs) are decimal 4xxx
 ERROR( unknown_error, 4000, "An unknown error occurred" )  // C++ exception not of type Error
diff --git a/flow/include/flow/flow.h b/flow/include/flow/flow.h
index a3c156c9d1..3cc7470dbf 100644
--- a/flow/include/flow/flow.h
+++ b/flow/include/flow/flow.h
@@ -46,6 +46,7 @@
 #include <memory>
 #include <mutex>
 
+#include "flow/CodeProbe.h"
 #include "flow/Platform.h"
 #include "flow/FastAlloc.h"
 #include "flow/IRandom.h"
@@ -59,15 +60,9 @@
 #include <boost/version.hpp>
 
 #define TEST(condition)                                                                                                \
-	if (!(condition)) {                                                                                                \
-	} else {                                                                                                           \
-		static BaseTraceEvent* __test =                                                                                \
-		    &(TraceEvent(intToSeverity(FLOW_KNOBS->CODE_COV_TRACE_EVENT_SEVERITY), "CodeCoverage")                     \
-		          .detail("File", __FILE__)                                                                            \
-		          .detail("Line", __LINE__)                                                                            \
-		          .detail("Condition", #condition));                                                                   \
-		(void)__test;                                                                                                  \
-	}
+	do {                                                                                                               \
+		static_assert(false, "TEST macros are deprecated, please use CODE_PROBE instead");                             \
+	} while (false)
 
 /*
 usage:
@@ -1172,9 +1167,9 @@ public:
 	// stream.send( request )
 	//   Unreliable at most once delivery: Delivers request unless there is a connection failure (zero or one times)
 
-	void send(const T& value) const { queue->send(value); }
-	void send(T&& value) const { queue->send(std::move(value)); }
-	void sendError(const Error& error) const { queue->sendError(error); }
+	void send(const T& value) { queue->send(value); }
+	void send(T&& value) { queue->send(std::move(value)); }
+	void sendError(const Error& error) { queue->sendError(error); }
 
 	// stream.getReply( request )
 	//   Reliable at least once delivery: Eventually delivers request at least once and returns one of the replies if
@@ -1183,22 +1178,22 @@ public:
 	//   If a reply is returned, request was or will be delivered one or more times.
 	//   If cancelled, request was or will be delivered zero or more times.
 	template <class X>
-	Future<REPLY_TYPE(X)> getReply(const X& value) const {
+	Future<REPLY_TYPE(X)> getReply(const X& value) {
 		send(value);
 		return getReplyPromise(value).getFuture();
 	}
 	template <class X>
-	Future<REPLY_TYPE(X)> getReply(const X& value, TaskPriority taskID) const {
+	Future<REPLY_TYPE(X)> getReply(const X& value, TaskPriority taskID) {
 		setReplyPriority(value, taskID);
 		return getReplyPromise(value).getFuture();
 	}
 
 	template <class X>
-	Future<X> getReply() const {
+	Future<X> getReply() {
 		return getReply(Promise<X>());
 	}
 	template <class X>
-	Future<X> getReplyWithTaskID(TaskPriority taskID) const {
+	Future<X> getReplyWithTaskID(TaskPriority taskID) {
 		Promise<X> reply;
 		reply.getEndpoint(taskID);
 		return getReply(reply);
@@ -1370,5 +1365,7 @@ inline bool check_yield(TaskPriority taskID = TaskPriority::DefaultYield) {
 	return g_network->check_yield(taskID);
 }
 
+void bindDeterministicRandomToOpenssl();
+
 #include "flow/genericactors.actor.h"
 #endif
diff --git a/flow/include/flow/genericactors.actor.h b/flow/include/flow/genericactors.actor.h
index c548ee02c3..5aecc04215 100644
--- a/flow/include/flow/genericactors.actor.h
+++ b/flow/include/flow/genericactors.actor.h
@@ -83,7 +83,7 @@ Future<Optional<T>> stopAfter(Future<T> what) {
 		ret = Optional<T>(_);
 	} catch (Error& e) {
 		bool ok = e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete ||
-		          e.code() == error_code_actor_cancelled || e.code() == error_code_please_reboot_remote_kv_store;
+		          e.code() == error_code_actor_cancelled;
 		TraceEvent(ok ? SevInfo : SevError, "StopAfterError").error(e);
 		if (!ok) {
 			fprintf(stderr, "Fatal Error: %s\n", e.what());
@@ -224,14 +224,6 @@ Future<T> delayed(Future<T> what, double time = 0.0, TaskPriority taskID = TaskP
 	}
 }
 
-// wait <interval> then call what() in a loop forever
-ACTOR template <class Func>
-Future<Void> recurring(Func what, double interval, TaskPriority taskID = TaskPriority::DefaultDelay) {
-	loop choose {
-		when(wait(delay(interval, taskID))) { what(); }
-	}
-}
-
 ACTOR template <class Func>
 Future<Void> trigger(Func what, Future<Void> signal) {
 	wait(signal);
@@ -858,7 +850,7 @@ Future<Void> timeoutWarningCollector(FutureStream<Void> const& input,
                                      double const& logDelay,
                                      const char* const& context,
                                      UID const& id);
-Future<bool> quorumEqualsTrue(std::vector<Future<bool>> const& futures, int const& required);
+ACTOR Future<bool> quorumEqualsTrue(std::vector<Future<bool>> futures, int required);
 Future<Void> lowPriorityDelay(double const& waitTime);
 
 ACTOR template <class T>
@@ -1004,6 +996,11 @@ Future<Void> waitForAny(std::vector<Future<T>> const& results) {
 	return quorum(results, 1);
 }
 
+ACTOR Future<Void> waitForMost(std::vector<Future<ErrorOr<Void>>> futures,
+                               int faultTolerance,
+                               Error e,
+                               double waitMultiplierForSlowFutures = 1.0);
+
 ACTOR Future<bool> shortCircuitAny(std::vector<Future<bool>> f);
 
 ACTOR template <class T>
@@ -1199,6 +1196,48 @@ inline Future<Void> operator||(Future<Void> const& lhs, Future<Void> const& rhs)
 	return chooseActor(lhs, rhs);
 }
 
+// wait <interval> then call what() in a loop forever
+ACTOR template <class Func>
+Future<Void> recurring(Func what, double interval, TaskPriority taskID = TaskPriority::DefaultDelay) {
+	loop choose {
+		when(wait(delay(interval, taskID))) { what(); }
+	}
+}
+
+// Invoke actorFunc() forever in a loop
+// At least wait<interval> between two actor functor invocations
+ACTOR template <class F>
+Future<Void> recurringAsync(
+    F actorFunc, // Callback actor functor
+    double interval, // Interval between two subsequent invocations of actor functor.
+    bool absoluteIntervalDelay, // Flag guarantees "interval" delay between two subequent actor functor invocations. If
+                                // not selected, guarantees provided are "at least 'interval' delay" between two
+                                // subsequent actor functor invocations, however, due to either 'poorly choose' interval
+                                // value AND/OR actor functor taking longer than expected to return, could cause actor
+                                // functor to run with no-delay
+    double initialDelay, // Initial delay interval
+    TaskPriority taskID = TaskPriority::DefaultDelay) {
+
+	wait(delay(initialDelay));
+
+	state Future<Void> val;
+
+	loop {
+		val = actorFunc();
+
+		if (absoluteIntervalDelay) {
+			wait(val);
+			// Ensure subsequent actorFunc executions observe client supplied delay interval.
+			wait(delay(interval));
+		} else {
+			// Guarantee at-least client supplied interval delay; two possible scenarios:
+			// 1. The actorFunc executions finishes before 'interval' delay
+			// 2. The actorFunc executions takes > 'interval' delay.
+			wait(val && delay(interval));
+		}
+	}
+}
+
 ACTOR template <class T>
 Future<T> brokenPromiseToNever(Future<T> in) {
 	try {
@@ -1420,7 +1459,8 @@ private:
 			}
 			return Void();
 		} catch (...) {
-			TEST(true); // If we get cancelled here, we are holding the lock but the caller doesn't know, so release it
+			CODE_PROBE(true,
+			           "If we get cancelled here, we are holding the lock but the caller doesn't know, so release it");
 			lock->release(amount);
 			throw;
 		}
@@ -1977,27 +2017,31 @@ Future<decltype(std::declval<Fun>()(std::declval<T>()).getValue())> runAfter(Fut
 	return res;
 }
 
-ACTOR template <class T, class U>
-Future<U> runAfter(Future<T> lhs, Future<U> rhs) {
-	T val1 = wait(lhs);
-	U res = wait(rhs);
-	return res;
-}
-
 template <class T, class Fun>
 auto operator>>=(Future<T> lhs, Fun&& rhs) -> Future<decltype(rhs(std::declval<T>()))> {
 	return runAfter(lhs, std::forward<Fun>(rhs));
 }
 
+/*
+ * NOTE: This implementation can't guarantee the doesn't really enforce the ACTOR execution order. See issue #7708
+ACTOR template <class T, class U>
+Future<U> runAfter(Future<T> lhs, Future<U> rhs) {
+    T val1 = wait(lhs);
+    U res = wait(rhs);
+    return res;
+}
+
 template <class T, class U>
 Future<U> operator>>(Future<T> const& lhs, Future<U> const& rhs) {
-	return runAfter(lhs, rhs);
+    return runAfter(lhs, rhs);
 }
+ */
 
 /*
  * IAsyncListener is similar to AsyncVar, but it decouples the input and output, so the translation unit
  * responsible for handling the output does not need to have knowledge of how the output is generated
  */
+
 template <class Output>
 class IAsyncListener : public ReferenceCounted<IAsyncListener<Output>> {
 public:
diff --git a/flow/include/flow/network.h b/flow/include/flow/network.h
index 62613f402e..d1bf24973d 100644
--- a/flow/include/flow/network.h
+++ b/flow/include/flow/network.h
@@ -467,11 +467,18 @@ public:
 	// this may not be an address we can connect to!
 	virtual NetworkAddress getPeerAddress() const = 0;
 
+	// Returns whether the peer is trusted.
+	// For TLS-enabled connections, this is true if the peer has presented a valid chain of certificates trusted by the
+	// local endpoint. For non-TLS connections this is always true for any valid open connection.
+	virtual bool hasTrustedPeer() const = 0;
+
 	virtual UID getDebugID() const = 0;
 
 	// At present, implemented by Sim2Conn where we want to disable bits flip for connections between parent process and
 	// child process, also reduce latency for this kind of connection
 	virtual bool isStableConnection() const { throw unsupported_operation(); }
+
+	virtual boost::asio::ip::tcp::socket& getSocket() = 0;
 };
 
 class IListener {
@@ -520,6 +527,7 @@ public:
 		enDiskFailureInjector = 16,
 		enBitFlipper = 17,
 		enHistogram = 18,
+		enTokenCache = 19,
 		COUNT // Add new fields before this enumerator
 	};
 
@@ -576,7 +584,7 @@ public:
 	// Returns true if the current thread is the main thread
 
 	virtual void onMainThread(Promise<Void>&& signal, TaskPriority taskID) = 0;
-	// Executes signal.send(Void()) on a/the thread belonging to this network
+	// Executes signal.send(Void()) on a/the thread belonging to this network in FIFO order
 
 	virtual THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*),
 	                                  void* arg,
@@ -688,9 +696,10 @@ public:
 
 	// Make an outgoing connection to the given address.  May return an error or block indefinitely in case of
 	// connection problems!
-	virtual Future<Reference<IConnection>> connect(NetworkAddress toAddr, const std::string& host = "") = 0;
+	virtual Future<Reference<IConnection>> connect(NetworkAddress toAddr,
+	                                               boost::asio::ip::tcp::socket* existingSocket = nullptr) = 0;
 
-	virtual Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr, const std::string& host = "") = 0;
+	virtual Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr) = 0;
 
 	// Make an outgoing udp connection and connect to the passed address.
 	virtual Future<Reference<IUDPSocket>> createUDPSocket(NetworkAddress toAddr) = 0;
@@ -732,6 +741,22 @@ public:
 		return static_cast<INetworkConnections*>((void*)g_network->global(INetwork::enNetworkConnections));
 	}
 
+	// If a DNS name can be resolved to both and IPv4 and IPv6 addresses, we want IPv6 addresses when running the
+	// clusters on IPv6.
+	// This function takes a vector of addresses and return a random one, preferring IPv6 over IPv4.
+	static NetworkAddress pickOneAddress(const std::vector<NetworkAddress>& addresses) {
+		std::vector<NetworkAddress> ipV6Addresses;
+		for (const NetworkAddress& addr : addresses) {
+			if (addr.isV6()) {
+				ipV6Addresses.push_back(addr);
+			}
+		}
+		if (ipV6Addresses.size() > 0) {
+			return ipV6Addresses[deterministicRandom()->randomInt(0, ipV6Addresses.size())];
+		}
+		return addresses[deterministicRandom()->randomInt(0, addresses.size())];
+	}
+
 	void removeCachedDNS(const std::string& host, const std::string& service) { dnsCache.remove(host, service); }
 
 	DNSCache dnsCache;
diff --git a/flow/include/flow/serialize.h b/flow/include/flow/serialize.h
index 5fe7ddbc3f..4c0deddcc7 100644
--- a/flow/include/flow/serialize.h
+++ b/flow/include/flow/serialize.h
@@ -91,6 +91,8 @@ inline typename Archive::READER& operator>>(Archive& ar, Item& item) {
 
 template <class Archive, class Item, class... Items>
 typename Archive::WRITER& serializer(Archive& ar, const Item& item, const Items&... items) {
+	static_assert(fb_appears_last_property(pack<Item, Items...>{}),
+	              "An argument to a serializer call that must appear last (Arena?) does not appear last");
 	save(ar, item);
 	if constexpr (sizeof...(Items) > 0) {
 		serializer(ar, items...);
@@ -100,6 +102,8 @@ typename Archive::WRITER& serializer(Archive& ar, const Item& item, const Items&
 
 template <class Archive, class Item, class... Items>
 typename Archive::READER& serializer(Archive& ar, Item& item, Items&... items) {
+	static_assert(fb_appears_last_property(pack<Item, Items...>{}),
+	              "An argument to a serializer call that must appear last (Arena?) does not appear last");
 	load(ar, item);
 	if constexpr (sizeof...(Items) > 0) {
 		serializer(ar, items...);
@@ -373,7 +377,7 @@ struct _Unversioned {
 };
 
 // These functions return valid options to the VersionOptions parameter of the constructor of each archive type
-inline _IncludeVersion IncludeVersion(ProtocolVersion defaultVersion = currentProtocolVersion) {
+inline _IncludeVersion IncludeVersion(ProtocolVersion defaultVersion = currentProtocolVersion()) {
 	return _IncludeVersion(defaultVersion);
 }
 inline _AssumeVersion AssumeVersion(ProtocolVersion version) {
@@ -449,17 +453,17 @@ public:
 	void serializeAsTuple(StringRef str) {
 		size_t last_pos = 0;
 
-		serializeBytes(LiteralStringRef("\x01"));
+		serializeBytes("\x01"_sr);
 
 		for (size_t pos = 0; pos < str.size(); ++pos) {
 			if (str[pos] == '\x00') {
 				serializeBytes(str.substr(last_pos, pos - last_pos));
-				serializeBytes(LiteralStringRef("\x00\xff"));
+				serializeBytes("\x00\xff"_sr);
 				last_pos = pos + 1;
 			}
 		}
 		serializeBytes(str.substr(last_pos, str.size() - last_pos));
-		serializeBytes(LiteralStringRef("\x00"));
+		serializeBytes("\x00"_sr);
 	}
 
 	void serializeAsTuple(bool t) {
@@ -854,10 +858,6 @@ struct PacketWriter {
 	}
 	ProtocolVersion protocolVersion() const { return m_protocolVersion; }
 	void setProtocolVersion(ProtocolVersion pv) { m_protocolVersion = pv; }
-
-private:
-	void serializeBytesAcrossBoundary(const void* data, int bytes);
-	void nextBuffer(size_t size = 0 /* downstream it will default to at least 4k minus some padding */);
 	uint8_t* writeBytes(size_t size) {
 		if (size > buffer->bytes_unwritten()) {
 			nextBuffer(size);
@@ -868,6 +868,9 @@ private:
 		return result;
 	}
 
+private:
+	void serializeBytesAcrossBoundary(const void* data, int bytes);
+	void nextBuffer(size_t size = 0 /* downstream it will default to at least 4k minus some padding */);
 	template <class, class>
 	friend class MakeSerializeSource;
 
diff --git a/flow/network.cpp b/flow/network.cpp
index f97e66cffc..85c73f8e85 100644
--- a/flow/network.cpp
+++ b/flow/network.cpp
@@ -279,7 +279,7 @@ Future<Reference<IConnection>> INetworkConnections::connect(const std::string& h
 	// Use map to create an actor that returns an endpoint or throws
 	Future<NetworkAddress> pickEndpoint =
 	    map(resolveTCPEndpoint(host, service), [=](std::vector<NetworkAddress> const& addresses) -> NetworkAddress {
-		    NetworkAddress addr = addresses[deterministicRandom()->randomInt(0, addresses.size())];
+		    NetworkAddress addr = INetworkConnections::pickOneAddress(addresses);
 		    addr.fromHostname = true;
 		    if (isTLS) {
 			    addr.flags = NetworkAddress::FLAG_TLS;
@@ -293,7 +293,7 @@ Future<Reference<IConnection>> INetworkConnections::connect(const std::string& h
 	                std::function<Future<Reference<IConnection>>(NetworkAddress const&)>,
 	                Reference<IConnection>>(
 	    pickEndpoint,
-	    [=](NetworkAddress const& addr) -> Future<Reference<IConnection>> { return connectExternal(addr, host); });
+	    [=](NetworkAddress const& addr) -> Future<Reference<IConnection>> { return connectExternal(addr); });
 }
 
 IUDPSocket::~IUDPSocket() {}
@@ -347,4 +347,22 @@ TEST_CASE("/flow/network/ipaddress") {
 	return Void();
 }
 
+TEST_CASE("/flow/network/ipV6Preferred") {
+	std::vector<NetworkAddress> addresses;
+	for (int i = 0; i < 50; ++i) {
+		std::string s = fmt::format("{}.{}.{}.{}:{}", i, i, i, i, i);
+		addresses.push_back(NetworkAddress::parse(s));
+	}
+	std::string ipv6 = "[2001:db8:85a3::8a2e:370:7334]:4800";
+	addresses.push_back(NetworkAddress::parse(ipv6));
+	for (int i = 50; i < 100; ++i) {
+		std::string s = fmt::format("{}.{}.{}.{}:{}", i, i, i, i, i);
+		addresses.push_back(NetworkAddress::parse(s));
+	}
+	// Confirm IPv6 is always preferred.
+	ASSERT(INetworkConnections::pickOneAddress(addresses).toString() == ipv6);
+
+	return Void();
+}
+
 NetworkInfo::NetworkInfo() : handshakeLock(new FlowLock(FLOW_KNOBS->TLS_HANDSHAKE_LIMIT)) {}
diff --git a/flowbench/BenchEncrypt.cpp b/flowbench/BenchEncrypt.cpp
index e99e80a209..ca10cfc83e 100644
--- a/flowbench/BenchEncrypt.cpp
+++ b/flowbench/BenchEncrypt.cpp
@@ -25,7 +25,7 @@
 
 static StreamCipher::IV getRandomIV() {
 	StreamCipher::IV iv;
-	generateRandomData(iv.data(), iv.size());
+	deterministicRandom()->randomBytes(iv.data(), iv.size());
 	return iv;
 }
 
@@ -47,7 +47,7 @@ static void bench_encrypt(benchmark::State& state) {
 	auto key = StreamCipherKey::getGlobalCipherKey();
 	auto iv = getRandomIV();
 	auto data = getKey(bytes);
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		for (int chunk = 0; chunk < chunks; ++chunk) {
 			benchmark::DoNotOptimize(encrypt(key, iv, data.begin() + chunk * chunkSize, chunkSize));
 		}
@@ -64,7 +64,7 @@ static void bench_decrypt(benchmark::State& state) {
 	auto iv = getRandomIV();
 	auto data = getKey(bytes);
 	auto encrypted = encrypt(key, iv, data.begin(), data.size());
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		Arena arena;
 		DecryptionStreamCipher decryptor(key, iv);
 		for (int chunk = 0; chunk < chunks; ++chunk) {
diff --git a/flowbench/BenchHash.cpp b/flowbench/BenchHash.cpp
index 40d15db4ab..92d4a5e5a1 100644
--- a/flowbench/BenchHash.cpp
+++ b/flowbench/BenchHash.cpp
@@ -58,7 +58,7 @@ template <HashType hashType>
 static void bench_hash(benchmark::State& state) {
 	auto length = 1 << state.range(0);
 	auto key = getKey(length);
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		hash<hashType>(key, length);
 	}
 	state.SetItemsProcessed(static_cast<long>(state.iterations()));
diff --git a/flowbench/BenchIterate.cpp b/flowbench/BenchIterate.cpp
index 793c3fc307..8f2e8c6921 100644
--- a/flowbench/BenchIterate.cpp
+++ b/flowbench/BenchIterate.cpp
@@ -50,7 +50,7 @@ static void bench_iterate(benchmark::State& state) {
 	auto kv = getKV(size, size);
 	ListImpl mutations;
 	populate(mutations, items, size, kv.key, kv.value);
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		for (const auto& mutation : mutations) {
 			benchmark::DoNotOptimize(mutation);
 		}
diff --git a/flowbench/BenchMem.cpp b/flowbench/BenchMem.cpp
index 8696461efd..6db2ce28a8 100644
--- a/flowbench/BenchMem.cpp
+++ b/flowbench/BenchMem.cpp
@@ -31,7 +31,7 @@ static void bench_memcmp(benchmark::State& state) {
 	memset(b2.get(), 0, kLength);
 	b2.get()[kLength - 1] = 1;
 
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		benchmark::DoNotOptimize(memcmp(b1.get(), b2.get(), kLength));
 	}
 }
@@ -42,7 +42,7 @@ static void bench_memcpy(benchmark::State& state) {
 	std::unique_ptr<char[]> b2{ new char[kLength] };
 	memset(b1.get(), 0, kLength);
 
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		benchmark::DoNotOptimize(memcpy(b2.get(), b1.get(), kLength));
 	}
 }
diff --git a/flowbench/BenchMetadataCheck.cpp b/flowbench/BenchMetadataCheck.cpp
index c028e3deec..48d2b0bcde 100644
--- a/flowbench/BenchMetadataCheck.cpp
+++ b/flowbench/BenchMetadataCheck.cpp
@@ -29,17 +29,17 @@
 
 static const std::array<MutationRef, 5> mutations = {
 	MutationRef(MutationRef::Type::ClearRange, normalKeys.begin, normalKeys.end),
-	MutationRef(MutationRef::Type::ClearRange, LiteralStringRef("a"), LiteralStringRef("b")),
-	MutationRef(MutationRef::Type::ClearRange, LiteralStringRef("aaaaaaaaaa"), LiteralStringRef("bbbbbbbbbb")),
+	MutationRef(MutationRef::Type::ClearRange, "a"_sr, "b"_sr),
+	MutationRef(MutationRef::Type::ClearRange, "aaaaaaaaaa"_sr, "bbbbbbbbbb"_sr),
 	MutationRef(MutationRef::Type::ClearRange, normalKeys.begin, systemKeys.end),
 	MutationRef(MutationRef::Type::ClearRange,
-	            LiteralStringRef("a").withPrefix(systemKeys.begin),
-	            LiteralStringRef("b").withPrefix(systemKeys.begin)),
+	            "a"_sr.withPrefix(systemKeys.begin),
+	            "b"_sr.withPrefix(systemKeys.begin)),
 };
 
 static void bench_check_metadata1(benchmark::State& state) {
 	const auto& m = mutations[state.range(0)];
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		benchmark::DoNotOptimize(KeyRangeRef(m.param1, m.param2).intersects(systemKeys));
 	}
 	state.SetItemsProcessed(static_cast<long>(state.iterations()));
@@ -47,7 +47,7 @@ static void bench_check_metadata1(benchmark::State& state) {
 
 static void bench_check_metadata2(benchmark::State& state) {
 	const auto& m = mutations[state.range(0)];
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		benchmark::DoNotOptimize(m.param2.size() > 1 && m.param2[0] == systemKeys.begin[0]);
 	}
 	state.SetItemsProcessed(static_cast<long>(state.iterations()));
diff --git a/flowbench/BenchPopulate.cpp b/flowbench/BenchPopulate.cpp
index 35213b4171..37dacc0478 100644
--- a/flowbench/BenchPopulate.cpp
+++ b/flowbench/BenchPopulate.cpp
@@ -35,7 +35,7 @@ static void bench_populate(benchmark::State& state) {
 	size_t items = state.range(0);
 	size_t size = state.range(1);
 	auto kv = getKV(size, size);
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		Standalone<VectorRef<MutationRef>> mutations;
 		mutations.reserve(mutations.arena(), items);
 		for (int i = 0; i < items; ++i) {
diff --git a/flowbench/BenchRandom.cpp b/flowbench/BenchRandom.cpp
index 47d607cffe..bd478a4ac5 100644
--- a/flowbench/BenchRandom.cpp
+++ b/flowbench/BenchRandom.cpp
@@ -23,9 +23,8 @@
 #include "flow/IRandom.h"
 
 static void bench_random(benchmark::State& state) {
-	while (state.KeepRunning()) {
-		double r = deterministicRandom()->random01();
-		benchmark::DoNotOptimize(r);
+	for (auto _ : state) {
+		benchmark::DoNotOptimize(deterministicRandom()->random01());
 	}
 	state.SetItemsProcessed(static_cast<long>(state.iterations()));
 }
diff --git a/flowbench/BenchRef.cpp b/flowbench/BenchRef.cpp
index 297e32a66b..434abf38c4 100644
--- a/flowbench/BenchRef.cpp
+++ b/flowbench/BenchRef.cpp
@@ -71,7 +71,7 @@ struct Factory<RefType::FlowReferenceThreadSafe> {
 
 template <RefType refType>
 static void bench_ref_create_and_destroy(benchmark::State& state) {
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		auto ptr = Factory<refType>::create();
 		benchmark::DoNotOptimize(ptr);
 		Factory<refType>::cleanup(ptr);
@@ -82,7 +82,7 @@ static void bench_ref_create_and_destroy(benchmark::State& state) {
 template <RefType refType>
 static void bench_ref_copy(benchmark::State& state) {
 	auto ptr = Factory<refType>::create();
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		auto ptr2 = ptr;
 		benchmark::DoNotOptimize(ptr2);
 	}
diff --git a/flowbench/BenchSamples.cpp b/flowbench/BenchSamples.cpp
new file mode 100644
index 0000000000..687a371048
--- /dev/null
+++ b/flowbench/BenchSamples.cpp
@@ -0,0 +1,101 @@
+/*
+ * BenchSamples.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "benchmark/benchmark.h"
+#include "flow/IRandom.h"
+#include "flowbench/GlobalData.h"
+#include "fdbrpc/Stats.h"
+#include "flow/Histogram.h"
+
+static void bench_continuousSampleInt(benchmark::State& state) {
+	ContinuousSample<int64_t> cs(state.range(0));
+	InputGenerator<int64_t> data(1e6, []() { return deterministicRandom()->randomInt64(0, 1e9); });
+
+	for (auto _ : state) {
+		cs.addSample(data.next());
+	}
+
+	state.SetItemsProcessed(state.iterations());
+}
+BENCHMARK(bench_continuousSampleInt)->Arg(1000)->Arg(10000)->ReportAggregatesOnly(true);
+
+static void bench_continuousSampleLatency(benchmark::State& state) {
+	ContinuousSample<double> cs(state.range(0));
+	InputGenerator<double> data(1e6, []() { return deterministicRandom()->random01() * 2.0; });
+
+	for (auto _ : state) {
+		cs.addSample(data.next());
+	}
+
+	state.SetItemsProcessed(state.iterations());
+}
+BENCHMARK(bench_continuousSampleLatency)->Arg(1000)->Arg(10000)->ReportAggregatesOnly(true);
+
+static void bench_latencyBands(benchmark::State& state) {
+	constexpr double max = 2.0;
+	InputGenerator<double> data(1e6, []() { return deterministicRandom()->random01() * max; });
+	LatencyBands lb("test", UID(), 5);
+	for (int i = 0; i < state.range(0); ++i) {
+		lb.addThreshold(max * (double)(i + 1) / (state.range(0) + 1));
+	}
+
+	for (auto _ : state) {
+		lb.addMeasurement(data.next(), false);
+	}
+
+	state.SetItemsProcessed(state.iterations());
+}
+BENCHMARK(bench_latencyBands)->Arg(1)->Arg(4)->Arg(7)->Arg(10)->ReportAggregatesOnly(true);
+
+static void bench_histogramInt(benchmark::State& state) {
+	Reference<Histogram> h = Histogram::getHistogram("histogramTest"_sr, "bytes"_sr, Histogram::Unit::bytes);
+	InputGenerator<int32_t> data(1e6, []() { return deterministicRandom()->randomInt64(0, 1e9); });
+
+	for (auto _ : state) {
+		h->sample(data.next());
+	}
+
+	state.SetItemsProcessed(state.iterations());
+}
+BENCHMARK(bench_histogramInt)->ReportAggregatesOnly(true);
+
+static void bench_histogramPct(benchmark::State& state) {
+	Reference<Histogram> h = Histogram::getHistogram("histogramTest"_sr, "pct"_sr, Histogram::Unit::percentageLinear);
+	InputGenerator<double> data(1e6, []() { return deterministicRandom()->random01() * 1.5; });
+
+	for (auto _ : state) {
+		h->samplePercentage(data.next());
+	}
+
+	state.SetItemsProcessed(state.iterations());
+}
+BENCHMARK(bench_histogramPct)->ReportAggregatesOnly(true);
+
+static void bench_histogramTime(benchmark::State& state) {
+	Reference<Histogram> h = Histogram::getHistogram("histogramTest"_sr, "latency"_sr, Histogram::Unit::microseconds);
+	InputGenerator<double> data(1e6, []() { return deterministicRandom()->random01() * 5; });
+
+	for (auto _ : state) {
+		h->sampleSeconds(data.next());
+	}
+
+	state.SetItemsProcessed(state.iterations());
+}
+BENCHMARK(bench_histogramTime)->ReportAggregatesOnly(true);
diff --git a/flowbench/BenchTimer.cpp b/flowbench/BenchTimer.cpp
index 90a542a3f9..1299fd1c07 100644
--- a/flowbench/BenchTimer.cpp
+++ b/flowbench/BenchTimer.cpp
@@ -23,17 +23,15 @@
 #include "flow/Platform.h"
 
 static void bench_timer(benchmark::State& state) {
-	while (state.KeepRunning()) {
-		double time = timer();
-		benchmark::DoNotOptimize(time);
+	for (auto _ : state) {
+		benchmark::DoNotOptimize(timer());
 	}
 	state.SetItemsProcessed(static_cast<long>(state.iterations()));
 }
 
 static void bench_timer_monotonic(benchmark::State& state) {
-	while (state.KeepRunning()) {
-		double time = timer_monotonic();
-		benchmark::DoNotOptimize(time);
+	for (auto _ : state) {
+		benchmark::DoNotOptimize(timer_monotonic());
 	}
 	state.SetItemsProcessed(static_cast<long>(state.iterations()));
 }
diff --git a/flowbench/BenchVersionVector.cpp b/flowbench/BenchVersionVector.cpp
index db8d1036cd..45079bf139 100644
--- a/flowbench/BenchVersionVector.cpp
+++ b/flowbench/BenchVersionVector.cpp
@@ -34,7 +34,7 @@ static void bench_vv_getdelta(benchmark::State& benchState) {
 
 	i = 0;
 	const int64_t numDeltas = benchState.range(1);
-	while (benchState.KeepRunning()) {
+	for (auto _ : benchState) {
 		vv.setVersion(Tag(0, i++), ++version);
 		i %= tags;
 
diff --git a/flowbench/BenchVersionVectorSerialization.cpp b/flowbench/BenchVersionVectorSerialization.cpp
index ea8de78fd8..66c14549f5 100644
--- a/flowbench/BenchVersionVectorSerialization.cpp
+++ b/flowbench/BenchVersionVectorSerialization.cpp
@@ -41,7 +41,7 @@ static void bench_serializable_traits_version(benchmark::State& state) {
 
 	size_t size = 0;
 	VersionVector deserializedVV;
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		Standalone<StringRef> msg = ObjectWriter::toValue(serializedVV, Unversioned());
 
 		// Capture the serialized buffer size.
@@ -71,7 +71,7 @@ static void bench_dynamic_size_traits_version(benchmark::State& state) {
 
 	size_t size = 0;
 	VersionVector deserializedVV;
-	while (state.KeepRunning()) {
+	for (auto _ : state) {
 		size = dynamic_size_traits<VersionVector>::size(serializedVV, context);
 
 		uint8_t* buf = context.allocate(size);
diff --git a/flowbench/GlobalData.cpp b/flowbench/GlobalData.cpp
index 8cdd38a844..88883909b5 100644
--- a/flowbench/GlobalData.cpp
+++ b/flowbench/GlobalData.cpp
@@ -28,7 +28,7 @@ static inline void initGlobalData() {
 	if (!globalData) {
 		globalData = static_cast<uint8_t*>(allocateFast(globalDataSize));
 	}
-	generateRandomData(globalData, globalDataSize);
+	deterministicRandom()->randomBytes(globalData, globalDataSize);
 }
 
 KeyValueRef getKV(size_t keySize, size_t valueSize) {
diff --git a/flowbench/include/flowbench/GlobalData.h b/flowbench/include/flowbench/GlobalData.h
index 3f2d1f53db..66ba8c283f 100644
--- a/flowbench/include/flowbench/GlobalData.h
+++ b/flowbench/include/flowbench/GlobalData.h
@@ -28,4 +28,32 @@
 KeyValueRef getKV(size_t keySize, size_t valueSize);
 KeyRef getKey(size_t keySize);
 
+// Pre-generate a vector of T using a lambda then return them
+// via next() one by one with wrap-around
+template <typename T>
+struct InputGenerator {
+	InputGenerator() {}
+
+	template <typename Fn>
+	InputGenerator(int n, Fn genFunc) {
+		ASSERT(n > 0);
+		data.reserve(n);
+		for (int i = 0; i < n; ++i) {
+			data.push_back(genFunc());
+		}
+		lastIndex = -1;
+	}
+
+	const T& next() {
+		if (++lastIndex == data.size()) {
+			lastIndex = 0;
+		}
+
+		return data[lastIndex];
+	}
+
+	std::vector<T> data;
+	int lastIndex;
+};
+
 #endif
diff --git a/packaging/deb/builddebs.sh b/packaging/deb/builddebs.sh
index e08f1befd5..8840b08287 100755
--- a/packaging/deb/builddebs.sh
+++ b/packaging/deb/builddebs.sh
@@ -52,8 +52,8 @@ mkdir -p -m 0755 $CLIENTSDIR/usr/share/doc/foundationdb-clients
 mkdir -p -m 0755 $CLIENTSDIR/usr/lib/foundationdb/backup_agent
 
 install -m 0755 bin/fdbcli $CLIENTSDIR/usr/bin
-install -m 0644 lib/libfdb_c.so $CLIENTSDIR/usr/lib
-install -m 0644 bindings/c/foundationdb/fdb_c.h bindings/c/foundationdb/fdb_c_types.h bindings/c/foundationdb/fdb_c_internal.h bindings/c/foundationdb/fdb_c_options.g.h fdbclient/vexillographer/fdb.options $CLIENTSDIR/usr/include/foundationdb
+install -m 0644 lib/libfdb_c.so lib/libfdb_c_shim.so $CLIENTSDIR/usr/lib
+install -m 0644 bindings/c/foundationdb/fdb_c.h bindings/c/foundationdb/fdb_c_types.h bindings/c/foundationdb/fdb_c_internal.h bindings/c/foundationdb/fdb_c_options.g.h fdbclient/vexillographer/fdb.options bindings/c/foundationdb/fdb_c_shim.h $CLIENTSDIR/usr/include/foundationdb
 dos2unix -q -n README.md $CLIENTSDIR/usr/share/doc/foundationdb-clients/README
 chmod 0644 $CLIENTSDIR/usr/share/doc/foundationdb-clients/README
 install -m 0755 bin/fdbbackup $CLIENTSDIR/usr/lib/foundationdb/backup_agent/backup_agent
diff --git a/packaging/docker/build-images.sh b/packaging/docker/build-images.sh
index 02574515d9..2ce492645d 100755
--- a/packaging/docker/build-images.sh
+++ b/packaging/docker/build-images.sh
@@ -2,12 +2,17 @@
 set -Eeuo pipefail
 script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P)
 reset=$(tput sgr0)
+red=$(tput setaf 1)
 blue=$(tput setaf 4)
 
-function logg() {
+function logg () {
     printf "${blue}##### $(date +"%H:%M:%S") #  %-56.55s #####${reset}\n" "${1}"
 }
 
+function loge () {
+    printf "${red}##### $(date +"%H:%M:%S") #  %-56.55s #####${reset}\n" "${1}"
+}
+
 function pushd () {
     command pushd "$@" > /dev/null
 }
@@ -16,7 +21,19 @@ function popd () {
     command popd > /dev/null
 }
 
-function create_fake_website_directory() {
+function error_exit () {
+    echo "${red}################################################################################${reset}"
+    loge "${0} FAILED"
+    echo "${red}################################################################################${reset}"
+}
+
+trap error_exit ERR
+
+function create_fake_website_directory () {
+    if [ ${#} -ne 1 ]; then
+        loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}"
+    fi
+    local stripped_binaries_and_from_where="${1}"
     fdb_binaries=( 'fdbbackup' 'fdbcli' 'fdbserver' 'fdbmonitor' )
     logg "PREPARING WEBSITE"
     website_directory="${script_dir}/website"
@@ -112,7 +129,7 @@ function create_fake_website_directory() {
     fdb_website="file:///tmp/website"
 }
 
-function compile_ycsb() {
+function compile_ycsb () {
     logg "COMPILING YCSB"
     if [ "${use_development_java_bindings}" == "true" ]; then
         logg "INSTALL JAVA BINDINGS"
@@ -150,7 +167,13 @@ function compile_ycsb() {
     popd || exit 128
 }
 
-function build_and_push_images(){
+function build_and_push_images () {
+    if [ ${#} -ne 3 ]; then
+        loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}"
+    fi
+    local dockerfile_name="${1}"
+    local use_development_java_bindings="${2}"
+    local push_docker_images="${3}"
     declare -a tags_to_push=()
     for image in "${image_list[@]}"; do
         logg "BUILDING ${image}"
@@ -237,11 +260,6 @@ image_list=(
 )
 registry=""
 tag_base="foundationdb/"
-# THESE CONTROL THE PATH OF FUNCTIONS THAT ARE CALLED BELOW
-stripped_binaries_and_from_where="stripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" )
-dockerfile_name="Dockerfile"
-use_development_java_bindings="false"
-push_docker_images="false"
 
 if [ -n "${OKTETO_NAMESPACE+x}" ]; then
     logg "RUNNING IN OKTETO/AWS"
@@ -258,19 +276,24 @@ if [ -n "${OKTETO_NAMESPACE+x}" ]; then
     else
         tag_postfix="${OKTETO_NAME:-dev}"
     fi
-    stripped_binaries_and_from_where="unstripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" )
-    dockerfile_name="Dockerfile.eks"
-    use_development_java_bindings="true"
-    push_docker_images="true"
+
+    # build regular images
+    create_fake_website_directory stripped_local
+    build_and_push_images Dockerfile true true
+
+    # build debug images
+    create_fake_website_directory unstripped_local
+    build_and_push_images Dockerfile.eks true true
 else
     echo "Dear ${USER}, you probably need to edit this file before running it. "
     echo "${0} has a very narrow set of situations where it will be successful,"
     echo "or even useful, when executed unedited"
     exit 1
+    # this set of options will creat standard images from a local build
+    # create_fake_website_directory stripped_local
+    # build_and_push_images Dockerfile false false
 fi
 
-create_fake_website_directory
-build_and_push_images
 
 echo "${blue}################################################################################${reset}"
 logg "COMPLETED ${0}"
diff --git a/packaging/rpm/buildrpms.sh b/packaging/rpm/buildrpms.sh
index a59fb5caf2..5256cb051c 100755
--- a/packaging/rpm/buildrpms.sh
+++ b/packaging/rpm/buildrpms.sh
@@ -33,7 +33,8 @@ install -m 0755 bin/fdbcli $INSTDIR/usr/bin
 install -m 0755 bin/fdbserver $INSTDIR/usr/sbin
 install -m 0755 bin/fdbmonitor $INSTDIR/usr/lib/foundationdb
 install -m 0755 lib/libfdb_c.so $INSTDIR/usr/lib64
-install -m 0644 bindings/c/foundationdb/fdb_c.h bindings/c/foundationdb/fdb_c_options.g.h bindings/c/foundationdb/fdb_c_types.h bindings/c/foundationdb/fdb_c_internal.h fdbclient/vexillographer/fdb.options $INSTDIR/usr/include/foundationdb
+install -m 0755 lib/libfdb_c_shim.so $INSTDIR/usr/lib64
+install -m 0644 bindings/c/foundationdb/fdb_c.h bindings/c/foundationdb/fdb_c_options.g.h bindings/c/foundationdb/fdb_c_types.h bindings/c/foundationdb/fdb_c_internal.h bindings/c/foundationdb/fdb_c_shim.h fdbclient/vexillographer/fdb.options $INSTDIR/usr/include/foundationdb
 dos2unix -q -n README.md $INSTDIR/usr/share/doc/foundationdb-clients/README
 dos2unix -q -n README.md $INSTDIR/usr/share/doc/foundationdb-server/README
 chmod 0644 $INSTDIR/usr/share/doc/foundationdb-clients/README
diff --git a/pull_request_template.md b/pull_request_template.md
index f4b6480fd9..416adbc743 100644
--- a/pull_request_template.md
+++ b/pull_request_template.md
@@ -1,8 +1,8 @@
-Put description here...
+Replace this text with your description here...
 
 # Code-Reviewer Section
 
-The general guidelines can be found [here](https://github.com/apple/foundationdb/wiki/FoundationDB-Commit-Process).
+The general pull request guidelines can be found [here](https://github.com/apple/foundationdb/wiki/FoundationDB-Commit-Process).
 
 Please check each of the following things and check *all* boxes before accepting a PR.
 
diff --git a/recipes/go-recipes/blob.go b/recipes/go-recipes/blob.go
index f6590acfde..744101a77a 100644
--- a/recipes/go-recipes/blob.go
+++ b/recipes/go-recipes/blob.go
@@ -29,6 +29,8 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
+const API_VERSION int = 720
+
 const CHUNK_SIZE int = 5
 
 func write_blob(t fdb.Transactor, blob_subspace subspace.Subspace, blob []byte) (err error) {
@@ -79,7 +81,7 @@ func read_blob(t fdb.ReadTransactor, blob_subspace subspace.Subspace) ([]byte, e
 }
 
 func main() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/doc.go b/recipes/go-recipes/doc.go
index 9ae34ad10c..40edd60ffa 100644
--- a/recipes/go-recipes/doc.go
+++ b/recipes/go-recipes/doc.go
@@ -32,6 +32,8 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
+const API_VERSION int = 720
+
 func clear_subspace(trtr fdb.Transactor, sub subspace.Subspace) error {
 	_, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
 		tr.ClearRange(sub)
@@ -220,7 +222,7 @@ func (doc Doc) GetDoc(trtr fdb.Transactor, doc_id int) interface{} {
 }
 
 func main() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/graph.go b/recipes/go-recipes/graph.go
index 9487b09e89..b46d33b26d 100644
--- a/recipes/go-recipes/graph.go
+++ b/recipes/go-recipes/graph.go
@@ -30,6 +30,8 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
+const API_VERSION int = 720
+
 func clear_subspace(trtr fdb.Transactor, sub subspace.Subspace) error {
 	_, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
 		tr.ClearRange(sub)
@@ -125,7 +127,7 @@ func (graph *Graph) get_in_neighbors(trtr fdb.Transactor, node int) ([]int, erro
 }
 
 func main() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/indirect.go b/recipes/go-recipes/indirect.go
index e49d9640e3..20f8c5c90f 100644
--- a/recipes/go-recipes/indirect.go
+++ b/recipes/go-recipes/indirect.go
@@ -30,6 +30,8 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
+const API_VERSION int = 720
+
 func clear_subspace(trtr fdb.Transactor, sub subspace.Subspace) error {
 	_, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
 		tr.ClearRange(sub)
@@ -94,7 +96,7 @@ func (wrkspc Workspace) Session(foo func(directory.DirectorySubspace)) (err erro
 }
 
 func main() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/multi.go b/recipes/go-recipes/multi.go
index 06311feb41..5ced2da6c9 100644
--- a/recipes/go-recipes/multi.go
+++ b/recipes/go-recipes/multi.go
@@ -30,6 +30,8 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
+const API_VERSION int = 720
+
 func clear_subspace(db fdb.Transactor, ss subspace.Subspace) {
 	db.Transact(func(tr fdb.Transaction) (interface{}, error) {
 		tr.ClearRange(ss)
@@ -131,8 +133,7 @@ func (multi MultiMap) MultiIsElement(trtr fdb.Transactor, index, value interface
 }
 
 func main() {
-
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/priority.go b/recipes/go-recipes/priority.go
index 897253ce8a..3323c85630 100644
--- a/recipes/go-recipes/priority.go
+++ b/recipes/go-recipes/priority.go
@@ -31,6 +31,8 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
+const API_VERSION int = 720
+
 func clear_subspace(trtr fdb.Transactor, sub subspace.Subspace) error {
 	_, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
 		tr.ClearRange(sub)
@@ -118,7 +120,7 @@ func (prty Priority) Peek(trtr fdb.Transactor, max bool) interface{} {
 }
 
 func main() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/queue.go b/recipes/go-recipes/queue.go
index e1d0091755..c87725e7fe 100644
--- a/recipes/go-recipes/queue.go
+++ b/recipes/go-recipes/queue.go
@@ -30,6 +30,8 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
+const API_VERSION int = 720
+
 type EmptyQueueError struct{}
 
 func (q EmptyQueueError) Error() string {
@@ -108,7 +110,7 @@ func (q *Queue) FirstItem(trtr fdb.Transactor) (interface{}, error) {
 func main() {
 	fmt.Println("Queue Example Program")
 
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/table.go b/recipes/go-recipes/table.go
index b2b9b5493e..704b4d70c4 100644
--- a/recipes/go-recipes/table.go
+++ b/recipes/go-recipes/table.go
@@ -30,6 +30,8 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
+const API_VERSION int = 720
+
 func clear_subspace(trtr fdb.Transactor, sub subspace.Subspace) error {
 	_, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
 		tr.ClearRange(sub)
@@ -145,7 +147,7 @@ func (tbl Table) TableGetCol(tr fdb.ReadTransactor, col int) ([]interface{}, err
 }
 
 func main() {
-	fdb.MustAPIVersion(720)
+	fdb.MustAPIVersion(API_VERSION)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index bacbe312dd..a2bbcb8a4e 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -10,6 +10,7 @@ set(TEST_AGGREGATE_TRACES "NONE" CACHE STRING "Create aggregated trace files (NO
 set(TEST_LOG_FORMAT "xml" CACHE STRING "Format for test trace files (xml, json)")
 set(TEST_INCLUDE ".*" CACHE STRING "Include only tests that match the given regex")
 set(TEST_EXCLUDE ".^" CACHE STRING "Exclude all tests matching the given regex")
+set(SANITIZER_OPTIONS "UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1;TSAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/contrib/tsan.suppressions" CACHE STRING "Environment variables setting sanitizer options")
 
 # for the restart test we optimally want to use the last stable fdbserver
 # to test upgrades
@@ -39,7 +40,7 @@ if(WITH_PYTHON)
 
   configure_testing(TEST_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
                     ERROR_ON_ADDITIONAL_FILES
-                    IGNORE_PATTERNS ".*/CMakeLists.txt")
+                    IGNORE_PATTERNS ".*/CMakeLists.txt" ".*/requirements.txt")
 
   add_fdb_test(TEST_FILES AsyncFileCorrectness.txt UNIT IGNORE)
   add_fdb_test(TEST_FILES AsyncFileMix.txt UNIT IGNORE)
@@ -125,21 +126,29 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/BackupAzureBlobCorrectness.toml IGNORE)
   add_fdb_test(TEST_FILES fast/BackupS3BlobCorrectness.toml IGNORE)
   add_fdb_test(TEST_FILES fast/BackupCorrectness.toml)
+  add_fdb_test(TEST_FILES fast/EncryptedBackupCorrectness.toml)
   add_fdb_test(TEST_FILES fast/BackupCorrectnessClean.toml)
   add_fdb_test(TEST_FILES fast/BackupToDBCorrectness.toml)
   add_fdb_test(TEST_FILES fast/BackupToDBCorrectnessClean.toml)
-  add_fdb_test(TEST_FILES fast/BlobGranuleVerifySmall.toml)
-  add_fdb_test(TEST_FILES fast/BlobGranuleVerifySmallClean.toml)
   add_fdb_test(TEST_FILES fast/BlobGranuleVerifyAtomicOps.toml)
   add_fdb_test(TEST_FILES fast/BlobGranuleVerifyCycle.toml)
+  add_fdb_test(TEST_FILES fast/BlobGranuleVerifySmall.toml)
+  add_fdb_test(TEST_FILES fast/BlobGranuleVerifySmallClean.toml)
+  # TODO: test occasionally times out due to too many change feed shard parts
+  add_fdb_test(TEST_FILES fast/BlobGranuleMoveVerifyCycle.toml IGNORE)
   add_fdb_test(TEST_FILES fast/CacheTest.toml)
   add_fdb_test(TEST_FILES fast/CloggedSideband.toml)
   add_fdb_test(TEST_FILES fast/CompressionUtilsUnit.toml)
   add_fdb_test(TEST_FILES fast/ConfigureLocked.toml)
+  add_fdb_test(TEST_FILES fast/ConfigIncrement.toml)
+  add_fdb_test(TEST_FILES fast/ConfigIncrementChangeCoordinators.toml)
+  add_fdb_test(TEST_FILES fast/ConfigIncrementWithKills.toml)
   add_fdb_test(TEST_FILES fast/ConstrainedRandomSelector.toml)
   add_fdb_test(TEST_FILES fast/CycleAndLock.toml)
   add_fdb_test(TEST_FILES fast/CycleTest.toml)
   add_fdb_test(TEST_FILES fast/ChangeFeeds.toml)
+  add_fdb_test(TEST_FILES fast/ChangeFeedOperations.toml)
+  add_fdb_test(TEST_FILES fast/ChangeFeedOperationsMove.toml)
   add_fdb_test(TEST_FILES fast/DataLossRecovery.toml)
   add_fdb_test(TEST_FILES fast/EncryptionOps.toml)
   # TODO: fix failures and renable the test
@@ -160,6 +169,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/MoveKeysCycle.toml)
   add_fdb_test(TEST_FILES fast/MutationLogReaderCorrectness.toml)
   add_fdb_test(TEST_FILES fast/GetMappedRange.toml)
+  add_fdb_test(TEST_FILES fast/IDDTxnProcessorApiCorrectness.toml)
   add_fdb_test(TEST_FILES fast/PrivateEndpoints.toml)
   add_fdb_test(TEST_FILES fast/ProtocolVersion.toml)
   add_fdb_test(TEST_FILES fast/RandomSelector.toml)
@@ -178,6 +188,9 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/SwizzledRollbackSideband.toml)
   add_fdb_test(TEST_FILES fast/SystemRebootTestCycle.toml)
   add_fdb_test(TEST_FILES fast/TaskBucketCorrectness.toml)
+  add_fdb_test(TEST_FILES fast/TenantCycle.toml)
+  add_fdb_test(TEST_FILES fast/TenantCycleTokenless.toml)
+  add_fdb_test(TEST_FILES fast/TenantEntryCache.toml)
   add_fdb_test(TEST_FILES fast/TimeKeeperCorrectness.toml)
   add_fdb_test(TEST_FILES fast/TxnStateStoreCycleTest.toml)
   add_fdb_test(TEST_FILES fast/UDP.toml)
@@ -187,26 +200,28 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/WriteDuringRead.toml)
   add_fdb_test(TEST_FILES fast/WriteDuringReadClean.toml)
   add_fdb_test(TEST_FILES noSim/RandomUnitTests.toml UNIT)
-  add_fdb_test(TEST_FILES noSim/SystemDataTest.toml UNIT)
   if (WITH_ROCKSDB_EXPERIMENTAL)
-    add_fdb_test(TEST_FILES noSim/KeyValueStoreRocksDBTest.toml)
+    add_fdb_test(TEST_FILES noSim/KeyValueStoreRocksDBTest.toml UNIT)
     add_fdb_test(TEST_FILES noSim/ShardedRocksDBTest.toml UNIT)
     add_fdb_test(TEST_FILES fast/PhysicalShardMove.toml)
+    add_fdb_test(TEST_FILES fast/StorageServerCheckpointRestore.toml)
   else()
     add_fdb_test(TEST_FILES noSim/KeyValueStoreRocksDBTest.toml IGNORE)
     add_fdb_test(TEST_FILES noSim/ShardedRocksDBTest.toml IGNORE)
     add_fdb_test(TEST_FILES fast/PhysicalShardMove.toml IGNORE)
+    add_fdb_test(TEST_FILES fast/StorageServerCheckpointRestore.toml  IGNORE)
   endif()
+  add_fdb_test(TEST_FILES rare/BlobGranuleRanges.toml)
   add_fdb_test(TEST_FILES rare/CheckRelocation.toml)
   add_fdb_test(TEST_FILES rare/ClogUnclog.toml)
   add_fdb_test(TEST_FILES rare/CloggedCycleWithKills.toml)
-  add_fdb_test(TEST_FILES rare/ConfigIncrement.toml)
-  add_fdb_test(TEST_FILES rare/ConfigIncrementWithKills.toml)
+  add_fdb_test(TEST_FILES rare/ConfigDBUnitTest.toml)
   add_fdb_test(TEST_FILES rare/ConflictRangeCheck.toml)
   add_fdb_test(TEST_FILES rare/ConflictRangeRYOWCheck.toml)
   add_fdb_test(TEST_FILES rare/CycleRollbackClogged.toml)
   add_fdb_test(TEST_FILES rare/CycleWithKills.toml)
   add_fdb_test(TEST_FILES rare/CycleWithDeadHall.toml)
+  add_fdb_test(TEST_FILES rare/DataDistributionMetrics.toml)
   add_fdb_test(TEST_FILES rare/FuzzTest.toml)
   add_fdb_test(TEST_FILES rare/GlobalTagThrottling.toml IGNORE)
   add_fdb_test(TEST_FILES rare/HighContentionPrefixAllocator.toml)
@@ -217,6 +232,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES rare/RandomReadWriteTest.toml)
   add_fdb_test(TEST_FILES rare/ReadSkewReadWrite.toml)
   add_fdb_test(TEST_FILES rare/SpecificUnitTests.toml)
+  add_fdb_test(TEST_FILES rare/StorageQuotaTest.toml)
   add_fdb_test(TEST_FILES rare/SwizzledLargeApiCorrectness.toml)
   add_fdb_test(TEST_FILES rare/RedwoodCorrectnessBTree.toml)
   add_fdb_test(TEST_FILES rare/RedwoodDeltaTree.toml)
@@ -251,12 +267,12 @@ if(WITH_PYTHON)
   add_fdb_test(
     TEST_FILES restarting/from_6.3.13/CycleTestRestart-1.txt
                restarting/from_6.3.13/CycleTestRestart-2.txt)
-  add_fdb_test(
-    TEST_FILES restarting/from_6.3.13/DrUpgradeRestart-1.txt
-               restarting/from_6.3.13/DrUpgradeRestart-2.txt)
   add_fdb_test(
     TEST_FILES restarting/from_6.3.13/StorefrontTestRestart-1.txt
                restarting/from_6.3.13/StorefrontTestRestart-2.txt)
+  add_fdb_test(
+    TEST_FILES restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt
+               restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt)
   add_fdb_test(
     TEST_FILES restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml
                restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml)
@@ -287,6 +303,9 @@ if(WITH_PYTHON)
   add_fdb_test(
     TEST_FILES restarting/from_7.1.0/VersionVectorEnableRestart-1.toml
                restarting/from_7.1.0/VersionVectorEnableRestart-2.toml)
+  add_fdb_test(
+    TEST_FILES restarting/from_7.2.0/DrUpgradeRestart-1.txt
+               restarting/from_7.2.0/DrUpgradeRestart-2.txt)
 
 
   add_fdb_test(TEST_FILES slow/ApiCorrectness.toml)
@@ -311,12 +330,14 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES slow/DiskFailureCycle.toml)
   add_fdb_test(TEST_FILES slow/FastTriggeredWatches.toml)
   add_fdb_test(TEST_FILES slow/LowLatencyWithFailures.toml)
+  add_fdb_test(TEST_FILES slow/MetaclusterManagement.toml)
   add_fdb_test(TEST_FILES slow/MoveKeysClean.toml)
   add_fdb_test(TEST_FILES slow/MoveKeysSideband.toml)
   add_fdb_test(TEST_FILES slow/RyowCorrectness.toml)
   add_fdb_test(TEST_FILES slow/Serializability.toml)
   add_fdb_test(TEST_FILES slow/SharedBackupCorrectness.toml)
   add_fdb_test(TEST_FILES slow/SharedBackupToDBCorrectness.toml)
+  add_fdb_test(TEST_FILES slow/SharedDefaultBackupCorrectness.toml)
   add_fdb_test(TEST_FILES slow/StorefrontTest.toml)
   add_fdb_test(TEST_FILES slow/SwizzledApiCorrectness.toml)
   add_fdb_test(TEST_FILES slow/SwizzledCycleTest.toml)
@@ -324,7 +345,9 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES slow/SwizzledRollbackTimeLapse.toml)
   add_fdb_test(TEST_FILES slow/SwizzledRollbackTimeLapseIncrement.toml)
   add_fdb_test(TEST_FILES slow/SwizzledTenantManagement.toml)
+  add_fdb_test(TEST_FILES slow/SwizzledTenantManagementMetacluster.toml)
   add_fdb_test(TEST_FILES slow/TenantManagement.toml)
+  add_fdb_test(TEST_FILES slow/TenantManagementConcurrency.toml)
   add_fdb_test(TEST_FILES slow/VersionStampBackupToDB.toml)
   add_fdb_test(TEST_FILES slow/VersionStampSwitchover.toml)
   add_fdb_test(TEST_FILES slow/WriteDuringReadAtomicRestore.toml)
@@ -358,19 +381,20 @@ if(WITH_PYTHON)
       NAME multiversion_client/unit_tests
       COMMAND $<TARGET_FILE:fdbserver> -r unittests -f /fdbclient/multiversionclient/
     )
-    set_tests_properties("multiversion_client/unit_tests" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
+    set_tests_properties("multiversion_client/unit_tests" PROPERTIES ENVIRONMENT "${SANITIZER_OPTIONS}")
     add_test(
       NAME threadsafe_threadfuture_to_future/unit_tests
       COMMAND $<TARGET_FILE:fdbserver> -r unittests -f /flow/safeThreadFutureToFuture/
     )
-    set_tests_properties("threadsafe_threadfuture_to_future/unit_tests" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
+    set_tests_properties("threadsafe_threadfuture_to_future/unit_tests" PROPERTIES ENVIRONMENT "${SANITIZER_OPTIONS}")
   endif()
 
   if(NOT OPEN_FOR_IDE)
     add_test(
       NAME command_line_argument_test
-      COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/argument_parsing/test_argument_parsing.py ${CMAKE_BINARY_DIR}
+      COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/argument_parsing/test_argument_parsing.py ${CMAKE_BINARY_DIR}
     )
+    set_tests_properties(command_line_argument_test PROPERTIES ENVIRONMENT "FDB_CLUSTER_FILE=${CMAKE_BINARY_DIR}/fdb.cluster")
   endif()
 
   verify_testing()
@@ -380,6 +404,42 @@ if(WITH_PYTHON)
       create_valgrind_correctness_package()
     endif()
   endif()
+
+  if (NOT OPEN_FOR_IDE AND NOT WIN32 AND NOT USE_SANITIZER)
+    # setup venv for testing token-based authorization
+    if (APPLE)
+      set(ld_env_name "DYLD_LIBRARY_PATH")
+    else()
+      set(ld_env_name "LD_LIBRARY_PATH")
+    endif()
+    set(authz_venv_dir ${CMAKE_CURRENT_BINARY_DIR}/authorization_test_venv)
+    set(authz_venv_activate ". ${authz_venv_dir}/bin/activate")
+    set(authz_venv_stamp_file ${authz_venv_dir}/venv.ready)
+    set(authz_venv_cmd "")
+    string(APPEND authz_venv_cmd "[[ ! -f ${authz_venv_stamp_file} ]] && ")
+    string(APPEND authz_venv_cmd "${Python3_EXECUTABLE} -m venv ${authz_venv_dir} ")
+    string(APPEND authz_venv_cmd "&& ${authz_venv_activate} ")
+    string(APPEND authz_venv_cmd "&& pip install --upgrade pip ")
+    string(APPEND authz_venv_cmd "&& pip install --upgrade -r ${CMAKE_SOURCE_DIR}/tests/authorization/requirements.txt ")
+    string(APPEND authz_venv_cmd "&& (cd ${CMAKE_BINARY_DIR}/bindings/python && python3 setup.py install) ")
+    string(APPEND authz_venv_cmd "&& touch ${authz_venv_stamp_file} ")
+    string(APPEND authz_venv_cmd "|| echo 'venv already set up'")
+    add_test(
+      NAME authorization_venv_setup
+      COMMAND bash -c ${authz_venv_cmd}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+    set_tests_properties(authorization_venv_setup PROPERTIES FIXTURES_SETUP authz_virtual_env TIMEOUT 60)
+
+    set(authz_script_dir ${CMAKE_SOURCE_DIR}/tests/authorization)
+    set(authz_test_cmd "${authz_venv_activate} && pytest ${authz_script_dir}/authz_test.py -rA --build-dir ${CMAKE_BINARY_DIR} -vvv")
+    add_test(
+      NAME token_based_tenant_authorization
+      WORKING_DIRECTORY ${authz_venv_dir}
+      COMMAND bash -c ${authz_test_cmd})
+    set_tests_properties(token_based_tenant_authorization PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_SOURCE_DIR}/tests/TestRunner;${ld_env_name}=${CMAKE_BINARY_DIR}/lib")
+    set_tests_properties(token_based_tenant_authorization PROPERTIES FIXTURES_REQUIRED authz_virtual_env)
+    set_tests_properties(token_based_tenant_authorization PROPERTIES TIMEOUT 120)
+  endif()
 else()
   message(WARNING "Python not found, won't configure ctest")
 endif()
diff --git a/tests/CTestCustom.ctest.cmake b/tests/CTestCustom.ctest.cmake
index 484bc07c71..b29793158d 100644
--- a/tests/CTestCustom.ctest.cmake
+++ b/tests/CTestCustom.ctest.cmake
@@ -1 +1 @@
-set(CTEST_CUSTOM_PRE_TEST ${CTEST_CUSTOM_PRE_TEST} "@Python_EXECUTABLE@ @PROJECT_SOURCE_DIR@/tests/TestRunner/TestDirectory.py @PROJECT_BINARY_DIR@")
+set(CTEST_CUSTOM_PRE_TEST ${CTEST_CUSTOM_PRE_TEST} "@Python3_EXECUTABLE@ @PROJECT_SOURCE_DIR@/tests/TestRunner/TestDirectory.py @PROJECT_BINARY_DIR@")
diff --git a/tests/ParallelRestoreApiCorrectnessAtomicRestore.txt b/tests/ParallelRestoreApiCorrectnessAtomicRestore.txt
index 1ecb871d31..82a78a7dcf 100644
--- a/tests/ParallelRestoreApiCorrectnessAtomicRestore.txt
+++ b/tests/ParallelRestoreApiCorrectnessAtomicRestore.txt
@@ -4,6 +4,8 @@ simBackupAgents=BackupToFile
 ;timeout is in seconds
 timeout=360000
 runSetup=true
+; tenants are not supported with parallel restore
+allowDefaultTenant=false
 
     testName=ApiCorrectness
     numKeys=5000
diff --git a/tests/ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml b/tests/ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml
index 15a0089787..e37d8d2572 100644
--- a/tests/ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml
+++ b/tests/ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml
@@ -5,6 +5,8 @@ simBackupAgents = 'BackupToFile'
 #timeout is in seconds
 timeout = 360000
 runSetup = true
+# tenants are not supported with parallel restore
+allowDefaultTenant = false
 
     [[test.workload]]
     testName = 'ApiCorrectness'
diff --git a/tests/TestRunner/binary_download.py b/tests/TestRunner/binary_download.py
index 2863d1e1f2..ea224dc360 100644
--- a/tests/TestRunner/binary_download.py
+++ b/tests/TestRunner/binary_download.py
@@ -10,9 +10,13 @@ import hashlib
 
 from local_cluster import random_secret_string
 
+CURRENT_VERSION = "7.2.0"
+FUTURE_VERSION = "7.3.0"
+
 SUPPORTED_PLATFORMS = ["x86_64", "aarch64"]
 SUPPORTED_VERSIONS = [
-    "7.2.0",
+    FUTURE_VERSION,
+    CURRENT_VERSION,
     "7.1.9",
     "7.1.8",
     "7.1.7",
@@ -67,7 +71,6 @@ SUPPORTED_VERSIONS = [
 ]
 FDB_DOWNLOAD_ROOT = "https://github.com/apple/foundationdb/releases/download/"
 LOCAL_OLD_BINARY_REPO = "/opt/foundationdb/old/"
-CURRENT_VERSION = "7.2.0"
 MAX_DOWNLOAD_ATTEMPTS = 5
 
 
@@ -93,18 +96,17 @@ def read_to_str(filename):
         return f.read()
 
 
+def is_local_build_version(version):
+    return version == CURRENT_VERSION or version == FUTURE_VERSION
+
+
 class FdbBinaryDownloader:
-    def __init__(
-        self,
-        build_dir
-    ):
+    def __init__(self, build_dir):
         self.build_dir = Path(build_dir).resolve()
         assert self.build_dir.exists(), "{} does not exist".format(build_dir)
         assert self.build_dir.is_dir(), "{} is not a directory".format(build_dir)
         self.platform = platform.machine()
-        assert self.platform in SUPPORTED_PLATFORMS, "Unsupported platform {}".format(
-            self.platform
-        )
+        assert self.platform in SUPPORTED_PLATFORMS, "Unsupported platform {}".format(self.platform)
         self.tmp_dir = self.build_dir.joinpath("tmp", random_secret_string(16))
         self.tmp_dir.mkdir(parents=True)
         self.download_dir = self.build_dir.joinpath("tmp", "old_binaries")
@@ -117,7 +119,7 @@ class FdbBinaryDownloader:
         return (self.local_binary_repo is not None) and (self.local_binary_repo.joinpath(version).exists())
 
     def binary_path(self, version, bin_name):
-        if version == CURRENT_VERSION:
+        if is_local_build_version(version):
             return self.build_dir.joinpath("bin", bin_name)
         elif self.version_in_local_repo(version):
             return self.local_binary_repo.joinpath(version, "bin", "{}-{}".format(bin_name, version))
@@ -125,7 +127,7 @@ class FdbBinaryDownloader:
             return self.download_dir.joinpath(version, bin_name)
 
     def lib_dir(self, version):
-        if version == CURRENT_VERSION:
+        if is_local_build_version(version):
             return self.build_dir.joinpath("lib")
         else:
             return self.download_dir.joinpath(version)
@@ -134,9 +136,7 @@ class FdbBinaryDownloader:
         return self.lib_dir(version).joinpath("libfdb_c.so")
 
     # Download an old binary of a given version from a remote repository
-    def download_old_binary(
-        self, version, target_bin_name, remote_bin_name, make_executable
-    ):
+    def download_old_binary(self, version, target_bin_name, remote_bin_name, make_executable):
         local_file = self.download_dir.joinpath(version, target_bin_name)
         if local_file.exists():
             return
@@ -152,9 +152,7 @@ class FdbBinaryDownloader:
 
         for attempt_cnt in range(MAX_DOWNLOAD_ATTEMPTS + 1):
             if attempt_cnt == MAX_DOWNLOAD_ATTEMPTS:
-                assert False, "Failed to download {} after {} attempts".format(
-                    local_file_tmp, MAX_DOWNLOAD_ATTEMPTS
-                )
+                assert False, "Failed to download {} after {} attempts".format(local_file_tmp, MAX_DOWNLOAD_ATTEMPTS)
             try:
                 print("Downloading '{}' to '{}'...".format(remote_file, local_file_tmp))
                 request.urlretrieve(remote_file, local_file_tmp)
@@ -172,11 +170,7 @@ class FdbBinaryDownloader:
             if expected_checksum == actual_checkum:
                 print("Checksum OK")
                 break
-            print(
-                "Checksum mismatch. Expected: {} Actual: {}".format(
-                    expected_checksum, actual_checkum
-                )
-            )
+            print("Checksum mismatch. Expected: {} Actual: {}".format(expected_checksum, actual_checkum))
 
         os.rename(local_file_tmp, local_file)
         os.remove(local_sha256)
@@ -202,22 +196,14 @@ class FdbBinaryDownloader:
 
     # Download all old binaries required for testing the specified upgrade path
     def download_old_binaries(self, version):
-        if version == CURRENT_VERSION:
+        if is_local_build_version(version):
             return
 
         if self.version_in_local_repo(version):
             self.copy_clientlib_from_local_repo(version)
             return
 
-        self.download_old_binary(
-            version, "fdbserver", "fdbserver.{}".format(self.platform), True
-        )
-        self.download_old_binary(
-            version, "fdbmonitor", "fdbmonitor.{}".format(self.platform), True
-        )
-        self.download_old_binary(
-            version, "fdbcli", "fdbcli.{}".format(self.platform), True
-        )
-        self.download_old_binary(
-            version, "libfdb_c.so", "libfdb_c.{}.so".format(self.platform), False
-        )
+        self.download_old_binary(version, "fdbserver", "fdbserver.{}".format(self.platform), True)
+        self.download_old_binary(version, "fdbmonitor", "fdbmonitor.{}".format(self.platform), True)
+        self.download_old_binary(version, "fdbcli", "fdbcli.{}".format(self.platform), True)
+        self.download_old_binary(version, "libfdb_c.so", "libfdb_c.{}.so".format(self.platform), False)
diff --git a/tests/TestRunner/local_cluster.py b/tests/TestRunner/local_cluster.py
index 9500e94b67..6375a7a01e 100644
--- a/tests/TestRunner/local_cluster.py
+++ b/tests/TestRunner/local_cluster.py
@@ -39,18 +39,20 @@ def is_port_in_use(port):
 
 valid_letters_for_secret = string.ascii_letters + string.digits
 
+
 class TLSConfig:
     # Passing a negative chain length generates expired leaf certificate
     def __init__(
         self,
         server_chain_len: int = 3,
         client_chain_len: int = 2,
-        verify_peers = "Check.Valid=1",
+        verify_peers="Check.Valid=1",
     ):
         self.server_chain_len = server_chain_len
         self.client_chain_len = client_chain_len
         self.verify_peers = verify_peers
 
+
 def random_secret_string(length):
     return "".join(random.choice(valid_letters_for_secret) for _ in range(length))
 
@@ -84,6 +86,9 @@ datadir = {datadir}/$ID
 logdir = {logdir}
 {bg_knob_line}
 {tls_config}
+{authz_public_key_config}
+{custom_config}
+{use_future_protocol_version}
 # logsize = 10MiB
 # maxlogssize = 100MiB
 # machine-id =
@@ -110,9 +115,12 @@ logdir = {logdir}
         port=None,
         ip_address=None,
         blob_granules_enabled: bool = False,
+        use_future_protocol_version: bool = False,
         redundancy: str = "single",
         tls_config: TLSConfig = None,
         mkcert_binary: str = "",
+        custom_config: dict = {},
+        public_key_json_str: str = "",
     ):
         self.basedir = Path(basedir)
         self.etc = self.basedir.joinpath("etc")
@@ -133,10 +141,12 @@ logdir = {logdir}
         self.redundancy = redundancy
         self.ip_address = "127.0.0.1" if ip_address is None else ip_address
         self.first_port = port
+        self.custom_config = custom_config
         self.blob_granules_enabled = blob_granules_enabled
         if blob_granules_enabled:
             # add extra process for blob_worker
             self.process_number += 1
+        self.use_future_protocol_version = use_future_protocol_version
 
         if self.first_port is not None:
             self.last_used_port = int(self.first_port) - 1
@@ -153,6 +163,7 @@ logdir = {logdir}
         self.coordinators = set()
         self.active_servers = set(self.server_ports.keys())
         self.tls_config = tls_config
+        self.public_key_json_file = None
         self.mkcert_binary = Path(mkcert_binary)
         self.server_cert_file = self.cert.joinpath("server_cert.pem")
         self.client_cert_file = self.cert.joinpath("client_cert.pem")
@@ -161,6 +172,11 @@ logdir = {logdir}
         self.server_ca_file = self.cert.joinpath("server_ca.pem")
         self.client_ca_file = self.cert.joinpath("client_ca.pem")
 
+        if public_key_json_str:
+            self.public_key_json_file = self.etc.joinpath("public_keys.json")
+            with open(self.public_key_json_file, "w") as pubkeyfile:
+                pubkeyfile.write(public_key_json_str)
+
         if create_config:
             self.create_cluster_file()
             self.save_config()
@@ -168,6 +184,8 @@ logdir = {logdir}
         if self.tls_config is not None:
             self.create_tls_cert()
 
+        self.cluster_file = self.etc.joinpath("fdb.cluster")
+
     def __next_port(self):
         if self.first_port is None:
             return get_free_port()
@@ -193,7 +211,10 @@ logdir = {logdir}
                     ip_address=self.ip_address,
                     bg_knob_line=bg_knob_line,
                     tls_config=self.tls_conf_string(),
+                    authz_public_key_config=self.authz_public_key_conf_string(),
                     optional_tls=":tls" if self.tls_config is not None else "",
+                    custom_config='\n'.join(["{} = {}".format(key, value) for key, value in self.custom_config.items()]),
+                    use_future_protocol_version="use-future-protocol-version = true" if self.use_future_protocol_version else "",
                 )
             )
             # By default, the cluster only has one process
@@ -278,9 +299,16 @@ logdir = {logdir}
     def __fdbcli_exec(self, cmd, stdout, stderr, timeout):
         args = [self.fdbcli_binary, "-C", self.cluster_file, "--exec", cmd]
         if self.tls_config:
-            args += ["--tls-certificate-file", self.client_cert_file,
-                     "--tls-key-file", self.client_key_file,
-                     "--tls-ca-file", self.server_ca_file]
+            args += [
+                "--tls-certificate-file",
+                self.client_cert_file,
+                "--tls-key-file",
+                self.client_key_file,
+                "--tls-ca-file",
+                self.server_ca_file,
+            ]
+        if self.use_future_protocol_version:
+            args += ["--use-future-protocol-version"]
         res = subprocess.run(args, env=self.process_env(), stderr=stderr, stdout=stdout, timeout=timeout)
         assert res.returncode == 0, "fdbcli command {} failed with {}".format(cmd, res.returncode)
         return res.stdout
@@ -307,22 +335,32 @@ logdir = {logdir}
     # Generate and install test certificate chains and keys
     def create_tls_cert(self):
         assert self.tls_config is not None, "TLS not enabled"
-        assert self.mkcert_binary.exists() and self.mkcert_binary.is_file(), "{} does not exist".format(self.mkcert_binary)
+        assert self.mkcert_binary.exists() and self.mkcert_binary.is_file(), "{} does not exist".format(
+            self.mkcert_binary
+        )
         self.cert.mkdir(exist_ok=True)
         server_chain_len = abs(self.tls_config.server_chain_len)
         client_chain_len = abs(self.tls_config.client_chain_len)
-        expire_server_cert = (self.tls_config.server_chain_len < 0)
-        expire_client_cert = (self.tls_config.client_chain_len < 0)
+        expire_server_cert = self.tls_config.server_chain_len < 0
+        expire_client_cert = self.tls_config.client_chain_len < 0
         args = [
             str(self.mkcert_binary),
-            "--server-chain-length", str(server_chain_len),
-            "--client-chain-length", str(client_chain_len),
-            "--server-cert-file", str(self.server_cert_file),
-            "--client-cert-file", str(self.client_cert_file),
-            "--server-key-file", str(self.server_key_file),
-            "--client-key-file", str(self.client_key_file),
-            "--server-ca-file", str(self.server_ca_file),
-            "--client-ca-file", str(self.client_ca_file),
+            "--server-chain-length",
+            str(server_chain_len),
+            "--client-chain-length",
+            str(client_chain_len),
+            "--server-cert-file",
+            str(self.server_cert_file),
+            "--client-cert-file",
+            str(self.client_cert_file),
+            "--server-key-file",
+            str(self.server_key_file),
+            "--client-key-file",
+            str(self.client_key_file),
+            "--server-ca-file",
+            str(self.server_ca_file),
+            "--client-ca-file",
+            str(self.client_ca_file),
             "--print-args",
         ]
         if expire_server_cert:
@@ -344,6 +382,12 @@ logdir = {logdir}
             }
             return "\n".join("{} = {}".format(k, v) for k, v in conf_map.items())
 
+    def authz_public_key_conf_string(self):
+        if self.public_key_json_file is not None:
+            return "authorization-public-key-file = {}".format(self.public_key_json_file)
+        else:
+            return ""
+
     # Get cluster status using fdbcli
     def get_status(self):
         status_output = self.fdbcli_exec_and_get("status json")
@@ -372,6 +416,7 @@ logdir = {logdir}
     def get_coordinators_from_status(self):
         def is_coordinator(proc_status):
             return any(entry["role"] == "coordinator" for entry in proc_status["roles"])
+
         return self.get_servers_from_status(is_coordinator)
 
     def process_env(self):
@@ -404,35 +449,38 @@ logdir = {logdir}
     def wait_for_server_update(self, timeout=CLUSTER_UPDATE_TIMEOUT_SEC):
         time_limit = time.time() + timeout
         servers_found = set()
-        while (time.time() <= time_limit):
+        while time.time() <= time_limit:
             servers_found = self.get_all_servers_from_status()
-            if (servers_found != self.active_servers):
+            if servers_found != self.active_servers:
                 break
             time.sleep(RETRY_INTERVAL_SEC)
         assert "Failed to apply server changes after {}sec. Expected: {}, Actual: {}".format(
-            timeout,  self.active_servers, servers_found)
+            timeout, self.active_servers, servers_found
+        )
 
     # Apply changes to the set of the coordinators, based on the current value of self.coordinators
     def update_coordinators(self):
         urls = ["{}:{}".format(self.ip_address, self.server_ports[id]) for id in self.coordinators]
         self.fdbcli_exec("coordinators {}".format(" ".join(urls)))
 
-     # Wait until the changes to the set of the coordinators are applied
+    # Wait until the changes to the set of the coordinators are applied
     def wait_for_coordinator_update(self, timeout=CLUSTER_UPDATE_TIMEOUT_SEC):
         time_limit = time.time() + timeout
         coord_found = set()
-        while (time.time() <= time_limit):
+        while time.time() <= time_limit:
             coord_found = self.get_coordinators_from_status()
-            if (coord_found != self.coordinators):
+            if coord_found != self.coordinators:
                 break
             time.sleep(RETRY_INTERVAL_SEC)
         assert "Failed to apply coordinator changes after {}sec. Expected: {}, Actual: {}".format(
-            timeout, self.coordinators, coord_found)
+            timeout, self.coordinators, coord_found
+        )
         # Check if the cluster file was successfully updated too
         connection_string = open(self.cluster_file, "r").read()
         for server_id in self.coordinators:
-            assert connection_string.find(str(self.server_ports[server_id])) != -1, \
-                "Missing coordinator {} port {} in the cluster file".format(server_id, self.server_ports[server_id])
+            assert (
+                connection_string.find(str(self.server_ports[server_id])) != -1
+            ), "Missing coordinator {} port {} in the cluster file".format(server_id, self.server_ports[server_id])
 
     # Exclude the servers with the given ID from the cluster, i.e. move out their data
     # The method waits until the changes are applied
@@ -445,19 +493,25 @@ logdir = {logdir}
         old_servers = self.active_servers.copy()
         new_servers = set()
         print("Starting cluster wiggle")
-        print("Old servers: {} on ports {}".format(old_servers, [
-              self.server_ports[server_id] for server_id in old_servers]))
+        print(
+            "Old servers: {} on ports {}".format(
+                old_servers, [self.server_ports[server_id] for server_id in old_servers]
+            )
+        )
         print("Old coordinators: {}".format(self.coordinators))
 
         # Step 1: add new servers
         start_time = time.time()
         for _ in range(len(old_servers)):
             new_servers.add(self.add_server())
-        print("New servers: {} on ports {}".format(new_servers, [
-              self.server_ports[server_id] for server_id in new_servers]))
+        print(
+            "New servers: {} on ports {}".format(
+                new_servers, [self.server_ports[server_id] for server_id in new_servers]
+            )
+        )
         self.save_config()
         self.wait_for_server_update()
-        print("New servers successfully added to the cluster. Time: {}s".format(time.time()-start_time))
+        print("New servers successfully added to the cluster. Time: {}s".format(time.time() - start_time))
 
         # Step 2: change coordinators
         start_time = time.time()
@@ -466,12 +520,12 @@ logdir = {logdir}
         self.coordinators = new_coordinators.copy()
         self.update_coordinators()
         self.wait_for_coordinator_update()
-        print("Coordinators successfully changed. Time: {}s".format(time.time()-start_time))
+        print("Coordinators successfully changed. Time: {}s".format(time.time() - start_time))
 
         # Step 3: exclude old servers from the cluster, i.e. move out their data
         start_time = time.time()
         self.exclude_servers(old_servers)
-        print("Old servers successfully excluded from the cluster. Time: {}s".format(time.time()-start_time))
+        print("Old servers successfully excluded from the cluster. Time: {}s".format(time.time() - start_time))
 
         # Step 4: remove the old servers
         start_time = time.time()
@@ -479,4 +533,4 @@ logdir = {logdir}
             self.remove_server(server_id)
         self.save_config()
         self.wait_for_server_update()
-        print("Old servers successfully removed from the cluster. Time: {}s".format(time.time()-start_time))
+        print("Old servers successfully removed from the cluster. Time: {}s".format(time.time() - start_time))
diff --git a/tests/TestRunner/tmp_cluster.py b/tests/TestRunner/tmp_cluster.py
index ebe789ab6b..dc3f62f391 100755
--- a/tests/TestRunner/tmp_cluster.py
+++ b/tests/TestRunner/tmp_cluster.py
@@ -18,6 +18,10 @@ class TempCluster(LocalCluster):
         port: str = None,
         blob_granules_enabled: bool = False,
         tls_config: TLSConfig = None,
+        public_key_json_str: str = None,
+        remove_at_exit: bool = True,
+        custom_config: dict = {},
+        enable_tenants: bool = True,
     ):
         self.build_dir = Path(build_dir).resolve()
         assert self.build_dir.exists(), "{} does not exist".format(build_dir)
@@ -25,6 +29,8 @@ class TempCluster(LocalCluster):
         tmp_dir = self.build_dir.joinpath("tmp", random_secret_string(16))
         tmp_dir.mkdir(parents=True)
         self.tmp_dir = tmp_dir
+        self.remove_at_exit = remove_at_exit
+        self.enable_tenants = enable_tenants
         super().__init__(
             tmp_dir,
             self.build_dir.joinpath("bin", "fdbserver"),
@@ -35,20 +41,27 @@ class TempCluster(LocalCluster):
             blob_granules_enabled=blob_granules_enabled,
             tls_config=tls_config,
             mkcert_binary=self.build_dir.joinpath("bin", "mkcert"),
+            public_key_json_str=public_key_json_str,
+            custom_config=custom_config,
         )
 
     def __enter__(self):
         super().__enter__()
-        super().create_database()
+        if self.enable_tenants:
+            super().create_database()
+        else:
+            super().create_database(enable_tenants=False)
         return self
 
     def __exit__(self, xc_type, exc_value, traceback):
         super().__exit__(xc_type, exc_value, traceback)
-        shutil.rmtree(self.tmp_dir)
+        if self.remove_at_exit:
+            shutil.rmtree(self.tmp_dir)
 
     def close(self):
         super().__exit__(None, None, None)
-        shutil.rmtree(self.tmp_dir)
+        if self.remove_at_exit:
+            shutil.rmtree(self.tmp_dir)
 
 
 if __name__ == "__main__":
@@ -91,6 +104,12 @@ if __name__ == "__main__":
         help="Do not dump cluster log on error",
         action="store_true",
     )
+    parser.add_argument(
+        "--disable-tenants",
+        help="Do not enable tenant mode",
+        action="store_true",
+        default=False
+    )
     parser.add_argument(
         "--blob-granules-enabled", help="Enable blob granules", action="store_true"
     )
@@ -115,6 +134,12 @@ if __name__ == "__main__":
         default="Check.Valid=1",
     )
     args = parser.parse_args()
+
+    if args.disable_tenants:
+        enable_tenants = False
+    else:
+        enable_tenants = True
+
     tls_config = None
     if args.tls_enabled:
         tls_config = TLSConfig(server_chain_len=args.server_cert_chain_len,
@@ -125,15 +150,16 @@ if __name__ == "__main__":
         args.process_number,
         blob_granules_enabled=args.blob_granules_enabled,
         tls_config=tls_config,
+        enable_tenants=enable_tenants,
     ) as cluster:
         print("log-dir: {}".format(cluster.log))
         print("etc-dir: {}".format(cluster.etc))
         print("data-dir: {}".format(cluster.data))
-        print("cluster-file: {}".format(cluster.etc.joinpath("fdb.cluster")))
+        print("cluster-file: {}".format(cluster.cluster_file))
         cmd_args = []
         for cmd in args.cmd:
             if cmd == "@CLUSTER_FILE@":
-                cmd_args.append(str(cluster.etc.joinpath("fdb.cluster")))
+                cmd_args.append(str(cluster.cluster_file))
             elif cmd == "@DATA_DIR@":
                 cmd_args.append(str(cluster.data))
             elif cmd == "@LOG_DIR@":
@@ -160,7 +186,7 @@ if __name__ == "__main__":
                 cmd_args.append(cmd)
         env = dict(**os.environ)
         env["FDB_CLUSTER_FILE"] = env.get(
-            "FDB_CLUSTER_FILE", cluster.etc.joinpath("fdb.cluster")
+            "FDB_CLUSTER_FILE", cluster.cluster_file
         )
         errcode = subprocess.run(
             cmd_args, stdout=sys.stdout, stderr=sys.stderr, env=env
diff --git a/tests/TestRunner/tmp_multi_cluster.py b/tests/TestRunner/tmp_multi_cluster.py
old mode 100644
new mode 100755
diff --git a/tests/TestRunner/upgrade_test.py b/tests/TestRunner/upgrade_test.py
index 62055ce0a7..8f43e0cddf 100755
--- a/tests/TestRunner/upgrade_test.py
+++ b/tests/TestRunner/upgrade_test.py
@@ -11,15 +11,18 @@ import sys
 from threading import Thread, Event
 import traceback
 import time
-from binary_download import FdbBinaryDownloader, SUPPORTED_VERSIONS, CURRENT_VERSION
+from binary_download import FdbBinaryDownloader, SUPPORTED_VERSIONS, CURRENT_VERSION, FUTURE_VERSION
 from local_cluster import LocalCluster, random_secret_string
 
+TENANT_API_VERSION = 720
+
 CLUSTER_ACTIONS = ["wiggle"]
 HEALTH_CHECK_TIMEOUT_SEC = 5
 PROGRESS_CHECK_TIMEOUT_SEC = 30
 TESTER_STATS_INTERVAL_SEC = 5
 TRANSACTION_RETRY_LIMIT = 100
 RUN_WITH_GDB = False
+CLEANUP_ON_EXIT = True
 
 
 def version_from_str(ver_str):
@@ -44,10 +47,7 @@ def random_sleep(min_sec, max_sec):
 
 
 class UpgradeTest:
-    def __init__(
-        self,
-        args
-    ):
+    def __init__(self, args):
         self.build_dir = Path(args.build_dir).resolve()
         assert self.build_dir.exists(), "{} does not exist".format(args.build_dir)
         assert self.build_dir.is_dir(), "{} is not a directory".format(args.build_dir)
@@ -62,6 +62,10 @@ class UpgradeTest:
         self.downloader = FdbBinaryDownloader(args.build_dir)
         self.download_old_binaries()
         self.create_external_lib_dir()
+        self.testing_future_version = FUTURE_VERSION in self.upgrade_path
+        self.future_version_client_lib_path = (
+            self.downloader.lib_path(FUTURE_VERSION) if self.testing_future_version else None
+        )
         init_version = self.upgrade_path[0]
         self.cluster = LocalCluster(
             self.tmp_dir,
@@ -70,19 +74,16 @@ class UpgradeTest:
             self.downloader.binary_path(init_version, "fdbcli"),
             args.process_number,
             create_config=False,
-            redundancy=args.redundancy
+            redundancy=args.redundancy,
+            blob_granules_enabled=args.blob_granules_enabled,
         )
         self.cluster.create_cluster_file()
         self.configure_version(init_version)
         self.log = self.cluster.log
         self.etc = self.cluster.etc
         self.data = self.cluster.data
-        self.input_pipe_path = self.tmp_dir.joinpath(
-            "input.{}".format(random_secret_string(8))
-        )
-        self.output_pipe_path = self.tmp_dir.joinpath(
-            "output.{}".format(random_secret_string(8))
-        )
+        self.input_pipe_path = self.tmp_dir.joinpath("input.{}".format(random_secret_string(8)))
+        self.output_pipe_path = self.tmp_dir.joinpath("output.{}".format(random_secret_string(8)))
         os.mkfifo(self.input_pipe_path)
         os.mkfifo(self.output_pipe_path)
         self.progress_event = Event()
@@ -91,6 +92,7 @@ class UpgradeTest:
         self.tester_proc = None
         self.output_pipe = None
         self.ctrl_pipe = None
+        self.determine_api_version()
 
     # Download all old binaries required for testing the specified upgrade path
     def download_old_binaries(self):
@@ -103,11 +105,11 @@ class UpgradeTest:
         self.external_lib_dir = self.tmp_dir.joinpath("client_libs")
         self.external_lib_dir.mkdir(parents=True)
         for version in self.used_versions:
+            if version == FUTURE_VERSION:
+                continue
             src_file_path = self.downloader.lib_path(version)
             assert src_file_path.exists(), "{} does not exist".format(src_file_path)
-            target_file_path = self.external_lib_dir.joinpath(
-                "libfdb_c.{}.so".format(version)
-            )
+            target_file_path = self.external_lib_dir.joinpath("libfdb_c.{}.so".format(version))
             shutil.copyfile(src_file_path, target_file_path)
 
     # Perform a health check of the cluster: Use fdbcli status command to check if the number of
@@ -123,19 +125,16 @@ class UpgradeTest:
                 continue
             num_proc = len(status["cluster"]["processes"])
             if num_proc != self.cluster.process_number:
-                print(
-                    "Health check: {} of {} processes found. Retrying".format(
-                        num_proc, self.cluster.process_number
-                    )
-                )
+                print("Health check: {} of {} processes found. Retrying".format(num_proc, self.cluster.process_number))
                 time.sleep(1)
                 continue
+            expected_version = self.cluster_version
+            if expected_version == FUTURE_VERSION:
+                expected_version = CURRENT_VERSION
             for (_, proc_stat) in status["cluster"]["processes"].items():
                 proc_ver = proc_stat["version"]
-                assert (
-                    proc_ver == self.cluster_version
-                ), "Process version: expected: {}, actual: {}".format(
-                    self.cluster_version, proc_ver
+                assert proc_ver == expected_version, "Process version: expected: {}, actual: {}".format(
+                    expected_version, proc_ver
                 )
             print("Health check: OK")
             return
@@ -146,9 +145,10 @@ class UpgradeTest:
         self.cluster.fdbmonitor_binary = self.downloader.binary_path(version, "fdbmonitor")
         self.cluster.fdbserver_binary = self.downloader.binary_path(version, "fdbserver")
         self.cluster.fdbcli_binary = self.downloader.binary_path(version, "fdbcli")
-        self.cluster.set_env_var("LD_LIBRARY_PATH", self.downloader.lib_dir(version))
-        if version_before(version, "7.1.0"):
-            self.cluster.use_legacy_conf_syntax = True
+        self.cluster.set_env_var("LD_LIBRARY_PATH", "%s:%s" % (
+            self.downloader.lib_dir(version), os.getenv("LD_LIBRARY_PATH")))
+        self.cluster.use_legacy_conf_syntax = version_before(version, "7.1.0")
+        self.cluster.use_future_protocol_version = version == FUTURE_VERSION
         self.cluster.save_config()
         self.cluster_version = version
 
@@ -164,12 +164,13 @@ class UpgradeTest:
     def __enter__(self):
         print("Starting cluster version {}".format(self.cluster_version))
         self.cluster.start_cluster()
-        self.cluster.create_database(enable_tenants=False)
+        self.cluster.create_database(enable_tenants=(self.api_version >= TENANT_API_VERSION))
         return self
 
     def __exit__(self, xc_type, exc_value, traceback):
         self.cluster.stop_cluster()
-        shutil.rmtree(self.tmp_dir)
+        if CLEANUP_ON_EXIT:
+            shutil.rmtree(self.tmp_dir)
 
     # Determine FDB API version matching the upgrade path
     def determine_api_version(self):
@@ -181,7 +182,6 @@ class UpgradeTest:
     def exec_workload(self, test_file):
         self.tester_retcode = 1
         try:
-            self.determine_api_version()
             cmd_args = [
                 self.tester_bin,
                 "--cluster-file",
@@ -205,19 +205,17 @@ class UpgradeTest:
                 "--transaction-retry-limit",
                 str(TRANSACTION_RETRY_LIMIT),
                 "--stats-interval",
-                str(TESTER_STATS_INTERVAL_SEC * 1000)
+                str(TESTER_STATS_INTERVAL_SEC * 1000),
             ]
             if RUN_WITH_GDB:
                 cmd_args = ["gdb", "-ex", "run", "--args"] + cmd_args
-            print(
-                "Executing test command: {}".format(
-                    " ".join([str(c) for c in cmd_args])
-                )
-            )
+            if FUTURE_VERSION in self.upgrade_path:
+                cmd_args += ["--future-version-client-library", self.future_version_client_lib_path]
+            if self.cluster.blob_granules_enabled:
+                cmd_args += ["--blob-granule-local-file-path", str(self.cluster.data.joinpath("fdbblob")) + "/"]
+            print("Executing test command: {}".format(" ".join([str(c) for c in cmd_args])))
 
-            self.tester_proc = subprocess.Popen(
-                cmd_args, stdout=sys.stdout, stderr=sys.stderr
-            )
+            self.tester_proc = subprocess.Popen(cmd_args, stdout=sys.stdout, stderr=sys.stderr)
             self.tester_retcode = self.tester_proc.wait()
             self.tester_proc = None
 
@@ -241,9 +239,7 @@ class UpgradeTest:
         if self.progress_event.is_set():
             print("Progress check: OK")
         else:
-            assert False, "Progress check failed after upgrade to version {}".format(
-                self.cluster_version
-            )
+            assert False, "Progress check failed after upgrade to version {}".format(self.cluster_version)
 
     # The main function of a thread for reading and processing
     # the notifications received from the tester
@@ -324,11 +320,7 @@ class UpgradeTest:
 
     def grep_logs_for_events(self, severity):
         return (
-            subprocess.getoutput(
-                "grep -r 'Severity=\"{}\"' {}".format(
-                    severity, self.cluster.log.as_posix()
-                )
-            )
+            subprocess.getoutput("grep -r 'Severity=\"{}\"' {}".format(severity, self.cluster.log.as_posix()))
             .rstrip()
             .splitlines()
         )
@@ -336,9 +328,7 @@ class UpgradeTest:
     # Check the cluster log for errors
     def check_cluster_logs(self, error_limit=100):
         sev40s = (
-            subprocess.getoutput(
-                "grep -r 'Severity=\"40\"' {}".format(self.cluster.log.as_posix())
-            )
+            subprocess.getoutput("grep -r 'Severity=\"40\"' {}".format(self.cluster.log.as_posix()))
             .rstrip()
             .splitlines()
         )
@@ -349,7 +339,7 @@ class UpgradeTest:
             # correct asan annotations so that it shouldn't produce any false positives.
             if line.endswith(
                 "WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false "
-                "positives in some cases! "
+                "positives in some cases!"
             ):
                 continue
             if err_cnt < error_limit:
@@ -368,9 +358,7 @@ class UpgradeTest:
     # Check the server and client logs for warnings and dump them
     def dump_warnings_in_logs(self, limit=100):
         sev30s = (
-            subprocess.getoutput(
-                "grep -r 'Severity=\"30\"' {}".format(self.cluster.log.as_posix())
-            )
+            subprocess.getoutput("grep -r 'Severity=\"30\"' {}".format(self.cluster.log.as_posix()))
             .rstrip()
             .splitlines()
         )
@@ -378,11 +366,7 @@ class UpgradeTest:
         if len(sev30s) == 0:
             print("No warnings found in logs")
         else:
-            print(
-                ">>>>>>>>>>>>>>>>>>>> Found {} severity 30 events (warnings):".format(
-                    len(sev30s)
-                )
-            )
+            print(">>>>>>>>>>>>>>>>>>>> Found {} severity 30 events (warnings):".format(len(sev30s)))
             for line in sev30s[:limit]:
                 print(line)
 
@@ -418,8 +402,8 @@ if __name__ == "__main__":
     parser.add_argument(
         "--upgrade-path",
         nargs="+",
-        help="Cluster upgrade path: a space separated list of versions.\n" +
-        "The list may also contain cluster change actions: {}".format(CLUSTER_ACTIONS),
+        help="Cluster upgrade path: a space separated list of versions.\n"
+        + "The list may also contain cluster change actions: {}".format(CLUSTER_ACTIONS),
         default=[CURRENT_VERSION],
     )
     parser.add_argument(
@@ -446,8 +430,12 @@ if __name__ == "__main__":
         action="store_true",
     )
     parser.add_argument(
-        "--run-with-gdb", help="Execute the tester binary from gdb", action="store_true"
+        "--no-cleanup-on-error",
+        help="In case of an error do not remove any of the generated files",
+        action="store_true",
     )
+    parser.add_argument("--blob-granules-enabled", help="Enable blob granules", action="store_true")
+    parser.add_argument("--run-with-gdb", help="Execute the tester binary from gdb", action="store_true")
     args = parser.parse_args()
     if args.process_number == 0:
         args.process_number = random.randint(1, 5)
@@ -471,5 +459,7 @@ if __name__ == "__main__":
         test.dump_warnings_in_logs()
         if errcode != 0 and not args.disable_log_dump:
             test.dump_cluster_logs()
+        if errcode != 0 and args.no_cleanup_on_error:
+            CLEANUP_ON_EXIT = False
 
     sys.exit(errcode)
diff --git a/tests/authorization/admin_server.py b/tests/authorization/admin_server.py
new file mode 100644
index 0000000000..9f2c2adc9d
--- /dev/null
+++ b/tests/authorization/admin_server.py
@@ -0,0 +1,135 @@
+#!/usr/bin/python
+#
+# admin_server.py
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import fdb
+from multiprocessing import Pipe, Process
+from typing import Union, List
+from util import to_str, to_bytes, cleanup_tenant
+
+class _admin_request(object):
+    def __init__(self, op: str, args: List[Union[str, bytes]]=[]):
+        self.op = op
+        self.args = args
+
+    def __str__(self):
+        return f"admin_request({self.op}, {self.args})"
+
+    def __repr__(self):
+        return f"admin_request({self.op}, {self.args})"
+
+def main_loop(main_pipe, pipe):
+    main_pipe.close()
+    db = None
+    while True:
+        try:
+            req = pipe.recv()
+        except EOFError:
+            return
+        if not isinstance(req, _admin_request):
+            pipe.send(TypeError("unexpected type {}".format(type(req))))
+            continue
+        op = req.op
+        args = req.args
+        resp = True
+        try:
+            if op == "connect":
+                db = fdb.open(req.args[0])
+            elif op == "configure_tls":
+                keyfile, certfile, cafile = req.args[:3]
+                fdb.options.set_tls_key_path(keyfile)
+                fdb.options.set_tls_cert_path(certfile)
+                fdb.options.set_tls_ca_path(cafile)
+            elif op == "create_tenant":
+                if db is None:
+                    resp = Exception("db not open")
+                else:
+                    for tenant in req.args:
+                        tenant_str = to_str(tenant)
+                        tenant_bytes = to_bytes(tenant)
+                        fdb.tenant_management.create_tenant(db, tenant_bytes)
+            elif op == "delete_tenant":
+                if db is None:
+                    resp = Exception("db not open")
+                else:
+                    for tenant in req.args:
+                        tenant_str = to_str(tenant)
+                        tenant_bytes = to_bytes(tenant)
+                        cleanup_tenant(db, tenant_bytes)
+            elif op == "cleanup_database":
+                if db is None:
+                    resp = Exception("db not open")
+                else:
+                    tr = db.create_transaction()
+                    del tr[b'':b'\xff']
+                    tr.commit().wait()
+                    tenants = list(map(lambda x: x.key, list(fdb.tenant_management.list_tenants(db, b'', b'\xff', 0).to_list())))
+                    for tenant in tenants:
+                        fdb.tenant_management.delete_tenant(db, tenant)
+            elif op == "terminate":
+                pipe.send(True)
+                return
+            else:
+                resp = ValueError("unknown operation: {}".format(req))
+        except Exception as e:
+            resp = e
+        pipe.send(resp)
+
+_admin_server = None
+
+def get():
+    return _admin_server
+
+# server needs to be a singleton running in subprocess, because FDB network layer (including active TLS config) is a global var
+class Server(object):
+    def __init__(self):
+        global _admin_server
+        assert _admin_server is None, "admin server may be setup once per process"
+        _admin_server = self
+        self._main_pipe, self._admin_pipe = Pipe(duplex=True)
+        self._admin_proc = Process(target=main_loop, args=(self._main_pipe, self._admin_pipe))
+
+    def start(self):
+        self._admin_proc.start()
+
+    def join(self):
+        self._main_pipe.close()
+        self._admin_pipe.close()
+        self._admin_proc.join()
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        self.join()
+
+    def request(self, op, args=[]):
+        req = _admin_request(op, args)
+        try:
+            self._main_pipe.send(req)
+            resp = self._main_pipe.recv()
+            if resp != True:
+                print("{} failed: {}".format(req, resp))
+                raise resp
+            else:
+                print("{} succeeded".format(req))
+        except Exception as e:
+            print("{} failed by exception: {}".format(req, e))
+            raise
diff --git a/tests/authorization/authz_test.py b/tests/authorization/authz_test.py
new file mode 100644
index 0000000000..fe379144fa
--- /dev/null
+++ b/tests/authorization/authz_test.py
@@ -0,0 +1,297 @@
+#!/usr/bin/python
+#
+# authz_test.py
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import admin_server
+import argparse
+import authlib
+import fdb
+import os
+import pytest
+import random
+import sys
+import time
+from multiprocessing import Process, Pipe
+from typing import Union
+from util import alg_from_kty, public_keyset_from_keys, random_alphanum_str, random_alphanum_bytes, to_str, to_bytes, KeyFileReverter, token_claim_1h, wait_until_tenant_tr_succeeds, wait_until_tenant_tr_fails
+
+special_key_ranges = [
+    ("transaction description", b"/description", b"/description\x00"),
+    ("global knobs", b"/globalKnobs", b"/globalKnobs\x00"),
+    ("knobs", b"/knobs0", b"/knobs0\x00"),
+    ("conflicting keys", b"/transaction/conflicting_keys/", b"/transaction/conflicting_keys/\xff\xff"),
+    ("read conflict range", b"/transaction/read_conflict_range/", b"/transaction/read_conflict_range/\xff\xff"),
+    ("conflicting keys", b"/transaction/write_conflict_range/", b"/transaction/write_conflict_range/\xff\xff"),
+    ("data distribution stats", b"/metrics/data_distribution_stats/", b"/metrics/data_distribution_stats/\xff\xff"),
+    ("kill storage", b"/globals/killStorage", b"/globals/killStorage\x00"),
+]
+
+def test_simple_tenant_access(private_key, token_gen, default_tenant, tenant_tr_gen):
+    token = token_gen(private_key, token_claim_1h(default_tenant))
+    tr = tenant_tr_gen(default_tenant)
+    tr.options.set_authorization_token(token)
+    tr[b"abc"] = b"def"
+    tr.commit().wait()
+    tr = tenant_tr_gen(default_tenant)
+    tr.options.set_authorization_token(token)
+    assert tr[b"abc"] == b"def", "tenant write transaction not visible"
+
+def test_cross_tenant_access_disallowed(private_key, default_tenant, token_gen, tenant_gen, tenant_tr_gen):
+    # use default tenant token with second tenant transaction and see it fail
+    second_tenant = random_alphanum_bytes(12)
+    tenant_gen(second_tenant)
+    token_second = token_gen(private_key, token_claim_1h(second_tenant))
+    tr_second = tenant_tr_gen(second_tenant)
+    tr_second.options.set_authorization_token(token_second)
+    tr_second[b"abc"] = b"def"
+    tr_second.commit().wait()
+    token_default = token_gen(private_key, token_claim_1h(default_tenant))
+    tr_second = tenant_tr_gen(second_tenant)
+    tr_second.options.set_authorization_token(token_default)
+    # test that read transaction fails
+    try:
+        value = tr_second[b"abc"].value
+        assert False, f"expected permission denied, but read transaction went through, value: {value}"
+    except fdb.FDBError as e:
+        assert e.code == 6000, f"expected permission_denied, got {e} instead"
+    # test that write transaction fails
+    tr_second = tenant_tr_gen(second_tenant)
+    tr_second.options.set_authorization_token(token_default)
+    try:
+        tr_second[b"def"] = b"ghi"
+        tr_second.commit().wait()
+        assert False, "expected permission denied, but write transaction went through"
+    except fdb.FDBError as e:
+        assert e.code == 6000, f"expected permission_denied, got {e} instead"
+
+def test_system_and_special_key_range_disallowed(db, tenant_tr_gen, token_gen):
+    second_tenant = random_alphanum_bytes(12)
+    try:
+        fdb.tenant_management.create_tenant(db, second_tenant)
+        assert False, "disallowed create_tenant has succeeded"
+    except fdb.FDBError as e:
+        assert e.code == 6000, f"expected permission_denied, got {e} instead"
+
+    try:
+        tr = db.create_transaction()
+        tr.options.set_access_system_keys()
+        kvs = tr.get_range(b"\xff", b"\xff\xff", limit=1).to_list()
+        assert False, f"disallowed system keyspace read has succeeded. found item: {kvs}"
+    except fdb.FDBError as e:
+        assert e.code == 6000, f"expected permission_denied, got {e} instead"
+
+    for range_name, special_range_begin, special_range_end in special_key_ranges:
+        tr = db.create_transaction()
+        tr.options.set_access_system_keys()
+        tr.options.set_special_key_space_relaxed()
+        try:
+            kvs = tr.get_range(special_range_begin, special_range_end, limit=1).to_list()
+            assert False, f"disallowed special keyspace read for range {range_name} has succeeded. found item {kvs}"
+        except fdb.FDBError as e:
+            assert e.code == 6000, f"expected permission_denied from attempted read to range {range_name}, got {e} instead"
+
+    try:
+        tr = db.create_transaction()
+        tr.options.set_access_system_keys()
+        del tr[b"\xff":b"\xff\xff"]
+        tr.commit().wait()
+        assert False, f"disallowed system keyspace write has succeeded"
+    except fdb.FDBError as e:
+        assert e.code == 6000, f"expected permission_denied, got {e} instead"
+
+    for range_name, special_range_begin, special_range_end in special_key_ranges:
+        tr = db.create_transaction()
+        tr.options.set_access_system_keys()
+        tr.options.set_special_key_space_relaxed()
+        try:
+            del tr[special_range_begin:special_range_end]
+            tr.commit().wait()
+            assert False, f"write to disallowed special keyspace range {range_name} has succeeded"
+        except fdb.FDBError as e:
+            assert e.code == 6000, f"expected permission_denied from attempted write to range {range_name}, got {e} instead"
+
+    try:
+        tr = db.create_transaction()
+        tr.options.set_access_system_keys()
+        kvs = tr.get_range(b"", b"\xff", limit=1).to_list()
+        assert False, f"disallowed normal keyspace read has succeeded. found item {kvs}"
+    except fdb.FDBError as e:
+        assert e.code == 6000, f"expected permission_denied, got {e} instead"
+
+def test_public_key_set_rollover(
+        kty, private_key_gen, private_key, public_key_refresh_interval,
+        cluster, default_tenant, token_gen, tenant_gen, tenant_tr_gen):
+    new_kid = random_alphanum_str(12)
+    new_kty = "EC" if kty == "RSA" else "RSA"
+    new_key = private_key_gen(kty=new_kty, kid=new_kid)
+    token_default = token_gen(private_key, token_claim_1h(default_tenant))
+
+    second_tenant = random_alphanum_bytes(12)
+    tenant_gen(second_tenant)
+    token_second = token_gen(new_key, token_claim_1h(second_tenant))
+
+    interim_set = public_keyset_from_keys([new_key, private_key])
+    max_repeat = 10
+
+    print(f"interim keyset: {interim_set}")
+    old_key_json = None
+    with open(cluster.public_key_json_file, "r") as keyfile:
+        old_key_json = keyfile.read()
+
+    delay = public_key_refresh_interval
+
+    with KeyFileReverter(cluster.public_key_json_file, old_key_json, delay):
+        with open(cluster.public_key_json_file, "w") as keyfile:
+            keyfile.write(interim_set)
+        wait_until_tenant_tr_succeeds(second_tenant, new_key, tenant_tr_gen, token_gen, max_repeat, delay)
+        print("interim key set activated")
+        final_set = public_keyset_from_keys([new_key])
+        print(f"final keyset: {final_set}")
+        with open(cluster.public_key_json_file, "w") as keyfile:
+            keyfile.write(final_set)
+        wait_until_tenant_tr_fails(default_tenant, private_key, tenant_tr_gen, token_gen, max_repeat, delay)
+
+def test_public_key_set_broken_file_tolerance(
+        private_key, public_key_refresh_interval,
+        cluster, public_key_jwks_str, default_tenant, token_gen, tenant_tr_gen):
+    delay = public_key_refresh_interval
+    # retry limit in waiting for keyset file update to propagate to FDB server's internal keyset
+    max_repeat = 10
+
+    with KeyFileReverter(cluster.public_key_json_file, public_key_jwks_str, delay):
+        # key file update should take effect even after witnessing broken key file
+        with open(cluster.public_key_json_file, "w") as keyfile:
+            keyfile.write(public_key_jwks_str.strip()[:10]) # make the file partial, injecting parse error
+        time.sleep(delay * 2)
+        # should still work; internal key set only clears with a valid, empty key set file
+        tr_default = tenant_tr_gen(default_tenant)
+        tr_default.options.set_authorization_token(token_gen(private_key, token_claim_1h(default_tenant)))
+        tr_default[b"abc"] = b"def"
+        tr_default.commit().wait()
+        with open(cluster.public_key_json_file, "w") as keyfile:
+            keyfile.write('{"keys":[]}')
+        # eventually internal key set will become empty and won't accept any new tokens
+        wait_until_tenant_tr_fails(default_tenant, private_key, tenant_tr_gen, token_gen, max_repeat, delay)
+
+def test_public_key_set_deletion_tolerance(
+        private_key, public_key_refresh_interval,
+        cluster, public_key_jwks_str, default_tenant, token_gen, tenant_tr_gen):
+    delay = public_key_refresh_interval
+    # retry limit in waiting for keyset file update to propagate to FDB server's internal keyset
+    max_repeat = 10
+
+    with KeyFileReverter(cluster.public_key_json_file, public_key_jwks_str, delay):
+        # key file update should take effect even after witnessing deletion of key file
+        with open(cluster.public_key_json_file, "w") as keyfile:
+            keyfile.write('{"keys":[]}')
+        time.sleep(delay)
+        wait_until_tenant_tr_fails(default_tenant, private_key, tenant_tr_gen, token_gen, max_repeat, delay)
+        os.remove(cluster.public_key_json_file)
+        time.sleep(delay * 2)
+        with open(cluster.public_key_json_file, "w") as keyfile:
+            keyfile.write(public_key_jwks_str)
+        # eventually updated key set should take effect and transaction should be accepted
+        wait_until_tenant_tr_succeeds(default_tenant, private_key, tenant_tr_gen, token_gen, max_repeat, delay)
+
+def test_public_key_set_empty_file_tolerance(
+        private_key, public_key_refresh_interval,
+        cluster, public_key_jwks_str, default_tenant, token_gen, tenant_tr_gen):
+    delay = public_key_refresh_interval
+    # retry limit in waiting for keyset file update to propagate to FDB server's internal keyset
+    max_repeat = 10
+
+    with KeyFileReverter(cluster.public_key_json_file, public_key_jwks_str, delay):
+        # key file update should take effect even after witnessing an empty file
+        with open(cluster.public_key_json_file, "w") as keyfile:
+            keyfile.write('{"keys":[]}')
+        # eventually internal key set will become empty and won't accept any new tokens
+        wait_until_tenant_tr_fails(default_tenant, private_key, tenant_tr_gen, token_gen, max_repeat, delay)
+        # empty the key file
+        with open(cluster.public_key_json_file, "w") as keyfile:
+            pass
+        time.sleep(delay * 2)
+        with open(cluster.public_key_json_file, "w") as keyfile:
+            keyfile.write(public_key_jwks_str)
+        # eventually key file should update and transactions should go through
+        wait_until_tenant_tr_succeeds(default_tenant, private_key, tenant_tr_gen, token_gen, max_repeat, delay)
+
+def test_bad_token(private_key, token_gen, default_tenant, tenant_tr_gen):
+    def del_attr(d, attr):
+        del d[attr]
+        return d
+
+    def set_attr(d, attr, value):
+        d[attr] = value
+        return d
+
+    claim_mutations = [
+        ("no nbf", lambda claim: del_attr(claim, "nbf")),
+        ("no exp", lambda claim: del_attr(claim, "exp")),
+        ("no iat", lambda claim: del_attr(claim, "iat")),
+        ("too early", lambda claim: set_attr(claim, "nbf", time.time() + 30)),
+        ("too late", lambda claim: set_attr(claim, "exp", time.time() - 10)),
+        ("no tenants", lambda claim: del_attr(claim, "tenants")),
+        ("empty tenants", lambda claim: set_attr(claim, "tenants", [])),
+    ]
+    for case_name, mutation in claim_mutations:
+        tr = tenant_tr_gen(default_tenant)
+        tr.options.set_authorization_token(token_gen(private_key, mutation(token_claim_1h(default_tenant))))
+        try:
+            value = tr[b"abc"].value
+            assert False, f"expected permission_denied for case {case_name}, but read transaction went through"
+        except fdb.FDBError as e:
+            assert e.code == 6000, f"expected permission_denied for case {case_name}, got {e} instead"
+        tr = tenant_tr_gen(default_tenant)
+        tr.options.set_authorization_token(token_gen(private_key, mutation(token_claim_1h(default_tenant))))
+        tr[b"abc"] = b"def"
+        try:
+            tr.commit().wait()
+            assert False, f"expected permission_denied for case {case_name}, but write transaction went through"
+        except fdb.FDBError as e:
+            assert e.code == 6000, f"expected permission_denied for case {case_name}, got {e} instead"
+
+    # unknown key case: override "kid" field in header
+    # first, update only the kid field of key with export-update-import
+    key_dict = private_key.as_dict(is_private=True)
+    key_dict["kid"] = random_alphanum_str(10)
+    renamed_key = authlib.jose.JsonWebKey.import_key(key_dict)
+    unknown_key_token = token_gen(
+                renamed_key,
+                token_claim_1h(default_tenant),
+                headers={
+                    "typ": "JWT",
+                    "kty": renamed_key.kty,
+                    "alg": alg_from_kty(renamed_key.kty),
+                    "kid": renamed_key.kid,
+                })
+    tr = tenant_tr_gen(default_tenant)
+    tr.options.set_authorization_token(unknown_key_token)
+    try:
+        value = tr[b"abc"].value
+        assert False, f"expected permission_denied for 'unknown key' case, but read transaction went through"
+    except fdb.FDBError as e:
+        assert e.code == 6000, f"expected permission_denied for 'unknown key' case, got {e} instead"
+    tr = tenant_tr_gen(default_tenant)
+    tr.options.set_authorization_token(unknown_key_token)
+    tr[b"abc"] = b"def"
+    try:
+        tr.commit().wait()
+        assert False, f"expected permission_denied for 'unknown key' case, but write transaction went through"
+    except fdb.FDBError as e:
+        assert e.code == 6000, f"expected permission_denied for 'unknown key' case, got {e} instead"
diff --git a/tests/authorization/conftest.py b/tests/authorization/conftest.py
new file mode 100644
index 0000000000..5780f39b89
--- /dev/null
+++ b/tests/authorization/conftest.py
@@ -0,0 +1,173 @@
+#!/usr/bin/python
+#
+# conftest.py
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import fdb
+import pytest
+import subprocess
+import admin_server
+from authlib.jose import JsonWebKey, KeySet, jwt
+from local_cluster import TLSConfig
+from tmp_cluster import TempCluster
+from typing import Union
+from util import alg_from_kty, public_keyset_from_keys, random_alphanum_str, random_alphanum_bytes, to_str, to_bytes
+
+fdb.api_version(720)
+
+cluster_scope = "module"
+
+def pytest_addoption(parser):
+    parser.addoption(
+            "--build-dir", action="store", dest="build_dir", help="FDB build directory", required=True)
+    parser.addoption(
+            "--kty", action="store", choices=["EC", "RSA"], default="EC", dest="kty", help="Token signature algorithm")
+    parser.addoption(
+            "--trusted-client",
+            action="store_true",
+            default=False,
+            dest="trusted_client",
+            help="Whether client shall be configured trusted, i.e. mTLS-ready")
+    parser.addoption(
+            "--public-key-refresh-interval",
+            action="store",
+            default=1,
+            dest="public_key_refresh_interval",
+            help="How frequently server refreshes authorization public key file")
+
+@pytest.fixture(scope="session")
+def build_dir(request):
+    return request.config.option.build_dir
+
+@pytest.fixture(scope="session")
+def kty(request):
+    return request.config.option.kty
+
+@pytest.fixture(scope="session")
+def trusted_client(request):
+    return request.config.option.trusted_client
+
+@pytest.fixture(scope="session")
+def public_key_refresh_interval(request):
+    return request.config.option.public_key_refresh_interval
+
+@pytest.fixture(scope="session")
+def alg(kty):
+    if kty == "EC":
+        return "ES256"
+    else:
+        return "RS256"
+
+@pytest.fixture(scope="session")
+def kid():
+    return random_alphanum_str(12)
+
+@pytest.fixture(scope="session")
+def private_key_gen():
+    def fn(kty: str, kid: str):
+        if kty == "EC":
+            return JsonWebKey.generate_key(kty=kty, crv_or_size="P-256", is_private=True, options={"kid": kid})
+        else:
+            return JsonWebKey.generate_key(kty=kty, crv_or_size=4096, is_private=True, options={"kid": kid})
+    return fn
+
+@pytest.fixture(scope="session")
+def private_key(kty, kid, private_key_gen):
+    return private_key_gen(kty, kid)
+
+@pytest.fixture(scope="session")
+def public_key_jwks_str(private_key):
+    return public_keyset_from_keys([private_key])
+
+@pytest.fixture(scope="session")
+def token_gen():
+    def fn(private_key, claims, headers={}):
+        if not headers:
+            headers = {
+                "typ": "JWT",
+                "kty": private_key.kty,
+                "alg": alg_from_kty(private_key.kty),
+                "kid": private_key.kid,
+            }
+        return jwt.encode(headers, claims, private_key)
+    return fn
+
+@pytest.fixture(scope=cluster_scope)
+def admin_ipc():
+    server = admin_server.Server()
+    server.start()
+    yield server
+    server.join()
+
+@pytest.fixture(autouse=True, scope=cluster_scope)
+def cluster(admin_ipc, build_dir, public_key_jwks_str, public_key_refresh_interval, trusted_client):
+    with TempCluster(
+            build_dir=build_dir,
+            tls_config=TLSConfig(server_chain_len=3, client_chain_len=2),
+            public_key_json_str=public_key_jwks_str,
+            remove_at_exit=True,
+            custom_config={
+                "knob-public-key-file-refresh-interval-seconds": public_key_refresh_interval,
+            }) as cluster:
+        keyfile = str(cluster.client_key_file)
+        certfile = str(cluster.client_cert_file)
+        cafile = str(cluster.server_ca_file)
+        fdb.options.set_tls_key_path(keyfile if trusted_client else "")
+        fdb.options.set_tls_cert_path(certfile if trusted_client else "")
+        fdb.options.set_tls_ca_path(cafile)
+        fdb.options.set_trace_enable()
+        admin_ipc.request("configure_tls", [keyfile, certfile, cafile])
+        admin_ipc.request("connect", [str(cluster.cluster_file)])
+        yield cluster
+
+@pytest.fixture
+def db(cluster, admin_ipc):
+    db = fdb.open(str(cluster.cluster_file))
+    db.options.set_transaction_timeout(2000) # 2 seconds
+    db.options.set_transaction_retry_limit(3)
+    yield db
+    admin_ipc.request("cleanup_database")
+    db = None
+
+@pytest.fixture
+def tenant_gen(db, admin_ipc):
+    def fn(tenant):
+        tenant = to_bytes(tenant)
+        admin_ipc.request("create_tenant", [tenant])
+    return fn
+
+@pytest.fixture
+def tenant_del(db, admin_ipc):
+    def fn(tenant):
+        tenant = to_str(tenant)
+        admin_ipc.request("delete_tenant", [tenant])
+    return fn
+
+@pytest.fixture
+def default_tenant(tenant_gen, tenant_del):
+    tenant = random_alphanum_bytes(8)
+    tenant_gen(tenant)
+    yield tenant
+    tenant_del(tenant)
+
+@pytest.fixture
+def tenant_tr_gen(db):
+    def fn(tenant):
+        tenant = db.open_tenant(to_bytes(tenant))
+        return tenant.create_transaction()
+    return fn
diff --git a/tests/authorization/requirements.txt b/tests/authorization/requirements.txt
new file mode 100644
index 0000000000..a60c422406
--- /dev/null
+++ b/tests/authorization/requirements.txt
@@ -0,0 +1,12 @@
+attrs==22.1.0
+Authlib==1.0.1
+cffi==1.15.1
+cryptography==37.0.4
+iniconfig==1.1.1
+packaging==21.3
+pluggy==1.0.0
+py==1.11.0
+pycparser==2.21
+pyparsing==3.0.9
+pytest==7.1.2
+tomli==2.0.1
diff --git a/tests/authorization/util.py b/tests/authorization/util.py
new file mode 100644
index 0000000000..aa38c05f32
--- /dev/null
+++ b/tests/authorization/util.py
@@ -0,0 +1,124 @@
+import fdb
+import json
+import random
+import string
+import time
+from typing import Union, List
+
+def to_str(s: Union[str, bytes]):
+    if isinstance(s, bytes):
+        s = s.decode("utf8")
+    return s
+
+def to_bytes(s: Union[str, bytes]):
+    if isinstance(s, str):
+        s = s.encode("utf8")
+    return s
+
+def random_alphanum_str(k: int):
+    return ''.join(random.choices(string.ascii_letters + string.digits, k=k))
+
+def random_alphanum_bytes(k: int):
+    return random_alphanum_str(k).encode("ascii")
+
+def cleanup_tenant(db, tenant_name):
+    try:
+        tenant = db.open_tenant(tenant_name)
+        del tenant[:]
+        fdb.tenant_management.delete_tenant(db, tenant_name)
+    except fdb.FDBError as e:
+        if e.code == 2131: # tenant not found
+            pass
+        else:
+            raise
+
+def alg_from_kty(kty: str):
+    if kty == "EC":
+        return "ES256"
+    else:
+        return "RS256"
+
+def public_keyset_from_keys(keys: List):
+    keys = list(map(lambda key: key.as_dict(is_private=False, alg=alg_from_kty(key.kty)), keys))
+    return json.dumps({ "keys": keys })
+
+class KeyFileReverter(object):
+    def __init__(self, filename: str, content: str, refresh_delay: int):
+        self.filename = filename
+        self.content = content
+        self.refresh_delay = refresh_delay
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        with open(self.filename, "w") as keyfile:
+            keyfile.write(self.content)
+        print(f"key file reverted. waiting {self.refresh_delay * 2} seconds for the update to take effect...")
+        time.sleep(self.refresh_delay * 2)
+
+# JWT claim that is valid for 1 hour since time of invocation
+def token_claim_1h(tenant_name):
+    now = time.time()
+    return {
+        "iss": "fdb-authz-tester",
+        "sub": "authz-test",
+        "aud": ["tmp-cluster"],
+        "iat": now,
+        "nbf": now - 1,
+        "exp": now + 60 * 60, 
+        "jti": random_alphanum_str(10),
+        "tenants": [to_str(tenant_name)],
+    }
+
+# repeat try-wait loop up to max_repeat times until both read and write tr fails for tenant with permission_denied
+# important: only use this function if you don't have any data dependencies to key "abc"
+def wait_until_tenant_tr_fails(tenant, private_key, tenant_tr_gen, token_gen, max_repeat, delay):
+    repeat = 0
+    read_blocked = False
+    write_blocked = False
+    while (not read_blocked or not write_blocked) and repeat < max_repeat:
+        time.sleep(delay)
+        tr = tenant_tr_gen(tenant)
+        # a token needs to be generated at every iteration because once it is accepted/cached,
+        # it will pass verification by caching until it expires
+        tr.options.set_authorization_token(token_gen(private_key, token_claim_1h(tenant)))
+        try:
+            if not read_blocked:
+                value = tr[b"abc"].value
+        except fdb.FDBError as e:
+            assert e.code == 6000, f"expected permission_denied, got {e} instead"
+            read_blocked = True
+        if not read_blocked:
+            repeat += 1
+            continue
+
+        try:
+            if not write_blocked:
+                tr[b"abc"] = b"def"
+                tr.commit().wait()
+        except fdb.FDBError as e:
+            assert e.code == 6000, f"expected permission_denied, got {e} instead"
+            write_blocked = True
+        if not write_blocked:
+            repeat += 1
+    assert repeat < max_repeat, f"tenant transaction did not start to fail in {max_repeat * delay} seconds"
+
+# repeat try-wait loop up to max_repeat times until both read and write tr succeeds for tenant
+# important: only use this function if you don't have any data dependencies to key "abc"
+def wait_until_tenant_tr_succeeds(tenant, private_key, tenant_tr_gen, token_gen, max_repeat, delay):
+    repeat = 0
+    token = token_gen(private_key, token_claim_1h(tenant))
+    while repeat < max_repeat:
+        try:
+            time.sleep(delay)
+            tr = tenant_tr_gen(tenant)
+            tr.options.set_authorization_token(token)
+            value = tr[b"abc"].value
+            tr[b"abc"] = b"qwe"
+            tr.commit().wait()
+            break
+        except fdb.FDBError as e:
+            assert e.code == 6000, f"expected permission_denied, got {e} instead"
+            repeat += 1
+    assert repeat < max_repeat, f"tenant transaction did not start to succeed in {max_repeat * delay} seconds"
diff --git a/tests/fast/AtomicBackupToDBCorrectness.toml b/tests/fast/AtomicBackupToDBCorrectness.toml
index 1b601da923..84214337d9 100644
--- a/tests/fast/AtomicBackupToDBCorrectness.toml
+++ b/tests/fast/AtomicBackupToDBCorrectness.toml
@@ -1,5 +1,5 @@
 [configuration]
-extraDB = 1
+extraDatabaseMode = 'LocalOrSingle'
 
 [[test]]
 testTitle = 'BackupAndRestore'
diff --git a/tests/fast/BackupAzureBlobCorrectness.toml b/tests/fast/BackupAzureBlobCorrectness.toml
index 48598d5c6b..2401f395df 100644
--- a/tests/fast/BackupAzureBlobCorrectness.toml
+++ b/tests/fast/BackupAzureBlobCorrectness.toml
@@ -1,3 +1,5 @@
+testClass = "Backup"
+
 [[test]]
 testTitle = 'Cycle'
 clearAfterTest = 'false'
@@ -39,7 +41,9 @@ testTitle = 'Backup'
     testName = 'BackupToBlob'
     backupAfter = 0.0
     backupTag = 'default'
-    backupURL = 'azure://0.0.0.0:10000/devstoreaccount1/test_container/'
+    # azure storage account style format: azure://<account_name>.blob.core.windows.net/<container_name>
+    # general ip-port style format: azure://<IP|Hostname>:<Port>/<account_name>/<container_name>
+    backupURL = 'azure://0.0.0.0:10000/devstoreaccount1/testcontainer'
     
     [[test.workload]]
     testName = 'RandomClogging'
@@ -72,7 +76,7 @@ clearAfterTest = 'false'
     testName = 'RestoreFromBlob'
     restoreAfter = 0.0
     backupTag = 'default'
-    backupURL = 'azure://0.0.0.0:10000/devstoreaccount1/test_container/'
+    backupURL = 'azure://0.0.0.0:10000/devstoreaccount1/testcontainer'
 
     [[test.workload]]
     testName = 'RandomClogging'
diff --git a/tests/fast/BackupCorrectness.toml b/tests/fast/BackupCorrectness.toml
index 1315b1af5f..0582bfa0eb 100644
--- a/tests/fast/BackupCorrectness.toml
+++ b/tests/fast/BackupCorrectness.toml
@@ -1,3 +1,5 @@
+testClass = "Backup"
+
 [[test]]
 testTitle = 'BackupAndRestore'
 clearAfterTest = false
diff --git a/tests/fast/BackupCorrectnessClean.toml b/tests/fast/BackupCorrectnessClean.toml
index d5fc3d945e..d581e0caea 100644
--- a/tests/fast/BackupCorrectnessClean.toml
+++ b/tests/fast/BackupCorrectnessClean.toml
@@ -1,3 +1,5 @@
+testClass = "Backup"
+
 [[test]]
 testTitle = 'BackupAndRestore'
 clearAfterTest = false
diff --git a/tests/fast/BackupS3BlobCorrectness.toml b/tests/fast/BackupS3BlobCorrectness.toml
index e80fa847b0..d642cd4811 100644
--- a/tests/fast/BackupS3BlobCorrectness.toml
+++ b/tests/fast/BackupS3BlobCorrectness.toml
@@ -1,3 +1,5 @@
+testClass = "Backup"
+
 [[test]]
 testTitle = 'Cycle'
 clearAfterTest = 'false'
diff --git a/tests/fast/BackupToDBCorrectness.toml b/tests/fast/BackupToDBCorrectness.toml
index cf50093657..b8c9e859b8 100644
--- a/tests/fast/BackupToDBCorrectness.toml
+++ b/tests/fast/BackupToDBCorrectness.toml
@@ -1,5 +1,7 @@
+testClass = "Backup"
+
 [configuration]
-extraDB = 1
+extraDatabaseMode = 'LocalOrSingle'
 
 [[test]]
 testTitle = 'BackupAndRestore'
diff --git a/tests/fast/BackupToDBCorrectnessClean.toml b/tests/fast/BackupToDBCorrectnessClean.toml
index 9c2e9135e5..e6a9921383 100644
--- a/tests/fast/BackupToDBCorrectnessClean.toml
+++ b/tests/fast/BackupToDBCorrectnessClean.toml
@@ -1,5 +1,7 @@
+testClass = "Backup"
+
 [configuration]
-extraDB = 1
+extraDatabaseMode = 'LocalOrSingle'
 
 [[test]]
 testTitle = 'BackupAndRestore'
diff --git a/tests/fast/BlobGranuleMoveVerifyCycle.toml b/tests/fast/BlobGranuleMoveVerifyCycle.toml
new file mode 100644
index 0000000000..4f1267aac7
--- /dev/null
+++ b/tests/fast/BlobGranuleMoveVerifyCycle.toml
@@ -0,0 +1,48 @@
+[configuration]
+testClass = "BlobGranule"
+blobGranulesEnabled = true 
+allowDefaultTenant = false
+# FIXME: re-enable rocks at some point
+storageEngineExcludeTypes = [4]
+
+[[test]]
+testTitle = 'BlobGranuleMoveVerifyCycle'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    transactionsPerSecond = 250.0
+    testDuration = 60.0
+    expectedRate = 0
+
+    [[test.workload]]
+    testName = 'RandomMoveKeys'
+    testDuration = 60.0
+
+    [[test.workload]]
+    testName = 'BlobGranuleVerifier'
+    testDuration = 60.0
+    # cycle does its own workload checking, don't want clear racing with its checking
+    clearAndMergeCheck = false
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 60.0
+
+    [[test.workload]]
+    testName = 'Rollback'
+    meanDelay = 60.0
+    testDuration = 60.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 60.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 60.0
diff --git a/tests/fast/BlobGranuleVerifyAtomicOps.toml b/tests/fast/BlobGranuleVerifyAtomicOps.toml
index b593a4814d..84e946c004 100644
--- a/tests/fast/BlobGranuleVerifyAtomicOps.toml
+++ b/tests/fast/BlobGranuleVerifyAtomicOps.toml
@@ -1,11 +1,11 @@
 [configuration]
+testClass = "BlobGranule"
 blobGranulesEnabled = true 
 allowDefaultTenant = false
+injectTargetedSSRestart = true
+injectSSDelay = true
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [4]
-
-[[knobs]]
-bg_range_source = "blobRangeKeys"
+storageEngineExcludeTypes = [4, 5]
 
 [[test]]
 testTitle = 'BlobGranuleVerifyAtomicOps'
diff --git a/tests/fast/BlobGranuleVerifyCycle.toml b/tests/fast/BlobGranuleVerifyCycle.toml
index 4d7767d422..1cf056b87b 100644
--- a/tests/fast/BlobGranuleVerifyCycle.toml
+++ b/tests/fast/BlobGranuleVerifyCycle.toml
@@ -1,11 +1,11 @@
 [configuration]
+testClass = "BlobGranule"
 blobGranulesEnabled = true 
 allowDefaultTenant = false
+injectTargetedSSRestart = true
+injectSSDelay = true
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [4]
-
-[[knobs]]
-bg_range_source = "blobRangeKeys"
+storageEngineExcludeTypes = [4, 5]
 
 [[test]]
 testTitle = 'BlobGranuleVerifyCycle'
@@ -19,6 +19,8 @@ testTitle = 'BlobGranuleVerifyCycle'
     [[test.workload]]
     testName = 'BlobGranuleVerifier'
     testDuration = 60.0
+    # cycle does its own workload checking, don't want clear racing with its checking
+    clearAndMergeCheck = false
 
     [[test.workload]]
     testName = 'RandomClogging'
diff --git a/tests/fast/BlobGranuleVerifySmall.toml b/tests/fast/BlobGranuleVerifySmall.toml
index 151594e455..ba50b0fda6 100644
--- a/tests/fast/BlobGranuleVerifySmall.toml
+++ b/tests/fast/BlobGranuleVerifySmall.toml
@@ -1,12 +1,11 @@
 [configuration]
+testClass = "BlobGranule"
 blobGranulesEnabled = true 
 allowDefaultTenant = false
-# FIXME: exclude redwood because WriteDuringRead can write massive KV pairs and we don't chunk change feed data on disk yet
+injectTargetedSSRestart = true
+injectSSDelay = true
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [3, 4] 
-
-[[knobs]]
-bg_range_source = "blobRangeKeys"
+storageEngineExcludeTypes = [4, 5]
 
 [[test]]
 testTitle = 'BlobGranuleVerifySmall'
diff --git a/tests/fast/BlobGranuleVerifySmallClean.toml b/tests/fast/BlobGranuleVerifySmallClean.toml
index ca4b5f81cf..ef957b9f53 100644
--- a/tests/fast/BlobGranuleVerifySmallClean.toml
+++ b/tests/fast/BlobGranuleVerifySmallClean.toml
@@ -1,12 +1,9 @@
 [configuration]
 blobGranulesEnabled = true
 allowDefaultTenant = false
-# FIXME: exclude redwood because WriteDuringRead can write massive KV pairs and we don't chunk change feed data on disk yet
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [3, 4]
-
-[[knobs]]
-bg_range_source = "blobRangeKeys"
+storageEngineExcludeTypes = [4, 5]
+testClass = "BlobGranule"
 
 [[test]]
 testTitle = 'BlobGranuleVerifySmallClean'
@@ -20,3 +17,4 @@ testTitle = 'BlobGranuleVerifySmallClean'
     [[test.workload]]
     testName = 'BlobGranuleVerifier'
     testDuration = 60.0
+    enablePurging = false
diff --git a/tests/fast/ChangeFeedOperations.toml b/tests/fast/ChangeFeedOperations.toml
new file mode 100644
index 0000000000..344b912657
--- /dev/null
+++ b/tests/fast/ChangeFeedOperations.toml
@@ -0,0 +1,34 @@
+[configuration]
+allowDefaultTenant = false
+testClass = "ChangeFeeds"
+
+[[test]]
+testTitle = 'ChangeFeedOperationsTest'
+
+    [[test.workload]]
+    testName = 'ChangeFeedOperations'
+    testDuration = 60.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 60.0
+
+    [[test.workload]]
+    testName = 'Rollback'
+    meanDelay = 30.0
+    testDuration = 60.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 60.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 60.0
+
diff --git a/tests/fast/ChangeFeedOperationsMove.toml b/tests/fast/ChangeFeedOperationsMove.toml
new file mode 100644
index 0000000000..a44666b5fe
--- /dev/null
+++ b/tests/fast/ChangeFeedOperationsMove.toml
@@ -0,0 +1,38 @@
+[configuration]
+testClass = "ChangeFeeds"
+allowDefaultTenant = false
+
+[[test]]
+testTitle = 'ChangeFeedOperationsTest'
+
+    [[test.workload]]
+    testName = 'ChangeFeedOperations'
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'RandomMoveKeys'
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'Rollback'
+    meanDelay = 30.0
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 30.0
+
diff --git a/tests/fast/ChangeFeeds.toml b/tests/fast/ChangeFeeds.toml
index 8f2d348dd3..e51e8c7c7a 100644
--- a/tests/fast/ChangeFeeds.toml
+++ b/tests/fast/ChangeFeeds.toml
@@ -1,4 +1,5 @@
 [configuration]
+testClass = "ChangeFeeds"
 allowDefaultTenant = false
 
 [[test]]
diff --git a/tests/rare/ConfigIncrement.toml b/tests/fast/ConfigIncrement.toml
similarity index 78%
rename from tests/rare/ConfigIncrement.toml
rename to tests/fast/ConfigIncrement.toml
index eee28710f6..2684e90df9 100644
--- a/tests/rare/ConfigIncrement.toml
+++ b/tests/fast/ConfigIncrement.toml
@@ -17,3 +17,8 @@ testTitle = 'ConfigIncrement'
   machinesToLeave = 3
   reboot = true
   testDuration = 10.0
+
+  [[test.workload]]
+  testName = 'ChangeConfig'
+  maxDelayBeforeChange = 120.0
+  coordinators = 'auto'
diff --git a/tests/fast/ConfigIncrementChangeCoordinators.toml b/tests/fast/ConfigIncrementChangeCoordinators.toml
new file mode 100644
index 0000000000..caf5f10309
--- /dev/null
+++ b/tests/fast/ConfigIncrementChangeCoordinators.toml
@@ -0,0 +1,28 @@
+# Similar to the ConfigIncrement workload, but with a larger amount of time
+# between transactions to allow a coordinator change to take place in between
+# transactions.
+[configuration]
+configDB = 'paxos'
+
+[[test]]
+testTitle = 'ConfigIncrementChangeCoordinators'
+
+  [[test.workload]]
+  testName = 'ConfigIncrement'
+  incrementActors = 2
+  incrementsPerActor = 10
+  meanSleepWithinTransactions = 0.01
+  meanSleepBetweenTransactions = 10
+
+  [[test.workload]]
+  testName = 'Attrition'
+  machinesToKill = 10
+  machinesToLeave = 3
+  reboot = true
+  testDuration = 10.0
+
+  [[test.workload]]
+  testName = 'ChangeConfig'
+  maxDelayBeforeChange = 120.0
+  coordinators = 'auto'
+  coordinatorChanges = 2
diff --git a/tests/rare/ConfigIncrementWithKills.toml b/tests/fast/ConfigIncrementWithKills.toml
similarity index 62%
rename from tests/rare/ConfigIncrementWithKills.toml
rename to tests/fast/ConfigIncrementWithKills.toml
index 6d840f0103..ea2b6b4a70 100644
--- a/tests/rare/ConfigIncrementWithKills.toml
+++ b/tests/fast/ConfigIncrementWithKills.toml
@@ -2,7 +2,7 @@
 configDB = 'paxos'
 
 [[test]]
-testTitle = 'ConfigIncrement'
+testTitle = 'ConfigIncrementWithKills'
 
   [[test.workload]]
   testName = 'ConfigIncrement'
@@ -14,3 +14,9 @@ testTitle = 'ConfigIncrement'
   [[test.workload]]
   testName = 'Attrition'
   reboot = false
+
+  [[test.workload]]
+  testName = 'ChangeConfig'
+  maxDelayBeforeChange = 120.0
+  coordinators = 'auto'
+  coordinatorChanges = 2
diff --git a/tests/fast/EncryptKeyProxyTest.toml b/tests/fast/EncryptKeyProxyTest.toml
index 3f5f7e12f1..199cc72da6 100644
--- a/tests/fast/EncryptKeyProxyTest.toml
+++ b/tests/fast/EncryptKeyProxyTest.toml
@@ -1,3 +1,6 @@
+[configuration]
+testClass = "Encryption"
+
 [[knobs]]
 enable_encryption = true
 
diff --git a/tests/fast/EncryptedBackupCorrectness.toml b/tests/fast/EncryptedBackupCorrectness.toml
new file mode 100644
index 0000000000..ac522cd441
--- /dev/null
+++ b/tests/fast/EncryptedBackupCorrectness.toml
@@ -0,0 +1,62 @@
+[configuration]
+allowDefaultTenant = false
+allowDisablingTenants = false
+
+[[knobs]]
+enable_encryption = true
+
+[[test]]
+testTitle = 'TenantCreation'
+
+    [[test.workload]]
+    testName = 'CreateTenant'
+    name = 'First'
+
+    [[test.workload]]
+    testName = 'CreateTenant'
+    name = 'Second'
+
+    [[test.workload]]
+    testName = 'CreateTenant'
+    name = 'Third'
+
+[[test]]
+testTitle = 'EncryptedBackupAndRestore'
+clearAfterTest = false
+simBackupAgents = 'BackupToFile'
+
+    [[test.workload]]
+    testName = 'BulkSetup'
+    tenants = 'First,Second,Third'
+    transactionsPerSecond = 2500.0
+
+    [[test.workload]]
+    testName = 'BackupAndRestoreCorrectness'
+    enableBackupEncryption = true
+    defaultBackup = true
+    backupAfter = 10.0
+    restoreAfter = 60.0
+    backupRangesCount = -1
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 90.0
+
+    [[test.workload]]
+    testName = 'Rollback'
+    meanDelay = 90.0
+    testDuration = 90.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 90.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 90.0
diff --git a/tests/fast/EncryptionOps.toml b/tests/fast/EncryptionOps.toml
index 72d54369fe..5002e5adfc 100644
--- a/tests/fast/EncryptionOps.toml
+++ b/tests/fast/EncryptionOps.toml
@@ -1,3 +1,10 @@
+[configuration]
+buggify = false
+testClass = "Encryption"
+
+[[knobs]]
+enable_encryption = false
+
 [[test]]
 testTitle = 'EncryptDecrypt'
 
diff --git a/tests/fast/IDDTxnProcessorApiCorrectness.toml b/tests/fast/IDDTxnProcessorApiCorrectness.toml
new file mode 100644
index 0000000000..8a3ee524d3
--- /dev/null
+++ b/tests/fast/IDDTxnProcessorApiCorrectness.toml
@@ -0,0 +1,9 @@
+[configuration]
+generateFearless = false # prevent generating remote dc because in MGS there's no region setting yet
+disableTss = true # There's no TSS in MGS this prevent the DD operate TSS mapping
+
+[[test]]
+testTitle = 'IDDTxnProcessorApiCorrectness'
+    [[test.workload]]
+    testName = 'IDDTxnProcessorApiCorrectness'
+    testDuration = 10.0
\ No newline at end of file
diff --git a/tests/fast/PhysicalShardMove.toml b/tests/fast/PhysicalShardMove.toml
index 66948ce559..c1b0d32134 100644
--- a/tests/fast/PhysicalShardMove.toml
+++ b/tests/fast/PhysicalShardMove.toml
@@ -1,14 +1,18 @@
 [configuration]
 config = 'triple'
-storageEngineType = 4
+storageEngineType = 5
 processesPerMachine = 1
 coordinators = 3
 machineCount = 15
 allowDefaultTenant = false
 
+[[knobs]]
+shard_encode_location_metadata = true
+enable_dd_physical_shard = true
+
 [[test]]
 testTitle = 'PhysicalShardMove'
 useDB = true
 
     [[test.workload]]
-    testName = 'SSCheckpointWorkload'
+    testName = 'PhysicalShardMove'
diff --git a/tests/fast/RandomUnitTests.toml b/tests/fast/RandomUnitTests.toml
index 265921f84d..e34a71fe46 100644
--- a/tests/fast/RandomUnitTests.toml
+++ b/tests/fast/RandomUnitTests.toml
@@ -5,5 +5,8 @@ startDelay = 0
 
     [[test.workload]]
     testName = 'UnitTests'
-    maxTestCases = 1
+    maxTestCases = 10
     testsMatching = '/'
+    # ConfigDB tests persist state, so running these tests
+    # sequentially can cause issues
+    testsIgnored = '/fdbserver/ConfigDB/'
diff --git a/tests/fast/SpecialKeySpaceCorrectness.toml b/tests/fast/SpecialKeySpaceCorrectness.toml
index d88477d29d..fcabb8222e 100644
--- a/tests/fast/SpecialKeySpaceCorrectness.toml
+++ b/tests/fast/SpecialKeySpaceCorrectness.toml
@@ -1,3 +1,6 @@
+[configuration]
+allowDisablingTenants = false
+
 [[test]]
 testTitle = 'SpecialKeySpaceCorrectnessTest'
 
diff --git a/tests/fast/StorageServerCheckpointRestore.toml b/tests/fast/StorageServerCheckpointRestore.toml
new file mode 100644
index 0000000000..ab81c70a7b
--- /dev/null
+++ b/tests/fast/StorageServerCheckpointRestore.toml
@@ -0,0 +1,14 @@
+[configuration]
+config = 'triple'
+storageEngineType = 4
+processesPerMachine = 1
+coordinators = 3
+machineCount = 15
+allowDefaultTenant = false
+
+[[test]]
+testTitle = 'SSCheckpointRestoreWorkload'
+useDB = true
+
+    [[test.workload]]
+    testName = 'SSCheckpointRestoreWorkload'
diff --git a/tests/fast/TenantCycle.toml b/tests/fast/TenantCycle.toml
new file mode 100644
index 0000000000..9566ab7a6e
--- /dev/null
+++ b/tests/fast/TenantCycle.toml
@@ -0,0 +1,31 @@
+[configuration]
+allowDefaultTenant = false
+allowDisablingTenants = false
+
+[[test]]
+testTitle = 'TenantCreation'
+
+    [[test.workload]]
+    testName = 'CreateTenant'
+    name = 'First'
+
+    [[test.workload]]
+    testName = 'CreateTenant'
+    name = 'Second'
+
+[[test]]
+testTitle = 'Cycle'
+
+    [[test.workload]]
+    testName = 'TenantCycle'
+    tenant = 'First'
+    transactionsPerSecond = 250.0
+    testDuration = 10.0
+    expectedRate = 0.80
+
+    [[test.workload]]
+    testName = 'TenantCycle'
+    tenant = 'Second'
+    transactionsPerSecond = 2500.0
+    testDuration = 10.0
+    expectedRate = 0.80
diff --git a/tests/fast/TenantCycleTokenless.toml b/tests/fast/TenantCycleTokenless.toml
new file mode 100644
index 0000000000..6e192c1f7f
--- /dev/null
+++ b/tests/fast/TenantCycleTokenless.toml
@@ -0,0 +1,36 @@
+[configuration]
+allowDefaultTenant = false
+allowDisablingTenants = false
+
+[[knobs]]
+allow_tokenless_tenant_access = true
+
+[[test]]
+testTitle = 'TenantCreation'
+
+    [[test.workload]]
+    testName = 'CreateTenant'
+    name = 'First'
+
+    [[test.workload]]
+    testName = 'CreateTenant'
+    name = 'Second'
+
+[[test]]
+testTitle = 'Cycle'
+
+    [[test.workload]]
+    testName = 'TenantCycle'
+    tenant = 'First'
+    transactionsPerSecond = 250.0
+    testDuration = 10.0
+    expectedRate = 0.80
+    useToken = false
+
+    [[test.workload]]
+    testName = 'TenantCycle'
+    tenant = 'Second'
+    transactionsPerSecond = 2500.0
+    testDuration = 10.0
+    expectedRate = 0.80
+    useToken = false
diff --git a/tests/fast/TenantEntryCache.toml b/tests/fast/TenantEntryCache.toml
new file mode 100644
index 0000000000..c60423ef89
--- /dev/null
+++ b/tests/fast/TenantEntryCache.toml
@@ -0,0 +1,10 @@
+[configuration]
+allowDefaultTenant = false
+allowDisablingTenants = false
+
+[[test]]
+testTitle = 'TenantEntryCacheTest'
+
+    [[test.workload]]
+    testName = 'TenantEntryCache'
+    maxTenants = 100
\ No newline at end of file
diff --git a/tests/loopback_cluster/run_custom_cluster.sh b/tests/loopback_cluster/run_custom_cluster.sh
index 37f048047a..72d9b239fc 100755
--- a/tests/loopback_cluster/run_custom_cluster.sh
+++ b/tests/loopback_cluster/run_custom_cluster.sh
@@ -7,12 +7,14 @@ PORT_PREFIX=1500
 
 # default cluster settings, override with options
 STATELESS_COUNT=4
+REPLICATION_COUNT=1
 LOGS_COUNT=8
 STORAGE_COUNT=16
 KNOBS=""
 LOGS_TASKSET=""
 STATELESS_TASKSET=""
 STORAGE_TASKSET=""
+LOGROUTER_COUNT=0
 
 function usage {
 	echo "Usage"
@@ -25,6 +27,7 @@ function usage {
 	printf "\t--logs_taskset BITMASK\n\r\t\tbitmask of CPUs to pin logs to. Default is all CPUs.\n\r"
 	printf "\t--storage_count COUNT\n\r\t\tnumber of storage daemons to start.  Default ${STORAGE_COUNT}\n\r"
 	printf "\t--storage_taskset BITMASK\n\r\t\tBitmask of CPUs to pin storage to. Default is all CPUs.\n\r"
+	printf "\t--replication_count COUNT\n\r\t\tReplication count may be 1,2 or 3. Default is 1.\n\r"
 	echo "Example"
 	printf "\t${0} . --knobs '--knob_proxy_use_resolver_private_mutations=1' --stateless_count 4 --stateless_taskset 0xf --logs_count 8 --logs_taskset 0xff0 --storage_taskset 0xffff000\n\r"
 	exit 1
@@ -36,11 +39,36 @@ function start_servers {
 		DATA=${DIR}/${SERVER_COUNT}/data
 		mkdir -p ${LOG} ${DATA}
 		PORT=$(( $PORT_PREFIX + $SERVER_COUNT ))
-		$2 ${FDB} -p auto:${PORT} "$KNOBS" -c $3 -d $DATA -L $LOG -C $CLUSTER &
+		ZONE=$4-Z-$(( $j % $REPLICATION_COUNT ))
+		$2 ${FDB} -p auto:${PORT} $KNOBS -c $3 -d $DATA -L $LOG -C $CLUSTER --datacenter_id=$4 --locality-zoneid $ZONE --locality-machineid M-$SERVER_COUNT &
 		SERVER_COUNT=$(( $SERVER_COUNT + 1 ))
 	done
 }
 
+function create_fileconfig {
+	cat > /tmp/fdbfileconfig.json <<EOF
+{
+	"regions": [{
+		"datacenters": [{
+			"id": "DC1",
+			"priority": 1
+		}, {
+			"id": "DC2",
+			"priority": 0,
+			"satellite": 1,
+			"satellite_logs": 8
+		}],
+		"satellite_redundancy_mode": "one_satellite_double"
+	}, {
+		"datacenters": [{
+			"id": "DC3",
+			"priority": -1
+		}]
+	}]
+}
+EOF
+}
+
 if (( $# < 1 )) ; then
 	echo Wrong number of arguments
 	usage
@@ -76,6 +104,12 @@ while [[ $# -gt 0 ]]; do
 		--storage_count)
 			STORAGE_COUNT=$2
 			;;	
+		--replication_count)
+			REPLICATION_COUNT=$2
+			;;
+		--logrouter_count)
+			LOGROUTER_COUNT=$2
+			;;
 	esac
 	shift; shift
 done
@@ -86,6 +120,15 @@ if [ ! -f ${FDB} ]; then
 	usage
 fi
 
+if [ $REPLICATION_COUNT -eq 1 ]; then
+	replication="single"
+elif [ $REPLICATION_COUNT -eq 2 ]; then
+	replication="double"
+elif [ $REPLICATION_COUNT -eq 3 ]; then
+	replication="triple"
+else
+	usage
+fi
 
 DIR=./loopback-cluster
 rm -rf $DIR
@@ -97,12 +140,23 @@ echo $CLUSTER_FILE > $CLUSTER
 
 echo "Starting Cluster: " $CLUSTER_FILE
 
-start_servers $STATELESS_COUNT "$STATELESS_TASKSET" stateless
-start_servers $LOGS_COUNT "$LOGS_TASKSET" log
-start_servers $STORAGE_COUNT "$STORAGE_TASKSET" storage
+start_servers $STATELESS_COUNT "$STATELESS_TASKSET" stateless DC1
+start_servers $LOGS_COUNT "$LOGS_TASKSET" log DC1
+start_servers $STORAGE_COUNT "$STORAGE_TASKSET" storage DC1
 
 CLI="$BUILD/bin/fdbcli -C ${CLUSTER} --exec"
-echo "configure new ssd single - stand by"
+echo "configure new ssd $replication - stand by"
 
-# sleep 2 seconds to wait for workers to join cluster, then configure database
-( sleep 2 ; $CLI "configure new ssd single" )
+# sleep 2 seconds to wait for workers to join cluster, then configure database and coordinators
+( sleep 2 ; $CLI "configure new ssd $replication" ; $CLI "coordinators auto")
+
+if [ $LOGROUTER_COUNT -gt 0 ]; then
+	start_servers $LOGROUTER_COUNT "$STORAGE_TASKSET" router DC3
+	# Same number remote/satellite logs and ss as primary
+	start_servers $LOGS_COUNT "$LOGS_TASKSET" log DC2
+	start_servers $LOGS_COUNT "$LOGS_TASKSET" log DC3
+	start_servers $STORAGE_COUNT "$STORAGE_TASKSET" storage DC3
+	create_fileconfig
+	$CLI "fileconfigure /tmp/fdbfileconfig.json"
+	echo "Wait for data to be fully replicated (Healthy), then issue: $CLI configure usable_regions=2"
+fi
diff --git a/tests/rare/BlobGranuleRanges.toml b/tests/rare/BlobGranuleRanges.toml
new file mode 100644
index 0000000000..aa38cdf5f8
--- /dev/null
+++ b/tests/rare/BlobGranuleRanges.toml
@@ -0,0 +1,38 @@
+[configuration]
+blobGranulesEnabled = true 
+allowDefaultTenant = false
+injectTargetedSSRestart = true
+injectSSDelay = true
+# FIXME: re-enable rocks at some point
+storageEngineExcludeTypes = [4, 5]
+
+[[test]]
+testTitle = 'BlobGranuleRanges'
+
+    [[test.workload]]
+    testName = 'BlobGranuleRanges'
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'Rollback'
+    meanDelay = 20.0
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 30.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 30.0
+
diff --git a/tests/rare/ConfigDBUnitTest.toml b/tests/rare/ConfigDBUnitTest.toml
new file mode 100644
index 0000000000..3aca62613e
--- /dev/null
+++ b/tests/rare/ConfigDBUnitTest.toml
@@ -0,0 +1,9 @@
+[[test]]
+testTitle = 'ConfigDBUnitTest'
+useDB = false
+startDelay = 0
+
+    [[test.workload]]
+    testName = 'UnitTests'
+    maxTestCases = 1
+    testsMatching = '/fdbserver/ConfigDB/'
diff --git a/tests/rare/DataDistributionMetrics.toml b/tests/rare/DataDistributionMetrics.toml
new file mode 100644
index 0000000000..9ec0b40f9c
--- /dev/null
+++ b/tests/rare/DataDistributionMetrics.toml
@@ -0,0 +1,8 @@
+[configuration]
+buggify = false
+
+[[test]]
+testTitle = 'DataDistributionMetricsTest'
+
+    [[test.workload]]
+    testName = 'DataDistributionMetrics'
diff --git a/tests/rare/GlobalTagThrottling.toml b/tests/rare/GlobalTagThrottling.toml
index 58cda2312e..499f2990f2 100644
--- a/tests/rare/GlobalTagThrottling.toml
+++ b/tests/rare/GlobalTagThrottling.toml
@@ -8,7 +8,7 @@ testTitle='GlobalTagThrottling'
   [[test.workload]]
   testName='GlobalTagThrottling'
   transactionTag='sampleTag1'
-  totalReadQuota=1.0
+  totalQuota=1.0
 
   [[test.workload]]
   testName='ReadWrite'
diff --git a/tests/rare/StorageQuotaTest.toml b/tests/rare/StorageQuotaTest.toml
new file mode 100644
index 0000000000..b55ddcc688
--- /dev/null
+++ b/tests/rare/StorageQuotaTest.toml
@@ -0,0 +1,5 @@
+[[test]]
+testTitle = 'StorageQuota'
+
+    [[test.workload]]
+    testName = 'StorageQuota'
diff --git a/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt b/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt
index 439ccedb5e..366d8bc2a3 100644
--- a/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt
+++ b/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt
@@ -1,4 +1,4 @@
-extraDB=3
+extraDatabaseMode=Local
 
 testTitle=DrUpgrade
 runSetup=false
diff --git a/tests/restarting/from_6.3.13/DrUpgradeRestart-1.txt b/tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt
similarity index 100%
rename from tests/restarting/from_6.3.13/DrUpgradeRestart-1.txt
rename to tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt
diff --git a/tests/restarting/from_6.3.13/DrUpgradeRestart-2.txt b/tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt
similarity index 93%
rename from tests/restarting/from_6.3.13/DrUpgradeRestart-2.txt
rename to tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt
index 439ccedb5e..366d8bc2a3 100644
--- a/tests/restarting/from_6.3.13/DrUpgradeRestart-2.txt
+++ b/tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt
@@ -1,4 +1,4 @@
-extraDB=3
+extraDatabaseMode=Local
 
 testTitle=DrUpgrade
 runSetup=false
diff --git a/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml b/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml
index f0f2141778..04144efd6b 100644
--- a/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml
+++ b/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml
@@ -1,3 +1,4 @@
+testClass = "Backup"
 storageEngineExcludeTypes=3
 
 [[test]]
diff --git a/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml b/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml
index b5ac855d52..22f016deab 100644
--- a/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml
+++ b/tests/restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml
@@ -1,3 +1,5 @@
+testClass = "Backup"
+
 [[test]]
 testTitle = 'SecondCycleTest'
 simBackupAgents = 'BackupToFile'
diff --git a/tests/restarting/from_7.1.0/ConfigureTestRestart-2.toml b/tests/restarting/from_7.1.0/ConfigureTestRestart-2.toml
index 8230bec479..96f1ed9ea9 100644
--- a/tests/restarting/from_7.1.0/ConfigureTestRestart-2.toml
+++ b/tests/restarting/from_7.1.0/ConfigureTestRestart-2.toml
@@ -1,4 +1,4 @@
-[[configuration]]
+[configuration]
 randomlyRenameZoneId=true
 
 [[test]]
diff --git a/tests/restarting/from_7.1.0/SnapCycleRestart-1.txt b/tests/restarting/from_7.1.0/SnapCycleRestart-1.txt
index 2636c7e236..84eadec755 100644
--- a/tests/restarting/from_7.1.0/SnapCycleRestart-1.txt
+++ b/tests/restarting/from_7.1.0/SnapCycleRestart-1.txt
@@ -1,4 +1,5 @@
-storageEngineExcludeTypes=3
+testClass=SnapshotTest
+storageEngineExcludeTypes=[3, 4, 5]
 
 ;Take snap and do cycle test
 testTitle=SnapCyclePre
diff --git a/tests/restarting/from_7.1.0/SnapCycleRestart-2.txt b/tests/restarting/from_7.1.0/SnapCycleRestart-2.txt
index 2fae7418d4..c0366ae2f4 100644
--- a/tests/restarting/from_7.1.0/SnapCycleRestart-2.txt
+++ b/tests/restarting/from_7.1.0/SnapCycleRestart-2.txt
@@ -1,3 +1,5 @@
+testClass=SnapshotTest
+storageEngineExcludeTypes=[4, 5]
 buggify=off
 
 testTitle=SnapCycleRestore
diff --git a/tests/restarting/from_7.1.0/SnapIncrementalRestore-1.txt b/tests/restarting/from_7.1.0/SnapIncrementalRestore-1.txt
index 569a05ef64..22bd11c01f 100644
--- a/tests/restarting/from_7.1.0/SnapIncrementalRestore-1.txt
+++ b/tests/restarting/from_7.1.0/SnapIncrementalRestore-1.txt
@@ -1,4 +1,5 @@
-storageEngineExcludeTypes=3
+testClass=SnapshotTest
+storageEngineExcludeTypes=[3, 4, 5]
 
 logAntiQuorum = 0
 
diff --git a/tests/restarting/from_7.1.0/SnapIncrementalRestore-2.txt b/tests/restarting/from_7.1.0/SnapIncrementalRestore-2.txt
index 2145c68e6e..fc6034373a 100644
--- a/tests/restarting/from_7.1.0/SnapIncrementalRestore-2.txt
+++ b/tests/restarting/from_7.1.0/SnapIncrementalRestore-2.txt
@@ -1,3 +1,6 @@
+testClass=SnapshotTest
+storageEngineExcludeTypes=[4, 5]
+
 testTitle=RestoreBackup
 simBackupAgents=BackupToFile
 clearAfterTest=false
diff --git a/tests/restarting/from_7.1.0/SnapTestAttrition-1.txt b/tests/restarting/from_7.1.0/SnapTestAttrition-1.txt
index a4f26975c7..826d756e88 100644
--- a/tests/restarting/from_7.1.0/SnapTestAttrition-1.txt
+++ b/tests/restarting/from_7.1.0/SnapTestAttrition-1.txt
@@ -1,4 +1,5 @@
-storageEngineExcludeTypes=3
+testClass=SnapshotTest
+storageEngineExcludeTypes=[3, 4, 5]
 
 ;write 1000 Keys ending with even numbers
 testTitle=SnapTestPre
diff --git a/tests/restarting/from_7.1.0/SnapTestAttrition-2.txt b/tests/restarting/from_7.1.0/SnapTestAttrition-2.txt
index d7d9131549..8a62641c1e 100644
--- a/tests/restarting/from_7.1.0/SnapTestAttrition-2.txt
+++ b/tests/restarting/from_7.1.0/SnapTestAttrition-2.txt
@@ -1,3 +1,6 @@
+testClass=SnapshotTest
+storageEngineExcludeTypes=[4, 5]
+
 buggify=off
 
 ; verify all keys are even numbered
diff --git a/tests/restarting/from_7.1.0/SnapTestRestart-1.txt b/tests/restarting/from_7.1.0/SnapTestRestart-1.txt
index ae229b0458..d501cad91a 100644
--- a/tests/restarting/from_7.1.0/SnapTestRestart-1.txt
+++ b/tests/restarting/from_7.1.0/SnapTestRestart-1.txt
@@ -1,4 +1,5 @@
-storageEngineExcludeTypes=3
+testClass=SnapshotTest
+storageEngineExcludeTypes=[3, 4, 5]
 
 ;write 1000 Keys ending with even numbers
 testTitle=SnapTestPre
diff --git a/tests/restarting/from_7.1.0/SnapTestRestart-2.txt b/tests/restarting/from_7.1.0/SnapTestRestart-2.txt
index 0d4aae1d16..2c67691278 100644
--- a/tests/restarting/from_7.1.0/SnapTestRestart-2.txt
+++ b/tests/restarting/from_7.1.0/SnapTestRestart-2.txt
@@ -1,3 +1,6 @@
+testClass=SnapshotTest
+storageEngineExcludeTypes=[4, 5]
+
 buggify=off
 
 ; verify all keys are even numbered
diff --git a/tests/restarting/from_7.1.0/SnapTestSimpleRestart-1.txt b/tests/restarting/from_7.1.0/SnapTestSimpleRestart-1.txt
index 118e00a16b..82532d4c90 100644
--- a/tests/restarting/from_7.1.0/SnapTestSimpleRestart-1.txt
+++ b/tests/restarting/from_7.1.0/SnapTestSimpleRestart-1.txt
@@ -1,4 +1,5 @@
-storageEngineExcludeTypes=3
+testClass=SnapshotTest
+storageEngineExcludeTypes=[3, 4, 5]
 
 ;write 1000 Keys ending with even number
 testTitle=SnapSimplePre
diff --git a/tests/restarting/from_7.1.0/SnapTestSimpleRestart-2.txt b/tests/restarting/from_7.1.0/SnapTestSimpleRestart-2.txt
index eeaa599184..2e03bf7eac 100644
--- a/tests/restarting/from_7.1.0/SnapTestSimpleRestart-2.txt
+++ b/tests/restarting/from_7.1.0/SnapTestSimpleRestart-2.txt
@@ -1,3 +1,6 @@
+testClass=SnapshotTest
+storageEngineExcludeTypes=[4, 5]
+
 buggify=off
 
 ; verify all keys are even numbered
diff --git a/tests/restarting/from_7.2.0/DrUpgradeRestart-1.txt b/tests/restarting/from_7.2.0/DrUpgradeRestart-1.txt
new file mode 100644
index 0000000000..81f932bada
--- /dev/null
+++ b/tests/restarting/from_7.2.0/DrUpgradeRestart-1.txt
@@ -0,0 +1,21 @@
+storageEngineExcludeTypes=3
+extraDatabaseMode=Local
+
+testTitle=DrUpgrade
+clearAfterTest=false
+simBackupAgents=BackupToDB
+
+    testName=Cycle
+    nodeCount=30000
+    transactionsPerSecond=2500.0
+    testDuration=30.0
+    expectedRate=0
+
+    testName=BackupToDBUpgrade
+    backupAfter=10.0
+    stopDifferentialAfter=50.0
+    backupRangesCount=-1
+    
+    testName=SaveAndKill
+    restartInfoLocation=simfdb/restartInfo.ini
+    testDuration=40.0
diff --git a/tests/restarting/from_7.2.0/DrUpgradeRestart-2.txt b/tests/restarting/from_7.2.0/DrUpgradeRestart-2.txt
new file mode 100644
index 0000000000..366d8bc2a3
--- /dev/null
+++ b/tests/restarting/from_7.2.0/DrUpgradeRestart-2.txt
@@ -0,0 +1,18 @@
+extraDatabaseMode=Local
+
+testTitle=DrUpgrade
+runSetup=false
+clearAfterTest=false
+simBackupAgents=BackupToDB
+waitForQuiescenceBegin=false
+
+    testName=Cycle
+    nodeCount=30000
+    transactionsPerSecond=2500.0
+    testDuration=30.0
+    expectedRate=0
+
+    testName=BackupToDBUpgrade
+    backupAfter=10.0
+    backupRangesCount=-1
+    stopDifferentialAfter=70.0
diff --git a/tests/slow/ApiCorrectnessSwitchover.toml b/tests/slow/ApiCorrectnessSwitchover.toml
index 3b4bfd5680..42679e9445 100644
--- a/tests/slow/ApiCorrectnessSwitchover.toml
+++ b/tests/slow/ApiCorrectnessSwitchover.toml
@@ -1,5 +1,5 @@
 [configuration]
-extraDB = 2
+extraDatabaseMode = 'Single'
 
 [[test]]
 testTitle = 'ApiCorrectnessTest'
diff --git a/tests/slow/BlobGranuleCorrectness.toml b/tests/slow/BlobGranuleCorrectness.toml
index f7ab5204a3..00aa86b07d 100644
--- a/tests/slow/BlobGranuleCorrectness.toml
+++ b/tests/slow/BlobGranuleCorrectness.toml
@@ -2,12 +2,14 @@
 blobGranulesEnabled = true 
 allowDefaultTenant = false
 allowDisablingTenants = false
+injectTargetedSSRestart = true
+injectSSDelay = true
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [4]
+storageEngineExcludeTypes = [4, 5]
 
 [[knobs]]
-bg_range_source = "tenant"
 bg_metadata_source = "tenant"
+bg_key_tuple_truncate_offset = 1
 enable_encryption = true
 
 [[test]]
@@ -15,7 +17,7 @@ testTitle = 'BlobGranuleCorrectness'
 
     [[test.workload]]
     testName = 'BlobGranuleCorrectnessWorkload'
-    testDuration = 120.0
+    testDuration = 240.0
 
     [[test.workload]]
     testName = 'RandomClogging'
@@ -38,4 +40,4 @@ testTitle = 'BlobGranuleCorrectness'
     machinesToKill = 10
     machinesToLeave = 3
     reboot = true
-    testDuration = 120.0
\ No newline at end of file
+    testDuration = 120.0
diff --git a/tests/slow/BlobGranuleCorrectnessClean.toml b/tests/slow/BlobGranuleCorrectnessClean.toml
index bf58ef1630..832e3d2b86 100644
--- a/tests/slow/BlobGranuleCorrectnessClean.toml
+++ b/tests/slow/BlobGranuleCorrectnessClean.toml
@@ -3,10 +3,9 @@ blobGranulesEnabled = true
 allowDefaultTenant = false
 allowDisablingTenants = false
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [4]
+storageEngineExcludeTypes = [4, 5]
 
 [[knobs]]
-bg_range_source = "tenant"
 bg_metadata_source = "tenant"
 enable_encryption = true
 
diff --git a/tests/slow/BlobGranuleVerifyBalance.toml b/tests/slow/BlobGranuleVerifyBalance.toml
index 8ef280284f..91f97d6d04 100644
--- a/tests/slow/BlobGranuleVerifyBalance.toml
+++ b/tests/slow/BlobGranuleVerifyBalance.toml
@@ -1,11 +1,10 @@
 [configuration]
 blobGranulesEnabled = true
 allowDefaultTenant = false
+injectTargetedSSRestart = true
+injectSSDelay = true
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [4]
-
-[[knobs]]
-bg_range_source = "blobRangeKeys"
+storageEngineExcludeTypes = [4, 5]
 
 [[test]]
 testTitle = 'BlobGranuleVerifyBalance'
diff --git a/tests/slow/BlobGranuleVerifyBalanceClean.toml b/tests/slow/BlobGranuleVerifyBalanceClean.toml
index 045d5debe9..4ea976020e 100644
--- a/tests/slow/BlobGranuleVerifyBalanceClean.toml
+++ b/tests/slow/BlobGranuleVerifyBalanceClean.toml
@@ -2,10 +2,7 @@
 blobGranulesEnabled = true
 allowDefaultTenant = false
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [4]
-
-[[knobs]]
-bg_range_source = "blobRangeKeys"
+storageEngineExcludeTypes = [4, 5]
 
 [[test]]
 testTitle = 'BlobGranuleVerifyBalanceClean'
@@ -24,3 +21,4 @@ testTitle = 'BlobGranuleVerifyBalanceClean'
     [[test.workload]]
     testName = 'BlobGranuleVerifier'
     testDuration = 120.0
+    enablePurging = false
diff --git a/tests/slow/BlobGranuleVerifyLarge.toml b/tests/slow/BlobGranuleVerifyLarge.toml
index 53d203d59f..01aac91356 100644
--- a/tests/slow/BlobGranuleVerifyLarge.toml
+++ b/tests/slow/BlobGranuleVerifyLarge.toml
@@ -1,11 +1,10 @@
 [configuration]
 blobGranulesEnabled = true 
 allowDefaultTenant = false
+injectTargetedSSRestart = true
+injectSSDelay = true
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [4]
-
-[[knobs]]
-bg_range_source = "blobRangeKeys"
+storageEngineExcludeTypes = [4, 5]
 
 [[test]]
 testTitle = 'BlobGranuleVerifyLarge'
diff --git a/tests/slow/BlobGranuleVerifyLargeClean.toml b/tests/slow/BlobGranuleVerifyLargeClean.toml
index 974703e1c8..a7adc4243a 100644
--- a/tests/slow/BlobGranuleVerifyLargeClean.toml
+++ b/tests/slow/BlobGranuleVerifyLargeClean.toml
@@ -2,10 +2,7 @@
 blobGranulesEnabled = true 
 allowDefaultTenant = false
 # FIXME: re-enable rocks at some point
-storageEngineExcludeTypes = [4]
-
-[[knobs]]
-bg_range_source = "blobRangeKeys"
+storageEngineExcludeTypes = [4, 5]
 
 [[test]]
 testTitle = 'BlobGranuleVerifyLargeClean'
@@ -28,3 +25,4 @@ testTitle = 'BlobGranuleVerifyLargeClean'
     [[test.workload]]
     testName = 'BlobGranuleVerifier'
     testDuration = 120.0
+    enablePurging = false
diff --git a/tests/slow/DifferentClustersSameRV.toml b/tests/slow/DifferentClustersSameRV.toml
index add7a2377e..ee0d34a8fa 100644
--- a/tests/slow/DifferentClustersSameRV.toml
+++ b/tests/slow/DifferentClustersSameRV.toml
@@ -1,5 +1,6 @@
 [configuration]
-extraDB = 2
+extraDatabaseMode = 'Single'
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'DifferentClustersSameRV'
diff --git a/tests/slow/DiskFailureCycle.toml b/tests/slow/DiskFailureCycle.toml
index 3f09821bf4..74e1b3a613 100644
--- a/tests/slow/DiskFailureCycle.toml
+++ b/tests/slow/DiskFailureCycle.toml
@@ -3,7 +3,7 @@ buggify = false
 minimumReplication = 3
 minimumRegions = 3
 logAntiQuorum = 0
-storageEngineExcludeTypes = [4]
+storageEngineExcludeTypes = [4, 5]
 disableRemoteKVS = true
 
 [[test]]
diff --git a/tests/slow/MetaclusterManagement.toml b/tests/slow/MetaclusterManagement.toml
new file mode 100644
index 0000000000..5e8db254d7
--- /dev/null
+++ b/tests/slow/MetaclusterManagement.toml
@@ -0,0 +1,18 @@
+[configuration]
+allowDefaultTenant = false
+allowDisablingTenants = false
+allowCreatingTenants = false
+extraDatabaseMode = 'Multiple'
+extraDatabaseCount = 5
+
+[[test]]
+testTitle = 'MetaclusterManagementTest'
+clearAfterTest = true
+timeout = 2100
+runSetup = true
+
+    [[test.workload]]
+    testName = 'MetaclusterManagement'
+	maxTenants = 1000
+	maxTenantGroups = 20
+	testDuration = 60
diff --git a/tests/slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml b/tests/slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml
index ba56f68d31..fe894b2df7 100644
--- a/tests/slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml
+++ b/tests/slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml
@@ -1,6 +1,5 @@
-# Disable buggify for parallel restore
-#[configuration]
-#buggify=on
+[configuration]
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'BackupAndParallelRestoreWithAtomicOp'
diff --git a/tests/slow/ParallelRestoreNewBackupCorrectnessCycle.toml b/tests/slow/ParallelRestoreNewBackupCorrectnessCycle.toml
index 3acece923d..f4dac7069c 100644
--- a/tests/slow/ParallelRestoreNewBackupCorrectnessCycle.toml
+++ b/tests/slow/ParallelRestoreNewBackupCorrectnessCycle.toml
@@ -1,6 +1,6 @@
-# Disable buggify for parallel restore
-#[configuration]
-#buggify=off
+[configuration]
+# tenants are not supported with parallel restore
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'BackupAndRestore'
diff --git a/tests/slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml b/tests/slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml
index 7862f5784a..0d0336d7ed 100644
--- a/tests/slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml
+++ b/tests/slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml
@@ -1,6 +1,6 @@
-# Disable buggify for parallel restore
-#[configuration]
-#buggify=off
+[configuration]
+# tenants are not supported with parallel restore
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'BackupAndRestore'
diff --git a/tests/slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml b/tests/slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml
index 4b305660bc..4f2b6b2797 100644
--- a/tests/slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml
+++ b/tests/slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml
@@ -1,5 +1,6 @@
 [configuration]
 StderrSeverity = 30
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'WriteDuringReadTest'
diff --git a/tests/slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml b/tests/slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml
index 647d15ec26..ea324b7a04 100644
--- a/tests/slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml
+++ b/tests/slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml
@@ -1,6 +1,6 @@
-# Disable buggify for parallel restore
-#[configuration]
-#buggify=on
+[configuration]
+# tenants are not supported with parallel restore
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'BackupAndParallelRestoreWithAtomicOp'
diff --git a/tests/slow/ParallelRestoreOldBackupCorrectnessCycle.toml b/tests/slow/ParallelRestoreOldBackupCorrectnessCycle.toml
index ac237864e2..bdb4e2b013 100644
--- a/tests/slow/ParallelRestoreOldBackupCorrectnessCycle.toml
+++ b/tests/slow/ParallelRestoreOldBackupCorrectnessCycle.toml
@@ -1,5 +1,6 @@
-# Disable buggify for parallel restore
-#buggify=off
+[configuration]
+# tenants are not supported with parallel restore
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'BackupAndRestore'
diff --git a/tests/slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml b/tests/slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml
index 8f6f7b92aa..78d465688b 100644
--- a/tests/slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml
+++ b/tests/slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml
@@ -1,6 +1,6 @@
-# Disable buggify for parallel restore
-#[configuration]
-#buggify=off
+[configuration]
+# tenants are not supported with parallel restore
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'BackupAndRestore'
diff --git a/tests/slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml b/tests/slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml
index 0479031d75..9827f1034f 100644
--- a/tests/slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml
+++ b/tests/slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml
@@ -1,5 +1,7 @@
 [configuration]
 StderrSeverity = 30
+# tenants are not supported with parallel restore
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'WriteDuringReadTest'
diff --git a/tests/slow/SharedBackupCorrectness.toml b/tests/slow/SharedBackupCorrectness.toml
index c03b89831a..f202cae840 100644
--- a/tests/slow/SharedBackupCorrectness.toml
+++ b/tests/slow/SharedBackupCorrectness.toml
@@ -1,5 +1,5 @@
 [configuration]
-extraDB = 1
+extraDatabaseMode = 'LocalOrSingle'
 
 [[test]]
 testTitle = 'BackupAndRestore'
diff --git a/tests/slow/SharedBackupToDBCorrectness.toml b/tests/slow/SharedBackupToDBCorrectness.toml
index 3a3a07dfbd..cebee48a2e 100644
--- a/tests/slow/SharedBackupToDBCorrectness.toml
+++ b/tests/slow/SharedBackupToDBCorrectness.toml
@@ -1,5 +1,5 @@
 [configuration]
-extraDB = 1
+extraDatabaseMode = 'LocalOrSingle'
 
 [[test]]
 testTitle = 'BackupAndRestore'
diff --git a/tests/slow/SharedDefaultBackupCorrectness.toml b/tests/slow/SharedDefaultBackupCorrectness.toml
new file mode 100644
index 0000000000..7fc89d04c2
--- /dev/null
+++ b/tests/slow/SharedDefaultBackupCorrectness.toml
@@ -0,0 +1,35 @@
+[configuration]
+extraDatabaseMode = 'Single'
+
+[[test]]
+testTitle = 'SharedDefaultBackupToFileThenDB'
+clearAfterTest = false
+simBackupAgents = 'BackupToFileAndDB'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 3000
+    transactionsPerSecond = 500.0
+    testDuration = 30.0
+    expectedRate = 0
+
+    [[test.workload]]
+    testName = 'BackupAndRestoreCorrectness'
+    backupTag = 'backup1'
+    backupAfter = 20.0
+    minBackupAfter = 10.0
+    restoreAfter = 60.0
+    shareLogRange = true
+    performRestore = true
+    allowPauses = false
+    defaultBackup = true
+
+    [[test.workload]]
+    testName = 'BackupToDBCorrectness'
+    backupTag = 'backup2'
+    backupAfter = 20.0
+    minBackupAfter = 10.0
+    restoreAfter = 60.0
+    performRestore = false
+    shareLogRange = true
+    defaultBackup = true
diff --git a/tests/slow/SwizzledTenantManagement.toml b/tests/slow/SwizzledTenantManagement.toml
index e7129f1061..8a8731915a 100644
--- a/tests/slow/SwizzledTenantManagement.toml
+++ b/tests/slow/SwizzledTenantManagement.toml
@@ -10,36 +10,36 @@ runSetup = true
 
     [[test.workload]]
     testName = 'TenantManagement'
-	maxTenants = 1000
-    testDuration = 60
+    maxTenants = 1000
+    testDuration = 120
 
     [[test.workload]]
     testName = 'RandomClogging'
-    testDuration = 120.0
+    testDuration = 180.0
     swizzle = 1
 
     [[test.workload]]
     testName = 'Rollback'
-    testDuration = 120.0
+    testDuration = 180.0
     meanDelay = 10.0
 
     [[test.workload]]
     testName = 'Attrition'
-    testDuration = 120.0
+    testDuration = 180.0
 
     [[test.workload]]
     testName = 'Attrition'
     machinesToKill = 10
     machinesToLeave = 3
     reboot = true
-    testDuration = 120.0
+    testDuration = 180.0
 
     [[test.workload]]
     testName = 'Attrition'
     machinesToKill = 10
     machinesToLeave = 3
     reboot = true
-    testDuration = 120.0
+    testDuration = 180.0
 
     [[test.workload]]
     testName = 'ChangeConfig'
diff --git a/tests/slow/SwizzledTenantManagementMetacluster.toml b/tests/slow/SwizzledTenantManagementMetacluster.toml
new file mode 100644
index 0000000000..af3a40d639
--- /dev/null
+++ b/tests/slow/SwizzledTenantManagementMetacluster.toml
@@ -0,0 +1,40 @@
+[configuration]
+allowDefaultTenant = false
+allowDisablingTenants = false
+allowCreatingTenants = false
+extraDatabaseMode = 'Single'
+
+[[test]]
+testTitle = 'TenantManagementTest'
+clearAfterTest = true
+timeout = 2100
+runSetup = true
+
+    [[test.workload]]
+    testName = 'TenantManagement'
+    maxTenants = 1000
+    testDuration = 120
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 180.0
+    swizzle = 1
+
+    [[test.workload]]
+    testName = 'Rollback'
+    testDuration = 180.0
+    meanDelay = 10.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 180.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 180.0
diff --git a/tests/slow/TenantManagement.toml b/tests/slow/TenantManagement.toml
index 5848bdf4e3..023c826153 100644
--- a/tests/slow/TenantManagement.toml
+++ b/tests/slow/TenantManagement.toml
@@ -1,6 +1,8 @@
 [configuration]
 allowDefaultTenant = false
 allowDisablingTenants = false
+allowCreatingTenants = false
+extraDatabaseMode = 'Single'
 
 [[test]]
 testTitle = 'TenantManagementTest'
@@ -10,5 +12,5 @@ runSetup = true
 
     [[test.workload]]
     testName = 'TenantManagement'
-	maxTenants = 1000
-	testDuration = 60
+    maxTenants = 1000
+    testDuration = 120
diff --git a/tests/slow/TenantManagementConcurrency.toml b/tests/slow/TenantManagementConcurrency.toml
new file mode 100644
index 0000000000..3b04d76586
--- /dev/null
+++ b/tests/slow/TenantManagementConcurrency.toml
@@ -0,0 +1,16 @@
+[configuration]
+allowDefaultTenant = false
+allowDisablingTenants = false
+allowCreatingTenants = false
+extraDatabaseMode = 'Single' 
+
+[[test]]
+testTitle = 'TenantManagementConcurrencyTest'
+clearAfterTest = true
+timeout = 2100
+runSetup = true
+
+    [[test.workload]]
+    testName = 'TenantManagementConcurrency'
+    maxTenants = 100
+    testDuration = 120
diff --git a/tests/slow/VersionStampBackupToDB.toml b/tests/slow/VersionStampBackupToDB.toml
index 4b36182dd0..88e2182cb6 100644
--- a/tests/slow/VersionStampBackupToDB.toml
+++ b/tests/slow/VersionStampBackupToDB.toml
@@ -1,5 +1,5 @@
 [configuration]
-extraDB = 2
+extraDatabaseMode = 'Single'
 
 [[test]]
 testTitle = 'VersionStampBackupToDB'
diff --git a/tests/slow/VersionStampSwitchover.toml b/tests/slow/VersionStampSwitchover.toml
index 328c199b93..16603193be 100644
--- a/tests/slow/VersionStampSwitchover.toml
+++ b/tests/slow/VersionStampSwitchover.toml
@@ -1,5 +1,5 @@
 [configuration]
-extraDB = 2
+extraDatabaseMode = 'Single'
 
 [[test]]
 testTitle = 'VersionStampCorrectnessTest'
diff --git a/tests/slow/WriteDuringReadAtomicRestore.toml b/tests/slow/WriteDuringReadAtomicRestore.toml
index 79afd3911d..a148f0a1c9 100644
--- a/tests/slow/WriteDuringReadAtomicRestore.toml
+++ b/tests/slow/WriteDuringReadAtomicRestore.toml
@@ -1,6 +1,5 @@
 [configuration]
 StderrSeverity = 30
-allowDefaultTenant = false
 
 [[test]]
 testTitle = 'WriteDuringReadTest'
diff --git a/tests/slow/WriteDuringReadSwitchover.toml b/tests/slow/WriteDuringReadSwitchover.toml
index b5232c9119..f7eb0cf01f 100644
--- a/tests/slow/WriteDuringReadSwitchover.toml
+++ b/tests/slow/WriteDuringReadSwitchover.toml
@@ -1,6 +1,6 @@
 [configuration]
 StderrSeverity = 30
-extraDB = 2
+extraDatabaseMode = 'Single'
 
 [[test]]
 testTitle = 'WriteDuringReadTest'