resolve conflicts

CMakeLists.txt

@@ -16,7 +16,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-cmake_minimum_required(VERSION 3.13)
+if(WIN32)
+  cmake_minimum_required(VERSION 3.15)
+else()
+  cmake_minimum_required(VERSION 3.13)
+endif()
+
 project(foundationdb
   VERSION 7.1.0
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."

build/Dockerfile.windows

@@ -0,0 +1,65 @@
+# escape=`
+ARG IMAGE_TAG=0.1.0
+
+# Use the latest Windows Server Core image with .NET Framework 4.8.
+FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019
+
+# Restore the default Windows shell for correct batch processing.
+SHELL ["cmd", "/S", "/C"]
+
+# Download the Build Tools bootstrapper.
+ADD https://aka.ms/vs/16/release/vs_buildtools.exe C:\TEMP\vs_buildtools.exe
+
+# Install Build Tools with the Microsoft.VisualStudio.Workload.AzureBuildTools workload, excluding workloads and components with known issues.
+RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache `
+    --installPath C:\BuildTools `
+    --add Microsoft.VisualStudio.Workload.VCTools `
+    --add Microsoft.VisualStudio.Component.TestTools.BuildTools `
+    --add Microsoft.VisualStudio.Component.VC.ASAN `
+    --add Microsoft.VisualStudio.Component.VC.CMake.Project `
+    --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
+    --add Microsoft.VisualStudio.Component.Windows10SDK.18362 `
+    --add Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset `
+    --add Microsoft.VisualStudio.Component.VC.Llvm.Clang `
+ || IF "%ERRORLEVEL%"=="3010" EXIT 0
+
+# Install Choco package manager
+RUN powershell -Command "iwr https://chocolatey.org/install.ps1 -UseBasicParsing | iex"
+RUN powershell -Command choco install 7zip -y
+
+# Download boost
+# We want to make this as early as possible as downloading and unpacking boost takes a very long time
+RUN mkdir C:\Downloads && `
+    powershell -Command iwr -Uri https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.7z -OutFile C:\Downloads\boost_1_72_0.7z && `
+    powershell -Command (Get-FileHash C:\Downloads\boost_1_72_0.7z).Hash -eq \"247a91dd7e4d9dd3c4b954b532fbc167ba62dc15ab834e5ad893d7c3f9eb5f0f\" && `
+    cd \Downloads && "C:\Program Files\7-Zip\7z" x boost_1_72_0.7z && del boost_1_72_0.7z
+
+# install other dependencies
+RUN powershell -Command choco install cmake openjdk12 python -y
+
+# add cmake to the path
+RUN setx path "%path%;C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\Program Files (x86)\Microsoft Visual Studio\installer"
+
+# Compile boost context
+RUN C:\BuildTools\Common7\Tools\VsDevCmd.bat && `
+    cd \Downloads\boost_1_72_0 && `
+    .\bootstrap.bat && `
+    powershell -Command `
+    .\b2 variant=release address-model=64 architecture=x86 link=static --with-context --build-type=minimal --layout=system `
+        -j((Get-WmiObject -Class Win32_Processor).NumberOfLogicalProcessors)
+
+# CMake's find_package wouldn't be able to find this otherwise
+RUN setx CMAKE_PREFIX_PATH "C:\Downloads\boost_1_72_0\stage\"
+
+LABEL version=${IMAGE_TAG}
+ENV DOCKER_IMAGEVER=${IMAGE_TAG}
+
+# Enable Windows Update Service (which is required to get .Net Core which is a dependency for wix) and install .Net framework
+RUN powershell "Set-Service -Name wuauserv -StartupType Manual; Install-WindowsFeature -Name NET-Framework-Features -Verbose"
+
+# Install WIX
+RUN powershell -Command choco install wixtoolset --version 3.11.2 -y
+
+# Define the entry point for the docker container.
+# This entry point starts the developer command prompt and launches the PowerShell shell.
+ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"]

build/Dockerfile.windows.devel

@@ -0,0 +1,9 @@
+# Use the latest FoundationDB Windows Build image
+FROM foundationdb/foundationdb-build-windows:0.1.0
+
+# Install git
+RUN powershell -Command choco install git -y
+
+# Define the entry point for the docker container.
+# This entry point starts the developer command prompt and launches the PowerShell shell.
+ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"]

build/build.ps1

@@ -0,0 +1,63 @@
+param (
+    [string]$SourceDir = (Resolve-Path "$PSScriptRoot\.."),
+    [string]$ImageName = "fdb-windows",
+    # By default we want to leave one CPU core for the OS so the user has some minimal control over the system
+    [string]$Cpus = (Get-CimInstance -ClassName Win32_Processor -Filter "DeviceID='CPU0'").NumberOfLogicalProcessors - 2,
+    # We want to leave at least 1GB of memory for the OS
+    [string]$Memory = (Get-CimInstance -ClassName Win32_ComputerSystem).TotalPhysicalMemory - 2*[Math]::Pow(2, 30),
+    [Parameter(Mandatory=$true)][string]$BuildDir,
+    [switch]$DryRun = $false,
+    [switch]$ForceConfigure = $false,
+    [switch]$SkipDockerBuild = $false,
+    [Parameter(Position=0)][string]$Target = "installer"
+)
+
+$BuildDir = Resolve-Path $BuildDir
+# we don't want a trailing \ in the build dir
+if ($BuildDir.EndsWith("\")) {
+    $BuildDir = $BuildDir.Substring(0, $BuildDir.Length - 1)
+}
+$exponent = 0
+$Memory = $Memory.ToUpper()
+if ($Memory.EndsWith("K")) {
+    $exponent = 10
+} elseif ($Memory.EndsWith("M")) {
+    $exponent = 20
+} elseif ($Memory.EndsWith("G")) {
+    $exponent = 30
+} elseif ($Memory.EndsWith("T")) {
+    $exponent = 40
+}
+if ($exponent -gt 0) {
+    $Memory = $Memory.Substring(0, $Memory.Length - 1) * [Math]::Pow(2, $exponent)
+}
+
+$buildCommand = [string]::Format("Get-Content {0}\build\Dockerfile.windows.devel | docker build -t {1} -m {2} -", 
+                                 $SourceDir, $ImageName, [Math]::Min(16 * [Math]::Pow(2, 30), $Memory))
+if ($DryRun -and !$SkipDockerBuild) {
+    Write-Output $buildCommand
+} elseif (!$SkipDockerBuild) {
+    Invoke-Expression -Command $buildCommand
+}
+
+# Write build instructions into file
+$cmdFile = "docker_command.ps1"
+$batFile = "$BuildDir\$cmdFile"
+$batFileDocker = "C:\fdbbuild\$cmdFile"
+# "C:\BuildTools\Common7\Tools\VsDevCmd.bat" | Out-File $batFile
+"cd \fdbbuild" | Out-File -Append $batFile
+if ($ForceConfigure -or ![System.IO.File]::Exists("$BuildDir\CMakeCache.txt") -or ($Target -eq "configure")) {
+    "cmake -G ""Visual Studio 16 2019"" -A x64 -T""ClangCL"" -S C:\foundationdb -B C:\fdbbuild --debug-trycompile" | Out-File -Append $batFile
+}
+if ($Target -ne "configure") {
+    "msbuild /p:CL_MPCount=$Cpus /m /p:UseMultiToolTask=true /p:Configuration=Release foundationdb.sln" | Out-File -Append $batFile
+}
+
+$dockerCommand = "powershell.exe -NoLogo -ExecutionPolicy Bypass -File $batFileDocker"
+$runCommand = [string]::Format("docker run -v {0}:C:\foundationdb -v {1}:C:\fdbbuild --name fdb-build -m {2} --cpus={3} --rm {4} ""{5}""",
+                                $SourceDir, $BuildDir, $Memory, $Cpus, $ImageName, $dockerCommand);
+if ($DryRun) {
+    Write-Output $runCommand
+} else {
+    Invoke-Expression $runCommand
+}

cmake/ConfigureCompiler.cmake

@@ -100,8 +100,7 @@ if(WIN32)
   endif()
   add_compile_options(/W0 /EHsc /bigobj $<$<CONFIG:Release>:/Zi> /MP /FC /Gm-)
   add_compile_definitions(NOMINMAX)
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
+  set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 else()
   set(GCC NO)
   set(CLANG NO)

design/tlog-spilling.md.html

@@ -352,7 +352,7 @@ API for random reads to the DiskQueue.  That ability is now required for
 peeking, and thus, `IDiskQueue`'s API has been enhanced correspondingly:
 
 ``` CPP
-enum class CheckHashes { NO, YES };
+BOOLEAN_PARAM(CheckHashes);
 
 class IDiskQueue {
     // ...
@@ -369,9 +369,9 @@ and not `(start, length)`.
 Spilled data, when using spill-by-value, was resistant to bitrot via data being
 checksummed interally within SQLite's B-tree.  Now that reads can be done
 directly, the responsibility for verifying data integrity falls upon the
-DiskQueue.  `CheckHashes::YES` will cause the DiskQueue to use the checksum in
+DiskQueue.  `CheckHashes::TRUE` will cause the DiskQueue to use the checksum in
 each DiskQueue page to verify data integrity.  If an externally maintained
-checksums exists to verify the returned data, then `CheckHashes::NO` can be
+checksums exists to verify the returned data, then `CheckHashes::FALSE` can be
 used to elide the checksumming.  A page failing its checksum will cause the
 transaction log to die with an `io_error()`.  
 

documentation/CMakeLists.txt

@@ -1,4 +1,8 @@
 add_subdirectory(tutorial)
+if(WIN32)
+  return()
+endif()
+
 # build a virtualenv
 set(sphinx_dir ${CMAKE_CURRENT_SOURCE_DIR}/sphinx)
 set(venv_dir ${CMAKE_CURRENT_BINARY_DIR}/venv)

fdbbackup/FileConverter.actor.cpp

@@ -598,7 +598,7 @@ int main(int argc, char** argv) {
 		Error::init();
 
 		StringRef url(param.container_url);
-		setupNetwork(0, true);
+		setupNetwork(0, UseMetrics::TRUE);
 
 		TraceEvent::setNetworkThread();
 		openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, param.log_dir, "convert", param.trace_log_group);

fdbbackup/FileDecoder.actor.cpp

@@ -579,7 +579,7 @@ int main(int argc, char** argv) {
 		Error::init();
 
 		StringRef url(param.container_url);
-		setupNetwork(0, true);
+		setupNetwork(0, UseMetrics::TRUE);
 
 		TraceEvent::setNetworkThread();
 		openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, param.log_dir, "decode", param.trace_log_group);

fdbbackup/backup.actor.cpp

@@ -133,6 +133,7 @@ enum {
 	OPT_WAITFORDONE,
 	OPT_BACKUPKEYS_FILTER,
 	OPT_INCREMENTALONLY,
+	OPT_ENCRYPTION_KEY_FILE,
 
 	// Backup Modify
 	OPT_MOD_ACTIVE_INTERVAL,
@@ -259,6 +260,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
 	{ OPT_KNOB, "--knob_", SO_REQ_SEP },
 	{ OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
 	{ OPT_INCREMENTALONLY, "--incremental", SO_NONE },
+	{ OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP },
 #ifndef TLS_DISABLED
 	TLS_OPTION_FLAGS
 #endif
@@ -697,6 +699,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = {
 	{ OPT_INCREMENTALONLY, "--incremental", SO_NONE },
 	{ OPT_RESTORE_BEGIN_VERSION, "--begin_version", SO_REQ_SEP },
 	{ OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY, "--inconsistent_snapshot_only", SO_NONE },
+	{ OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP },
 #ifndef TLS_DISABLED
 	TLS_OPTION_FLAGS
 #endif
@@ -1089,6 +1092,8 @@ static void printBackupUsage(bool devhelp) {
 	       "                 Performs incremental backup without the base backup.\n"
 	       "                 This option indicates to the backup agent that it will only need to record the log files, "
 	       "and ignore the range files.\n");
+	printf("  --encryption_key_file"
+	       "                 The AES-128-GCM key in the provided file is used for encrypting backup files.\n");
 #ifndef TLS_DISABLED
 	printf(TLS_HELP);
 #endif
@@ -1162,6 +1167,8 @@ static void printRestoreUsage(bool devhelp) {
 	       "                 To be used in conjunction with incremental restore.\n"
 	       "                 Indicates to the backup agent to only begin replaying log files from a certain version, "
 	       "instead of the entire set.\n");
+	printf("  --encryption_key_file"
+	       "                 The AES-128-GCM key in the provided file is used for decrypting backup files.\n");
 #ifndef TLS_DISABLED
 	printf(TLS_HELP);
 #endif
@@ -1463,7 +1470,7 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
                                          std::string id,
                                          ProgramExe exe,
                                          Database dest,
-                                         bool snapshot = false) {
+                                         Snapshot snapshot = Snapshot::FALSE) {
 	// This process will write a document that looks like this:
 	// { backup : { $expires : {<subdoc>}, version: <version from approximately 30 seconds from now> }
 	// so that the value under 'backup' will eventually expire to null and thus be ignored by
@@ -1639,7 +1646,7 @@ ACTOR Future<Void> cleanupStatus(Reference<ReadYourWritesTransaction> tr,
                                  std::string name,
                                  std::string id,
                                  int limit = 1) {
-	state RangeResult docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true));
+	state RangeResult docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::TRUE));
 	state bool readMore = false;
 	state int i;
 	for (i = 0; i < docs.size(); ++i) {
@@ -1668,7 +1675,7 @@ ACTOR Future<Void> cleanupStatus(Reference<ReadYourWritesTransaction> tr,
 		}
 		if (readMore) {
 			limit = 10000;
-			RangeResult docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true));
+			RangeResult docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::TRUE));
 			docs = std::move(docs2);
 			readMore = false;
 		}
@@ -1705,7 +1712,10 @@ ACTOR Future<json_spirit::mObject> getLayerStatus(Database src, std::string root
 
 // Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay
 // based on that and BACKUP_AGGREGATE_POLL_RATE
-ACTOR Future<Void> updateAgentPollRate(Database src, std::string rootKey, std::string name, double* pollDelay) {
+ACTOR Future<Void> updateAgentPollRate(Database src,
+                                       std::string rootKey,
+                                       std::string name,
+                                       std::shared_ptr<double> pollDelay) {
 	loop {
 		try {
 			json_spirit::mObject status = wait(getLayerStatus(src, rootKey));
@@ -1727,7 +1737,7 @@ ACTOR Future<Void> updateAgentPollRate(Database src, std::string rootKey, std::s
 ACTOR Future<Void> statusUpdateActor(Database statusUpdateDest,
                                      std::string name,
                                      ProgramExe exe,
-                                     double* pollDelay,
+                                     std::shared_ptr<double> pollDelay,
                                      Database taskDest = Database(),
                                      std::string id = nondeterministicRandom()->randomUniqueID().toString()) {
 	state std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name;
@@ -1757,7 +1767,8 @@ ACTOR Future<Void> statusUpdateActor(Database statusUpdateDest,
 				try {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-					state Future<std::string> futureStatusDoc = getLayerStatus(tr, name, id, exe, taskDest, true);
+					state Future<std::string> futureStatusDoc =
+					    getLayerStatus(tr, name, id, exe, taskDest, Snapshot::TRUE);
 					wait(cleanupStatus(tr, rootKey, name, id));
 					std::string statusdoc = wait(futureStatusDoc);
 					tr->set(instanceKey, statusdoc);
@@ -1774,7 +1785,7 @@ ACTOR Future<Void> statusUpdateActor(Database statusUpdateDest,
 
 			// Now that status was written at least once by this process (and hopefully others), start the poll rate
 			// control updater if it wasn't started yet
-			if (!pollRateUpdater.isValid() && pollDelay != nullptr)
+			if (!pollRateUpdater.isValid())
 				pollRateUpdater = updateAgentPollRate(statusUpdateDest, rootKey, name, pollDelay);
 		} catch (Error& e) {
 			TraceEvent(SevWarnAlways, "UnableToWriteStatus").error(e);
@@ -1784,17 +1795,17 @@ ACTOR Future<Void> statusUpdateActor(Database statusUpdateDest,
 }
 
 ACTOR Future<Void> runDBAgent(Database src, Database dest) {
-	state double pollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
+	state std::shared_ptr<double> pollDelay = std::make_shared<double>(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE);
 	std::string id = nondeterministicRandom()->randomUniqueID().toString();
-	state Future<Void> status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, &pollDelay, dest, id);
+	state Future<Void> status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, pollDelay, dest, id);
 	state Future<Void> status_other =
-	    statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, &pollDelay, dest, id);
+	    statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, pollDelay, dest, id);
 
 	state DatabaseBackupAgent backupAgent(src);
 
 	loop {
 		try {
-			wait(backupAgent.run(dest, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT));
+			wait(backupAgent.run(dest, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT));
 			break;
 		} catch (Error& e) {
 			if (e.code() == error_code_operation_cancelled)
@@ -1811,14 +1822,14 @@ ACTOR Future<Void> runDBAgent(Database src, Database dest) {
 }
 
 ACTOR Future<Void> runAgent(Database db) {
-	state double pollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
-	state Future<Void> status = statusUpdateActor(db, "backup", ProgramExe::AGENT, &pollDelay);
+	state std::shared_ptr<double> pollDelay = std::make_shared<double>(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE);
+	state Future<Void> status = statusUpdateActor(db, "backup", ProgramExe::AGENT, pollDelay);
 
 	state FileBackupAgent backupAgent;
 
 	loop {
 		try {
-			wait(backupAgent.run(db, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT));
+			wait(backupAgent.run(db, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT));
 			break;
 		} catch (Error& e) {
 			if (e.code() == error_code_operation_cancelled)
@@ -1846,7 +1857,8 @@ ACTOR Future<Void> submitDBBackup(Database src,
 			backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
 		}
 
-		wait(backupAgent.submitBackup(dest, KeyRef(tagName), backupRanges, false, StringRef(), StringRef(), true));
+		wait(backupAgent.submitBackup(
+		    dest, KeyRef(tagName), backupRanges, StopWhenDone::FALSE, StringRef(), StringRef(), LockDB::TRUE));
 
 		// Check if a backup agent is running
 		bool agentRunning = wait(backupAgent.checkActive(dest));
@@ -1890,10 +1902,10 @@ ACTOR Future<Void> submitBackup(Database db,
                                 Standalone<VectorRef<KeyRangeRef>> backupRanges,
                                 std::string tagName,
                                 bool dryRun,
-                                bool waitForCompletion,
-                                bool stopWhenDone,
-                                bool usePartitionedLog,
-                                bool incrementalBackupOnly) {
+                                WaitForComplete waitForCompletion,
+                                StopWhenDone stopWhenDone,
+                                UsePartitionedLog usePartitionedLog,
+                                IncrementalBackupOnly incrementalBackupOnly) {
 	try {
 		state FileBackupAgent backupAgent;
 
@@ -1996,7 +2008,7 @@ ACTOR Future<Void> switchDBBackup(Database src,
                                   Database dest,
                                   Standalone<VectorRef<KeyRangeRef>> backupRanges,
                                   std::string tagName,
-                                  bool forceAction) {
+                                  ForceAction forceAction) {
 	try {
 		state DatabaseBackupAgent backupAgent(src);
 
@@ -2046,7 +2058,7 @@ ACTOR Future<Void> statusDBBackup(Database src, Database dest, std::string tagNa
 	return Void();
 }
 
-ACTOR Future<Void> statusBackup(Database db, std::string tagName, bool showErrors, bool json) {
+ACTOR Future<Void> statusBackup(Database db, std::string tagName, ShowErrors showErrors, bool json) {
 	try {
 		state FileBackupAgent backupAgent;
 
@@ -2063,11 +2075,15 @@ ACTOR Future<Void> statusBackup(Database db, std::string tagName, bool showError
 	return Void();
 }
 
-ACTOR Future<Void> abortDBBackup(Database src, Database dest, std::string tagName, bool partial, bool dstOnly) {
+ACTOR Future<Void> abortDBBackup(Database src,
+                                 Database dest,
+                                 std::string tagName,
+                                 PartialBackup partial,
+                                 DstOnly dstOnly) {
 	try {
 		state DatabaseBackupAgent backupAgent(src);
 
-		wait(backupAgent.abortBackup(dest, Key(tagName), partial, false, dstOnly));
+		wait(backupAgent.abortBackup(dest, Key(tagName), partial, AbortOldBackup::FALSE, dstOnly));
 		wait(backupAgent.unlockBackup(dest, Key(tagName)));
 
 		printf("The DR on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str());
@@ -2118,7 +2134,7 @@ ACTOR Future<Void> abortBackup(Database db, std::string tagName) {
 	return Void();
 }
 
-ACTOR Future<Void> cleanupMutations(Database db, bool deleteData) {
+ACTOR Future<Void> cleanupMutations(Database db, DeleteData deleteData) {
 	try {
 		wait(cleanupBackup(db, deleteData));
 	} catch (Error& e) {
@@ -2131,7 +2147,7 @@ ACTOR Future<Void> cleanupMutations(Database db, bool deleteData) {
 	return Void();
 }
 
-ACTOR Future<Void> waitBackup(Database db, std::string tagName, bool stopWhenDone) {
+ACTOR Future<Void> waitBackup(Database db, std::string tagName, StopWhenDone stopWhenDone) {
 	try {
 		state FileBackupAgent backupAgent;
 
@@ -2150,7 +2166,7 @@ ACTOR Future<Void> waitBackup(Database db, std::string tagName, bool stopWhenDon
 	return Void();
 }
 
-ACTOR Future<Void> discontinueBackup(Database db, std::string tagName, bool waitForCompletion) {
+ACTOR Future<Void> discontinueBackup(Database db, std::string tagName, WaitForComplete waitForCompletion) {
 	try {
 		state FileBackupAgent backupAgent;
 
@@ -2220,7 +2236,9 @@ ACTOR Future<Void> changeDBBackupResumed(Database src, Database dest, bool pause
 	return Void();
 }
 
-Reference<IBackupContainer> openBackupContainer(const char* name, std::string destinationContainer) {
+Reference<IBackupContainer> openBackupContainer(const char* name,
+                                                std::string destinationContainer,
+                                                Optional<std::string> const& encryptionKeyFile = {}) {
 	// Error, if no dest container was specified
 	if (destinationContainer.empty()) {
 		fprintf(stderr, "ERROR: No backup destination was specified.\n");
@@ -2230,7 +2248,7 @@ Reference<IBackupContainer> openBackupContainer(const char* name, std::string de
 
 	Reference<IBackupContainer> c;
 	try {
-		c = IBackupContainer::openContainer(destinationContainer);
+		c = IBackupContainer::openContainer(destinationContainer, encryptionKeyFile);
 	} catch (Error& e) {
 		std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str());
 		if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) {
@@ -2255,12 +2273,13 @@ ACTOR Future<Void> runRestore(Database db,
                               Version targetVersion,
                               std::string targetTimestamp,
                               bool performRestore,
-                              bool verbose,
-                              bool waitForDone,
+                              Verbose verbose,
+                              WaitForComplete waitForDone,
                               std::string addPrefix,
                               std::string removePrefix,
-                              bool onlyAppyMutationLogs,
-                              bool inconsistentSnapshotOnly) {
+                              OnlyApplyMutationLogs onlyApplyMutationLogs,
+                              InconsistentSnapshotOnly inconsistentSnapshotOnly,
+                              Optional<std::string> encryptionKeyFile) {
 	if (ranges.empty()) {
 		ranges.push_back_deep(ranges.arena(), normalKeys);
 	}
@@ -2296,7 +2315,8 @@ ACTOR Future<Void> runRestore(Database db,
 	try {
 		state FileBackupAgent backupAgent;
 
-		state Reference<IBackupContainer> bc = openBackupContainer(exeRestore.toString().c_str(), container);
+		state Reference<IBackupContainer> bc =
+		    openBackupContainer(exeRestore.toString().c_str(), container, encryptionKeyFile);
 
 		// If targetVersion is unset then use the maximum restorable version from the backup description
 		if (targetVersion == invalidVersion) {
@@ -2306,7 +2326,7 @@ ACTOR Future<Void> runRestore(Database db,
 
 			BackupDescription desc = wait(bc->describeBackup());
 
-			if (onlyAppyMutationLogs && desc.contiguousLogEnd.present()) {
+			if (onlyApplyMutationLogs && desc.contiguousLogEnd.present()) {
 				targetVersion = desc.contiguousLogEnd.get() - 1;
 			} else if (desc.maxRestorableVersion.present()) {
 				targetVersion = desc.maxRestorableVersion.get();
@@ -2330,10 +2350,11 @@ ACTOR Future<Void> runRestore(Database db,
 			                                                   verbose,
 			                                                   KeyRef(addPrefix),
 			                                                   KeyRef(removePrefix),
-			                                                   true,
-			                                                   onlyAppyMutationLogs,
+			                                                   LockDB::TRUE,
+			                                                   onlyApplyMutationLogs,
 			                                                   inconsistentSnapshotOnly,
-			                                                   beginVersion));
+			                                                   beginVersion,
+			                                                   encryptionKeyFile));
 
 			if (waitForDone && verbose) {
 				// If restore is now complete then report version restored
@@ -2369,8 +2390,8 @@ ACTOR Future<Void> runFastRestoreTool(Database db,
                                       Standalone<VectorRef<KeyRangeRef>> ranges,
                                       Version dbVersion,
                                       bool performRestore,
-                                      bool verbose,
-                                      bool waitForDone) {
+                                      Verbose verbose,
+                                      WaitForComplete waitForDone) {
 	try {
 		state FileBackupAgent backupAgent;
 		state Version restoreVersion = invalidVersion;
@@ -2413,7 +2434,7 @@ ACTOR Future<Void> runFastRestoreTool(Database db,
 			                                       ranges,
 			                                       KeyRef(container),
 			                                       dbVersion,
-			                                       true,
+			                                       LockDB::TRUE,
 			                                       randomUID,
 			                                       LiteralStringRef(""),
 			                                       LiteralStringRef("")));
@@ -2512,7 +2533,8 @@ ACTOR Future<Void> expireBackupData(const char* name,
                                     Database db,
                                     bool force,
                                     Version restorableAfterVersion,
-                                    std::string restorableAfterDatetime) {
+                                    std::string restorableAfterDatetime,
+                                    Optional<std::string> encryptionKeyFile) {
 	if (!endDatetime.empty()) {
 		Version v = wait(timeKeeperVersionFromDatetime(endDatetime, db));
 		endVersion = v;
@@ -2531,7 +2553,7 @@ ACTOR Future<Void> expireBackupData(const char* name,
 	}
 
 	try {
-		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer);
+		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, encryptionKeyFile);
 
 		state IBackupContainer::ExpireProgress progress;
 		state std::string lastProgress;
@@ -2613,9 +2635,10 @@ ACTOR Future<Void> describeBackup(const char* name,
                                   std::string destinationContainer,
                                   bool deep,
                                   Optional<Database> cx,
-                                  bool json) {
+                                  bool json,
+                                  Optional<std::string> encryptionKeyFile) {
 	try {
-		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer);
+		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, encryptionKeyFile);
 		state BackupDescription desc = wait(c->describeBackup(deep));
 		if (cx.present())
 			wait(desc.resolveVersionTimes(cx.get()));
@@ -2645,7 +2668,7 @@ ACTOR Future<Void> queryBackup(const char* name,
                                Version restoreVersion,
                                std::string originalClusterFile,
                                std::string restoreTimestamp,
-                               bool verbose) {
+                               Verbose verbose) {
 	state UID operationId = deterministicRandom()->randomUniqueID();
 	state JsonBuilderObject result;
 	state std::string errorMessage;
@@ -2838,7 +2861,7 @@ ACTOR Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOp
 			}
 
 			state BackupConfig config(uidFlag.get().first);
-			EBackupState s = wait(config.stateEnum().getOrThrow(tr, false, backup_invalid_info()));
+			EBackupState s = wait(config.stateEnum().getOrThrow(tr, Snapshot::FALSE, backup_invalid_info()));
 			if (!FileBackupAgent::isRunnable(s)) {
 				fprintf(stderr, "Backup on tag '%s' is not runnable.\n", tagName.c_str());
 				throw backup_error();
@@ -2858,7 +2881,7 @@ ACTOR Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOp
 			}
 
 			if (options.activeSnapshotIntervalSeconds.present()) {
-				Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, false, backup_error()));
+				Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, Snapshot::FALSE, backup_error()));
 				config.snapshotTargetEndVersion().set(tr,
 				                                      begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() *
 				                                               CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
@@ -3244,13 +3267,13 @@ int main(int argc, char* argv[]) {
 		Version beginVersion = invalidVersion;
 		Version restoreVersion = invalidVersion;
 		std::string restoreTimestamp;
-		bool waitForDone = false;
-		bool stopWhenDone = true;
-		bool usePartitionedLog = false; // Set to true to use new backup system
-		bool incrementalBackupOnly = false;
-		bool onlyAppyMutationLogs = false;
-		bool inconsistentSnapshotOnly = false;
-		bool forceAction = false;
+		WaitForComplete waitForDone{ false };
+		StopWhenDone stopWhenDone{ true };
+		UsePartitionedLog usePartitionedLog{ false }; // Set to true to use new backup system
+		IncrementalBackupOnly incrementalBackupOnly{ false };
+		OnlyApplyMutationLogs onlyApplyMutationLogs{ false };
+		InconsistentSnapshotOnly inconsistentSnapshotOnly{ false };
+		ForceAction forceAction{ false };
 		bool trace = false;
 		bool quietDisplay = false;
 		bool dryRun = false;
@@ -3260,8 +3283,8 @@ int main(int argc, char* argv[]) {
 		uint64_t traceRollSize = TRACE_DEFAULT_ROLL_SIZE;
 		uint64_t traceMaxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE;
 		ESOError lastError;
-		bool partial = true;
-		bool dstOnly = false;
+		PartialBackup partial{ true };
+		DstOnly dstOnly{ false };
 		LocalityData localities;
 		uint64_t memLimit = 8LL << 30;
 		Optional<uint64_t> ti;
@@ -3271,7 +3294,8 @@ int main(int argc, char* argv[]) {
 		std::string restoreClusterFileDest;
 		std::string restoreClusterFileOrig;
 		bool jsonOutput = false;
-		bool deleteData = false;
+		DeleteData deleteData{ false };
+		Optional<std::string> encryptionKeyFile;
 
 		BackupModifyOptions modifyOptions;
 
@@ -3355,13 +3379,13 @@ int main(int argc, char* argv[]) {
 				dryRun = true;
 				break;
 			case OPT_DELETE_DATA:
-				deleteData = true;
+				deleteData.set(true);
 				break;
 			case OPT_MIN_CLEANUP_SECONDS:
 				knobs.emplace_back("min_cleanup_seconds", args->OptionArg());
 				break;
 			case OPT_FORCE:
-				forceAction = true;
+				forceAction.set(true);
 				break;
 			case OPT_TRACE:
 				trace = true;
@@ -3441,10 +3465,10 @@ int main(int argc, char* argv[]) {
 				sourceClusterFile = args->OptionArg();
 				break;
 			case OPT_CLEANUP:
-				partial = false;
+				partial.set(false);
 				break;
 			case OPT_DSTONLY:
-				dstOnly = true;
+				dstOnly.set(true);
 				break;
 			case OPT_KNOB: {
 				std::string syn = args->OptionSyntax();
@@ -3503,17 +3527,20 @@ int main(int argc, char* argv[]) {
 				modifyOptions.verifyUID = args->OptionArg();
 				break;
 			case OPT_WAITFORDONE:
-				waitForDone = true;
+				waitForDone.set(true);
 				break;
 			case OPT_NOSTOPWHENDONE:
-				stopWhenDone = false;
+				stopWhenDone.set(false);
 				break;
 			case OPT_USE_PARTITIONED_LOG:
-				usePartitionedLog = true;
+				usePartitionedLog.set(true);
 				break;
 			case OPT_INCREMENTALONLY:
-				incrementalBackupOnly = true;
-				onlyAppyMutationLogs = true;
+				incrementalBackupOnly.set(true);
+				onlyApplyMutationLogs.set(true);
+				break;
+			case OPT_ENCRYPTION_KEY_FILE:
+				encryptionKeyFile = args->OptionArg();
 				break;
 			case OPT_RESTORECONTAINER:
 				restoreContainer = args->OptionArg();
@@ -3565,7 +3592,7 @@ int main(int argc, char* argv[]) {
 				break;
 			}
 			case OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY: {
-				inconsistentSnapshotOnly = true;
+				inconsistentSnapshotOnly.set(true);
 				break;
 			}
 #ifdef _WIN32
@@ -3704,7 +3731,7 @@ int main(int argc, char* argv[]) {
 			}
 		}
 
-		IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO);
+		IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::FALSE, IsSimulated::FALSE);
 		auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
 		for (const auto& [knobName, knobValueString] : knobs) {
 			try {
@@ -3731,7 +3758,7 @@ int main(int argc, char* argv[]) {
 		}
 
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		g_knobs.initialize(Randomize::NO, IsSimulated::NO);
+		g_knobs.initialize(Randomize::FALSE, IsSimulated::FALSE);
 
 		if (trace) {
 			if (!traceLogGroup.empty())
@@ -3769,7 +3796,7 @@ int main(int argc, char* argv[]) {
 		Reference<IBackupContainer> c;
 
 		try {
-			setupNetwork(0, true);
+			setupNetwork(0, UseMetrics::TRUE);
 		} catch (Error& e) {
 			fprintf(stderr, "ERROR: %s\n", e.what());
 			return FDB_EXIT_ERROR;
@@ -3813,7 +3840,7 @@ int main(int argc, char* argv[]) {
 			}
 
 			try {
-				db = Database::createDatabase(ccf, -1, true, localities);
+				db = Database::createDatabase(ccf, -1, IsInternal::TRUE, localities);
 			} catch (Error& e) {
 				fprintf(stderr, "ERROR: %s\n", e.what());
 				fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getFilename().c_str());
@@ -3833,7 +3860,7 @@ int main(int argc, char* argv[]) {
 			}
 
 			try {
-				sourceDb = Database::createDatabase(sourceCcf, -1, true, localities);
+				sourceDb = Database::createDatabase(sourceCcf, -1, IsInternal::TRUE, localities);
 			} catch (Error& e) {
 				fprintf(stderr, "ERROR: %s\n", e.what());
 				fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", sourceCcf->getFilename().c_str());
@@ -3853,7 +3880,7 @@ int main(int argc, char* argv[]) {
 				if (!initCluster())
 					return FDB_EXIT_ERROR;
 				// Test out the backup url to make sure it parses.  Doesn't test to make sure it's actually writeable.
-				openBackupContainer(argv[0], destinationContainer);
+				openBackupContainer(argv[0], destinationContainer, encryptionKeyFile);
 				f = stopAfter(submitBackup(db,
 				                           destinationContainer,
 				                           initialSnapshotIntervalSeconds,
@@ -3879,7 +3906,7 @@ int main(int argc, char* argv[]) {
 			case BackupType::STATUS:
 				if (!initCluster())
 					return FDB_EXIT_ERROR;
-				f = stopAfter(statusBackup(db, tagName, true, jsonOutput));
+				f = stopAfter(statusBackup(db, tagName, ShowErrors::TRUE, jsonOutput));
 				break;
 
 			case BackupType::ABORT:
@@ -3932,7 +3959,8 @@ int main(int argc, char* argv[]) {
 				                               db,
 				                               forceAction,
 				                               expireRestorableAfterVersion,
-				                               expireRestorableAfterDatetime));
+				                               expireRestorableAfterDatetime,
+				                               encryptionKeyFile));
 				break;
 
 			case BackupType::DELETE_BACKUP:
@@ -3952,7 +3980,8 @@ int main(int argc, char* argv[]) {
 				                             destinationContainer,
 				                             describeDeep,
 				                             describeTimestamps ? Optional<Database>(db) : Optional<Database>(),
-				                             jsonOutput));
+				                             jsonOutput,
+				                             encryptionKeyFile));
 				break;
 
 			case BackupType::LIST:
@@ -3968,7 +3997,7 @@ int main(int argc, char* argv[]) {
 				                          restoreVersion,
 				                          restoreClusterFileOrig,
 				                          restoreTimestamp,
-				                          !quietDisplay));
+				                          Verbose{ !quietDisplay }));
 				break;
 
 			case BackupType::DUMP:
@@ -4029,15 +4058,16 @@ int main(int argc, char* argv[]) {
 				                         restoreVersion,
 				                         restoreTimestamp,
 				                         !dryRun,
-				                         !quietDisplay,
+				                         Verbose{ !quietDisplay },
 				                         waitForDone,
 				                         addPrefix,
 				                         removePrefix,
-				                         onlyAppyMutationLogs,
-				                         inconsistentSnapshotOnly));
+				                         onlyApplyMutationLogs,
+				                         inconsistentSnapshotOnly,
+				                         encryptionKeyFile));
 				break;
 			case RestoreType::WAIT:
-				f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), true)));
+				f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), Verbose::TRUE)));
 				break;
 			case RestoreType::ABORT:
 				f = stopAfter(
@@ -4097,8 +4127,14 @@ int main(int argc, char* argv[]) {
 			// TODO: We have not implemented the code commented out in this case
 			switch (restoreType) {
 			case RestoreType::START:
-				f = stopAfter(runFastRestoreTool(
-				    db, tagName, restoreContainer, backupKeys, restoreVersion, !dryRun, !quietDisplay, waitForDone));
+				f = stopAfter(runFastRestoreTool(db,
+				                                 tagName,
+				                                 restoreContainer,
+				                                 backupKeys,
+				                                 restoreVersion,
+				                                 !dryRun,
+				                                 Verbose{ !quietDisplay },
+				                                 waitForDone));
 				break;
 			case RestoreType::WAIT:
 				printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n");

fdbcli/fdbcli.actor.cpp

@@ -3151,7 +3151,7 @@ struct CLIOptions {
 		}
 
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		g_knobs.initialize(Randomize::NO, IsSimulated::NO);
+		g_knobs.initialize(Randomize::FALSE, IsSimulated::FALSE);
 	}
 
 	int processArg(CSimpleOpt& args) {
@@ -3322,7 +3322,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	TraceEvent::setNetworkThread();
 
 	try {
-		db = Database::createDatabase(ccf, -1, false);
+		db = Database::createDatabase(ccf, -1, IsInternal::FALSE);
 		if (!opt.exec.present()) {
 			printf("Using cluster file `%s'.\n", ccf->getFilename().c_str());
 		}
@@ -4924,7 +4924,7 @@ int main(int argc, char** argv) {
 
 	registerCrashHandler();
 
-	IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO);
+	IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::FALSE, IsSimulated::FALSE);
 
 #ifdef __unixish__
 	struct sigaction act;

fdbclient/AsyncFileS3BlobStore.actor.h

@@ -256,7 +256,7 @@ public:
 	    m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) {
 
 		// Add first part
-		m_parts.push_back(Reference<Part>(new Part(1, m_bstore->knobs.multipart_min_part_size)));
+		m_parts.push_back(makeReference<Part>(1, m_bstore->knobs.multipart_min_part_size));
 	}
 };
 

fdbclient/AsyncTaskThread.actor.cpp

@@ -83,6 +83,6 @@ TEST_CASE("/asynctaskthread/add") {
 		clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, 100));
 	}
 	wait(waitForAll(clients));
-	ASSERT(sum == 1000);
+	ASSERT_EQ(sum, 1000);
 	return Void();
 }

fdbclient/BackupAgent.actor.h

@@ -36,6 +36,26 @@
 #include "fdbclient/BackupContainer.h"
 #include "flow/actorcompiler.h" // has to be last include
 
+FDB_DECLARE_BOOLEAN_PARAM(LockDB);
+FDB_DECLARE_BOOLEAN_PARAM(UnlockDB);
+FDB_DECLARE_BOOLEAN_PARAM(StopWhenDone);
+FDB_DECLARE_BOOLEAN_PARAM(Verbose);
+FDB_DECLARE_BOOLEAN_PARAM(WaitForComplete);
+FDB_DECLARE_BOOLEAN_PARAM(ForceAction);
+FDB_DECLARE_BOOLEAN_PARAM(Terminator);
+FDB_DECLARE_BOOLEAN_PARAM(IncrementalBackupOnly);
+FDB_DECLARE_BOOLEAN_PARAM(UsePartitionedLog);
+FDB_DECLARE_BOOLEAN_PARAM(OnlyApplyMutationLogs);
+FDB_DECLARE_BOOLEAN_PARAM(InconsistentSnapshotOnly);
+FDB_DECLARE_BOOLEAN_PARAM(ShowErrors);
+FDB_DECLARE_BOOLEAN_PARAM(AbortOldBackup);
+FDB_DECLARE_BOOLEAN_PARAM(DstOnly); // TODO: More descriptive name?
+FDB_DECLARE_BOOLEAN_PARAM(WaitForDestUID);
+FDB_DECLARE_BOOLEAN_PARAM(CheckBackupUID);
+FDB_DECLARE_BOOLEAN_PARAM(DeleteData);
+FDB_DECLARE_BOOLEAN_PARAM(SetValidation);
+FDB_DECLARE_BOOLEAN_PARAM(PartialBackup);
+
 class BackupAgentBase : NonCopyable {
 public:
 	// Time formatter for anything backup or restore related
@@ -65,6 +85,7 @@ public:
 	static const Key keyConfigStopWhenDoneKey;
 	static const Key keyStateStatus;
 	static const Key keyStateStop;
+	static const Key keyStateLogBeginVersion;
 	static const Key keyLastUid;
 	static const Key keyBeginKey;
 	static const Key keyEndKey;
@@ -124,7 +145,11 @@ public:
 
 	KeyBackedProperty<Key> lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); }
 
-	Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
+	Future<Void> run(Database cx, double pollDelay, int maxConcurrentTasks) {
+		return taskBucket->run(cx, futureBucket, std::make_shared<double const>(pollDelay), maxConcurrentTasks);
+	}
+
+	Future<Void> run(Database cx, std::shared_ptr<double const> pollDelay, int maxConcurrentTasks) {
 		return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
 	}
 
@@ -135,13 +160,13 @@ public:
 	static Key getPauseKey();
 
 	// parallel restore
-	Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true);
+	Future<Void> parallelRestoreFinish(Database cx, UID randomUID, UnlockDB = UnlockDB::TRUE);
 	Future<Void> submitParallelRestore(Database cx,
 	                                   Key backupTag,
 	                                   Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                                   Key bcUrl,
 	                                   Version targetVersion,
-	                                   bool lockDB,
+	                                   LockDB lockDB,
 	                                   UID randomUID,
 	                                   Key addPrefix,
 	                                   Key removePrefix);
@@ -163,29 +188,31 @@ public:
 	                        Key tagName,
 	                        Key url,
 	                        Standalone<VectorRef<KeyRangeRef>> ranges,
-	                        bool waitForComplete = true,
-	                        Version targetVersion = -1,
-	                        bool verbose = true,
+	                        WaitForComplete = WaitForComplete::TRUE,
+	                        Version targetVersion = ::invalidVersion,
+	                        Verbose = Verbose::TRUE,
 	                        Key addPrefix = Key(),
 	                        Key removePrefix = Key(),
-	                        bool lockDB = true,
-	                        bool onlyAppyMutationLogs = false,
-	                        bool inconsistentSnapshotOnly = false,
-	                        Version beginVersion = -1);
+	                        LockDB = LockDB::TRUE,
+	                        OnlyApplyMutationLogs = OnlyApplyMutationLogs::FALSE,
+	                        InconsistentSnapshotOnly = InconsistentSnapshotOnly::FALSE,
+	                        Version beginVersion = ::invalidVersion,
+	                        Optional<std::string> const& encryptionKeyFileName = {});
 	Future<Version> restore(Database cx,
 	                        Optional<Database> cxOrig,
 	                        Key tagName,
 	                        Key url,
-	                        bool waitForComplete = true,
-	                        Version targetVersion = -1,
-	                        bool verbose = true,
+	                        WaitForComplete waitForComplete = WaitForComplete::TRUE,
+	                        Version targetVersion = ::invalidVersion,
+	                        Verbose verbose = Verbose::TRUE,
 	                        KeyRange range = normalKeys,
 	                        Key addPrefix = Key(),
 	                        Key removePrefix = Key(),
-	                        bool lockDB = true,
-	                        bool onlyAppyMutationLogs = false,
-	                        bool inconsistentSnapshotOnly = false,
-	                        Version beginVersion = -1) {
+	                        LockDB lockDB = LockDB::TRUE,
+	                        OnlyApplyMutationLogs onlyApplyMutationLogs = OnlyApplyMutationLogs::FALSE,
+	                        InconsistentSnapshotOnly inconsistentSnapshotOnly = InconsistentSnapshotOnly::FALSE,
+	                        Version beginVersion = ::invalidVersion,
+	                        Optional<std::string> const& encryptionKeyFileName = {}) {
 		Standalone<VectorRef<KeyRangeRef>> rangeRef;
 		rangeRef.push_back_deep(rangeRef.arena(), range);
 		return restore(cx,
@@ -199,9 +226,10 @@ public:
 		               addPrefix,
 		               removePrefix,
 		               lockDB,
-		               onlyAppyMutationLogs,
+		               onlyApplyMutationLogs,
 		               inconsistentSnapshotOnly,
-		               beginVersion);
+		               beginVersion,
+		               encryptionKeyFileName);
 	}
 	Future<Version> atomicRestore(Database cx,
 	                              Key tagName,
@@ -222,7 +250,7 @@ public:
 	Future<ERestoreState> abortRestore(Database cx, Key tagName);
 
 	// Waits for a restore tag to reach a final (stable) state.
-	Future<ERestoreState> waitRestore(Database cx, Key tagName, bool verbose);
+	Future<ERestoreState> waitRestore(Database cx, Key tagName, Verbose);
 
 	// Get a string describing the status of a tag
 	Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName);
@@ -237,20 +265,22 @@ public:
 	                          Key outContainer,
 	                          int initialSnapshotIntervalSeconds,
 	                          int snapshotIntervalSeconds,
-	                          std::string tagName,
+	                          std::string const& tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                          bool stopWhenDone = true,
-	                          bool partitionedLog = false,
-	                          bool incrementalBackupOnly = false);
+	                          StopWhenDone = StopWhenDone::TRUE,
+	                          UsePartitionedLog = UsePartitionedLog::FALSE,
+	                          IncrementalBackupOnly = IncrementalBackupOnly::FALSE,
+	                          Optional<std::string> const& encryptionKeyFileName = {});
 	Future<Void> submitBackup(Database cx,
 	                          Key outContainer,
 	                          int initialSnapshotIntervalSeconds,
 	                          int snapshotIntervalSeconds,
-	                          std::string tagName,
+	                          std::string const& tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                          bool stopWhenDone = true,
-	                          bool partitionedLog = false,
-	                          bool incrementalBackupOnly = false) {
+	                          StopWhenDone stopWhenDone = StopWhenDone::TRUE,
+	                          UsePartitionedLog partitionedLog = UsePartitionedLog::FALSE,
+	                          IncrementalBackupOnly incrementalBackupOnly = IncrementalBackupOnly::FALSE,
+	                          Optional<std::string> const& encryptionKeyFileName = {}) {
 		return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
 			return submitBackup(tr,
 			                    outContainer,
@@ -260,7 +290,8 @@ public:
 			                    backupRanges,
 			                    stopWhenDone,
 			                    partitionedLog,
-			                    incrementalBackupOnly);
+			                    incrementalBackupOnly,
+			                    encryptionKeyFileName);
 		});
 	}
 
@@ -282,19 +313,19 @@ public:
 		return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return abortBackup(tr, tagName); });
 	}
 
-	Future<std::string> getStatus(Database cx, bool showErrors, std::string tagName);
+	Future<std::string> getStatus(Database cx, ShowErrors, std::string tagName);
 	Future<std::string> getStatusJSON(Database cx, std::string tagName);
 
 	Future<Optional<Version>> getLastRestorable(Reference<ReadYourWritesTransaction> tr,
 	                                            Key tagName,
-	                                            bool snapshot = false);
+	                                            Snapshot = Snapshot::FALSE);
 	void setLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName, Version version);
 
 	// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
 	// will return when the backup directory is restorable.
 	Future<EnumState> waitBackup(Database cx,
 	                             std::string tagName,
-	                             bool stopWhenDone = true,
+	                             StopWhenDone = StopWhenDone::TRUE,
 	                             Reference<IBackupContainer>* pContainer = nullptr,
 	                             UID* pUID = nullptr);
 
@@ -353,7 +384,11 @@ public:
 		sourceTagNames = std::move(r.sourceTagNames);
 	}
 
-	Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
+	Future<Void> run(Database cx, double pollDelay, int maxConcurrentTasks) {
+		return taskBucket->run(cx, futureBucket, std::make_shared<double const>(pollDelay), maxConcurrentTasks);
+	}
+
+	Future<Void> run(Database cx, std::shared_ptr<double const> pollDelay, int maxConcurrentTasks) {
 		return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
 	}
 
@@ -362,7 +397,7 @@ public:
 	                              Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                              Key addPrefix,
 	                              Key removePrefix,
-	                              bool forceAction = false);
+	                              ForceAction = ForceAction::FALSE);
 
 	Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
 	Future<Void> unlockBackup(Database cx, Key tagName) {
@@ -381,18 +416,18 @@ public:
 	Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
 	                          Key tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                          bool stopWhenDone = true,
+	                          StopWhenDone = StopWhenDone::TRUE,
 	                          Key addPrefix = StringRef(),
 	                          Key removePrefix = StringRef(),
-	                          bool lockDatabase = false,
+	                          LockDB lockDatabase = LockDB::FALSE,
 	                          PreBackupAction backupAction = PreBackupAction::VERIFY);
 	Future<Void> submitBackup(Database cx,
 	                          Key tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                          bool stopWhenDone = true,
+	                          StopWhenDone stopWhenDone = StopWhenDone::TRUE,
 	                          Key addPrefix = StringRef(),
 	                          Key removePrefix = StringRef(),
-	                          bool lockDatabase = false,
+	                          LockDB lockDatabase = LockDB::FALSE,
 	                          PreBackupAction backupAction = PreBackupAction::VERIFY) {
 		return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
 			return submitBackup(
@@ -408,35 +443,36 @@ public:
 
 	Future<Void> abortBackup(Database cx,
 	                         Key tagName,
-	                         bool partial = false,
-	                         bool abortOldBackup = false,
-	                         bool dstOnly = false,
-	                         bool waitForDestUID = false);
+	                         PartialBackup = PartialBackup::FALSE,
+	                         AbortOldBackup = AbortOldBackup::FALSE,
+	                         DstOnly = DstOnly::FALSE,
+	                         WaitForDestUID = WaitForDestUID::FALSE);
 
 	Future<std::string> getStatus(Database cx, int errorLimit, Key tagName);
 
-	Future<EnumState> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
+	Future<EnumState> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, Snapshot = Snapshot::FALSE);
 	Future<EnumState> getStateValue(Database cx, UID logUid) {
 		return runRYWTransaction(cx,
 		                         [=](Reference<ReadYourWritesTransaction> tr) { return getStateValue(tr, logUid); });
 	}
 
-	Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
+	Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, Snapshot = Snapshot::FALSE);
 	Future<UID> getDestUid(Database cx, UID logUid) {
 		return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getDestUid(tr, logUid); });
 	}
 
-	Future<UID> getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot = false);
+	Future<UID> getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, Snapshot = Snapshot::FALSE);
 	Future<UID> getLogUid(Database cx, Key tagName) {
 		return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getLogUid(tr, tagName); });
 	}
 
-	Future<int64_t> getRangeBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
-	Future<int64_t> getLogBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
-
+	Future<int64_t> getRangeBytesWritten(Reference<ReadYourWritesTransaction> tr,
+	                                     UID logUid,
+	                                     Snapshot = Snapshot::FALSE);
+	Future<int64_t> getLogBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, Snapshot = Snapshot::FALSE);
 	// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
 	// will return when the backup directory is restorable.
-	Future<EnumState> waitBackup(Database cx, Key tagName, bool stopWhenDone = true);
+	Future<EnumState> waitBackup(Database cx, Key tagName, StopWhenDone = StopWhenDone::TRUE);
 	Future<EnumState> waitSubmitted(Database cx, Key tagName);
 	Future<Void> waitUpgradeToLatestDrVersion(Database cx, Key tagName);
 
@@ -494,7 +530,7 @@ Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
                           Key logUidValue,
                           Key destUidValue,
                           Optional<Version> endVersion = Optional<Version>(),
-                          bool checkBackupUid = false,
+                          CheckBackupUID = CheckBackupUID::FALSE,
                           Version backupUid = 0);
 Key getApplyKey(Version version, Key backupUid);
 Version getLogKeyVersion(Key key);
@@ -506,18 +542,18 @@ ACTOR Future<Void> readCommitted(Database cx,
                                  PromiseStream<RangeResultWithVersion> results,
                                  Reference<FlowLock> lock,
                                  KeyRangeRef range,
-                                 bool terminator = true,
-                                 bool systemAccess = false,
-                                 bool lockAware = false);
+                                 Terminator terminator = Terminator::TRUE,
+                                 AccessSystemKeys systemAccess = AccessSystemKeys::FALSE,
+                                 LockAware lockAware = LockAware::FALSE);
 ACTOR Future<Void> readCommitted(Database cx,
                                  PromiseStream<RCGroup> results,
                                  Future<Void> active,
                                  Reference<FlowLock> lock,
                                  KeyRangeRef range,
                                  std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
-                                 bool terminator = true,
-                                 bool systemAccess = false,
-                                 bool lockAware = false);
+                                 Terminator terminator = Terminator::TRUE,
+                                 AccessSystemKeys systemAccess = AccessSystemKeys::FALSE,
+                                 LockAware lockAware = LockAware::FALSE);
 ACTOR Future<Void> applyMutations(Database cx,
                                   Key uid,
                                   Key addPrefix,
@@ -527,7 +563,7 @@ ACTOR Future<Void> applyMutations(Database cx,
                                   RequestStream<CommitTransactionRequest> commit,
                                   NotifiedVersion* committedVersion,
                                   Reference<KeyRangeMap<Version>> keyVersion);
-ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData);
+ACTOR Future<Void> cleanupBackup(Database cx, DeleteData deleteData);
 
 using EBackupState = BackupAgentBase::EnumState;
 template <>
@@ -570,14 +606,15 @@ public:
 typedef KeyBackedMap<std::string, UidAndAbortedFlagT> TagMap;
 // Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace.
 class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
+	ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
+	                                                           Reference<ReadYourWritesTransaction> tr,
+	                                                           Snapshot snapshot);
+
 public:
 	TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
 
-	ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
-	                                                           Reference<ReadYourWritesTransaction> tr,
-	                                                           bool snapshot);
-
-	Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
+	Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr,
+	                                         Snapshot snapshot = Snapshot::FALSE) {
 		return getAll_impl(this, tr, snapshot);
 	}
 
@@ -593,12 +630,12 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) {
 }
 
 static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr,
-                                                                  bool snapshot = false) {
+                                                                  Snapshot snapshot = Snapshot::FALSE) {
 	return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot);
 }
 
 static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr,
-                                                                 bool snapshot = false) {
+                                                                 Snapshot snapshot = Snapshot::FALSE) {
 	return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot);
 }
 
@@ -613,7 +650,9 @@ public:
 
 	KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {}
 
-	Future<Void> toTask(Reference<ReadYourWritesTransaction> tr, Reference<Task> task, bool setValidation = true) {
+	Future<Void> toTask(Reference<ReadYourWritesTransaction> tr,
+	                    Reference<Task> task,
+	                    SetValidation setValidation = SetValidation::TRUE) {
 		// Set the uid task parameter
 		TaskParams.uid().set(task, uid);
 

fdbclient/BackupAgentBase.actor.cpp

@@ -26,6 +26,24 @@
 #include "flow/ActorCollection.h"
 #include "flow/actorcompiler.h" // has to be last include
 
+FDB_DEFINE_BOOLEAN_PARAM(LockDB);
+FDB_DEFINE_BOOLEAN_PARAM(UnlockDB);
+FDB_DEFINE_BOOLEAN_PARAM(StopWhenDone);
+FDB_DEFINE_BOOLEAN_PARAM(Verbose);
+FDB_DEFINE_BOOLEAN_PARAM(WaitForComplete);
+FDB_DEFINE_BOOLEAN_PARAM(ForceAction);
+FDB_DEFINE_BOOLEAN_PARAM(Terminator);
+FDB_DEFINE_BOOLEAN_PARAM(UsePartitionedLog);
+FDB_DEFINE_BOOLEAN_PARAM(InconsistentSnapshotOnly);
+FDB_DEFINE_BOOLEAN_PARAM(ShowErrors);
+FDB_DEFINE_BOOLEAN_PARAM(AbortOldBackup);
+FDB_DEFINE_BOOLEAN_PARAM(DstOnly);
+FDB_DEFINE_BOOLEAN_PARAM(WaitForDestUID);
+FDB_DEFINE_BOOLEAN_PARAM(CheckBackupUID);
+FDB_DEFINE_BOOLEAN_PARAM(DeleteData);
+FDB_DEFINE_BOOLEAN_PARAM(SetValidation);
+FDB_DEFINE_BOOLEAN_PARAM(PartialBackup);
+
 std::string BackupAgentBase::formatTime(int64_t epochs) {
 	time_t curTime = (time_t)epochs;
 	char buffer[30];
@@ -105,6 +123,7 @@ const Key BackupAgentBase::keyConfigBackupRanges = "config_backup_ranges"_sr;
 const Key BackupAgentBase::keyConfigStopWhenDoneKey = "config_stop_when_done"_sr;
 const Key BackupAgentBase::keyStateStop = "state_stop"_sr;
 const Key BackupAgentBase::keyStateStatus = "state_status"_sr;
+const Key BackupAgentBase::keyStateLogBeginVersion = "last_begin_version"_sr;
 const Key BackupAgentBase::keyLastUid = "last_uid"_sr;
 const Key BackupAgentBase::keyBeginKey = "beginKey"_sr;
 const Key BackupAgentBase::keyEndKey = "endKey"_sr;
@@ -374,9 +393,9 @@ ACTOR Future<Void> readCommitted(Database cx,
                                  PromiseStream<RangeResultWithVersion> results,
                                  Reference<FlowLock> lock,
                                  KeyRangeRef range,
-                                 bool terminator,
-                                 bool systemAccess,
-                                 bool lockAware) {
+                                 Terminator terminator,
+                                 AccessSystemKeys systemAccess,
+                                 LockAware lockAware) {
 	state KeySelector begin = firstGreaterOrEqual(range.begin);
 	state KeySelector end = firstGreaterOrEqual(range.end);
 	state Transaction tr(cx);
@@ -450,9 +469,9 @@ ACTOR Future<Void> readCommitted(Database cx,
                                  Reference<FlowLock> lock,
                                  KeyRangeRef range,
                                  std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
-                                 bool terminator,
-                                 bool systemAccess,
-                                 bool lockAware) {
+                                 Terminator terminator,
+                                 AccessSystemKeys systemAccess,
+                                 LockAware lockAware) {
 	state KeySelector nextKey = firstGreaterOrEqual(range.begin);
 	state KeySelector end = firstGreaterOrEqual(range.end);
 
@@ -559,7 +578,8 @@ Future<Void> readCommitted(Database cx,
                            Reference<FlowLock> lock,
                            KeyRangeRef range,
                            std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy) {
-	return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true);
+	return readCommitted(
+	    cx, results, Void(), lock, range, groupBy, Terminator::TRUE, AccessSystemKeys::TRUE, LockAware::TRUE);
 }
 
 ACTOR Future<int> dumpData(Database cx,
@@ -770,7 +790,7 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
                                         Key logUidValue,
                                         Key destUidValue,
                                         Optional<Version> endVersion,
-                                        bool checkBackupUid,
+                                        CheckBackupUID checkBackupUid,
                                         Version backupUid) {
 	state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
 	state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
@@ -898,7 +918,7 @@ Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
                           Key logUidValue,
                           Key destUidValue,
                           Optional<Version> endVersion,
-                          bool checkBackupUid,
+                          CheckBackupUID checkBackupUid,
                           Version backupUid) {
 	return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid);
 }
@@ -995,7 +1015,7 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
 	}
 }
 
-ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData) {
+ACTOR Future<Void> cleanupBackup(Database cx, DeleteData deleteData) {
 	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 	loop {
 		try {

fdbclient/BackupContainer.actor.cpp

@@ -58,6 +58,7 @@ ACTOR Future<Void> appendStringRefWithLen(Reference<IBackupFile> file, Standalon
 	wait(file->append(s.begin(), s.size()));
 	return Void();
 }
+
 } // namespace IBackupFile_impl
 
 Future<Void> IBackupFile::appendStringRefWithLen(Standalone<StringRef> s) {
@@ -253,7 +254,8 @@ std::vector<std::string> IBackupContainer::getURLFormats() {
 }
 
 // Get an IBackupContainer based on a container URL string
-Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& url) {
+Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& url,
+                                                            Optional<std::string> const& encryptionKeyFileName) {
 	static std::map<std::string, Reference<IBackupContainer>> m_cache;
 
 	Reference<IBackupContainer>& r = m_cache[url];
@@ -262,9 +264,9 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 
 	try {
 		StringRef u(url);
-		if (u.startsWith(LiteralStringRef("file://"))) {
-			r = Reference<IBackupContainer>(new BackupContainerLocalDirectory(url));
-		} else if (u.startsWith(LiteralStringRef("blobstore://"))) {
+		if (u.startsWith("file://"_sr)) {
+			r = makeReference<BackupContainerLocalDirectory>(url, encryptionKeyFileName);
+		} else if (u.startsWith("blobstore://"_sr)) {
 			std::string resource;
 
 			// The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options.
@@ -277,15 +279,16 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 			for (auto c : resource)
 				if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/')
 					throw backup_invalid_url();
-			r = Reference<IBackupContainer>(new BackupContainerS3BlobStore(bstore, resource, backupParams));
+			r = makeReference<BackupContainerS3BlobStore>(bstore, resource, backupParams, encryptionKeyFileName);
 		}
 #ifdef BUILD_AZURE_BACKUP
-		else if (u.startsWith(LiteralStringRef("azure://"))) {
-			u.eat(LiteralStringRef("azure://"));
-			auto address = NetworkAddress::parse(u.eat(LiteralStringRef("/")).toString());
-			auto containerName = u.eat(LiteralStringRef("/")).toString();
-			auto accountName = u.eat(LiteralStringRef("/")).toString();
-			r = Reference<IBackupContainer>(new BackupContainerAzureBlobStore(address, containerName, accountName));
+		else if (u.startsWith("azure://"_sr)) {
+			u.eat("azure://"_sr);
+			auto address = NetworkAddress::parse(u.eat("/"_sr).toString());
+			auto containerName = u.eat("/"_sr).toString();
+			auto accountName = u.eat("/"_sr).toString();
+			r = makeReference<BackupContainerAzureBlobStore>(
+			    address, containerName, accountName, encryptionKeyFileName);
 		}
 #endif
 		else {
@@ -315,10 +318,10 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL) {
 	try {
 		StringRef u(baseURL);
-		if (u.startsWith(LiteralStringRef("file://"))) {
+		if (u.startsWith("file://"_sr)) {
 			std::vector<std::string> results = wait(BackupContainerLocalDirectory::listURLs(baseURL));
 			return results;
-		} else if (u.startsWith(LiteralStringRef("blobstore://"))) {
+		} else if (u.startsWith("blobstore://"_sr)) {
 			std::string resource;
 
 			S3BlobStoreEndpoint::ParametersT backupParams;
@@ -333,14 +336,14 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL)
 			}
 
 			// Create a dummy container to parse the backup-specific parameters from the URL and get a final bucket name
-			BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams);
+			BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams, {});
 
 			std::vector<std::string> results = wait(BackupContainerS3BlobStore::listURLs(bstore, dummy.getBucket()));
 			return results;
 		}
 		// TODO: Enable this when Azure backups are ready
 		/*
-		else if (u.startsWith(LiteralStringRef("azure://"))) {
+		else if (u.startsWith("azure://"_sr)) {
 		    std::vector<std::string> results = wait(BackupContainerAzureBlobStore::listURLs(baseURL));
 		    return results;
 		}
@@ -386,7 +389,7 @@ ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Databa
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			state std::vector<std::pair<int64_t, Version>> results =
-			    wait(versionMap.getRange(tr, 0, time, 1, false, true));
+			    wait(versionMap.getRange(tr, 0, time, 1, Snapshot::FALSE, Reverse::TRUE));
 			if (results.size() != 1) {
 				// No key less than time was found in the database
 				// Look for a key >= time.
@@ -425,7 +428,7 @@ ACTOR Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version v, Reference
 
 		// Find the highest time < mid
 		state std::vector<std::pair<int64_t, Version>> results =
-		    wait(versionMap.getRange(tr, min, mid, 1, false, true));
+		    wait(versionMap.getRange(tr, min, mid, 1, Snapshot::FALSE, Reverse::TRUE));
 
 		if (results.size() != 1) {
 			if (mid == min) {

fdbclient/BackupContainer.h

@@ -293,7 +293,8 @@ public:
 	                                                          Version beginVersion = -1) = 0;
 
 	// Get an IBackupContainer based on a container spec string
-	static Reference<IBackupContainer> openContainer(const std::string& url);
+	static Reference<IBackupContainer> openContainer(const std::string& url,
+	                                                 const Optional<std::string>& encryptionKeyFileName = {});
 	static std::vector<std::string> getURLFormats();
 	static Future<std::vector<std::string>> listContainers(const std::string& baseURL);
 

fdbclient/BackupContainerAzureBlobStore.actor.cpp

@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/BackupContainerAzureBlobStore.h"
+#include "fdbrpc/AsyncFileEncrypted.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -167,8 +168,12 @@ public:
 		if (!exists) {
 			throw file_not_found();
 		}
-		return Reference<IAsyncFile>(
-		    new ReadFile(self->asyncTaskThread, self->containerName, fileName, self->client.get()));
+		Reference<IAsyncFile> f =
+		    makeReference<ReadFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
+		if (self->usesEncryption()) {
+			f = makeReference<AsyncFileEncrypted>(f, false);
+		}
+		return f;
 	}
 
 	ACTOR static Future<Reference<IBackupFile>> writeFile(BackupContainerAzureBlobStore* self, std::string fileName) {
@@ -177,10 +182,11 @@ public:
 			    auto outcome = client->create_append_blob(containerName, fileName).get();
 			    return Void();
 		    }));
-		return Reference<IBackupFile>(
-		    new BackupFile(fileName,
-		                   Reference<IAsyncFile>(new WriteFile(
-		                       self->asyncTaskThread, self->containerName, fileName, self->client.get()))));
+		auto f = makeReference<WriteFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
+		if (self->usesEncryption()) {
+			f = makeReference<AsyncFileEncrypted>(f, true);
+		}
+		return makeReference<BackupFile>(fileName, f);
 	}
 
 	static void listFiles(AzureClient* client,
@@ -213,6 +219,16 @@ public:
 		}
 		return Void();
 	}
+
+	ACTOR static Future<Void> create(BackupContainerAzureBlobStore* self) {
+		state Future<Void> f1 =
+		    self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client.get()] {
+			    client->create_container(containerName).wait();
+			    return Void();
+		    });
+		state Future<Void> f2 = self->usesEncryption() ? self->encryptionSetupComplete() : Void();
+		return f1 && f2;
+	}
 };
 
 Future<bool> BackupContainerAzureBlobStore::blobExists(const std::string& fileName) {
@@ -225,10 +241,11 @@ Future<bool> BackupContainerAzureBlobStore::blobExists(const std::string& fileNa
 
 BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const NetworkAddress& address,
                                                              const std::string& accountName,
-                                                             const std::string& containerName)
+                                                             const std::string& containerName,
+                                                             const Optional<std::string>& encryptionKeyFileName)
   : containerName(containerName) {
+	setEncryptionKey(encryptionKeyFileName);
 	std::string accountKey = std::getenv("AZURE_KEY");
-
 	auto credential = std::make_shared<azure::storage_lite::shared_key_credential>(accountName, accountKey);
 	auto storageAccount = std::make_shared<azure::storage_lite::storage_account>(
 	    accountName, credential, false, format("http://%s/%s", address.toString().c_str(), accountName.c_str()));
@@ -244,10 +261,7 @@ void BackupContainerAzureBlobStore::delref() {
 }
 
 Future<Void> BackupContainerAzureBlobStore::create() {
-	return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] {
-		client->create_container(containerName).wait();
-		return Void();
-	});
+	return BackupContainerAzureBlobStoreImpl::create(this);
 }
 Future<bool> BackupContainerAzureBlobStore::exists() {
 	return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] {

fdbclient/BackupContainerAzureBlobStore.h

@@ -44,7 +44,8 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem,
 public:
 	BackupContainerAzureBlobStore(const NetworkAddress& address,
 	                              const std::string& accountName,
-	                              const std::string& containerName);
+	                              const std::string& containerName,
+	                              const Optional<std::string>& encryptionKeyFileName);
 
 	void addref() override;
 	void delref() override;

fdbclient/BackupContainerFileSystem.actor.cpp

@@ -23,6 +23,7 @@
 #include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbclient/BackupContainerLocalDirectory.h"
 #include "fdbclient/JsonBuilder.h"
+#include "flow/StreamCipher.h"
 #include "flow/UnitTest.h"
 
 #include <algorithm>
@@ -290,13 +291,13 @@ public:
 
 		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
 		for (int i = 0; i < logs.size(); i++) {
-			ASSERT(logs[i].tagId >= 0);
-			ASSERT(logs[i].tagId < logs[i].totalTags);
+			ASSERT_GE(logs[i].tagId, 0);
+			ASSERT_LT(logs[i].tagId, logs[i].totalTags);
 			auto& indices = tagIndices[logs[i].tagId];
 			// filter out if indices.back() is subset of files[i] or vice versa
 			if (!indices.empty()) {
 				if (logs[indices.back()].isSubset(logs[i])) {
-					ASSERT(logs[indices.back()].fileSize <= logs[i].fileSize);
+					ASSERT_LE(logs[indices.back()].fileSize, logs[i].fileSize);
 					indices.back() = i;
 				} else if (!logs[i].isSubset(logs[indices.back()])) {
 					indices.push_back(i);
@@ -864,7 +865,7 @@ public:
 		int i = 0;
 		for (int j = 1; j < logs.size(); j++) {
 			if (logs[j].isSubset(logs[i])) {
-				ASSERT(logs[j].fileSize <= logs[i].fileSize);
+				ASSERT_LE(logs[j].fileSize, logs[i].fileSize);
 				continue;
 			}
 
@@ -1032,10 +1033,10 @@ public:
 	}
 
 	static std::string versionFolderString(Version v, int smallestBucket) {
-		ASSERT(smallestBucket < 14);
+		ASSERT_LT(smallestBucket, 14);
 		// Get a 0-padded fixed size representation of v
 		std::string vFixedPrecision = format("%019lld", v);
-		ASSERT(vFixedPrecision.size() == 19);
+		ASSERT_EQ(vFixedPrecision.size(), 19);
 		// Truncate smallestBucket from the fixed length representation
 		vFixedPrecision.resize(vFixedPrecision.size() - smallestBucket);
 
@@ -1126,6 +1127,42 @@ public:
 		return false;
 	}
 
+	ACTOR static Future<Void> createTestEncryptionKeyFile(std::string filename) {
+		state Reference<IAsyncFile> keyFile = wait(IAsyncFileSystem::filesystem()->open(
+		    filename,
+		    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE,
+		    0600));
+		StreamCipher::Key::RawKeyType testKey;
+		generateRandomData(testKey.data(), testKey.size());
+		keyFile->write(testKey.data(), testKey.size(), 0);
+		wait(keyFile->sync());
+		return Void();
+	}
+
+	ACTOR static Future<Void> readEncryptionKey(std::string encryptionKeyFileName) {
+		state Reference<IAsyncFile> keyFile;
+		state StreamCipher::Key::RawKeyType key;
+		try {
+			Reference<IAsyncFile> _keyFile =
+			    wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400));
+			keyFile = _keyFile;
+		} catch (Error& e) {
+			TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile")
+			    .detail("FileName", encryptionKeyFileName)
+			    .error(e);
+			throw e;
+		}
+		int bytesRead = wait(keyFile->read(key.data(), key.size(), 0));
+		if (bytesRead != key.size()) {
+			TraceEvent(SevWarnAlways, "InvalidEncryptionKeyFileSize")
+			    .detail("ExpectedSize", key.size())
+			    .detail("ActualSize", bytesRead);
+			throw invalid_encryption_key_file();
+		}
+		ASSERT_EQ(bytesRead, key.size());
+		StreamCipher::Key::initializeKey(std::move(key));
+		return Void();
+	}
 }; // class BackupContainerFileSystemImpl
 
 Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion,
@@ -1432,6 +1469,20 @@ BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::unreliable
 BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::logType() {
 	return { Reference<BackupContainerFileSystem>::addRef(this), "mutation_log_type" };
 }
+bool BackupContainerFileSystem::usesEncryption() const {
+	return encryptionSetupFuture.isValid();
+}
+Future<Void> BackupContainerFileSystem::encryptionSetupComplete() const {
+	return encryptionSetupFuture;
+}
+void BackupContainerFileSystem::setEncryptionKey(Optional<std::string> const& encryptionKeyFileName) {
+	if (encryptionKeyFileName.present()) {
+		encryptionSetupFuture = BackupContainerFileSystemImpl::readEncryptionKey(encryptionKeyFileName.get());
+	}
+}
+Future<Void> BackupContainerFileSystem::createTestEncryptionKeyFile(std::string const &filename) {
+	return BackupContainerFileSystemImpl::createTestEncryptionKeyFile(filename);
+}
 
 namespace backup_test {
 
@@ -1466,12 +1517,12 @@ ACTOR Future<Void> writeAndVerifyFile(Reference<IBackupContainer> c, Reference<I
 
 	state Reference<IAsyncFile> inputFile = wait(c->readFile(f->getFileName()));
 	int64_t fileSize = wait(inputFile->size());
-	ASSERT(size == fileSize);
+	ASSERT_EQ(size, fileSize);
 	if (size > 0) {
 		state Standalone<VectorRef<uint8_t>> buf;
 		buf.resize(buf.arena(), fileSize);
 		int b = wait(inputFile->read(buf.begin(), buf.size(), 0));
-		ASSERT(b == buf.size());
+		ASSERT_EQ(b, buf.size());
 		ASSERT(buf == content);
 	}
 	return Void();
@@ -1485,7 +1536,7 @@ Version nextVersion(Version v) {
 
 // Write a snapshot file with only begin & end key
 ACTOR static Future<Void> testWriteSnapshotFile(Reference<IBackupFile> file, Key begin, Key end, uint32_t blockSize) {
-	ASSERT(blockSize > 3 * sizeof(uint32_t) + begin.size() + end.size());
+	ASSERT_GT(blockSize, 3 * sizeof(uint32_t) + begin.size() + end.size());
 
 	uint32_t fileVersion = BACKUP_AGENT_SNAPSHOT_FILE_VERSION;
 	// write Header
@@ -1506,12 +1557,16 @@ ACTOR static Future<Void> testWriteSnapshotFile(Reference<IBackupFile> file, Key
 	return Void();
 }
 
-ACTOR static Future<Void> testBackupContainer(std::string url) {
+ACTOR Future<Void> testBackupContainer(std::string url, Optional<std::string> encryptionKeyFileName) {
 	state FlowLock lock(100e6);
 
+	if (encryptionKeyFileName.present()) {
+		wait(BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get()));
+	}
+
 	printf("BackupContainerTest URL %s\n", url.c_str());
 
-	state Reference<IBackupContainer> c = IBackupContainer::openContainer(url);
+	state Reference<IBackupContainer> c = IBackupContainer::openContainer(url, encryptionKeyFileName);
 
 	// Make sure container doesn't exist, then create it.
 	try {
@@ -1597,9 +1652,9 @@ ACTOR static Future<Void> testBackupContainer(std::string url) {
 	wait(waitForAll(writes));
 
 	state BackupFileList listing = wait(c->dumpFileList());
-	ASSERT(listing.ranges.size() == nRangeFiles);
-	ASSERT(listing.logs.size() == logs.size());
-	ASSERT(listing.snapshots.size() == snapshots.size());
+	ASSERT_EQ(listing.ranges.size(), nRangeFiles);
+	ASSERT_EQ(listing.logs.size(), logs.size());
+	ASSERT_EQ(listing.snapshots.size(), snapshots.size());
 
 	state BackupDescription desc = wait(c->describeBackup());
 	printf("\n%s\n", desc.toString().c_str());
@@ -1629,8 +1684,8 @@ ACTOR static Future<Void> testBackupContainer(std::string url) {
 
 		// If there is an error, it must be backup_cannot_expire and we have to be on the last snapshot
 		if (f.isError()) {
-			ASSERT(f.getError().code() == error_code_backup_cannot_expire);
-			ASSERT(i == listing.snapshots.size() - 1);
+			ASSERT_EQ(f.getError().code(), error_code_backup_cannot_expire);
+			ASSERT_EQ(i, listing.snapshots.size() - 1);
 			wait(c->expireData(expireVersion, true));
 		}
 
@@ -1646,31 +1701,34 @@ ACTOR static Future<Void> testBackupContainer(std::string url) {
 	ASSERT(d.isError() && d.getError().code() == error_code_backup_does_not_exist);
 
 	BackupFileList empty = wait(c->dumpFileList());
-	ASSERT(empty.ranges.size() == 0);
-	ASSERT(empty.logs.size() == 0);
-	ASSERT(empty.snapshots.size() == 0);
+	ASSERT_EQ(empty.ranges.size(), 0);
+	ASSERT_EQ(empty.logs.size(), 0);
+	ASSERT_EQ(empty.snapshots.size(), 0);
 
 	printf("BackupContainerTest URL=%s PASSED.\n", url.c_str());
 
 	return Void();
 }
 
-TEST_CASE("/backup/containers/localdir") {
-	if (g_network->isSimulated())
-		wait(testBackupContainer(format("file://simfdb/backups/%llx", timer_int())));
-	else
-		wait(testBackupContainer(format("file:///private/tmp/fdb_backups/%llx", timer_int())));
+TEST_CASE("/backup/containers/localdir/unencrypted") {
+	wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {}));
 	return Void();
-};
+}
+
+TEST_CASE("/backup/containers/localdir/encrypted") {
+	wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()),
+	                         format("%s/test_encryption_key", params.getDataDir().c_str())));
+	return Void();
+}
 
 TEST_CASE("/backup/containers/url") {
 	if (!g_network->isSimulated()) {
 		const char* url = getenv("FDB_TEST_BACKUP_URL");
 		ASSERT(url != nullptr);
-		wait(testBackupContainer(url));
+		wait(testBackupContainer(url, {}));
 	}
 	return Void();
-};
+}
 
 TEST_CASE("/backup/containers_list") {
 	if (!g_network->isSimulated()) {
@@ -1683,7 +1741,7 @@ TEST_CASE("/backup/containers_list") {
 		}
 	}
 	return Void();
-};
+}
 
 TEST_CASE("/backup/time") {
 	// test formatTime()

fdbclient/BackupContainerFileSystem.h

@@ -153,6 +153,13 @@ public:
 	                                                  bool logsOnly,
 	                                                  Version beginVersion) final;
 
+	static Future<Void> createTestEncryptionKeyFile(std::string const& filename);
+
+protected:
+	bool usesEncryption() const;
+	void setEncryptionKey(Optional<std::string> const& encryptionKeyFileName);
+	Future<Void> encryptionSetupComplete() const;
+
 private:
 	struct VersionProperty {
 		VersionProperty(Reference<BackupContainerFileSystem> bc, const std::string& name)
@@ -186,6 +193,8 @@ private:
 	Future<std::vector<RangeFile>> old_listRangeFiles(Version beginVersion, Version endVersion);
 
 	friend class BackupContainerFileSystemImpl;
+
+	Future<Void> encryptionSetupFuture;
 };
 
 #endif

fdbclient/BackupContainerLocalDirectory.actor.cpp

@@ -131,7 +131,10 @@ std::string BackupContainerLocalDirectory::getURLFormat() {
 	return "file://</path/to/base/dir/>";
 }
 
-BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url) {
+BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url,
+                                                             const Optional<std::string>& encryptionKeyFileName) {
+	setEncryptionKey(encryptionKeyFileName);
+
 	std::string path;
 	if (url.find("file://") != 0) {
 		TraceEvent(SevWarn, "BackupContainerLocalDirectory")
@@ -193,7 +196,10 @@ Future<std::vector<std::string>> BackupContainerLocalDirectory::listURLs(const s
 }
 
 Future<Void> BackupContainerLocalDirectory::create() {
-	// Nothing should be done here because create() can be called by any process working with the container URL,
+	if (usesEncryption()) {
+		return encryptionSetupComplete();
+	}
+	// No directory should be created here because create() can be called by any process working with the container URL,
 	// such as fdbbackup. Since "local directory" containers are by definition local to the machine they are
 	// accessed from, the container's creation (in this case the creation of a directory) must be ensured prior to
 	// every file creation, which is done in openFile(). Creating the directory here will result in unnecessary
@@ -207,6 +213,9 @@ Future<bool> BackupContainerLocalDirectory::exists() {
 
 Future<Reference<IAsyncFile>> BackupContainerLocalDirectory::readFile(const std::string& path) {
 	int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED;
+	if (usesEncryption()) {
+		flags |= IAsyncFile::OPEN_ENCRYPTED;
+	}
 	// Simulation does not properly handle opening the same file from multiple machines using a shared filesystem,
 	// so create a symbolic link to make each file opening appear to be unique.  This could also work in production
 	// but only if the source directory is writeable which shouldn't be required for a restore.
@@ -258,8 +267,11 @@ Future<Reference<IAsyncFile>> BackupContainerLocalDirectory::readFile(const std:
 }
 
 Future<Reference<IBackupFile>> BackupContainerLocalDirectory::writeFile(const std::string& path) {
-	int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE |
-	            IAsyncFile::OPEN_READWRITE;
+	int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE |
+	            IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE;
+	if (usesEncryption()) {
+		flags |= IAsyncFile::OPEN_ENCRYPTED;
+	}
 	std::string fullPath = joinPath(m_path, path);
 	platform::createDirectory(parentDirectory(fullPath));
 	std::string temp = fullPath + "." + deterministicRandom()->randomUniqueID().toString() + ".temp";

fdbclient/BackupContainerLocalDirectory.h

@@ -33,7 +33,7 @@ public:
 
 	static std::string getURLFormat();
 
-	BackupContainerLocalDirectory(const std::string& url);
+	BackupContainerLocalDirectory(const std::string& url, Optional<std::string> const& encryptionKeyFileName);
 
 	static Future<std::vector<std::string>> listURLs(const std::string& url);
 

fdbclient/BackupContainerS3BlobStore.actor.cpp

@@ -20,6 +20,7 @@
 
 #include "fdbclient/AsyncFileS3BlobStore.actor.h"
 #include "fdbclient/BackupContainerS3BlobStore.h"
+#include "fdbrpc/AsyncFileEncrypted.h"
 #include "fdbrpc/AsyncFileReadAhead.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -103,6 +104,10 @@ public:
 			wait(bc->m_bstore->writeEntireFile(bc->m_bucket, bc->indexEntry(), ""));
 		}
 
+		if (bc->usesEncryption()) {
+			wait(bc->encryptionSetupComplete());
+		}
+
 		return Void();
 	}
 
@@ -137,9 +142,10 @@ std::string BackupContainerS3BlobStore::indexEntry() {
 
 BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
                                                        const std::string& name,
-                                                       const S3BlobStoreEndpoint::ParametersT& params)
+                                                       const S3BlobStoreEndpoint::ParametersT& params,
+                                                       const Optional<std::string>& encryptionKeyFileName)
   : m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") {
-
+	setEncryptionKey(encryptionKeyFileName);
 	// Currently only one parameter is supported, "bucket"
 	for (const auto& [name, value] : params) {
 		if (name == "bucket") {
@@ -164,12 +170,16 @@ std::string BackupContainerS3BlobStore::getURLFormat() {
 }
 
 Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::string& path) {
-	return Reference<IAsyncFile>(new AsyncFileReadAheadCache(
-	    Reference<IAsyncFile>(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))),
-	    m_bstore->knobs.read_block_size,
-	    m_bstore->knobs.read_ahead_blocks,
-	    m_bstore->knobs.concurrent_reads_per_file,
-	    m_bstore->knobs.read_cache_blocks_per_file));
+	Reference<IAsyncFile> f = makeReference<AsyncFileS3BlobStoreRead>(m_bstore, m_bucket, dataPath(path));
+	if (usesEncryption()) {
+		f = makeReference<AsyncFileEncrypted>(f, AsyncFileEncrypted::Mode::READ_ONLY);
+	}
+	f = makeReference<AsyncFileReadAheadCache>(f,
+	                                           m_bstore->knobs.read_block_size,
+	                                           m_bstore->knobs.read_ahead_blocks,
+	                                           m_bstore->knobs.concurrent_reads_per_file,
+	                                           m_bstore->knobs.read_cache_blocks_per_file);
+	return f;
 }
 
 Future<std::vector<std::string>> BackupContainerS3BlobStore::listURLs(Reference<S3BlobStoreEndpoint> bstore,
@@ -178,8 +188,11 @@ Future<std::vector<std::string>> BackupContainerS3BlobStore::listURLs(Reference<
 }
 
 Future<Reference<IBackupFile>> BackupContainerS3BlobStore::writeFile(const std::string& path) {
-	return Reference<IBackupFile>(new BackupContainerS3BlobStoreImpl::BackupFile(
-	    path, Reference<IAsyncFile>(new AsyncFileS3BlobStoreWrite(m_bstore, m_bucket, dataPath(path)))));
+	Reference<IAsyncFile> f = makeReference<AsyncFileS3BlobStoreWrite>(m_bstore, m_bucket, dataPath(path));
+	if (usesEncryption()) {
+		f = makeReference<AsyncFileEncrypted>(f, AsyncFileEncrypted::Mode::APPEND_ONLY);
+	}
+	return Future<Reference<IBackupFile>>(makeReference<BackupContainerS3BlobStoreImpl::BackupFile>(path, f));
 }
 
 Future<Void> BackupContainerS3BlobStore::deleteFile(const std::string& path) {

fdbclient/BackupContainerS3BlobStore.h

@@ -43,7 +43,8 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem,
 public:
 	BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
 	                           const std::string& name,
-	                           const S3BlobStoreEndpoint::ParametersT& params);
+	                           const S3BlobStoreEndpoint::ParametersT& params,
+	                           const Optional<std::string>& encryptionKeyFileName);
 
 	void addref() override;
 	void delref() override;

fdbclient/CMakeLists.txt

@@ -15,6 +15,8 @@ set(FDBCLIENT_SRCS
   BackupContainerLocalDirectory.h
   BackupContainerS3BlobStore.actor.cpp
   BackupContainerS3BlobStore.h
+  ClientBooleanParams.cpp
+  ClientBooleanParams.h
   ClientKnobCollection.cpp
   ClientKnobCollection.h
   ClientKnobs.cpp

fdbclient/ClientBooleanParams.cpp

@@ -0,0 +1,30 @@
+/*
+ * ClientBooleanParams.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ClientBooleanParams.h"
+
+FDB_DEFINE_BOOLEAN_PARAM(EnableLocalityLoadBalance);
+FDB_DEFINE_BOOLEAN_PARAM(LockAware);
+FDB_DEFINE_BOOLEAN_PARAM(Reverse);
+FDB_DEFINE_BOOLEAN_PARAM(Snapshot);
+FDB_DEFINE_BOOLEAN_PARAM(IsInternal);
+FDB_DEFINE_BOOLEAN_PARAM(AddConflictRange);
+FDB_DEFINE_BOOLEAN_PARAM(UseMetrics);
+FDB_DEFINE_BOOLEAN_PARAM(IsSwitchable);

fdbclient/ClientBooleanParams.h

@@ -0,0 +1,32 @@
+/*
+ * ClientBooleanParams.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "flow/BooleanParam.h"
+
+FDB_DECLARE_BOOLEAN_PARAM(EnableLocalityLoadBalance);
+FDB_DECLARE_BOOLEAN_PARAM(LockAware);
+FDB_DECLARE_BOOLEAN_PARAM(Reverse);
+FDB_DECLARE_BOOLEAN_PARAM(Snapshot);
+FDB_DECLARE_BOOLEAN_PARAM(IsInternal);
+FDB_DECLARE_BOOLEAN_PARAM(AddConflictRange);
+FDB_DECLARE_BOOLEAN_PARAM(UseMetrics);
+FDB_DECLARE_BOOLEAN_PARAM(IsSwitchable);

fdbclient/ClientKnobs.cpp

@@ -29,8 +29,7 @@ ClientKnobs::ClientKnobs(Randomize randomize) {
 	initialize(randomize);
 }
 
-void ClientKnobs::initialize(Randomize _randomize) {
-	bool const randomize = (_randomize == Randomize::YES);
+void ClientKnobs::initialize(Randomize randomize) {
 	// clang-format off
 
 	init( TOO_MANY,                            1000000 );
@@ -253,13 +252,13 @@ void ClientKnobs::initialize(Randomize _randomize) {
 
 TEST_CASE("/fdbclient/knobs/initialize") {
 	// This test depends on TASKBUCKET_TIMEOUT_VERSIONS being defined as a constant multiple of CORE_VERSIONSPERSECOND
-	ClientKnobs clientKnobs(Randomize::NO);
+	ClientKnobs clientKnobs(Randomize::FALSE);
 	int64_t initialCoreVersionsPerSecond = clientKnobs.CORE_VERSIONSPERSECOND;
 	int initialTaskBucketTimeoutVersions = clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS;
 	clientKnobs.setKnob("core_versionspersecond", initialCoreVersionsPerSecond * 2);
 	ASSERT_EQ(clientKnobs.CORE_VERSIONSPERSECOND, initialCoreVersionsPerSecond * 2);
 	ASSERT_EQ(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS, initialTaskBucketTimeoutVersions);
-	clientKnobs.initialize(Randomize::NO);
+	clientKnobs.initialize(Randomize::FALSE);
 	ASSERT_EQ(clientKnobs.CORE_VERSIONSPERSECOND, initialCoreVersionsPerSecond * 2);
 	ASSERT_EQ(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS, initialTaskBucketTimeoutVersions * 2);
 	return Void();

fdbclient/ClientKnobs.h

@@ -22,9 +22,13 @@
 #define FDBCLIENT_KNOBS_H
 #pragma once
 
+#include "flow/BooleanParam.h"
 #include "flow/Knobs.h"
 #include "flow/flow.h"
 
+FDB_DECLARE_BOOLEAN_PARAM(Randomize);
+FDB_DECLARE_BOOLEAN_PARAM(IsSimulated);
+
 class ClientKnobs : public KnobsImpl<ClientKnobs> {
 public:
 	int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically

fdbclient/CommitProxyInterface.h

@@ -64,6 +64,7 @@ struct CommitProxyInterface {
 	bool operator==(CommitProxyInterface const& r) const { return id() == r.id(); }
 	bool operator!=(CommitProxyInterface const& r) const { return id() != r.id(); }
 	NetworkAddress address() const { return commit.getEndpoint().getPrimaryAddress(); }
+	NetworkAddressList addresses() const { return commit.getEndpoint().addresses; }
 
 	template <class Archive>
 	void serialize(Archive& ar) {

fdbclient/DatabaseBackupAgent.actor.cpp

@@ -47,8 +47,11 @@ DatabaseBackupAgent::DatabaseBackupAgent()
   : subspace(Subspace(databaseBackupPrefixRange.begin)), tagNames(subspace.get(BackupAgentBase::keyTagName)),
     states(subspace.get(BackupAgentBase::keyStates)), config(subspace.get(BackupAgentBase::keyConfig)),
     errors(subspace.get(BackupAgentBase::keyErrors)), ranges(subspace.get(BackupAgentBase::keyRanges)),
-    taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), true, false, true)),
-    futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), true, true)),
+    taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks),
+                              AccessSystemKeys::TRUE,
+                              PriorityBatch::FALSE,
+                              LockAware::TRUE)),
+    futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::TRUE, LockAware::TRUE)),
     sourceStates(subspace.get(BackupAgentBase::keySourceStates)),
     sourceTagNames(subspace.get(BackupAgentBase::keyTagName)) {}
 
@@ -56,8 +59,11 @@ DatabaseBackupAgent::DatabaseBackupAgent(Database src)
   : subspace(Subspace(databaseBackupPrefixRange.begin)), tagNames(subspace.get(BackupAgentBase::keyTagName)),
     states(subspace.get(BackupAgentBase::keyStates)), config(subspace.get(BackupAgentBase::keyConfig)),
     errors(subspace.get(BackupAgentBase::keyErrors)), ranges(subspace.get(BackupAgentBase::keyRanges)),
-    taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), true, false, true)),
-    futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), true, true)),
+    taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks),
+                              AccessSystemKeys::TRUE,
+                              PriorityBatch::FALSE,
+                              LockAware::TRUE)),
+    futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::TRUE, LockAware::TRUE)),
     sourceStates(subspace.get(BackupAgentBase::keySourceStates)),
     sourceTagNames(subspace.get(BackupAgentBase::keyTagName)) {
 	taskBucket->src = src;
@@ -234,7 +240,8 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 		// retrieve kvData
 		state PromiseStream<RangeResultWithVersion> results;
 
-		state Future<Void> rc = readCommitted(taskBucket->src, results, lock, range, true, true, true);
+		state Future<Void> rc = readCommitted(
+		    taskBucket->src, results, lock, range, Terminator::TRUE, AccessSystemKeys::TRUE, LockAware::TRUE);
 		state Key rangeBegin = range.begin;
 		state Key rangeEnd;
 		state bool endOfStream = false;
@@ -316,16 +323,20 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 					    applyMutationsKeyVersionCountRange.begin);
 					state Future<RangeResult> backupVersions =
 					    krmGetRanges(tr, prefix, KeyRangeRef(rangeBegin, rangeEnd), BUGGIFY ? 2 : 2000, 1e5);
-					state Future<Optional<Value>> logVersionValue = tr->get(
-					    task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin), true);
-					state Future<Optional<Value>> rangeCountValue = tr->get(rangeCountKey, true);
-					state Future<RangeResult> prevRange = tr->getRange(
-					    firstGreaterOrEqual(prefix), lastLessOrEqual(rangeBegin.withPrefix(prefix)), 1, true, true);
+					state Future<Optional<Value>> logVersionValue =
+					    tr->get(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin),
+					            Snapshot::TRUE);
+					state Future<Optional<Value>> rangeCountValue = tr->get(rangeCountKey, Snapshot::TRUE);
+					state Future<RangeResult> prevRange = tr->getRange(firstGreaterOrEqual(prefix),
+					                                                   lastLessOrEqual(rangeBegin.withPrefix(prefix)),
+					                                                   1,
+					                                                   Snapshot::TRUE,
+					                                                   Reverse::TRUE);
 					state Future<RangeResult> nextRange = tr->getRange(firstGreaterOrEqual(rangeEnd.withPrefix(prefix)),
 					                                                   firstGreaterOrEqual(strinc(prefix)),
 					                                                   1,
-					                                                   true,
-					                                                   false);
+					                                                   Snapshot::TRUE,
+					                                                   Reverse::FALSE);
 					state Future<Void> verified = taskBucket->keepRunning(tr, task);
 
 					wait(checkDatabaseLock(tr,
@@ -363,7 +374,7 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 					Version logVersion =
 					    logVersionValue.get().present()
 					        ? BinaryReader::fromStringRef<Version>(logVersionValue.get().get(), Unversioned())
-					        : -1;
+					        : ::invalidVersion;
 					if (logVersion >= values.second) {
 						task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey] = rangeBegin;
 						return Void();
@@ -633,7 +644,7 @@ struct EraseLogRangeTaskFunc : TaskFuncBase {
 				    task->params[BackupAgentBase::keyConfigLogUid],
 				    task->params[BackupAgentBase::destUid],
 				    Optional<Version>(endVersion),
-				    true,
+				    CheckBackupUID::TRUE,
 				    BinaryReader::fromStringRef<Version>(task->params[BackupAgentBase::keyFolderId], Unversioned())));
 				wait(tr->commit());
 				return Void();
@@ -886,9 +897,9 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 				                           locks[j],
 				                           ranges[j],
 				                           decodeBKMutationLogKey,
-				                           true,
-				                           true,
-				                           true));
+				                           Terminator::TRUE,
+				                           AccessSystemKeys::TRUE,
+				                           LockAware::TRUE));
 			}
 
 			// copy the range
@@ -1191,7 +1202,7 @@ struct FinishedFullBackupTaskFunc : TaskFuncBase {
 				                           task->params[DatabaseBackupAgent::keyFolderId], Unversioned()))
 					return Void();
 
-				wait(eraseLogData(tr, logUidValue, destUidValue, Optional<Version>(), true, backupUid));
+				wait(eraseLogData(tr, logUidValue, destUidValue, Optional<Version>(), CheckBackupUID::TRUE, backupUid));
 				wait(tr->commit());
 				return Void();
 			} catch (Error& e) {
@@ -1321,6 +1332,10 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase {
 			    .detail("LogUID", task->params[BackupAgentBase::keyConfigLogUid]);
 		}
 
+		// set the log version to the state
+		tr->set(StringRef(states.pack(DatabaseBackupAgent::keyStateLogBeginVersion)),
+		        BinaryWriter::toValue(beginVersion, Unversioned()));
+
 		if (!stopWhenDone.present()) {
 			state Reference<TaskFuture> allPartsDone = futureBucket->future(tr);
 			std::vector<Future<Key>> addTaskVector;
@@ -1592,9 +1607,9 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 			                           lock,
 			                           ranges[i],
 			                           decodeBKMutationLogKey,
-			                           true,
-			                           true,
-			                           true));
+			                           Terminator::TRUE,
+			                           AccessSystemKeys::TRUE,
+			                           LockAware::TRUE));
 			dump.push_back(dumpData(cx, task, results[i], lock.getPtr(), taskBucket));
 		}
 
@@ -1701,7 +1716,7 @@ struct AbortOldBackupTaskFunc : TaskFuncBase {
 		}
 
 		TraceEvent("DBA_AbortOldBackup").detail("TagName", tagNameKey.printable());
-		wait(srcDrAgent.abortBackup(cx, tagNameKey, false, true));
+		wait(srcDrAgent.abortBackup(cx, tagNameKey, PartialBackup::FALSE, AbortOldBackup::TRUE));
 
 		return Void();
 	}
@@ -2445,7 +2460,7 @@ public:
 	ACTOR static Future<EBackupState> waitBackup(DatabaseBackupAgent* backupAgent,
 	                                             Database cx,
 	                                             Key tagName,
-	                                             bool stopWhenDone) {
+	                                             StopWhenDone stopWhenDone) {
 		state std::string backTrace;
 		state UID logUid = wait(backupAgent->getLogUid(cx, tagName));
 		state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
@@ -2510,10 +2525,10 @@ public:
 	                                       Reference<ReadYourWritesTransaction> tr,
 	                                       Key tagName,
 	                                       Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                       bool stopWhenDone,
+	                                       StopWhenDone stopWhenDone,
 	                                       Key addPrefix,
 	                                       Key removePrefix,
-	                                       bool lockDB,
+	                                       LockDB lockDB,
 	                                       DatabaseBackupAgent::PreBackupAction backupAction) {
 		state UID logUid = deterministicRandom()->randomUniqueID();
 		state Key logUidValue = BinaryWriter::toValue(logUid, Unversioned());
@@ -2667,7 +2682,7 @@ public:
 	                                           Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                                           Key addPrefix,
 	                                           Key removePrefix,
-	                                           bool forceAction) {
+	                                           ForceAction forceAction) {
 		state DatabaseBackupAgent drAgent(dest);
 		state UID destlogUid = wait(backupAgent->getLogUid(dest, tagName));
 		state EBackupState status = wait(backupAgent->getStateValue(dest, destlogUid));
@@ -2751,7 +2766,7 @@ public:
 				throw;
 		}
 
-		wait(success(backupAgent->waitBackup(dest, tagName, true)));
+		wait(success(backupAgent->waitBackup(dest, tagName, StopWhenDone::TRUE)));
 
 		TraceEvent("DBA_SwitchoverStopped");
 
@@ -2780,10 +2795,10 @@ public:
 			wait(drAgent.submitBackup(backupAgent->taskBucket->src,
 			                          tagName,
 			                          backupRanges,
-			                          false,
+			                          StopWhenDone::FALSE,
 			                          addPrefix,
 			                          removePrefix,
-			                          true,
+			                          LockDB::TRUE,
 			                          DatabaseBackupAgent::PreBackupAction::NONE));
 		} catch (Error& e) {
 			if (e.code() != error_code_backup_duplicate)
@@ -2835,10 +2850,10 @@ public:
 	ACTOR static Future<Void> abortBackup(DatabaseBackupAgent* backupAgent,
 	                                      Database cx,
 	                                      Key tagName,
-	                                      bool partial,
-	                                      bool abortOldBackup,
-	                                      bool dstOnly,
-	                                      bool waitForDestUID) {
+	                                      PartialBackup partial,
+	                                      AbortOldBackup abortOldBackup,
+	                                      DstOnly dstOnly,
+	                                      WaitForDestUID waitForDestUID) {
 		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 		state Key logUidValue, destUidValue;
 		state UID logUid, destUid;
@@ -3063,8 +3078,8 @@ public:
 				    errorLimit > 0
 				        ? tr->getRange(backupAgent->errors.get(BinaryWriter::toValue(logUid, Unversioned())).range(),
 				                       errorLimit,
-				                       false,
-				                       true)
+				                       Snapshot::FALSE,
+				                       Reverse::TRUE)
 				        : Future<RangeResult>();
 				state Future<Optional<Value>> fBackupUid =
 				    tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
@@ -3080,6 +3095,9 @@ public:
 				state Future<Optional<Key>> fBackupKeysPacked =
 				    tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned()))
 				                .pack(BackupAgentBase::keyConfigBackupRanges));
+				state Future<Optional<Value>> flogVersionKey = 
+				    tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
+				                .pack(BackupAgentBase::keyStateLogBeginVersion));
 
 				state EBackupState backupState = wait(backupAgent->getStateValue(tr, logUid));
 
@@ -3095,7 +3113,14 @@ public:
 					}
 
 					state Optional<Value> stopVersionKey = wait(fStopVersionKey);
-
+					Optional<Value> logVersionKey = wait(flogVersionKey);
+					state std::string logVersionText
+					    = ". Last log version is " 
+					      + (
+						logVersionKey.present()
+					        ? format("%lld", BinaryReader::fromStringRef<Version>(logVersionKey.get(), Unversioned()))
+					        : "unset"
+					      );
 					Optional<Key> backupKeysPacked = wait(fBackupKeysPacked);
 
 					state Standalone<VectorRef<KeyRangeRef>> backupRanges;
@@ -3115,7 +3140,7 @@ public:
 						break;
 					case EBackupState::STATE_RUNNING_DIFFERENTIAL:
 						statusText +=
-						    "The DR on tag `" + tagNameDisplay + "' is a complete copy of the primary database.\n";
+						    "The DR on tag `" + tagNameDisplay + "' is a complete copy of the primary database" + logVersionText + ".\n";
 						break;
 					case EBackupState::STATE_COMPLETED: {
 						Version stopVersion =
@@ -3127,13 +3152,13 @@ public:
 					} break;
 					case EBackupState::STATE_PARTIALLY_ABORTED: {
 						statusText += "The previous DR on tag `" + tagNameDisplay + "' " +
-						              BackupAgentBase::getStateText(backupState) + ".\n";
+						              BackupAgentBase::getStateText(backupState) + logVersionText + ".\n";
 						statusText += "Abort the DR with --cleanup before starting a new DR.\n";
 						break;
 					}
 					default:
 						statusText += "The previous DR on tag `" + tagNameDisplay + "' " +
-						              BackupAgentBase::getStateText(backupState) + ".\n";
+						              BackupAgentBase::getStateText(backupState) + logVersionText + ".\n";
 						break;
 					}
 				}
@@ -3191,7 +3216,7 @@ public:
 	ACTOR static Future<EBackupState> getStateValue(DatabaseBackupAgent* backupAgent,
 	                                                Reference<ReadYourWritesTransaction> tr,
 	                                                UID logUid,
-	                                                bool snapshot) {
+	                                                Snapshot snapshot) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
@@ -3204,7 +3229,7 @@ public:
 	ACTOR static Future<UID> getDestUid(DatabaseBackupAgent* backupAgent,
 	                                    Reference<ReadYourWritesTransaction> tr,
 	                                    UID logUid,
-	                                    bool snapshot) {
+	                                    Snapshot snapshot) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		state Key destUidKey =
@@ -3217,7 +3242,7 @@ public:
 	ACTOR static Future<UID> getLogUid(DatabaseBackupAgent* backupAgent,
 	                                   Reference<ReadYourWritesTransaction> tr,
 	                                   Key tagName,
-	                                   bool snapshot) {
+	                                   Snapshot snapshot) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		state Optional<Value> logUid = wait(tr->get(backupAgent->tagNames.pack(tagName), snapshot));
@@ -3235,7 +3260,7 @@ Future<Void> DatabaseBackupAgent::atomicSwitchover(Database dest,
                                                    Standalone<VectorRef<KeyRangeRef>> backupRanges,
                                                    Key addPrefix,
                                                    Key removePrefix,
-                                                   bool forceAction) {
+                                                   ForceAction forceAction) {
 	return DatabaseBackupAgentImpl::atomicSwitchover(
 	    this, dest, tagName, backupRanges, addPrefix, removePrefix, forceAction);
 }
@@ -3243,10 +3268,10 @@ Future<Void> DatabaseBackupAgent::atomicSwitchover(Database dest,
 Future<Void> DatabaseBackupAgent::submitBackup(Reference<ReadYourWritesTransaction> tr,
                                                Key tagName,
                                                Standalone<VectorRef<KeyRangeRef>> backupRanges,
-                                               bool stopWhenDone,
+                                               StopWhenDone stopWhenDone,
                                                Key addPrefix,
                                                Key removePrefix,
-                                               bool lockDatabase,
+                                               LockDB lockDatabase,
                                                PreBackupAction backupAction) {
 	return DatabaseBackupAgentImpl::submitBackup(
 	    this, tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, backupAction);
@@ -3258,10 +3283,10 @@ Future<Void> DatabaseBackupAgent::discontinueBackup(Reference<ReadYourWritesTran
 
 Future<Void> DatabaseBackupAgent::abortBackup(Database cx,
                                               Key tagName,
-                                              bool partial,
-                                              bool abortOldBackup,
-                                              bool dstOnly,
-                                              bool waitForDestUID) {
+                                              PartialBackup partial,
+                                              AbortOldBackup abortOldBackup,
+                                              DstOnly dstOnly,
+                                              WaitForDestUID waitForDestUID) {
 	return DatabaseBackupAgentImpl::abortBackup(this, cx, tagName, partial, abortOldBackup, dstOnly, waitForDestUID);
 }
 
@@ -3271,15 +3296,15 @@ Future<std::string> DatabaseBackupAgent::getStatus(Database cx, int errorLimit,
 
 Future<EBackupState> DatabaseBackupAgent::getStateValue(Reference<ReadYourWritesTransaction> tr,
                                                         UID logUid,
-                                                        bool snapshot) {
+                                                        Snapshot snapshot) {
 	return DatabaseBackupAgentImpl::getStateValue(this, tr, logUid, snapshot);
 }
 
-Future<UID> DatabaseBackupAgent::getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot) {
+Future<UID> DatabaseBackupAgent::getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, Snapshot snapshot) {
 	return DatabaseBackupAgentImpl::getDestUid(this, tr, logUid, snapshot);
 }
 
-Future<UID> DatabaseBackupAgent::getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot) {
+Future<UID> DatabaseBackupAgent::getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, Snapshot snapshot) {
 	return DatabaseBackupAgentImpl::getLogUid(this, tr, tagName, snapshot);
 }
 
@@ -3287,7 +3312,7 @@ Future<Void> DatabaseBackupAgent::waitUpgradeToLatestDrVersion(Database cx, Key
 	return DatabaseBackupAgentImpl::waitUpgradeToLatestDrVersion(this, cx, tagName);
 }
 
-Future<EBackupState> DatabaseBackupAgent::waitBackup(Database cx, Key tagName, bool stopWhenDone) {
+Future<EBackupState> DatabaseBackupAgent::waitBackup(Database cx, Key tagName, StopWhenDone stopWhenDone) {
 	return DatabaseBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone);
 }
 
@@ -3297,12 +3322,12 @@ Future<EBackupState> DatabaseBackupAgent::waitSubmitted(Database cx, Key tagName
 
 Future<int64_t> DatabaseBackupAgent::getRangeBytesWritten(Reference<ReadYourWritesTransaction> tr,
                                                           UID logUid,
-                                                          bool snapshot) {
+                                                          Snapshot snapshot) {
 	return DRConfig(logUid).rangeBytesWritten().getD(tr, snapshot);
 }
 
 Future<int64_t> DatabaseBackupAgent::getLogBytesWritten(Reference<ReadYourWritesTransaction> tr,
                                                         UID logUid,
-                                                        bool snapshot) {
+                                                        Snapshot snapshot) {
 	return DRConfig(logUid).logBytesWritten().getD(tr, snapshot);
 }

fdbclient/DatabaseContext.h

@@ -157,11 +157,11 @@ public:
 	static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo,
 	                       Future<Void> clientInfoMonitor,
 	                       LocalityData clientLocality,
-	                       bool enableLocalityLoadBalance,
+	                       EnableLocalityLoadBalance,
 	                       TaskPriority taskID = TaskPriority::DefaultEndpoint,
-	                       bool lockAware = false,
+	                       LockAware = LockAware::FALSE,
 	                       int apiVersion = Database::API_VERSION_LATEST,
-	                       bool switchable = false);
+	                       IsSwitchable = IsSwitchable::FALSE);
 
 	~DatabaseContext();
 
@@ -180,13 +180,13 @@ public:
 		                                    switchable));
 	}
 
-	std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation(const KeyRef&, bool isBackward = false);
+	std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation(const KeyRef&, Reverse isBackward = Reverse::FALSE);
 	bool getCachedLocations(const KeyRangeRef&,
 	                        vector<std::pair<KeyRange, Reference<LocationInfo>>>&,
 	                        int limit,
-	                        bool reverse);
+	                        Reverse reverse);
 	Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const vector<struct StorageServerInterface>&);
-	void invalidateCache(const KeyRef&, bool isBackward = false);
+	void invalidateCache(const KeyRef&, Reverse isBackward = Reverse::FALSE);
 	void invalidateCache(const KeyRangeRef&);
 
 	bool sampleReadTags() const;
@@ -217,7 +217,7 @@ public:
 	void setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value);
 
 	Error deferredError;
-	bool lockAware;
+	LockAware lockAware{ LockAware::FALSE };
 
 	bool isError() const { return deferredError.code() != invalid_error_code; }
 
@@ -242,7 +242,7 @@ public:
 	// new cluster.
 	Future<Void> switchConnectionFile(Reference<ClusterConnectionFile> standby);
 	Future<Void> connectionFileChanged();
-	bool switchable = false;
+	IsSwitchable switchable{ false };
 
 	// Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
 	Future<int64_t> rebootWorker(StringRef address, bool check = false, int duration = 0);
@@ -259,11 +259,11 @@ public:
 	                         Future<Void> clientInfoMonitor,
 	                         TaskPriority taskID,
 	                         LocalityData const& clientLocality,
-	                         bool enableLocalityLoadBalance,
-	                         bool lockAware,
-	                         bool internal = true,
+	                         EnableLocalityLoadBalance,
+	                         LockAware,
+	                         IsInternal = IsInternal::TRUE,
 	                         int apiVersion = Database::API_VERSION_LATEST,
-	                         bool switchable = false);
+	                         IsSwitchable = IsSwitchable::FALSE);
 
 	explicit DatabaseContext(const Error& err);
 
@@ -282,7 +282,7 @@ public:
 	UID proxiesLastChange;
 	LocalityData clientLocality;
 	QueueModel queueModel;
-	bool enableLocalityLoadBalance;
+	EnableLocalityLoadBalance enableLocalityLoadBalance{ EnableLocalityLoadBalance::FALSE };
 
 	struct VersionRequest {
 		SpanID spanContext;
@@ -329,7 +329,7 @@ public:
 	std::unordered_map<UID, Reference<TSSMetrics>> tssMetrics;
 
 	UID dbId;
-	bool internal; // Only contexts created through the C client and fdbcli are non-internal
+	IsInternal internal; // Only contexts created through the C client and fdbcli are non-internal
 
 	PrioritizedTransactionTagMap<ClientTagThrottleData> throttledTags;
 

fdbclient/FileBackupAgent.actor.cpp

@@ -42,6 +42,9 @@
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+FDB_DEFINE_BOOLEAN_PARAM(IncrementalBackupOnly);
+FDB_DEFINE_BOOLEAN_PARAM(OnlyApplyMutationLogs);
+
 #define SevFRTestInfo SevVerbose
 //#define SevFRTestInfo SevInfo
 
@@ -117,7 +120,7 @@ Key FileBackupAgent::getPauseKey() {
 
 ACTOR Future<std::vector<KeyBackedTag>> TagUidMap::getAll_impl(TagUidMap* tagsMap,
                                                                Reference<ReadYourWritesTransaction> tr,
-                                                               bool snapshot) {
+                                                               Snapshot snapshot) {
 	state Key prefix = tagsMap->prefix; // Copying it here as tagsMap lifetime is not tied to this actor
 	TagMap::PairsType tagPairs = wait(tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot));
 	std::vector<KeyBackedTag> results;
@@ -142,7 +145,7 @@ public:
 	}
 	KeyBackedProperty<Key> addPrefix() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
 	KeyBackedProperty<Key> removePrefix() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
-	KeyBackedProperty<bool> onlyAppyMutationLogs() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
+	KeyBackedProperty<bool> onlyApplyMutationLogs() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
 	KeyBackedProperty<bool> inconsistentSnapshotOnly() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
 	// XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges
 	KeyBackedProperty<KeyRange> restoreRange() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
@@ -248,9 +251,9 @@ public:
 	Key applyMutationsMapPrefix() { return uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid); }
 
 	ACTOR static Future<int64_t> getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid) {
-		// Both of these are snapshot reads
-		state Future<Optional<Value>> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), true);
-		state Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), true);
+		state Future<Optional<Value>> beginVal =
+		    tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::TRUE);
+		state Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::TRUE);
 		wait(success(beginVal) && success(endVal));
 
 		if (!beginVal.get().present() || !endVal.get().present())
@@ -440,8 +443,12 @@ FileBackupAgent::FileBackupAgent()
     // The other subspaces have logUID -> value
     ,
     config(subspace.get(BackupAgentBase::keyConfig)), lastRestorable(subspace.get(FileBackupAgent::keyLastRestorable)),
-    taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), true, false, true)),
-    futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), true, true)) {}
+    taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks),
+                              AccessSystemKeys::TRUE,
+                              PriorityBatch::FALSE,
+                              LockAware::TRUE)),
+    futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::TRUE, LockAware::TRUE)) {
+}
 
 namespace fileBackup {
 
@@ -863,10 +870,10 @@ ACTOR static Future<Void> abortFiveOneBackup(FileBackupAgent* backupAgent,
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 	state KeyBackedTag tag = makeBackupTag(tagName);
-	state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, false, backup_unneeded()));
+	state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::FALSE, backup_unneeded()));
 
 	state BackupConfig config(current.first);
-	EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));
+	EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN));
 
 	if (!backupAgent->isRunnable(status)) {
 		throw backup_unneeded();
@@ -952,7 +959,7 @@ ACTOR static Future<Key> addBackupTask(StringRef name,
                                        Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
                                        std::function<void(Reference<Task>)> setupTaskFn = NOP_SETUP_TASK_FN,
                                        int priority = 0,
-                                       bool setValidation = true) {
+                                       SetValidation setValidation = SetValidation::TRUE) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
@@ -1107,7 +1114,7 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 				Params.beginKey().set(task, range.end);
 
 				// Save and extend the task with the new begin parameter
-				state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, true));
+				state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, UpdateParams::TRUE));
 
 				// Update the range bytes written in the backup config
 				backup.rangeBytesWritten().atomicOp(tr, file->size(), MutationRef::AddValue);
@@ -1201,7 +1208,13 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		// retrieve kvData
 		state PromiseStream<RangeResultWithVersion> results;
 
-		state Future<Void> rc = readCommitted(cx, results, lock, KeyRangeRef(beginKey, endKey), true, true, true);
+		state Future<Void> rc = readCommitted(cx,
+		                                      results,
+		                                      lock,
+		                                      KeyRangeRef(beginKey, endKey),
+		                                      Terminator::TRUE,
+		                                      AccessSystemKeys::TRUE,
+		                                      LockAware::TRUE);
 		state RangeFileWriter rangeFile;
 		state BackupConfig backup(task);
 
@@ -2044,7 +2057,8 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 		state std::vector<Future<Void>> rc;
 
 		for (auto& range : ranges) {
-			rc.push_back(readCommitted(cx, results, lock, range, false, true, true));
+			rc.push_back(
+			    readCommitted(cx, results, lock, range, Terminator::FALSE, AccessSystemKeys::TRUE, LockAware::TRUE));
 		}
 
 		state Future<Void> sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr<Void> const& result) {
@@ -2222,7 +2236,7 @@ struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
 			    Params.destUidValue().set(task, destUidValue);
 		    },
 		    0,
-		    false));
+		    SetValidation::FALSE));
 
 		return key;
 	}
@@ -3580,9 +3594,9 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		state int64_t remainingInBatch = Params.remainingInBatch().get(task);
 		state bool addingToExistingBatch = remainingInBatch > 0;
 		state Version restoreVersion;
-		state Future<Optional<bool>> onlyAppyMutationLogs = restore.onlyAppyMutationLogs().get(tr);
+		state Future<Optional<bool>> onlyApplyMutationLogs = restore.onlyApplyMutationLogs().get(tr);
 
-		wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) && success(onlyAppyMutationLogs) &&
+		wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) && success(onlyApplyMutationLogs) &&
 		     checkTaskVersion(tr->getDatabase(), task, name, version));
 
 		// If not adding to an existing batch then update the apply mutations end version so the mutations from the
@@ -4058,12 +4072,13 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 				wait(checkTaskVersion(tr->getDatabase(), task, name, version));
-				wait(store(beginVersion, restore.beginVersion().getD(tr, false, invalidVersion)));
+				wait(store(beginVersion, restore.beginVersion().getD(tr, Snapshot::FALSE, ::invalidVersion)));
 
 				wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)));
 				wait(store(ranges, restore.getRestoreRangesOrDefault(tr)));
-				wait(store(logsOnly, restore.onlyAppyMutationLogs().getD(tr, false, false)));
-				wait(store(inconsistentSnapshotOnly, restore.inconsistentSnapshotOnly().getD(tr, false, false)));
+				wait(store(logsOnly, restore.onlyApplyMutationLogs().getD(tr, Snapshot::FALSE, false)));
+				wait(store(inconsistentSnapshotOnly,
+				           restore.inconsistentSnapshotOnly().getD(tr, Snapshot::FALSE, false)));
 
 				wait(taskBucket->keepRunning(tr, task));
 
@@ -4245,7 +4260,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 		    tr, taskBucket, task, 0, "", 0, CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE)));
 
 		wait(taskBucket->finish(tr, task));
-		state Future<Optional<bool>> logsOnly = restore.onlyAppyMutationLogs().get(tr);
+		state Future<Optional<bool>> logsOnly = restore.onlyApplyMutationLogs().get(tr);
 		wait(success(logsOnly));
 		if (logsOnly.get().present() && logsOnly.get().get()) {
 			// If this is an incremental restore, we need to set the applyMutationsMapPrefix
@@ -4314,7 +4329,7 @@ public:
 	static constexpr int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8;
 
 	// Parallel restore
-	ACTOR static Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true) {
+	ACTOR static Future<Void> parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB = UnlockDB::TRUE) {
 		state ReadYourWritesTransaction tr(cx);
 		state Optional<Value> restoreRequestDoneKeyValue;
 		TraceEvent("FastRestoreToolWaitForRestoreToFinish").detail("DBLock", randomUID);
@@ -4365,7 +4380,7 @@ public:
 	                                                Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                                                Key bcUrl,
 	                                                Version targetVersion,
-	                                                bool lockDB,
+	                                                LockDB lockDB,
 	                                                UID randomUID,
 	                                                Key addPrefix,
 	                                                Key removePrefix) {
@@ -4458,7 +4473,7 @@ public:
 	ACTOR static Future<EBackupState> waitBackup(FileBackupAgent* backupAgent,
 	                                             Database cx,
 	                                             std::string tagName,
-	                                             bool stopWhenDone,
+	                                             StopWhenDone stopWhenDone,
 	                                             Reference<IBackupContainer>* pContainer = nullptr,
 	                                             UID* pUID = nullptr) {
 		state std::string backTrace;
@@ -4476,7 +4491,8 @@ public:
 				}
 
 				state BackupConfig config(oldUidAndAborted.get().first);
-				state EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));
+				state EBackupState status =
+				    wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN));
 
 				// Break, if one of the following is true
 				//  - no longer runnable
@@ -4486,7 +4502,7 @@ public:
 
 					if (pContainer != nullptr) {
 						Reference<IBackupContainer> c =
-						    wait(config.backupContainer().getOrThrow(tr, false, backup_invalid_info()));
+						    wait(config.backupContainer().getOrThrow(tr, Snapshot::FALSE, backup_invalid_info()));
 						*pContainer = c;
 					}
 
@@ -4506,6 +4522,7 @@ public:
 		}
 	}
 
+	// TODO: Get rid of all of these confusing boolean flags
 	ACTOR static Future<Void> submitBackup(FileBackupAgent* backupAgent,
 	                                       Reference<ReadYourWritesTransaction> tr,
 	                                       Key outContainer,
@@ -4513,9 +4530,10 @@ public:
 	                                       int snapshotIntervalSeconds,
 	                                       std::string tagName,
 	                                       Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                       bool stopWhenDone,
-	                                       bool partitionedLog,
-	                                       bool incrementalBackupOnly) {
+	                                       StopWhenDone stopWhenDone,
+	                                       UsePartitionedLog partitionedLog,
+	                                       IncrementalBackupOnly incrementalBackupOnly,
+	                                       Optional<std::string> encryptionKeyFileName) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
@@ -4531,7 +4549,7 @@ public:
 		if (uidAndAbortedFlag.present()) {
 			state BackupConfig prevConfig(uidAndAbortedFlag.get().first);
 			state EBackupState prevBackupStatus =
-			    wait(prevConfig.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));
+			    wait(prevConfig.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN));
 			if (FileBackupAgent::isRunnable(prevBackupStatus)) {
 				throw backup_duplicate();
 			}
@@ -4553,7 +4571,7 @@ public:
 			backupContainer = joinPath(backupContainer, std::string("backup-") + nowStr.toString());
 		}
 
-		state Reference<IBackupContainer> bc = IBackupContainer::openContainer(backupContainer);
+		state Reference<IBackupContainer> bc = IBackupContainer::openContainer(backupContainer, encryptionKeyFileName);
 		try {
 			wait(timeoutError(bc->create(), 30));
 		} catch (Error& e) {
@@ -4644,9 +4662,9 @@ public:
 	                                        Version restoreVersion,
 	                                        Key addPrefix,
 	                                        Key removePrefix,
-	                                        bool lockDB,
-	                                        bool onlyAppyMutationLogs,
-	                                        bool inconsistentSnapshotOnly,
+	                                        LockDB lockDB,
+	                                        OnlyApplyMutationLogs onlyApplyMutationLogs,
+	                                        InconsistentSnapshotOnly inconsistentSnapshotOnly,
 	                                        Version beginVersion,
 	                                        UID uid) {
 		KeyRangeMap<int> restoreRangeSet;
@@ -4698,7 +4716,7 @@ public:
 			                                .removePrefix(removePrefix)
 			                                .withPrefix(addPrefix);
 			RangeResult existingRows = wait(tr->getRange(restoreIntoRange, 1));
-			if (existingRows.size() > 0 && !onlyAppyMutationLogs) {
+			if (existingRows.size() > 0 && !onlyApplyMutationLogs) {
 				throw restore_destination_not_empty();
 			}
 		}
@@ -4715,7 +4733,7 @@ public:
 		restore.sourceContainer().set(tr, bc);
 		restore.stateEnum().set(tr, ERestoreState::QUEUED);
 		restore.restoreVersion().set(tr, restoreVersion);
-		restore.onlyAppyMutationLogs().set(tr, onlyAppyMutationLogs);
+		restore.onlyApplyMutationLogs().set(tr, onlyApplyMutationLogs);
 		restore.inconsistentSnapshotOnly().set(tr, inconsistentSnapshotOnly);
 		restore.beginVersion().set(tr, beginVersion);
 		if (BUGGIFY && restoreRanges.size() == 1) {
@@ -4738,7 +4756,7 @@ public:
 	}
 
 	// This method will return the final status of the backup
-	ACTOR static Future<ERestoreState> waitRestore(Database cx, Key tagName, bool verbose) {
+	ACTOR static Future<ERestoreState> waitRestore(Database cx, Key tagName, Verbose verbose) {
 		state ERestoreState status;
 		loop {
 			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
@@ -4794,9 +4812,9 @@ public:
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 		state KeyBackedTag tag = makeBackupTag(tagName.toString());
-		state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, false, backup_unneeded()));
+		state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::FALSE, backup_unneeded()));
 		state BackupConfig config(current.first);
-		state EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));
+		state EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN));
 
 		if (!FileBackupAgent::isRunnable(status)) {
 			throw backup_unneeded();
@@ -4845,11 +4863,11 @@ public:
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 		state KeyBackedTag tag = makeBackupTag(tagName);
-		state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, false, backup_unneeded()));
+		state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::FALSE, backup_unneeded()));
 
 		state BackupConfig config(current.first);
 		state Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
-		EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));
+		EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN));
 
 		if (!backupAgent->isRunnable(status)) {
 			throw backup_unneeded();
@@ -4951,7 +4969,7 @@ public:
 					state BackupConfig config(uidAndAbortedFlag.get().first);
 
 					state EBackupState backupState =
-					    wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));
+					    wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN));
 					JsonBuilderObject statusDoc;
 					statusDoc.setKey("Name", BackupAgentBase::getStateName(backupState));
 					statusDoc.setKey("Description", BackupAgentBase::getStateText(backupState));
@@ -5075,7 +5093,7 @@ public:
 
 	ACTOR static Future<std::string> getStatus(FileBackupAgent* backupAgent,
 	                                           Database cx,
-	                                           bool showErrors,
+	                                           ShowErrors showErrors,
 	                                           std::string tagName) {
 		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 		state std::string statusText;
@@ -5095,7 +5113,8 @@ public:
 				state Future<Optional<Value>> fPaused = tr->get(backupAgent->taskBucket->getPauseKey());
 				if (uidAndAbortedFlag.present()) {
 					config = BackupConfig(uidAndAbortedFlag.get().first);
-					EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));
+					EBackupState status =
+					    wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN));
 					backupState = status;
 				}
 
@@ -5257,7 +5276,7 @@ public:
 	ACTOR static Future<Optional<Version>> getLastRestorable(FileBackupAgent* backupAgent,
 	                                                         Reference<ReadYourWritesTransaction> tr,
 	                                                         Key tagName,
-	                                                         bool snapshot) {
+	                                                         Snapshot snapshot) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		state Optional<Value> version = wait(tr->get(backupAgent->lastRestorable.pack(tagName), snapshot));
@@ -5290,7 +5309,7 @@ public:
 	//   removePrefix: for each key to be restored, remove this prefix first.
 	//   lockDB: if set lock the database with randomUid before performing restore;
 	//           otherwise, check database is locked with the randomUid
-	//   onlyAppyMutationLogs: only perform incremental restore, by only applying mutation logs
+	//   onlyApplyMutationLogs: only perform incremental restore, by only applying mutation logs
 	//   inconsistentSnapshotOnly: Ignore mutation log files during the restore to speedup the process.
 	//                             When set to true, gives an inconsistent snapshot, thus not recommended
 	//   beginVersion: restore's begin version
@@ -5301,15 +5320,16 @@ public:
 	                                     Key tagName,
 	                                     Key url,
 	                                     Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                     bool waitForComplete,
+	                                     WaitForComplete waitForComplete,
 	                                     Version targetVersion,
-	                                     bool verbose,
+	                                     Verbose verbose,
 	                                     Key addPrefix,
 	                                     Key removePrefix,
-	                                     bool lockDB,
-	                                     bool onlyAppyMutationLogs,
-	                                     bool inconsistentSnapshotOnly,
+	                                     LockDB lockDB,
+	                                     OnlyApplyMutationLogs onlyApplyMutationLogs,
+	                                     InconsistentSnapshotOnly inconsistentSnapshotOnly,
 	                                     Version beginVersion,
+	                                     Optional<std::string> encryptionKeyFileName,
 	                                     UID randomUid) {
 		// The restore command line tool won't allow ranges to be empty, but correctness workloads somehow might.
 		if (ranges.empty()) {
@@ -5327,12 +5347,12 @@ public:
 		if (targetVersion == invalidVersion && desc.maxRestorableVersion.present())
 			targetVersion = desc.maxRestorableVersion.get();
 
-		if (targetVersion == invalidVersion && onlyAppyMutationLogs && desc.contiguousLogEnd.present()) {
+		if (targetVersion == invalidVersion && onlyApplyMutationLogs && desc.contiguousLogEnd.present()) {
 			targetVersion = desc.contiguousLogEnd.get() - 1;
 		}
 
 		Optional<RestorableFileSet> restoreSet =
-		    wait(bc->getRestoreSet(targetVersion, ranges, onlyAppyMutationLogs, beginVersion));
+		    wait(bc->getRestoreSet(targetVersion, ranges, onlyApplyMutationLogs, beginVersion));
 
 		if (!restoreSet.present()) {
 			TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
@@ -5364,7 +5384,7 @@ public:
 				                   addPrefix,
 				                   removePrefix,
 				                   lockDB,
-				                   onlyAppyMutationLogs,
+				                   onlyApplyMutationLogs,
 				                   inconsistentSnapshotOnly,
 				                   beginVersion,
 				                   randomUid));
@@ -5395,7 +5415,7 @@ public:
 	                                           Standalone<VectorRef<KeyRangeRef>> ranges,
 	                                           Key addPrefix,
 	                                           Key removePrefix,
-	                                           bool fastRestore) {
+	                                           UsePartitionedLog fastRestore) {
 		state Reference<ReadYourWritesTransaction> ryw_tr =
 		    Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
 		state BackupConfig backupConfig;
@@ -5468,7 +5488,7 @@ public:
 			}
 		}
 
-		wait(success(waitBackup(backupAgent, cx, tagName.toString(), true)));
+		wait(success(waitBackup(backupAgent, cx, tagName.toString(), StopWhenDone::TRUE)));
 		TraceEvent("AS_BackupStopped");
 
 		ryw_tr->reset();
@@ -5493,13 +5513,19 @@ public:
 
 		if (fastRestore) {
 			TraceEvent("AtomicParallelRestoreStartRestore");
-			Version targetVersion = -1;
-			bool lockDB = true;
-			wait(submitParallelRestore(
-			    cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, lockDB, randomUid, addPrefix, removePrefix));
+			Version targetVersion = ::invalidVersion;
+			wait(submitParallelRestore(cx,
+			                           tagName,
+			                           ranges,
+			                           KeyRef(bc->getURL()),
+			                           targetVersion,
+			                           LockDB::TRUE,
+			                           randomUid,
+			                           addPrefix,
+			                           removePrefix));
 			state bool hasPrefix = (addPrefix.size() > 0 || removePrefix.size() > 0);
 			TraceEvent("AtomicParallelRestoreWaitForRestoreFinish").detail("HasPrefix", hasPrefix);
-			wait(parallelRestoreFinish(cx, randomUid, !hasPrefix));
+			wait(parallelRestoreFinish(cx, randomUid, UnlockDB{ !hasPrefix }));
 			// If addPrefix or removePrefix set, we want to transform the effect by copying data
 			if (hasPrefix) {
 				wait(transformRestoredDatabase(cx, ranges, addPrefix, removePrefix));
@@ -5514,15 +5540,16 @@ public:
 			                           tagName,
 			                           KeyRef(bc->getURL()),
 			                           ranges,
-			                           true,
-			                           -1,
-			                           true,
+			                           WaitForComplete::TRUE,
+			                           ::invalidVersion,
+			                           Verbose::TRUE,
 			                           addPrefix,
 			                           removePrefix,
-			                           true,
-			                           false,
-			                           false,
-			                           invalidVersion,
+			                           LockDB::TRUE,
+			                           OnlyApplyMutationLogs::FALSE,
+			                           InconsistentSnapshotOnly::FALSE,
+			                           ::invalidVersion,
+			                           {},
 			                           randomUid));
 			return ver;
 		}
@@ -5537,14 +5564,15 @@ public:
 	                                          Standalone<VectorRef<KeyRangeRef>> ranges,
 	                                          Key addPrefix,
 	                                          Key removePrefix) {
-		return success(atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, true));
+		return success(
+		    atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::TRUE));
 	}
 };
 
 const int FileBackupAgent::dataFooterSize = 20;
 
 // Return if parallel restore has finished
-Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB) {
+Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB) {
 	return FileBackupAgentImpl::parallelRestoreFinish(cx, randomUID, unlockDB);
 }
 
@@ -5553,7 +5581,7 @@ Future<Void> FileBackupAgent::submitParallelRestore(Database cx,
                                                     Standalone<VectorRef<KeyRangeRef>> backupRanges,
                                                     Key bcUrl,
                                                     Version targetVersion,
-                                                    bool lockDB,
+                                                    LockDB lockDB,
                                                     UID randomUID,
                                                     Key addPrefix,
                                                     Key removePrefix) {
@@ -5574,15 +5602,16 @@ Future<Version> FileBackupAgent::restore(Database cx,
                                          Key tagName,
                                          Key url,
                                          Standalone<VectorRef<KeyRangeRef>> ranges,
-                                         bool waitForComplete,
+                                         WaitForComplete waitForComplete,
                                          Version targetVersion,
-                                         bool verbose,
+                                         Verbose verbose,
                                          Key addPrefix,
                                          Key removePrefix,
-                                         bool lockDB,
-                                         bool onlyAppyMutationLogs,
-                                         bool inconsistentSnapshotOnly,
-                                         Version beginVersion) {
+                                         LockDB lockDB,
+                                         OnlyApplyMutationLogs onlyApplyMutationLogs,
+                                         InconsistentSnapshotOnly inconsistentSnapshotOnly,
+                                         Version beginVersion,
+                                         Optional<std::string> const& encryptionKeyFileName) {
 	return FileBackupAgentImpl::restore(this,
 	                                    cx,
 	                                    cxOrig,
@@ -5595,9 +5624,10 @@ Future<Version> FileBackupAgent::restore(Database cx,
 	                                    addPrefix,
 	                                    removePrefix,
 	                                    lockDB,
-	                                    onlyAppyMutationLogs,
+	                                    onlyApplyMutationLogs,
 	                                    inconsistentSnapshotOnly,
 	                                    beginVersion,
+	                                    encryptionKeyFileName,
 	                                    deterministicRandom()->randomUniqueID());
 }
 
@@ -5606,7 +5636,8 @@ Future<Version> FileBackupAgent::atomicRestore(Database cx,
                                                Standalone<VectorRef<KeyRangeRef>> ranges,
                                                Key addPrefix,
                                                Key removePrefix) {
-	return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix, false);
+	return FileBackupAgentImpl::atomicRestore(
+	    this, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::FALSE);
 }
 
 Future<ERestoreState> FileBackupAgent::abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName) {
@@ -5621,7 +5652,7 @@ Future<std::string> FileBackupAgent::restoreStatus(Reference<ReadYourWritesTrans
 	return fileBackup::restoreStatus(tr, tagName);
 }
 
-Future<ERestoreState> FileBackupAgent::waitRestore(Database cx, Key tagName, bool verbose) {
+Future<ERestoreState> FileBackupAgent::waitRestore(Database cx, Key tagName, Verbose verbose) {
 	return FileBackupAgentImpl::waitRestore(cx, tagName, verbose);
 };
 
@@ -5629,11 +5660,12 @@ Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction>
                                            Key outContainer,
                                            int initialSnapshotIntervalSeconds,
                                            int snapshotIntervalSeconds,
-                                           std::string tagName,
+                                           std::string const& tagName,
                                            Standalone<VectorRef<KeyRangeRef>> backupRanges,
-                                           bool stopWhenDone,
-                                           bool partitionedLog,
-                                           bool incrementalBackupOnly) {
+                                           StopWhenDone stopWhenDone,
+                                           UsePartitionedLog partitionedLog,
+                                           IncrementalBackupOnly incrementalBackupOnly,
+                                           Optional<std::string> const& encryptionKeyFileName) {
 	return FileBackupAgentImpl::submitBackup(this,
 	                                         tr,
 	                                         outContainer,
@@ -5643,7 +5675,8 @@ Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction>
 	                                         backupRanges,
 	                                         stopWhenDone,
 	                                         partitionedLog,
-	                                         incrementalBackupOnly);
+	                                         incrementalBackupOnly,
+	                                         encryptionKeyFileName);
 }
 
 Future<Void> FileBackupAgent::discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName) {
@@ -5654,7 +5687,7 @@ Future<Void> FileBackupAgent::abortBackup(Reference<ReadYourWritesTransaction> t
 	return FileBackupAgentImpl::abortBackup(this, tr, tagName);
 }
 
-Future<std::string> FileBackupAgent::getStatus(Database cx, bool showErrors, std::string tagName) {
+Future<std::string> FileBackupAgent::getStatus(Database cx, ShowErrors showErrors, std::string tagName) {
 	return FileBackupAgentImpl::getStatus(this, cx, showErrors, tagName);
 }
 
@@ -5664,7 +5697,7 @@ Future<std::string> FileBackupAgent::getStatusJSON(Database cx, std::string tagN
 
 Future<Optional<Version>> FileBackupAgent::getLastRestorable(Reference<ReadYourWritesTransaction> tr,
                                                              Key tagName,
-                                                             bool snapshot) {
+                                                             Snapshot snapshot) {
 	return FileBackupAgentImpl::getLastRestorable(this, tr, tagName, snapshot);
 }
 
@@ -5676,7 +5709,7 @@ void FileBackupAgent::setLastRestorable(Reference<ReadYourWritesTransaction> tr,
 
 Future<EBackupState> FileBackupAgent::waitBackup(Database cx,
                                                  std::string tagName,
-                                                 bool stopWhenDone,
+                                                 StopWhenDone stopWhenDone,
                                                  Reference<IBackupContainer>* pContainer,
                                                  UID* pUID) {
 	return FileBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone, pContainer, pUID);
@@ -5737,8 +5770,8 @@ ACTOR static Future<Void> writeKVs(Database cx, Standalone<VectorRef<KeyValueRef
 	state ReadYourWritesTransaction tr(cx);
 	loop {
 		try {
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			KeyRef k1 = kvs[begin].key;
 			KeyRef k2 = end < kvs.size() ? kvs[end].key : normalKeys.end;
 			TraceEvent(SevFRTestInfo, "TransformDatabaseContentsWriteKVReadBack")

fdbclient/GrvProxyInterface.h

@@ -46,6 +46,7 @@ struct GrvProxyInterface {
 	bool operator==(GrvProxyInterface const& r) const { return id() == r.id(); }
 	bool operator!=(GrvProxyInterface const& r) const { return id() != r.id(); }
 	NetworkAddress address() const { return getConsistentReadVersion.getEndpoint().getPrimaryAddress(); }
+	NetworkAddressList addresses() const { return getConsistentReadVersion.getEndpoint().addresses; }
 
 	template <class Archive>
 	void serialize(Archive& ar) {

fdbclient/IConfigTransaction.h

@@ -45,7 +45,9 @@ public:
 
 	// Not implemented:
 	void setVersion(Version) override { throw client_invalid_operation(); }
-	Future<Key> getKey(KeySelector const& key, bool snapshot = false) override { throw client_invalid_operation(); }
+	Future<Key> getKey(KeySelector const& key, Snapshot snapshot = Snapshot::FALSE) override {
+		throw client_invalid_operation();
+	}
 	Future<Standalone<VectorRef<const char*>>> getAddressesForKey(Key const& key) override {
 		throw client_invalid_operation();
 	}

fdbclient/IKnobCollection.cpp

@@ -56,17 +56,17 @@ KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::stri
 	static std::unique_ptr<IKnobCollection> clientKnobCollection, serverKnobCollection, testKnobCollection;
 	if (type == Type::CLIENT) {
 		if (!clientKnobCollection) {
-			clientKnobCollection = create(type, Randomize::NO, IsSimulated::NO);
+			clientKnobCollection = create(type, Randomize::FALSE, IsSimulated::FALSE);
 		}
 		return clientKnobCollection->parseKnobValue(knobName, knobValue);
 	} else if (type == Type::SERVER) {
 		if (!serverKnobCollection) {
-			serverKnobCollection = create(type, Randomize::NO, IsSimulated::NO);
+			serverKnobCollection = create(type, Randomize::FALSE, IsSimulated::FALSE);
 		}
 		return serverKnobCollection->parseKnobValue(knobName, knobValue);
 	} else if (type == Type::TEST) {
 		if (!testKnobCollection) {
-			testKnobCollection = create(type, Randomize::NO, IsSimulated::NO);
+			testKnobCollection = create(type, Randomize::FALSE, IsSimulated::FALSE);
 		}
 		return testKnobCollection->parseKnobValue(knobName, knobValue);
 	}
@@ -74,7 +74,7 @@ KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::stri
 }
 
 std::unique_ptr<IKnobCollection> IKnobCollection::globalKnobCollection =
-    IKnobCollection::create(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO);
+    IKnobCollection::create(IKnobCollection::Type::CLIENT, Randomize::FALSE, IsSimulated::FALSE);
 
 void IKnobCollection::setGlobalKnobCollection(Type type, Randomize randomize, IsSimulated isSimulated) {
 	globalKnobCollection = create(type, randomize, isSimulated);

fdbclient/ISingleThreadTransaction.h

@@ -23,6 +23,7 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/KeyRangeMap.h"
+#include "fdbclient/NativeAPI.actor.h"
 #include "flow/Error.h"
 #include "flow/FastRef.h"
 
@@ -49,18 +50,18 @@ public:
 	virtual void setVersion(Version v) = 0;
 	virtual Future<Version> getReadVersion() = 0;
 	virtual Optional<Version> getCachedReadVersion() const = 0;
-	virtual Future<Optional<Value>> get(const Key& key, bool snapshot = false) = 0;
-	virtual Future<Key> getKey(const KeySelector& key, bool snapshot = false) = 0;
+	virtual Future<Optional<Value>> get(const Key& key, Snapshot = Snapshot::FALSE) = 0;
+	virtual Future<Key> getKey(const KeySelector& key, Snapshot = Snapshot::FALSE) = 0;
 	virtual Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
 	                                                    const KeySelector& end,
 	                                                    int limit,
-	                                                    bool snapshot = false,
-	                                                    bool reverse = false) = 0;
+	                                                    Snapshot = Snapshot::FALSE,
+	                                                    Reverse = Reverse::FALSE) = 0;
 	virtual Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
 	                                                    KeySelector end,
 	                                                    GetRangeLimits limits,
-	                                                    bool snapshot = false,
-	                                                    bool reverse = false) = 0;
+	                                                    Snapshot = Snapshot::FALSE,
+	                                                    Reverse = Reverse::FALSE) = 0;
 	virtual Future<Standalone<VectorRef<const char*>>> getAddressesForKey(Key const& key) = 0;
 	virtual Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(KeyRange const& range, int64_t chunkSize) = 0;
 	virtual Future<int64_t> getEstimatedRangeSizeBytes(KeyRange const& keys) = 0;

fdbclient/KeyBackedTypes.h

@@ -150,7 +150,7 @@ template <typename T>
 class KeyBackedProperty {
 public:
 	KeyBackedProperty(KeyRef key) : key(key) {}
-	Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const {
+	Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, Snapshot snapshot = Snapshot::FALSE) const {
 		return map(tr->get(key, snapshot), [](Optional<Value> const& val) -> Optional<T> {
 			if (val.present())
 				return Codec<T>::unpack(Tuple::unpack(val.get()));
@@ -158,12 +158,14 @@ public:
 		});
 	}
 	// Get property's value or defaultValue if it doesn't exist
-	Future<T> getD(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, T defaultValue = T()) const {
+	Future<T> getD(Reference<ReadYourWritesTransaction> tr,
+	               Snapshot snapshot = Snapshot::FALSE,
+	               T defaultValue = T()) const {
 		return map(get(tr, snapshot), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
 	}
 	// Get property's value or throw error if it doesn't exist
 	Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr,
-	                     bool snapshot = false,
+	                     Snapshot snapshot = Snapshot::FALSE,
 	                     Error err = key_not_found()) const {
 		auto keyCopy = key;
 		auto backtrace = platform::get_backtrace();
@@ -180,7 +182,7 @@ public:
 		});
 	}
 
-	Future<Optional<T>> get(Database cx, bool snapshot = false) const {
+	Future<Optional<T>> get(Database cx, Snapshot snapshot = Snapshot::FALSE) const {
 		auto& copy = *this;
 		return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -190,7 +192,7 @@ public:
 		});
 	}
 
-	Future<T> getD(Database cx, bool snapshot = false, T defaultValue = T()) const {
+	Future<T> getD(Database cx, Snapshot snapshot = Snapshot::FALSE, T defaultValue = T()) const {
 		auto& copy = *this;
 		return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -200,7 +202,7 @@ public:
 		});
 	}
 
-	Future<T> getOrThrow(Database cx, bool snapshot = false, Error err = key_not_found()) const {
+	Future<T> getOrThrow(Database cx, Snapshot snapshot = Snapshot::FALSE, Error err = key_not_found()) const {
 		auto& copy = *this;
 		return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -235,7 +237,7 @@ template <typename T>
 class KeyBackedBinaryValue {
 public:
 	KeyBackedBinaryValue(KeyRef key) : key(key) {}
-	Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const {
+	Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, Snapshot snapshot = Snapshot::FALSE) const {
 		return map(tr->get(key, snapshot), [](Optional<Value> const& val) -> Optional<T> {
 			if (val.present())
 				return BinaryReader::fromStringRef<T>(val.get(), Unversioned());
@@ -243,8 +245,11 @@ public:
 		});
 	}
 	// Get property's value or defaultValue if it doesn't exist
-	Future<T> getD(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, T defaultValue = T()) const {
-		return map(get(tr, false), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
+	Future<T> getD(Reference<ReadYourWritesTransaction> tr,
+	               Snapshot snapshot = Snapshot::FALSE,
+	               T defaultValue = T()) const {
+		return map(get(tr, Snapshot::FALSE),
+		           [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
 	}
 	void set(Reference<ReadYourWritesTransaction> tr, T const& val) {
 		return tr->set(key, BinaryWriter::toValue<T>(val, Unversioned()));
@@ -273,8 +278,8 @@ public:
 	                           KeyType const& begin,
 	                           Optional<KeyType> const& end,
 	                           int limit,
-	                           bool snapshot = false,
-	                           bool reverse = false) const {
+	                           Snapshot snapshot = Snapshot::FALSE,
+	                           Reverse reverse = Reverse::FALSE) const {
 		Subspace s = space; // 'this' could be invalid inside lambda
 		Key endKey = end.present() ? s.pack(Codec<KeyType>::pack(end.get())) : space.range().end;
 		return map(
@@ -293,7 +298,7 @@ public:
 
 	Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr,
 	                                KeyType const& key,
-	                                bool snapshot = false) const {
+	                                Snapshot snapshot = Snapshot::FALSE) const {
 		return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot),
 		           [](Optional<Value> const& val) -> Optional<ValueType> {
 			           if (val.present())
@@ -339,7 +344,7 @@ public:
 	                        ValueType const& begin,
 	                        Optional<ValueType> const& end,
 	                        int limit,
-	                        bool snapshot = false) const {
+	                        Snapshot snapshot = Snapshot::FALSE) const {
 		Subspace s = space; // 'this' could be invalid inside lambda
 		Key endKey = end.present() ? s.pack(Codec<ValueType>::pack(end.get())) : space.range().end;
 		return map(
@@ -353,7 +358,9 @@ public:
 		    });
 	}
 
-	Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const& val, bool snapshot = false) const {
+	Future<bool> exists(Reference<ReadYourWritesTransaction> tr,
+	                    ValueType const& val,
+	                    Snapshot snapshot = Snapshot::FALSE) const {
 		return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot),
 		           [](Optional<Value> const& val) -> bool { return val.present(); });
 	}

fdbclient/KeyRangeMap.actor.cpp

@@ -119,7 +119,8 @@ void krmSetPreviouslyEmptyRange(CommitTransactionRef& tr,
 ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) {
 	state KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
-	RangeResult old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
+	RangeResult old =
+	    wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::TRUE));
 
 	Value oldValue;
 	bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
@@ -140,7 +141,8 @@ ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, V
 ACTOR Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value) {
 	state KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
-	RangeResult old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
+	RangeResult old =
+	    wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::TRUE));
 
 	Value oldValue;
 	bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
@@ -175,8 +177,10 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
 	    KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString());
 
 	state vector<Future<RangeResult>> keys;
-	keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true));
-	keys.push_back(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, true));
+	keys.push_back(
+	    tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, Snapshot::TRUE));
+	keys.push_back(
+	    tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, Snapshot::TRUE));
 	wait(waitForAll(keys));
 
 	// Determine how far to extend this range at the beginning

fdbclient/ManagementAPI.actor.cpp

@@ -2473,7 +2473,8 @@ ACTOR Future<Void> changeCachedRange(Database cx, KeyRangeRef range, bool add) {
 			tr.clear(sysRangeClear);
 			tr.clear(privateRange);
 			tr.addReadConflictRange(privateRange);
-			RangeResult previous = wait(tr.getRange(KeyRangeRef(storageCachePrefix, sysRange.begin), 1, true));
+			RangeResult previous =
+			    wait(tr.getRange(KeyRangeRef(storageCachePrefix, sysRange.begin), 1, Snapshot::TRUE));
 			bool prevIsCached = false;
 			if (!previous.empty()) {
 				std::vector<uint16_t> prevVal;
@@ -2489,7 +2490,7 @@ ACTOR Future<Void> changeCachedRange(Database cx, KeyRangeRef range, bool add) {
 				tr.set(sysRange.begin, trueValue);
 				tr.set(privateRange.begin, serverKeysTrue);
 			}
-			RangeResult after = wait(tr.getRange(KeyRangeRef(sysRange.end, storageCacheKeys.end), 1, false));
+			RangeResult after = wait(tr.getRange(KeyRangeRef(sysRange.end, storageCacheKeys.end), 1, Snapshot::FALSE));
 			bool afterIsCached = false;
 			if (!after.empty()) {
 				std::vector<uint16_t> afterVal;

fdbclient/NativeAPI.actor.h

@@ -27,10 +27,12 @@
 #elif !defined(FDBCLIENT_NATIVEAPI_ACTOR_H)
 #define FDBCLIENT_NATIVEAPI_ACTOR_H
 
+#include "flow/BooleanParam.h"
 #include "flow/flow.h"
 #include "flow/TDMetric.actor.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/CommitProxyInterface.h"
+#include "fdbclient/ClientBooleanParams.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/ClusterInterface.h"
@@ -51,7 +53,8 @@ void addref(DatabaseContext* ptr);
 template <>
 void delref(DatabaseContext* ptr);
 
-void validateOptionValue(Optional<StringRef> value, bool shouldBePresent);
+void validateOptionValuePresent(Optional<StringRef> value);
+void validateOptionValueNotPresent(Optional<StringRef> value);
 
 void enableClientInfoLogging();
 
@@ -81,13 +84,13 @@ public:
 	// on another thread
 	static Database createDatabase(Reference<ClusterConnectionFile> connFile,
 	                               int apiVersion,
-	                               bool internal = true,
+	                               IsInternal internal = IsInternal::TRUE,
 	                               LocalityData const& clientLocality = LocalityData(),
 	                               DatabaseContext* preallocatedDb = nullptr);
 
 	static Database createDatabase(std::string connFileName,
 	                               int apiVersion,
-	                               bool internal = true,
+	                               IsInternal internal = IsInternal::TRUE,
 	                               LocalityData const& clientLocality = LocalityData());
 
 	Database() {} // an uninitialized database can be destructed or reassigned safely; that's it
@@ -112,7 +115,7 @@ private:
 void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>());
 
 // Configures the global networking machinery
-void setupNetwork(uint64_t transportId = 0, bool useMetrics = false);
+void setupNetwork(uint64_t transportId = 0, UseMetrics = UseMetrics::FALSE);
 
 // This call blocks while the network is running.  To use the API in a single-threaded
 //  environment, the calling program must have ACTORs already launched that are waiting
@@ -248,24 +251,24 @@ public:
 	Future<Version> getRawReadVersion();
 	Optional<Version> getCachedReadVersion() const;
 
-	[[nodiscard]] Future<Optional<Value>> get(const Key& key, bool snapshot = false);
+	[[nodiscard]] Future<Optional<Value>> get(const Key& key, Snapshot = Snapshot::FALSE);
 	[[nodiscard]] Future<Void> watch(Reference<Watch> watch);
-	[[nodiscard]] Future<Key> getKey(const KeySelector& key, bool snapshot = false);
+	[[nodiscard]] Future<Key> getKey(const KeySelector& key, Snapshot = Snapshot::FALSE);
 	// Future< Optional<KeyValue> > get( const KeySelectorRef& key );
 	[[nodiscard]] Future<RangeResult> getRange(const KeySelector& begin,
 	                                           const KeySelector& end,
 	                                           int limit,
-	                                           bool snapshot = false,
-	                                           bool reverse = false);
+	                                           Snapshot = Snapshot::FALSE,
+	                                           Reverse = Reverse::FALSE);
 	[[nodiscard]] Future<RangeResult> getRange(const KeySelector& begin,
 	                                           const KeySelector& end,
 	                                           GetRangeLimits limits,
-	                                           bool snapshot = false,
-	                                           bool reverse = false);
+	                                           Snapshot = Snapshot::FALSE,
+	                                           Reverse = Reverse::FALSE);
 	[[nodiscard]] Future<RangeResult> getRange(const KeyRange& keys,
 	                                           int limit,
-	                                           bool snapshot = false,
-	                                           bool reverse = false) {
+	                                           Snapshot snapshot = Snapshot::FALSE,
+	                                           Reverse reverse = Reverse::FALSE) {
 		return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
 		                KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
 		                limit,
@@ -274,8 +277,8 @@ public:
 	}
 	[[nodiscard]] Future<RangeResult> getRange(const KeyRange& keys,
 	                                           GetRangeLimits limits,
-	                                           bool snapshot = false,
-	                                           bool reverse = false) {
+	                                           Snapshot snapshot = Snapshot::FALSE,
+	                                           Reverse reverse = Reverse::FALSE) {
 		return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
 		                KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
 		                limits,
@@ -289,19 +292,19 @@ public:
 	                                          const KeySelector& begin,
 	                                          const KeySelector& end,
 	                                          int limit,
-	                                          bool snapshot = false,
-	                                          bool reverse = false);
+	                                          Snapshot = Snapshot::FALSE,
+	                                          Reverse = Reverse::FALSE);
 	[[nodiscard]] Future<Void> getRangeStream(const PromiseStream<Standalone<RangeResultRef>>& results,
 	                                          const KeySelector& begin,
 	                                          const KeySelector& end,
 	                                          GetRangeLimits limits,
-	                                          bool snapshot = false,
-	                                          bool reverse = false);
+	                                          Snapshot = Snapshot::FALSE,
+	                                          Reverse = Reverse::FALSE);
 	[[nodiscard]] Future<Void> getRangeStream(const PromiseStream<Standalone<RangeResultRef>>& results,
 	                                          const KeyRange& keys,
 	                                          int limit,
-	                                          bool snapshot = false,
-	                                          bool reverse = false) {
+	                                          Snapshot snapshot = Snapshot::FALSE,
+	                                          Reverse reverse = Reverse::FALSE) {
 		return getRangeStream(results,
 		                      KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
 		                      KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
@@ -312,8 +315,8 @@ public:
 	[[nodiscard]] Future<Void> getRangeStream(const PromiseStream<Standalone<RangeResultRef>>& results,
 	                                          const KeyRange& keys,
 	                                          GetRangeLimits limits,
-	                                          bool snapshot = false,
-	                                          bool reverse = false) {
+	                                          Snapshot snapshot = Snapshot::FALSE,
+	                                          Reverse reverse = Reverse::FALSE) {
 		return getRangeStream(results,
 		                      KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
 		                      KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
@@ -348,13 +351,13 @@ public:
 	// The returned list would still be in form of [keys.begin, splitPoint1, splitPoint2, ... , keys.end]
 	Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(KeyRange const& keys, int64_t chunkSize);
 	// If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key
-	void set(const KeyRef& key, const ValueRef& value, bool addConflictRange = true);
+	void set(const KeyRef& key, const ValueRef& value, AddConflictRange = AddConflictRange::TRUE);
 	void atomicOp(const KeyRef& key,
 	              const ValueRef& value,
 	              MutationRef::Type operationType,
-	              bool addConflictRange = true);
-	void clear(const KeyRangeRef& range, bool addConflictRange = true);
-	void clear(const KeyRef& key, bool addConflictRange = true);
+	              AddConflictRange = AddConflictRange::TRUE);
+	void clear(const KeyRangeRef& range, AddConflictRange = AddConflictRange::TRUE);
+	void clear(const KeyRef& key, AddConflictRange = AddConflictRange::TRUE);
 	[[nodiscard]] Future<Void> commit(); // Throws not_committed or commit_unknown_result errors in normal operation
 
 	void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>());
@@ -451,7 +454,7 @@ inline uint64_t getWriteOperationCost(uint64_t bytes) {
 
 // Create a transaction to set the value of system key \xff/conf/perpetual_storage_wiggle. If enable == true, the value
 // will be 1. Otherwise, the value will be 0.
-ACTOR Future<Void> setPerpetualStorageWiggle(Database cx, bool enable, bool lock_aware = false);
+ACTOR Future<Void> setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware = LockAware::FALSE);
 
 #include "flow/unactorcompiler.h"
 #endif

fdbclient/PaxosConfigTransaction.actor.cpp

@@ -33,7 +33,7 @@ Optional<Version> PaxosConfigTransaction::getCachedReadVersion() const {
 	return ::invalidVersion;
 }
 
-Future<Optional<Value>> PaxosConfigTransaction::get(Key const& key, bool snapshot) {
+Future<Optional<Value>> PaxosConfigTransaction::get(Key const& key, Snapshot snapshot) {
 	// TODO: Implement
 	return Optional<Value>{};
 }
@@ -41,8 +41,8 @@ Future<Optional<Value>> PaxosConfigTransaction::get(Key const& key, bool snapsho
 Future<Standalone<RangeResultRef>> PaxosConfigTransaction::getRange(KeySelector const& begin,
                                                                     KeySelector const& end,
                                                                     int limit,
-                                                                    bool snapshot,
-                                                                    bool reverse) {
+                                                                    Snapshot snapshot,
+                                                                    Reverse reverse) {
 	// TODO: Implement
 	ASSERT(false);
 	return Standalone<RangeResultRef>{};
@@ -51,9 +51,9 @@ Future<Standalone<RangeResultRef>> PaxosConfigTransaction::getRange(KeySelector
 Future<Standalone<RangeResultRef>> PaxosConfigTransaction::getRange(KeySelector begin,
                                                                     KeySelector end,
                                                                     GetRangeLimits limits,
-                                                                    bool snapshot,
-                                                                    bool reverse) {
-	// TODO: Implememnt
+                                                                    Snapshot snapshot,
+                                                                    Reverse reverse) {
+	// TODO: Implement
 	ASSERT(false);
 	return Standalone<RangeResultRef>{};
 }

fdbclient/PaxosConfigTransaction.h

@@ -38,17 +38,17 @@ public:
 	Future<Version> getReadVersion() override;
 	Optional<Version> getCachedReadVersion() const override;
 
-	Future<Optional<Value>> get(Key const& key, bool snapshot = false) override;
+	Future<Optional<Value>> get(Key const& key, Snapshot = Snapshot::FALSE) override;
 	Future<Standalone<RangeResultRef>> getRange(KeySelector const& begin,
 	                                            KeySelector const& end,
 	                                            int limit,
-	                                            bool snapshot = false,
-	                                            bool reverse = false) override;
+	                                            Snapshot = Snapshot::FALSE,
+	                                            Reverse = Reverse::FALSE) override;
 	Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
 	                                            KeySelector end,
 	                                            GetRangeLimits limits,
-	                                            bool snapshot = false,
-	                                            bool reverse = false) override;
+	                                            Snapshot = Snapshot::FALSE,
+	                                            Reverse = Reverse::FALSE) override;
 	void set(KeyRef const& key, ValueRef const& value) override;
 	void clear(KeyRangeRef const&) override { throw client_invalid_operation(); }
 	void clear(KeyRef const&) override;

fdbclient/ReadYourWrites.actor.cpp

@@ -65,7 +65,7 @@ public:
 		typedef Key Result;
 	};
 
-	template <bool Reverse>
+	template <bool reverse>
 	struct GetRangeReq {
 		GetRangeReq(KeySelector begin, KeySelector end, GetRangeLimits limits)
 		  : begin(begin), end(end), limits(limits) {}
@@ -99,7 +99,7 @@ public:
 		} else if (it->is_empty_range()) {
 			return Optional<Value>();
 		} else {
-			Optional<Value> res = wait(ryw->tr.get(read.key, true));
+			Optional<Value> res = wait(ryw->tr.get(read.key, Snapshot::TRUE));
 			KeyRef k(ryw->arena, read.key);
 
 			if (res.present()) {
@@ -162,20 +162,22 @@ public:
 	// transaction. Responsible for clipping results to the non-system keyspace when appropriate, since NativeAPI
 	// doesn't do that.
 
-	static Future<Optional<Value>> readThrough(ReadYourWritesTransaction* ryw, GetValueReq read, bool snapshot) {
+	static Future<Optional<Value>> readThrough(ReadYourWritesTransaction* ryw, GetValueReq read, Snapshot snapshot) {
 		return ryw->tr.get(read.key, snapshot);
 	}
 
-	ACTOR static Future<Key> readThrough(ReadYourWritesTransaction* ryw, GetKeyReq read, bool snapshot) {
+	ACTOR static Future<Key> readThrough(ReadYourWritesTransaction* ryw, GetKeyReq read, Snapshot snapshot) {
 		Key key = wait(ryw->tr.getKey(read.key, snapshot));
 		if (ryw->getMaxReadKey() < key)
 			return ryw->getMaxReadKey(); // Filter out results in the system keys if they are not accessible
 		return key;
 	}
 
-	ACTOR template <bool Reverse>
-	static Future<RangeResult> readThrough(ReadYourWritesTransaction* ryw, GetRangeReq<Reverse> read, bool snapshot) {
-		if (Reverse && read.end.offset > 1) {
+	ACTOR template <bool backwards>
+	static Future<RangeResult> readThrough(ReadYourWritesTransaction* ryw,
+	                                       GetRangeReq<backwards> read,
+	                                       Snapshot snapshot) {
+		if (backwards && read.end.offset > 1) {
 			// FIXME: Optimistically assume that this will not run into the system keys, and only reissue if the result
 			// actually does.
 			Key key = wait(ryw->tr.getKey(read.end, snapshot));
@@ -185,10 +187,11 @@ public:
 				read.end = KeySelector(firstGreaterOrEqual(key), key.arena());
 		}
 
-		RangeResult v = wait(ryw->tr.getRange(read.begin, read.end, read.limits, snapshot, Reverse));
+		RangeResult v = wait(
+		    ryw->tr.getRange(read.begin, read.end, read.limits, snapshot, backwards ? Reverse::TRUE : Reverse::FALSE));
 		KeyRef maxKey = ryw->getMaxReadKey();
 		if (v.size() > 0) {
-			if (!Reverse && v[v.size() - 1].key >= maxKey) {
+			if (!backwards && v[v.size() - 1].key >= maxKey) {
 				state RangeResult _v = v;
 				int i = _v.size() - 2;
 				for (; i >= 0 && _v[i].key >= maxKey; --i) {
@@ -299,7 +302,7 @@ public:
 	ACTOR template <class Req>
 	static Future<typename Req::Result> readWithConflictRangeThrough(ReadYourWritesTransaction* ryw,
 	                                                                 Req req,
-	                                                                 bool snapshot) {
+	                                                                 Snapshot snapshot) {
 		choose {
 			when(typename Req::Result result = wait(readThrough(ryw, req, snapshot))) { return result; }
 			when(wait(ryw->resetPromise.getFuture())) { throw internal_error(); }
@@ -316,7 +319,7 @@ public:
 	ACTOR template <class Req>
 	static Future<typename Req::Result> readWithConflictRangeRYW(ReadYourWritesTransaction* ryw,
 	                                                             Req req,
-	                                                             bool snapshot) {
+	                                                             Snapshot snapshot) {
 		state RYWIterator it(&ryw->cache, &ryw->writes);
 		choose {
 			when(typename Req::Result result = wait(read(ryw, req, &it))) {
@@ -332,7 +335,7 @@ public:
 	template <class Req>
 	static inline Future<typename Req::Result> readWithConflictRange(ReadYourWritesTransaction* ryw,
 	                                                                 Req const& req,
-	                                                                 bool snapshot) {
+	                                                                 Snapshot snapshot) {
 		if (ryw->options.readYourWritesDisabled) {
 			return readWithConflictRangeThrough(ryw, req, snapshot);
 		} else if (snapshot && ryw->options.snapshotRywEnabled <= 0) {
@@ -690,7 +693,8 @@ public:
 				//TraceEvent("RYWIssuing", randomID).detail("Begin", read_begin.toString()).detail("End", read_end.toString()).detail("Bytes", requestLimit.bytes).detail("Rows", requestLimit.rows).detail("Limits", limits.bytes).detail("Reached", limits.isReached()).detail("RequestCount", requestCount).detail("SingleClears", singleClears).detail("UcEnd", ucEnd.beginKey()).detail("MinRows", requestLimit.minRows);
 
 				additionalRows = 0;
-				RangeResult snapshot_read = wait(ryw->tr.getRange(read_begin, read_end, requestLimit, true, false));
+				RangeResult snapshot_read =
+				    wait(ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::TRUE, Reverse::FALSE));
 				KeyRangeRef range = getKnownKeyRange(snapshot_read, read_begin, read_end, ryw->arena);
 
 				//TraceEvent("RYWCacheInsert", randomID).detail("Range", range).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", snapshot_read).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", snapshot_read.readThrough);
@@ -993,7 +997,8 @@ public:
 				//TraceEvent("RYWIssuing", randomID).detail("Begin", read_begin.toString()).detail("End", read_end.toString()).detail("Bytes", requestLimit.bytes).detail("Rows", requestLimit.rows).detail("Limits", limits.bytes).detail("Reached", limits.isReached()).detail("RequestCount", requestCount).detail("SingleClears", singleClears).detail("UcEnd", ucEnd.beginKey()).detail("MinRows", requestLimit.minRows);
 
 				additionalRows = 0;
-				RangeResult snapshot_read = wait(ryw->tr.getRange(read_begin, read_end, requestLimit, true, true));
+				RangeResult snapshot_read =
+				    wait(ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::TRUE, Reverse::TRUE));
 				KeyRangeRef range = getKnownKeyRangeBack(snapshot_read, read_begin, read_end, ryw->arena);
 
 				//TraceEvent("RYWCacheInsert", randomID).detail("Range", range).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", snapshot_read).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", snapshot_read.readThrough);
@@ -1110,7 +1115,7 @@ public:
 
 		if (!ryw->options.readYourWritesDisabled) {
 			ryw->watchMap[key].push_back(watch);
-			val = readWithConflictRange(ryw, GetValueReq(key), false);
+			val = readWithConflictRange(ryw, GetValueReq(key), Snapshot::FALSE);
 		} else {
 			ryw->approximateSize += 2 * key.expectedSize() + 1;
 			val = ryw->tr.get(key);
@@ -1352,7 +1357,7 @@ ACTOR Future<RangeResult> getWorkerInterfaces(Reference<ClusterConnectionFile> c
 	}
 }
 
-Future<Optional<Value>> ReadYourWritesTransaction::get(const Key& key, bool snapshot) {
+Future<Optional<Value>> ReadYourWritesTransaction::get(const Key& key, Snapshot snapshot) {
 	TEST(true); // ReadYourWritesTransaction::get
 
 	if (getDatabase()->apiVersionAtLeast(630)) {
@@ -1416,7 +1421,7 @@ Future<Optional<Value>> ReadYourWritesTransaction::get(const Key& key, bool snap
 	return result;
 }
 
-Future<Key> ReadYourWritesTransaction::getKey(const KeySelector& key, bool snapshot) {
+Future<Key> ReadYourWritesTransaction::getKey(const KeySelector& key, Snapshot snapshot) {
 	if (checkUsedDuringCommit()) {
 		return used_during_commit();
 	}
@@ -1435,8 +1440,8 @@ Future<Key> ReadYourWritesTransaction::getKey(const KeySelector& key, bool snaps
 Future<RangeResult> ReadYourWritesTransaction::getRange(KeySelector begin,
                                                         KeySelector end,
                                                         GetRangeLimits limits,
-                                                        bool snapshot,
-                                                        bool reverse) {
+                                                        Snapshot snapshot,
+                                                        Reverse reverse) {
 	if (getDatabase()->apiVersionAtLeast(630)) {
 		if (specialKeys.contains(begin.getKey()) && specialKeys.begin <= end.getKey() &&
 		    end.getKey() <= specialKeys.end) {
@@ -1495,8 +1500,8 @@ Future<RangeResult> ReadYourWritesTransaction::getRange(KeySelector begin,
 Future<RangeResult> ReadYourWritesTransaction::getRange(const KeySelector& begin,
                                                         const KeySelector& end,
                                                         int limit,
-                                                        bool snapshot,
-                                                        bool reverse) {
+                                                        Snapshot snapshot,
+                                                        Reverse reverse) {
 	return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse);
 }
 
@@ -1627,13 +1632,14 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const&
 			clearBegin = std::max(ExtStringRef(keys.begin), it.beginKey());
 			inClearRange = true;
 		} else if (!it.is_cleared_range() && inClearRange) {
-			tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena)), false);
+			tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena)),
+			         AddConflictRange::FALSE);
 			inClearRange = false;
 		}
 	}
 
 	if (inClearRange) {
-		tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), keys.end), false);
+		tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), keys.end), AddConflictRange::FALSE);
 	}
 
 	it.skip(keys.begin);
@@ -1657,9 +1663,9 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const&
 				switch (op[i].type) {
 				case MutationRef::SetValue:
 					if (op[i].value.present()) {
-						tr.set(it.beginKey().assertRef(), op[i].value.get(), false);
+						tr.set(it.beginKey().assertRef(), op[i].value.get(), AddConflictRange::FALSE);
 					} else {
-						tr.clear(it.beginKey().assertRef(), false);
+						tr.clear(it.beginKey().assertRef(), AddConflictRange::FALSE);
 					}
 					break;
 				case MutationRef::AddValue:
@@ -1676,7 +1682,7 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const&
 				case MutationRef::MinV2:
 				case MutationRef::AndV2:
 				case MutationRef::CompareAndClear:
-					tr.atomicOp(it.beginKey().assertRef(), op[i].value.get(), op[i].type, false);
+					tr.atomicOp(it.beginKey().assertRef(), op[i].value.get(), op[i].type, AddConflictRange::FALSE);
 					break;
 				default:
 					break;
@@ -1845,7 +1851,7 @@ RangeResult ReadYourWritesTransaction::getWriteConflictRangeIntersecting(KeyRang
 }
 
 void ReadYourWritesTransaction::atomicOp(const KeyRef& key, const ValueRef& operand, uint32_t operationType) {
-	bool addWriteConflict = !options.getAndResetWriteConflictDisabled();
+	AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() };
 
 	if (checkUsedDuringCommit()) {
 		throw used_during_commit();
@@ -1893,7 +1899,7 @@ void ReadYourWritesTransaction::atomicOp(const KeyRef& key, const ValueRef& oper
 		// this does validation of the key and needs to be performed before the readYourWritesDisabled path
 		KeyRangeRef range = getVersionstampKeyRange(arena, k, tr.getCachedReadVersion().orDefault(0), getMaxReadKey());
 		versionStampKeys.push_back(arena, k);
-		addWriteConflict = false;
+		addWriteConflict = AddConflictRange::FALSE;
 		if (!options.readYourWritesDisabled) {
 			writeRangeToNativeTransaction(range);
 			writes.addUnmodifiedAndUnreadableRange(range);
@@ -1953,7 +1959,7 @@ void ReadYourWritesTransaction::set(const KeyRef& key, const ValueRef& value) {
 		}
 	}
 
-	bool addWriteConflict = !options.getAndResetWriteConflictDisabled();
+	AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() };
 
 	if (checkUsedDuringCommit()) {
 		throw used_during_commit();
@@ -1983,7 +1989,7 @@ void ReadYourWritesTransaction::set(const KeyRef& key, const ValueRef& value) {
 }
 
 void ReadYourWritesTransaction::clear(const KeyRangeRef& range) {
-	bool addWriteConflict = !options.getAndResetWriteConflictDisabled();
+	AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() };
 
 	if (checkUsedDuringCommit()) {
 		throw used_during_commit();
@@ -2036,7 +2042,7 @@ void ReadYourWritesTransaction::clear(const KeyRangeRef& range) {
 }
 
 void ReadYourWritesTransaction::clear(const KeyRef& key) {
-	bool addWriteConflict = !options.getAndResetWriteConflictDisabled();
+	AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() };
 
 	if (checkUsedDuringCommit()) {
 		throw used_during_commit();
@@ -2165,7 +2171,7 @@ void ReadYourWritesTransaction::setOption(FDBTransactionOptions::Option option,
 void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option option, Optional<StringRef> value) {
 	switch (option) {
 	case FDBTransactionOptions::READ_YOUR_WRITES_DISABLE:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 
 		if (!reading.isReady() || !cache.empty() || !writes.empty())
 			throw client_invalid_operation();
@@ -2174,26 +2180,26 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti
 		break;
 
 	case FDBTransactionOptions::READ_AHEAD_DISABLE:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 
 		options.readAheadDisabled = true;
 		break;
 
 	case FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 
 		options.nextWriteDisableConflictRange = true;
 		break;
 
 	case FDBTransactionOptions::ACCESS_SYSTEM_KEYS:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 
 		options.readSystemKeys = true;
 		options.writeSystemKeys = true;
 		break;
 
 	case FDBTransactionOptions::READ_SYSTEM_KEYS:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 
 		options.readSystemKeys = true;
 		break;
@@ -2217,30 +2223,30 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti
 		transactionDebugInfo->transactionName = value.present() ? value.get().toString() : "";
 		break;
 	case FDBTransactionOptions::SNAPSHOT_RYW_ENABLE:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 
 		options.snapshotRywEnabled++;
 		break;
 	case FDBTransactionOptions::SNAPSHOT_RYW_DISABLE:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 
 		options.snapshotRywEnabled--;
 		break;
 	case FDBTransactionOptions::USED_DURING_COMMIT_PROTECTION_DISABLE:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 
 		options.disableUsedDuringCommitProtection = true;
 		break;
 	case FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 		options.specialKeySpaceRelaxed = true;
 		break;
 	case FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 		options.specialKeySpaceChangeConfiguration = true;
 		break;
 	case FDBTransactionOptions::BYPASS_UNREADABLE:
-		validateOptionValue(value, false);
+		validateOptionValueNotPresent(value);
 		options.bypassUnreadable = true;
 		break;
 	default:

fdbclient/ReadYourWrites.h

@@ -71,22 +71,22 @@ public:
 	void setVersion(Version v) override { tr.setVersion(v); }
 	Future<Version> getReadVersion() override;
 	Optional<Version> getCachedReadVersion() const override { return tr.getCachedReadVersion(); }
-	Future<Optional<Value>> get(const Key& key, bool snapshot = false) override;
-	Future<Key> getKey(const KeySelector& key, bool snapshot = false) override;
+	Future<Optional<Value>> get(const Key& key, Snapshot = Snapshot::FALSE) override;
+	Future<Key> getKey(const KeySelector& key, Snapshot = Snapshot::FALSE) override;
 	Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
 	                                            const KeySelector& end,
 	                                            int limit,
-	                                            bool snapshot = false,
-	                                            bool reverse = false) override;
+	                                            Snapshot = Snapshot::FALSE,
+	                                            Reverse = Reverse::FALSE) override;
 	Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
 	                                            KeySelector end,
 	                                            GetRangeLimits limits,
-	                                            bool snapshot = false,
-	                                            bool reverse = false) override;
+	                                            Snapshot = Snapshot::FALSE,
+	                                            Reverse = Reverse::FALSE) override;
 	Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys,
 	                                            int limit,
-	                                            bool snapshot = false,
-	                                            bool reverse = false) {
+	                                            Snapshot snapshot = Snapshot::FALSE,
+	                                            Reverse reverse = Reverse::FALSE) {
 		return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
 		                KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
 		                limit,
@@ -95,8 +95,8 @@ public:
 	}
 	Future<RangeResult> getRange(const KeyRange& keys,
 	                             GetRangeLimits limits,
-	                             bool snapshot = false,
-	                             bool reverse = false) {
+	                             Snapshot snapshot = Snapshot::FALSE,
+	                             Reverse reverse = Reverse::FALSE) {
 		return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
 		                KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
 		                limits,

fdbclient/ServerKnobs.cpp

@@ -26,9 +26,7 @@ ServerKnobs::ServerKnobs(Randomize randomize, ClientKnobs* clientKnobs, IsSimula
 	initialize(randomize, clientKnobs, isSimulated);
 }
 
-void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsSimulated _isSimulated) {
-	bool const randomize = _randomize == Randomize::YES;
-	bool const isSimulated = _isSimulated == IsSimulated::YES;
+void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSimulated isSimulated) {
 	// clang-format off
 	// Versions
 	init( VERSIONS_PER_SECOND,                                   1e6 );
@@ -102,6 +100,8 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS
 	init( PUSH_STATS_SLOW_AMOUNT,                                  2 );
 	init( PUSH_STATS_SLOW_RATIO,                                 0.5 );
 	init( TLOG_POP_BATCH_SIZE,                                  1000 ); if ( randomize && BUGGIFY ) TLOG_POP_BATCH_SIZE = 10;
+	init( TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE,     250e6 );
+	init( ENABLE_DETAILED_TLOG_POP_TRACE,                       true );
 
 	// disk snapshot max timeout, to be put in TLog, storage and coordinator nodes
 	init( MAX_FORKED_PROCESS_OUTPUT,                            1024 );
@@ -462,7 +462,15 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS
 	init( REPLACE_INTERFACE_CHECK_DELAY,                         5.0 );
 	init( COORDINATOR_REGISTER_INTERVAL,                         5.0 );
 	init( CLIENT_REGISTER_INTERVAL,                            600.0 );
-	init( CLUSTER_CONTROLLER_ENABLE_WORKER_HEALTH_MONITOR,     false );
+	init( CC_ENABLE_WORKER_HEALTH_MONITOR,                     false );
+	init( CC_WORKER_HEALTH_CHECKING_INTERVAL,                   60.0 );
+	init( CC_DEGRADED_LINK_EXPIRATION_INTERVAL,                300.0 );
+	init( CC_MIN_DEGRADATION_INTERVAL,                         120.0 );
+	init( CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE,                      3 );
+	init( CC_MAX_EXCLUSION_DUE_TO_HEALTH,                          2 );
+	init( CC_HEALTH_TRIGGER_RECOVERY,                          false );
+	init( CC_TRACKING_HEALTH_RECOVERY_INTERVAL,               3600.0 );
+	init( CC_MAX_HEALTH_RECOVERY_COUNT,                            2 );
 
 	init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL,                   600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0;
 	init( EXPECTED_MASTER_FITNESS,            ProcessClass::UnsetFit );
@@ -720,6 +728,7 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS
 	init( REDWOOD_DEFAULT_EXTENT_READ_SIZE,              1024 * 1024 );
 	init( REDWOOD_EXTENT_CONCURRENT_READS,                         4 );
 	init( REDWOOD_KVSTORE_CONCURRENT_READS,                       64 );
+	init( REDWOOD_KVSTORE_RANGE_PREFETCH,                       true );
 	init( REDWOOD_PAGE_REBUILD_MAX_SLACK,                       0.33 );
 	init( REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES,                    10 );
 	init( REDWOOD_LAZY_CLEAR_MIN_PAGES,                            0 );

fdbclient/ServerKnobs.h

@@ -20,6 +20,7 @@
 
 #pragma once
 
+#include "flow/BooleanParam.h"
 #include "flow/Knobs.h"
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/Locality.h"
@@ -64,6 +65,8 @@ public:
 	                                              // message (measured in 1/1024ths, e.g. a value of 2048 yields a
 	                                              // factor of 2).
 	int64_t VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD;
+	int64_t TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE;
+	bool ENABLE_DETAILED_TLOG_POP_TRACE;
 	double TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR;
 	int64_t TLOG_MESSAGE_BLOCK_BYTES;
 	int64_t MAX_MESSAGE_SIZE;
@@ -222,10 +225,6 @@ public:
 	double DD_FAILURE_TIME;
 	double DD_ZERO_HEALTHY_TEAM_DELAY;
 
-	// Redwood Storage Engine
-	int PREFIX_TREE_IMMEDIATE_KEY_SIZE_LIMIT;
-	int PREFIX_TREE_IMMEDIATE_KEY_SIZE_MIN;
-
 	// KeyValueStore SQLITE
 	int CLEAR_BUFFER_SIZE;
 	double READ_VALUE_TIME_ESTIMATE;
@@ -389,7 +388,23 @@ public:
 	double REPLACE_INTERFACE_CHECK_DELAY;
 	double COORDINATOR_REGISTER_INTERVAL;
 	double CLIENT_REGISTER_INTERVAL;
-	bool CLUSTER_CONTROLLER_ENABLE_WORKER_HEALTH_MONITOR;
+	bool CC_ENABLE_WORKER_HEALTH_MONITOR;
+	double CC_WORKER_HEALTH_CHECKING_INTERVAL; // The interval of refreshing the degraded server list.
+	double CC_DEGRADED_LINK_EXPIRATION_INTERVAL; // The time period from the last degradation report after which a
+	                                             // degraded server is considered healthy.
+	double CC_MIN_DEGRADATION_INTERVAL; // The minimum interval that a server is reported as degraded to be considered
+	                                    // as degraded by Cluster Controller.
+	int CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE; // The maximum number of degraded peers when excluding a server. When the
+	                                        // number of degraded peers is more than this value, we will not exclude
+	                                        // this server since it may because of server overload.
+	int CC_MAX_EXCLUSION_DUE_TO_HEALTH; // The max number of degraded servers to exclude by Cluster Controller due to
+	                                    // degraded health.
+	bool CC_HEALTH_TRIGGER_RECOVERY; // If true, cluster controller will kill the master to trigger recovery when
+	                                 // detecting degraded servers. If false, cluster controller only prints a warning.
+	double CC_TRACKING_HEALTH_RECOVERY_INTERVAL; // The number of recovery count should not exceed
+	                                             // CC_MAX_HEALTH_RECOVERY_COUNT within
+	                                             // CC_TRACKING_HEALTH_RECOVERY_INTERVAL.
+	int CC_MAX_HEALTH_RECOVERY_COUNT;
 
 	// Knobs used to select the best policy (via monte carlo)
 	int POLICY_RATING_TESTS; // number of tests per policy (in order to compare)
@@ -659,7 +674,7 @@ public:
 	int REDWOOD_DEFAULT_EXTENT_READ_SIZE; // Extent read size for Redwood files
 	int REDWOOD_EXTENT_CONCURRENT_READS; // Max number of simultaneous extent disk reads in progress.
 	int REDWOOD_KVSTORE_CONCURRENT_READS; // Max number of simultaneous point or range reads in progress.
-	int REDWOOD_COMMIT_CONCURRENT_READS; // Max number of concurrent reads done to support commit operations
+	bool REDWOOD_KVSTORE_RANGE_PREFETCH; // Whether to use range read prefetching
 	double REDWOOD_PAGE_REBUILD_MAX_SLACK; // When rebuilding pages, max slack to allow in page
 	int REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES; // Number of pages to try to pop from the lazy delete queue and process at
 	                                         // once

fdbclient/SimpleConfigTransaction.actor.cpp

@@ -221,23 +221,23 @@ Optional<Version> SimpleConfigTransaction::getCachedReadVersion() const {
 	return impl().getCachedReadVersion();
 }
 
-Future<Optional<Value>> SimpleConfigTransaction::get(Key const& key, bool snapshot) {
+Future<Optional<Value>> SimpleConfigTransaction::get(Key const& key, Snapshot snapshot) {
 	return impl().get(key);
 }
 
 Future<Standalone<RangeResultRef>> SimpleConfigTransaction::getRange(KeySelector const& begin,
                                                                      KeySelector const& end,
                                                                      int limit,
-                                                                     bool snapshot,
-                                                                     bool reverse) {
+                                                                     Snapshot snapshot,
+                                                                     Reverse reverse) {
 	return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey()));
 }
 
 Future<Standalone<RangeResultRef>> SimpleConfigTransaction::getRange(KeySelector begin,
                                                                      KeySelector end,
                                                                      GetRangeLimits limits,
-                                                                     bool snapshot,
-                                                                     bool reverse) {
+                                                                     Snapshot snapshot,
+                                                                     Reverse reverse) {
 	return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey()));
 }
 

fdbclient/SimpleConfigTransaction.h

@@ -47,17 +47,17 @@ public:
 	Future<Version> getReadVersion() override;
 	Optional<Version> getCachedReadVersion() const override;
 
-	Future<Optional<Value>> get(Key const& key, bool snapshot = false) override;
+	Future<Optional<Value>> get(Key const& key, Snapshot = Snapshot::FALSE) override;
 	Future<Standalone<RangeResultRef>> getRange(KeySelector const& begin,
 	                                            KeySelector const& end,
 	                                            int limit,
-	                                            bool snapshot = false,
-	                                            bool reverse = false) override;
+	                                            Snapshot = Snapshot::FALSE,
+	                                            Reverse = Reverse::FALSE) override;
 	Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
 	                                            KeySelector end,
 	                                            GetRangeLimits limits,
-	                                            bool snapshot = false,
-	                                            bool reverse = false) override;
+	                                            Snapshot = Snapshot::FALSE,
+	                                            Reverse = Reverse::FALSE) override;
 	Future<Void> commit() override;
 	Version getCommittedVersion() const override;
 	void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;

fdbclient/SpecialKeySpace.actor.cpp

@@ -277,7 +277,7 @@ ACTOR Future<RangeResult> SpecialKeySpace::checkRYWValid(SpecialKeySpace* sks,
                                                          KeySelector begin,
                                                          KeySelector end,
                                                          GetRangeLimits limits,
-                                                         bool reverse) {
+                                                         Reverse reverse) {
 	ASSERT(ryw);
 	choose {
 		when(RangeResult result =
@@ -293,7 +293,7 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
                                                                     KeySelector begin,
                                                                     KeySelector end,
                                                                     GetRangeLimits limits,
-                                                                    bool reverse) {
+                                                                    Reverse reverse) {
 	// This function handles ranges which cover more than one keyrange and aggregates all results
 	// KeySelector, GetRangeLimits and reverse are all handled here
 	state RangeResult result;
@@ -413,7 +413,7 @@ Future<RangeResult> SpecialKeySpace::getRange(ReadYourWritesTransaction* ryw,
                                               KeySelector begin,
                                               KeySelector end,
                                               GetRangeLimits limits,
-                                              bool reverse) {
+                                              Reverse reverse) {
 	// validate limits here
 	if (!limits.isValid())
 		return range_limits_invalid();
@@ -441,7 +441,7 @@ ACTOR Future<Optional<Value>> SpecialKeySpace::getActor(SpecialKeySpace* sks,
 	                                        KeySelector(firstGreaterOrEqual(key)),
 	                                        KeySelector(firstGreaterOrEqual(keyAfter(key))),
 	                                        GetRangeLimits(CLIENT_KNOBS->TOO_MANY),
-	                                        false));
+	                                        Reverse::FALSE));
 	ASSERT(result.size() <= 1);
 	if (result.size()) {
 		return Optional<Value>(result[0].value);

fdbclient/SpecialKeySpace.actor.h

@@ -168,7 +168,7 @@ public:
 	                             KeySelector begin,
 	                             KeySelector end,
 	                             GetRangeLimits limits,
-	                             bool reverse = false);
+	                             Reverse = Reverse::FALSE);
 
 	void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value);
 
@@ -209,13 +209,13 @@ private:
 	                                               KeySelector begin,
 	                                               KeySelector end,
 	                                               GetRangeLimits limits,
-	                                               bool reverse);
+	                                               Reverse reverse);
 	ACTOR static Future<RangeResult> getRangeAggregationActor(SpecialKeySpace* sks,
 	                                                          ReadYourWritesTransaction* ryw,
 	                                                          KeySelector begin,
 	                                                          KeySelector end,
 	                                                          GetRangeLimits limits,
-	                                                          bool reverse);
+	                                                          Reverse reverse);
 
 	KeyRangeMap<SpecialKeyRangeReadImpl*> readImpls;
 	KeyRangeMap<SpecialKeySpace::MODULE> modules;

fdbclient/StorageServerInterface.cpp

@@ -145,6 +145,47 @@ void TSS_traceMismatch(TraceEvent& event,
 	    .detail("Version", req.version)
 	    .detail("Limit", req.limit)
 	    .detail("LimitBytes", req.limitBytes)
+	    .setMaxFieldLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE * 4 / 10)
+	    .detail("SSReply", ssResultsString)
+	    .detail("TSSReply", tssResultsString);
+}
+
+// streaming range reads
+template <>
+bool TSS_doCompare(const GetKeyValuesStreamReply& src, const GetKeyValuesStreamReply& tss) {
+	return src.more == tss.more && src.data == tss.data;
+}
+
+template <>
+const char* TSS_mismatchTraceName(const GetKeyValuesStreamRequest& req) {
+	return "TSSMismatchGetKeyValuesStream";
+}
+
+// TODO this is all duplicated from above, simplify?
+template <>
+void TSS_traceMismatch(TraceEvent& event,
+                       const GetKeyValuesStreamRequest& req,
+                       const GetKeyValuesStreamReply& src,
+                       const GetKeyValuesStreamReply& tss) {
+	std::string ssResultsString = format("(%d)%s:\n", src.data.size(), src.more ? "+" : "");
+	for (auto& it : src.data) {
+		ssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value);
+	}
+
+	std::string tssResultsString = format("(%d)%s:\n", tss.data.size(), tss.more ? "+" : "");
+	for (auto& it : tss.data) {
+		tssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value);
+	}
+	event
+	    .detail(
+	        "Begin",
+	        format("%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset))
+	    .detail("End",
+	            format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset))
+	    .detail("Version", req.version)
+	    .detail("Limit", req.limit)
+	    .detail("LimitBytes", req.limitBytes)
+	    .setMaxFieldLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE * 4 / 10)
 	    .detail("SSReply", ssResultsString)
 	    .detail("TSSReply", tssResultsString);
 }
@@ -290,6 +331,9 @@ void TSSMetrics::recordLatency(const ReadHotSubRangeRequest& req, double ssLaten
 template <>
 void TSSMetrics::recordLatency(const SplitRangeRequest& req, double ssLatency, double tssLatency) {}
 
+template <>
+void TSSMetrics::recordLatency(const GetKeyValuesStreamRequest& req, double ssLatency, double tssLatency) {}
+
 // -------------------
 
 TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") {

fdbclient/TaskBucket.actor.cpp

@@ -22,6 +22,11 @@
 #include "fdbclient/ReadYourWrites.h"
 #include "flow/actorcompiler.h" // has to be last include
 
+FDB_DEFINE_BOOLEAN_PARAM(AccessSystemKeys);
+FDB_DEFINE_BOOLEAN_PARAM(PriorityBatch);
+FDB_DEFINE_BOOLEAN_PARAM(VerifyTask);
+FDB_DEFINE_BOOLEAN_PARAM(UpdateParams);
+
 Reference<TaskFuture> Task::getDoneFuture(Reference<FutureBucket> fb) {
 	return fb->unpack(params[reservedTaskParamKeyDone]);
 }
@@ -168,14 +173,14 @@ public:
 
 		{
 			// Get a task key that is <= a random UID task key, if successful then return it
-			Key k = wait(tr->getKey(lastLessOrEqual(space.pack(uid)), true));
+			Key k = wait(tr->getKey(lastLessOrEqual(space.pack(uid)), Snapshot::TRUE));
 			if (space.contains(k))
 				return Optional<Key>(k);
 		}
 
 		{
 			// Get a task key that is <= the maximum possible UID, if successful return it.
-			Key k = wait(tr->getKey(lastLessOrEqual(space.pack(maxUIDKey)), true));
+			Key k = wait(tr->getKey(lastLessOrEqual(space.pack(maxUIDKey)), Snapshot::TRUE));
 			if (space.contains(k))
 				return Optional<Key>(k);
 		}
@@ -328,7 +333,7 @@ public:
 	                                        Reference<FutureBucket> futureBucket,
 	                                        Reference<Task> task,
 	                                        Reference<TaskFuncBase> taskFunc,
-	                                        bool verifyTask) {
+	                                        VerifyTask verifyTask) {
 		bool isFinished = wait(taskBucket->isFinished(tr, task));
 		if (isFinished) {
 			return Void();
@@ -390,7 +395,7 @@ public:
 					taskBucket->setOptions(tr);
 
 					// Attempt to extend the task's timeout
-					state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, false));
+					state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, UpdateParams::FALSE));
 					wait(tr->commit());
 					task->timeoutVersion = newTimeout;
 					versionNow = tr->getCommittedVersion();
@@ -406,15 +411,16 @@ public:
 	                                 Reference<TaskBucket> taskBucket,
 	                                 Reference<FutureBucket> futureBucket,
 	                                 Reference<Task> task) {
+		state Reference<TaskFuncBase> taskFunc;
+		state VerifyTask verifyTask = false;
+
 		if (!task || !TaskFuncBase::isValidTask(task))
 			return false;
 
-		state Reference<TaskFuncBase> taskFunc;
-
 		try {
 			taskFunc = TaskFuncBase::create(task->params[Task::reservedTaskParamKeyType]);
 			if (taskFunc) {
-				state bool verifyTask = (task->params.find(Task::reservedTaskParamValidKey) != task->params.end());
+				verifyTask.set(task->params.find(Task::reservedTaskParamValidKey) != task->params.end());
 
 				if (verifyTask) {
 					loop {
@@ -472,7 +478,7 @@ public:
 	ACTOR static Future<Void> dispatch(Database cx,
 	                                   Reference<TaskBucket> taskBucket,
 	                                   Reference<FutureBucket> futureBucket,
-	                                   double* pollDelay,
+	                                   std::shared_ptr<double const> pollDelay,
 	                                   int maxConcurrentTasks) {
 		state std::vector<Future<bool>> tasks(maxConcurrentTasks);
 		for (auto& f : tasks)
@@ -569,7 +575,7 @@ public:
 	ACTOR static Future<Void> run(Database cx,
 	                              Reference<TaskBucket> taskBucket,
 	                              Reference<FutureBucket> futureBucket,
-	                              double* pollDelay,
+	                              std::shared_ptr<double const> pollDelay,
 	                              int maxConcurrentTasks) {
 		state Reference<AsyncVar<bool>> paused = makeReference<AsyncVar<bool>>(true);
 		state Future<Void> watchPausedFuture = watchPaused(cx, taskBucket, paused);
@@ -812,7 +818,7 @@ public:
 	ACTOR static Future<Version> extendTimeout(Reference<ReadYourWritesTransaction> tr,
 	                                           Reference<TaskBucket> taskBucket,
 	                                           Reference<Task> task,
-	                                           bool updateParams,
+	                                           UpdateParams updateParams,
 	                                           Version newTimeoutVersion) {
 		taskBucket->setOptions(tr);
 
@@ -863,11 +869,14 @@ public:
 	}
 };
 
-TaskBucket::TaskBucket(const Subspace& subspace, bool sysAccess, bool priorityBatch, bool lockAware)
+TaskBucket::TaskBucket(const Subspace& subspace,
+                       AccessSystemKeys sysAccess,
+                       PriorityBatch priorityBatch,
+                       LockAware lockAware)
   : prefix(subspace), active(prefix.get(LiteralStringRef("ac"))), available(prefix.get(LiteralStringRef("av"))),
     available_prioritized(prefix.get(LiteralStringRef("avp"))), timeouts(prefix.get(LiteralStringRef("to"))),
     pauseKey(prefix.pack(LiteralStringRef("pause"))), timeout(CLIENT_KNOBS->TASKBUCKET_TIMEOUT_VERSIONS),
-    system_access(sysAccess), priority_batch(priorityBatch), lock_aware(lockAware), cc("TaskBucket"),
+    system_access(sysAccess), priority_batch(priorityBatch), lockAware(lockAware), cc("TaskBucket"),
     dbgid(deterministicRandom()->randomUniqueID()), dispatchSlotChecksStarted("DispatchSlotChecksStarted", cc),
     dispatchErrors("DispatchErrors", cc), dispatchDoTasks("DispatchDoTasks", cc),
     dispatchEmptyTasks("DispatchEmptyTasks", cc), dispatchSlotChecksComplete("DispatchSlotChecksComplete", cc) {}
@@ -971,7 +980,7 @@ Future<bool> TaskBucket::doTask(Database cx, Reference<FutureBucket> futureBucke
 
 Future<Void> TaskBucket::run(Database cx,
                              Reference<FutureBucket> futureBucket,
-                             double* pollDelay,
+                             std::shared_ptr<double const> pollDelay,
                              int maxConcurrentTasks) {
 	return TaskBucketImpl::run(cx, Reference<TaskBucket>::addRef(this), futureBucket, pollDelay, maxConcurrentTasks);
 }
@@ -1001,7 +1010,7 @@ Future<Void> TaskBucket::finish(Reference<ReadYourWritesTransaction> tr, Referen
 
 Future<Version> TaskBucket::extendTimeout(Reference<ReadYourWritesTransaction> tr,
                                           Reference<Task> task,
-                                          bool updateParams,
+                                          UpdateParams updateParams,
                                           Version newTimeoutVersion) {
 	return TaskBucketImpl::extendTimeout(
 	    tr, Reference<TaskBucket>::addRef(this), task, updateParams, newTimeoutVersion);
@@ -1041,8 +1050,8 @@ public:
 	}
 };
 
-FutureBucket::FutureBucket(const Subspace& subspace, bool sysAccess, bool lockAware)
-  : prefix(subspace), system_access(sysAccess), lock_aware(lockAware) {}
+FutureBucket::FutureBucket(const Subspace& subspace, AccessSystemKeys sysAccess, LockAware lockAware)
+  : prefix(subspace), system_access(sysAccess), lockAware(lockAware) {}
 
 FutureBucket::~FutureBucket() {}
 

fdbclient/TaskBucket.h

@@ -35,6 +35,11 @@
 class FutureBucket;
 class TaskFuture;
 
+FDB_DECLARE_BOOLEAN_PARAM(AccessSystemKeys);
+FDB_DECLARE_BOOLEAN_PARAM(PriorityBatch);
+FDB_DECLARE_BOOLEAN_PARAM(VerifyTask);
+FDB_DECLARE_BOOLEAN_PARAM(UpdateParams);
+
 // A Task is a set of key=value parameters that constitute a unit of work for a TaskFunc to perform.
 // The parameter keys are specific to the TaskFunc that the Task is for, except for a set of reserved
 // parameter keys which are used by TaskBucket to determine which TaskFunc to run and provide
@@ -134,13 +139,16 @@ class FutureBucket;
 // instance may declare the Task a failure and move it back to the available subspace.
 class TaskBucket : public ReferenceCounted<TaskBucket> {
 public:
-	TaskBucket(const Subspace& subspace, bool sysAccess = false, bool priorityBatch = false, bool lockAware = false);
+	TaskBucket(const Subspace& subspace,
+	           AccessSystemKeys = AccessSystemKeys::FALSE,
+	           PriorityBatch = PriorityBatch::FALSE,
+	           LockAware = LockAware::FALSE);
 	virtual ~TaskBucket();
 
 	void setOptions(Reference<ReadYourWritesTransaction> tr) {
 		if (system_access)
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		if (lock_aware)
+		if (lockAware)
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	}
 
@@ -191,7 +199,10 @@ public:
 
 	Future<bool> doOne(Database cx, Reference<FutureBucket> futureBucket);
 
-	Future<Void> run(Database cx, Reference<FutureBucket> futureBucket, double* pollDelay, int maxConcurrentTasks);
+	Future<Void> run(Database cx,
+	                 Reference<FutureBucket> futureBucket,
+	                 std::shared_ptr<double const> pollDelay,
+	                 int maxConcurrentTasks);
 	Future<Void> watchPaused(Database cx, Reference<AsyncVar<bool>> paused);
 
 	Future<bool> isEmpty(Reference<ReadYourWritesTransaction> tr);
@@ -207,11 +218,11 @@ public:
 	// Extend the task's timeout as if it just started and also save any parameter changes made to the task
 	Future<Version> extendTimeout(Reference<ReadYourWritesTransaction> tr,
 	                              Reference<Task> task,
-	                              bool updateParams,
+	                              UpdateParams updateParams,
 	                              Version newTimeoutVersion = invalidVersion);
 	Future<Void> extendTimeout(Database cx,
 	                           Reference<Task> task,
-	                           bool updateParams,
+	                           UpdateParams updateParams,
 	                           Version newTimeoutVersion = invalidVersion) {
 		return map(runRYWTransaction(cx,
 		                             [=](Reference<ReadYourWritesTransaction> tr) {
@@ -250,7 +261,7 @@ public:
 
 	bool getSystemAccess() const { return system_access; }
 
-	bool getLockAware() const { return lock_aware; }
+	bool getLockAware() const { return lockAware; }
 
 	Key getPauseKey() const { return pauseKey; }
 
@@ -293,20 +304,20 @@ private:
 	uint32_t timeout;
 	bool system_access;
 	bool priority_batch;
-	bool lock_aware;
+	bool lockAware;
 };
 
 class TaskFuture;
 
 class FutureBucket : public ReferenceCounted<FutureBucket> {
 public:
-	FutureBucket(const Subspace& subspace, bool sysAccess = false, bool lockAware = false);
+	FutureBucket(const Subspace& subspace, AccessSystemKeys = AccessSystemKeys::FALSE, LockAware = LockAware::FALSE);
 	virtual ~FutureBucket();
 
 	void setOptions(Reference<ReadYourWritesTransaction> tr) {
 		if (system_access)
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		if (lock_aware)
+		if (lockAware)
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	}
 
@@ -324,7 +335,7 @@ public:
 
 	Reference<TaskFuture> unpack(Key key);
 	bool isSystemAccess() const { return system_access; };
-	bool isLockAware() const { return lock_aware; };
+	bool isLockAware() const { return lockAware; };
 
 private:
 	friend class TaskFuture;
@@ -333,7 +344,7 @@ private:
 
 	Subspace prefix;
 	bool system_access;
-	bool lock_aware;
+	bool lockAware;
 };
 
 class TaskFuture : public ReferenceCounted<TaskFuture> {

fdbclient/ThreadSafeTransaction.cpp

@@ -122,7 +122,7 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion)
 	    [db, connFile, apiVersion]() {
 		    try {
 			    Database::createDatabase(
-			        Reference<ClusterConnectionFile>(connFile), apiVersion, false, LocalityData(), db)
+			        Reference<ClusterConnectionFile>(connFile), apiVersion, IsInternal::FALSE, LocalityData(), db)
 			        .extractPtr();
 		    } catch (Error& e) {
 			    new (db) DatabaseContext(e);
@@ -192,7 +192,7 @@ ThreadFuture<Optional<Value>> ThreadSafeTransaction::get(const KeyRef& key, bool
 	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, k, snapshot]() -> Future<Optional<Value>> {
 		tr->checkDeferredError();
-		return tr->get(k, snapshot);
+		return tr->get(k, Snapshot{ snapshot });
 	});
 }
 
@@ -202,7 +202,7 @@ ThreadFuture<Key> ThreadSafeTransaction::getKey(const KeySelectorRef& key, bool
 	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, k, snapshot]() -> Future<Key> {
 		tr->checkDeferredError();
-		return tr->getKey(k, snapshot);
+		return tr->getKey(k, Snapshot{ snapshot });
 	});
 }
 
@@ -238,7 +238,7 @@ ThreadFuture<RangeResult> ThreadSafeTransaction::getRange(const KeySelectorRef&
 	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, b, e, limit, snapshot, reverse]() -> Future<RangeResult> {
 		tr->checkDeferredError();
-		return tr->getRange(b, e, limit, snapshot, reverse);
+		return tr->getRange(b, e, limit, Snapshot{ snapshot }, Reverse{ reverse });
 	});
 }
 
@@ -253,7 +253,7 @@ ThreadFuture<RangeResult> ThreadSafeTransaction::getRange(const KeySelectorRef&
 	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, b, e, limits, snapshot, reverse]() -> Future<RangeResult> {
 		tr->checkDeferredError();
-		return tr->getRange(b, e, limits, snapshot, reverse);
+		return tr->getRange(b, e, limits, Snapshot{ snapshot }, Reverse{ reverse });
 	});
 }
 

fdbrpc/AsyncFileEncrypted.actor.cpp

@@ -0,0 +1,274 @@
+/*
+ * AsyncFileEncrypted.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbrpc/AsyncFileEncrypted.h"
+#include "flow/StreamCipher.h"
+#include "flow/UnitTest.h"
+#include "flow/xxhash.h"
+#include "flow/actorcompiler.h" // must be last include
+
+class AsyncFileEncryptedImpl {
+public:
+	// Determine the initialization for the first block of a file based on a hash of
+	// the filename.
+	static auto getFirstBlockIV(const std::string& filename) {
+		StreamCipher::IV iv;
+		auto salt = basename(filename);
+		auto pos = salt.find('.');
+		salt = salt.substr(0, pos);
+		auto hash = XXH3_128bits(salt.c_str(), salt.size());
+		auto high = reinterpret_cast<unsigned char*>(&hash.high64);
+		auto low = reinterpret_cast<unsigned char*>(&hash.low64);
+		std::copy(high, high + 8, &iv[0]);
+		std::copy(low, low + 6, &iv[8]);
+		iv[14] = iv[15] = 0; // last 16 bits identify block
+		return iv;
+	}
+
+	// Read a single block of size ENCRYPTION_BLOCK_SIZE bytes, and decrypt.
+	ACTOR static Future<Standalone<StringRef>> readBlock(AsyncFileEncrypted* self, uint16_t block) {
+		state Arena arena;
+		state unsigned char* encrypted = new (arena) unsigned char[FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE];
+		int bytes = wait(
+		    self->file->read(encrypted, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * block));
+		DecryptionStreamCipher decryptor(StreamCipher::Key::getKey(), self->getIV(block));
+		auto decrypted = decryptor.decrypt(encrypted, bytes, arena);
+		return Standalone<StringRef>(decrypted, arena);
+	}
+
+	ACTOR static Future<int> read(AsyncFileEncrypted* self, void* data, int length, int offset) {
+		state const uint16_t firstBlock = offset / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE;
+		state const uint16_t lastBlock = (offset + length - 1) / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE;
+		state uint16_t block;
+		state unsigned char* output = reinterpret_cast<unsigned char*>(data);
+		state int bytesRead = 0;
+		ASSERT(self->mode == AsyncFileEncrypted::Mode::READ_ONLY);
+		for (block = firstBlock; block <= lastBlock; ++block) {
+			state StringRef plaintext;
+
+			auto cachedBlock = self->readBuffers.get(block);
+			if (cachedBlock.present()) {
+				plaintext = cachedBlock.get();
+			} else {
+				Standalone<StringRef> _plaintext = wait(readBlock(self, block));
+				self->readBuffers.insert(block, _plaintext);
+				plaintext = _plaintext;
+			}
+			auto start = (block == firstBlock) ? plaintext.begin() + (offset % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE)
+			                                   : plaintext.begin();
+			auto end = (block == lastBlock)
+			               ? plaintext.begin() + ((offset + length) % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE)
+			               : plaintext.end();
+			if ((offset + length) % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE == 0) {
+				end = plaintext.end();
+			}
+			std::copy(start, end, output);
+			output += (end - start);
+			bytesRead += (end - start);
+		}
+		return bytesRead;
+	}
+
+	ACTOR static Future<Void> write(AsyncFileEncrypted* self, void const* data, int length, int64_t offset) {
+		ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY);
+		// All writes must append to the end of the file:
+		ASSERT_EQ(offset, self->currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + self->offsetInBlock);
+		state unsigned char const* input = reinterpret_cast<unsigned char const*>(data);
+		while (length > 0) {
+			const auto chunkSize = std::min(length, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE - self->offsetInBlock);
+			Arena arena;
+			auto encrypted = self->encryptor->encrypt(input, chunkSize, arena);
+			std::copy(encrypted.begin(), encrypted.end(), &self->writeBuffer[self->offsetInBlock]);
+			offset += encrypted.size();
+			self->offsetInBlock += chunkSize;
+			length -= chunkSize;
+			input += chunkSize;
+			if (self->offsetInBlock == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) {
+				wait(self->writeLastBlockToFile());
+				self->offsetInBlock = 0;
+				ASSERT_LT(self->currentBlock, std::numeric_limits<uint16_t>::max());
+				++self->currentBlock;
+				self->encryptor = std::make_unique<EncryptionStreamCipher>(StreamCipher::Key::getKey(),
+				                                                           self->getIV(self->currentBlock));
+			}
+		}
+		return Void();
+	}
+
+	ACTOR static Future<Void> sync(AsyncFileEncrypted* self) {
+		ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY);
+		wait(self->writeLastBlockToFile());
+		wait(self->file->sync());
+		return Void();
+	}
+
+	ACTOR static Future<Void> zeroRange(AsyncFileEncrypted* self, int64_t offset, int64_t length) {
+		ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY);
+		// TODO: Could optimize this
+		Arena arena;
+		auto zeroes = new (arena) unsigned char[length];
+		memset(zeroes, 0, length);
+		wait(self->write(zeroes, length, offset));
+		return Void();
+	}
+};
+
+AsyncFileEncrypted::AsyncFileEncrypted(Reference<IAsyncFile> file, Mode mode)
+  : file(file), mode(mode), currentBlock(0), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) {
+	firstBlockIV = AsyncFileEncryptedImpl::getFirstBlockIV(file->getFilename());
+	if (mode == Mode::APPEND_ONLY) {
+		encryptor = std::make_unique<EncryptionStreamCipher>(StreamCipher::Key::getKey(), getIV(currentBlock));
+		writeBuffer = std::vector<unsigned char>(FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, 0);
+	}
+}
+
+void AsyncFileEncrypted::addref() {
+	ReferenceCounted<AsyncFileEncrypted>::addref();
+}
+
+void AsyncFileEncrypted::delref() {
+	ReferenceCounted<AsyncFileEncrypted>::delref();
+}
+
+Future<int> AsyncFileEncrypted::read(void* data, int length, int64_t offset) {
+	return AsyncFileEncryptedImpl::read(this, data, length, offset);
+}
+
+Future<Void> AsyncFileEncrypted::write(void const* data, int length, int64_t offset) {
+	return AsyncFileEncryptedImpl::write(this, data, length, offset);
+}
+
+Future<Void> AsyncFileEncrypted::zeroRange(int64_t offset, int64_t length) {
+	return AsyncFileEncryptedImpl::zeroRange(this, offset, length);
+}
+
+Future<Void> AsyncFileEncrypted::truncate(int64_t size) {
+	ASSERT(mode == Mode::APPEND_ONLY);
+	return file->truncate(size);
+}
+
+Future<Void> AsyncFileEncrypted::sync() {
+	ASSERT(mode == Mode::APPEND_ONLY);
+	return AsyncFileEncryptedImpl::sync(this);
+}
+
+Future<Void> AsyncFileEncrypted::flush() {
+	ASSERT(mode == Mode::APPEND_ONLY);
+	return Void();
+}
+
+Future<int64_t> AsyncFileEncrypted::size() const {
+	ASSERT(mode == Mode::READ_ONLY);
+	return file->size();
+}
+
+std::string AsyncFileEncrypted::getFilename() const {
+	return file->getFilename();
+}
+
+Future<Void> AsyncFileEncrypted::readZeroCopy(void** data, int* length, int64_t offset) {
+	throw io_error();
+	return Void();
+}
+
+void AsyncFileEncrypted::releaseZeroCopy(void* data, int length, int64_t offset) {
+	throw io_error();
+}
+
+int64_t AsyncFileEncrypted::debugFD() const {
+	return file->debugFD();
+}
+
+StreamCipher::IV AsyncFileEncrypted::getIV(uint16_t block) const {
+	auto iv = firstBlockIV;
+	iv[14] = block / 256;
+	iv[15] = block % 256;
+	return iv;
+}
+
+Future<Void> AsyncFileEncrypted::writeLastBlockToFile() {
+	return file->write(&writeBuffer[0], offsetInBlock, currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE);
+}
+
+size_t AsyncFileEncrypted::RandomCache::evict() {
+	ASSERT_EQ(vec.size(), maxSize);
+	auto index = deterministicRandom()->randomInt(0, maxSize);
+	hashMap.erase(vec[index]);
+	return index;
+}
+
+AsyncFileEncrypted::RandomCache::RandomCache(size_t maxSize) : maxSize(maxSize) {
+	vec.reserve(maxSize);
+}
+
+void AsyncFileEncrypted::RandomCache::insert(uint16_t block, const Standalone<StringRef>& value) {
+	auto [_, found] = hashMap.insert({ block, value });
+	if (found) {
+		return;
+	} else if (vec.size() < maxSize) {
+		vec.push_back(block);
+	} else {
+		auto index = evict();
+		vec[index] = block;
+	}
+}
+
+Optional<Standalone<StringRef>> AsyncFileEncrypted::RandomCache::get(uint16_t block) const {
+	auto it = hashMap.find(block);
+	if (it == hashMap.end()) {
+		return {};
+	} else {
+		return it->second;
+	}
+}
+
+// This test writes random data into an encrypted file in random increments,
+// then reads this data back from the file in random increments, then confirms that
+// the bytes read match the bytes written.
+TEST_CASE("fdbrpc/AsyncFileEncrypted") {
+	state const int bytes = FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * deterministicRandom()->randomInt(0, 1000);
+	state std::vector<unsigned char> writeBuffer(bytes, 0);
+	generateRandomData(&writeBuffer.front(), bytes);
+	state std::vector<unsigned char> readBuffer(bytes, 0);
+	ASSERT(g_network->isSimulated());
+	StreamCipher::Key::initializeRandomTestKey();
+	int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE |
+	            IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_ENCRYPTED | IAsyncFile::OPEN_UNCACHED |
+	            IAsyncFile::OPEN_NO_AIO;
+	state Reference<IAsyncFile> file =
+	    wait(IAsyncFileSystem::filesystem()->open(joinPath(params.getDataDir(), "test-encrypted-file"), flags, 0600));
+	state int bytesWritten = 0;
+	while (bytesWritten < bytes) {
+		chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesWritten);
+		wait(file->write(&writeBuffer[bytesWritten], chunkSize, bytesWritten));
+		bytesWritten += chunkSize;
+	}
+	wait(file->sync());
+	state int bytesRead = 0;
+	state int chunkSize;
+	while (bytesRead < bytes) {
+		chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesRead);
+		int bytesReadInChunk = wait(file->read(&readBuffer[bytesRead], chunkSize, bytesRead));
+		ASSERT_EQ(bytesReadInChunk, chunkSize);
+		bytesRead += bytesReadInChunk;
+	}
+	ASSERT(writeBuffer == readBuffer);
+	return Void();
+}

fdbrpc/AsyncFileEncrypted.h

@@ -0,0 +1,81 @@
+/*
+ * AsyncFileEncrypted.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbrpc/IAsyncFile.h"
+#include "flow/FastRef.h"
+#include "flow/flow.h"
+#include "flow/IRandom.h"
+#include "flow/StreamCipher.h"
+
+#include <array>
+
+/*
+ * Append-only file encrypted using AES-128-GCM.
+ * */
+class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted<AsyncFileEncrypted> {
+public:
+	enum class Mode { APPEND_ONLY, READ_ONLY };
+
+private:
+	Reference<IAsyncFile> file;
+	StreamCipher::IV firstBlockIV;
+	StreamCipher::IV getIV(uint16_t block) const;
+	Mode mode;
+	Future<Void> writeLastBlockToFile();
+	friend class AsyncFileEncryptedImpl;
+
+	// Reading:
+	class RandomCache {
+		size_t maxSize;
+		std::vector<uint16_t> vec;
+		std::unordered_map<uint16_t, Standalone<StringRef>> hashMap;
+		size_t evict();
+
+	public:
+		RandomCache(size_t maxSize);
+		void insert(uint16_t block, const Standalone<StringRef>& value);
+		Optional<Standalone<StringRef>> get(uint16_t block) const;
+	} readBuffers;
+
+	// Writing (append only):
+	std::unique_ptr<EncryptionStreamCipher> encryptor;
+	uint16_t currentBlock{ 0 };
+	int offsetInBlock{ 0 };
+	std::vector<unsigned char> writeBuffer;
+	Future<Void> initialize();
+
+public:
+	AsyncFileEncrypted(Reference<IAsyncFile>, Mode);
+	void addref() override;
+	void delref() override;
+	Future<int> read(void* data, int length, int64_t offset) override;
+	Future<Void> write(void const* data, int length, int64_t offset) override;
+	Future<Void> zeroRange(int64_t offset, int64_t length) override;
+	Future<Void> truncate(int64_t size) override;
+	Future<Void> sync() override;
+	Future<Void> flush() override;
+	Future<int64_t> size() const override;
+	std::string getFilename() const override;
+	Future<Void> readZeroCopy(void** data, int* length, int64_t offset) override;
+	void releaseZeroCopy(void* data, int length, int64_t offset) override;
+	int64_t debugFD() const override;
+};

fdbrpc/CMakeLists.txt

@@ -33,6 +33,13 @@ set(FDBRPC_SRCS
   TraceFileIO.cpp
   TSSComparison.h)
 
+if(WITH_TLS AND NOT WIN32)
+  set(FDBRPC_SRCS
+    ${FDBRPC_SRCS}
+    AsyncFileEncrypted.h
+    AsyncFileEncrypted.actor.cpp)
+endif()
+
 set(COMPILE_EIO OFF)
 
 if(NOT WIN32)

fdbrpc/IAsyncFile.h

@@ -53,7 +53,8 @@ public:
 		OPEN_LARGE_PAGES = 0x100000,
 		OPEN_NO_AIO =
 		    0x200000, // Don't use AsyncFileKAIO or similar implementations that rely on filesystem support for AIO
-		OPEN_CACHED_READ_ONLY = 0x400000 // AsyncFileCached opens files read/write even if you specify read only
+		OPEN_CACHED_READ_ONLY = 0x400000, // AsyncFileCached opens files read/write even if you specify read only
+		OPEN_ENCRYPTED = 0x800000 // File is encrypted using AES-128-GCM (must be either read-only or write-only)
 	};
 
 	virtual void addref() = 0;

fdbrpc/LoadBalance.actor.cpp

@@ -18,9 +18,13 @@
  * limitations under the License.
  */
 
+#include "fdbrpc/LoadBalance.actor.h"
 #include "flow/flow.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+FDB_DEFINE_BOOLEAN_PARAM(AtMostOnce);
+FDB_DEFINE_BOOLEAN_PARAM(TriedAllOptions);
+
 // Throwing all_alternatives_failed will cause the client to issue a GetKeyLocationRequest to the proxy, so this actor
 // attempts to limit the number of these errors thrown by a single client to prevent it from saturating the proxies with
 // these requests
@@ -49,4 +53,4 @@ ACTOR Future<Void> allAlternativesFailedDelay(Future<Void> okFuture) {
 		when(wait(::delayJittered(delay))) { throw all_alternatives_failed(); }
 	}
 	return Void();
-}
+}

fdbrpc/LoadBalance.actor.h

@@ -28,6 +28,7 @@
 #elif !defined(FLOW_LOADBALANCE_ACTOR_H)
 #define FLOW_LOADBALANCE_ACTOR_H
 
+#include "flow/BooleanParam.h"
 #include "flow/flow.h"
 #include "flow/Knobs.h"
 
@@ -147,6 +148,7 @@ Future<Void> tssComparison(Req req,
 				        ? SevWarnAlways
 				        : SevError,
 				    TSS_mismatchTraceName(req));
+				mismatchEvent.setMaxEventLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE);
 				mismatchEvent.detail("TSSID", tssData.tssId);
 
 				if (FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS && ssTeam->size() > 1) {
@@ -238,6 +240,9 @@ Future<Void> tssComparison(Req req,
 	return Void();
 }
 
+FDB_DECLARE_BOOLEAN_PARAM(AtMostOnce);
+FDB_DECLARE_BOOLEAN_PARAM(TriedAllOptions);
+
 // Stores state for a request made by the load balancer
 template <class Request, class Interface, class Multi>
 struct RequestData : NonCopyable {
@@ -245,7 +250,7 @@ struct RequestData : NonCopyable {
 
 	Future<Reply> response;
 	Reference<ModelHolder> modelHolder;
-	bool triedAllOptions = false;
+	TriedAllOptions triedAllOptions{ false };
 
 	bool requestStarted = false; // true once the request has been sent to an alternative
 	bool requestProcessed = false; // true once a response has been received and handled by checkAndProcessResult
@@ -284,7 +289,7 @@ struct RequestData : NonCopyable {
 	// Initializes the request state and starts it, possibly after a backoff delay
 	void startRequest(
 	    double backoff,
-	    bool triedAllOptions,
+	    TriedAllOptions triedAllOptions,
 	    RequestStream<Request> const* stream,
 	    Request& request,
 	    QueueModel* model,
@@ -320,8 +325,8 @@ struct RequestData : NonCopyable {
 	// A return value with an error means that the error should be thrown back to original caller
 	static ErrorOr<bool> checkAndProcessResultImpl(Reply const& result,
 	                                               Reference<ModelHolder> modelHolder,
-	                                               bool atMostOnce,
-	                                               bool triedAllOptions) {
+	                                               AtMostOnce atMostOnce,
+	                                               TriedAllOptions triedAllOptions) {
 		ASSERT(modelHolder);
 
 		Optional<LoadBalancedReply> loadBalancedReply;
@@ -377,7 +382,7 @@ struct RequestData : NonCopyable {
 	// A return value of true means that the request completed successfully
 	// A return value of false means that the request failed but should be retried
 	// In the event of a non-retryable failure, an error is thrown indicating the failure
-	bool checkAndProcessResult(bool atMostOnce) {
+	bool checkAndProcessResult(AtMostOnce atMostOnce) {
 		ASSERT(response.isReady());
 		requestProcessed = true;
 
@@ -412,9 +417,9 @@ struct RequestData : NonCopyable {
 
 		// We need to process the lagging request in order to update the queue model
 		Reference<ModelHolder> holderCapture = std::move(modelHolder);
-		bool triedAllOptionsCapture = triedAllOptions;
+		auto triedAllOptionsCapture = triedAllOptions;
 		Future<Void> updateModel = map(response, [holderCapture, triedAllOptionsCapture](Reply result) {
-			checkAndProcessResultImpl(result, holderCapture, false, triedAllOptionsCapture);
+			checkAndProcessResultImpl(result, holderCapture, AtMostOnce::FALSE, triedAllOptionsCapture);
 			return Void();
 		});
 		model->addActor.send(updateModel);
@@ -441,7 +446,8 @@ Future<REPLY_TYPE(Request)> loadBalance(
     RequestStream<Request> Interface::*channel,
     Request request = Request(),
     TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint,
-    bool atMostOnce = false, // if true, throws request_maybe_delivered() instead of retrying automatically
+    AtMostOnce atMostOnce =
+        AtMostOnce::FALSE, // if true, throws request_maybe_delivered() instead of retrying automatically
     QueueModel* model = nullptr) {
 
 	state RequestData<Request, Interface, Multi> firstRequestData;
@@ -453,6 +459,8 @@ Future<REPLY_TYPE(Request)> loadBalance(
 	state Promise<Void> requestFinished;
 	state double startTime = now();
 
+	state TriedAllOptions triedAllOptions = TriedAllOptions::FALSE;
+
 	setReplyPriority(request, taskID);
 	if (!alternatives)
 		return Never();
@@ -556,7 +564,6 @@ Future<REPLY_TYPE(Request)> loadBalance(
 
 	state int numAttempts = 0;
 	state double backoff = 0;
-	state bool triedAllOptions = false;
 	// Issue requests to selected servers.
 	loop {
 		if (now() - startTime > (g_network->isSimulated() ? 30.0 : 600.0)) {
@@ -595,7 +602,7 @@ Future<REPLY_TYPE(Request)> loadBalance(
 				break;
 			nextAlt = (nextAlt + 1) % alternatives->size();
 			if (nextAlt == startAlt)
-				triedAllOptions = true;
+				triedAllOptions = TriedAllOptions::TRUE;
 			stream = nullptr;
 		}
 
@@ -702,7 +709,7 @@ Future<REPLY_TYPE(Request)> loadBalance(
 
 		nextAlt = (nextAlt + 1) % alternatives->size();
 		if (nextAlt == startAlt)
-			triedAllOptions = true;
+			triedAllOptions = TriedAllOptions::TRUE;
 		resetReply(request, taskID);
 		secondDelay = Never();
 	}
@@ -724,7 +731,7 @@ Future<REPLY_TYPE(Request)> basicLoadBalance(Reference<ModelInterface<Multi>> al
                                              RequestStream<Request> Interface::*channel,
                                              Request request = Request(),
                                              TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint,
-                                             bool atMostOnce = false) {
+                                             AtMostOnce atMostOnce = AtMostOnce::FALSE) {
 	setReplyPriority(request, taskID);
 	if (!alternatives)
 		return Never();

fdbrpc/Net2FileSystem.cpp

@@ -32,6 +32,9 @@
 
 #include "fdbrpc/AsyncFileCached.actor.h"
 #include "fdbrpc/AsyncFileEIO.actor.h"
+#if (!defined(TLS_DISABLED) && !defined(_WIN32))
+#include "fdbrpc/AsyncFileEncrypted.h"
+#endif
 #include "fdbrpc/AsyncFileWinASIO.actor.h"
 #include "fdbrpc/AsyncFileKAIO.actor.h"
 #include "flow/AsioReactor.h"
@@ -76,6 +79,14 @@ Future<Reference<class IAsyncFile>> Net2FileSystem::open(const std::string& file
 		    static_cast<boost::asio::io_service*>((void*)g_network->global(INetwork::enASIOService)));
 	if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0)
 		f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
+#if (!defined(TLS_DISABLED) && !defined(_WIN32))
+	if (flags & IAsyncFile::OPEN_ENCRYPTED)
+		f = map(f, [flags](Reference<IAsyncFile> r) {
+			auto mode = flags & IAsyncFile::OPEN_READWRITE ? AsyncFileEncrypted::Mode::APPEND_ONLY
+			                                               : AsyncFileEncrypted::Mode::READ_ONLY;
+			return Reference<IAsyncFile>(new AsyncFileEncrypted(r, mode));
+		});
+#endif
 	return f;
 }
 

fdbrpc/TSSComparison.h

@@ -41,6 +41,7 @@ struct DetailedTSSMismatch {
 struct TSSMetrics : ReferenceCounted<TSSMetrics>, NonCopyable {
 	CounterCollection cc;
 	Counter requests;
+	Counter streamComparisons;
 	Counter ssErrors;
 	Counter tssErrors;
 	Counter tssTimeouts;
@@ -99,9 +100,10 @@ struct TSSMetrics : ReferenceCounted<TSSMetrics>, NonCopyable {
 	}
 
 	TSSMetrics()
-	  : cc("TSSClientMetrics"), requests("Requests", cc), ssErrors("SSErrors", cc), tssErrors("TSSErrors", cc),
-	    tssTimeouts("TSSTimeouts", cc), mismatches("Mismatches", cc), SSgetValueLatency(1000), SSgetKeyLatency(1000),
-	    SSgetKeyValuesLatency(1000), TSSgetValueLatency(1000), TSSgetKeyLatency(1000), TSSgetKeyValuesLatency(1000) {}
+	  : cc("TSSClientMetrics"), requests("Requests", cc), streamComparisons("StreamComparisons", cc),
+	    ssErrors("SSErrors", cc), tssErrors("TSSErrors", cc), tssTimeouts("TSSTimeouts", cc),
+	    mismatches("Mismatches", cc), SSgetValueLatency(1000), SSgetKeyLatency(1000), SSgetKeyValuesLatency(1000),
+	    TSSgetValueLatency(1000), TSSgetKeyLatency(1000), TSSgetKeyValuesLatency(1000) {}
 };
 
 template <class Rep>

fdbrpc/fdbrpc.h

@@ -533,6 +533,8 @@ public:
 		}
 	}
 
+	void reset() { *this = ReplyPromiseStream<T>(); }
+
 private:
 	NetNotifiedQueueWithAcknowledgements<T>* queue;
 	SAV<Void>* errors;

fdbrpc/sim2.actor.cpp

@@ -33,6 +33,9 @@
 #include "flow/Util.h"
 #include "fdbrpc/IAsyncFile.h"
 #include "fdbrpc/AsyncFileCached.actor.h"
+#if (!defined(TLS_DISABLED) && !defined(_WIN32))
+#include "fdbrpc/AsyncFileEncrypted.h"
+#endif
 #include "fdbrpc/AsyncFileNonDurable.actor.h"
 #include "flow/crc32c.h"
 #include "fdbrpc/TraceFileIO.h"
@@ -2473,6 +2476,14 @@ Future<Reference<class IAsyncFile>> Sim2FileSystem::open(const std::string& file
 		f = AsyncFileDetachable::open(f);
 		if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0)
 			f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
+#if (!defined(TLS_DISABLED) && !defined(_WIN32))
+		if (flags & IAsyncFile::OPEN_ENCRYPTED)
+			f = map(f, [flags](Reference<IAsyncFile> r) {
+				auto mode = flags & IAsyncFile::OPEN_READWRITE ? AsyncFileEncrypted::Mode::APPEND_ONLY
+				                                               : AsyncFileEncrypted::Mode::READ_ONLY;
+				return Reference<IAsyncFile>(new AsyncFileEncrypted(r, mode));
+			});
+#endif
 		return f;
 	} else
 		return AsyncFileCached::open(filename, flags, mode);

fdbserver/BackupWorker.actor.cpp

@@ -243,7 +243,7 @@ struct BackupData {
 	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion - 1), popVersion(req.startVersion - 1),
 	    cc("BackupWorker", myId.toString()), pulledVersion(0), paused(false),
 	    lock(new FlowLock(SERVER_KNOBS->BACKUP_LOCK_BYTES)) {
-		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
+		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE);
 
 		specialCounter(cc, "SavedVersion", [this]() { return this->savedVersion; });
 		specialCounter(cc, "MinKnownCommittedVersion", [this]() { return this->minKnownCommittedVersion; });

fdbserver/ClusterController.actor.cpp

@@ -133,9 +133,9 @@ public:
 		    serverInfo(new AsyncVar<ServerDBInfo>()), db(DatabaseContext::create(clientInfo,
 		                                                                         Future<Void>(),
 		                                                                         LocalityData(),
-		                                                                         true,
+		                                                                         EnableLocalityLoadBalance::TRUE,
 		                                                                         TaskPriority::DefaultEndpoint,
-		                                                                         true)) // SOMEDAY: Locality!
+		                                                                         LockAware::TRUE)) // SOMEDAY: Locality!
 		{}
 
 		void setDistributor(const DataDistributorInterface& interf) {
@@ -289,6 +289,7 @@ public:
 		for (auto& it : id_worker) {
 			auto fitness = it.second.details.processClass.machineClassFitness(ProcessClass::Storage);
 			if (workerAvailable(it.second, false) && !conf.isExcludedServer(it.second.details.interf.addresses()) &&
+			    !isExcludedDegradedServer(it.second.details.interf.addresses()) &&
 			    fitness != ProcessClass::NeverAssign &&
 			    (!dcId.present() || it.second.details.interf.locality.dcId() == dcId.get())) {
 				fitness_workers[fitness].push_back(it.second.details);
@@ -529,6 +530,16 @@ public:
 				                     dcIds);
 				continue;
 			}
+			if (isExcludedDegradedServer(worker_details.interf.addresses())) {
+				logWorkerUnavailable(SevInfo,
+				                     id,
+				                     "complex",
+				                     "Worker server is excluded from the cluster due to degradation",
+				                     worker_details,
+				                     fitness,
+				                     dcIds);
+				continue;
+			}
 			if (fitness == ProcessClass::NeverAssign) {
 				logWorkerUnavailable(
 				    SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds);
@@ -764,6 +775,16 @@ public:
 				                     dcIds);
 				continue;
 			}
+			if (isExcludedDegradedServer(worker_details.interf.addresses())) {
+				logWorkerUnavailable(SevInfo,
+				                     id,
+				                     "simple",
+				                     "Worker server is excluded from the cluster due to degradation",
+				                     worker_details,
+				                     fitness,
+				                     dcIds);
+				continue;
+			}
 			if (fitness == ProcessClass::NeverAssign) {
 				logWorkerUnavailable(
 				    SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds);
@@ -897,6 +918,16 @@ public:
 				                     dcIds);
 				continue;
 			}
+			if (isExcludedDegradedServer(worker_details.interf.addresses())) {
+				logWorkerUnavailable(SevInfo,
+				                     id,
+				                     "deprecated",
+				                     "Worker server is excluded from the cluster due to degradation",
+				                     worker_details,
+				                     fitness,
+				                     dcIds);
+				continue;
+			}
 			if (fitness == ProcessClass::NeverAssign) {
 				logWorkerUnavailable(
 				    SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds);
@@ -1312,7 +1343,8 @@ public:
 
 		for (auto& it : id_worker) {
 			auto fitness = it.second.details.processClass.machineClassFitness(role);
-			if (conf.isExcludedServer(it.second.details.interf.addresses())) {
+			if (conf.isExcludedServer(it.second.details.interf.addresses()) ||
+			    isExcludedDegradedServer(it.second.details.interf.addresses())) {
 				fitness = std::max(fitness, ProcessClass::ExcludeFit);
 			}
 			if (workerAvailable(it.second, checkStable) && fitness < unacceptableFitness &&
@@ -1359,6 +1391,7 @@ public:
 			auto fitness = it.second.details.processClass.machineClassFitness(role);
 			if (workerAvailable(it.second, checkStable) &&
 			    !conf.isExcludedServer(it.second.details.interf.addresses()) &&
+			    !isExcludedDegradedServer(it.second.details.interf.addresses()) &&
 			    it.second.details.interf.locality.dcId() == dcId &&
 			    (!minWorker.present() ||
 			     (it.second.details.interf.id() != minWorker.get().worker.interf.id() &&
@@ -1493,7 +1526,9 @@ public:
 	                                                         bool checkStable = false) {
 		std::set<Optional<Standalone<StringRef>>> result;
 		for (auto& it : id_worker)
-			if (workerAvailable(it.second, checkStable) && !conf.isExcludedServer(it.second.details.interf.addresses()))
+			if (workerAvailable(it.second, checkStable) &&
+			    !conf.isExcludedServer(it.second.details.interf.addresses()) &&
+			    !isExcludedDegradedServer(it.second.details.interf.addresses()))
 				result.insert(it.second.details.interf.locality.dcId());
 		return result;
 	}
@@ -2779,6 +2814,162 @@ public:
         }
     }
 
+	// Checks that if any worker or their degraded peers have recovered. If so, remove them from `workerHealth`.
+	void updateRecoveredWorkers() {
+		double currentTime = now();
+		for (auto& [workerAddress, health] : workerHealth) {
+			for (auto it = health.degradedPeers.begin(); it != health.degradedPeers.end();) {
+				if (currentTime - it->second.lastRefreshTime > SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL) {
+					TraceEvent("WorkerPeerHealthRecovered").detail("Worker", workerAddress).detail("Peer", it->first);
+					health.degradedPeers.erase(it++);
+				} else {
+					++it;
+				}
+			}
+		}
+
+		for (auto it = workerHealth.begin(); it != workerHealth.end();) {
+			if (it->second.degradedPeers.empty()) {
+				TraceEvent("WorkerAllPeerHealthRecovered").detail("Worker", it->first);
+				workerHealth.erase(it++);
+			} else {
+				++it;
+			}
+		}
+	}
+
+	// Returns a list of servers who are experiencing degraded links. These are candidates to perform exclusion. Note
+	// that only one endpoint of a bad link will be included in this list.
+	std::unordered_set<NetworkAddress> getServersWithDegradedLink() {
+		updateRecoveredWorkers();
+
+		// Build a map keyed by measured degraded peer. This map gives the info that who complains a particular server.
+		std::unordered_map<NetworkAddress, std::unordered_set<NetworkAddress>> degradedLinkDst2Src;
+		double currentTime = now();
+		for (const auto& [server, health] : workerHealth) {
+			for (const auto& [degradedPeer, times] : health.degradedPeers) {
+				if (currentTime - times.startTime < SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL) {
+					// This degraded link is not long enough to be considered as degraded.
+					continue;
+				}
+				degradedLinkDst2Src[degradedPeer].insert(server);
+			}
+		}
+
+		// Sort degraded peers based on the number of workers complaining about it.
+		std::vector<std::pair<int, NetworkAddress>> count2DegradedPeer;
+		for (const auto& [degradedPeer, complainers] : degradedLinkDst2Src) {
+			count2DegradedPeer.push_back({ complainers.size(), degradedPeer });
+		}
+		std::sort(count2DegradedPeer.begin(), count2DegradedPeer.end(), std::greater<>());
+
+		// Go through all reported degraded peers by decreasing order of the number of complainers. For a particular
+		// degraded peer, if a complainer has already be considered as degraded, we skip the current examine degraded
+		// peer since there has been one endpoint on the link between degradedPeer and complainer considered as
+		// degraded. This is to address the issue that both endpoints on a bad link may be considered as degraded
+		// server.
+		//
+		// For example, if server A is already considered as a degraded server, and A complains B, we won't add B as
+		// degraded since A is already considered as degraded.
+		std::unordered_set<NetworkAddress> currentDegradedServers;
+		for (const auto& [complainerCount, badServer] : count2DegradedPeer) {
+			for (const auto& complainer : degradedLinkDst2Src[badServer]) {
+				if (currentDegradedServers.find(complainer) == currentDegradedServers.end()) {
+					currentDegradedServers.insert(badServer);
+					break;
+				}
+			}
+		}
+
+		// For degraded server that are complained by more than SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE, we
+		// don't know if it is a hot server, or the network is bad. We remove from the returned degraded server list.
+		std::unordered_set<NetworkAddress> currentDegradedServersWithinLimit;
+		for (const auto& badServer : currentDegradedServers) {
+			if (degradedLinkDst2Src[badServer].size() <= SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE) {
+				currentDegradedServersWithinLimit.insert(badServer);
+			}
+		}
+		return currentDegradedServersWithinLimit;
+	}
+
+	// Returns true when the cluster controller should trigger a recovery due to degraded servers are used in the
+	// transaction system in the primary data center.
+	bool shouldTriggerRecoveryDueToDegradedServers() {
+		if (degradedServers.size() > SERVER_KNOBS->CC_MAX_EXCLUSION_DUE_TO_HEALTH) {
+			return false;
+		}
+
+		const ServerDBInfo dbi = db.serverInfo->get();
+		if (dbi.recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+			return false;
+		}
+
+		// Do not trigger recovery if the cluster controller is excluded, since the master will change
+		// anyways once the cluster controller is moved
+		if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) {
+			return false;
+		}
+
+		if (db.config.regions.size() > 1 && db.config.regions[0].priority > db.config.regions[1].priority &&
+		    db.config.regions[0].dcId != clusterControllerDcId.get() && versionDifferenceUpdated &&
+		    datacenterVersionDifference < SERVER_KNOBS->MAX_VERSION_DIFFERENCE) {
+			checkRegions(db.config.regions);
+		}
+
+		for (const auto& excludedServer : degradedServers) {
+			if (dbi.master.addresses().contains(excludedServer)) {
+				return true;
+			}
+
+			for (auto& logSet : dbi.logSystemConfig.tLogs) {
+				if (!logSet.isLocal || logSet.locality == tagLocalitySatellite) {
+					continue;
+				}
+				for (const auto& tlog : logSet.tLogs) {
+					if (tlog.present() && tlog.interf().addresses().contains(excludedServer)) {
+						return true;
+					}
+				}
+			}
+
+			for (auto& proxy : dbi.client.grvProxies) {
+				if (proxy.addresses().contains(excludedServer)) {
+					return true;
+				}
+			}
+
+			for (auto& proxy : dbi.client.commitProxies) {
+				if (proxy.addresses().contains(excludedServer)) {
+					return true;
+				}
+			}
+
+			for (auto& resolver : dbi.resolvers) {
+				if (resolver.addresses().contains(excludedServer)) {
+					return true;
+				}
+			}
+		}
+
+		return false;
+	}
+
+	int recentRecoveryCountDueToHealth() {
+		while (!recentHealthTriggeredRecoveryTime.empty() &&
+		       now() - recentHealthTriggeredRecoveryTime.front() > SERVER_KNOBS->CC_TRACKING_HEALTH_RECOVERY_INTERVAL) {
+			recentHealthTriggeredRecoveryTime.pop();
+		}
+		return recentHealthTriggeredRecoveryTime.size();
+	}
+
+	bool isExcludedDegradedServer(const NetworkAddressList& a) {
+		for (const auto& server : excludedDegradedServers) {
+			if (a.contains(server))
+				return true;
+		}
+		return false;
+	}
+
 	std::map<Optional<Standalone<StringRef>>, WorkerInfo> id_worker;
 	std::map<Optional<Standalone<StringRef>>, ProcessClass>
 	    id_class; // contains the mapping from process id to process class from the database
@@ -2828,6 +3019,12 @@ public:
         // TODO(zhewu): Include disk and CPU signals.
     };
 	std::unordered_map<NetworkAddress, WorkerHealth> workerHealth;
+	std::unordered_set<NetworkAddress>
+	    degradedServers; // The servers that the cluster controller is considered as degraded. The servers in this list
+	                     // are not excluded unless they are added to `excludedDegradedServers`.
+	std::unordered_set<NetworkAddress>
+	    excludedDegradedServers; // The degraded servers to be excluded when assigning workers to roles.
+	std::queue<double> recentHealthTriggeredRecoveryTime;
 
 	CounterCollection clusterControllerMetrics;
 
@@ -2860,7 +3057,7 @@ public:
 		serverInfo.clusterInterface = ccInterface;
 		serverInfo.myLocality = locality;
 		db.serverInfo->set(serverInfo);
-		cx = openDBOnServer(db.serverInfo, TaskPriority::DefaultEndpoint, true, true);
+		cx = openDBOnServer(db.serverInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE);
 	}
 
 	~ClusterControllerData() {
@@ -4499,6 +4696,58 @@ ACTOR Future<Void> dbInfoUpdater(ClusterControllerData* self) {
 	}
 }
 
+// The actor that periodically monitors the health of tracked workers.
+ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
+	loop {
+		try {
+			while (!self->goodRecruitmentTime.isReady()) {
+				wait(self->goodRecruitmentTime);
+			}
+
+			self->degradedServers = self->getServersWithDegradedLink();
+
+			// Compare `self->degradedServers` with `self->excludedDegradedServers` and remove those that have
+			// recovered.
+			for (auto it = self->excludedDegradedServers.begin(); it != self->excludedDegradedServers.end();) {
+				if (self->degradedServers.find(*it) == self->degradedServers.end()) {
+					self->excludedDegradedServers.erase(it++);
+				} else {
+					++it;
+				}
+			}
+
+			if (!self->degradedServers.empty()) {
+				std::string degradedServerString;
+				for (const auto& server : self->degradedServers) {
+					degradedServerString += server.toString() + " ";
+				}
+				TraceEvent("ClusterControllerHealthMonitor").detail("DegradedServers", degradedServerString);
+
+				// Check if the cluster controller should trigger a recovery to exclude any degraded servers from the
+				// transaction system.
+				if (self->shouldTriggerRecoveryDueToDegradedServers()) {
+					if (SERVER_KNOBS->CC_HEALTH_TRIGGER_RECOVERY) {
+						if (self->recentRecoveryCountDueToHealth() < SERVER_KNOBS->CC_MAX_HEALTH_RECOVERY_COUNT) {
+							self->recentHealthTriggeredRecoveryTime.push(now());
+							self->excludedDegradedServers = self->degradedServers;
+							TraceEvent("DegradedServerDetectedAndTriggerRecovery")
+							    .detail("RecentRecoveryCountDueToHealth", self->recentRecoveryCountDueToHealth());
+							self->db.forceMasterFailure.trigger();
+						}
+					} else {
+						self->excludedDegradedServers.clear();
+						TraceEvent("DegradedServerDetectedAndSuggestRecovery");
+					}
+				}
+			}
+
+			wait(delay(SERVER_KNOBS->CC_WORKER_HEALTH_CHECKING_INTERVAL));
+		} catch (Error& e) {
+			TraceEvent(SevWarnAlways, "ClusterControllerHealthMonitorError").error(e);
+		}
+	}
+}
+
 ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
                                          Future<Void> leaderFail,
                                          ServerCoordinators coordinators,
@@ -4539,6 +4788,10 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 	self.addActor.send(traceRole(Role::CLUSTER_CONTROLLER, interf.id()));
 	// printf("%s: I am the cluster controller\n", g_network->getLocalAddress().toString().c_str());
 
+	if (SERVER_KNOBS->CC_ENABLE_WORKER_HEALTH_MONITOR) {
+		self.addActor.send(workerHealthMonitor(&self));
+	}
+
 	loop choose {
 		when(ErrorOr<Void> err = wait(error)) {
 			if (err.isError()) {
@@ -4610,7 +4863,7 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 			clusterRegisterMaster(&self, req);
 		}
 		when(UpdateWorkerHealthRequest req = waitNext(interf.updateWorkerHealth.getFuture())) {
-			if (SERVER_KNOBS->CLUSTER_CONTROLLER_ENABLE_WORKER_HEALTH_MONITOR) {
+			if (SERVER_KNOBS->CC_ENABLE_WORKER_HEALTH_MONITOR) {
 				self.updateWorkerHealth(req);
 			}
 		}
@@ -4771,4 +5024,264 @@ TEST_CASE("/fdbserver/clustercontroller/updateWorkerHealth") {
     return Void();
 }
 
+TEST_CASE("/fdbserver/clustercontroller/updateRecoveredWorkers") {
+	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
+	ClusterControllerData data(ClusterControllerFullInterface(),
+	                           LocalityData(),
+	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	NetworkAddress worker1(IPAddress(0x01010101), 1);
+	NetworkAddress worker2(IPAddress(0x11111111), 1);
+	NetworkAddress badPeer1(IPAddress(0x02020202), 1);
+	NetworkAddress badPeer2(IPAddress(0x03030303), 1);
+
+	// Create following test scenario:
+	// 	 worker1 -> badPeer1 active
+	// 	 worker1 -> badPeer2 recovered
+	// 	 worker2 -> badPeer2 recovered
+	data.workerHealth[worker1].degradedPeers[badPeer1] = {
+		now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1, now()
+	};
+	data.workerHealth[worker1].degradedPeers[badPeer2] = {
+		now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1,
+		now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1
+	};
+	data.workerHealth[worker2].degradedPeers[badPeer2] = {
+		now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1,
+		now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1
+	};
+	data.updateRecoveredWorkers();
+
+	ASSERT_EQ(data.workerHealth.size(), 1);
+	ASSERT(data.workerHealth.find(worker1) != data.workerHealth.end());
+	ASSERT(data.workerHealth[worker1].degradedPeers.find(badPeer1) != data.workerHealth[worker1].degradedPeers.end());
+	ASSERT(data.workerHealth[worker1].degradedPeers.find(badPeer2) == data.workerHealth[worker1].degradedPeers.end());
+	ASSERT(data.workerHealth.find(worker2) == data.workerHealth.end());
+
+	return Void();
+}
+
+TEST_CASE("/fdbserver/clustercontroller/getServersWithDegradedLink") {
+	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
+	ClusterControllerData data(ClusterControllerFullInterface(),
+	                           LocalityData(),
+	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	NetworkAddress worker(IPAddress(0x01010101), 1);
+	NetworkAddress badPeer1(IPAddress(0x02020202), 1);
+	NetworkAddress badPeer2(IPAddress(0x03030303), 1);
+	NetworkAddress badPeer3(IPAddress(0x04040404), 1);
+	NetworkAddress badPeer4(IPAddress(0x05050505), 1);
+
+	// Test that a reported degraded link should stay for sometime before being considered as a degraded link by cluster
+	// controller.
+	{
+		data.workerHealth[worker].degradedPeers[badPeer1] = { now(), now() };
+		ASSERT(data.getServersWithDegradedLink().empty());
+		data.workerHealth.clear();
+	}
+
+	// Test that when there is only one reported degraded link, getServersWithDegradedLink can return correct degraded
+	// server.
+	{
+		data.workerHealth[worker].degradedPeers[badPeer1] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		auto degradedServers = data.getServersWithDegradedLink();
+		ASSERT(degradedServers.size() == 1);
+		ASSERT(degradedServers.find(badPeer1) != degradedServers.end());
+		data.workerHealth.clear();
+	}
+
+	// Test that if both A complains B and B compalins A, only one of the server will be chosen as degraded server.
+	{
+		data.workerHealth[worker].degradedPeers[badPeer1] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer1].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		auto degradedServers = data.getServersWithDegradedLink();
+		ASSERT(degradedServers.size() == 1);
+		ASSERT(degradedServers.find(worker) != degradedServers.end() ||
+		       degradedServers.find(badPeer1) != degradedServers.end());
+		data.workerHealth.clear();
+	}
+
+	// Test that if B complains A and C complains A, A is selected as degraded server instead of B or C.
+	{
+		ASSERT(SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE < 4);
+		data.workerHealth[worker].degradedPeers[badPeer1] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer1].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[worker].degradedPeers[badPeer2] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer2].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		auto degradedServers = data.getServersWithDegradedLink();
+		ASSERT(degradedServers.size() == 1);
+		ASSERT(degradedServers.find(worker) != degradedServers.end());
+		data.workerHealth.clear();
+	}
+
+	// Test that if the number of complainers exceeds the threshold, no degraded server is returned.
+	{
+		ASSERT(SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE < 4);
+		data.workerHealth[badPeer1].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer2].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer3].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer4].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		ASSERT(data.getServersWithDegradedLink().empty());
+		data.workerHealth.clear();
+	}
+
+	// Test that if the degradation is reported both ways between A and other 4 servers, no degraded server is returned.
+	{
+		ASSERT(SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE < 4);
+		data.workerHealth[worker].degradedPeers[badPeer1] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer1].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[worker].degradedPeers[badPeer2] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer2].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[worker].degradedPeers[badPeer3] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer3].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[worker].degradedPeers[badPeer4] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		data.workerHealth[badPeer4].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1,
+			                                                  now() };
+		ASSERT(data.getServersWithDegradedLink().empty());
+		data.workerHealth.clear();
+	}
+
+	return Void();
+}
+
+TEST_CASE("/fdbserver/clustercontroller/recentRecoveryCountDueToHealth") {
+	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
+	ClusterControllerData data(ClusterControllerFullInterface(),
+	                           LocalityData(),
+	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+
+	ASSERT_EQ(data.recentRecoveryCountDueToHealth(), 0);
+
+	data.recentHealthTriggeredRecoveryTime.push(now() - SERVER_KNOBS->CC_TRACKING_HEALTH_RECOVERY_INTERVAL - 1);
+	ASSERT_EQ(data.recentRecoveryCountDueToHealth(), 0);
+
+	data.recentHealthTriggeredRecoveryTime.push(now() - SERVER_KNOBS->CC_TRACKING_HEALTH_RECOVERY_INTERVAL + 1);
+	ASSERT_EQ(data.recentRecoveryCountDueToHealth(), 1);
+
+	data.recentHealthTriggeredRecoveryTime.push(now());
+	ASSERT_EQ(data.recentRecoveryCountDueToHealth(), 2);
+
+	return Void();
+}
+
+TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServers") {
+	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
+	ClusterControllerData data(ClusterControllerFullInterface(),
+	                           LocalityData(),
+	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	NetworkAddress master(IPAddress(0x01010101), 1);
+	NetworkAddress tlog(IPAddress(0x02020202), 1);
+	NetworkAddress satelliteTlog(IPAddress(0x03030303), 1);
+	NetworkAddress remoteTlog(IPAddress(0x04040404), 1);
+	NetworkAddress logRouter(IPAddress(0x05050505), 1);
+	NetworkAddress backup(IPAddress(0x06060606), 1);
+	NetworkAddress proxy(IPAddress(0x07070707), 1);
+	NetworkAddress resolver(IPAddress(0x08080808), 1);
+
+	// Create a ServerDBInfo using above addresses.
+	ServerDBInfo testDbInfo;
+	testDbInfo.master.changeCoordinators =
+	    RequestStream<struct ChangeCoordinatorsRequest>(Endpoint({ master }, UID(1, 2)));
+
+	TLogInterface localTLogInterf;
+	localTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ tlog }, UID(1, 2)));
+	TLogInterface localLogRouterInterf;
+	localLogRouterInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ logRouter }, UID(1, 2)));
+	BackupInterface backupInterf;
+	backupInterf.waitFailure = RequestStream<ReplyPromise<Void>>(Endpoint({ backup }, UID(1, 2)));
+	TLogSet localTLogSet;
+	localTLogSet.isLocal = true;
+	localTLogSet.tLogs.push_back(OptionalInterface(localTLogInterf));
+	localTLogSet.logRouters.push_back(OptionalInterface(localLogRouterInterf));
+	localTLogSet.backupWorkers.push_back(OptionalInterface(backupInterf));
+	testDbInfo.logSystemConfig.tLogs.push_back(localTLogSet);
+
+	TLogInterface sateTLogInterf;
+	sateTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ satelliteTlog }, UID(1, 2)));
+	TLogSet sateTLogSet;
+	sateTLogSet.isLocal = true;
+	sateTLogSet.locality = tagLocalitySatellite;
+	sateTLogSet.tLogs.push_back(OptionalInterface(sateTLogInterf));
+	testDbInfo.logSystemConfig.tLogs.push_back(sateTLogSet);
+
+	TLogInterface remoteTLogInterf;
+	remoteTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ remoteTlog }, UID(1, 2)));
+	TLogSet remoteTLogSet;
+	remoteTLogSet.isLocal = false;
+	remoteTLogSet.tLogs.push_back(OptionalInterface(remoteTLogInterf));
+	testDbInfo.logSystemConfig.tLogs.push_back(remoteTLogSet);
+
+	GrvProxyInterface proxyInterf;
+	proxyInterf.getConsistentReadVersion = RequestStream<struct GetReadVersionRequest>(Endpoint({ proxy }, UID(1, 2)));
+	testDbInfo.client.grvProxies.push_back(proxyInterf);
+
+	ResolverInterface resolverInterf;
+	resolverInterf.resolve = RequestStream<struct ResolveTransactionBatchRequest>(Endpoint({ resolver }, UID(1, 2)));
+	testDbInfo.resolvers.push_back(resolverInterf);
+
+	testDbInfo.recoveryState = RecoveryState::ACCEPTING_COMMITS;
+
+	// No recovery when no degraded servers.
+	data.db.serverInfo->set(testDbInfo);
+	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
+
+	// Trigger recovery when master is degraded.
+	data.degradedServers.insert(master);
+	ASSERT(data.shouldTriggerRecoveryDueToDegradedServers());
+	data.degradedServers.clear();
+
+	// Trigger recovery when primary TLog is degraded.
+	data.degradedServers.insert(tlog);
+	ASSERT(data.shouldTriggerRecoveryDueToDegradedServers());
+	data.degradedServers.clear();
+
+	// No recovery when satellite Tlog is degraded.
+	data.degradedServers.insert(satelliteTlog);
+	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
+	data.degradedServers.clear();
+
+	// No recovery when remote tlog is degraded.
+	data.degradedServers.insert(remoteTlog);
+	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
+	data.degradedServers.clear();
+
+	// No recovery when log router is degraded.
+	data.degradedServers.insert(logRouter);
+	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
+	data.degradedServers.clear();
+
+	// No recovery when backup worker is degraded.
+	data.degradedServers.insert(backup);
+	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
+	data.degradedServers.clear();
+
+	// Trigger recovery when proxy is degraded.
+	data.degradedServers.insert(proxy);
+	ASSERT(data.shouldTriggerRecoveryDueToDegradedServers());
+	data.degradedServers.clear();
+
+	// Trigger recovery when resolver is degraded.
+	data.degradedServers.insert(resolver);
+	ASSERT(data.shouldTriggerRecoveryDueToDegradedServers());
+
+	return Void();
+}
+
 } // namespace

fdbserver/ConfigDatabaseUnitTests.actor.cpp

@@ -158,13 +158,13 @@ public:
 	ReadFromLocalConfigEnvironment(std::string const& dataDir,
 	                               std::string const& configPath,
 	                               std::map<std::string, std::string> const& manualKnobOverrides)
-	  : dataDir(dataDir), localConfiguration(dataDir, configPath, manualKnobOverrides, IsTest::YES), consumer(Never()) {
-	}
+	  : dataDir(dataDir), localConfiguration(dataDir, configPath, manualKnobOverrides, IsTest::TRUE),
+	    consumer(Never()) {}
 
 	Future<Void> setup() { return setup(this); }
 
 	Future<Void> restartLocalConfig(std::string const& newConfigPath) {
-		localConfiguration = LocalConfiguration(dataDir, newConfigPath, {}, IsTest::YES);
+		localConfiguration = LocalConfiguration(dataDir, newConfigPath, {}, IsTest::TRUE);
 		return setup();
 	}
 

fdbserver/DataDistribution.actor.cpp

@@ -5765,7 +5765,7 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 	state double lastLimited = 0;
 	self->addActor.send(monitorBatchLimitedTime(self->dbInfo, &lastLimited));
 
-	state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DataDistributionLaunch, true, true);
+	state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DataDistributionLaunch, LockAware::TRUE);
 	cx->locationCacheSize = SERVER_KNOBS->DD_LOCATION_CACHE_SIZE;
 
 	// cx->setOption( FDBDatabaseOptions::LOCATION_CACHE_SIZE, StringRef((uint8_t*)
@@ -6106,7 +6106,7 @@ static std::set<int> const& normalDataDistributorErrors() {
 }
 
 ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<AsyncVar<struct ServerDBInfo>> db) {
-	state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, true, true);
+	state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::TRUE);
 	state ReadYourWritesTransaction tr(cx);
 	loop {
 		try {
@@ -6447,7 +6447,7 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 	state Reference<DataDistributorData> self(new DataDistributorData(db, di.id()));
 	state Future<Void> collection = actorCollection(self->addActor.getFuture());
 	state PromiseStream<GetMetricsListRequest> getShardMetricsList;
-	state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, true, true);
+	state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::TRUE);
 	state ActorCollection actors(false);
 	state DDEnabledState ddEnabledState;
 	self->addActor.send(actors.getResult());
@@ -6498,8 +6498,8 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 std::unique_ptr<DDTeamCollection> testTeamCollection(int teamSize,
                                                      Reference<IReplicationPolicy> policy,
                                                      int processCount) {
-	Database database =
-	    DatabaseContext::create(makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), false);
+	Database database = DatabaseContext::create(
+	    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::FALSE);
 
 	DatabaseConfiguration conf;
 	conf.storageTeamSize = teamSize;
@@ -6541,8 +6541,8 @@ std::unique_ptr<DDTeamCollection> testTeamCollection(int teamSize,
 std::unique_ptr<DDTeamCollection> testMachineTeamCollection(int teamSize,
                                                             Reference<IReplicationPolicy> policy,
                                                             int processCount) {
-	Database database =
-	    DatabaseContext::create(makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), false);
+	Database database = DatabaseContext::create(
+	    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::FALSE);
 
 	DatabaseConfiguration conf;
 	conf.storageTeamSize = teamSize;

fdbserver/DiskQueue.actor.cpp

@@ -29,6 +29,8 @@
 typedef bool (*compare_pages)(void*, void*);
 typedef int64_t loc_t;
 
+FDB_DEFINE_BOOLEAN_PARAM(CheckHashes);
+
 // 0 -> 0
 // 1 -> 4k
 // 4k -> 4k
@@ -1241,9 +1243,9 @@ private:
 			// start and end are on the same page
 			ASSERT(pagedData.size() == sizeof(Page));
 			Page* data = reinterpret_cast<Page*>(const_cast<uint8_t*>(pagedData.begin()));
-			if (ch == CheckHashes::YES && !data->checkHash())
+			if (ch && !data->checkHash())
 				throw io_error();
-			if (ch == CheckHashes::NO && data->payloadSize > Page::maxPayload)
+			if (!ch && data->payloadSize > Page::maxPayload)
 				throw io_error();
 			pagedData.contents() = pagedData.substr(sizeof(PageHeader) + startingOffset, endingOffset - startingOffset);
 			return pagedData;
@@ -1252,9 +1254,9 @@ private:
 			// we don't have to double allocate in a hot, memory hungry call.
 			uint8_t* buf = mutateString(pagedData);
 			Page* data = reinterpret_cast<Page*>(const_cast<uint8_t*>(pagedData.begin()));
-			if (ch == CheckHashes::YES && !data->checkHash())
+			if (ch && !data->checkHash())
 				throw io_error();
-			if (ch == CheckHashes::NO && data->payloadSize > Page::maxPayload)
+			if (!ch && data->payloadSize > Page::maxPayload)
 				throw io_error();
 
 			// Only start copying from `start` in the first page.
@@ -1264,9 +1266,9 @@ private:
 				buf += length;
 			}
 			data++;
-			if (ch == CheckHashes::YES && !data->checkHash())
+			if (ch && !data->checkHash())
 				throw io_error();
-			if (ch == CheckHashes::NO && data->payloadSize > Page::maxPayload)
+			if (!ch && data->payloadSize > Page::maxPayload)
 				throw io_error();
 
 			// Copy all the middle pages
@@ -1277,9 +1279,9 @@ private:
 				memmove(buf, data->payload, length);
 				buf += length;
 				data++;
-				if (ch == CheckHashes::YES && !data->checkHash())
+				if (ch && !data->checkHash())
 					throw io_error();
-				if (ch == CheckHashes::NO && data->payloadSize > Page::maxPayload)
+				if (!ch && data->payloadSize > Page::maxPayload)
 					throw io_error();
 			}
 

fdbserver/GrvProxyServer.actor.cpp

@@ -253,7 +253,7 @@ struct GrvProxyData {
 	             RequestStream<GetReadVersionRequest> getConsistentReadVersion,
 	             Reference<AsyncVar<ServerDBInfo>> db)
 	  : dbgid(dbgid), stats(dbgid), master(master), getConsistentReadVersion(getConsistentReadVersion),
-	    cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true)), db(db), lastStartCommit(0),
+	    cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE)), db(db), lastStartCommit(0),
 	    lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), updateCommitRequests(0), lastCommitTime(0),
 	    minKnownCommittedVersion(invalidVersion) {}
 };

fdbserver/IDiskQueue.h

@@ -24,11 +24,9 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbserver/IKeyValueStore.h"
+#include "flow/BooleanParam.h"
 
-enum class CheckHashes {
-	NO,
-	YES,
-};
+FDB_DECLARE_BOOLEAN_PARAM(CheckHashes);
 
 class IDiskQueue : public IClosable {
 public:

fdbserver/KeyValueStoreRocksDB.actor.cpp

@@ -282,7 +282,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				a.result.send(Value(StringRef(reinterpret_cast<const uint8_t*>(value.data()),
 				                              std::min(value.size(), size_t(a.maxLength)))));
 			} else {
-				TraceEvent(SevError, "RocksDBError").detail("Error", s.ToString()).detail("Method", "ReadValuePrefix");
+				if (!s.IsNotFound()) {
+					TraceEvent(SevError, "RocksDBError").detail("Error", s.ToString()).detail("Method", "ReadValuePrefix");
+				}
 				a.result.send(Optional<Value>());
 			}
 		}

fdbserver/LocalConfiguration.actor.cpp

@@ -28,6 +28,8 @@
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+FDB_DEFINE_BOOLEAN_PARAM(IsTest);
+
 namespace {
 
 const KeyRef configPathKey = "configPath"_sr;
@@ -228,11 +230,11 @@ class LocalConfigurationImpl {
 	void updateInMemoryState(Version lastSeenVersion) {
 		this->lastSeenVersion = lastSeenVersion;
 		// TODO: Support randomization?
-		getKnobs().reset(Randomize::NO, g_network->isSimulated() ? IsSimulated::YES : IsSimulated::NO);
+		getKnobs().reset(Randomize::FALSE, g_network->isSimulated() ? IsSimulated::TRUE : IsSimulated::FALSE);
 		configKnobOverrides.update(getKnobs());
 		manualKnobOverrides.update(getKnobs());
 		// Must reinitialize in order to update dependent knobs
-		getKnobs().initialize(Randomize::NO, g_network->isSimulated() ? IsSimulated::YES : IsSimulated::NO);
+		getKnobs().initialize(Randomize::FALSE, g_network->isSimulated() ? IsSimulated::TRUE : IsSimulated::FALSE);
 	}
 
 	ACTOR static Future<Void> setSnapshot(LocalConfigurationImpl* self,
@@ -329,10 +331,11 @@ public:
 	    broadcasterChanges("BroadcasterChanges", cc), snapshots("Snapshots", cc),
 	    changeRequestsFetched("ChangeRequestsFetched", cc), mutations("Mutations", cc), configKnobOverrides(configPath),
 	    manualKnobOverrides(manualKnobOverrides) {
-		if (isTest == IsTest::YES) {
-			testKnobCollection = IKnobCollection::create(IKnobCollection::Type::TEST,
-			                                             Randomize::NO,
-			                                             g_network->isSimulated() ? IsSimulated::YES : IsSimulated::NO);
+		if (isTest) {
+			testKnobCollection =
+			    IKnobCollection::create(IKnobCollection::Type::TEST,
+			                            Randomize::FALSE,
+			                            g_network->isSimulated() ? IsSimulated::TRUE : IsSimulated::FALSE);
 		}
 		logger = traceCounters(
 		    "LocalConfigurationMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "LocalConfigurationMetrics");
@@ -401,7 +404,8 @@ public:
 		ConfigKnobOverrides configKnobOverrides;
 		configKnobOverrides.set(
 		    {}, "knob_name_that_does_not_exist"_sr, KnobValueRef::create(ParsedKnobValue(int{ 1 })));
-		auto testKnobCollection = IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::NO, IsSimulated::NO);
+		auto testKnobCollection =
+		    IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::FALSE, IsSimulated::FALSE);
 		// Should only trace and not throw an error:
 		configKnobOverrides.update(*testKnobCollection);
 	}
@@ -409,7 +413,8 @@ public:
 	static void testConfigKnobOverridesInvalidValue() {
 		ConfigKnobOverrides configKnobOverrides;
 		configKnobOverrides.set({}, "test_int"_sr, KnobValueRef::create(ParsedKnobValue("not_an_int")));
-		auto testKnobCollection = IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::NO, IsSimulated::NO);
+		auto testKnobCollection =
+		    IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::FALSE, IsSimulated::FALSE);
 		// Should only trace and not throw an error:
 		configKnobOverrides.update(*testKnobCollection);
 	}

fdbserver/LocalConfiguration.h

@@ -29,8 +29,7 @@
 #include "flow/Arena.h"
 #include "flow/Knobs.h"
 
-// To be used effectively as a boolean parameter with added type safety
-enum class IsTest { NO, YES };
+FDB_DECLARE_BOOLEAN_PARAM(IsTest);
 
 /*
  * Each worker maintains a LocalConfiguration object used to update its knob collection.
@@ -52,7 +51,7 @@ public:
 	LocalConfiguration(std::string const& dataFolder,
 	                   std::string const& configPath,
 	                   std::map<std::string, std::string> const& manualKnobOverrides,
-	                   IsTest isTest = IsTest::NO);
+	                   IsTest = IsTest::FALSE);
 	LocalConfiguration(LocalConfiguration&&);
 	LocalConfiguration& operator=(LocalConfiguration&&);
 	~LocalConfiguration();

fdbserver/MetricLogger.actor.cpp

@@ -182,7 +182,7 @@ public:
 	// levelKey is the prefix for the entire level, no timestamp at the end
 	ACTOR static Future<Optional<Standalone<StringRef>>> getLastBlock_impl(ReadYourWritesTransaction* tr,
 	                                                                       Standalone<StringRef> levelKey) {
-		RangeResult results = wait(tr->getRange(normalKeys.withPrefix(levelKey), 1, true, true));
+		RangeResult results = wait(tr->getRange(normalKeys.withPrefix(levelKey), 1, Snapshot::TRUE, Reverse::TRUE));
 		if (results.size() == 1)
 			return results[0].value;
 		return Optional<Standalone<StringRef>>();

fdbserver/MoveKeys.actor.cpp

@@ -1039,8 +1039,9 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
 				    LocalityData::ExcludeLocalityPrefix.toString() + l.first + ":" + l.second))));
 			}
 
-			state Future<RangeResult> fTags = tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY, true);
-			state Future<RangeResult> fHistoryTags = tr->getRange(serverTagHistoryKeys, CLIENT_KNOBS->TOO_MANY, true);
+			state Future<RangeResult> fTags = tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY, Snapshot::TRUE);
+			state Future<RangeResult> fHistoryTags =
+			    tr->getRange(serverTagHistoryKeys, CLIENT_KNOBS->TOO_MANY, Snapshot::TRUE);
 
 			wait(success(fTagLocalities) && success(fv) && success(fTags) && success(fHistoryTags) &&
 			     success(fExclProc) && success(fExclIP) && success(fFailProc) && success(fFailIP) &&

fdbserver/OldTLogServer_6_0.actor.cpp

@@ -311,7 +311,7 @@ struct TLogData : NonCopyable {
 	    targetVolatileBytes(SERVER_KNOBS->TLOG_SPILL_THRESHOLD), overheadBytesInput(0), overheadBytesDurable(0),
 	    concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS), ignorePopRequest(false),
 	    ignorePopDeadline(), ignorePopUid(), dataFolder(folder), toBePopped() {
-		cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, true);
+		cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE);
 	}
 };
 

fdbserver/OldTLogServer_6_2.actor.cpp

@@ -375,7 +375,7 @@ struct TLogData : NonCopyable {
 	    peekMemoryLimiter(SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES),
 	    concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS), ignorePopRequest(false),
 	    ignorePopDeadline(), ignorePopUid(), dataFolder(folder), toBePopped() {
-		cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, true);
+		cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE);
 	}
 };
 
@@ -1440,6 +1440,19 @@ ACTOR Future<Void> tLogPopCore(TLogData* self, Tag inputTag, Version to, Referen
 			}
 		}
 
+		uint64_t PoppedVersionLag = logData->persistentDataDurableVersion - logData->queuePoppedVersion;
+		if ( SERVER_KNOBS->ENABLE_DETAILED_TLOG_POP_TRACE &&
+			(logData->queuePoppedVersion > 0) && //avoid generating massive events at beginning 
+			(tagData->unpoppedRecovered || PoppedVersionLag >= SERVER_KNOBS->TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE)) { //when recovery or long lag
+			TraceEvent("TLogPopDetails", logData->logId)
+				.detail("Tag", tagData->tag.toString())
+				.detail("UpTo", upTo)
+				.detail("PoppedVersionLag", PoppedVersionLag)
+				.detail("MinPoppedTag", logData->minPoppedTag.toString())
+				.detail("QueuePoppedVersion", logData->queuePoppedVersion)
+				.detail("UnpoppedRecovered", tagData->unpoppedRecovered ? "True" : "False")
+				.detail("NothingPersistent", tagData->nothingPersistent ? "True" : "False");
+		}
 		if (upTo > logData->persistentDataDurableVersion)
 			wait(tagData->eraseMessagesBefore(upTo, self, logData, TaskPriority::TLogPop));
 		//TraceEvent("TLogPop", self->dbgid).detail("Tag", tag.toString()).detail("To", upTo);
@@ -1744,7 +1757,7 @@ ACTOR Future<Void> tLogPeekMessages(TLogData* self, TLogPeekRequest req, Referen
 			state std::vector<Future<Standalone<StringRef>>> messageReads;
 			messageReads.reserve(commitLocations.size());
 			for (const auto& pair : commitLocations) {
-				messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::YES));
+				messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::TRUE));
 			}
 			commitLocations.clear();
 			wait(waitForAll(messageReads));

fdbserver/ProxyCommitData.actor.h

@@ -247,7 +247,7 @@ struct ProxyCommitData {
 	    mostRecentProcessedRequestNumber(0), getConsistentReadVersion(getConsistentReadVersion), commit(commit),
 	    lastCoalesceTime(0), localCommitBatchesStarted(0), locked(false),
 	    commitBatchInterval(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_MIN), firstProxy(firstProxy),
-	    cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true)), db(db),
+	    cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE)), db(db),
 	    singleKeyMutationEvent(LiteralStringRef("SingleKeyMutation")), commitBatchesMemBytesCount(0), lastTxsPop(0),
 	    lastStartCommit(0), lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), lastCommitTime(0),
 	    lastMasterReset(now()), lastResolverReset(now()) {

fdbserver/QuietDatabase.actor.cpp

@@ -637,7 +637,7 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
 
 	// The quiet database check (which runs at the end of every test) will always time out due to active data movement.
 	// To get around this, quiet Database will disable the perpetual wiggle in the setup phase.
-	wait(setPerpetualStorageWiggle(cx, false, true));
+	wait(setPerpetualStorageWiggle(cx, false, LockAware::TRUE));
 
 	// Require 3 consecutive successful quiet database checks spaced 2 second apart
 	state int numSuccesses = 0;

fdbserver/Ratekeeper.actor.cpp

@@ -1409,7 +1409,7 @@ ACTOR Future<Void> configurationMonitor(RatekeeperData* self) {
 }
 
 ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo>> dbInfo) {
-	state RatekeeperData self(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, true));
+	state RatekeeperData self(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE));
 	state Future<Void> timeout = Void();
 	state std::vector<Future<Void>> tlogTrackers;
 	state std::vector<TLogInterface> tlogInterfs;

fdbserver/ResolverInterface.h

@@ -49,6 +49,7 @@ struct ResolverInterface {
 	bool operator==(ResolverInterface const& r) const { return id() == r.id(); }
 	bool operator!=(ResolverInterface const& r) const { return id() != r.id(); }
 	NetworkAddress address() const { return resolve.getEndpoint().getPrimaryAddress(); }
+	NetworkAddressList addresses() const { return resolve.getEndpoint().addresses; }
 	void initEndpoints() {
 		metrics.getEndpoint(TaskPriority::ResolutionMetrics);
 		split.getEndpoint(TaskPriority::ResolutionMetrics);

fdbserver/RestoreCommon.actor.cpp

@@ -141,8 +141,8 @@ Key RestoreConfigFR::applyMutationsMapPrefix() {
 
 ACTOR Future<int64_t> RestoreConfigFR::getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid) {
 	// Both of these are snapshot reads
-	state Future<Optional<Value>> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), true);
-	state Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), true);
+	state Future<Optional<Value>> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::TRUE);
+	state Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::TRUE);
 	wait(success(beginVal) && success(endVal));
 
 	if (!beginVal.get().present() || !endVal.get().present())

fdbserver/RestoreWorker.actor.cpp

@@ -410,7 +410,7 @@ ACTOR Future<Void> restoreWorker(Reference<ClusterConnectionFile> connFile,
                                  LocalityData locality,
                                  std::string coordFolder) {
 	try {
-		Database cx = Database::createDatabase(connFile, Database::API_VERSION_LATEST, true, locality);
+		Database cx = Database::createDatabase(connFile, Database::API_VERSION_LATEST, IsInternal::TRUE, locality);
 		wait(reportErrors(_restoreWorker(cx, locality), "RestoreWorker"));
 	} catch (Error& e) {
 		TraceEvent("FastRestoreWorker").detail("Error", e.what());

fdbserver/SimulatedCluster.actor.cpp

@@ -347,8 +347,8 @@ ACTOR Future<Void> runBackup(Reference<ClusterConnectionFile> connFile) {
 		Database cx = Database::createDatabase(connFile, -1);
 
 		state FileBackupAgent fileAgent;
-		state double backupPollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
-		agentFutures.push_back(fileAgent.run(cx, &backupPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
+		agentFutures.push_back(fileAgent.run(
+		    cx, 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
 
 		while (g_simulator.backupAgents == ISimulator::BackupAgentType::BackupToFile) {
 			wait(delay(1.0));
@@ -383,11 +383,10 @@ ACTOR Future<Void> runDr(Reference<ClusterConnectionFile> connFile) {
 		state DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx);
 		state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(extraDB);
 
-		state double dr1PollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
-		state double dr2PollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
+		auto drPollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
 
-		agentFutures.push_back(extraAgent.run(cx, &dr1PollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
-		agentFutures.push_back(dbAgent.run(extraDB, &dr2PollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
+		agentFutures.push_back(extraAgent.run(cx, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
+		agentFutures.push_back(dbAgent.run(extraDB, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
 
 		while (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) {
 			wait(delay(1.0));

fdbserver/Status.actor.cpp

@@ -2605,10 +2605,9 @@ ACTOR Future<JsonBuilderObject> lockedStatusFetcher(Reference<AsyncVar<ServerDBI
                                                     std::set<std::string>* incomplete_reasons) {
 	state JsonBuilderObject statusObj;
 
-	state Database cx = openDBOnServer(db,
-	                                   TaskPriority::DefaultEndpoint,
-	                                   true,
-	                                   false); // Open a new database connection that isn't lock-aware
+	state Database cx =
+	    openDBOnServer(db,
+	                   TaskPriority::DefaultEndpoint); // Open a new database connection that isn't lock-aware
 	state Transaction tr(cx);
 	state int timeoutSeconds = 5;
 	state Future<Void> getTimeout = delay(timeoutSeconds);

fdbserver/StorageCache.actor.cpp

@@ -251,7 +251,7 @@ public:
 		newestAvailableVersion.insert(allKeys, invalidVersion);
 		newestDirtyVersion.insert(allKeys, invalidVersion);
 		addCacheRange(CacheRangeInfo::newNotAssigned(allKeys));
-		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
+		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE);
 	}
 
 	// Puts the given cacheRange into cachedRangeMap.  The caller is responsible for adding cacheRanges
@@ -1194,7 +1194,7 @@ ACTOR Future<RangeResult> tryFetchRange(Database cx,
 
 	try {
 		loop {
-			RangeResult rep = wait(tr.getRange(begin, end, limits, true));
+			RangeResult rep = wait(tr.getRange(begin, end, limits, Snapshot::TRUE));
 			limits.decrement(rep);
 
 			if (limits.isReached() || !rep.more) {
@@ -1392,7 +1392,7 @@ ACTOR Future<Void> fetchKeys(StorageCacheData* data, AddingCacheRange* cacheRang
 					// TODO: NEELAM: what's this for?
 					// FIXME: remove when we no longer support upgrades from 5.X
 					if (debug_getRangeRetries >= 100) {
-						data->cx->enableLocalityLoadBalance = false;
+						data->cx->enableLocalityLoadBalance = EnableLocalityLoadBalance::FALSE;
 					}
 
 					debug_getRangeRetries++;

fdbserver/TLogServer.actor.cpp

@@ -386,7 +386,7 @@ struct TLogData : NonCopyable {
 	    commitLatencyDist(Histogram::getHistogram(LiteralStringRef("tLog"),
 	                                              LiteralStringRef("commit"),
 	                                              Histogram::Unit::microseconds)) {
-		cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, true);
+		cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE);
 	}
 };
 
@@ -1166,6 +1166,19 @@ ACTOR Future<Void> tLogPopCore(TLogData* self, Tag inputTag, Version to, Referen
 			}
 		}
 
+		uint64_t PoppedVersionLag = logData->persistentDataDurableVersion - logData->queuePoppedVersion;
+		if ( SERVER_KNOBS->ENABLE_DETAILED_TLOG_POP_TRACE &&
+			(logData->queuePoppedVersion > 0) && //avoid generating massive events at beginning 
+			(tagData->unpoppedRecovered || PoppedVersionLag >= SERVER_KNOBS->TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE)) { //when recovery or long lag
+			TraceEvent("TLogPopDetails", logData->logId)
+				.detail("Tag", tagData->tag.toString())
+				.detail("UpTo", upTo)
+				.detail("PoppedVersionLag", PoppedVersionLag)
+				.detail("MinPoppedTag", logData->minPoppedTag.toString())
+				.detail("QueuePoppedVersion", logData->queuePoppedVersion)
+				.detail("UnpoppedRecovered", tagData->unpoppedRecovered ? "True" : "False")
+				.detail("NothingPersistent", tagData->nothingPersistent ? "True" : "False");
+		}
 		if (upTo > logData->persistentDataDurableVersion)
 			wait(tagData->eraseMessagesBefore(upTo, self, logData, TaskPriority::TLogPop));
 		//TraceEvent("TLogPop", logData->logId).detail("Tag", tag.toString()).detail("To", upTo);
@@ -1785,7 +1798,7 @@ ACTOR Future<Void> tLogPeekMessages(TLogData* self, TLogPeekRequest req, Referen
 			state std::vector<Future<Standalone<StringRef>>> messageReads;
 			messageReads.reserve(commitLocations.size());
 			for (const auto& pair : commitLocations) {
-				messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::YES));
+				messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::TRUE));
 			}
 			commitLocations.clear();
 			wait(waitForAll(messageReads));

fdbserver/VersionedBTree.actor.cpp

@@ -6532,20 +6532,19 @@ public:
 		//     If there is a record in the tree > query then moveNext() will move to it.
 		// If non-zero is returned then the cursor is valid and the return value is logically equivalent
 		// to query.compare(cursor.get())
-		ACTOR Future<int> seek_impl(PagerEventReasons r, BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) {
+
+		ACTOR Future<int> seek_impl(PagerEventReasons r, BTreeCursor* self, RedwoodRecordRef query) {
 			state RedwoodRecordRef internalPageQuery = query.withMaxPageID();
 			self->path.resize(1);
-			debug_printf(
-			    "seek(%s, %d) start cursor = %s\n", query.toString().c_str(), prefetchBytes, self->toString().c_str());
+			debug_printf("seek(%s) start cursor = %s\n", query.toString().c_str(), self->toString().c_str());
 
 			loop {
 				auto& entry = self->path.back();
 				if (entry.btPage()->isLeaf()) {
 					int cmp = entry.cursor.seek(query);
 					self->valid = entry.cursor.valid() && !entry.cursor.isErased();
-					debug_printf("seek(%s, %d) loop exit cmp=%d cursor=%s\n",
+					debug_printf("seek(%s) loop exit cmp=%d cursor=%s\n",
 					             query.toString().c_str(),
-					             prefetchBytes,
 					             cmp,
 					             self->toString().c_str());
 					return self->valid ? cmp : 0;
@@ -6556,68 +6555,97 @@ public:
 				// to and will be updated if anything is inserted into the cleared range, so if the seek fails
 				// or it finds an entry with a null child page then query does not exist in the BTree.
 				if (entry.cursor.seekLessThan(internalPageQuery) && entry.cursor.get().value.present()) {
-					debug_printf("seek(%s, %d) loop seek success cursor=%s\n",
-					             query.toString().c_str(),
-					             prefetchBytes,
-					             self->toString().c_str());
-					Future<Void> f = self->pushPage(r, self->getHeight(), entry.cursor);
-
-					// Prefetch siblings, at least prefetchBytes, at level 2 but without jumping to another level 2
-					// sibling
-					if (prefetchBytes != 0 && entry.btPage()->height == 2) {
-						auto c = entry.cursor;
-						bool fwd = prefetchBytes > 0;
-						prefetchBytes = abs(prefetchBytes);
-						// While we should still preload more bytes and a move in the target direction is successful
-						while (prefetchBytes > 0 && (fwd ? c.moveNext() : c.movePrev())) {
-							// If there is a page link, preload it.
-							if (c.get().value.present()) {
-								BTreePageIDRef childPage = c.get().getChildPage();
-								preLoadPage(self->pager.getPtr(), self->getHeight()-1, childPage);
-								prefetchBytes -= self->btree->m_blockSize * childPage.size();
-							}
-						}
-					}
-
+					debug_printf(
+					    "seek(%s) loop seek success cursor=%s\n", query.toString().c_str(), self->toString().c_str());
+					Future<Void> f = self->pushPage(r, 0, entry.cursor);
 					wait(f);
 				} else {
 					self->valid = false;
-					debug_printf("seek(%s, %d) loop exit cmp=0 cursor=%s\n",
-					             query.toString().c_str(),
-					             prefetchBytes,
-					             self->toString().c_str());
+					debug_printf(
+					    "seek(%s) loop exit cmp=0 cursor=%s\n", query.toString().c_str(), self->toString().c_str());
 					return 0;
 				}
 			}
 		}
 
-		Future<int> seek(PagerEventReasons r, RedwoodRecordRef query, int prefetchBytes) { return seek_impl(r, this, query, prefetchBytes); }
+		Future<int> seek(PagerEventReasons r, RedwoodRecordRef query) { return seek_impl(r, this, query); }
 
-		ACTOR Future<Void> seekGTE_impl(PagerEventReasons r, BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) {
-			debug_printf("seekGTE(%s, %d) start\n", query.toString().c_str(), prefetchBytes);
-			int cmp = wait(self->seek(r, query, prefetchBytes));
+		ACTOR Future<Void> seekGTE_impl(PagerEventReasons r, BTreeCursor* self, RedwoodRecordRef query) {
+			debug_printf("seekGTE(%s) start\n", query.toString().c_str());
+			int cmp = wait(self->seek(r, query));
 			if (cmp > 0 || (cmp == 0 && !self->isValid())) {
 				wait(self->moveNext());
 			}
 			return Void();
 		}
 
-		Future<Void> seekGTE(PagerEventReasons r, RedwoodRecordRef query, int prefetchBytes) {
-			return seekGTE_impl(r, this, query, prefetchBytes);
+		Future<Void> seekGTE(PagerEventReasons r, RedwoodRecordRef query) { return seekGTE_impl(r, this, query); }
+
+		// Start fetching sibling nodes in the forward or backward direction, stopping after recordLimit or byteLimit
+		void prefetch(KeyRef rangeEnd, bool directionForward, int recordLimit, int byteLimit) {
+			// Prefetch scans level 2 so if there are less than 2 nodes in the path there is no level 2
+			if (path.size() < 2) {
+				return;
+			}
+
+			auto firstLeaf = path.back().btPage();
+
+			// We know the first leaf's record count, so assume they are all relevant to the query,
+			// even though some may not be.
+			int recordsRead = firstLeaf->tree()->numItems;
+
+			// We can't know for sure how many records are in a node without reading it, so just guess
+			// that siblings have about the same record count as the first leaf.
+			int estRecordsPerPage = recordsRead;
+
+			// Use actual KVBytes stored for the first leaf, but use node capacity for siblings below
+			int bytesRead = firstLeaf->kvBytes;
+
+			// Cursor for moving through siblings.
+			// Note that only immediate siblings under the same parent are considered for prefetch so far.
+			BTreePage::BinaryTree::Cursor c = path[path.size() - 2].cursor;
+
+			// The loop conditions are split apart into different if blocks for readability.
+			// While query limits are not exceeded
+			while (recordsRead < recordLimit && bytesRead < byteLimit) {
+				// If prefetching right siblings
+				if (directionForward) {
+					// If there is no right sibling or its lower boundary is greater
+					// or equal to than the range end then stop.
+					if(!c.moveNext() || c.get().key >= rangeEnd) {
+						break;
+					}
+				}
+				else {
+					// Prefetching left siblings
+					// If the current leaf lower boundary is less than or equal to the range end
+					// or there is no left sibling then stop
+					if(c.get().key <= rangeEnd || !c.movePrev()) {
+						break;
+					}
+				}
+
+				// Prefetch the sibling if the link is not null
+				if (c.get().value.present()) {
+					BTreePageIDRef childPage = c.get().getChildPage();
+					preLoadPage(pager.getPtr(), path[path.size() - 2].getHeight()-1, childPage);
+					recordsRead += estRecordsPerPage;
+					// Use sibling node capacity as an estimate of bytes read.
+					bytesRead += childPage.size() * this->btree->m_blockSize;
+				}
+			}
 		}
 
-		ACTOR Future<Void> seekLT_impl(PagerEventReasons r, BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) {
-			debug_printf("seekLT(%s, %d) start\n", query.toString().c_str(), prefetchBytes);
-			int cmp = wait(self->seek(r, query, prefetchBytes));
+		ACTOR Future<Void> seekLT_impl(PagerEventReasons r, BTreeCursor* self, RedwoodRecordRef query) {
+			debug_printf("seekLT(%s) start\n", query.toString().c_str());
+			int cmp = wait(self->seek(r, query));
 			if (cmp <= 0) {
 				wait(self->movePrev());
 			}
 			return Void();
 		}
 
-		Future<Void> seekLT(PagerEventReasons r, RedwoodRecordRef query, int prefetchBytes) {
-			return seekLT_impl(r, this, query, -prefetchBytes);
-		}
+		Future<Void> seekLT(PagerEventReasons r, RedwoodRecordRef query) { return seekLT_impl(r, this, query); }
 
 		ACTOR Future<Void> move_impl(BTreeCursor* self, bool forward) {
 			// Try to the move cursor at the end of the path in the correct direction
@@ -6702,7 +6730,8 @@ RedwoodRecordRef VersionedBTree::dbEnd(LiteralStringRef("\xff\xff\xff\xff\xff"))
 class KeyValueStoreRedwoodUnversioned : public IKeyValueStore {
 public:
 	KeyValueStoreRedwoodUnversioned(std::string filePrefix, UID logID)
-	  : m_filePrefix(filePrefix), m_concurrentReads(SERVER_KNOBS->REDWOOD_KVSTORE_CONCURRENT_READS) {
+	  : m_filePrefix(filePrefix), m_concurrentReads(SERVER_KNOBS->REDWOOD_KVSTORE_CONCURRENT_READS),
+	    prefetch(SERVER_KNOBS->REDWOOD_KVSTORE_RANGE_PREFETCH) {
 
 		int pageSize =
 		    BUGGIFY ? deterministicRandom()->randomInt(1000, 4096 * 4) : SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE;
@@ -6810,11 +6839,12 @@ public:
 			return result;
 		}
 
-		// Prefetch is disabled for now pending some decent logic for deciding how much to fetch
-		state int prefetchBytes = 0;
-
 		if (rowLimit > 0) {
-			wait(cur.seekGTE(PagerEventReasons::rangeRead, keys.begin, prefetchBytes));
+			wait(cur.seekGTE(PagerEventReasons::rangeRead, keys.begin));
+
+			if (self->prefetch) {
+				cur.prefetch(keys.end, true, rowLimit, byteLimit);
+			}
 			while (cur.isValid()) {
 				// Read page contents without using waits
 				BTreePage::BinaryTree::Cursor leafCursor = cur.back().cursor;
@@ -6856,7 +6886,12 @@ public:
 				wait(cur.moveNext());
 			}
 		} else {
-			wait(cur.seekLT(PagerEventReasons::rangeRead, keys.end, prefetchBytes));
+			wait(cur.seekLT(PagerEventReasons::rangeRead, keys.end));
+
+			if (self->prefetch) {
+				cur.prefetch(keys.begin, false, -rowLimit, byteLimit);
+			}
+
 			while (cur.isValid()) {
 				// Read page contents without using waits
 				BTreePage::BinaryTree::Cursor leafCursor = cur.back().cursor;
@@ -6918,7 +6953,7 @@ public:
 		state FlowLock::Releaser releaser(self->m_concurrentReads);
 		++g_redwoodMetrics.metric.opGet;
 
-		wait(cur.seekGTE(PagerEventReasons::pointRead, key, 0));
+		wait(cur.seekGTE(PagerEventReasons::pointRead, key));
 		if (cur.isValid() && cur.get().key == key) {
 			// Return a Value whose arena depends on the source page arena
 			Value v;
@@ -6955,6 +6990,7 @@ private:
 	Promise<Void> m_closed;
 	Promise<Void> m_error;
 	FlowLock m_concurrentReads;
+	bool prefetch;
 
 	template <typename T>
 	inline Future<T> catchError(Future<T> f) {
@@ -7035,12 +7071,13 @@ ACTOR Future<int> verifyRangeBTreeCursor(VersionedBTree* btree,
 		             start.printable().c_str(),
 		             end.printable().c_str(),
 		             randomKey.toString().c_str());
-		wait(success(cur.seek(PagerEventReasons::rangeRead, randomKey, 0)));
+		wait(success(cur.seek(PagerEventReasons::rangeRead, randomKey)));
 	}
 
 	debug_printf(
 	    "VerifyRange(@%" PRId64 ", %s, %s): Actual seek\n", v, start.printable().c_str(), end.printable().c_str());
-	wait(cur.seekGTE(PagerEventReasons::rangeRead, start, 0));
+
+	wait(cur.seekGTE(PagerEventReasons::rangeRead, start));
 
 	state Standalone<VectorRef<KeyValueRef>> results;
 
@@ -7140,7 +7177,7 @@ ACTOR Future<int> verifyRangeBTreeCursor(VersionedBTree* btree,
 	}
 
 	// Now read the range from the tree in reverse order and compare to the saved results
-	wait(cur.seekLT(PagerEventReasons::rangeRead, end, 0));
+	wait(cur.seekLT(PagerEventReasons::rangeRead, end));
 
 	state std::reverse_iterator<const KeyValueRef*> r = results.rbegin();
 
@@ -7217,7 +7254,7 @@ ACTOR Future<int> seekAllBTreeCursor(VersionedBTree* btree,
 			state Optional<std::string> val = i->second;
 			debug_printf("Verifying @%" PRId64 " '%s'\n", ver, key.c_str());
 			state Arena arena;
-			wait(cur.seekGTE(PagerEventReasons::metaData, RedwoodRecordRef(KeyRef(arena, key)), 0));
+			wait(cur.seekGTE(PagerEventReasons::metaData, RedwoodRecordRef(KeyRef(arena, key))));
 			bool foundKey = cur.isValid() && cur.get().key == key;
 			bool hasValue = foundKey && cur.get().value.present();
 
@@ -7342,7 +7379,7 @@ ACTOR Future<Void> randomReader(VersionedBTree* btree) {
 			}
 
 			state KeyValue kv = randomKV(10, 0);
-			wait(cur.seekGTE(PagerEventReasons::pointRead, kv.key, 0));
+			wait(cur.seekGTE(PagerEventReasons::pointRead,kv.key));
 			state int c = deterministicRandom()->randomInt(0, 100);
 			state bool direction = deterministicRandom()->coinflip();
 			while (cur.isValid() && c-- > 0) {
@@ -8922,7 +8959,7 @@ ACTOR Future<Void> randomSeeks(VersionedBTree* btree, int count, char firstChar,
 	wait(btree->initBTreeCursor(&cur, readVer));
 	while (c < count) {
 		state Key k = randomString(20, firstChar, lastChar);
-		wait(cur.seekGTE(PagerEventReasons::pointRead, k, 0));
+		wait(cur.seekGTE(PagerEventReasons::pointRead,k));
 		++c;
 	}
 	double elapsed = timer() - readStart;
@@ -8933,7 +8970,7 @@ ACTOR Future<Void> randomSeeks(VersionedBTree* btree, int count, char firstChar,
 ACTOR Future<Void> randomScans(VersionedBTree* btree,
                                int count,
                                int width,
-                               int readAhead,
+                               int prefetchBytes,
                                char firstChar,
                                char lastChar) {
 	state Version readVer = btree->getLatestVersion();
@@ -8942,29 +8979,34 @@ ACTOR Future<Void> randomScans(VersionedBTree* btree,
 	state VersionedBTree::BTreeCursor cur;
 	wait(btree->initBTreeCursor(&cur, readVer));
 
-	state bool adaptive = readAhead < 0;
 	state int totalScanBytes = 0;
 	while (c++ < count) {
 		state Key k = randomString(20, firstChar, lastChar);
-		wait(cur.seekGTE(PagerEventReasons::pointRead, k, readAhead));
-		if (adaptive) {
-			readAhead = totalScanBytes / c;
-		}
+		wait(cur.seekGTE(PagerEventReasons::pointRead,k));
 		state int w = width;
-		state bool direction = deterministicRandom()->coinflip();
+		state bool directionFwd = deterministicRandom()->coinflip();
+
+		if (prefetchBytes > 0) {
+			cur.prefetch(directionFwd ? VersionedBTree::dbEnd.key : VersionedBTree::dbBegin.key,
+			             directionFwd,
+			             width,
+			             prefetchBytes);
+		}
+
 		while (w > 0 && cur.isValid()) {
 			totalScanBytes += cur.get().expectedSize();
-			wait(success(direction ? cur.moveNext() : cur.movePrev()));
+			wait(success(directionFwd ? cur.moveNext() : cur.movePrev()));
 			--w;
 		}
 	}
 	double elapsed = timer() - readStart;
-	printf("Completed %d scans: readAhead=%d width=%d bytesRead=%d scansRate=%d/s\n",
+	printf("Completed %d scans: width=%d totalbytesRead=%d prefetchBytes=%d scansRate=%d scans/s  %.2f MB/s\n",
 	       count,
-	       readAhead,
 	       width,
 	       totalScanBytes,
-	       int(count / elapsed));
+	       prefetchBytes,
+	       int(count / elapsed),
+	       double(totalScanBytes) / 1e6 / elapsed);
 	return Void();
 }
 
@@ -9192,6 +9234,8 @@ TEST_CASE(":/redwood/performance/set") {
 	state int concurrentScans = params.getInt("concurrentScans").orDefault(64);
 	state int seeks = params.getInt("seeks").orDefault(1000000);
 	state int scans = params.getInt("scans").orDefault(20000);
+	state int scanWidth = params.getInt("scanWidth").orDefault(50);
+	state int scanPrefetchBytes = params.getInt("scanPrefetchBytes").orDefault(0);
 	state bool pagerMemoryOnly = params.getInt("pagerMemoryOnly").orDefault(0);
 	state bool traceMetrics = params.getInt("traceMetrics").orDefault(0);
 
@@ -9215,6 +9259,8 @@ TEST_CASE(":/redwood/performance/set") {
 	printf("concurrentSeeks: %d\n", concurrentSeeks);
 	printf("seeks: %d\n", seeks);
 	printf("scans: %d\n", scans);
+	printf("scanWidth: %d\n", scanWidth);
+	printf("scanPrefetchBytes: %d\n", scanPrefetchBytes);
 	printf("fileName: %s\n", fileName.c_str());
 	printf("openExisting: %d\n", openExisting);
 	printf("insertRecords: %d\n", insertRecords);
@@ -9327,9 +9373,14 @@ TEST_CASE(":/redwood/performance/set") {
 	}
 
 	if (scans > 0) {
-		printf("Parallel scans, count=%d, concurrency=%d, no readAhead ...\n", scans, concurrentScans);
+		printf("Parallel scans, concurrency=%d, scans=%d, scanWidth=%d, scanPreftchBytes=%d ...\n",
+		       concurrentScans,
+		       scans,
+		       scanWidth,
+		       scanPrefetchBytes);
 		for (int x = 0; x < concurrentScans; ++x) {
-			actors.add(randomScans(btree, scans / concurrentScans, 50, 0, firstKeyChar, lastKeyChar));
+			actors.add(
+			    randomScans(btree, scans / concurrentScans, scanWidth, scanPrefetchBytes, firstKeyChar, lastKeyChar));
 		}
 		wait(actors.signalAndReset());
 		if (!traceMetrics) {
@@ -9338,7 +9389,7 @@ TEST_CASE(":/redwood/performance/set") {
 	}
 
 	if (seeks > 0) {
-		printf("Parallel seeks, count=%d, concurrency=%d ...\n", seeks, concurrentSeeks);
+		printf("Parallel seeks, concurrency=%d, seeks=%d ...\n", concurrentSeeks, seeks);
 		for (int x = 0; x < concurrentSeeks; ++x) {
 			actors.add(randomSeeks(btree, seeks / concurrentSeeks, firstKeyChar, lastKeyChar));
 		}

fdbserver/WorkerInterface.actor.h

@@ -31,6 +31,7 @@
 #include "fdbserver/TLogInterface.h"
 #include "fdbserver/RatekeeperInterface.h"
 #include "fdbserver/ResolverInterface.h"
+#include "fdbclient/ClientBooleanParams.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbclient/FDBTypes.h"
@@ -832,8 +833,8 @@ struct ServerDBInfo;
 
 class Database openDBOnServer(Reference<AsyncVar<ServerDBInfo>> const& db,
                               TaskPriority taskID = TaskPriority::DefaultEndpoint,
-                              bool enableLocalityLoadBalance = true,
-                              bool lockAware = false);
+                              LockAware = LockAware::FALSE,
+                              EnableLocalityLoadBalance = EnableLocalityLoadBalance::TRUE);
 ACTOR Future<Void> extractClusterInterface(Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>>> a,
                                            Reference<AsyncVar<Optional<struct ClusterInterface>>> b);
 

fdbserver/fdbserver.actor.cpp

@@ -1641,8 +1641,9 @@ int main(int argc, char* argv[]) {
 		enableBuggify(opts.buggifyEnabled, BuggifyType::General);
 
 		IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::SERVER,
-		                                         Randomize::YES,
-		                                         role == ServerRole::Simulation ? IsSimulated::YES : IsSimulated::NO);
+		                                         Randomize::TRUE,
+		                                         role == ServerRole::Simulation ? IsSimulated::TRUE
+		                                                                        : IsSimulated::FALSE);
 		IKnobCollection::getMutableGlobalKnobCollection().setKnob("log_directory", KnobValue::create(opts.logFolder));
 		if (role != ServerRole::Simulation) {
 			IKnobCollection::getMutableGlobalKnobCollection().setKnob("commit_batches_mem_bytes_hard_limit",
@@ -1677,7 +1678,7 @@ int main(int argc, char* argv[]) {
 		                                                          KnobValue::create(int64_t{ opts.memLimit }));
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
 		IKnobCollection::getMutableGlobalKnobCollection().initialize(
-		    Randomize::YES, role == ServerRole::Simulation ? IsSimulated::YES : IsSimulated::NO);
+		    Randomize::TRUE, role == ServerRole::Simulation ? IsSimulated::TRUE : IsSimulated::FALSE);
 
 		// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
 		EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY);

fdbserver/masterserver.actor.cpp

@@ -577,7 +577,7 @@ Future<Void> sendMasterRegistration(MasterData* self,
 }
 
 ACTOR Future<Void> updateRegistration(Reference<MasterData> self, Reference<ILogSystem> logSystem) {
-	state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, true, true);
+	state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE);
 	state Future<Void> trigger = self->registrationTrigger.onTrigger();
 	state Future<Void> updateLogsKey;
 
@@ -1965,7 +1965,7 @@ ACTOR Future<Void> masterCore(Reference<MasterData> self) {
 		self->addActor.send(resolutionBalancing(self));
 
 	self->addActor.send(changeCoordinators(self));
-	Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, true, true);
+	Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE);
 	self->addActor.send(configurationMonitor(self, cx));
 	if (self->configuration.backupWorkerEnabled) {
 		self->addActor.send(recruitBackupWorkers(self, cx));

fdbserver/storageserver.actor.cpp

@@ -852,7 +852,7 @@ public:
 		newestDirtyVersion.insert(allKeys, invalidVersion);
 		addShard(ShardInfo::newNotAssigned(allKeys));
 
-		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
+		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE);
 	}
 
 	//~StorageServer() { fclose(log); }
@@ -2790,7 +2790,7 @@ ACTOR Future<Void> tryGetRange(PromiseStream<RangeResult> results, Transaction*
 		loop {
 			GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED, SERVER_KNOBS->FETCH_BLOCK_BYTES);
 			limits.minRows = 0;
-			state RangeResult rep = wait(tr->getRange(begin, end, limits, true));
+			state RangeResult rep = wait(tr->getRange(begin, end, limits, Snapshot::TRUE));
 			if (!rep.more) {
 				rep.readThrough = keys.end;
 			}
@@ -2903,7 +2903,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			tr.info.taskID = TaskPriority::FetchKeys;
 			state PromiseStream<RangeResult> results;
 			state Future<Void> hold = SERVER_KNOBS->FETCH_USING_STREAMING
-			                              ? tr.getRangeStream(results, keys, GetRangeLimits(), true)
+			                              ? tr.getRangeStream(results, keys, GetRangeLimits(), Snapshot::TRUE)
 			                              : tryGetRange(results, &tr, keys);
 			state Key nfk = keys.begin;
 
@@ -2970,7 +2970,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 
 					// FIXME: remove when we no longer support upgrades from 5.X
 					if (debug_getRangeRetries >= 100) {
-						data->cx->enableLocalityLoadBalance = false;
+						data->cx->enableLocalityLoadBalance = EnableLocalityLoadBalance::FALSE;
 						TraceEvent(SevWarnAlways, "FKDisableLB").detail("FKID", fetchKeysID);
 					}
 
@@ -3018,7 +3018,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		}
 
 		// FIXME: remove when we no longer support upgrades from 5.X
-		data->cx->enableLocalityLoadBalance = true;
+		data->cx->enableLocalityLoadBalance = EnableLocalityLoadBalance::TRUE;
 		TraceEvent(SevWarnAlways, "FKReenableLB").detail("FKID", fetchKeysID);
 
 		// We have completed the fetch and write of the data, now we wait for MVCC window to pass.

fdbserver/tester.actor.cpp

@@ -607,7 +607,7 @@ ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
 		startRole(Role::TESTER, workIface.id(), UID(), details);
 
 		if (work.useDatabase) {
-			cx = Database::createDatabase(ccf, -1, true, locality);
+			cx = Database::createDatabase(ccf, -1, IsInternal::TRUE, locality);
 			wait(delay(1.0));
 		}
 
@@ -1468,7 +1468,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 		}
 
 		if (perpetualWiggleEnabled) { // restore the enabled perpetual storage wiggle setting
-			wait(setPerpetualStorageWiggle(cx, true, true));
+			wait(setPerpetualStorageWiggle(cx, true, LockAware::TRUE));
 		}
 	}
 

fdbserver/worker.actor.cpp

@@ -138,8 +138,8 @@ ACTOR static Future<Void> extractClientInfo(Reference<AsyncVar<ServerDBInfo>> db
 
 Database openDBOnServer(Reference<AsyncVar<ServerDBInfo>> const& db,
                         TaskPriority taskID,
-                        bool enableLocalityLoadBalance,
-                        bool lockAware) {
+                        LockAware lockAware,
+                        EnableLocalityLoadBalance enableLocalityLoadBalance) {
 	auto info = makeReference<AsyncVar<ClientDBInfo>>();
 	auto cx = DatabaseContext::create(info,
 	                                  extractClientInfo(db, info),
@@ -1215,15 +1215,15 @@ ACTOR Future<Void> workerServer(Reference<ClusterConnectionFile> connFile,
 		if (metricsConnFile.size() > 0) {
 			try {
 				state Database db =
-				    Database::createDatabase(metricsConnFile, Database::API_VERSION_LATEST, true, locality);
+				    Database::createDatabase(metricsConnFile, Database::API_VERSION_LATEST, IsInternal::TRUE, locality);
 				metricsLogger = runMetrics(db, KeyRef(metricsPrefix));
 			} catch (Error& e) {
 				TraceEvent(SevWarnAlways, "TDMetricsBadClusterFile").error(e).detail("ConnFile", metricsConnFile);
 			}
 		} else {
-			bool lockAware = metricsPrefix.size() && metricsPrefix[0] == '\xff';
-			metricsLogger = runMetrics(openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, lockAware),
-			                           KeyRef(metricsPrefix));
+			auto lockAware = metricsPrefix.size() && metricsPrefix[0] == '\xff' ? LockAware::TRUE : LockAware::FALSE;
+			metricsLogger =
+			    runMetrics(openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, lockAware), KeyRef(metricsPrefix));
 		}
 	}
 

fdbserver/workloads/ApiCorrectness.actor.cpp

@@ -435,6 +435,8 @@ public:
 	// Gets a single range of values from the database and memory stores and compares them, returning true if the
 	// results were the same
 	ACTOR Future<bool> runGetRange(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
+		state Reverse reverse = deterministicRandom()->coinflip();
+
 		// Generate a random range
 		Key key = self->selectRandomKey(data, 0.5);
 		Key key2 = self->selectRandomKey(data, 0.5);
@@ -444,7 +446,6 @@ public:
 
 		// Generate a random maximum number of results
 		state int limit = deterministicRandom()->randomInt(0, 101);
-		state bool reverse = deterministicRandom()->random01() > 0.5 ? false : true;
 
 		// Get the range from memory
 		state RangeResult storeResults = self->store.getRange(KeyRangeRef(start, end), limit, reverse);
@@ -480,6 +481,8 @@ public:
 	// Gets a single range of values using key selectors from the database and memory store and compares them, returning
 	// true if the results were the same
 	ACTOR Future<bool> runGetRangeSelector(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
+		state Reverse reverse = deterministicRandom()->coinflip();
+
 		KeySelector selectors[2];
 		Key keys[2];
 
@@ -530,7 +533,6 @@ public:
 
 		// Choose a random maximum number of results
 		state int limit = deterministicRandom()->randomInt(0, 101);
-		state bool reverse = deterministicRandom()->random01() < 0.5 ? false : true;
 
 		// Get the range from the memory store
 		state RangeResult storeResults = self->store.getRange(KeyRangeRef(startKey, endKey), limit, reverse);

fdbserver/workloads/ApiWorkload.actor.cpp

@@ -167,14 +167,15 @@ ACTOR Future<bool> compareDatabaseToMemory(ApiWorkload* self) {
 
 	loop {
 		// Fetch a subset of the results from each of the database and the memory store and compare them
-		state RangeResult storeResults = self->store.getRange(KeyRangeRef(startKey, endKey), resultsPerRange, false);
+		state RangeResult storeResults =
+		    self->store.getRange(KeyRangeRef(startKey, endKey), resultsPerRange, Reverse::FALSE);
 
 		state Reference<TransactionWrapper> transaction = self->createTransaction();
 		state KeyRangeRef range(startKey, endKey);
 
 		loop {
 			try {
-				state RangeResult dbResults = wait(transaction->getRange(range, resultsPerRange, false));
+				state RangeResult dbResults = wait(transaction->getRange(range, resultsPerRange, Reverse::FALSE));
 
 				// Compare results of database and memory store
 				Version v = wait(transaction->getReadVersion());