From 7bcc0e15f2b0f63004534ebfa104619193b4e3ad Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 5 Feb 2020 14:23:54 -0800
Subject: [PATCH 001/176] Backup worker: enable 50% of time in simulation

Make this randomization a separate one.
---
 fdbserver/SimulatedCluster.actor.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 46ab8fb8ff..e548fa7db3 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -816,7 +816,6 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR
 		set_config(format("log_spill:=%d", logSpill));
 		int logVersion = deterministicRandom()->randomInt( TLogVersion::MIN_RECRUITABLE, TLogVersion::MAX_SUPPORTED+1 );
 		set_config(format("log_version:=%d", logVersion));
-		set_config("backup_worker_enabled:=1");
 	} else {
 		if (deterministicRandom()->random01() < 0.7)
 			set_config(format("log_version:=%d", TLogVersion::MAX_SUPPORTED));
@@ -824,6 +823,10 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR
 			set_config(format("log_spill:=%d", TLogSpillType::DEFAULT));
 	}
 
+	if (deterministicRandom()->random01() < 0.5) {
+		set_config("backup_worker_enabled:=1");
+	}
+
 	if(generateFearless || (datacenters == 2 && deterministicRandom()->random01() < 0.5)) {
 		//The kill region workload relies on the fact that all "0", "2", and "4" are all of the possible primary dcids.
 		StatusObject primaryObj;

From b4ab78764cafb56d3d3979f64f4357201478e46b Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Mon, 3 Feb 2020 16:17:39 -0800
Subject: [PATCH 002/176] FastRestore:Add comment for integrating with new
 backup format

---
 fdbserver/RestoreLoader.actor.cpp   | 14 +++++++++++---
 fdbserver/RestoreMaster.actor.cpp   |  2 ++
 fdbserver/RestoreRoleCommon.actor.h |  2 ++
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 70eedf06f0..bf75158c4f 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -612,6 +612,11 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 }
 
 // Parsing the data blocks in a range file
+// kvOpsIter: saves the parsed versioned-mutations for the sepcific LoadingParam;
+// samplesIter: saves the sampled mutations from the parsed versioned-mutations;
+// bc: backup container to read the backup file
+// version: the version the parsed mutations should be at
+// asset: RestoreAsset about which backup data should be parsed
 ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
     std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
     std::map<LoadingParam, MutationsVec>::iterator samplesIter, LoaderCounters* cc, Reference<IBackupContainer> bc,
@@ -691,9 +696,12 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 	return Void();
 }
 
-// Parse data blocks in a log file into a vector of <string, string> pairs. Each pair.second contains the mutations at a
-// version encoded in pair.first Step 1: decodeLogFileBlock into <string, string> pairs Step 2: Concatenate the
-// pair.second of pairs with the same pair.first.
+// Parse data blocks in a log file into a vector of <string, string> pairs.
+// Each pair.second contains the mutations at a version encoded in pair.first;
+// Step 1: decodeLogFileBlock into <string, string> pairs;
+// Step 2: Concatenate the second of pairs with the same pair.first.
+// pProcessedFileOffset: ensure each data block is processed in order exactly once;
+// pMutationMap: concatenated mutation list string at the mutation's commit version
 ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(NotifiedVersion* pProcessedFileOffset,
                                                            SerializedMutationListMap* pMutationMap,
                                                            SerializedMutationPartMap* pMutationPartMap,
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 59a3c1d63c..a216ba08b4 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -276,6 +276,7 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 	self->initBackupContainer(request.url);
 
 	// Get all backup files' description and save them to files
+	// TODO for Jingyu: Verify all backup files in new backup are collected
 	wait(collectBackupFiles(self->bc, &rangeFiles, &logFiles, cx, request));
 
 	std::sort(rangeFiles.begin(), rangeFiles.end());
@@ -284,6 +285,7 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 		       std::tie(f2.endVersion, f2.beginVersion, f2.fileIndex, f2.fileName);
 	});
 
+	// TODO for Jingyu: Verify new backup files are grouped into correct version batches.
 	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches); // Divide files into version batches
 	self->dumpVersionBatches(self->versionBatches);
 
diff --git a/fdbserver/RestoreRoleCommon.actor.h b/fdbserver/RestoreRoleCommon.actor.h
index 531e652a7e..8679a5e0a2 100644
--- a/fdbserver/RestoreRoleCommon.actor.h
+++ b/fdbserver/RestoreRoleCommon.actor.h
@@ -51,6 +51,8 @@ struct RestoreMasterData;
 
 struct RestoreSimpleRequest;
 
+// VersionedMutationsMap: Key is the version of parsed backup mutations
+// Value MutationsVec is the vector of parsed backup mutations
 using VersionedMutationsMap = std::map<Version, MutationsVec>;
 
 ACTOR Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name);

From fe26037488ec974c8c342b72d52917bd0e569f54 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 13 Feb 2020 15:40:39 -0800
Subject: [PATCH 003/176] Add partitioned logs to BackupContainer

---
 fdbclient/BackupContainer.actor.cpp | 112 +++++++++++++++++-----------
 1 file changed, 67 insertions(+), 45 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 53ddf397df..b097ee692d 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -240,9 +240,13 @@ std::string BackupDescription::toJSON() const {
  *     file written will be after the start version of the snapshot's execution.
  * 
  *   Log files are at file paths like
- *       /logs/.../log,startVersion,endVersion,blockSize
+ *       /plogs/...log,startVersion,endVersion,UID,blocksize,tagID
+ *       /logs/.../log,startVersion,endVersion,UID,blockSize
  *     where ... is a multi level path which sorts lexically into version order and results in approximately 1
- *     unique folder per day containing about 5,000 files.
+ *     unique folder per day containing about 5,000 files. Logs after 7.0 are stored in "plogs"
+ *     directory and are partitioned according to tagIDs (0, 1, 2, ...). Logs before 7.0 are
+ *     stored in "logs" directory and are not partitioned.
+ *
  *
  *   BACKWARD COMPATIBILITY
  *
@@ -329,18 +333,18 @@ public:
 	}
 
 	// The innermost folder covers 100,000 seconds (1e11 versions) which is 5,000 mutation log files at current settings.
-	static std::string logVersionFolderString(Version v, bool mlogs) {
-		return format("%s/%s/", (mlogs ? "mlogs" : "logs"), versionFolderString(v, 11).c_str());
+	static std::string logVersionFolderString(Version v, bool partitioned) {
+		return format("%s/%s/", (partitioned ? "plogs" : "logs"), versionFolderString(v, 11).c_str());
 	}
 
-	Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) override {
+	Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) final {
 		return writeFile(logVersionFolderString(beginVersion, false) +
 		                 format("log,%lld,%lld,%s,%d", beginVersion, endVersion,
 		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize));
 	}
 
 	Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
-	                                                  uint16_t tagId) override {
+	                                                  uint16_t tagId) final {
 		return writeFile(logVersionFolderString(beginVersion, true) +
 		                 format("log,%lld,%lld,%s,%d,%d", beginVersion, endVersion,
 		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize, tagId));
@@ -528,18 +532,19 @@ public:
 		return writeKeyspaceSnapshotFile_impl(Reference<BackupContainerFileSystem>::addRef(this), fileNames, totalBytes);
 	};
 
-	// List log files, unsorted, which contain data at any version >= beginVersion and <= targetVersion
-	Future<std::vector<LogFile>> listLogFiles(Version beginVersion = 0, Version targetVersion = std::numeric_limits<Version>::max()) {
-		// The first relevant log file could have a begin version less than beginVersion based on the knobs which determine log file range size,
-		// so start at an earlier version adjusted by how many versions a file could contain.
+	// List log files, unsorted, which contain data at any version >= beginVersion and <= targetVersion.
+	// "partitioned" flag indicates if new partitioned mutation logs or old logs should be listed.
+	Future<std::vector<LogFile>> listLogFiles(Version beginVersion, Version targetVersion, bool partitioned) {
+		// The first relevant log file could have a begin version less than beginVersion based on the knobs which
+		// determine log file range size, so start at an earlier version adjusted by how many versions a file could
+		// contain.
 		//
 		// Get the cleaned (without slashes) first and last folders that could contain relevant results.
-		bool mlogs = false; // tagged mutation logs
 		std::string firstPath = cleanFolderString(
 		    logVersionFolderString(std::max<Version>(0, beginVersion - CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES *
 		                                                                   CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE),
-		                           mlogs));
-		std::string lastPath = cleanFolderString(logVersionFolderString(targetVersion, mlogs));
+		                           partitioned));
+		std::string lastPath = cleanFolderString(logVersionFolderString(targetVersion, partitioned));
 
 		std::function<bool(std::string const &)> pathFilter = [=](const std::string &folderPath) {
 			// Remove slashes in the given folder path so that the '/' positions in the version folder string do not matter
@@ -549,7 +554,7 @@ public:
 				|| (cleaned > firstPath && cleaned < lastPath);
 		};
 
-		return map(listFiles("logs/", pathFilter), [=](const FilesAndSizesT &files) {
+		return map(listFiles((partitioned ? "plogs/" : "logs/"), pathFilter), [=](const FilesAndSizesT& files) {
 			std::vector<LogFile> results;
 			LogFile lf;
 			for(auto &f : files) {
@@ -636,11 +641,15 @@ public:
 	ACTOR static Future<BackupFileList> dumpFileList_impl(Reference<BackupContainerFileSystem> bc, Version begin, Version end) {
 		state Future<std::vector<RangeFile>> fRanges = bc->listRangeFiles(begin, end);
 		state Future<std::vector<KeyspaceSnapshotFile>> fSnapshots = bc->listKeyspaceSnapshots(begin, end);
-		state Future<std::vector<LogFile>> fLogs = bc->listLogFiles(begin, end);
+		state std::vector<LogFile> logs;
+		state std::vector<LogFile> pLogs;
 
-		wait(success(fRanges) && success(fSnapshots) && success(fLogs));
+		wait(success(fRanges) && success(fSnapshots) &&
+		     store(logs, bc->listLogFiles(begin, end, false)) &&
+		     store(pLogs, bc->listLogFiles(begin, end, true)));
+		logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
-		return BackupFileList({fRanges.get(), fLogs.get(), fSnapshots.get()});
+		return BackupFileList({ fRanges.get(), std::move(logs), fSnapshots.get() });
 	}
 
 	Future<BackupFileList> dumpFileList(Version begin, Version end) override {
@@ -767,7 +776,12 @@ public:
 		}
 
 		state std::vector<LogFile> logs;
-		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd)) && store(desc.snapshots, bc->listKeyspaceSnapshots()));
+		state std::vector<LogFile> pLogs;
+		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) &&
+		     store(pLogs, bc->listLogFiles(scanBegin, scanEnd, true)) &&
+		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
+		// FIXME: check partitioned logs & maybe enable the below line
+		// logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
 		// List logs in version order so log continuity can be analyzed
 		std::sort(logs.begin(), logs.end());
@@ -879,8 +893,10 @@ public:
 		state BackupDescription desc = wait(bc->describeBackup(false, expireEndVersion));
 
 		// Resolve relative versions using max log version
-		expireEndVersion =       resolveRelativeVersion(desc.maxLogEnd, expireEndVersion,       "ExpireEndVersion",       invalid_option_value());
-		restorableBeginVersion = resolveRelativeVersion(desc.maxLogEnd, restorableBeginVersion, "RestorableBeginVersion", invalid_option_value());
+		expireEndVersion =
+		    resolveRelativeVersion(desc.maxLogEnd, expireEndVersion, "ExpireEndVersion", invalid_option_value());
+		restorableBeginVersion = resolveRelativeVersion(desc.maxLogEnd, restorableBeginVersion,
+		                                                "RestorableBeginVersion", invalid_option_value());
 
 		// It would be impossible to have restorability to any version < expireEndVersion after expiring to that version
 		if(restorableBeginVersion < expireEndVersion)
@@ -921,13 +937,17 @@ public:
 			.detail("ScanBeginVersion", scanBegin);
 
 		state std::vector<LogFile> logs;
+		state std::vector<LogFile> pLogs; // partitioned mutation logs
 		state std::vector<RangeFile> ranges;
 
 		if(progress != nullptr) {
 			progress->step = "Listing files";
 		}
 		// Get log files or range files that contain any data at or before expireEndVersion
-		wait(store(logs, bc->listLogFiles(scanBegin, expireEndVersion - 1)) && store(ranges, bc->listRangeFiles(scanBegin, expireEndVersion - 1)));
+		wait(store(logs, bc->listLogFiles(scanBegin, expireEndVersion - 1, false)) &&
+		     store(pLogs, bc->listLogFiles(scanBegin, expireEndVersion - 1, true)) &&
+		     store(ranges, bc->listRangeFiles(scanBegin, expireEndVersion - 1)));
+		logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
 		// The new logBeginVersion will be taken from the last log file, if there is one
 		state Optional<Version> newLogBeginVersion;
@@ -1067,7 +1087,8 @@ public:
 				return Optional<RestorableFileSet>(restorable);
 			}
 
-			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion));
+			// FIXME: check if there are tagged logs. for each tag, there is no version gap.
+			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false));
 
 			// List logs in version order so log continuity can be analyzed
 			std::sort(logs.begin(), logs.end());
@@ -1098,7 +1119,7 @@ public:
 		return Optional<RestorableFileSet>();
 	}
 
-	Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) override {
+	Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) final {
 		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion);
 	}
 
@@ -1183,8 +1204,8 @@ public:
 
 class BackupContainerLocalDirectory : public BackupContainerFileSystem, ReferenceCounted<BackupContainerLocalDirectory> {
 public:
-	void addref() override { return ReferenceCounted<BackupContainerLocalDirectory>::addref(); }
-	void delref() override { return ReferenceCounted<BackupContainerLocalDirectory>::delref(); }
+	void addref() final { return ReferenceCounted<BackupContainerLocalDirectory>::addref(); }
+	void delref() final { return ReferenceCounted<BackupContainerLocalDirectory>::delref(); }
 
 	static std::string getURLFormat() { return "file://</path/to/base/dir/>"; }
 
@@ -1233,7 +1254,7 @@ public:
 		return results;
 	}
 
-	Future<Void> create() override {
+	Future<Void> create() final {
 		// Nothing should be done here because create() can be called by any process working with the container URL, such as fdbbackup.
 		// Since "local directory" containers are by definition local to the machine they are accessed from,
 		// the container's creation (in this case the creation of a directory) must be ensured prior to every file creation,
@@ -1243,11 +1264,11 @@ public:
 	}
 
 	// The container exists if the folder it resides in exists
-	Future<bool> exists() override {
+	Future<bool> exists() final {
 		return directoryExists(m_path);
 	}
 
-	Future<Reference<IAsyncFile>> readFile(std::string path) override {
+	Future<Reference<IAsyncFile>> readFile(std::string path) final {
 		int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED;
 		// Simulation does not properly handle opening the same file from multiple machines using a shared filesystem,
 		// so create a symbolic link to make each file opening appear to be unique.  This could also work in production
@@ -1272,10 +1293,10 @@ public:
 			int blockSize = 0;
 			// Extract block size from the filename, if present
 			size_t lastComma = path.find_last_of(',');
-			if(lastComma != path.npos) {
+			if (lastComma != path.npos) {
 				blockSize = atoi(path.substr(lastComma + 1).c_str());
 			}
-			if(blockSize <= 0) {
+			if (blockSize <= 0) {
 				blockSize = deterministicRandom()->randomInt(1e4, 1e6);
 			}
 			if(deterministicRandom()->random01() < .01) {
@@ -1324,7 +1345,7 @@ public:
 		std::string m_finalFullPath;
 	};
 
-	Future<Reference<IBackupFile>> writeFile(std::string path) override {
+	Future<Reference<IBackupFile>> writeFile(std::string path) final {
 		int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE;
 		std::string fullPath = joinPath(m_path, path);
 		platform::createDirectory(parentDirectory(fullPath));
@@ -1335,12 +1356,12 @@ public:
 		});
 	}
 
-	Future<Void> deleteFile(std::string path) override {
+	Future<Void> deleteFile(std::string path) final {
 		::deleteFile(joinPath(m_path, path));
 		return Void();
 	}
 
-	Future<FilesAndSizesT> listFiles(std::string path, std::function<bool(std::string const &)>) {
+	Future<FilesAndSizesT> listFiles(std::string path, std::function<bool(std::string const&)>) final {
 		FilesAndSizesT results;
 
 		std::vector<std::string> files;
@@ -1360,7 +1381,7 @@ public:
 		return results;
 	}
 
-	Future<Void> deleteContainer(int* pNumDeleted) override {
+	Future<Void> deleteContainer(int* pNumDeleted) final {
 		// In order to avoid deleting some random directory due to user error, first describe the backup
 		// and make sure it has something in it.
 		return map(describeBackup(false, invalidVersion), [=](BackupDescription const &desc) {
@@ -1420,8 +1441,8 @@ public:
 		}
 	}
 
-	void addref() override { return ReferenceCounted<BackupContainerBlobStore>::addref(); }
-	void delref() override { return ReferenceCounted<BackupContainerBlobStore>::delref(); }
+	void addref() final { return ReferenceCounted<BackupContainerBlobStore>::addref(); }
+	void delref() final { return ReferenceCounted<BackupContainerBlobStore>::delref(); }
 
 	static std::string getURLFormat() {
 		return BlobStoreEndpoint::getURLFormat(true) + " (Note: The 'bucket' parameter is required.)";
@@ -1429,7 +1450,7 @@ public:
 
 	virtual ~BackupContainerBlobStore() {}
 
-	Future<Reference<IAsyncFile>> readFile(std::string path) override {
+	Future<Reference<IAsyncFile>> readFile(std::string path) final {
 			return Reference<IAsyncFile>(
 				new AsyncFileReadAheadCache(
 					Reference<IAsyncFile>(new AsyncFileBlobStoreRead(m_bstore, m_bucket, dataPath(path))),
@@ -1466,17 +1487,18 @@ public:
 			return map(m_file->sync(), [=](Void _) { self->m_file.clear(); return Void(); });
 		}
 
-		void addref() override { return ReferenceCounted<BackupFile>::addref(); }
-		void delref() override { return ReferenceCounted<BackupFile>::delref(); }
+		void addref() final { return ReferenceCounted<BackupFile>::addref(); }
+		void delref() final { return ReferenceCounted<BackupFile>::delref(); }
+
 	private:
 		Reference<IAsyncFile> m_file;
 	};
 
-	Future<Reference<IBackupFile>> writeFile(std::string path) override {
+	Future<Reference<IBackupFile>> writeFile(std::string path) final {
 		return Reference<IBackupFile>(new BackupFile(path, Reference<IAsyncFile>(new AsyncFileBlobStoreWrite(m_bstore, m_bucket, dataPath(path)))));
 	}
 
-	Future<Void> deleteFile(std::string path) override {
+	Future<Void> deleteFile(std::string path) final {
 		return m_bstore->deleteObject(m_bucket, dataPath(path));
 	}
 
@@ -1498,7 +1520,7 @@ public:
 		return files;
 	}
 
-	Future<FilesAndSizesT> listFiles(std::string path, std::function<bool(std::string const &)> pathFilter) {
+	Future<FilesAndSizesT> listFiles(std::string path, std::function<bool(std::string const &)> pathFilter) final {
 		return listFiles_impl(Reference<BackupContainerBlobStore>::addRef(this), path, pathFilter);
 	}
 
@@ -1514,12 +1536,12 @@ public:
 		return Void();
 	}
 
-	Future<Void> create() override {
+	Future<Void> create() final {
 		return create_impl(Reference<BackupContainerBlobStore>::addRef(this));
 	}
 
 	// The container exists if the index entry in the blob bucket exists
-	Future<bool> exists() override {
+	Future<bool> exists() final {
 		return m_bstore->objectExists(m_bucket, indexEntry());
 	}
 
@@ -1539,7 +1561,7 @@ public:
 		return Void();
 	}
 
-	Future<Void> deleteContainer(int* pNumDeleted) override {
+	Future<Void> deleteContainer(int* pNumDeleted) final {
 		return deleteContainer_impl(Reference<BackupContainerBlobStore>::addRef(this), pNumDeleted);
 	}
 

From 21feb78f8af1bfd73dfd6042e188cfb0681c5628 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 14 Feb 2020 11:27:02 -0800
Subject: [PATCH 004/176] Add mutation log version names

I.e., BACKUP_AGENT_MLOG_VERSION for 2001 and PARTITIONED_MLOG_VERSION for 4110.
---
 fdbbackup/FileConverter.actor.cpp   |  3 +++
 fdbbackup/FileConverter.h           |  3 ---
 fdbbackup/FileDecoder.actor.cpp     |  2 +-
 fdbclient/BackupContainer.h         |  6 ++++++
 fdbclient/FileBackupAgent.actor.cpp | 12 ++++++------
 fdbserver/BackupWorker.actor.cpp    |  3 ++-
 fdbserver/RestoreCommon.actor.cpp   |  4 ++--
 fdbserver/RestoreLoader.actor.cpp   |  2 +-
 8 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp
index 46beea723b..f0bffa73e1 100644
--- a/fdbbackup/FileConverter.actor.cpp
+++ b/fdbbackup/FileConverter.actor.cpp
@@ -162,6 +162,9 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
 			Version msgVersion = invalidVersion;
 
 			try {
+				// Read block header
+				if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
+
 				while (1) {
 					// If eof reached or first key len bytes is 0xFF then end of block was reached.
 					if (reader.eof() || *reader.rptr == 0xFF) break;
diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index a342a41dd8..fc82e5dfb2 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -59,7 +59,4 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 
 }  // namespace file_converter
 
-// Mutation log version written by old FileBackupAgent
-static const uint32_t BACKUP_AGENT_MLOG_VERSION = 2001;
-
 #endif  // FDBBACKUP_FILECONVERTER_H
diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index ab4257885c..02b98e4825 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -290,7 +290,7 @@ struct DecodeProgress {
 		StringRefReader reader(block, restore_corrupted_data());
 
 		try {
-			// Read header, currently only decoding version 2001
+			// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
 			if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version();
 
 			// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 4d1ec5ecbe..7bc4ed70e5 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -62,6 +62,12 @@ protected:
 
 // Structures for various backup components
 
+// Mutation log version written by old FileBackupAgent
+static const uint32_t BACKUP_AGENT_MLOG_VERSION = 2001;
+
+// Mutation log version written by BackupWorker
+static const uint32_t PARTITIONED_MLOG_VERSION = 4110;
+
 struct LogFile {
 	Version beginVersion;
 	Version endVersion;
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 0eec26fa8a..8c58cbc162 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -612,7 +612,8 @@ namespace fileBackup {
 	struct LogFileWriter {
 		static const std::string &FFs;
 
-		LogFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0) : file(file), blockSize(blockSize), blockEnd(0), fileVersion(2001) {}
+		LogFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0)
+		  : file(file), blockSize(blockSize), blockEnd(0) {}
 
 		// Start a new block if needed, then write the key and value
 		ACTOR static Future<Void> writeKV_impl(LogFileWriter *self, Key k, Value v) {
@@ -629,8 +630,8 @@ namespace fileBackup {
 				// Set new blockEnd
 				self->blockEnd += self->blockSize;
 
-				// write Header
-				wait(self->file->append((uint8_t *)&self->fileVersion, sizeof(self->fileVersion)));
+				// write the block header
+				wait(self->file->append((uint8_t *)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION)));
 			}
 
 			wait(self->file->appendStringRefWithLen(k));
@@ -650,7 +651,6 @@ namespace fileBackup {
 
 	private:
 		int64_t blockEnd;
-		uint32_t fileVersion;
 	};
 
 	ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file, int64_t offset, int len) {
@@ -663,8 +663,8 @@ namespace fileBackup {
 		state StringRefReader reader(buf, restore_corrupted_data());
 
 		try {
-			// Read header, currently only decoding version 2001
-			if(reader.consume<int32_t>() != 2001)
+			// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
+			if(reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
 				throw restore_unsupported_file_version();
 
 			// Read k/v pairs.  Block ends either at end of last value exactly or with 0xFF as first key len byte.
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 07ddbc06a7..5229340d8d 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -371,7 +371,8 @@ ACTOR Future<Void> addMutation(Reference<IBackupFile> logFile, VersionedMessage
 		}
 
 		*blockEnd += blockSize;
-		// TODO: add block header
+		// write block Header
+		wait(logFile->append((uint8_t*)&PARTITIONED_MLOG_VERSION, sizeof(PARTITIONED_MLOG_VERSION)));
 	}
 
 	wait(logFile->append((void*)header.begin(), header.size()));
diff --git a/fdbserver/RestoreCommon.actor.cpp b/fdbserver/RestoreCommon.actor.cpp
index 0c336538da..7ea783c04a 100644
--- a/fdbserver/RestoreCommon.actor.cpp
+++ b/fdbserver/RestoreCommon.actor.cpp
@@ -396,8 +396,8 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IA
 	state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
 
 	try {
-		// Read header, currently only decoding version 2001
-		if (reader.consume<int32_t>() != 2001) throw restore_unsupported_file_version();
+		// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
+		if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version();
 
 		// Read k/v pairs.  Block ends either at end of last value exactly or with 0xFF as first key len byte.
 		while (1) {
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index bf75158c4f..22646b307a 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

From 3c088b2352a84372dc50c65401bbb7cb568ab81b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 17 Feb 2020 14:36:09 -0800
Subject: [PATCH 005/176] Integrate parallel restore with partitioned logs

In parallel restore, use new getPartitionedRestoreSet() to get a set containing
partitioned mutation logs. The loader uses a new parser to extract mutations
from partitioned logs.

TODO: fix unable to restore errors.
---
 fdbclient/BackupContainer.actor.cpp      |  14 ++-
 fdbclient/BackupContainer.h              |   5 +
 fdbclient/RestoreWorkerInterface.actor.h |  18 ++--
 fdbserver/RestoreCommon.actor.h          |  26 +++--
 fdbserver/RestoreLoader.actor.cpp        | 119 ++++++++++++++++++++---
 fdbserver/RestoreMaster.actor.cpp        |  15 +--
 fdbserver/RestoreMaster.actor.h          |   2 +-
 7 files changed, 153 insertions(+), 46 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index b097ee692d..98215aabcd 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1064,7 +1064,7 @@ public:
 		return expireData_impl(Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
 	}
 
-	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion) {
+	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion, bool partitioned) {
 		// Find the most recent keyrange snapshot to end at or before targetVersion
 		state Optional<KeyspaceSnapshotFile> snapshot;
 		std::vector<KeyspaceSnapshotFile> snapshots = wait(bc->listKeyspaceSnapshots());
@@ -1088,11 +1088,15 @@ public:
 			}
 
 			// FIXME: check if there are tagged logs. for each tag, there is no version gap.
-			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false));
+			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion, partitioned));
 
 			// List logs in version order so log continuity can be analyzed
 			std::sort(logs.begin(), logs.end());
 
+			// TODO(jingyu): for partitioned logs, the continuity checking should be based on
+			// epochs and versions, which should be saved in a metadata file by backup worker and
+			// thus is available here.
+
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
 			if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {
 				auto i = logs.begin();
@@ -1120,7 +1124,11 @@ public:
 	}
 
 	Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) final {
-		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion);
+		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion, false);
+	}
+
+	Future<Optional<RestorableFileSet>> getPartitionedRestoreSet(Version targetVersion) final {
+		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion, true);
 	}
 
 private:
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 7bc4ed70e5..9c4526e6f4 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -260,6 +260,11 @@ public:
 	// restore to given version is not possible.
 	virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) = 0;
 
+	// Get exactly the files necessary to restore to targetVersion. Returns non-present if
+	// restore to given version is not possible. This is intended for parallel
+	// restore in FDB 7.0, which reads partitioned mutation logs.
+	virtual Future<Optional<RestorableFileSet>> getPartitionedRestoreSet(Version targetVersion) = 0;
+
 	// Get an IBackupContainer based on a container spec string
 	static Reference<IBackupContainer> openContainer(std::string url);
 	static std::vector<std::string> getURLFormats();
diff --git a/fdbclient/RestoreWorkerInterface.actor.h b/fdbclient/RestoreWorkerInterface.actor.h
index 684a12c44e..84c2f603e8 100644
--- a/fdbclient/RestoreWorkerInterface.actor.h
+++ b/fdbclient/RestoreWorkerInterface.actor.h
@@ -209,6 +209,8 @@ struct RestoreAsset {
 	KeyRange range; // Only use mutations in range
 
 	int fileIndex;
+	// Partition ID for mutation log files, which is also encoded in the filename of mutation logs.
+	int partitionId = -1;
 	std::string filename;
 	int64_t offset;
 	int64_t len;
@@ -218,12 +220,12 @@ struct RestoreAsset {
 	RestoreAsset() = default;
 
 	bool operator==(const RestoreAsset& r) const {
-		return fileIndex == r.fileIndex && filename == r.filename && offset == r.offset && len == r.len &&
-		       beginVersion == r.beginVersion && endVersion == r.endVersion && range == r.range;
+		return beginVersion == r.beginVersion && endVersion == r.endVersion && range == r.range &&
+		       fileIndex == r.fileIndex && partitionId == r.partitionId && filename == r.filename &&
+		       offset == r.offset && len == r.len;
 	}
 	bool operator!=(const RestoreAsset& r) const {
-		return fileIndex != r.fileIndex || filename != r.filename || offset != r.offset || len != r.len ||
-		       beginVersion != r.beginVersion || endVersion != r.endVersion || range != r.range;
+		return !(*this == r);
 	}
 	bool operator<(const RestoreAsset& r) const {
 		return std::make_tuple(fileIndex, filename, offset, len, beginVersion, endVersion, range.begin, range.end) <
@@ -233,14 +235,14 @@ struct RestoreAsset {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, beginVersion, endVersion, range, filename, fileIndex, offset, len, uid);
+		serializer(ar, beginVersion, endVersion, range, filename, fileIndex, partitionId, offset, len, uid);
 	}
 
 	std::string toString() {
 		std::stringstream ss;
 		ss << "UID:" << uid.toString() << " begin:" << beginVersion << " end:" << endVersion
 		   << " range:" << range.toString() << " filename:" << filename << " fileIndex:" << fileIndex
-		   << " offset:" << offset << " len:" << len;
+		   << " partitionId:" << partitionId << " offset:" << offset << " len:" << len;
 		return ss.str();
 	}
 
@@ -269,6 +271,10 @@ struct LoadingParam {
 		return (isRangeFile < r.isRangeFile) || (isRangeFile == r.isRangeFile && asset < r.asset);
 	}
 
+	bool isPartitionedLog() const {
+		return !isRangeFile && asset.partitionId >= 0;
+	}
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, isRangeFile, url, rangeVersion, blockSize, asset);
diff --git a/fdbserver/RestoreCommon.actor.h b/fdbserver/RestoreCommon.actor.h
index 72d86d8d49..ea0e54837d 100644
--- a/fdbserver/RestoreCommon.actor.h
+++ b/fdbserver/RestoreCommon.actor.h
@@ -188,6 +188,7 @@ struct RestoreFileFR {
 	int64_t cursor; // The start block location to be restored. All blocks before cursor have been scheduled to load and
 	                // restore
 	int fileIndex; // index of backup file. Must be identical per file.
+	int partitionId = -1; // Partition ID (Log Router Tag ID) for mutation files.
 
 	Tuple pack() const {
 		return Tuple()
@@ -199,7 +200,8 @@ struct RestoreFileFR {
 		    .append(endVersion)
 		    .append(beginVersion)
 		    .append(cursor)
-		    .append(fileIndex);
+		    .append(fileIndex)
+		    .append(partitionId);
 	}
 	static RestoreFileFR unpack(Tuple const& t) {
 		RestoreFileFR r;
@@ -213,6 +215,7 @@ struct RestoreFileFR {
 		r.beginVersion = t.getInt(i++);
 		r.cursor = t.getInt(i++);
 		r.fileIndex = t.getInt(i++);
+		r.partitionId = t.getInt(i++);
 		return r;
 	}
 
@@ -225,18 +228,21 @@ struct RestoreFileFR {
 	  : version(invalidVersion), isRange(false), blockSize(0), fileSize(0), endVersion(invalidVersion),
 	    beginVersion(invalidVersion), cursor(0), fileIndex(0) {}
 
-	RestoreFileFR(Version version, std::string fileName, bool isRange, int64_t blockSize, int64_t fileSize,
-	              Version endVersion, Version beginVersion)
-	  : version(version), fileName(fileName), isRange(isRange), blockSize(blockSize), fileSize(fileSize),
-	    endVersion(endVersion), beginVersion(beginVersion), cursor(0), fileIndex(0) {}
+	explicit RestoreFileFR(const RangeFile& f)
+	  : version(f.version), fileName(f.fileName), isRange(true), blockSize(f.blockSize), fileSize(f.fileSize),
+	    endVersion(f.version), beginVersion(f.version), cursor(0), fileIndex(0) {}
+
+	explicit RestoreFileFR(const LogFile& f)
+	  : version(f.beginVersion), fileName(f.fileName), isRange(false), blockSize(f.blockSize), fileSize(f.fileSize),
+	    endVersion(f.endVersion), beginVersion(f.beginVersion), cursor(0), fileIndex(0), partitionId(f.tagId) {}
 
 	std::string toString() const {
 		std::stringstream ss;
-		ss << "version:" << std::to_string(version) << " fileName:" << fileName
-		   << " isRange:" << std::to_string(isRange) << " blockSize:" << std::to_string(blockSize)
-		   << " fileSize:" << std::to_string(fileSize) << " endVersion:" << std::to_string(endVersion)
-		   << " beginVersion:" << std::to_string(beginVersion) << " cursor:" << std::to_string(cursor)
-		   << " fileIndex:" << std::to_string(fileIndex);
+		ss << "version:" << version << " fileName:" << fileName
+		   << " isRange:" << isRange << " blockSize:" << blockSize
+		   << " fileSize:" << fileSize << " endVersion:" << endVersion
+		   << " beginVersion:" << beginVersion << " cursor:" << cursor
+		   << " fileIndex:" << fileIndex << " partitionId:" << partitionId;
 		return ss.str();
 	}
 };
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 22646b307a..79bfb057d3 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -23,6 +23,7 @@
 
 #include "fdbclient/BackupContainer.h"
 #include "fdbserver/RestoreLoader.actor.h"
+#include "fdbserver/RestoreRoleCommon.actor.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -140,6 +141,90 @@ void handleRestoreSysInfoRequest(const RestoreSysInfoRequest& req, Reference<Res
 	req.reply.send(RestoreCommonReply(self->id()));
 }
 
+// Parse a data block in a partitioned mutation log file and store mutations
+// into "kvOpsIter" and samples into "samplesIter".
+ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
+    NotifiedVersion* processedFileOffset, std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
+    std::map<LoadingParam, MutationsVec>::iterator samplesIter, Reference<IBackupContainer> bc, RestoreAsset asset) {
+	state Standalone<StringRef> buf = makeString(asset.len);
+	state Reference<IAsyncFile> file = wait(bc->readFile(asset.filename));
+	int rLen = wait(file->read(mutateString(buf), asset.len, asset.offset));
+	if (rLen != asset.len) throw restore_bad_read();
+
+	TraceEvent("FastRestore")
+	    .detail("DecodingLogFile", asset.filename)
+	    .detail("Offset", asset.offset)
+	    .detail("Length", asset.len);
+
+	// Ensure data blocks in the same file are processed in order
+	wait(processedFileOffset->whenAtLeast(asset.offset));
+	ASSERT(processedFileOffset->get() == asset.offset);
+
+	BackupStringRefReader reader(buf, restore_corrupted_data());
+	try {
+		// Read block header
+		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
+
+		Version lastVersion = invalidVersion;
+		VersionedMutationsMap& kvOps = kvOpsIter->second;
+		VersionedMutationsMap::iterator it = kvOps.end();
+		while (1) {
+			// If eof reached or first key len bytes is 0xFF then end of block was reached.
+			if (reader.eof() || *reader.rptr == 0xFF) break;
+
+			// Deserialize messages written in saveMutationsToFile().
+			Version msgVersion = bigEndian64(reader.consume<Version>());
+			uint32_t sub = bigEndian32(reader.consume<uint32_t>());
+			int msgSize = bigEndian32(reader.consume<int>());
+			const uint8_t* message = reader.consume(msgSize);
+
+			// Skip mutations out of the version range
+			if (!asset.isInVersionRange(msgVersion)) continue;
+
+			if (lastVersion != msgVersion) {
+				bool inserted;
+				std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec());
+				lastVersion = msgVersion;
+			}
+			ASSERT(it != kvOps.end());
+
+			ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(currentProtocolVersion));
+			MutationRef mutation;
+			rd >> mutation;
+
+			// Should this mutation be skipped?
+			if (mutation.param1 >= asset.range.end ||
+			    (isRangeMutation(mutation) && mutation.param2 < asset.range.begin) ||
+			    (!isRangeMutation(mutation) && mutation.param1 < asset.range.begin)) {
+				continue;
+			}
+			// Only apply mutation within the asset.range
+			if (isRangeMutation(mutation)) {
+				mutation.param1 = mutation.param1 >= asset.range.begin ? mutation.param1 : asset.range.begin;
+				mutation.param2 = mutation.param2 < asset.range.end ? mutation.param2 : asset.range.end;
+			}
+
+			TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
+			    .detail("CommitVersion", msgVersion)
+			    .detail("ParsedMutation", mutation.toString());
+			it->second.push_back_deep(it->second.arena(), mutation);
+			// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
+			if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {
+				samplesIter->second.push_back_deep(samplesIter->second.arena(), mutation);
+			}
+		}
+	} catch (Error& e) {
+		TraceEvent(SevWarn, "FileRestoreCorruptLogFileBlock")
+		    .error(e)
+		    .detail("Filename", file->getFilename())
+		    .detail("BlockOffset", asset.offset)
+		    .detail("BlockLen", asset.len);
+		throw;
+	}
+	processedFileOffset->set(asset.offset + asset.len);
+	return Void();
+}
+
 ACTOR Future<Void> _processLoadingParam(LoadingParam param, Reference<LoaderBatchData> batchData, UID loaderID,
                                         Reference<IBackupContainer> bc) {
 	// Temporary data structure for parsing log files into (version, <K, V, mutationType>)
@@ -155,15 +240,15 @@ ACTOR Future<Void> _processLoadingParam(LoadingParam param, Reference<LoaderBatc
 	// Q: How to record the  param's fields inside LoadingParam Refer to storageMetrics
 	TraceEvent("FastRestoreLoaderProcessLoadingParam", loaderID).detail("LoadingParam", param.toString());
 	ASSERT(param.blockSize > 0);
-	ASSERT(param.asset.offset % param.blockSize == 0); // Parse file must be at block bondary.
+	ASSERT(param.asset.offset % param.blockSize == 0); // Parse file must be at block boundary.
 	ASSERT(batchData->kvOpsPerLP.find(param) == batchData->kvOpsPerLP.end());
 
 	// NOTE: map's iterator is guaranteed to be stable, but pointer may not.
-	// state VersionedMutationsMap* kvOps = &self->kvOpsPerLP[param];
-	batchData->kvOpsPerLP.emplace(param, VersionedMutationsMap());
-	batchData->sampleMutations.emplace(param, MutationsVec());
-	kvOpsPerLPIter = batchData->kvOpsPerLP.find(param);
-	samplesIter = batchData->sampleMutations.find(param);
+	bool inserted;
+	std::tie(kvOpsPerLPIter, inserted) = batchData->kvOpsPerLP.emplace(param, VersionedMutationsMap());
+	ASSERT(inserted);
+	std::tie(samplesIter, inserted) = batchData->sampleMutations.emplace(param, MutationsVec());
+	ASSERT(inserted);
 
 	for (int64_t j = param.asset.offset; j < param.asset.len; j += param.blockSize) {
 		RestoreAsset subAsset = param.asset;
@@ -174,13 +259,18 @@ ACTOR Future<Void> _processLoadingParam(LoadingParam param, Reference<LoaderBatc
 			    kvOpsPerLPIter, samplesIter, &batchData->counters, bc, param.rangeVersion.get(), subAsset));
 		} else {
 			// TODO: Sanity check the log file's range is overlapped with the restored version range
-			fileParserFutures.push_back(
-			    _parseLogFileToMutationsOnLoader(&processedFileOffset, &mutationMap, &mutationPartMap, bc, subAsset));
+			if (param.isPartitionedLog()) {
+				fileParserFutures.push_back(_parsePartitionedLogFileOnLoader(&processedFileOffset, kvOpsPerLPIter,
+				                                                             samplesIter, bc, subAsset));
+			} else {
+				fileParserFutures.push_back(_parseLogFileToMutationsOnLoader(&processedFileOffset, &mutationMap,
+				                                                             &mutationPartMap, bc, subAsset));
+			}
 		}
 	}
 	wait(waitForAll(fileParserFutures));
 
-	if (!param.isRangeFile) {
+	if (!param.isRangeFile && !param.isPartitionedLog()) {
 		_parseSerializedMutation(kvOpsPerLPIter, &mutationMap, samplesIter, &batchData->counters, param.asset);
 	}
 
@@ -508,15 +598,15 @@ bool concatenateBackupMutationForLogFile(std::map<Standalone<StringRef>, Standal
 	// Use commitVersion as id
 	Standalone<StringRef> id = StringRef((uint8_t*)&commitVersion, sizeof(Version));
 
-	if (mutationMap.find(id) == mutationMap.end()) {
+	auto it = mutationMap.find(id);
+	if (it == mutationMap.end()) {
 		mutationMap.insert(std::make_pair(id, val_input));
 		if (part != 0) {
 			TraceEvent(SevError, "FastRestore").detail("FirstPartNotZero", part).detail("KeyInput", getHexString(key_input));
 		}
 		mutationPartMap.insert(std::make_pair(id, part));
 	} else { // Concatenate the val string with the same commitVersion
-		mutationMap[id] =
-		    mutationMap[id].contents().withSuffix(val_input.contents()); // Assign the new Areana to the map's value
+		it->second = it->second.contents().withSuffix(val_input.contents()); // Assign the new Areana to the map's value
 		if (part != (mutationPartMap[id] + 1)) {
 			// Check if the same range or log file has been processed more than once!
 			TraceEvent(SevError, "FastRestore")
@@ -722,14 +812,11 @@ ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(NotifiedVersion* pPro
 	if (pProcessedFileOffset->get() == asset.offset) {
 		int start = 0;
 		int end = data.size();
-		int numConcatenated = 0;
 		for (int i = start; i < end; ++i) {
 			// Key k = data[i].key.withPrefix(mutationLogPrefix);
 			// ValueRef v = data[i].value;
 			// Concatenate the backuped param1 and param2 (KV) at the same version.
-			bool concatenated =
-			    concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, data[i].key, data[i].value, asset);
-			numConcatenated += (concatenated ? 1 : 0);
+			concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, data[i].key, data[i].value, asset);
 		}
 		pProcessedFileOffset->set(asset.offset + asset.len);
 	}
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index a216ba08b4..51c168cc5f 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -276,7 +276,6 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 	self->initBackupContainer(request.url);
 
 	// Get all backup files' description and save them to files
-	// TODO for Jingyu: Verify all backup files in new backup are collected
 	wait(collectBackupFiles(self->bc, &rangeFiles, &logFiles, cx, request));
 
 	std::sort(rangeFiles.begin(), rangeFiles.end());
@@ -337,12 +336,7 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<MasterBatchData> batchDat
                                              Database cx, RestoreRequest request, VersionBatch versionBatch,
                                              bool isRangeFile) {
 	// set is internally sorted
-	std::set<RestoreFileFR>* files = nullptr;
-	if (isRangeFile) {
-		files = &versionBatch.rangeFiles;
-	} else {
-		files = &versionBatch.logFiles;
-	}
+	std::set<RestoreFileFR>* files = isRangeFile ? &versionBatch.rangeFiles : &versionBatch.logFiles;
 
 	TraceEvent("FastRestoreMasterPhaseLoadFilesStart")
 	    .detail("BatchIndex", batchIndex)
@@ -376,6 +370,7 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<MasterBatchData> batchDat
 		param.asset.uid = deterministicRandom()->randomUniqueID();
 		param.asset.filename = file.fileName;
 		param.asset.fileIndex = file.fileIndex;
+		param.asset.partitionId = file.partitionId;
 		param.asset.offset = 0;
 		param.asset.len = file.fileSize;
 		param.asset.range = request.range;
@@ -692,7 +687,7 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 		request.targetVersion = desc.maxRestorableVersion.get();
 	}
 
-	Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(request.targetVersion));
+	Optional<RestorableFileSet> restorable = wait(bc->getPartitionedRestoreSet(request.targetVersion));
 
 	if (!restorable.present()) {
 		TraceEvent(SevWarn, "FastRestoreMasterPhaseCollectBackupFiles").detail("NotRestorable", request.targetVersion);
@@ -709,7 +704,7 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 		if (f.fileSize <= 0) {
 			continue;
 		}
-		RestoreFileFR file(f.version, f.fileName, true, f.blockSize, f.fileSize, f.version, f.version);
+		RestoreFileFR file(f);
 		TraceEvent("FastRestoreMasterPhaseCollectBackupFiles").detail("RangeFileFR", file.toString());
 		uniqueRangeFiles.insert(file);
 	}
@@ -718,7 +713,7 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 		if (f.fileSize <= 0) {
 			continue;
 		}
-		RestoreFileFR file(f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion, f.beginVersion);
+		RestoreFileFR file(f);
 		TraceEvent("FastRestoreMasterPhaseCollectBackupFiles").detail("LogFileFR", file.toString());
 		logFiles->push_back(file);
 		uniqueLogFiles.insert(file);
diff --git a/fdbserver/RestoreMaster.actor.h b/fdbserver/RestoreMaster.actor.h
index 434e4d6f0d..fce748cc5b 100644
--- a/fdbserver/RestoreMaster.actor.h
+++ b/fdbserver/RestoreMaster.actor.h
@@ -251,7 +251,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 	// Split backup files into version batches, each of which has similar data size
 	// Input: sorted range files, sorted log files;
 	// Output: a set of version batches whose size is less than SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES
-	//    	   and each mutation in backup files is included in the version batches exactly once.
+	//         and each mutation in backup files is included in the version batches exactly once.
 	// Assumption 1: input files has no empty files;
 	// Assumption 2: range files at one version <= FASTRESTORE_VERSIONBATCH_MAX_BYTES.
 	// Note: We do not allow a versionBatch size larger than the FASTRESTORE_VERSIONBATCH_MAX_BYTES because the range

From 3664c6948b2c5c415e7fd8b6e31589c5f0272895 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 18 Feb 2020 13:21:29 -0800
Subject: [PATCH 006/176] Consolidate StringRefReader classes

Fix a compiler error of unused variable too.
---
 fdbclient/BackupAgent.actor.h       |  4 +++
 fdbserver/RestoreCommon.actor.cpp   | 48 +++++------------------------
 fdbserver/RestoreLoader.actor.cpp   | 10 +++---
 fdbserver/RestoreRoleCommon.actor.h | 47 ----------------------------
 4 files changed, 17 insertions(+), 92 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index ae6717c619..9ef90976d1 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -880,6 +880,10 @@ struct StringRefReader {
 	const int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
 	const uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
 
+	// Convert big Endian value (e.g., encoded in log file) into a littleEndian uint64_t value.
+	int64_t consumeNetworkInt64() { return (int64_t)bigEndian64((uint32_t)consume<int64_t>()); }
+	uint64_t consumeNetworkUInt64() { return bigEndian64(consume<uint64_t>()); }
+
 	bool eof() { return rptr == end; }
 
 	const uint8_t *rptr, *end;
diff --git a/fdbserver/RestoreCommon.actor.cpp b/fdbserver/RestoreCommon.actor.cpp
index 7ea783c04a..e5776e97d8 100644
--- a/fdbserver/RestoreCommon.actor.cpp
+++ b/fdbserver/RestoreCommon.actor.cpp
@@ -23,15 +23,15 @@
 
 #include "fdbserver/RestoreCommon.actor.h"
 
+// Backup agent header
+#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupContainer.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/MutationList.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
 
-// Backup agent header
-#include "fdbclient/BackupAgent.actor.h"
-//#include "FileBackupAgent.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/MutationList.h"
-#include "fdbclient/BackupContainer.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // Split RestoreConfigFR defined in FileBackupAgent.actor.cpp to declaration in Restore.actor.h and implementation in
@@ -296,38 +296,6 @@ std::string RestoreConfigFR::toString() {
 // The implementation of parallelFileRestore is copied from FileBackupAgent.actor.cpp
 // parallelFileRestore is copied from FileBackupAgent.actor.cpp for the same reason as RestoreConfigFR is copied
 namespace parallelFileRestore {
-// Helper class for reading restore data from a buffer and throwing the right errors.
-struct StringRefReader {
-	StringRefReader(StringRef s = StringRef(), Error e = Error()) : rptr(s.begin()), end(s.end()), failure_error(e) {}
-
-	// Return remainder of data as a StringRef
-	StringRef remainder() { return StringRef(rptr, end - rptr); }
-
-	// Return a pointer to len bytes at the current read position and advance read pos
-	const uint8_t* consume(unsigned int len) {
-		if (rptr == end && len != 0) throw end_of_stream();
-		const uint8_t* p = rptr;
-		rptr += len;
-		if (rptr > end) throw failure_error;
-		return p;
-	}
-
-	// Return a T from the current read position and advance read pos
-	template <typename T>
-	const T consume() {
-		return *(const T*)consume(sizeof(T));
-	}
-
-	// Functions for consuming big endian (network byte order) integers.
-	// Consumes a big endian number, swaps it to little endian, and returns it.
-	int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
-	uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
-
-	bool eof() { return rptr == end; }
-
-	const uint8_t *rptr, *end;
-	Error failure_error;
-};
 
 ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset,
                                                                       int len) {
@@ -336,7 +304,7 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
 	if (rLen != len) throw restore_bad_read();
 
 	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
-	state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
+	state StringRefReader reader(buf, restore_corrupted_data());
 
 	try {
 		// Read header, currently only decoding version 1001
@@ -393,7 +361,7 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IA
 	if (rLen != len) throw restore_bad_read();
 
 	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
-	state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
+	state StringRefReader reader(buf, restore_corrupted_data());
 
 	try {
 		// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 79bfb057d3..7eb3b0726c 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -160,7 +160,7 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 	wait(processedFileOffset->whenAtLeast(asset.offset));
 	ASSERT(processedFileOffset->get() == asset.offset);
 
-	BackupStringRefReader reader(buf, restore_corrupted_data());
+	StringRefReader reader(buf, restore_corrupted_data());
 	try {
 		// Read block header
 		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
@@ -174,7 +174,7 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 
 			// Deserialize messages written in saveMutationsToFile().
 			Version msgVersion = bigEndian64(reader.consume<Version>());
-			uint32_t sub = bigEndian32(reader.consume<uint32_t>());
+			bigEndian32(reader.consume<uint32_t>()); // subsequence number
 			int msgSize = bigEndian32(reader.consume<int>());
 			const uint8_t* message = reader.consume(msgSize);
 
@@ -576,7 +576,7 @@ bool concatenateBackupMutationForLogFile(std::map<Standalone<StringRef>, Standal
 	std::map<Standalone<StringRef>, uint32_t>& mutationPartMap = *pMutationPartMap;
 	const int key_prefix_len = sizeof(uint8_t) + sizeof(Version) + sizeof(uint32_t);
 
-	BackupStringRefReader readerKey(key_input, restore_corrupted_data()); // read key_input!
+	StringRefReader readerKey(key_input, restore_corrupted_data()); // read key_input!
 	int logRangeMutationFirstLength = key_input.size() - key_prefix_len;
 	bool concatenated = false;
 
@@ -646,13 +646,13 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 		StringRef k = m.first.contents();
 		StringRef val = m.second.contents();
 
-		BackupStringRefReader kReader(k, restore_corrupted_data());
+		StringRefReader kReader(k, restore_corrupted_data());
 		uint64_t commitVersion = kReader.consume<uint64_t>(); // Consume little Endian data
 		// We have already filter the commit not in [beginVersion, endVersion) when we concatenate kv pair in log file
 		ASSERT_WE_THINK(asset.isInVersionRange(commitVersion));
 		kvOps.insert(std::make_pair(commitVersion, MutationsVec()));
 
-		BackupStringRefReader vReader(val, restore_corrupted_data());
+		StringRefReader vReader(val, restore_corrupted_data());
 		vReader.consume<uint64_t>(); // Consume the includeVersion
 		// TODO(xumengpanda): verify the protocol version is compatible and raise error if needed
 
diff --git a/fdbserver/RestoreRoleCommon.actor.h b/fdbserver/RestoreRoleCommon.actor.h
index 8679a5e0a2..9ddbc3d82e 100644
--- a/fdbserver/RestoreRoleCommon.actor.h
+++ b/fdbserver/RestoreRoleCommon.actor.h
@@ -60,53 +60,6 @@ ACTOR Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id);
 ACTOR Future<Void> handleInitVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self);
 void handleFinishRestoreRequest(const RestoreFinishRequest& req, Reference<RestoreRoleData> self);
 
-// Helper class for reading restore data from a buffer and throwing the right errors.
-// This struct is mostly copied from StringRefReader. We add a sanity check in this struct.
-// We want to decouple code between fast restore and old restore. So we keep this duplicate struct
-struct BackupStringRefReader {
-	BackupStringRefReader(StringRef s = StringRef(), Error e = Error())
-	  : rptr(s.begin()), end(s.end()), failure_error(e), str_size(s.size()) {}
-
-	// Return remainder of data as a StringRef
-	StringRef remainder() { return StringRef(rptr, end - rptr); }
-
-	// Return a pointer to len bytes at the current read position and advance read pos
-	// Consume a little-Endian data. Since we only run on little-Endian machine, the data on storage is little Endian
-	const uint8_t* consume(unsigned int len) {
-		if (rptr == end && len != 0) throw end_of_stream();
-		const uint8_t* p = rptr;
-		rptr += len;
-		if (rptr > end) {
-			printf("[ERROR] BackupStringRefReader throw error! string length:%d\n", str_size);
-			printf("!!!!!!!!!!!![ERROR]!!!!!!!!!!!!!! Worker may die due to the error. Master will stuck when a worker "
-			       "die\n");
-			throw failure_error;
-		}
-		return p;
-	}
-
-	// Return a T from the current read position and advance read pos
-	template <typename T>
-	const T consume() {
-		return *(const T*)consume(sizeof(T));
-	}
-
-	// Functions for consuming big endian (network byte oselfer) integers.
-	// Consumes a big endian number, swaps it to little endian, and returns it.
-	int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
-	uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
-
-	// Convert big Endian value (e.g., encoded in log file) into a littleEndian uint64_t value.
-	int64_t consumeNetworkInt64() { return (int64_t)bigEndian64((uint32_t)consume<int64_t>()); }
-	uint64_t consumeNetworkUInt64() { return bigEndian64(consume<uint64_t>()); }
-
-	bool eof() { return rptr == end; }
-
-	const uint8_t *rptr, *end;
-	const int str_size;
-	Error failure_error;
-};
-
 class RoleVersionBatchState {
 public:
 	static const int INVALID = -1;

From f6c27ca0d0af43134f993af344569764190698f9 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 19 Feb 2020 11:24:17 -0800
Subject: [PATCH 007/176] Check block padding is \0xff for new mutation logs

---
 fdbserver/RestoreLoader.actor.cpp |  5 +++++
 fdbserver/RestoreMaster.actor.h   | 14 ++++++--------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 7eb3b0726c..3f15aebac1 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -213,6 +213,11 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 				samplesIter->second.push_back_deep(samplesIter->second.arena(), mutation);
 			}
 		}
+
+		// Make sure any remaining bytes in the block are 0xFF
+		for (auto b : reader.remainder()) {
+			if (b != 0xFF) throw restore_corrupted_data_padding();
+		}
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "FileRestoreCorruptLogFileBlock")
 		    .error(e)
diff --git a/fdbserver/RestoreMaster.actor.h b/fdbserver/RestoreMaster.actor.h
index fce748cc5b..4a4520f75d 100644
--- a/fdbserver/RestoreMaster.actor.h
+++ b/fdbserver/RestoreMaster.actor.h
@@ -231,19 +231,17 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 			}
 			++rangeIdx;
 		}
-		int logIdx = 0;
 		std::vector<RestoreFileFR> retLogs;
 		// Scan all logFiles every time to avoid assumption on log files' version ranges.
 		// For example, we do not assume each version range only exists in one log file
-		while (logIdx < logFiles.size()) {
-			Version begin = std::max(prevVersion, logFiles[logIdx].beginVersion);
-			Version end = std::min(nextVersion, logFiles[logIdx].endVersion);
+		for (const auto& file : logFiles) {
+			Version begin = std::max(prevVersion, file.beginVersion);
+			Version end = std::min(nextVersion, file.endVersion);
 			if (begin < end) { // logIdx file overlap in [prevVersion, nextVersion)
-				double ratio = (end - begin) * 1.0 / (logFiles[logIdx].endVersion - logFiles[logIdx].beginVersion);
-				size += logFiles[logIdx].fileSize * ratio;
-				retLogs.push_back(logFiles[logIdx]);
+				double ratio = (end - begin) * 1.0 / (file.endVersion - file.beginVersion);
+				size += file.fileSize * ratio;
+				retLogs.push_back(file);
 			}
-			++logIdx;
 		}
 		return std::make_tuple(size, rangeIdx, retLogs);
 	}

From 55005952f2f9bac273282ae92d145932213559b7 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 19 Feb 2020 15:50:12 -0800
Subject: [PATCH 008/176] Add a knob to switch mutation logs for parallel
 restore

Knob FASTRESTORE_USE_PARTITIONED_LOGS, default is true to enable partitioned
mutation logs. Otherwise, old mutation logs are used.
---
 fdbserver/Knobs.cpp               | 1 +
 fdbserver/Knobs.h                 | 1 +
 fdbserver/RestoreMaster.actor.cpp | 6 ++++--
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp
index dc0261ff06..40abb0d6e4 100644
--- a/fdbserver/Knobs.cpp
+++ b/fdbserver/Knobs.cpp
@@ -568,6 +568,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs, bool isSimula
 	init( FASTRESTORE_APPLYING_PARALLELISM,               	     100 ); if( randomize ) { FASTRESTORE_APPLYING_PARALLELISM = deterministicRandom()->random01() * 10 + 1; }
 	init( FASTRESTORE_MONITOR_LEADER_DELAY,                        5 ); if( randomize ) { FASTRESTORE_MONITOR_LEADER_DELAY = deterministicRandom()->random01() * 100; }
 	init( FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS,                60 ); if( randomize && BUGGIFY ) { FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS = deterministicRandom()->random01() * 240 + 10; }
+	init( FASTRESTORE_USE_PARTITIONED_LOGS,                     true );
 	init( FASTRESTORE_TRACK_REQUEST_LATENCY,              	    true ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_REQUEST_LATENCY = false; }
 	init( FASTRESTORE_TRACK_LOADER_SEND_REQUESTS,              false ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_LOADER_SEND_REQUESTS = true; }
 	init( FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT,                 6144 ); if( randomize && BUGGIFY ) { FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT = 1; }
diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h
index 475880fb2e..8b06c27aab 100644
--- a/fdbserver/Knobs.h
+++ b/fdbserver/Knobs.h
@@ -510,6 +510,7 @@ public:
 	int64_t FASTRESTORE_APPLYING_PARALLELISM; // number of outstanding txns writing to dest. DB
 	int64_t FASTRESTORE_MONITOR_LEADER_DELAY;
 	int64_t FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS;
+	bool FASTRESTORE_USE_PARTITIONED_LOGS;
 	bool FASTRESTORE_TRACK_REQUEST_LATENCY; // true to track reply latency of each request in a request batch
 	bool FASTRESTORE_TRACK_LOADER_SEND_REQUESTS; // track requests of load send mutations to appliers?
 	int64_t FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT; // threshold when pipelined actors should be delayed
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 51c168cc5f..2fba9204d2 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -284,7 +284,6 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 		       std::tie(f2.endVersion, f2.beginVersion, f2.fileIndex, f2.fileName);
 	});
 
-	// TODO for Jingyu: Verify new backup files are grouped into correct version batches.
 	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches); // Divide files into version batches
 	self->dumpVersionBatches(self->versionBatches);
 
@@ -686,8 +685,11 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 	if (request.targetVersion == invalidVersion && desc.maxRestorableVersion.present()) {
 		request.targetVersion = desc.maxRestorableVersion.get();
 	}
+	TraceEvent("FastRestore").detail("TargetVersion", request.targetVersion).detail("BackupDesc", desc.toString());
 
-	Optional<RestorableFileSet> restorable = wait(bc->getPartitionedRestoreSet(request.targetVersion));
+	Optional<RestorableFileSet> restorable =
+	    wait(SERVER_KNOBS->FASTRESTORE_USE_PARTITIONED_LOGS ? bc->getPartitionedRestoreSet(request.targetVersion)
+	                                                        : bc->getRestoreSet(request.targetVersion));
 
 	if (!restorable.present()) {
 		TraceEvent(SevWarn, "FastRestoreMasterPhaseCollectBackupFiles").detail("NotRestorable", request.targetVersion);

From c2623b5c20c24be39fe1a593ebf1e933775a2652 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 20 Feb 2020 14:35:09 -0800
Subject: [PATCH 009/176] Return partitioned logs for RestorableFileSet

---
 fdbclient/BackupContainer.actor.cpp | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 98215aabcd..9397be7b1d 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1093,9 +1093,24 @@ public:
 			// List logs in version order so log continuity can be analyzed
 			std::sort(logs.begin(), logs.end());
 
-			// TODO(jingyu): for partitioned logs, the continuity checking should be based on
-			// epochs and versions, which should be saved in a metadata file by backup worker and
-			// thus is available here.
+			if (partitioned) {
+				// Remove duplicated log files that can happen for old epochs.
+				std::vector<LogFile> filtered;
+				int i = 0;
+				for (int j = 1; j < logs.size(); j++) {
+					if (!logs[i].sameContent(logs[j])) {
+						filtered.push_back(logs[i]);
+						i = j;
+					}
+				}
+				if (i < logs.size()) filtered.push_back(logs[i]);
+
+				// TODO(jingyu): for partitioned logs, the continuity checking should be based on
+				// epochs and versions, which should be saved in a metadata file by backup worker and
+				// thus is available here. For now, assume it's continuous.
+				restorable.logs.swap(filtered);
+				return Optional<RestorableFileSet>(restorable);
+			}
 
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
 			if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {

From d8c6bf585d95d34ddae1b8d32f0fc8b0c272e292 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 20 Feb 2020 16:28:27 -0800
Subject: [PATCH 010/176] Include a total number of tags in partition log file
 names

This is needed for BackupContainer to check partitioned mutation logs are
continuous, i.e., restorable to a version.
---
 fdbclient/BackupContainer.actor.cpp | 14 +++++++-------
 fdbclient/BackupContainer.h         |  3 ++-
 fdbserver/BackupProgress.actor.cpp  | 18 ++++++++++--------
 fdbserver/BackupProgress.actor.h    |  6 ++++--
 fdbserver/BackupWorker.actor.cpp    | 10 ++++++----
 fdbserver/WorkerInterface.actor.h   |  3 ++-
 fdbserver/masterserver.actor.cpp    | 11 +++++++----
 7 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 9397be7b1d..3b9ce2389e 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -344,10 +344,11 @@ public:
 	}
 
 	Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
-	                                                  uint16_t tagId) final {
+	                                                  uint16_t tagId, int totalTags) final {
 		return writeFile(logVersionFolderString(beginVersion, true) +
-		                 format("log,%lld,%lld,%s,%d,%d", beginVersion, endVersion,
-		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize, tagId));
+		                 format("log,%lld,%lld,%s,%d,%d-of-%d", beginVersion, endVersion,
+		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize, tagId,
+		                        totalTags));
 	}
 
 	Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) override {
@@ -398,8 +399,8 @@ public:
 		if(sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n", &f.beginVersion, &f.endVersion, &f.blockSize, &len) == 3 && len == name.size()) {
 			out = f;
 			return true;
-		} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u,%d%n", &f.beginVersion, &f.endVersion,
-		                  &f.blockSize, &f.tagId, &len) == 4 &&
+		} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u,%d-of-%d%n", &f.beginVersion,
+		                  &f.endVersion, &f.blockSize, &f.tagId, &f.totalTags, &len) == 5 &&
 		           len == name.size() && f.tagId >= 0) {
 			out = f;
 			return true;
@@ -488,7 +489,6 @@ public:
 	ACTOR static Future<Void> writeKeyspaceSnapshotFile_impl(Reference<BackupContainerFileSystem> bc, std::vector<std::string> fileNames, int64_t totalBytes) {
 		ASSERT(!fileNames.empty());
 
-
 		state Version minVer = std::numeric_limits<Version>::max();
 		state Version maxVer = 0;
 		state RangeFile rf;
@@ -528,7 +528,7 @@ public:
 		return Void();
 	}
 
-	Future<Void> writeKeyspaceSnapshotFile(std::vector<std::string> fileNames, int64_t totalBytes) override {
+	Future<Void> writeKeyspaceSnapshotFile(std::vector<std::string> fileNames, int64_t totalBytes) final {
 		return writeKeyspaceSnapshotFile_impl(Reference<BackupContainerFileSystem>::addRef(this), fileNames, totalBytes);
 	};
 
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 9c4526e6f4..437f6e3eaa 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -75,6 +75,7 @@ struct LogFile {
 	std::string fileName;
 	int64_t fileSize;
 	int tagId = -1; // Log router tag. Non-negative for new backup format.
+	int totalTags = -1; // Total number of log router tags.
 
 	// Order by beginVersion, break ties with endVersion
 	bool operator< (const LogFile &rhs) const {
@@ -220,7 +221,7 @@ public:
 
 	// Open a tagged log file for writing, where tagId is the log router tag's id.
 	virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
-	                                                          uint16_t tagId) = 0;
+	                                                          uint16_t tagId, int totalTags) = 0;
 
 	// Write a KeyspaceSnapshotFile of range file names representing a full non overlapping
 	// snapshot of the key ranges this backup is targeting.
diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 0e7ccbaaa9..5492db7aa8 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -37,8 +37,8 @@ void BackupProgress::addBackupStatus(const WorkerBackupStatus& status) {
 	}
 }
 
-std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> BackupProgress::getUnfinishedBackup() {
-	std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> toRecruit;
+std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgress::getUnfinishedBackup() {
+	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit;
 
 	if (!backupStartedValue.present()) return toRecruit; // No active backups
 
@@ -68,7 +68,7 @@ std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> BackupProgress::g
 			    .detail("EndVersion", info.epochEnd);
 		}
 		if (!tagVersions.empty()) {
-			toRecruit[{ epoch, info.epochEnd }] = tagVersions;
+			toRecruit[{ epoch, info.epochEnd, info.logRouterTags }] = tagVersions;
 		}
 	}
 	return toRecruit;
@@ -115,11 +115,12 @@ TEST_CASE("/BackupProgress/Unfinished") {
 	BackupProgress progress(UID(0, 0), epochInfos);
 	progress.setBackupStartedValue(Optional<Value>(LiteralStringRef("1")));
 
-	std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> unfinished = progress.getUnfinishedBackup();
+	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> unfinished = progress.getUnfinishedBackup();
 
 	ASSERT(unfinished.size() == 1);
-	for (const auto [epochVersion, tagVersion] : unfinished) {
-		ASSERT(epochVersion.first == epoch1 && epochVersion.second == end1);
+	for (const auto [epochVersionCount, tagVersion] : unfinished) {
+		ASSERT(std::get<0>(epochVersionCount) == epoch1 && std::get<1>(epochVersionCount) == end1 &&
+		       std::get<2>(epochVersionCount) == 1);
 		ASSERT(tagVersion.size() == 1 && tagVersion.begin()->first == tag1 && tagVersion.begin()->second == begin1);
 	}
 
@@ -128,8 +129,9 @@ TEST_CASE("/BackupProgress/Unfinished") {
 	progress.addBackupStatus(status1);
 	unfinished = progress.getUnfinishedBackup();
 	ASSERT(unfinished.size() == 1);
-	for (const auto [epochVersion, tagVersion] : unfinished) {
-		ASSERT(epochVersion.first == epoch1 && epochVersion.second == end1);
+	for (const auto [epochVersionCount, tagVersion] : unfinished) {
+		ASSERT(std::get<0>(epochVersionCount) == epoch1 && std::get<1>(epochVersionCount) == end1 &&
+		       std::get<2>(epochVersionCount) == 1);
 		ASSERT(tagVersion.size() == 1 && tagVersion.begin()->first == tag1 && tagVersion.begin()->second == saved1 + 1);
 	}
 
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index f12002dbfe..90e93fc95e 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -25,6 +25,8 @@
 #define FDBSERVER_BACKUPPROGRESS_ACTOR_H
 
 #include <map>
+#include <tuple>
+
 #include "fdbclient/FDBTypes.h"
 #include "fdbserver/LogSystem.h"
 #include "flow/Arena.h"
@@ -41,7 +43,7 @@ public:
 	// savedVersion is used.
 	void addBackupStatus(const WorkerBackupStatus& status);
 
-	// Returns a map of pair<Epoch, endVersion> : map<tag, savedVersion>, so that
+	// Returns a map of tuple<Epoch, endVersion, logRouterTags> : map<tag, savedVersion>, so that
 	// the backup range should be [savedVersion + 1, endVersion) for the "tag" of the "Epoch".
 	//
 	// Specifically, the backup ranges for each old epoch are:
@@ -49,7 +51,7 @@ public:
 	//        backup [epochBegin, endVersion)
 	//    else if savedVersion < endVersion - 1 = knownCommittedVersion
 	//        backup [savedVersion + 1, endVersion)
-	std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> getUnfinishedBackup();
+	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> getUnfinishedBackup();
 
 	// Set the value for "backupStartedKey"
 	void setBackupStartedValue(Optional<Value> value) {
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 5229340d8d..d4546b6295 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -67,6 +67,7 @@ struct VersionedMessage {
 struct BackupData {
 	const UID myId;
 	const Tag tag; // LogRouter tag for this worker, i.e., (-2, i)
+	const int totalTags; // Total log router tags
 	const Version startVersion;
 	const Optional<Version> endVersion; // old epoch's end version (inclusive), or empty for current epoch
 	const LogEpoch recruitedEpoch;
@@ -102,9 +103,9 @@ struct BackupData {
 	Future<Void> logger;
 
 	explicit BackupData(UID id, Reference<AsyncVar<ServerDBInfo>> db, const InitializeBackupRequest& req)
-	  : myId(id), tag(req.routerTag), startVersion(req.startVersion), endVersion(req.endVersion),
-	    recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch), minKnownCommittedVersion(invalidVersion),
-	    savedVersion(invalidVersion), cc("BackupWorker", myId.toString()) {
+	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
+	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
+	    minKnownCommittedVersion(invalidVersion), savedVersion(invalidVersion), cc("BackupWorker", myId.toString()) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
 		pullFinished.set(false);
 
@@ -415,7 +416,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 			it->second.lastSavedVersion = self->messages[0].getVersion();
 		}
 		logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile(
-		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id));
+		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));
 		it++;
 	}
 	if (activeUids.empty()) {
@@ -646,6 +647,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 
 	TraceEvent("BackupWorkerStart", self.myId)
 	    .detail("Tag", req.routerTag.toString())
+	    .detail("TotalTags", req.totalTags)
 	    .detail("StartVersion", req.startVersion)
 	    .detail("EndVersion", req.endVersion.present() ? req.endVersion.get() : -1)
 	    .detail("LogEpoch", req.recruitedEpoch)
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index ee613912a1..c8885cb4a0 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -170,6 +170,7 @@ struct InitializeBackupRequest {
 	LogEpoch backupEpoch; // The epoch the worker should work on. If different from the recruitedEpoch, then it refers
 	                      // to some previous epoch with unfinished work.
 	Tag routerTag;
+	int totalTags;
 	Version startVersion;
 	Optional<Version> endVersion;
 	ReplyPromise<struct InitializeBackupReply> reply;
@@ -179,7 +180,7 @@ struct InitializeBackupRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, reqId, recruitedEpoch, backupEpoch, routerTag, startVersion, endVersion, reply);
+		serializer(ar, reqId, recruitedEpoch, backupEpoch, routerTag, totalTags, startVersion, endVersion, reply);
 	}
 };
 
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index a556c2fec2..7acf67b72a 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1261,6 +1261,7 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 		req.recruitedEpoch = epoch;
 		req.backupEpoch = epoch;
 		req.routerTag = idsTags[i].second;
+		req.totalTags = logRouterTags;
 		req.startVersion = startVersion;
 		TraceEvent("BackupRecruitment", self->dbgid)
 		    .detail("BKID", req.reqId)
@@ -1275,17 +1276,19 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 	}
 
 	wait(gotProgress);
-	std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> toRecruit = backupProgress->getUnfinishedBackup();
-	for (const auto& [epochVersion, tagVersions] : toRecruit) {
+	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
+	    backupProgress->getUnfinishedBackup();
+	for (const auto& [epochVersionCount, tagVersions] : toRecruit) {
 		for (const auto& [tag, version] : tagVersions) {
 			const auto& worker = self->backupWorkers[i % self->backupWorkers.size()];
 			i++;
 			InitializeBackupRequest req(deterministicRandom()->randomUniqueID());
 			req.recruitedEpoch = epoch;
-			req.backupEpoch = epochVersion.first;
+			req.backupEpoch = std::get<0>(epochVersionCount);
 			req.routerTag = tag;
+			req.totalTags = std::get<2>(epochVersionCount);
 			req.startVersion = version; // savedVersion + 1
-			req.endVersion = epochVersion.second - 1;
+			req.endVersion = std::get<1>(epochVersionCount) - 1;
 			TraceEvent("BackupRecruitment", self->dbgid)
 			    .detail("BKID", req.reqId)
 			    .detail("Tag", req.routerTag.toString())

From ace409b49a6f3f09ba951ea20fb95b07b3462d3f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 21 Feb 2020 11:47:51 -0800
Subject: [PATCH 011/176] Add subsequence number to restore loader & applier

The subsequence number is needed so that mutations of the same commit version
number, but from different partitioned logs can be correctly reassembled in
order.

For old backup files, the sub number is always 0. For partitioned mutation
logs, the actual sub number is used. For range files, the sub number is always
0.
---
 fdbclient/RestoreWorkerInterface.actor.h | 10 +--
 fdbserver/RestoreApplier.actor.cpp       | 21 ++++---
 fdbserver/RestoreApplier.actor.h         | 56 ++++++++---------
 fdbserver/RestoreLoader.actor.cpp        | 78 ++++++++++++------------
 fdbserver/RestoreRoleCommon.actor.h      |  8 ++-
 fdbserver/RestoreUtil.h                  |  1 +
 6 files changed, 89 insertions(+), 85 deletions(-)

diff --git a/fdbclient/RestoreWorkerInterface.actor.h b/fdbclient/RestoreWorkerInterface.actor.h
index 84c2f603e8..e51359b82a 100644
--- a/fdbclient/RestoreWorkerInterface.actor.h
+++ b/fdbclient/RestoreWorkerInterface.actor.h
@@ -453,26 +453,28 @@ struct RestoreSendVersionedMutationsRequest : TimedRequest {
 	Version prevVersion, version; // version is the commitVersion of the mutation vector.
 	bool isRangeFile;
 	MutationsVec mutations; // All mutations at the same version parsed by one loader
+	SubSequenceVec subs; // Sub-sequence number for mutations
 
 	ReplyPromise<RestoreCommonReply> reply;
 
 	RestoreSendVersionedMutationsRequest() = default;
 	explicit RestoreSendVersionedMutationsRequest(int batchIndex, const RestoreAsset& asset, Version prevVersion,
-	                                              Version version, bool isRangeFile, MutationsVec mutations)
+	                                              Version version, bool isRangeFile, MutationsVec mutations,
+	                                              SubSequenceVec subs)
 	  : batchIndex(batchIndex), asset(asset), prevVersion(prevVersion), version(version), isRangeFile(isRangeFile),
-	    mutations(mutations) {}
+	    mutations(mutations), subs(subs) {}
 
 	std::string toString() {
 		std::stringstream ss;
 		ss << "VersionBatchIndex:" << batchIndex << "RestoreAsset:" << asset.toString()
 		   << " prevVersion:" << prevVersion << " version:" << version << " isRangeFile:" << isRangeFile
-		   << " mutations.size:" << mutations.size();
+		   << " mutations.size:" << mutations.size() << " subs.size:" << subs.size();
 		return ss.str();
 	}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, batchIndex, asset, prevVersion, version, isRangeFile, mutations, reply);
+		serializer(ar, batchIndex, asset, prevVersion, version, isRangeFile, mutations, subs, reply);
 	}
 };
 
diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index 6df9baec32..fb31fea375 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -101,7 +101,7 @@ ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int
 
 // The actor may be invovked multiple times and executed async.
 // No race condition as long as we do not wait or yield when operate the shared data.
-// Multiple such actors can run on different fileIDs, because mutations in different files belong to different versions;
+// Multiple such actors can run on different fileIDs;
 // Only one actor can process mutations from the same file
 ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMutationsRequest req,
                                                           Reference<RestoreApplierData> self) {
@@ -126,21 +126,22 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMu
 	state bool isDuplicated = true;
 	if (curFilePos.get() == req.prevVersion) {
 		isDuplicated = false;
-		Version commitVersion = req.version;
+		const Version commitVersion = req.version;
 		uint16_t numVersionStampedKV = 0;
-		MutationsVec mutations(req.mutations);
 		// Sanity check: mutations in range file is in [beginVersion, endVersion);
 		// mutations in log file is in [beginVersion, endVersion], both inclusive.
 		ASSERT_WE_THINK(commitVersion >= req.asset.beginVersion);
 		// Loader sends the endVersion to ensure all useful versions are sent
 		ASSERT_WE_THINK(commitVersion <= req.asset.endVersion);
+		ASSERT(req.mutations.size() == req.subs.size());
 
-		for (int mIndex = 0; mIndex < mutations.size(); mIndex++) {
-			MutationRef mutation = mutations[mIndex];
+		for (int mIndex = 0; mIndex < req.mutations.size(); mIndex++) {
+			const MutationRef& mutation = req.mutations[mIndex];
+			const LogMessageVersion mutationVersion(commitVersion, req.subs[mIndex]);
 			TraceEvent(SevFRMutationInfo, "FastRestoreApplierPhaseReceiveMutations", self->id())
 			    .detail("ApplierNode", self->id())
 			    .detail("RestoreAsset", req.asset.toString())
-			    .detail("Version", commitVersion)
+			    .detail("Version", mutationVersion.toString())
 			    .detail("Index", mIndex)
 			    .detail("MutationReceived", mutation.toString());
 			batchData->counters.receivedBytes += mutation.totalSize();
@@ -159,10 +160,10 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMu
 			// Note: Log and range mutations may be delivered out of order. Can we handle it?
 			if (mutation.type == MutationRef::SetVersionstampedKey ||
 			    mutation.type == MutationRef::SetVersionstampedValue) {
-				batchData->addVersionStampedKV(mutation, commitVersion, numVersionStampedKV);
+				batchData->addVersionStampedKV(mutation, mutationVersion, numVersionStampedKV);
 				numVersionStampedKV++;
 			} else {
-				batchData->addMutation(mutation, commitVersion);
+				batchData->addMutation(mutation, mutationVersion);
 			}
 		}
 		curFilePos.set(req.version);
@@ -239,7 +240,7 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 			for (auto& vm : key.second->second.pendingMutations) {
 				for (auto& m : vm.second) {
 					TraceEvent(SevWarnAlways, "FastRestoreApplierGetAndComputeStagingKeysUnhandledError")
-					    .detail("PendingMutationVersion", vm.first)
+					    .detail("PendingMutationVersion", vm.first.toString())
 					    .detail("PendingMutation", m.toString());
 				}
 			}
@@ -250,7 +251,7 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 			// The key's version ideally should be the most recently committed version.
 			// But as long as it is > 1 and less than the start version of the version batch, it is the same result.
 			MutationRef m(MutationRef::SetValue, key.first, fValues[i].get().get());
-			key.second->second.add(m, (Version)1);
+			key.second->second.add(m, LogMessageVersion(1));
 			key.second->second.precomputeResult();
 			i++;
 		}
diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index 72424eed62..9d8f6b60d8 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -51,14 +51,14 @@ struct StagingKey {
 	Key key; // TODO: Maybe not needed?
 	Value val;
 	MutationRef::Type type; // set or clear
-	Version version; // largest version of set or clear for the key
-	std::map<Version, MutationsVec> pendingMutations; // mutations not set or clear type
+	LogMessageVersion version; // largest version of set or clear for the key
+	std::map<LogMessageVersion, MutationsVec> pendingMutations; // mutations not set or clear type
 
 	explicit StagingKey() : version(0), type(MutationRef::MAX_ATOMIC_OP) {}
 
 	// Add mutation m at newVersion to stagingKey
 	// Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set
-	void add(const MutationRef& m, Version newVersion) {
+	void add(const MutationRef& m, LogMessageVersion newVersion) {
 		ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue);
 		if (version < newVersion) {
 			if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
@@ -76,14 +76,14 @@ struct StagingKey {
 			}
 		} else if (version == newVersion) { // Sanity check
 			TraceEvent("FastRestoreApplierStagingKeyMutationAtSameVersion")
-			    .detail("Version", newVersion)
+			    .detail("Version", newVersion.toString())
 			    .detail("NewMutation", m.toString())
 			    .detail("ExistingKeyType", typeString[type]);
 			if (m.type == MutationRef::SetValue) {
 				if (type == MutationRef::SetValue) {
 					if (m.param2 != val) {
 						TraceEvent(SevError, "FastRestoreApplierStagingKeyMutationAtSameVersionUnhandled")
-						    .detail("Version", newVersion)
+						    .detail("Version", newVersion.toString())
 						    .detail("NewMutation", m.toString())
 						    .detail("ExistingKeyType", typeString[type])
 						    .detail("ExitingKeyValue", val)
@@ -92,7 +92,7 @@ struct StagingKey {
 					} // else {} Backup has duplicate set at the same version
 				} else {
 					TraceEvent(SevWarnAlways, "FastRestoreApplierStagingKeyMutationAtSameVersionOverride")
-					    .detail("Version", newVersion)
+					    .detail("Version", newVersion.toString())
 					    .detail("NewMutation", m.toString())
 					    .detail("ExistingKeyType", typeString[type])
 					    .detail("ExitingKeyValue", val);
@@ -101,7 +101,7 @@ struct StagingKey {
 				}
 			} else if (m.type == MutationRef::ClearRange) {
 				TraceEvent(SevWarnAlways, "FastRestoreApplierStagingKeyMutationAtSameVersionSkipped")
-				    .detail("Version", newVersion)
+				    .detail("Version", newVersion.toString())
 				    .detail("NewMutation", m.toString())
 				    .detail("ExistingKeyType", typeString[type])
 				    .detail("ExitingKeyValue", val);
@@ -113,9 +113,10 @@ struct StagingKey {
 	void precomputeResult() {
 		TraceEvent(SevDebug, "FastRestoreApplierPrecomputeResult")
 		    .detail("Key", key)
-		    .detail("Version", version)
-		    .detail("LargestPendingVersion", (pendingMutations.empty() ? -1 : pendingMutations.rbegin()->first));
-		std::map<Version, MutationsVec>::iterator lb = pendingMutations.lower_bound(version);
+		    .detail("Version", version.toString())
+		    .detail("LargestPendingVersion",
+		            (pendingMutations.empty() ? "-1" : pendingMutations.rbegin()->first.toString()));
+		std::map<LogMessageVersion, MutationsVec>::iterator lb = pendingMutations.lower_bound(version);
 		if (lb == pendingMutations.end()) {
 			return;
 		}
@@ -158,11 +159,11 @@ struct StagingKey {
 					type = MutationRef::SetValue; // Precomputed result should be set to DB.
 					TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet")
 					    .detail("Type", typeString[mutation.type])
-					    .detail("Version", lb->first);
+					    .detail("Version", lb->first.toString());
 				} else {
 					TraceEvent(SevWarnAlways, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation")
 					    .detail("Type", typeString[mutation.type])
-					    .detail("Version", lb->first);
+					    .detail("Version", lb->first.toString());
 				}
 			}
 			version = lb->first;
@@ -172,10 +173,10 @@ struct StagingKey {
 
 	// Does the key has at least 1 set or clear mutation to get the base value
 	bool hasBaseValue() {
-		if (version > 0) {
+		if (version.version > 0) {
 			ASSERT(type == MutationRef::SetValue || type == MutationRef::ClearRange);
 		}
-		return version > 0;
+		return version.version > 0;
 	}
 
 	// Has all pendingMutations been pre-applied to the val?
@@ -191,9 +192,9 @@ struct StagingKey {
 // Range mutations should be applied both to the destination DB and to the StagingKeys
 struct StagingKeyRange {
 	Standalone<MutationRef> mutation;
-	Version version;
+	LogMessageVersion version;
 
-	explicit StagingKeyRange(MutationRef m, Version newVersion) : mutation(m), version(newVersion) {}
+	explicit StagingKeyRange(MutationRef m, LogMessageVersion newVersion) : mutation(m), version(newVersion) {}
 
 	bool operator<(const StagingKeyRange& rhs) const {
 		return std::tie(version, mutation.type, mutation.param1, mutation.param2) <
@@ -263,7 +264,7 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 	}
 	~ApplierBatchData() = default;
 
-	void addMutation(MutationRef m, Version ver) {
+	void addMutation(MutationRef m, LogMessageVersion ver) {
 		if (!isRangeMutation(m)) {
 			auto item = stagingKeys.emplace(m.param1, StagingKey());
 			item.first->second.add(m, ver);
@@ -272,20 +273,20 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 		}
 	}
 
-	void addVersionStampedKV(MutationRef m, Version ver, uint16_t numVersionStampedKV) {
+	void addVersionStampedKV(MutationRef m, LogMessageVersion ver, uint16_t numVersionStampedKV) {
 		if (m.type == MutationRef::SetVersionstampedKey) {
 			// Assume transactionNumber = 0 does not affect result
 			TraceEvent(SevDebug, "FastRestoreApplierAddMutation")
 			    .detail("MutationType", typeString[m.type])
 			    .detail("FakedTransactionNumber", numVersionStampedKV);
-			transformVersionstampMutation(m, &MutationRef::param1, ver, numVersionStampedKV);
+			transformVersionstampMutation(m, &MutationRef::param1, ver.version, numVersionStampedKV);
 			addMutation(m, ver);
 		} else if (m.type == MutationRef::SetVersionstampedValue) {
 			// Assume transactionNumber = 0 does not affect result
 			TraceEvent(SevDebug, "FastRestoreApplierAddMutation")
 			    .detail("MutationType", typeString[m.type])
 			    .detail("FakedTransactionNumber", numVersionStampedKV);
-			transformVersionstampMutation(m, &MutationRef::param2, ver, numVersionStampedKV);
+			transformVersionstampMutation(m, &MutationRef::param2, ver.version, numVersionStampedKV);
 			addMutation(m, ver);
 		} else {
 			ASSERT(false);
@@ -298,8 +299,8 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 			if (!stagingKey.second.hasPrecomputed()) {
 				TraceEvent("FastRestoreApplierAllKeysPrecomputedFalse")
 				    .detail("Key", stagingKey.first)
-				    .detail("BufferedVersion", stagingKey.second.version)
-				    .detail("MaxPendingVersion", stagingKey.second.pendingMutations.rbegin()->first);
+				    .detail("BufferedVersion", stagingKey.second.version.toString())
+				    .detail("MaxPendingVersion", stagingKey.second.pendingMutations.rbegin()->first.toString());
 				return false;
 			}
 		}
@@ -320,20 +321,17 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 	}
 
 	bool isKVOpsSorted() {
-		bool ret = true;
 		auto prev = kvOps.begin();
 		for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
 			if (prev->first > it->first) {
-				ret = false;
-				break;
+				return false;
 			}
 			prev = it;
 		}
-		return ret;
+		return true;
 	}
 
 	bool allOpsAreKnown() {
-		bool ret = true;
 		for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
 			for (auto m = it->second.begin(); m != it->second.end(); ++m) {
 				if (m->type == MutationRef::SetValue || m->type == MutationRef::ClearRange ||
@@ -341,11 +339,11 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 					continue;
 				else {
 					TraceEvent(SevError, "FastRestore").detail("UnknownMutationType", m->type);
-					ret = false;
+					return false;
 				}
 			}
 		}
-		return ret;
+		return true;
 	}
 };
 
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 3f15aebac1..15dbb3e179 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -165,7 +165,6 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 		// Read block header
 		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
 
-		Version lastVersion = invalidVersion;
 		VersionedMutationsMap& kvOps = kvOpsIter->second;
 		VersionedMutationsMap::iterator it = kvOps.end();
 		while (1) {
@@ -173,20 +172,18 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 			if (reader.eof() || *reader.rptr == 0xFF) break;
 
 			// Deserialize messages written in saveMutationsToFile().
-			Version msgVersion = bigEndian64(reader.consume<Version>());
-			bigEndian32(reader.consume<uint32_t>()); // subsequence number
+			LogMessageVersion msgVersion;
+			msgVersion.version = bigEndian64(reader.consume<Version>());
+			msgVersion.sub = bigEndian32(reader.consume<uint32_t>());
 			int msgSize = bigEndian32(reader.consume<int>());
 			const uint8_t* message = reader.consume(msgSize);
 
 			// Skip mutations out of the version range
-			if (!asset.isInVersionRange(msgVersion)) continue;
+			if (!asset.isInVersionRange(msgVersion.version)) continue;
 
-			if (lastVersion != msgVersion) {
-				bool inserted;
-				std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec());
-				lastVersion = msgVersion;
-			}
-			ASSERT(it != kvOps.end());
+			bool inserted;
+			std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec());
+			ASSERT(inserted);
 
 			ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(currentProtocolVersion));
 			MutationRef mutation;
@@ -205,7 +202,7 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 			}
 
 			TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
-			    .detail("CommitVersion", msgVersion)
+			    .detail("CommitVersion", msgVersion.toString())
 			    .detail("ParsedMutation", mutation.toString());
 			it->second.push_back_deep(it->second.arena(), mutation);
 			// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
@@ -306,7 +303,6 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
 		    .detail("BatchIndex", req.batchIndex)
 		    .detail("ProcessLoadParam", req.param.toString());
 		ASSERT(batchData->sampleMutations.find(req.param) == batchData->sampleMutations.end());
-		batchData->processedFileParams[req.param] = Never(); // Ensure second exec. wait on _processLoadingParam()
 		batchData->processedFileParams[req.param] = _processLoadingParam(req.param, batchData, self->id(), self->bc);
 		isDuplicated = false;
 	} else {
@@ -314,8 +310,9 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
 		    .detail("BatchIndex", req.batchIndex)
 		    .detail("WaitOnProcessLoadParam", req.param.toString());
 	}
-	ASSERT(batchData->processedFileParams.find(req.param) != batchData->processedFileParams.end());
-	wait(batchData->processedFileParams[req.param]); // wait on the processing of the req.param.
+	auto it = batchData->processedFileParams.find(req.param);
+	ASSERT(it != batchData->processedFileParams.end());
+	wait(it->second); // wait on the processing of the req.param.
 
 	req.reply.send(RestoreLoadFileReply(req.param, batchData->sampleMutations[req.param], isDuplicated));
 	TraceEvent("FastRestoreLoaderPhaseLoadFileDone", self->id())
@@ -426,16 +423,15 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 	    .detail("RestoreAsset", asset.toString());
 
 	// There should be no mutation at asset.endVersion version because it is exclusive
-	if (kvOps.find(asset.endVersion) != kvOps.end()) {
+	if (kvOps.find(LogMessageVersion(asset.endVersion)) != kvOps.end()) {
 		TraceEvent(SevError, "FastRestoreLoaderSendMutationToApplier")
 		    .detail("BatchIndex", batchIndex)
 		    .detail("RestoreAsset", asset.toString())
 		    .detail("IsRangeFile", isRangeFile)
 		    .detail("Data loss at version", asset.endVersion);
-	}
-	// Ensure there is a mutation request sent at endVersion, so that applier can advance its notifiedVersion
-	if (kvOps.find(asset.endVersion) == kvOps.end()) {
-		kvOps[asset.endVersion] = MutationsVec(); // Empty mutation vector will be handled by applier
+	} else {
+		// Ensure there is a mutation request sent at endVersion, so that applier can advance its notifiedVersion
+		kvOps[LogMessageVersion(asset.endVersion)] = MutationsVec(); // Empty mutation vector will be handled by applier
 	}
 
 	splitMutationIndex = 0;
@@ -445,22 +441,24 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 		// applierMutationsBuffer is the mutation vector to be sent to each applier
 		// applierMutationsSize is buffered mutation vector size for each applier
 		std::map<UID, MutationsVec> applierMutationsBuffer;
+		std::map<UID, SubSequenceVec> applierSubsBuffer;
 		std::map<UID, double> applierMutationsSize;
 		for (auto& applierID : applierIDs) {
 			applierMutationsBuffer[applierID] = MutationsVec();
+			applierSubsBuffer[applierID] = SubSequenceVec();
 			applierMutationsSize[applierID] = 0.0;
 		}
-		Version commitVersion = kvOp->first;
-		if (!(commitVersion >= asset.beginVersion && commitVersion <= asset.endVersion)) { // Debug purpose
+		const LogMessageVersion& commitVersion = kvOp->first;
+		if (!(commitVersion.version >= asset.beginVersion &&
+		      commitVersion.version <= asset.endVersion)) { // Debug purpose
 			TraceEvent(SevError, "FastRestore_SendMutationsToApplier")
-			    .detail("CommitVersion", commitVersion)
+			    .detail("CommitVersion", commitVersion.version)
 			    .detail("RestoreAsset", asset.toString());
 		}
-		ASSERT(commitVersion >= asset.beginVersion);
-		ASSERT(commitVersion <= asset.endVersion); // endVersion is an empty commit to ensure progress
+		ASSERT(commitVersion.version >= asset.beginVersion);
+		ASSERT(commitVersion.version <= asset.endVersion); // endVersion is an empty commit to ensure progress
 
-		for (int mIndex = 0; mIndex < kvOp->second.size(); mIndex++) {
-			MutationRef kvm = kvOp->second[mIndex];
+		for (const MutationRef& kvm : kvOp->second) {
 			// Send the mutation to applier
 			if (isRangeMutation(kvm)) {
 				MutationsVec mvector;
@@ -478,6 +476,7 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 					// printf("SPLITTED MUTATION: %d: mutation:%s applierID:%s\n", splitMutationIndex,
 					// mutation.toString().c_str(), applierID.toString().c_str());
 					applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
+					applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
 					applierMutationsSize[applierID] += mutation.expectedSize();
 
 					kvCount++;
@@ -493,30 +492,30 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 				kvCount++;
 
 				applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
+				applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
 				applierMutationsSize[applierID] += mutation.expectedSize();
 			}
 		} // Mutations at the same version
 
 		// TODO: Sanity check each asset has been received exactly once!
 		// Send the mutations to appliers for each version
-		for (auto& applierID : applierIDs) {
-			requests.push_back(std::make_pair(
-			    applierID, RestoreSendVersionedMutationsRequest(batchIndex, asset, prevVersion, commitVersion,
-			                                                    isRangeFile, applierMutationsBuffer[applierID])));
+		for (const UID& applierID : applierIDs) {
+			requests.emplace_back(applierID, RestoreSendVersionedMutationsRequest(
+			                                     batchIndex, asset, prevVersion, commitVersion.version, isRangeFile,
+			                                     applierMutationsBuffer[applierID], applierSubsBuffer[applierID]));
 		}
 		TraceEvent(SevDebug, "FastRestore_SendMutationToApplier")
 		    .detail("PrevVersion", prevVersion)
-		    .detail("CommitVersion", commitVersion)
+		    .detail("CommitVersion", commitVersion.toString())
 		    .detail("RestoreAsset", asset.toString());
-		ASSERT(prevVersion < commitVersion);
-		prevVersion = commitVersion;
+		ASSERT(prevVersion <= commitVersion.version);
+		prevVersion = commitVersion.version;
 		// Tracking this request can be spammy
 		wait(sendBatchRequests(&RestoreApplierInterface::sendMutationVector, *pApplierInterfaces, requests,
 		                       TaskPriority::RestoreLoaderSendMutations,
 		                       SERVER_KNOBS->FASTRESTORE_TRACK_LOADER_SEND_REQUESTS));
 
 		requests.clear();
-
 	} // all versions of mutations in the same file
 
 	TraceEvent("FastRestore").detail("LoaderSendMutationOnAppliers", kvCount);
@@ -655,7 +654,8 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 		uint64_t commitVersion = kReader.consume<uint64_t>(); // Consume little Endian data
 		// We have already filter the commit not in [beginVersion, endVersion) when we concatenate kv pair in log file
 		ASSERT_WE_THINK(asset.isInVersionRange(commitVersion));
-		kvOps.insert(std::make_pair(commitVersion, MutationsVec()));
+		auto it = kvOps.insert(std::make_pair(LogMessageVersion(commitVersion), MutationsVec()));
+		ASSERT(it.second); // inserted is true
 
 		StringRefReader vReader(val, restore_corrupted_data());
 		vReader.consume<uint64_t>(); // Consume the includeVersion
@@ -695,7 +695,7 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 			TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
 			    .detail("CommitVersion", commitVersion)
 			    .detail("ParsedMutation", mutation.toString());
-			kvOps[commitVersion].push_back_deep(kvOps[commitVersion].arena(), mutation);
+			it.first->second.push_back_deep(it.first->second.arena(), mutation);
 			// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
 			if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {
 				samples.push_back_deep(samples.arena(), mutation);
@@ -774,13 +774,13 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 		cc->loadedRangeBytes += m.totalSize();
 
 		// We cache all kv operations into kvOps, and apply all kv operations later in one place
-		kvOps.insert(std::make_pair(version, MutationsVec()));
+		auto it = kvOps.insert(std::make_pair(LogMessageVersion(version), MutationsVec()));
 		TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
 		    .detail("CommitVersion", version)
 		    .detail("ParsedMutationKV", m.toString());
 
-		ASSERT_WE_THINK(kvOps.find(version) != kvOps.end());
-		kvOps[version].push_back_deep(kvOps[version].arena(), m);
+		ASSERT_WE_THINK(kvOps.find(LogMessageVersion(version)) != kvOps.end());
+		it.first->second.push_back_deep(it.first->second.arena(), m);
 		// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
 		if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {
 			cc->sampledRangeBytes += m.totalSize();
diff --git a/fdbserver/RestoreRoleCommon.actor.h b/fdbserver/RestoreRoleCommon.actor.h
index 9ddbc3d82e..cedbeb795c 100644
--- a/fdbserver/RestoreRoleCommon.actor.h
+++ b/fdbserver/RestoreRoleCommon.actor.h
@@ -51,9 +51,11 @@ struct RestoreMasterData;
 
 struct RestoreSimpleRequest;
 
-// VersionedMutationsMap: Key is the version of parsed backup mutations
-// Value MutationsVec is the vector of parsed backup mutations
-using VersionedMutationsMap = std::map<Version, MutationsVec>;
+// Key is the (version, subsequence) of parsed backup mutations.
+// Value MutationsVec is the vector of parsed backup mutations.
+// For old mutation logs, the subsequence number is always 0.
+// For partitioned mutation logs, each mutation has a unique LogMessageVersion.
+using VersionedMutationsMap = std::map<LogMessageVersion, MutationsVec>;
 
 ACTOR Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name);
 ACTOR Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id);
diff --git a/fdbserver/RestoreUtil.h b/fdbserver/RestoreUtil.h
index 3c1e1fa7d8..683d785fc2 100644
--- a/fdbserver/RestoreUtil.h
+++ b/fdbserver/RestoreUtil.h
@@ -39,6 +39,7 @@
 //#define SevFRMutationInfo SevInfo
 
 using MutationsVec = Standalone<VectorRef<MutationRef>>;
+using SubSequenceVec = Standalone<VectorRef<uint32_t>>;
 
 enum class RestoreRole { Invalid = 0, Master = 1, Loader, Applier };
 BINARY_SERIALIZABLE(RestoreRole);

From 84d79ce6f72d007e6c5be770ae5aa5331f6cce9e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 21 Feb 2020 14:07:46 -0800
Subject: [PATCH 012/176] Check partitioned log files are continuous for
 RestoreSet

The idea of checking is to use Tag 0 to find out ranges and their number of
tags. Then for each tag 1 and above, check versions are continuous.
---
 fdbclient/BackupContainer.actor.cpp | 143 +++++++++++++++++++++++++---
 1 file changed, 130 insertions(+), 13 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 3b9ce2389e..eed735eaa4 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -228,23 +228,24 @@ std::string BackupDescription::toJSON() const {
  *   Snapshot manifests (a complete set of files constituting a database snapshot for the backup's target ranges)
  *   are stored as JSON files at paths like
  *       /snapshots/snapshot,minVersion,maxVersion,totalBytes
- * 
+ *
  *   Key range files for snapshots are stored at paths like
  *       /kvranges/snapshot,startVersion/N/range,version,uid,blockSize
  *     where startVersion is the version at which the backup snapshot execution began and N is a number
- *     that is increased as key range files are generated over time (at varying rates) such that there 
+ *     that is increased as key range files are generated over time (at varying rates) such that there
  *     are around 5,000 key range files in each folder.
  *
- *     Note that startVersion will NOT correspond to the minVersion of a snapshot manifest because 
+ *     Note that startVersion will NOT correspond to the minVersion of a snapshot manifest because
  *     snapshot manifest min/max versions are based on the actual contained data and the first data
  *     file written will be after the start version of the snapshot's execution.
- * 
+ *
  *   Log files are at file paths like
- *       /plogs/...log,startVersion,endVersion,UID,blocksize,tagID
+ *       /plogs/...log,startVersion,endVersion,UID,blocksize,tagID-of-N
  *       /logs/.../log,startVersion,endVersion,UID,blockSize
  *     where ... is a multi level path which sorts lexically into version order and results in approximately 1
  *     unique folder per day containing about 5,000 files. Logs after 7.0 are stored in "plogs"
- *     directory and are partitioned according to tagIDs (0, 1, 2, ...). Logs before 7.0 are
+ *     directory and are partitioned according to tagIDs (0, 1, 2, ...) and the total number
+ *     partitions is N. Logs before 7.0 are
  *     stored in "logs" directory and are not partitioned.
  *
  *
@@ -252,8 +253,8 @@ std::string BackupDescription::toJSON() const {
  *
  *   Prior to FDB version 6.0.16, key range files were stored using a different folder scheme.  Newer versions
  *   still support this scheme for all restore and backup management operations but key range files generated
- *   by backup using version 6.0.16 or later use the scheme describe above.  
- * 
+ *   by backup using version 6.0.16 or later use the scheme describe above.
+ *
  *   The old format stored key range files at paths like
  *       /ranges/.../range,version,uid,blockSize
  *     where ... is a multi level path with sorts lexically into version order and results in up to approximately
@@ -1060,10 +1061,75 @@ public:
 	}
 
 	// Delete all data up to (but not including endVersion)
-	Future<Void> expireData(Version expireEndVersion, bool force, ExpireProgress *progress, Version restorableBeginVersion) override {
+	Future<Void> expireData(Version expireEndVersion, bool force, ExpireProgress* progress,
+	                        Version restorableBeginVersion) final {
 		return expireData_impl(Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
 	}
 
+	// For a list of log files specified by their indices (of the same tag),
+	// returns if they are continous in the range [begin, end].
+	static bool isContinuous(const std::vector<LogFile>& files, std::vector<int> indices, Version begin, Version end,
+	                         std::map<std::pair<Version, Version>, int>* tags) {
+		Version lastBegin = invalidVersion;
+		Version lastEnd = invalidVersion;
+		int lastTags = -1;
+
+		for (int idx : indices) {
+			const LogFile& file = files[idx];
+			if (lastEnd == invalidVersion) {
+				if (file.beginVersion > begin) return false;
+				if (file.endVersion > begin) {
+					lastBegin = begin;
+					lastTags = file.totalTags;
+				} else {
+					continue;
+				}
+			} else if (lastEnd != file.beginVersion) {
+				return false; // not continuous
+			}
+
+			if (lastTags != file.totalTags) {
+				if (tags != nullptr) {
+					tags->emplace(std::make_pair(lastBegin, file.beginVersion - 1), lastTags);
+				}
+				lastBegin = file.beginVersion;
+				lastTags = file.totalTags;
+			}
+			lastEnd = file.endVersion;
+			if (lastEnd > end) break;
+		}
+		if (lastBegin == invalidVersion || lastEnd <= end) return false; // not covering the range
+		if (tags != nullptr) {
+			tags->emplace(std::make_pair(lastBegin, end), lastTags);
+		}
+		return true;
+	}
+
+	// Returns true if logs are continuous in the range [begin, end].
+	// "files" should be pre-sorted according to version order.
+	static bool isPartitionedLogsContinuous(const std::vector<LogFile>& files, Version begin, Version end) {
+		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
+		for (int i = 0; i < files.size(); i++) {
+			ASSERT(files[i].tagId >= 0 && files[i].tagId < files[i].totalTags);
+			auto& indices = tagIndices[files[i].tagId];
+			indices.push_back(i);
+		}
+
+		// check tag 0 is continuous and create a map of ranges to tags
+		std::map<std::pair<Version, Version>, int> tags; // range [start, end) -> tags
+		if (!isContinuous(files, tagIndices[0], begin, end, &tags)) return false;
+
+		// for each range in tags, check all tags from 1 are continouous
+		for (const auto [beginEnd, count] : tags) {
+			for (int i = 1; i < count; i++) {
+				if (!isContinuous(files, tagIndices[i], beginEnd.first, beginEnd.second, nullptr)) {
+					return false;
+				}
+			}
+		}
+		return true;
+	}
+
 	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion, bool partitioned) {
 		// Find the most recent keyrange snapshot to end at or before targetVersion
 		state Optional<KeyspaceSnapshotFile> snapshot;
@@ -1105,11 +1171,11 @@ public:
 				}
 				if (i < logs.size()) filtered.push_back(logs[i]);
 
-				// TODO(jingyu): for partitioned logs, the continuity checking should be based on
-				// epochs and versions, which should be saved in a metadata file by backup worker and
-				// thus is available here. For now, assume it's continuous.
 				restorable.logs.swap(filtered);
-				return Optional<RestorableFileSet>(restorable);
+				if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) {
+					return Optional<RestorableFileSet>(restorable);
+				}
+				return Optional<RestorableFileSet>();
 			}
 
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
@@ -2008,3 +2074,54 @@ TEST_CASE("/backup/time") {
 
 	return Void();
 }
+
+TEST_CASE("/backup/continuous") {
+	std::vector<LogFile> files;
+
+	// [0, 100) 2 tags
+	files.push_back({ 0, 100, 10, "file1", 100, 0, 2 }); // Tag 0: 0-100
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
+
+	files.push_back({ 0, 100, 10, "file2", 200, 1, 2 }); // Tag 1: 0-100
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
+
+	// [100, 300) 3 tags
+	files.push_back({ 100, 200, 10, "file3", 200, 0, 3 }); // Tag 0: 100-200
+	files.push_back({ 100, 250, 10, "file4", 200, 1, 3 }); // Tag 1: 100-250
+	std::sort(files.begin(), files.end());
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150));
+
+	files.push_back({ 100, 300, 10, "file5", 200, 2, 3 }); // Tag 2: 100-300
+	std::sort(files.begin(), files.end());
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 200));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 10, 199));
+
+	files.push_back({ 250, 300, 10, "file6", 200, 0, 3 }); // Tag 0: 250-300, missing 200-250
+	std::sort(files.begin(), files.end());
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 240));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+
+	files.push_back({ 250, 300, 10, "file7", 200, 1, 3 }); // Tag 1: 250-300
+	std::sort(files.begin(), files.end());
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+
+	files.push_back({ 200, 250, 10, "file8", 200, 0, 3 }); // Tag 0: 200-250
+	std::sort(files.begin(), files.end());
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 299));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+
+	// [300, 400) 1 tag
+	// files.push_back({200, 250, 10, "file9", 200, 0, 3}); // Tag 0: 200-250, duplicate file
+	files.push_back({ 300, 400, 10, "file10", 200, 0, 1 }); // Tag 1: 300-400
+	std::sort(files.begin(), files.end());
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 399));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 399));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 150, 399));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 250, 399));
+
+	return Void();
+}
\ No newline at end of file

From eb6a889c780c550e5986222caa2181023c2df687 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 24 Feb 2020 16:53:57 -0800
Subject: [PATCH 013/176] Describe backup uses partitioned logs to find
 continuous end version

For partitioned logs, the continuous end version has to be done range by range,
where each range must contain continuous version for all tags.
---
 fdbclient/BackupContainer.actor.cpp | 119 +++++++++++++++++++++++-----
 1 file changed, 99 insertions(+), 20 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index eed735eaa4..7b7f27c14e 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -781,13 +781,21 @@ public:
 		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) &&
 		     store(pLogs, bc->listLogFiles(scanBegin, scanEnd, true)) &&
 		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
-		// FIXME: check partitioned logs & maybe enable the below line
-		// logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
 		// List logs in version order so log continuity can be analyzed
 		std::sort(logs.begin(), logs.end());
 
-		if(!logs.empty()) {
+		// Check partitioned logs
+		if (!pLogs.empty()) {
+			std::sort(pLogs.begin(), pLogs.end());
+			// If we didn't get log versions above then seed them using the first log file
+			if(!desc.contiguousLogEnd.present()) {
+				auto it = pLogs.begin();
+				desc.minLogBegin = it->beginVersion;
+				desc.contiguousLogEnd = it->endVersion;
+			}
+			desc.contiguousLogEnd.get() = getPartitionedLogsContinuousEndVersion(pLogs, scanBegin);
+		} else if (!logs.empty()) {
 			desc.maxLogEnd = logs.rbegin()->endVersion;
 
 			auto i = logs.begin();
@@ -875,7 +883,7 @@ public:
 	}
 
 	// Uses the virtual methods to describe the backup contents
-	Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) override {
+	Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) final {
 		return describeBackup_impl(Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride);
 	}
 
@@ -1067,15 +1075,20 @@ public:
 	}
 
 	// For a list of log files specified by their indices (of the same tag),
-	// returns if they are continous in the range [begin, end].
+	// returns if they are continous in the range [begin, end]. If "tags" is not
+	// nullptr, then it will be populated with [begin, end] -> tags, where next
+	// pair's begin == previous pair's end + 1. On return, the last pair's end
+	// version (inclusive) gives the continuous range from begin.
 	static bool isContinuous(const std::vector<LogFile>& files, std::vector<int> indices, Version begin, Version end,
 	                         std::map<std::pair<Version, Version>, int>* tags) {
 		Version lastBegin = invalidVersion;
 		Version lastEnd = invalidVersion;
 		int lastTags = -1;
 
+		ASSERT(tags == nullptr || tags->empty());
 		for (int idx : indices) {
 			const LogFile& file = files[idx];
+std::cout << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags" << lastTags << "\n";
 			if (lastEnd == invalidVersion) {
 				if (file.beginVersion > begin) return false;
 				if (file.endVersion > begin) {
@@ -1085,7 +1098,10 @@ public:
 					continue;
 				}
 			} else if (lastEnd != file.beginVersion) {
-				return false; // not continuous
+				if (tags != nullptr) {
+					tags->emplace(std::make_pair(lastBegin, lastEnd - 1), lastTags);
+				}
+				return false;
 			}
 
 			if (lastTags != file.totalTags) {
@@ -1098,11 +1114,11 @@ public:
 			lastEnd = file.endVersion;
 			if (lastEnd > end) break;
 		}
-		if (lastBegin == invalidVersion || lastEnd <= end) return false; // not covering the range
-		if (tags != nullptr) {
-			tags->emplace(std::make_pair(lastBegin, end), lastTags);
+std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags" << lastTags << "\n";
+		if (tags != nullptr && lastBegin != invalidVersion) {
+			tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags);
 		}
-		return true;
+		return lastBegin != invalidVersion && lastEnd > end;
 	}
 
 	// Returns true if logs are continuous in the range [begin, end].
@@ -1116,7 +1132,7 @@ public:
 		}
 
 		// check tag 0 is continuous and create a map of ranges to tags
-		std::map<std::pair<Version, Version>, int> tags; // range [start, end) -> tags
+		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
 		if (!isContinuous(files, tagIndices[0], begin, end, &tags)) return false;
 
 		// for each range in tags, check all tags from 1 are continouous
@@ -1130,6 +1146,67 @@ public:
 		return true;
 	}
 
+	// Returns log files that are not duplicated.
+	static std::vector<LogFile> filterDuplicates(std::vector<LogFile>& logs) {
+		std::sort(logs.begin(), logs.end());
+
+		std::vector<LogFile> filtered;
+		int i = 0;
+		for (int j = 1; j < logs.size(); j++) {
+			if (!logs[i].sameContent(logs[j])) {
+				filtered.push_back(logs[i]);
+				i = j;
+			}
+		}
+		if (i < logs.size()) filtered.push_back(logs[i]);
+		return filtered;
+	}
+
+	// Returns the end version such that [begin, end] is continuous.
+	static Version getPartitionedLogsContinuousEndVersion(std::vector<LogFile>& logs, Version begin) {
+		auto files = filterDuplicates(logs);
+for (auto file : files) std::cout << file.toString() << "\n";
+		Version end = 0;
+
+		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
+		for (int i = 0; i < files.size(); i++) {
+			ASSERT(files[i].tagId >= 0 && files[i].tagId < files[i].totalTags);
+			auto& indices = tagIndices[files[i].tagId];
+			indices.push_back(i);
+			end = files[i].endVersion - 1;
+		}
+std::cout << "Init end: " << end << "\n";
+
+		// check tag 0 is continuous in [begin, end] and create a map of ranges to tags
+		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
+		isContinuous(files, tagIndices[0], begin, end, &tags);
+		if (tags.empty() || end <= begin) return 0;
+		end = std::min(end, tags.rbegin()->first.second);
+std::cout << "Tag 0 end: " << end << "\n";
+for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " << v << "\n";
+
+		// for each range in tags, check all tags from 1 are continouous
+		Version lastEnd = begin;
+		for (const auto [beginEnd, count] : tags) {
+			Version tagEnd = end; // This range's minimum continous tag version
+			for (int i = 1; i < count; i++) {
+				std::map<std::pair<Version, Version>, int> rangeTags;
+				isContinuous(files, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
+				tagEnd = rangeTags.empty() ? 0 : std::min(tagEnd, rangeTags.rbegin()->first.second);
+std::cout << "Tag " << i << " end: " << tagEnd << "\n";
+				if (tagEnd == 0) return lastEnd;
+			}
+			if (tagEnd < beginEnd.second) {
+				end = tagEnd;
+				break;
+			}
+			lastEnd = beginEnd.second;
+		}
+
+std::cout << "Return end = " << end << "\n\n";
+		return end;
+	}
+
 	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion, bool partitioned) {
 		// Find the most recent keyrange snapshot to end at or before targetVersion
 		state Optional<KeyspaceSnapshotFile> snapshot;
@@ -1161,15 +1238,7 @@ public:
 
 			if (partitioned) {
 				// Remove duplicated log files that can happen for old epochs.
-				std::vector<LogFile> filtered;
-				int i = 0;
-				for (int j = 1; j < logs.size(); j++) {
-					if (!logs[i].sameContent(logs[j])) {
-						filtered.push_back(logs[i]);
-						i = j;
-					}
-				}
-				if (i < logs.size()) filtered.push_back(logs[i]);
+				std::vector<LogFile> filtered = filterDuplicates(logs);
 
 				restorable.logs.swap(filtered);
 				if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) {
@@ -2081,10 +2150,12 @@ TEST_CASE("/backup/continuous") {
 	// [0, 100) 2 tags
 	files.push_back({ 0, 100, 10, "file1", 100, 0, 2 }); // Tag 0: 0-100
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 0);
 
 	files.push_back({ 0, 100, 10, "file2", 200, 1, 2 }); // Tag 1: 0-100
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 99);
 
 	// [100, 300) 3 tags
 	files.push_back({ 100, 200, 10, "file3", 200, 0, 3 }); // Tag 0: 100-200
@@ -2093,17 +2164,21 @@ TEST_CASE("/backup/continuous") {
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 99);
 
 	files.push_back({ 100, 300, 10, "file5", 200, 2, 3 }); // Tag 2: 100-300
 	std::sort(files.begin(), files.end());
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 200));
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 10, 199));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 199);
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 100) == 199);
 
 	files.push_back({ 250, 300, 10, "file6", 200, 0, 3 }); // Tag 0: 250-300, missing 200-250
 	std::sort(files.begin(), files.end());
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 240));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 99) == 199);
 
 	files.push_back({ 250, 300, 10, "file7", 200, 1, 3 }); // Tag 1: 250-300
 	std::sort(files.begin(), files.end());
@@ -2113,6 +2188,7 @@ TEST_CASE("/backup/continuous") {
 	std::sort(files.begin(), files.end());
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 299));
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 150) == 299);
 
 	// [300, 400) 1 tag
 	// files.push_back({200, 250, 10, "file9", 200, 0, 3}); // Tag 0: 200-250, duplicate file
@@ -2122,6 +2198,9 @@ TEST_CASE("/backup/continuous") {
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 399));
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 150, 399));
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 250, 399));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 399);
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 99) == 399);
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 250) == 399);
 
 	return Void();
 }
\ No newline at end of file

From af967210ee9f1830a4ac188d3a47fe002193bc4a Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 24 Feb 2020 16:57:31 -0800
Subject: [PATCH 014/176] StagingKey can add out-of-order mutations

For partitioned logs, mutations of the same version may be sent to applier
out-of-order. If one loader advances to the next version, an applier may
receive later version mutations for different loaders. So, dropping of early
mutations is wrong.
---
 fdbserver/RestoreApplier.actor.h    | 36 ++++++++++++++-----------
 fdbserver/RestoreLoader.actor.cpp   | 41 +++++++++++++----------------
 fdbserver/RestoreRoleCommon.actor.h |  3 +++
 3 files changed, 41 insertions(+), 39 deletions(-)

diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index 9d8f6b60d8..a50c7f346f 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -60,21 +60,7 @@ struct StagingKey {
 	// Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set
 	void add(const MutationRef& m, LogMessageVersion newVersion) {
 		ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue);
-		if (version < newVersion) {
-			if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
-				key = m.param1;
-				val = m.param2;
-				type = (MutationRef::Type)m.type;
-				version = newVersion;
-			} else {
-				if (pendingMutations.find(newVersion) == pendingMutations.end()) {
-					pendingMutations.emplace(newVersion, MutationsVec());
-				}
-				// TODO: Do we really need deep copy?
-				MutationsVec& mutations = pendingMutations[newVersion];
-				mutations.push_back_deep(mutations.arena(), m);
-			}
-		} else if (version == newVersion) { // Sanity check
+		if (version == newVersion) { // Sanity check
 			TraceEvent("FastRestoreApplierStagingKeyMutationAtSameVersion")
 			    .detail("Version", newVersion.toString())
 			    .detail("NewMutation", m.toString())
@@ -106,7 +92,25 @@ struct StagingKey {
 				    .detail("ExistingKeyType", typeString[type])
 				    .detail("ExitingKeyValue", val);
 			}
-		} // else  input mutation is old and can be ignored
+		}
+		// newVersion can be smaller than version as different loaders can send
+		// mutations out of order.
+		if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
+			if (version < newVersion) {
+				key = m.param1;
+				val = m.param2;
+				type = (MutationRef::Type)m.type;
+				version = newVersion;
+			}
+		} else {
+			auto it = pendingMutations.find(newVersion);
+			if (it == pendingMutations.end()) {
+				bool inserted;
+				std::tie(it, inserted) = pendingMutations.emplace(newVersion, MutationsVec());
+			}
+			// TODO: Do we really need deep copy?
+			it->second.push_back_deep(it->second.arena(), m);
+		}
 	}
 
 	// Precompute the final value of the key.
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 15dbb3e179..109b25c43c 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -166,7 +166,6 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
 
 		VersionedMutationsMap& kvOps = kvOpsIter->second;
-		VersionedMutationsMap::iterator it = kvOps.end();
 		while (1) {
 			// If eof reached or first key len bytes is 0xFF then end of block was reached.
 			if (reader.eof() || *reader.rptr == 0xFF) break;
@@ -181,6 +180,7 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 			// Skip mutations out of the version range
 			if (!asset.isInVersionRange(msgVersion.version)) continue;
 
+			VersionedMutationsMap::iterator it;
 			bool inserted;
 			std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec());
 			ASSERT(inserted);
@@ -327,7 +327,6 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
 ACTOR Future<Void> handleSendMutationsRequest(RestoreSendMutationsToAppliersRequest req,
                                               Reference<RestoreLoaderData> self) {
 	state Reference<LoaderBatchData> batchData = self->batch[req.batchIndex];
-	state std::map<LoadingParam, VersionedMutationsMap>::iterator item = batchData->kvOpsPerLP.begin();
 	state Reference<LoaderBatchStatus> batchStatus = self->status[req.batchIndex];
 	state bool isDuplicated = true;
 
@@ -377,11 +376,11 @@ ACTOR Future<Void> handleSendMutationsRequest(RestoreSendMutationsToAppliersRequ
 	if (!isDuplicated) {
 		vector<Future<Void>> fSendMutations;
 		batchData->rangeToApplier = req.rangeToApplier;
-		for (; item != batchData->kvOpsPerLP.end(); item++) {
-			if (item->first.isRangeFile == req.useRangeFile) {
+		for (auto& [loadParam, kvOps] : batchData->kvOpsPerLP) {
+			if (loadParam.isRangeFile == req.useRangeFile) {
 				// Send the parsed mutation to applier who will apply the mutation to DB
-				fSendMutations.push_back(sendMutationsToApplier(&item->second, req.batchIndex, item->first.asset,
-				                                                item->first.isRangeFile, &batchData->rangeToApplier,
+				fSendMutations.push_back(sendMutationsToApplier(&kvOps, req.batchIndex, loadParam.asset,
+				                                                loadParam.isRangeFile, &batchData->rangeToApplier,
 				                                                &self->appliersInterf));
 			}
 		}
@@ -423,7 +422,7 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 	    .detail("RestoreAsset", asset.toString());
 
 	// There should be no mutation at asset.endVersion version because it is exclusive
-	if (kvOps.find(LogMessageVersion(asset.endVersion)) != kvOps.end()) {
+	if (kvOps.lower_bound(LogMessageVersion(asset.endVersion)) != kvOps.end()) {
 		TraceEvent(SevError, "FastRestoreLoaderSendMutationToApplier")
 		    .detail("BatchIndex", batchIndex)
 		    .detail("RestoreAsset", asset.toString())
@@ -449,12 +448,6 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 			applierMutationsSize[applierID] = 0.0;
 		}
 		const LogMessageVersion& commitVersion = kvOp->first;
-		if (!(commitVersion.version >= asset.beginVersion &&
-		      commitVersion.version <= asset.endVersion)) { // Debug purpose
-			TraceEvent(SevError, "FastRestore_SendMutationsToApplier")
-			    .detail("CommitVersion", commitVersion.version)
-			    .detail("RestoreAsset", asset.toString());
-		}
 		ASSERT(commitVersion.version >= asset.beginVersion);
 		ASSERT(commitVersion.version <= asset.endVersion); // endVersion is an empty commit to ensure progress
 
@@ -485,15 +478,14 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 				std::map<Key, UID>::iterator itlow = pRangeToApplier->upper_bound(kvm.param1);
 				--itlow; // make sure itlow->first <= m.param1
 				ASSERT(itlow->first <= kvm.param1);
-				MutationRef mutation = kvm;
 				UID applierID = itlow->second;
 				// printf("KV--Applier: K:%s ApplierID:%s\n", kvm.param1.toString().c_str(),
 				// applierID.toString().c_str());
 				kvCount++;
 
-				applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
+				applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), kvm);
 				applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
-				applierMutationsSize[applierID] += mutation.expectedSize();
+				applierMutationsSize[applierID] += kvm.expectedSize();
 			}
 		} // Mutations at the same version
 
@@ -606,20 +598,23 @@ bool concatenateBackupMutationForLogFile(std::map<Standalone<StringRef>, Standal
 	if (it == mutationMap.end()) {
 		mutationMap.insert(std::make_pair(id, val_input));
 		if (part != 0) {
-			TraceEvent(SevError, "FastRestore").detail("FirstPartNotZero", part).detail("KeyInput", getHexString(key_input));
+			TraceEvent(SevError, "FastRestore")
+			    .detail("FirstPartNotZero", part)
+			    .detail("KeyInput", getHexString(key_input));
 		}
 		mutationPartMap.insert(std::make_pair(id, part));
 	} else { // Concatenate the val string with the same commitVersion
 		it->second = it->second.contents().withSuffix(val_input.contents()); // Assign the new Areana to the map's value
-		if (part != (mutationPartMap[id] + 1)) {
+		auto& currentPart = mutationPartMap[id];
+		if (part != (currentPart + 1)) {
 			// Check if the same range or log file has been processed more than once!
 			TraceEvent(SevError, "FastRestore")
-				.detail("CurrentPart1", mutationPartMap[id])
-				.detail("CurrentPart2", part)
-				.detail("KeyInput", getHexString(key_input))
-				.detail("Hint", "Check if the same range or log file has been processed more than once");
+			    .detail("CurrentPart1", currentPart)
+			    .detail("CurrentPart2", part)
+			    .detail("KeyInput", getHexString(key_input))
+			    .detail("Hint", "Check if the same range or log file has been processed more than once");
 		}
-		mutationPartMap[id] = part;
+		currentPart = part;
 		concatenated = true;
 	}
 
diff --git a/fdbserver/RestoreRoleCommon.actor.h b/fdbserver/RestoreRoleCommon.actor.h
index cedbeb795c..5a3b30509a 100644
--- a/fdbserver/RestoreRoleCommon.actor.h
+++ b/fdbserver/RestoreRoleCommon.actor.h
@@ -55,6 +55,9 @@ struct RestoreSimpleRequest;
 // Value MutationsVec is the vector of parsed backup mutations.
 // For old mutation logs, the subsequence number is always 0.
 // For partitioned mutation logs, each mutation has a unique LogMessageVersion.
+// Note for partitioned logs, one LogMessageVersion can have multiple mutations,
+// because a clear mutation may be split into several smaller clear mutations by
+// backup workers.
 using VersionedMutationsMap = std::map<LogMessageVersion, MutationsVec>;
 
 ACTOR Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name);

From f697ccd1b906087c06dc4623b77020cebe5d2931 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 25 Feb 2020 16:37:25 -0800
Subject: [PATCH 015/176] Add describePartitionedBackup() for parallel restore

For partitioned logs, computing continuous log end version from min logs begin
version. Old backup test keeps using describeBackup() to be correctness clean.

Rename partitioned log file so that the last number is block size.
---
 fdbclient/BackupContainer.actor.cpp           | 125 +++++++++---------
 fdbclient/BackupContainer.h                   |   3 +
 fdbserver/RestoreMaster.actor.cpp             |   2 +-
 ...kupAndParallelRestoreCorrectness.actor.cpp |   4 +-
 4 files changed, 69 insertions(+), 65 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 7b7f27c14e..01d76bfb87 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -240,7 +240,7 @@ std::string BackupDescription::toJSON() const {
  *     file written will be after the start version of the snapshot's execution.
  *
  *   Log files are at file paths like
- *       /plogs/...log,startVersion,endVersion,UID,blocksize,tagID-of-N
+ *       /plogs/...log,startVersion,endVersion,UID,tagID-of-N,blocksize
  *       /logs/.../log,startVersion,endVersion,UID,blockSize
  *     where ... is a multi level path which sorts lexically into version order and results in approximately 1
  *     unique folder per day containing about 5,000 files. Logs after 7.0 are stored in "plogs"
@@ -347,9 +347,9 @@ public:
 	Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
 	                                                  uint16_t tagId, int totalTags) final {
 		return writeFile(logVersionFolderString(beginVersion, true) +
-		                 format("log,%lld,%lld,%s,%d,%d-of-%d", beginVersion, endVersion,
-		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize, tagId,
-		                        totalTags));
+		                 format("log,%lld,%lld,%s,%d-of-%d,%d", beginVersion, endVersion,
+		                        deterministicRandom()->randomUniqueID().toString().c_str(), tagId, totalTags,
+		                        blockSize));
 	}
 
 	Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) override {
@@ -400,8 +400,8 @@ public:
 		if(sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n", &f.beginVersion, &f.endVersion, &f.blockSize, &len) == 3 && len == name.size()) {
 			out = f;
 			return true;
-		} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u,%d-of-%d%n", &f.beginVersion,
-		                  &f.endVersion, &f.blockSize, &f.tagId, &f.totalTags, &len) == 5 &&
+		} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%d-of-%d,%u%n", &f.beginVersion,
+		                  &f.endVersion, &f.tagId, &f.totalTags, &f.blockSize, &len) == 5 &&
 		           len == name.size() && f.tagId >= 0) {
 			out = f;
 			return true;
@@ -672,7 +672,27 @@ public:
 		return v;
 	}
 
-	ACTOR static Future<BackupDescription> describeBackup_impl(Reference<BackupContainerFileSystem> bc, bool deepScan, Version logStartVersionOverride) {
+	// Computes the continuous end version for non-partitioned mutation logs up to
+	// the "targetVersion". If "outLogs" is not nullptr, it will be updated with
+	// continuous log files. "*end" is updated with the continuous end version.
+	static void computeRestoreEndVersion(const std::vector<LogFile>& logs, std::vector<LogFile>* outLogs, Version* end,
+	                                     Version targetVersion) {
+		auto i = logs.begin();
+		if (outLogs != nullptr) outLogs->push_back(*i);
+
+		// Add logs to restorable logs set until continuity is broken OR we reach targetVersion
+		while (++i != logs.end()) {
+			if (i->beginVersion > *end || i->beginVersion > targetVersion) break;
+
+			// If the next link in the log chain is found, update the end
+			if (i->beginVersion == *end) {
+				if (outLogs != nullptr) outLogs->push_back(*i);
+				*end = i->endVersion;
+			}
+		}
+	}
+
+	ACTOR static Future<BackupDescription> describeBackup_impl(Reference<BackupContainerFileSystem> bc, bool deepScan, Version logStartVersionOverride, bool partitioned) {
 		state BackupDescription desc;
 		desc.url = bc->getURL();
 
@@ -690,8 +710,10 @@ public:
 		// from which to resolve the relative version.
 		// This could be handled more efficiently without recursion but it's tricky, this will do for now.
 		if(logStartVersionOverride != invalidVersion && logStartVersionOverride < 0) {
-			BackupDescription tmp = wait(bc->describeBackup(false, invalidVersion));
-			logStartVersionOverride = resolveRelativeVersion(tmp.maxLogEnd, logStartVersionOverride, "LogStartVersionOverride", invalid_option_value());
+			BackupDescription tmp = wait(partitioned ? bc->describePartitionedBackup(false, invalidVersion)
+			                                         : bc->describeBackup(false, invalidVersion));
+			logStartVersionOverride = resolveRelativeVersion(tmp.maxLogEnd, logStartVersionOverride,
+			                                                 "LogStartVersionOverride", invalid_option_value());
 		}
 
 		// Get metadata versions
@@ -777,45 +799,31 @@ public:
 		}
 
 		state std::vector<LogFile> logs;
-		state std::vector<LogFile> pLogs;
-		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) &&
-		     store(pLogs, bc->listLogFiles(scanBegin, scanEnd, true)) &&
+		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, partitioned)) &&
 		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
 
 		// List logs in version order so log continuity can be analyzed
 		std::sort(logs.begin(), logs.end());
 
-		// Check partitioned logs
-		if (!pLogs.empty()) {
-			std::sort(pLogs.begin(), pLogs.end());
+		// Find out contiguous log end version
+		if (partitioned) {
 			// If we didn't get log versions above then seed them using the first log file
-			if(!desc.contiguousLogEnd.present()) {
-				auto it = pLogs.begin();
-				desc.minLogBegin = it->beginVersion;
-				desc.contiguousLogEnd = it->endVersion;
+			if (!desc.contiguousLogEnd.present()) {
+				desc.minLogBegin = logs.begin()->beginVersion;
+				desc.contiguousLogEnd = logs.begin()->endVersion;
 			}
-			desc.contiguousLogEnd.get() = getPartitionedLogsContinuousEndVersion(pLogs, scanBegin);
+			// contiguousLogEnd is not inclusive, so +1 here.
+			desc.contiguousLogEnd.get() = getPartitionedLogsContinuousEndVersion(logs, desc.minLogBegin.get()) + 1;
 		} else if (!logs.empty()) {
 			desc.maxLogEnd = logs.rbegin()->endVersion;
 
-			auto i = logs.begin();
 			// If we didn't get log versions above then seed them using the first log file
 			if(!desc.contiguousLogEnd.present()) {
-				desc.minLogBegin = i->beginVersion;
-				desc.contiguousLogEnd = i->endVersion;
-				++i;
-			}
-			auto &end = desc.contiguousLogEnd.get();  // For convenience to make loop cleaner
-
-			// Advance until continuity is broken
-			while(i != logs.end()) {
-				if(i->beginVersion > end)
-					break;
-				// If the next link in the log chain is found, update the end
-				if(i->beginVersion == end)
-					end = i->endVersion;
-				++i;
+				desc.minLogBegin = logs.begin()->beginVersion;
+				desc.contiguousLogEnd = logs.begin()->endVersion;
 			}
+			Version& end = desc.contiguousLogEnd.get();
+			computeRestoreEndVersion(logs, nullptr, &end, std::numeric_limits<Version>::max());
 		}
 
 		// Only update stored contiguous log begin and end versions if we did NOT use a log start override.
@@ -884,7 +892,11 @@ public:
 
 	// Uses the virtual methods to describe the backup contents
 	Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) final {
-		return describeBackup_impl(Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride);
+		return describeBackup_impl(Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride, false);
+	}
+
+	Future<BackupDescription> describePartitionedBackup(bool deepScan, Version logStartVersionOverride) final {
+		return describeBackup_impl(Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride, true);
 	}
 
 	ACTOR static Future<Void> expireData_impl(Reference<BackupContainerFileSystem> bc, Version expireEndVersion, bool force, ExpireProgress *progress, Version restorableBeginVersion) {
@@ -1175,7 +1187,7 @@ for (auto file : files) std::cout << file.toString() << "\n";
 			indices.push_back(i);
 			end = files[i].endVersion - 1;
 		}
-std::cout << "Init end: " << end << "\n";
+std::cout << "Init end: " << end << ", begin " << begin << "\n";
 
 		// check tag 0 is continuous in [begin, end] and create a map of ranges to tags
 		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
@@ -1249,22 +1261,9 @@ std::cout << "Return end = " << end << "\n\n";
 
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
 			if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {
-				auto i = logs.begin();
-				Version end = i->endVersion;
-				restorable.logs.push_back(*i);
-
-				// Add logs to restorable logs set until continuity is broken OR we reach targetVersion
-				while(++i != logs.end()) {
-					if(i->beginVersion > end || i->beginVersion > targetVersion)
-						break;
-					// If the next link in the log chain is found, update the end
-					if(i->beginVersion == end) {
-						restorable.logs.push_back(*i);
-						end = i->endVersion;
-					}
-				}
-
-				if(end >= targetVersion) {
+				Version end = logs.begin()->endVersion;
+				computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion);
+				if (end >= targetVersion) {
 					return Optional<RestorableFileSet>(restorable);
 				}
 			}
@@ -1460,6 +1459,7 @@ public:
 			if(deterministicRandom()->random01() < .01) {
 				blockSize /= deterministicRandom()->randomInt(1, 3);
 			}
+			ASSERT(blockSize > 0);
 
 			return map(f, [=](Reference<IAsyncFile> fr) {
 				int readAhead = deterministicRandom()->randomInt(0, 3);
@@ -1609,15 +1609,16 @@ public:
 	virtual ~BackupContainerBlobStore() {}
 
 	Future<Reference<IAsyncFile>> readFile(std::string path) final {
-			return Reference<IAsyncFile>(
-				new AsyncFileReadAheadCache(
-					Reference<IAsyncFile>(new AsyncFileBlobStoreRead(m_bstore, m_bucket, dataPath(path))),
-					m_bstore->knobs.read_block_size,
-					m_bstore->knobs.read_ahead_blocks,
-					m_bstore->knobs.concurrent_reads_per_file,
-					m_bstore->knobs.read_cache_blocks_per_file
-				)
-			);
+		ASSERT(m_bstore->knobs.read_ahead_blocks > 0);
+		return Reference<IAsyncFile>(
+			new AsyncFileReadAheadCache(
+				Reference<IAsyncFile>(new AsyncFileBlobStoreRead(m_bstore, m_bucket, dataPath(path))),
+				m_bstore->knobs.read_block_size,
+				m_bstore->knobs.read_ahead_blocks,
+				m_bstore->knobs.concurrent_reads_per_file,
+				m_bstore->knobs.read_cache_blocks_per_file
+			)
+		);
 	}
 
 	ACTOR static Future<std::vector<std::string>> listURLs(Reference<BlobStoreEndpoint> bstore, std::string bucket) {
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 437f6e3eaa..3eba09f06f 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -255,6 +255,9 @@ public:
 	// be after deleting all data prior to logStartVersionOverride.
 	virtual Future<BackupDescription> describeBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0;
 
+	// The same as above, except using partitioned mutation logs.
+	virtual Future<BackupDescription> describePartitionedBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0;
+
 	virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0;
 
 	// Get exactly the files necessary to restore to targetVersion.  Returns non-present if
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 2fba9204d2..a4da897650 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -676,7 +676,7 @@ ACTOR static Future<Standalone<VectorRef<RestoreRequest>>> collectRestoreRequest
 ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
                                              std::vector<RestoreFileFR>* logFiles, Database cx,
                                              RestoreRequest request) {
-	state BackupDescription desc = wait(bc->describeBackup());
+	state BackupDescription desc = wait(bc->describePartitionedBackup());
 
 	// Convert version to real time for operators to read the BackupDescription desc.
 	wait(desc.resolveVersionTimes(cx));
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 2435d3f4e0..6a1fec7129 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -209,7 +209,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 
 					state bool restorable = false;
 					if(lastBackupContainer) {
-						state Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
+						state Future<BackupDescription> fdesc = lastBackupContainer->describePartitionedBackup();
 						wait(ready(fdesc));
 
 						if(!fdesc.isError()) {
@@ -430,7 +430,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				    .detail("BackupTag", printable(self->backupTag));
 
 				auto container = IBackupContainer::openContainer(lastBackupContainer->getURL());
-				BackupDescription desc = wait(container->describeBackup());
+				BackupDescription desc = wait(container->describePartitionedBackup());
 
 				state Version targetVersion = -1;
 				if (desc.maxRestorableVersion.present()) {

From 4c93a23af901b77413dabfaa71e46c23f4df97cb Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 27 Feb 2020 14:04:19 -0800
Subject: [PATCH 016/176] Partitioned logs need to compute continuous begin
 Version

Because different tags may start at different versions, tag 0 can start at a
higher version. In this case, another tag's high version should be used as
the start version for continuous logs.
---
 fdbclient/BackupContainer.actor.cpp | 63 ++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 23 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 01d76bfb87..097feaf2af 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -799,6 +799,7 @@ public:
 		}
 
 		state std::vector<LogFile> logs;
+std::cout << "describe list: scanBegin:" << scanBegin << ", scanEnd:" << scanEnd << ", partitioned:" << partitioned << "\n";
 		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, partitioned)) &&
 		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
 
@@ -806,24 +807,20 @@ public:
 		std::sort(logs.begin(), logs.end());
 
 		// Find out contiguous log end version
-		if (partitioned) {
+		if (!logs.empty()) {
+			desc.maxLogEnd = logs.rbegin()->endVersion;
 			// If we didn't get log versions above then seed them using the first log file
 			if (!desc.contiguousLogEnd.present()) {
 				desc.minLogBegin = logs.begin()->beginVersion;
 				desc.contiguousLogEnd = logs.begin()->endVersion;
 			}
-			// contiguousLogEnd is not inclusive, so +1 here.
-			desc.contiguousLogEnd.get() = getPartitionedLogsContinuousEndVersion(logs, desc.minLogBegin.get()) + 1;
-		} else if (!logs.empty()) {
-			desc.maxLogEnd = logs.rbegin()->endVersion;
 
-			// If we didn't get log versions above then seed them using the first log file
-			if(!desc.contiguousLogEnd.present()) {
-				desc.minLogBegin = logs.begin()->beginVersion;
-				desc.contiguousLogEnd = logs.begin()->endVersion;
+			if (partitioned) {
+				determinePartitionedLogsBeginEnd(&desc, logs);
+			} else {
+				Version& end = desc.contiguousLogEnd.get();
+				computeRestoreEndVersion(logs, nullptr, &end, std::numeric_limits<Version>::max());
 			}
-			Version& end = desc.contiguousLogEnd.get();
-			computeRestoreEndVersion(logs, nullptr, &end, std::numeric_limits<Version>::max());
 		}
 
 		// Only update stored contiguous log begin and end versions if we did NOT use a log start override.
@@ -1091,8 +1088,8 @@ public:
 	// nullptr, then it will be populated with [begin, end] -> tags, where next
 	// pair's begin == previous pair's end + 1. On return, the last pair's end
 	// version (inclusive) gives the continuous range from begin.
-	static bool isContinuous(const std::vector<LogFile>& files, std::vector<int> indices, Version begin, Version end,
-	                         std::map<std::pair<Version, Version>, int>* tags) {
+	static bool isContinuous(const std::vector<LogFile>& files, const std::vector<int>& indices, Version begin,
+	                         Version end, std::map<std::pair<Version, Version>, int>* tags) {
 		Version lastBegin = invalidVersion;
 		Version lastEnd = invalidVersion;
 		int lastTags = -1;
@@ -1100,7 +1097,7 @@ public:
 		ASSERT(tags == nullptr || tags->empty());
 		for (int idx : indices) {
 			const LogFile& file = files[idx];
-std::cout << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags" << lastTags << "\n";
+std::cout << "  " << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags " << lastTags << "\n";
 			if (lastEnd == invalidVersion) {
 				if (file.beginVersion > begin) return false;
 				if (file.endVersion > begin) {
@@ -1126,7 +1123,7 @@ std::cout << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd "
 			lastEnd = file.endVersion;
 			if (lastEnd > end) break;
 		}
-std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags" << lastTags << "\n";
+std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags " << lastTags << "\n";
 		if (tags != nullptr && lastBegin != invalidVersion) {
 			tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags);
 		}
@@ -1159,9 +1156,8 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 	}
 
 	// Returns log files that are not duplicated.
-	static std::vector<LogFile> filterDuplicates(std::vector<LogFile>& logs) {
-		std::sort(logs.begin(), logs.end());
-
+	// PRE-CONDITION: logs are already sorted.
+	static std::vector<LogFile> filterDuplicates(const std::vector<LogFile>& logs) {
 		std::vector<LogFile> filtered;
 		int i = 0;
 		for (int j = 1; j < logs.size(); j++) {
@@ -1174,10 +1170,30 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 		return filtered;
 	}
 
+	// Analyze partitioned logs and set minLogBegin and contiguousLogEnd.
+	// For partitioned logs, different tags may start at different versions, so
+	// we need to find the "minLogBegin" version as well.
+	static void determinePartitionedLogsBeginEnd(BackupDescription* desc, const std::vector<LogFile>& logs) {
+		if (logs.empty()) return;
+
+		for (const LogFile& file : logs) {
+			Version end = getPartitionedLogsContinuousEndVersion(logs, file.beginVersion);
+std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
+			if (end > file.beginVersion) {
+				desc->minLogBegin = file.beginVersion;
+				// contiguousLogEnd is not inclusive, so +1 here.
+				desc->contiguousLogEnd.get() = end + 1;
+				return;
+			}
+		}
+	}
+
 	// Returns the end version such that [begin, end] is continuous.
-	static Version getPartitionedLogsContinuousEndVersion(std::vector<LogFile>& logs, Version begin) {
+	// "logs" should be already sorted.
+	static Version getPartitionedLogsContinuousEndVersion(const std::vector<LogFile>& logs, Version begin) {
 		auto files = filterDuplicates(logs);
-for (auto file : files) std::cout << file.toString() << "\n";
+std::cout << "getPartitionedLogsContinuousEndVersion begin:" << begin << "\n";
+for (auto file : files) std::cout << "    " << file.toString() << "\n";
 		Version end = 0;
 
 		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
@@ -1185,7 +1201,7 @@ for (auto file : files) std::cout << file.toString() << "\n";
 			ASSERT(files[i].tagId >= 0 && files[i].tagId < files[i].totalTags);
 			auto& indices = tagIndices[files[i].tagId];
 			indices.push_back(i);
-			end = files[i].endVersion - 1;
+			end = std::max(end, files[i].endVersion - 1);
 		}
 std::cout << "Init end: " << end << ", begin " << begin << "\n";
 
@@ -1194,7 +1210,7 @@ std::cout << "Init end: " << end << ", begin " << begin << "\n";
 		isContinuous(files, tagIndices[0], begin, end, &tags);
 		if (tags.empty() || end <= begin) return 0;
 		end = std::min(end, tags.rbegin()->first.second);
-std::cout << "Tag 0 end: " << end << "\n";
+std::cout << "  Tag 0 end: " << end << "\n";
 for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " << v << "\n";
 
 		// for each range in tags, check all tags from 1 are continouous
@@ -1205,7 +1221,7 @@ for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " <
 				std::map<std::pair<Version, Version>, int> rangeTags;
 				isContinuous(files, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
 				tagEnd = rangeTags.empty() ? 0 : std::min(tagEnd, rangeTags.rbegin()->first.second);
-std::cout << "Tag " << i << " end: " << tagEnd << "\n";
+std::cout << "  Tag " << i << " end: " << tagEnd << ", return end = "<< lastEnd << "\n";
 				if (tagEnd == 0) return lastEnd;
 			}
 			if (tagEnd < beginEnd.second) {
@@ -2154,6 +2170,7 @@ TEST_CASE("/backup/continuous") {
 	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 0);
 
 	files.push_back({ 0, 100, 10, "file2", 200, 1, 2 }); // Tag 1: 0-100
+	std::sort(files.begin(), files.end());
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
 	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 99);

From a20236a74dd868f2dcb550879a03ae5332dca9a0 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 27 Feb 2020 19:51:12 -0800
Subject: [PATCH 017/176] Fix backup worker does NOOP pop before getting backup
 key

The NOOP pop cuases some mutation ranges being dropped by backup workers. As a
result, the backup is incomplete. Specifically, the wait of BACKUP_NOOP_POP_DELAY
blocks the monitoring of backup key actor.
---
 fdbserver/BackupWorker.actor.cpp | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index d4546b6295..889931449b 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -589,25 +589,25 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 
 ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
 	state Future<Void> started, pullFinished;
+	state Future<GetReadVersionReply> replyFuture = Never();
 
 	loop {
 		started = monitorBackupStartedKeyChanges(self, true);
-		loop {
-			GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
-			                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
-
-			choose {
-				when(wait(started)) { break; }
-				when(wait(self->cx->onMasterProxiesChanged())) {}
-				when(GetReadVersionReply reply = wait(loadBalance(self->cx->getMasterProxies(false),
-				                                                  &MasterProxyInterface::getConsistentReadVersion,
-				                                                  request, self->cx->taskID))) {
-					self->savedVersion = std::max(reply.version, self->savedVersion);
-					self->minKnownCommittedVersion = std::max(reply.version, self->minKnownCommittedVersion);
-					TraceEvent("BackupWorkerNoopPop", self->myId).detail("SavedVersion", self->savedVersion);
-					self->pop(); // Pop while the worker is in this NOOP state.
-					wait(delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID));
-				}
+		loop choose {
+			when(wait(started)) { break; }
+			when(wait(self->cx->onMasterProxiesChanged() ||
+			          delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
+				GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
+				                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
+				replyFuture = loadBalance(self->cx->getMasterProxies(false),
+				                          &MasterProxyInterface::getConsistentReadVersion, request, self->cx->taskID);
+			}
+			when(GetReadVersionReply reply = wait(replyFuture)) {
+				replyFuture = Never();
+				self->savedVersion = std::max(reply.version, self->savedVersion);
+				self->minKnownCommittedVersion = std::max(reply.version, self->minKnownCommittedVersion);
+				TraceEvent("BackupWorkerNoopPop", self->myId).detail("SavedVersion", self->savedVersion);
+				self->pop(); // Pop while the worker is in this NOOP state.
 			}
 		}
 

From 07f1dcb5c98a53eeefb978ef55a03215095f06a8 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 28 Feb 2020 14:11:14 -0800
Subject: [PATCH 018/176] Fix contract changes: backup worker generate
 continuous versions

Before we allow holes in version ranges in partitioned mutation logs. This
has been changed so that restore can easily figure out if database is
restorable. A specific problem is that if the backup worker didn't find any
mutations for an old epoch, the worker can just exit without generating a
log file, thus leaving holes in version ranges.

Another contract change is that if a backup key is set, then we must store
all mutations for that key, especially for the worker for the old epoch. As a
result, the worker must first check backup key, before pulling mutations and
uploading logs. Otherwise, we may lose mutations.

Finally, when a backup key is removed, the saving of mutations should be up to
the current version so that backup worker doesn't exit too early. I.e., avoid
the case saved mutation versions are less than the snapshot version taken.
---
 fdbserver/BackupWorker.actor.cpp | 143 ++++++++++++++++++++++---------
 1 file changed, 102 insertions(+), 41 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 889931449b..bd33822dd5 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -78,15 +78,30 @@ struct BackupData {
 	Database cx;
 	std::vector<VersionedMessage> messages;
 	AsyncVar<bool> pullFinished;
+	NotifiedVersion pulledVersion;
 
 	struct PerBackupInfo {
 		PerBackupInfo() = default;
 		PerBackupInfo(BackupData* data, Version v) : self(data), startVersion(v) {}
 
-		bool isRunning() {
+		bool isReady() const {
+			return stopped || (container.isReady() && ranges.isReady());
+		}
+
+		bool isRunning() const {
 			return container.isReady() && ranges.isReady() && !stopped;
 		}
 
+		Future<Void> waitReady() {
+			if (stopped) return Void();
+			return _waitReady(this);
+		}
+
+		ACTOR static Future<Void> _waitReady(PerBackupInfo* info) {
+			wait(success(info->container) && success(info->ranges));
+			return Void();
+		}
+
 		BackupData* self = nullptr;
 		Version startVersion = invalidVersion;
 		Version lastSavedVersion = invalidVersion;
@@ -105,7 +120,8 @@ struct BackupData {
 	explicit BackupData(UID id, Reference<AsyncVar<ServerDBInfo>> db, const InitializeBackupRequest& req)
 	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
 	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
-	    minKnownCommittedVersion(invalidVersion), savedVersion(invalidVersion), cc("BackupWorker", myId.toString()) {
+	    minKnownCommittedVersion(invalidVersion), savedVersion(invalidVersion), cc("BackupWorker", myId.toString()),
+	    pulledVersion(0) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
 		pullFinished.set(false);
 
@@ -204,11 +220,38 @@ struct BackupData {
 		}
 		if (modified) changedTrigger.trigger();
 	}
+
+	ACTOR static Future<Void> _waitAllInfoReady(BackupData* self) {
+		std::vector<Future<Void>> all;
+		for (auto it = self->backups.begin(); it != self->backups.end(); ) {
+			if (it->second.stopped) {
+				TraceEvent("BackupWorkerRemoveStoppedContainer", self->myId).detail("BackupId", it->first);
+				it = self->backups.erase(it);
+				continue;
+			}
+			all.push_back(it->second.waitReady());
+			it++;
+		}
+		wait(waitForAll(all));
+		return Void();
+	}
+
+	Future<Void> waitAllInfoReady() {
+		return _waitAllInfoReady(this);
+	}
+
+	bool isAllInfoReady() const {
+		for (const auto& [uid, info] : backups) {
+			if (!info.isReady()) return false;
+		}
+		return true;
+	}
 };
 
 // Monitors "backupStartedKey". If "started" is true, wait until the key is set;
-// otherwise, wait until the key is cleared.
-ACTOR Future<Void> monitorBackupStartedKeyChanges(BackupData* self, bool started) {
+// otherwise, wait until the key is cleared. If "watch" is false, do not perform
+// the wait for key set/clear events. Returns if key present.
+ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started, bool watch) {
 	loop {
 		state ReadYourWritesTransaction tr(self->cx);
 
@@ -228,13 +271,13 @@ ACTOR Future<Void> monitorBackupStartedKeyChanges(BackupData* self, bool started
 						i++;
 					}
 					self->onBackupChanges(uidVersions);
-					if (started) return Void();
+					if (started || !watch) return true;
 				} else {
 					TraceEvent("BackupWorkerEmptyStartKey", self->myId);
 					self->onBackupChanges(uidVersions);
 
-					if (!started) {
-						return Void();
+					if (!started || !watch) {
+						return false;
 					}
 				}
 
@@ -383,7 +426,6 @@ ACTOR Future<Void> addMutation(Reference<IBackupFile> logFile, VersionedMessage
 
 // Saves messages in the range of [0, numMsg) to a file and then remove these
 // messages. The file format is a sequence of (Version, sub#, msgSize, message).
-// Note only ready backups are saved.
 ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int numMsg) {
 	state int blockSize = SERVER_KNOBS->BACKUP_FILE_BLOCK_BYTES;
 	state std::vector<Future<Reference<IBackupFile>>> logFileFutures;
@@ -394,36 +436,24 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 	state std::vector<Standalone<StringRef>> mutations;
 	state int idx;
 
-	for (auto it = self->backups.begin(); it != self->backups.end();) {
-		if (!it->second.isRunning()) {
-			if (it->second.stopped) {
-				TraceEvent("BackupWorkerRemoveStoppedContainer", self->myId).detail("BackupId", it->first);
-				it = self->backups.erase(it);
-			} else {
-				it++;
-			}
-			continue;
-		}
-		if (!it->second.container.get().present()) {
-			TraceEvent("BackupWorkerNoContainer", self->myId).detail("BackupId", it->first);
-			it = self->backups.erase(it);
-			continue;
-		}
+	// Make sure all backups are ready, otherwise mutations will be lost.
+	while (!self->isAllInfoReady()) {
+		wait(self->waitAllInfoReady());
+	}
+
+	for (auto it = self->backups.begin(); it != self->backups.end(); it++) {
+		ASSERT(it->second.container.get().present());
 		const int index = logFileFutures.size();
 		activeUids.insert(it->first);
 		self->insertRanges(keyRangeMap, it->second.ranges.get(), index);
 		if (it->second.lastSavedVersion == invalidVersion) {
-			it->second.lastSavedVersion = self->messages[0].getVersion();
+			it->second.lastSavedVersion = self->messages.empty() ? self->savedVersion : self->messages[0].getVersion();
 		}
 		logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile(
 		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));
-		it++;
-	}
-	if (activeUids.empty()) {
-		// stop early if there is no active backups
-		TraceEvent("BackupWorkerSkip", self->myId).detail("Count", numMsg);
-		return Void();
 	}
+	ASSERT(!activeUids.empty());
+
 	keyRangeMap.coalesce(allKeys);
 	wait(waitForAll(logFileFutures));
 
@@ -504,26 +534,28 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 
 		const Version maxPopVersion =
 		    self->endVersion.present() ? self->endVersion.get() : self->minKnownCommittedVersion;
+		state int numMsg = 0;
 		if (self->messages.empty()) {
 			// Even though messages is empty, we still want to advance popVersion.
 			popVersion = std::max(popVersion, maxPopVersion);
 		} else {
-			state int numMsg = 0;
 			for (const auto& message : self->messages) {
 				if (message.getVersion() > maxPopVersion) break;
 				popVersion = std::max(popVersion, message.getVersion());
 				numMsg++;
 			}
-			if (numMsg > 0) {
-				wait(saveMutationsToFile(self, popVersion, numMsg));
-				self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);
-			}
 		}
 		if (self->pullFinished.get() && self->messages.empty()) {
 			// Advance popVersion to the endVersion to avoid gap between last
 			// message version and the endVersion.
 			popVersion = self->endVersion.get();
 		}
+		if (numMsg > 0 || self->endVersion.present()) {
+			// save an empty file for old epochs so that log file versions are continuous
+			TraceEvent("BackupWorkerSave", self->myId).detail("PopVersion", popVersion).detail("MsgQ", self->messages.size());
+			wait(saveMutationsToFile(self, popVersion, numMsg));
+			self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);
+		}
 
 		if (popVersion > self->savedVersion) {
 			wait(saveProgress(self, popVersion));
@@ -572,6 +604,7 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		}
 
 		tagAt = r->version().version;
+		self->pulledVersion = tagAt;
 		TraceEvent("BackupWorkerGot", self->myId).suppressFor(1.0).detail("V", tagAt);
 		if (self->endVersion.present() && tagAt > self->endVersion.get()) {
 			self->eraseMessagesAfterEndVersion();
@@ -588,13 +621,17 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 }
 
 ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
-	state Future<Void> started, pullFinished;
+	state Future<Void> pullFinished = Void();
+	state Future<bool> started;
 	state Future<GetReadVersionReply> replyFuture = Never();
 
 	loop {
-		started = monitorBackupStartedKeyChanges(self, true);
+		started = monitorBackupStartedKeyChanges(self, true, true);
 		loop choose {
-			when(wait(started)) { break; }
+			when(bool present = wait(started)) {
+				replyFuture = Never();
+				break;
+			}
 			when(wait(self->cx->onMasterProxiesChanged() ||
 			          delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
 				GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
@@ -611,10 +648,28 @@ ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
 			}
 		}
 
-		Future<Void> stopped = monitorBackupStartedKeyChanges(self, false);
+		Future<bool> stopped = monitorBackupStartedKeyChanges(self, false, true);
 		pullFinished = pullAsyncData(self);
-		wait(stopped || pullFinished);
+		wait(success(stopped) || pullFinished);
 		if (pullFinished.isReady()) return Void(); // backup is done for some old epoch.
+
+		// Even though the snapshot is done, mutation logs may not be written
+		// out yet. We need to make usre mutations up to this point is written.
+		state Version currentVersion;
+		loop {
+			GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
+			                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
+			choose {
+				when(wait(self->cx->onMasterProxiesChanged())) {}
+				when(GetReadVersionReply reply = wait(loadBalance(self->cx->getMasterProxies(false),
+				                                                  &MasterProxyInterface::getConsistentReadVersion,
+				                                                  request, self->cx->taskID))) {
+					currentVersion = reply.version;
+					break;
+				}
+			}
+		}
+		wait(self->pulledVersion.whenAtLeast(currentVersion));
 		pullFinished = Future<Void>(); // cancels pullAsyncData()
 		TraceEvent("BackupWorkerPaused", self->myId);
 	}
@@ -653,13 +708,19 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 	    .detail("LogEpoch", req.recruitedEpoch)
 	    .detail("BackupEpoch", req.backupEpoch);
 	try {
-		addActor.send(monitorBackupKeyOrPullData(&self));
 		addActor.send(checkRemoved(db, req.recruitedEpoch, &self));
 		addActor.send(waitFailureServer(interf.waitFailure.getFuture()));
 		if (req.recruitedEpoch == req.backupEpoch && req.routerTag.id == 0) {
 			addActor.send(monitorAllWorkerStarted(&self));
 		}
 
+		// Check if backup key is present to avoid race between this check and
+		// noop pop as well as upload data: pop or skip upload before knowing
+		// there are backup keys.
+		bool present = wait(monitorBackupStartedKeyChanges(&self, true, false));
+		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present);
+
+		addActor.send(monitorBackupKeyOrPullData(&self));
 		state Future<Void> done = uploadData(&self);
 
 		loop choose {

From 8f57c46bc9650ce0312279c577ae47e321d2b41d Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 28 Feb 2020 14:38:11 -0800
Subject: [PATCH 019/176] Fix: backup worker savedVersion init to begin version

Choosing invalidVersion is wrong, as the worker starts at beginVersion.
---
 fdbserver/BackupWorker.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index bd33822dd5..1542c08bcc 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -120,7 +120,7 @@ struct BackupData {
 	explicit BackupData(UID id, Reference<AsyncVar<ServerDBInfo>> db, const InitializeBackupRequest& req)
 	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
 	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
-	    minKnownCommittedVersion(invalidVersion), savedVersion(invalidVersion), cc("BackupWorker", myId.toString()),
+	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion), cc("BackupWorker", myId.toString()),
 	    pulledVersion(0) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
 		pullFinished.set(false);
@@ -447,7 +447,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		activeUids.insert(it->first);
 		self->insertRanges(keyRangeMap, it->second.ranges.get(), index);
 		if (it->second.lastSavedVersion == invalidVersion) {
-			it->second.lastSavedVersion = self->messages.empty() ? self->savedVersion : self->messages[0].getVersion();
+			it->second.lastSavedVersion = self->savedVersion;
 		}
 		logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile(
 		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));

From ce3f0c6dfca3557e5ac8e98829e5f89d87c208e6 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 28 Feb 2020 16:05:34 -0800
Subject: [PATCH 020/176] Fix pulledVersion of backup worker

Not sure why, the cursor's version can be smaller than before.
---
 fdbserver/BackupWorker.actor.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 1542c08bcc..87bfd8523c 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -604,7 +604,9 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		}
 
 		tagAt = r->version().version;
-		self->pulledVersion = tagAt;
+		if (tagAt > self->pulledVersion.get()) {
+			self->pulledVersion.set(tagAt);
+		}
 		TraceEvent("BackupWorkerGot", self->myId).suppressFor(1.0).detail("V", tagAt);
 		if (self->endVersion.present() && tagAt > self->endVersion.get()) {
 			self->eraseMessagesAfterEndVersion();

From de9362748ecc1100dad19b527d7751375861de49 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 28 Feb 2020 17:14:18 -0800
Subject: [PATCH 021/176] Fix: backup worker ignores deleted container

---
 fdbserver/BackupWorker.actor.cpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 87bfd8523c..b32631f9b3 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -88,10 +88,6 @@ struct BackupData {
 			return stopped || (container.isReady() && ranges.isReady());
 		}
 
-		bool isRunning() const {
-			return container.isReady() && ranges.isReady() && !stopped;
-		}
-
 		Future<Void> waitReady() {
 			if (stopped) return Void();
 			return _waitReady(this);
@@ -229,6 +225,7 @@ struct BackupData {
 				it = self->backups.erase(it);
 				continue;
 			}
+
 			all.push_back(it->second.waitReady());
 			it++;
 		}
@@ -441,8 +438,12 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		wait(self->waitAllInfoReady());
 	}
 
-	for (auto it = self->backups.begin(); it != self->backups.end(); it++) {
-		ASSERT(it->second.container.get().present());
+	for (auto it = self->backups.begin(); it != self->backups.end();) {
+		if (!it->second.container.get().present()) {
+			TraceEvent("BackupWorkerNoContainer", self->myId).detail("BackupId", it->first);
+			it = self->backups.erase(it);
+			continue;
+		}
 		const int index = logFileFutures.size();
 		activeUids.insert(it->first);
 		self->insertRanges(keyRangeMap, it->second.ranges.get(), index);
@@ -451,6 +452,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		}
 		logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile(
 		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));
+		it++;
 	}
 	ASSERT(!activeUids.empty());
 

From 96eab2f3ecf1b27f5c89bfc688922623f1938b15 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 2 Mar 2020 13:29:42 -0800
Subject: [PATCH 022/176] Consider previously pulled version for pulling
 version

Saving files only happens if we are not pulling, i.e., not in NOOP mode.
---
 fdbserver/BackupWorker.actor.cpp | 56 +++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index b32631f9b3..b4526d012c 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -77,8 +77,8 @@ struct BackupData {
 	AsyncVar<Reference<ILogSystem>> logSystem;
 	Database cx;
 	std::vector<VersionedMessage> messages;
-	AsyncVar<bool> pullFinished;
 	NotifiedVersion pulledVersion;
+	bool pulling = false;
 
 	struct PerBackupInfo {
 		PerBackupInfo() = default;
@@ -119,7 +119,6 @@ struct BackupData {
 	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion), cc("BackupWorker", myId.toString()),
 	    pulledVersion(0) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
-		pullFinished.set(false);
 
 		specialCounter(cc, "SavedVersion", [this]() { return this->savedVersion; });
 		specialCounter(cc, "MinKnownCommittedVersion", [this]() { return this->minKnownCommittedVersion; });
@@ -128,6 +127,18 @@ struct BackupData {
 		                       "BackupWorkerMetrics");
 	}
 
+	bool pullFinished() const {
+		return endVersion.present() && pulledVersion.get() > endVersion.get();
+	}
+
+	bool allMessageSaved() const {
+		return endVersion.present() && savedVersion >= endVersion.get();
+	}
+
+	Version maxPopVersion() const {
+		return endVersion.present() ? endVersion.get() : minKnownCommittedVersion;
+	}
+
 	// Inserts a backup's single range into rangeMap.
 	template <class T>
 	void insertRange(KeyRangeMap<std::set<T>>& keyRangeMap, KeyRangeRef range, T value) {
@@ -525,7 +536,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 	state Version popVersion = invalidVersion;
 
 	loop {
-		if (self->endVersion.present() && self->savedVersion >= self->endVersion.get()) {
+		if (self->allMessageSaved()) {
 			self->messages.clear();
 			return Void();
 		}
@@ -534,27 +545,24 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 		// lag TLog might have. Changing to 20s may fail consistency check.
 		state Future<Void> uploadDelay = delay(10);
 
-		const Version maxPopVersion =
-		    self->endVersion.present() ? self->endVersion.get() : self->minKnownCommittedVersion;
 		state int numMsg = 0;
+		Version lastPopVersion = popVersion;
 		if (self->messages.empty()) {
 			// Even though messages is empty, we still want to advance popVersion.
-			popVersion = std::max(popVersion, maxPopVersion);
+			if (!self->endVersion.present()) {
+				popVersion = std::max(popVersion, self->minKnownCommittedVersion);
+			} else if (self->pullFinished()) {
+				popVersion = self->endVersion.get();
+			}
 		} else {
 			for (const auto& message : self->messages) {
-				if (message.getVersion() > maxPopVersion) break;
+				if (message.getVersion() > self->maxPopVersion()) break;
 				popVersion = std::max(popVersion, message.getVersion());
 				numMsg++;
 			}
 		}
-		if (self->pullFinished.get() && self->messages.empty()) {
-			// Advance popVersion to the endVersion to avoid gap between last
-			// message version and the endVersion.
-			popVersion = self->endVersion.get();
-		}
-		if (numMsg > 0 || self->endVersion.present()) {
+		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling)) {
 			// save an empty file for old epochs so that log file versions are continuous
-			TraceEvent("BackupWorkerSave", self->myId).detail("PopVersion", popVersion).detail("MsgQ", self->messages.size());
 			wait(saveMutationsToFile(self, popVersion, numMsg));
 			self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);
 		}
@@ -569,8 +577,8 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			self->pop();
 		}
 
-		if (!self->pullFinished.get()) {
-			wait(uploadDelay || self->pullFinished.onChange());
+		if (!self->pullFinished()) {
+			wait(uploadDelay);
 		}
 	}
 }
@@ -579,7 +587,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 ACTOR Future<Void> pullAsyncData(BackupData* self) {
 	state Future<Void> logSystemChange = Void();
 	state Reference<ILogSystem::IPeekCursor> r;
-	state Version tagAt = std::max(self->startVersion, self->savedVersion);
+	state Version tagAt = std::max(self->pulledVersion.get(), std::max(self->startVersion, self->savedVersion));
 
 	TraceEvent("BackupWorkerPull", self->myId);
 	loop {
@@ -606,18 +614,15 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		}
 
 		tagAt = r->version().version;
-		if (tagAt > self->pulledVersion.get()) {
-			self->pulledVersion.set(tagAt);
-		}
+		self->pulledVersion.set(tagAt);
 		TraceEvent("BackupWorkerGot", self->myId).suppressFor(1.0).detail("V", tagAt);
-		if (self->endVersion.present() && tagAt > self->endVersion.get()) {
+		if (self->pullFinished()) {
 			self->eraseMessagesAfterEndVersion();
 			TraceEvent("BackupWorkerFinishPull", self->myId)
 			    .detail("Tag", self->tag.toString())
 			    .detail("VersionGot", tagAt)
 			    .detail("EndVersion", self->endVersion.get())
 			    .detail("MsgQ", self->messages.size());
-			self->pullFinished.set(true);
 			return Void();
 		}
 		wait(yield());
@@ -654,8 +659,12 @@ ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
 
 		Future<bool> stopped = monitorBackupStartedKeyChanges(self, false, true);
 		pullFinished = pullAsyncData(self);
+		self->pulling = true;
 		wait(success(stopped) || pullFinished);
-		if (pullFinished.isReady()) return Void(); // backup is done for some old epoch.
+		if (pullFinished.isReady()) {
+			self->pulling = false;
+			return Void(); // backup is done for some old epoch.
+		}
 
 		// Even though the snapshot is done, mutation logs may not be written
 		// out yet. We need to make usre mutations up to this point is written.
@@ -675,6 +684,7 @@ ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
 		}
 		wait(self->pulledVersion.whenAtLeast(currentVersion));
 		pullFinished = Future<Void>(); // cancels pullAsyncData()
+		self->pulling = false;
 		TraceEvent("BackupWorkerPaused", self->myId);
 	}
 }

From 70487cee1bc67dd9a5a69c1bd664f3394a7131f4 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 3 Mar 2020 16:25:21 -0800
Subject: [PATCH 023/176] Handle partial recovery in BackupProgress

A partial recovery can result in empty epoch that copies previous epoch's
version range. In this case, getOldEpochTagsVersionsInfo() will not return
previous epoch's information. To correctly compute the start version for a
backup worker, we need to check previous epoch's saved version. If they are
larger than this epoch's begin version, use previously saved version as the
start version.
---
 fdbserver/BackupProgress.actor.cpp | 46 +++++++++++++++++++++++-------
 fdbserver/BackupProgress.actor.h   | 12 ++++++++
 2 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 5492db7aa8..cc7cadaaa0 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -37,6 +37,20 @@ void BackupProgress::addBackupStatus(const WorkerBackupStatus& status) {
 	}
 }
 
+void BackupProgress::updateTagVersions(std::map<Tag, Version>* tagVersions, std::set<Tag>* tags,
+                                       const std::map<Tag, Version>& progress, Version endVersion, LogEpoch epoch) {
+	for (const auto& [tag, savedVersion] : progress) {
+		tags->erase(tag);
+		if (savedVersion < endVersion - 1) {
+			tagVersions->insert({ tag, savedVersion + 1 });
+			TraceEvent("BW", dbgid)
+			    .detail("OldEpoch", epoch)
+			    .detail("Tag", tag.toString())
+			    .detail("BeginVersion", savedVersion + 1)
+			    .detail("EndVersion", endVersion);
+		}
+	}
+}
 std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgress::getUnfinishedBackup() {
 	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit;
 
@@ -45,20 +59,30 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 	for (const auto& [epoch, info] : epochInfos) {
 		std::set<Tag> tags = enumerateLogRouterTags(info.logRouterTags);
 		std::map<Tag, Version> tagVersions;
-		auto progressIt = progress.find(epoch);
-		if (progressIt != progress.end()) {
-			for (const auto& [tag, savedVersion] : progressIt->second) {
-				tags.erase(tag);
-				if (savedVersion < info.epochEnd - 1) {
-					tagVersions.insert({ tag, savedVersion + 1 });
-					TraceEvent("BW", dbgid)
-					    .detail("OldEpoch", epoch)
-					    .detail("Tag", tag.toString())
-					    .detail("BeginVersion", savedVersion + 1)
-					    .detail("EndVersion", info.epochEnd);
+		auto progressIt = progress.lower_bound(epoch);
+		if (progressIt != progress.end() && progressIt->first == epoch) {
+			updateTagVersions(&tagVersions, &tags, progressIt->second, info.epochEnd, epoch);
+		} else {
+			auto rit = findPreviousProgress(epoch);
+			if (rit != progress.rend()) {
+				// A partial recovery can result in empty epoch that copies previous
+				// epoch's version range. In this case, we should check previous
+				// epoch's savedVersion.
+				int savedMore = 0;
+				for (auto [tag, version] : rit->second) {
+					if (version > info.epochBegin) {
+						savedMore++;
+					}
+				}
+				if (savedMore > 1) {
+					ASSERT(savedMore == rit->second.size()); // all tags should saved more
+					ASSERT(savedMore == info.logRouterTags); // Smae number as logRouterTags
+
+					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
 			}
 		}
+
 		for (const Tag tag : tags) { // tags without progress data
 			tagVersions.insert({ tag, info.epochBegin });
 			TraceEvent("BW", dbgid)
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index 90e93fc95e..be7ed26e89 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -78,6 +78,18 @@ private:
 		return tags;
 	}
 
+	// For each tag in progress, the saved version is smaller than endVersion - 1,
+	// add {tag, savedVersion+1} to tagVersions and remove the tag from "tags".
+	void updateTagVersions(std::map<Tag, Version>* tagVersions, std::set<Tag>* tags,
+	                       const std::map<Tag, Version>& progress, Version endVersion, LogEpoch epoch);
+
+	std::map<LogEpoch, std::map<Tag, Version>>::reverse_iterator findPreviousProgress(LogEpoch epoch) {
+		for (auto it = progress.rbegin(); it != progress.rend(); ++it) {
+			if (it->first < epoch) return it;
+		}
+		return progress.rend();
+	}
+
 	const UID dbgid;
 
 	// Note this MUST be iterated in ascending order.

From a0fb8ad5fc2802f8b03d7d2c4eaf7ed28b61a226 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 3 Mar 2020 21:04:50 -0800
Subject: [PATCH 024/176] Fix version gap in old epoch's backup

When pull finished and message queue is empty, we should use end version as the
popVersion for backup files. Otherwise, there might be a version gap between
last message and end version.
---
 fdbserver/BackupWorker.actor.cpp | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index b4526d012c..08f4c8f8fd 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -551,8 +551,6 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			// Even though messages is empty, we still want to advance popVersion.
 			if (!self->endVersion.present()) {
 				popVersion = std::max(popVersion, self->minKnownCommittedVersion);
-			} else if (self->pullFinished()) {
-				popVersion = self->endVersion.get();
 			}
 		} else {
 			for (const auto& message : self->messages) {
@@ -561,7 +559,13 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 				numMsg++;
 			}
 		}
-		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling)) {
+		if (self->messages.empty() && self->pullFinished()) {
+			popVersion = self->endVersion.get();
+		}
+		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling) || self->pullFinished()) {
+			TraceEvent("BackupWorkerSave", self->myId)
+			    .detail("Version", popVersion)
+			    .detail("MsgQ", self->messages.size());
 			// save an empty file for old epochs so that log file versions are continuous
 			wait(saveMutationsToFile(self, popVersion, numMsg));
 			self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);

From a015277e4985238f93ee72228ec8576f3b2e5d5d Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 3 Mar 2020 21:15:36 -0800
Subject: [PATCH 025/176] Fix compiling error of reverse iterators

MacOS and Windows compiler doesn't like the use of "!=" operator of
std::map::reverse_iterator.
---
 fdbserver/BackupProgress.actor.cpp | 9 ++++-----
 fdbserver/BackupProgress.actor.h   | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index cc7cadaaa0..a9fd2fb365 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -64,19 +64,18 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 			updateTagVersions(&tagVersions, &tags, progressIt->second, info.epochEnd, epoch);
 		} else {
 			auto rit = findPreviousProgress(epoch);
-			if (rit != progress.rend()) {
+			if (!(rit == progress.rend())) {
 				// A partial recovery can result in empty epoch that copies previous
 				// epoch's version range. In this case, we should check previous
 				// epoch's savedVersion.
 				int savedMore = 0;
 				for (auto [tag, version] : rit->second) {
-					if (version > info.epochBegin) {
+					if (version >= info.epochBegin) {
 						savedMore++;
 					}
 				}
-				if (savedMore > 1) {
-					ASSERT(savedMore == rit->second.size()); // all tags should saved more
-					ASSERT(savedMore == info.logRouterTags); // Smae number as logRouterTags
+				if (savedMore > 0) {
+					ASSERT(info.logRouterTags == rit->second.size()); // Same number as logRouterTags
 
 					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index be7ed26e89..f7eacbe180 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -84,7 +84,7 @@ private:
 	                       const std::map<Tag, Version>& progress, Version endVersion, LogEpoch epoch);
 
 	std::map<LogEpoch, std::map<Tag, Version>>::reverse_iterator findPreviousProgress(LogEpoch epoch) {
-		for (auto it = progress.rbegin(); it != progress.rend(); ++it) {
+		for (auto it = progress.rbegin(); !(it == progress.rend()); ++it) {
 			if (it->first < epoch) return it;
 		}
 		return progress.rend();

From cc33a1e35e350372239b2aa4a5e3f77a676d466e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 4 Mar 2020 10:52:51 -0800
Subject: [PATCH 026/176] Filter partitioned logs with subset relationship

If a log file's progress is not saved, a new log file will be generated
with the same begin version. Then we can have a file that contains a subset
of contents in another log file. During restore, we should filter out files
that their contents are subset of other files.
---
 fdbbackup/FileConverter.actor.cpp   |  4 ++--
 fdbclient/BackupContainer.actor.cpp | 11 +++++++----
 fdbclient/BackupContainer.h         |  7 ++++---
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp
index f0bffa73e1..67cd4738b6 100644
--- a/fdbbackup/FileConverter.actor.cpp
+++ b/fdbbackup/FileConverter.actor.cpp
@@ -81,10 +81,10 @@ std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, Vers
 	std::vector<LogFile> sorted;
 	int i = 0;
 	for (int j = 1; j < filtered.size(); j++) {
-		if (!filtered[i].sameContent(filtered[j])) {
+		if (!filtered[i].isSubset(filtered[j])) {
 			sorted.push_back(filtered[i]);
-			i = j;
 		}
+		i = j;
 	}
 	if (i < filtered.size()) {
 		sorted.push_back(filtered[i]);
diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 097feaf2af..1f1e0ea281 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1155,16 +1155,19 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 		return true;
 	}
 
-	// Returns log files that are not duplicated.
+	// Returns log files that are not duplicated, or subset of another log.
+	// If a log file's progress is not saved, a new log file will be generated
+	// with the same begin version. So we can have a file that contains a subset
+	// of contents in another log file.
 	// PRE-CONDITION: logs are already sorted.
 	static std::vector<LogFile> filterDuplicates(const std::vector<LogFile>& logs) {
 		std::vector<LogFile> filtered;
 		int i = 0;
 		for (int j = 1; j < logs.size(); j++) {
-			if (!logs[i].sameContent(logs[j])) {
+			if (!logs[i].isSubset(logs[j])) {
 				filtered.push_back(logs[i]);
-				i = j;
 			}
+			i = j;
 		}
 		if (i < logs.size()) filtered.push_back(logs[i]);
 		return filtered;
@@ -1180,7 +1183,7 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 			Version end = getPartitionedLogsContinuousEndVersion(logs, file.beginVersion);
 std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
 			if (end > file.beginVersion) {
-				desc->minLogBegin = file.beginVersion;
+				// desc->minLogBegin = file.beginVersion;
 				// contiguousLogEnd is not inclusive, so +1 here.
 				desc->contiguousLogEnd.get() = end + 1;
 				return;
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 3eba09f06f..4bf144c07e 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -82,9 +82,10 @@ struct LogFile {
 		return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
 	}
 
-	// Returns if two log files have the same content by comparing version range and tag ID.
-	bool sameContent(const LogFile& rhs) const {
-		return beginVersion == rhs.beginVersion && endVersion == rhs.endVersion && tagId == rhs.tagId;
+	// Returns if this log file contains a subset of content of the given file
+	// by comparing version range and tag ID.
+	bool isSubset(const LogFile& rhs) const {
+		return beginVersion == rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
 	}
 
 	std::string toString() const {

From cade65768257395a001a47345c41dc1fbd2d26e0 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 4 Mar 2020 12:32:06 -0800
Subject: [PATCH 027/176] Give a chance for backup worker to finish writing
 files

If a backup worker is cancelled, wait until it finishes writing files so that
we don't need to create these files in the next epoch.
---
 fdbserver/BackupWorker.actor.cpp | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 08f4c8f8fd..75d06d5b7e 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -79,6 +79,7 @@ struct BackupData {
 	std::vector<VersionedMessage> messages;
 	NotifiedVersion pulledVersion;
 	bool pulling = false;
+	bool stopped = false;
 
 	struct PerBackupInfo {
 		PerBackupInfo() = default;
@@ -132,7 +133,7 @@ struct BackupData {
 	}
 
 	bool allMessageSaved() const {
-		return endVersion.present() && savedVersion >= endVersion.get();
+		return (endVersion.present() && savedVersion >= endVersion.get()) || stopped;
 	}
 
 	Version maxPopVersion() const {
@@ -559,7 +560,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 				numMsg++;
 			}
 		}
-		if (self->messages.empty() && self->pullFinished()) {
+		if (self->pullFinished()) {
 			popVersion = self->endVersion.get();
 		}
 		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling) || self->pullFinished()) {
@@ -717,6 +718,8 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 	state PromiseStream<Future<Void>> addActor;
 	state Future<Void> error = actorCollection(addActor.getFuture());
 	state Future<Void> dbInfoChange = Void();
+	state Future<Void> pull;
+	state Future<Void> done;
 
 	TraceEvent("BackupWorkerStart", self.myId)
 	    .detail("Tag", req.routerTag.toString())
@@ -738,8 +741,8 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		bool present = wait(monitorBackupStartedKeyChanges(&self, true, false));
 		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present);
 
-		addActor.send(monitorBackupKeyOrPullData(&self));
-		state Future<Void> done = uploadData(&self);
+		pull = monitorBackupKeyOrPullData(&self);
+		done = uploadData(&self);
 
 		loop choose {
 			when(wait(dbInfoChange)) {
@@ -768,9 +771,15 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 			when(wait(error)) {}
 		}
 	} catch (Error& e) {
-		TraceEvent("BackupWorkerTerminated", self.myId).error(e, true);
-		if (e.code() != error_code_actor_cancelled && e.code() != error_code_worker_removed) {
-			throw;
+		state Error err = e;
+		if (e.code() == error_code_worker_removed) {
+			pull = Void(); // cancels pulling
+			self.stopped = true;
+			wait(done);
+		}
+		TraceEvent("BackupWorkerTerminated", self.myId).error(err, true);
+		if (err.code() != error_code_actor_cancelled && err.code() != error_code_worker_removed) {
+			throw err;
 		}
 	}
 	return Void();

From 2c2d679a5d448456a1271a1dd03c66296244ac35 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 4 Mar 2020 14:07:42 -0800
Subject: [PATCH 028/176] Partitioned logs should be filtered after sorting by
 tag IDs

The default sorting by begin and end version doesn't work with duplicates
removal, as tags are also compared.
---
 fdbclient/BackupContainer.actor.cpp | 33 ++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 1f1e0ea281..e1a51ccf63 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1194,23 +1194,27 @@ std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
 	// Returns the end version such that [begin, end] is continuous.
 	// "logs" should be already sorted.
 	static Version getPartitionedLogsContinuousEndVersion(const std::vector<LogFile>& logs, Version begin) {
-		auto files = filterDuplicates(logs);
 std::cout << "getPartitionedLogsContinuousEndVersion begin:" << begin << "\n";
-for (auto file : files) std::cout << "    " << file.toString() << "\n";
+for (auto file : logs) std::cout << "    " << file.toString() << "\n";
 		Version end = 0;
 
 		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
-		for (int i = 0; i < files.size(); i++) {
-			ASSERT(files[i].tagId >= 0 && files[i].tagId < files[i].totalTags);
-			auto& indices = tagIndices[files[i].tagId];
-			indices.push_back(i);
-			end = std::max(end, files[i].endVersion - 1);
+		for (int i = 0; i < logs.size(); i++) {
+			ASSERT(logs[i].tagId >= 0 && logs[i].tagId < logs[i].totalTags);
+			auto& indices = tagIndices[logs[i].tagId];
+			// filter out if indices.back() is subset of files[i]
+			if (!indices.empty() && logs[indices.back()].isSubset(logs[i])) {
+				indices.back() = i;
+			} else {
+				indices.push_back(i);
+			}
+			end = std::max(end, logs[i].endVersion - 1);
 		}
 std::cout << "Init end: " << end << ", begin " << begin << "\n";
 
 		// check tag 0 is continuous in [begin, end] and create a map of ranges to tags
 		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
-		isContinuous(files, tagIndices[0], begin, end, &tags);
+		isContinuous(logs, tagIndices[0], begin, end, &tags);
 		if (tags.empty() || end <= begin) return 0;
 		end = std::min(end, tags.rbegin()->first.second);
 std::cout << "  Tag 0 end: " << end << "\n";
@@ -1222,7 +1226,7 @@ for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " <
 			Version tagEnd = end; // This range's minimum continous tag version
 			for (int i = 1; i < count; i++) {
 				std::map<std::pair<Version, Version>, int> rangeTags;
-				isContinuous(files, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
+				isContinuous(logs, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
 				tagEnd = rangeTags.empty() ? 0 : std::min(tagEnd, rangeTags.rbegin()->first.second);
 std::cout << "  Tag " << i << " end: " << tagEnd << ", return end = "<< lastEnd << "\n";
 				if (tagEnd == 0) return lastEnd;
@@ -1264,10 +1268,12 @@ std::cout << "Return end = " << end << "\n\n";
 			// FIXME: check if there are tagged logs. for each tag, there is no version gap.
 			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion, partitioned));
 
-			// List logs in version order so log continuity can be analyzed
-			std::sort(logs.begin(), logs.end());
-
 			if (partitioned) {
+				// sort by tag ID so that filterDuplicates works.
+				std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) {
+					return a.tagId < b.tagId || a.beginVersion < b.beginVersion || a.endVersion < b.endVersion;
+				});
+
 				// Remove duplicated log files that can happen for old epochs.
 				std::vector<LogFile> filtered = filterDuplicates(logs);
 
@@ -1278,6 +1284,9 @@ std::cout << "Return end = " << end << "\n\n";
 				return Optional<RestorableFileSet>();
 			}
 
+			// List logs in version order so log continuity can be analyzed
+			std::sort(logs.begin(), logs.end());
+
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
 			if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {
 				Version end = logs.begin()->endVersion;

From b8c362cf44f8c73dadb810bc0385f95825d1ab49 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 4 Mar 2020 16:27:24 -0800
Subject: [PATCH 029/176] Some correctness fixes

---
 fdbclient/BackupContainer.actor.cpp | 6 +++++-
 fdbserver/BackupProgress.actor.cpp  | 3 ++-
 fdbserver/BackupWorker.actor.cpp    | 6 ++++--
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index e1a51ccf63..69e0aa3378 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1271,13 +1271,17 @@ std::cout << "Return end = " << end << "\n\n";
 			if (partitioned) {
 				// sort by tag ID so that filterDuplicates works.
 				std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) {
-					return a.tagId < b.tagId || a.beginVersion < b.beginVersion || a.endVersion < b.endVersion;
+					return a.tagId == b.tagId ? (a.beginVersion == b.beginVersion ? a.endVersion < b.endVersion
+					                                                              : a.beginVersion < b.beginVersion)
+					                          : (a.tagId < b.tagId);
 				});
 
 				// Remove duplicated log files that can happen for old epochs.
 				std::vector<LogFile> filtered = filterDuplicates(logs);
 
 				restorable.logs.swap(filtered);
+				// sort by version order again for continuous analysis
+				std::sort(restorable.logs.begin(), restorable.logs.end());
 				if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) {
 					return Optional<RestorableFileSet>(restorable);
 				}
diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index a9fd2fb365..8a42be686c 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -75,7 +75,8 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 					}
 				}
 				if (savedMore > 0) {
-					ASSERT(info.logRouterTags == rit->second.size()); // Same number as logRouterTags
+					// TODO: check the logRouterTags are the same
+					// ASSERT(info.logRouterTags == rit->second.size());
 
 					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 75d06d5b7e..5ae7979c46 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -451,7 +451,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 	}
 
 	for (auto it = self->backups.begin(); it != self->backups.end();) {
-		if (!it->second.container.get().present()) {
+		if (it->second.stopped || !it->second.container.get().present()) {
 			TraceEvent("BackupWorkerNoContainer", self->myId).detail("BackupId", it->first);
 			it = self->backups.erase(it);
 			continue;
@@ -563,7 +563,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 		if (self->pullFinished()) {
 			popVersion = self->endVersion.get();
 		}
-		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling) || self->pullFinished()) {
+		if (((numMsg > 0 || popVersion > lastPopVersion) && self->pulling) || self->pullFinished()) {
 			TraceEvent("BackupWorkerSave", self->myId)
 			    .detail("Version", popVersion)
 			    .detail("MsgQ", self->messages.size());
@@ -572,6 +572,8 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);
 		}
 
+		// If transition into NOOP mode, should clear messages
+
 		if (popVersion > self->savedVersion) {
 			wait(saveProgress(self, popVersion));
 			TraceEvent("BackupWorkerSavedProgress", self->myId)

From 15437ffb53b8b779de95427b03d6d98601a53b15 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 5 Mar 2020 11:34:37 -0800
Subject: [PATCH 030/176] Add delay for master to recruit backup workers

This delay is to ensure old epoch's backup workers can save their progress in
the database. Otherwise, the new master could attempts to recruit backup
workers for the old epoch on version ranges that have already been popped. As
a result, the logs will lose data.
---
 fdbserver/BackupWorker.actor.cpp | 3 ++-
 fdbserver/Knobs.cpp              | 1 +
 fdbserver/Knobs.h                | 1 +
 fdbserver/masterserver.actor.cpp | 3 +++
 4 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 5ae7979c46..cc482d7984 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -373,6 +373,8 @@ ACTOR Future<Void> saveProgress(BackupData* self, Version backupVersion) {
 
 	loop {
 		try {
+			// It's critical to save progress immediately so that after a master
+			// recovery, the new master can know the progress so far.
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -466,7 +468,6 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));
 		it++;
 	}
-	ASSERT(!activeUids.empty());
 
 	keyRangeMap.coalesce(allKeys);
 	wait(waitForAll(logFileFutures));
diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp
index 40abb0d6e4..6b03d22f7c 100644
--- a/fdbserver/Knobs.cpp
+++ b/fdbserver/Knobs.cpp
@@ -355,6 +355,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs, bool isSimula
 	init( PROVISIONAL_START_DELAY,                               1.0 );
 	init( PROVISIONAL_MAX_DELAY,                                60.0 );
 	init( PROVISIONAL_DELAY_GROWTH,                              1.5 );
+	init( SECONDS_BEFORE_RECRUIT_BACKUP_WORKER,                  4.0 );
 
 	// Resolver
 	init( SAMPLE_OFFSET_PER_KEY,                                 100 );
diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h
index 8b06c27aab..9d3efa24b9 100644
--- a/fdbserver/Knobs.h
+++ b/fdbserver/Knobs.h
@@ -292,6 +292,7 @@ public:
 	double PROVISIONAL_START_DELAY;
 	double PROVISIONAL_DELAY_GROWTH;
 	double PROVISIONAL_MAX_DELAY;
+	double SECONDS_BEFORE_RECRUIT_BACKUP_WORKER;
 
 	// Resolver
 	int64_t KEY_BYTES_PER_SAMPLE;
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 7acf67b72a..943d1afb40 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1241,6 +1241,9 @@ ACTOR Future<Void> configurationMonitor(Reference<MasterData> self, Database cx)
 ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Database cx) {
 	ASSERT(self->backupWorkers.size() > 0);
 
+	// Avoid race between a backup worker's save progress and the reads below.
+	wait(delay(SERVER_KNOBS->SECONDS_BEFORE_RECRUIT_BACKUP_WORKER));
+
 	state LogEpoch epoch = self->cstate.myDBState.recoveryCount;
 	state Reference<BackupProgress> backupProgress(
 	    new BackupProgress(self->dbgid, self->logSystem->getOldEpochTagsVersionsInfo()));

From be1d36bed3f7aa763cc2da7e0b9cfb0df8b97e9c Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 6 Mar 2020 11:58:10 -0800
Subject: [PATCH 031/176] Backup worker updates latest log versions in
 BackupConfig

If backup worker is enabled, the current epoch's worker of tag (-2,0) will be
responsible for monitoring the backup progress of all workers and update the
BackupConfig with the latest saved log version, which is the minimum version
of all tags.

This change has been incorporated in the getLatestRestorableVersion() so that
it is transparent to clients.
---
 fdbclient/BackupAgent.actor.h       | 20 +++++++++--
 fdbclient/FileBackupAgent.actor.cpp |  6 +++-
 fdbserver/BackupWorker.actor.cpp    | 53 ++++++++++++++++++++++++-----
 3 files changed, 66 insertions(+), 13 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 9ef90976d1..896cd32509 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -787,6 +787,16 @@ public:
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
 	}
 
+	// Set to true if backup worker is enabled.
+	KeyBackedProperty<bool> backupWorkerEnabled() {
+		return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	}
+
+	// Latest version for which all prior versions have saved by backup workers.
+	KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
+		return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	}
+
 	// Stop differntial logging if already started or don't start after completing KV ranges
 	KeyBackedProperty<bool> stopWhenDone() {
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
@@ -816,10 +826,14 @@ public:
 		tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 		auto lastLog = latestLogEndVersion().get(tr);
 		auto firstSnapshot = firstSnapshotEndVersion().get(tr);
-		return map(success(lastLog) && success(firstSnapshot), [=](Void) -> Optional<Version> {
+		auto enabled = backupWorkerEnabled().get(tr);
+		auto workerVersion = latestBackupWorkerSavedVersion().get(tr);
+		return map(success(lastLog) && success(firstSnapshot) && success(enabled) && success(workerVersion), [=](Void) -> Optional<Version> {
 			// The latest log greater than the oldest snapshot is the restorable version
-			if(lastLog.get().present() && firstSnapshot.get().present() && lastLog.get().get() > firstSnapshot.get().get()) {
-				return std::max(lastLog.get().get() - 1, firstSnapshot.get().get());
+			Optional<Version> logVersion =
+			    enabled.get().present() && enabled.get().get() ? workerVersion.get() : lastLog.get();
+			if (logVersion.present() && firstSnapshot.get().present() && logVersion.get() > firstSnapshot.get().get()) {
+				return std::max(logVersion.get() - 1, firstSnapshot.get().get());
 			}
 			return {};
 		});
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 8c58cbc162..66f584e085 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -2388,7 +2388,8 @@ namespace fileBackup {
 
 			// Check if backup worker is enabled
 			DatabaseConfiguration dbConfig = wait(getDatabaseConfiguration(cx));
-			if (!dbConfig.backupWorkerEnabled) {
+			state bool backupWorkerEnabled = dbConfig.backupWorkerEnabled;
+			if (!backupWorkerEnabled) {
 				wait(success(changeConfig(cx, "backup_worker_enabled:=1", true)));
 			}
 
@@ -2420,6 +2421,9 @@ namespace fileBackup {
 					}
 
 					tr->set(backupStartedKey, encodeBackupStartedValue(ids));
+					if (backupWorkerEnabled) {
+						config.backupWorkerEnabled().set(tr, true);
+					}
 
 					// The task may be restarted. Set the watch if started key has NOT been set.
 					if (!taskStarted.get().present()) {
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index cc482d7984..fb8226c5fb 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -305,7 +305,10 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 // set the "allWorkerStarted" key of the BackupConfig to true, which in turn
 // unblocks StartFullBackupTaskFunc::_execute. Note only worker with Tag (-2,0)
 // runs this actor so that the key is set by one process.
-ACTOR Future<Void> monitorAllWorkerStarted(BackupData* self) {
+// Additionally, this actor updates the saved version for each BackupConfig in
+// the system space so that the client can know if a backup is restorable --
+// log saved version > snapshot version.
+ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 	loop {
 		wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger());
 		if (self->backups.empty()) {
@@ -319,23 +322,32 @@ ACTOR Future<Void> monitorAllWorkerStarted(BackupData* self) {
 		std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
 
 		state std::vector<UID> ready;
+		state std::map<UID, Version> savedLogVersions;
 		if (tagVersions.size() == self->logSystem.get()->getLogRouterTags()) {
 			// Check every version is larger than backup's startVersion
-			for (auto& uidInfo : self->backups) {
-				if (uidInfo.second.allWorkerStarted) continue;
+			for (auto& [uid, info] : self->backups) {
+				if (info.allWorkerStarted) {
+					// update update progress so far
+					Version v = std::numeric_limits<Version>::max();
+					for (const auto [tag, version] : tagVersions) {
+						v = std::min(v, version);
+					}
+					savedLogVersions.emplace(uid, v);
+					continue;
+				}
 				bool saved = true;
 				for (const std::pair<Tag, Version> tv : tagVersions) {
-					if (tv.second < uidInfo.second.startVersion) {
+					if (tv.second < info.startVersion) {
 						saved = false;
 						break;
 					}
 				}
 				if (saved) {
-					ready.push_back(uidInfo.first);
-					uidInfo.second.allWorkerStarted = true;
+					ready.push_back(uid);
+					info.allWorkerStarted = true;
 				}
 			}
-			if (ready.empty()) continue;
+			if (ready.empty() && savedLogVersions.empty()) continue;
 
 			// Set "allWorkerStarted" key for ready backups
 			loop {
@@ -350,13 +362,36 @@ ACTOR Future<Void> monitorAllWorkerStarted(BackupData* self) {
 						configs.emplace_back(uid);
 						readyValues.push_back(tr->get(configs.back().allWorkerStarted().key));
 					}
-					wait(waitForAll(readyValues));
+
+					state std::vector<Future<Optional<Version>>> prevVersions;
+					state std::vector<BackupConfig> versionConfigs;
+					for (const auto [uid, version] : savedLogVersions) {
+						versionConfigs.emplace_back(uid);
+						prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
+					}
+
+					wait(waitForAll(readyValues) && waitForAll(prevVersions));
+
 					for (int i = 0; i < readyValues.size(); i++) {
 						if (!readyValues[i].get().present()) {
 							configs[i].allWorkerStarted().set(tr, true);
 							TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", ready[i].toString());
 						}
 					}
+
+					for (int i = 0; i < prevVersions.size(); i++) {
+						const Version current = savedLogVersions[versionConfigs[i].getUid()];
+						if (prevVersions[i].get().present()) {
+							const Version prev = prevVersions[i].get().get();
+							ASSERT(prev <= current);
+						}
+						if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
+							TraceEvent("BackupWorkerSetVersion", self->myId)
+							    .detail("BackupID", versionConfigs[i].getUid())
+							    .detail("Version", current);
+							versionConfigs[i].latestBackupWorkerSavedVersion().set(tr, current);
+						}
+					}
 					wait(tr->commit());
 					break;
 				} catch (Error& e) {
@@ -735,7 +770,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		addActor.send(checkRemoved(db, req.recruitedEpoch, &self));
 		addActor.send(waitFailureServer(interf.waitFailure.getFuture()));
 		if (req.recruitedEpoch == req.backupEpoch && req.routerTag.id == 0) {
-			addActor.send(monitorAllWorkerStarted(&self));
+			addActor.send(monitorAllWorkerProgress(&self));
 		}
 
 		// Check if backup key is present to avoid race between this check and

From 524b275a94b3667a9ca3837e7025634c605f432f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 6 Mar 2020 15:40:06 -0800
Subject: [PATCH 032/176] Add a flag to submitBackup for partitioned log

This is to distinguish with old workloads so that they can work in simulation.
---
 fdbclient/BackupAgent.actor.h                 | 29 ++++++++++++++-----
 fdbclient/FileBackupAgent.actor.cpp           | 14 +++++++--
 ...kupAndParallelRestoreCorrectness.actor.cpp |  9 +++---
 3 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 896cd32509..7c84fa0121 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -308,9 +308,16 @@ public:
 
 	/** BACKUP METHODS **/
 
-	Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true);
-	Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true) {
-		return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone); });
+	Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds,
+	                          std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                          bool stopWhenDone = true, bool partitionedLog = false);
+	Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName,
+	                          Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true,
+	                          bool partitionedLog = false) {
+		return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
+			return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone,
+			                    partitionedLog);
+		});
 	}
 
 	Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
@@ -792,6 +799,11 @@ public:
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
 	}
 
+	// Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
+	KeyBackedProperty<bool> partitionedLogEnabled() {
+		return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	}
+
 	// Latest version for which all prior versions have saved by backup workers.
 	KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
@@ -826,12 +838,15 @@ public:
 		tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 		auto lastLog = latestLogEndVersion().get(tr);
 		auto firstSnapshot = firstSnapshotEndVersion().get(tr);
-		auto enabled = backupWorkerEnabled().get(tr);
+		auto workerEnabled = backupWorkerEnabled().get(tr);
+		auto plogEnabled = partitionedLogEnabled().get(tr);
 		auto workerVersion = latestBackupWorkerSavedVersion().get(tr);
-		return map(success(lastLog) && success(firstSnapshot) && success(enabled) && success(workerVersion), [=](Void) -> Optional<Version> {
+		return map(success(lastLog) && success(firstSnapshot) && success(workerEnabled) && success(plogEnabled) && success(workerVersion), [=](Void) -> Optional<Version> {
 			// The latest log greater than the oldest snapshot is the restorable version
-			Optional<Version> logVersion =
-			    enabled.get().present() && enabled.get().get() ? workerVersion.get() : lastLog.get();
+			Optional<Version> logVersion = workerEnabled.get().present() && workerEnabled.get().get() &&
+			                                       plogEnabled.get().present() && plogEnabled.get().get()
+			                                   ? workerVersion.get()
+			                                   : lastLog.get();
 			if (logVersion.present() && firstSnapshot.get().present() && logVersion.get() > firstSnapshot.get().get()) {
 				return std::max(logVersion.get() - 1, firstSnapshot.get().get());
 			}
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 66f584e085..660bb6c526 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3599,7 +3599,10 @@ public:
 		}
 	}
 
-	ACTOR static Future<Void> submitBackup(FileBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone) {
+	ACTOR static Future<Void> submitBackup(FileBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr,
+	                                       Key outContainer, int snapshotIntervalSeconds, std::string tagName,
+	                                       Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone,
+	                                       bool partitionedLog) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
@@ -3700,6 +3703,7 @@ public:
 		config.stopWhenDone().set(tr, stopWhenDone);
 		config.backupRanges().set(tr, normalizedRanges);
 		config.snapshotIntervalSeconds().set(tr, snapshotIntervalSeconds);
+		config.partitionedLogEnabled().set(tr, partitionedLog);
 
 		Key taskKey = wait(fileBackup::StartFullBackupTaskFunc::addTask(tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal()));
 
@@ -4444,8 +4448,12 @@ Future<ERestoreState> FileBackupAgent::waitRestore(Database cx, Key tagName, boo
 	return FileBackupAgentImpl::waitRestore(cx, tagName, verbose);
 };
 
-Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone) {
-	return FileBackupAgentImpl::submitBackup(this, tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone);
+Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer,
+                                           int snapshotIntervalSeconds, std::string tagName,
+                                           Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone,
+                                           bool partitionedLog) {
+	return FileBackupAgentImpl::submitBackup(this, tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges,
+	                                         stopWhenDone, partitionedLog);
 }
 
 Future<Void> FileBackupAgent::discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName){
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 6a1fec7129..1461419c34 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -180,7 +180,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 
 		try {
 			wait(backupAgent->submitBackup(cx, StringRef(backupContainer), deterministicRandom()->randomInt(0, 100),
-			                               tag.toString(), backupRanges, stopDifferentialDelay ? false : true));
+			                               tag.toString(), backupRanges, stopDifferentialDelay ? false : true,
+			                               /*partitionedLog=*/true));
 		} catch (Error& e) {
 			TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) throw;
@@ -395,9 +396,9 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 			if (!self->locked && BUGGIFY) {
 				TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
 				try {
-					extraBackup = backupAgent.submitBackup(cx, LiteralStringRef("file://simfdb/backups/"),
-					                                       deterministicRandom()->randomInt(0, 100),
-					                                       self->backupTag.toString(), self->backupRanges, true);
+					extraBackup = backupAgent.submitBackup(
+					    cx, LiteralStringRef("file://simfdb/backups/"), deterministicRandom()->randomInt(0, 100),
+					    self->backupTag.toString(), self->backupRanges, true, /*partitionedLog=*/true);
 				} catch (Error& e) {
 					TraceEvent("BARW_SubmitBackup2Exception", randomID)
 					    .error(e)

From 89d8f13038dfdcf7e2044afc288dcb44b77f0f58 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 8 Mar 2020 20:50:32 -0700
Subject: [PATCH 033/176] Fix backup worker start version when logset start
 version is lower

The start version of tlog set can be smaller than the last epoch's end version.
In this case, set backup worker's start version as last epoch's end version to
avoid overlapping of version ranges among backup workers.
---
 fdbserver/LogSystem.h                       | 3 ++-
 fdbserver/TagPartitionedLogSystem.actor.cpp | 6 ++++--
 fdbserver/masterserver.actor.cpp            | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h
index dad55b047f..7d8c79faba 100644
--- a/fdbserver/LogSystem.h
+++ b/fdbserver/LogSystem.h
@@ -721,7 +721,8 @@ struct ILogSystem {
 		// Call only on an ILogSystem obtained from recoverAndEndEpoch()
 		// Returns the first unreadable version number of the recovered epoch (i.e. message version numbers < (get_end(), 0) will be readable)
 
-	virtual Version getStartVersion() const = 0; // Returns the start version of current epoch.
+	// Returns the start version of current epoch for backup workers.
+	virtual Version getBackupStartVersion() const = 0;
 
 	struct EpochTagsVersionsInfo {
 		int32_t logRouterTags; // Number of log router tags.
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 31d3b1ff0b..66c4db462a 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -191,6 +191,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 	Optional<Version> recoverAt;
 	Optional<Version> recoveredAt;
 	Version knownCommittedVersion;
+	Version backupStartVersion = invalidVersion; // max(tLogs[0].startVersion, previous epochEnd).
 	LocalityData locality;
 	std::map< std::pair<UID, Tag>, std::pair<Version, Version> > outstandingPops;  // For each currently running popFromLog actor, (log server #, tag)->popped version
 	Optional<PromiseStream<Future<Void>>> addActor;
@@ -1349,9 +1350,9 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 
 	int getLogRouterTags() const override { return logRouterTags; }
 
-	Version getStartVersion() const override {
+	Version getBackupStartVersion() const final {
 		ASSERT(tLogs.size() > 0);
-		return tLogs[0]->startVersion;
+		return backupStartVersion;
 	}
 
 	std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo> getOldEpochTagsVersionsInfo() const override {
@@ -2213,6 +2214,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		logSystem->oldLogData.insert(logSystem->oldLogData.end(), oldLogSystem->oldLogData.begin(), oldLogSystem->oldLogData.end());
 
 		logSystem->tLogs[0]->startVersion = oldLogSystem->knownCommittedVersion + 1;
+		logSystem->backupStartVersion = oldLogSystem->knownCommittedVersion + 1;
 		state int lockNum = 0;
 		while(lockNum < oldLogSystem->lockResults.size()) {
 			if(oldLogSystem->lockResults[lockNum].logSet->locality == primaryLocality) {
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 943d1afb40..cda864a732 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1256,7 +1256,7 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 		idsTags.emplace_back(deterministicRandom()->randomUniqueID(), Tag(tagLocalityLogRouter, i));
 	}
 
-	const Version startVersion = self->logSystem->getStartVersion();
+	const Version startVersion = self->logSystem->getBackupStartVersion();
 	state int i = 0;
 	for (; i < logRouterTags; i++) {
 		const auto& worker = self->backupWorkers[i % self->backupWorkers.size()];

From ce2595821aa26aeb768d8638dd303302b3b94910 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 9 Mar 2020 10:17:38 -0700
Subject: [PATCH 034/176] Refactor to use std::find_if for more concise code

---
 fdbserver/BackupProgress.actor.cpp | 5 ++++-
 fdbserver/BackupProgress.actor.h   | 7 -------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 8a42be686c..c5e263e877 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -51,6 +51,7 @@ void BackupProgress::updateTagVersions(std::map<Tag, Version>* tagVersions, std:
 		}
 	}
 }
+
 std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgress::getUnfinishedBackup() {
 	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit;
 
@@ -63,7 +64,9 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 		if (progressIt != progress.end() && progressIt->first == epoch) {
 			updateTagVersions(&tagVersions, &tags, progressIt->second, info.epochEnd, epoch);
 		} else {
-			auto rit = findPreviousProgress(epoch);
+			auto rit =
+			    std::find_if(progress.rbegin(), progress.rend(),
+			                 [=](const std::pair<LogEpoch, std::map<Tag, Version>>& p) { return p.first < epoch; });
 			if (!(rit == progress.rend())) {
 				// A partial recovery can result in empty epoch that copies previous
 				// epoch's version range. In this case, we should check previous
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index f7eacbe180..3237fae6a0 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -83,13 +83,6 @@ private:
 	void updateTagVersions(std::map<Tag, Version>* tagVersions, std::set<Tag>* tags,
 	                       const std::map<Tag, Version>& progress, Version endVersion, LogEpoch epoch);
 
-	std::map<LogEpoch, std::map<Tag, Version>>::reverse_iterator findPreviousProgress(LogEpoch epoch) {
-		for (auto it = progress.rbegin(); !(it == progress.rend()); ++it) {
-			if (it->first < epoch) return it;
-		}
-		return progress.rend();
-	}
-
 	const UID dbgid;
 
 	// Note this MUST be iterated in ascending order.

From 6a302e66051fc6f132bf67fca88c00826cc799c9 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 9 Mar 2020 15:33:15 -0700
Subject: [PATCH 035/176] Add total number of tags to WorkerBackupStatus

This allows the backup worker to check the number of tags.
---
 fdbclient/FDBTypes.h               |  5 +++--
 fdbserver/BackupProgress.actor.cpp | 18 +++++++++++++-----
 fdbserver/BackupProgress.actor.h   |  4 ++++
 fdbserver/BackupWorker.actor.cpp   |  2 +-
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 6111b23138..6a4d1f73c4 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -978,13 +978,14 @@ struct WorkerBackupStatus {
 	LogEpoch epoch;
 	Version version;
 	Tag tag;
+	int32_t totalTags;
 
 	WorkerBackupStatus() : epoch(0), version(invalidVersion) {}
-	WorkerBackupStatus(LogEpoch e, Version v, Tag t) : epoch(e), version(v), tag(t) {}
+	WorkerBackupStatus(LogEpoch e, Version v, Tag t, int32_t total) : epoch(e), version(v), tag(t), totalTags(total) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, epoch, version, tag);
+		serializer(ar, epoch, version, tag, totalTags);
 	}
 };
 
diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index c5e263e877..1cb061af58 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -35,6 +35,13 @@ void BackupProgress::addBackupStatus(const WorkerBackupStatus& status) {
 	} else {
 		it.insert(lb, { status.tag, status.version });
 	}
+
+	auto tagIt = epochTags.find(status.epoch);
+	if (tagIt == epochTags.end()) {
+		epochTags.insert({ status.epoch, status.totalTags });
+	} else {
+		ASSERT(status.totalTags == tagIt->second);
+	}
 }
 
 void BackupProgress::updateTagVersions(std::map<Tag, Version>* tagVersions, std::set<Tag>* tags,
@@ -78,8 +85,8 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 					}
 				}
 				if (savedMore > 0) {
-					// TODO: check the logRouterTags are the same
-					// ASSERT(info.logRouterTags == rit->second.size());
+					// The logRouterTags are the same
+					ASSERT(info.logRouterTags == epochTags[rit->first]);
 
 					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
@@ -124,7 +131,8 @@ ACTOR Future<Void> getBackupProgress(Database cx, UID dbgid, Reference<BackupPro
 				    .detail("W", workerID)
 				    .detail("Epoch", status.epoch)
 				    .detail("Version", status.version)
-				    .detail("Tag", status.tag.toString());
+				    .detail("Tag", status.tag.toString())
+				    .detail("TotalTags", status.totalTags);
 			}
 			return Void();
 		} catch (Error& e) {
@@ -151,8 +159,8 @@ TEST_CASE("/BackupProgress/Unfinished") {
 		ASSERT(tagVersion.size() == 1 && tagVersion.begin()->first == tag1 && tagVersion.begin()->second == begin1);
 	}
 
-	const int saved1 = 50;
-	WorkerBackupStatus status1(epoch1, saved1, tag1);
+	const int saved1 = 50, totalTags = 1;
+	WorkerBackupStatus status1(epoch1, saved1, tag1, totalTags);
 	progress.addBackupStatus(status1);
 	unfinished = progress.getUnfinishedBackup();
 	ASSERT(unfinished.size() == 1);
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index 3237fae6a0..d17d2c9a15 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -93,6 +93,10 @@ private:
 	// the gap. "progress" MUST be iterated in ascending order.
 	std::map<LogEpoch, std::map<Tag, Version>> progress;
 
+	// LogRouterTags for each epoch obtained by decoding backup progress from
+	// the system keyspace.
+	std::map<LogEpoch, int32_t> epochTags;
+
 	// Value of the "backupStartedKey".
 	Optional<Value> backupStartedValue;
 };
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index fb8226c5fb..2e9a3d5a48 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -414,7 +414,7 @@ ACTOR Future<Void> saveProgress(BackupData* self, Version backupVersion) {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			WorkerBackupStatus status(self->backupEpoch, backupVersion, self->tag);
+			WorkerBackupStatus status(self->backupEpoch, backupVersion, self->tag, self->totalTags);
 			tr.set(key, backupProgressValue(status));
 			tr.addReadConflictRange(singleKeyRange(key));
 			wait(tr.commit());

From 7d1538a9fc2667f4604c8598b66d3bccc5f55ddd Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 9 Mar 2020 15:35:52 -0700
Subject: [PATCH 036/176] Fix wrong end version for restore loader

The restore cannot exceed the target version of the restore request. Otherwise,
the version restored is larger than the requested version.
---
 fdbserver/RestoreMaster.actor.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index a4da897650..e76d8eff05 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -314,8 +314,7 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 		TraceEvent("FastRestoreMasterDispatchVersionBatches")
 		    .detail("BatchIndex", batchIndex)
 		    .detail("BatchSize", versionBatch->size)
-		    .detail("RunningVersionBatches", self->runningVersionBatches.get())
-		    .detail("Start", now());
+		    .detail("RunningVersionBatches", self->runningVersionBatches.get());
 		self->batch[batchIndex] = Reference<MasterBatchData>(new MasterBatchData());
 		self->batchStatus[batchIndex] = Reference<MasterBatchStatus>(new MasterBatchStatus());
 		fBatches.push_back(distributeWorkloadPerVersionBatch(self, batchIndex, cx, request, *versionBatch));
@@ -374,7 +373,8 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<MasterBatchData> batchDat
 		param.asset.len = file.fileSize;
 		param.asset.range = request.range;
 		param.asset.beginVersion = versionBatch.beginVersion;
-		param.asset.endVersion = versionBatch.endVersion;
+		param.asset.endVersion =
+		    isRangeFile ? versionBatch.endVersion : std::min(versionBatch.endVersion, request.targetVersion + 1);
 
 		TraceEvent("FastRestoreMasterPhaseLoadFiles")
 		    .detail("BatchIndex", batchIndex)

From 937d8bcb8e231130d0dc187d89d374d1bc309d01 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 10 Mar 2020 15:45:57 -0700
Subject: [PATCH 037/176] Decode out of order mutations in old mutation logs

In the old mutation logs, a version's mutations are serialized as a buffer.
Then the buffer is split into smaller chunks, e.g., 10000 bytes each. When
writting chunks to the final mutation log file, these chunks can be flushed
out of order. For instance, the (version, chunck_part) can be in the order of
(3, 0), (4, 0), (3, 1). As a result, the decoder must read forward to find all
chunks of data for a version.

Another complication is that the files are organized into blocks, where (3, 1)
can be in a subsequent block. This change checks the value size for each
version, if the size is smaller than the right size, the decoder will look
for the missing chucks in the next block.
---
 fdbbackup/FileConverter.h       |   2 +
 fdbbackup/FileDecoder.actor.cpp | 140 ++++++++++++++++++++++----------
 2 files changed, 100 insertions(+), 42 deletions(-)

diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index fc82e5dfb2..e01566b889 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -31,6 +31,7 @@ namespace file_converter {
 enum {
 	OPT_CONTAINER,
 	OPT_BEGIN_VERSION,
+	OPT_CRASHONERROR,
 	OPT_END_VERSION,
 	OPT_TRACE,
 	OPT_TRACE_DIR,
@@ -44,6 +45,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 	                                        { OPT_CONTAINER, "--container", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "-b", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "--begin", SO_REQ_SEP },
+	                                        { OPT_CRASHONERROR, "--crash", SO_NONE },
 	                                        { OPT_END_VERSION, "-e", SO_REQ_SEP },
 	                                        { OPT_END_VERSION, "--end", SO_REQ_SEP },
 	                                        { OPT_TRACE, "--log", SO_NONE },
diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 02b98e4825..62716a562d 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -30,12 +30,15 @@
 #include "flow/serialize.h"
 #include "flow/actorcompiler.h" // has to be last include
 
+extern bool g_crashOnError;
+
 namespace file_converter {
 
 void printDecodeUsage() {
 	std::cout << "\n"
 	             "  -r, --container   Container URL.\n"
 	             "  -i, --input FILE  Log file to be decoded.\n"
+	             "  --crash           Crash on serious error.\n"
 	             "\n";
 	return;
 }
@@ -89,6 +92,10 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
 			param->container_url = args->OptionArg();
 			break;
 
+		case OPT_CRASHONERROR:
+			g_crashOnError = true;
+			break;
+
 		case OPT_INPUT_FILE:
 			param->file = args->OptionArg();
 			break;
@@ -161,7 +168,13 @@ std::vector<MutationRef> decode_value(const StringRef& value) {
 
 	reader.consume<uint64_t>(); // Consume the includeVersion
 	uint32_t val_length = reader.consume<uint32_t>();
-	ASSERT(val_length == value.size() - sizeof(uint64_t) - sizeof(uint32_t));
+	if (val_length != value.size() - sizeof(uint64_t) - sizeof(uint32_t)) {
+		TraceEvent("ValueError")
+		    .detail("ValueLen", val_length)
+		    .detail("ValueSize", value.size())
+		    .detail("Value", printable(value));
+		ASSERT(false);
+	}
 
 	std::vector<MutationRef> mutations;
 	while (1) {
@@ -217,54 +230,74 @@ struct DecodeProgress {
 
 	// The following are private APIs:
 
+	// Returns true if value contains complete data.
+	bool isValueComplete(StringRef value) {
+		StringRefReader reader(value, restore_corrupted_data());
+
+		reader.consume<uint64_t>(); // Consume the includeVersion
+		uint32_t val_length = reader.consume<uint32_t>();
+		return val_length == value.size() - sizeof(uint64_t) - sizeof(uint32_t);
+	}
+
 	// PRECONDITION: finished() must return false before calling this function.
 	// Returns the next batch of mutations along with the arena backing it.
 	ACTOR static Future<VersionedMutations> getNextBatchImpl(DecodeProgress* self) {
 		ASSERT(!self->finished());
 
-		state std::pair<Arena, KeyValueRef> arena_kv = self->keyValues[0];
-
-		// decode this batch's version
-		state std::pair<Version, int32_t> version_part = decode_key(arena_kv.second.key);
-		ASSERT(version_part.second == 0); // first part number must be 0.
-
-		// decode next versions, check if they are continuous parts
-		state int idx = 1; // next kv pair in "keyValues"
-		state int bufSize = arena_kv.second.value.size();
-		state int lastPart = 0;
 		loop {
-			// Try to decode another block if needed
-			if (idx == self->keyValues.size()) {
-				wait(readAndDecodeFile(self));
+			state std::tuple<Arena, Version, int32_t, StringRef> tuple = self->keyValues[0];
+
+			ASSERT(std::get<2>(tuple) == 0); // first part number must be 0.
+
+			// decode next versions, check if they are continuous parts
+			state int idx = 1; // next kv pair in "keyValues"
+			state int bufSize = std::get<3>(tuple).size();
+			state int lastPart = 0;
+			loop {
+				// Try to decode another block if needed
+				if (idx == self->keyValues.size()) {
+					wait(readAndDecodeFile(self));
+				}
+				if (idx == self->keyValues.size()) break;
+
+				auto next_tuple = self->keyValues[idx];
+				if (std::get<1>(tuple) != std::get<1>(next_tuple)) {
+					break;
+				}
+
+				if (lastPart + 1 != std::get<2>(next_tuple)) {
+					TraceEvent("DecodeError").detail("Part1", lastPart).detail("Part2", std::get<2>(next_tuple));
+					throw restore_corrupted_data();
+				}
+				bufSize += std::get<3>(next_tuple).size();
+				idx++;
+				lastPart++;
 			}
-			if (idx == self->keyValues.size()) break;
 
-			std::pair<Version, int32_t> next_version_part = decode_key(self->keyValues[idx].second.key);
-			if (version_part.first != next_version_part.first) break;
-
-			if (lastPart + 1 != next_version_part.second) {
-				TraceEvent("DecodeError").detail("Part1", lastPart).detail("Part2", next_version_part.second);
+			VersionedMutations m;
+			m.version = std::get<1>(tuple);
+			TraceEvent("Decode").detail("Version", m.version).detail("Idx", idx).detail("Q", self->keyValues.size());
+			StringRef value = std::get<3>(tuple);
+			if (idx > 1) {
+				// Stitch parts into one and then decode one by one
+				Standalone<StringRef> buf = self->combineValues(idx, bufSize);
+				value = buf;
+				m.arena = buf.arena();
+			} else {
+				m.arena = std::get<0>(tuple);
+			}
+			if (self->isValueComplete(value)) {
+				m.mutations = decode_value(value);
+				self->keyValues.erase(self->keyValues.begin(), self->keyValues.begin() + idx);
+				return m;
+			} else if (!self->eof) {
+				// Read one more block, hopefully the missing part of the value can be found.
+				wait(readAndDecodeFile(self));
+			} else {
+				TraceEvent(SevError, "MissingValue").detail("Version", m.version);
 				throw restore_corrupted_data();
 			}
-			bufSize += self->keyValues[idx].second.value.size();
-			idx++;
-			lastPart++;
 		}
-
-		VersionedMutations m;
-		m.version = version_part.first;
-		if (idx > 1) {
-			// Stitch parts into one and then decode one by one
-			Standalone<StringRef> buf = self->combineValues(idx, bufSize);
-			m.mutations = decode_value(buf);
-			m.arena = buf.arena();
-		} else {
-			m.mutations = decode_value(arena_kv.second.value);
-			m.arena = arena_kv.first;
-		}
-		self->keyValues.erase(self->keyValues.begin(), self->keyValues.begin() + idx);
-
-		return m;
 	}
 
 	// Returns a buffer which stitches first "idx" values into one.
@@ -275,7 +308,7 @@ struct DecodeProgress {
 		Standalone<StringRef> buf = makeString(len);
 		int n = 0;
 		for (int i = 0; i < idx; i++) {
-			const auto& value = keyValues[i].second.value;
+			const auto& value = std::get<3>(keyValues[i]);
 			memcpy(mutateString(buf) + n, value.begin(), value.size());
 			n += value.size();
 		}
@@ -301,9 +334,16 @@ struct DecodeProgress {
 				// Read key and value.  If anything throws then there is a problem.
 				uint32_t kLen = reader.consumeNetworkUInt32();
 				const uint8_t* k = reader.consume(kLen);
+				std::pair<Version, int32_t> version_part = decode_key(StringRef(k, kLen));
 				uint32_t vLen = reader.consumeNetworkUInt32();
 				const uint8_t* v = reader.consume(vLen);
-				keyValues.emplace_back(buf.arena(), KeyValueRef(StringRef(k, kLen), StringRef(v, vLen)));
+				TraceEvent("Block")
+				    .detail("KeySize", kLen)
+				    .detail("valueSize", vLen)
+				    .detail("Offset", reader.rptr - buf.begin())
+				    .detail("Version", version_part.first)
+				    .detail("Part", version_part.second);
+				keyValues.emplace_back(buf.arena(), version_part.first, version_part.second, StringRef(v, vLen));
 			}
 
 			// Make sure any remaining bytes in the block are 0xFF
@@ -311,6 +351,15 @@ struct DecodeProgress {
 				if (b != 0xFF) throw restore_corrupted_data_padding();
 			}
 
+			// The (version, part) in a block can be out of order, i.e., (3, 0)
+			// can be followed by (4, 0), and then (3, 1). So we need to sort them
+			// first by version, and then by part number.
+			std::sort(keyValues.begin(), keyValues.end(),
+			          [](const std::tuple<Arena, Version, int32_t, StringRef>& a,
+			             const std::tuple<Arena, Version, int32_t, StringRef>& b) {
+				          return std::get<1>(a) == std::get<1>(b) ? std::get<2>(a) < std::get<2>(b)
+				                                                  : std::get<1>(a) < std::get<1>(b);
+			          });
 			return;
 		} catch (Error& e) {
 			TraceEvent(SevWarn, "CorruptBlock").error(e).detail("Offset", reader.rptr - buf.begin());
@@ -360,14 +409,21 @@ struct DecodeProgress {
 	Reference<IAsyncFile> fd;
 	int64_t offset = 0;
 	bool eof = false;
-	// Key value pairs and their memory arenas.
-	std::vector<std::pair<Arena, KeyValueRef>> keyValues;
+	// A (version, part_number)'s mutations and memory arena.
+	std::vector<std::tuple<Arena, Version, int32_t, StringRef>> keyValues;
 };
 
 ACTOR Future<Void> decode_logs(DecodeParams params) {
 	state Reference<IBackupContainer> container = IBackupContainer::openContainer(params.container_url);
 
 	state BackupFileList listing = wait(container->dumpFileList());
+	// remove partitioned logs
+	listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(),
+	                                  [](const LogFile& file) {
+		                                  std::string prefix("plogs/");
+		                                  return file.fileName.substr(0, prefix.size()) == prefix;
+	                                  }),
+	                   listing.logs.end());
 	std::sort(listing.logs.begin(), listing.logs.end());
 	TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size());
 

From 7f3c64e32647f20f0fcad5bc65eeb018d4032111 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 10 Mar 2020 16:14:35 -0700
Subject: [PATCH 038/176] Ignore mutation logs of size 0 in converter

---
 fdbbackup/FileConverter.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp
index 67cd4738b6..006b311f87 100644
--- a/fdbbackup/FileConverter.actor.cpp
+++ b/fdbbackup/FileConverter.actor.cpp
@@ -68,7 +68,7 @@ void printLogFiles(std::string msg, const std::vector<LogFile>& files) {
 std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, Version begin, Version end) {
 	std::vector<LogFile> filtered;
 	for (const auto& file : files) {
-		if (file.beginVersion <= end && file.endVersion >= begin && file.tagId >= 0) {
+		if (file.beginVersion <= end && file.endVersion >= begin && file.tagId >= 0 && file.fileSize > 0) {
 			filtered.push_back(file);
 		}
 	}
@@ -76,7 +76,7 @@ std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, Vers
 
 	// Remove duplicates. This is because backup workers may store the log for
 	// old epochs successfully, but do not update the progress before another
-	// recovery happened.	As a result, next epoch will retry and creates
+	// recovery happened. As a result, next epoch will retry and creates
 	// duplicated log files.
 	std::vector<LogFile> sorted;
 	int i = 0;

From dbb05faa24f9e380532510e4ab5e6631613301c1 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 10 Mar 2020 19:42:36 -0700
Subject: [PATCH 039/176] Fix asset end version if request.targetVersion is -1

---
 fdbserver/RestoreMaster.actor.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index e76d8eff05..37fe740860 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -373,8 +373,9 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<MasterBatchData> batchDat
 		param.asset.len = file.fileSize;
 		param.asset.range = request.range;
 		param.asset.beginVersion = versionBatch.beginVersion;
-		param.asset.endVersion =
-		    isRangeFile ? versionBatch.endVersion : std::min(versionBatch.endVersion, request.targetVersion + 1);
+		param.asset.endVersion = (isRangeFile || request.targetVersion == -1)
+		                             ? versionBatch.endVersion
+		                             : std::min(versionBatch.endVersion, request.targetVersion + 1);
 
 		TraceEvent("FastRestoreMasterPhaseLoadFiles")
 		    .detail("BatchIndex", batchIndex)

From 472849e45c89a4f038ea46d5b26034a1f9e00190 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 10 Mar 2020 20:05:11 -0700
Subject: [PATCH 040/176] Fix MacOS compiling error

clang doesn't allow capture references, so use copy for lambda's capture list.
---
 fdbserver/BackupProgress.actor.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 1cb061af58..af236f86fa 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -71,9 +71,9 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 		if (progressIt != progress.end() && progressIt->first == epoch) {
 			updateTagVersions(&tagVersions, &tags, progressIt->second, info.epochEnd, epoch);
 		} else {
-			auto rit =
-			    std::find_if(progress.rbegin(), progress.rend(),
-			                 [=](const std::pair<LogEpoch, std::map<Tag, Version>>& p) { return p.first < epoch; });
+			auto rit = std::find_if(
+			    progress.rbegin(), progress.rend(),
+			    [epoch = epoch](const std::pair<LogEpoch, std::map<Tag, Version>>& p) { return p.first < epoch; });
 			if (!(rit == progress.rend())) {
 				// A partial recovery can result in empty epoch that copies previous
 				// epoch's version range. In this case, we should check previous

From 03fd5cf3fabcc39d92036c29fb6f1070dd7cd61f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 11 Mar 2020 15:39:09 -0700
Subject: [PATCH 041/176] Give maximum subsequence number for snapshot
 mutations

This is needed so that mutations in partitioned logs are applied first and
snapshot mutations are applied later for the same commit version.
---
 fdbserver/RestoreLoader.actor.cpp | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 109b25c43c..42fd83bf53 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -172,9 +172,9 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 
 			// Deserialize messages written in saveMutationsToFile().
 			LogMessageVersion msgVersion;
-			msgVersion.version = bigEndian64(reader.consume<Version>());
-			msgVersion.sub = bigEndian32(reader.consume<uint32_t>());
-			int msgSize = bigEndian32(reader.consume<int>());
+			msgVersion.version = reader.consumeNetworkUInt64();
+			msgVersion.sub = reader.consumeNetworkUInt32();
+			int msgSize = reader.consumeNetworkInt32();
 			const uint8_t* message = reader.consume(msgSize);
 
 			// Skip mutations out of the version range
@@ -769,12 +769,14 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 		cc->loadedRangeBytes += m.totalSize();
 
 		// We cache all kv operations into kvOps, and apply all kv operations later in one place
-		auto it = kvOps.insert(std::make_pair(LogMessageVersion(version), MutationsVec()));
+		// Note we give INT_MAX as the sub sequence number to override any log mutations.
+		const LogMessageVersion msgVersion(version, std::numeric_limits<int32_t>::max());
+		auto it = kvOps.insert(std::make_pair(msgVersion, MutationsVec()));
 		TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
 		    .detail("CommitVersion", version)
 		    .detail("ParsedMutationKV", m.toString());
 
-		ASSERT_WE_THINK(kvOps.find(LogMessageVersion(version)) != kvOps.end());
+		ASSERT_WE_THINK(kvOps.find(msgVersion) != kvOps.end());
 		it.first->second.push_back_deep(it.first->second.arena(), m);
 		// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
 		if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {

From 4e09c7be83f2fc63c24156c21573dd523912f78f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 11 Mar 2020 15:45:44 -0700
Subject: [PATCH 042/176] Remove debug print out

---
 fdbclient/BackupContainer.actor.cpp | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 69e0aa3378..6181fbb0fc 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -799,7 +799,6 @@ public:
 		}
 
 		state std::vector<LogFile> logs;
-std::cout << "describe list: scanBegin:" << scanBegin << ", scanEnd:" << scanEnd << ", partitioned:" << partitioned << "\n";
 		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, partitioned)) &&
 		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
 
@@ -1097,7 +1096,6 @@ std::cout << "describe list: scanBegin:" << scanBegin << ", scanEnd:" << scanEnd
 		ASSERT(tags == nullptr || tags->empty());
 		for (int idx : indices) {
 			const LogFile& file = files[idx];
-std::cout << "  " << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags " << lastTags << "\n";
 			if (lastEnd == invalidVersion) {
 				if (file.beginVersion > begin) return false;
 				if (file.endVersion > begin) {
@@ -1123,7 +1121,6 @@ std::cout << "  " << file.toString() << " " << "lastBegin " << lastBegin << ", l
 			lastEnd = file.endVersion;
 			if (lastEnd > end) break;
 		}
-std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags " << lastTags << "\n";
 		if (tags != nullptr && lastBegin != invalidVersion) {
 			tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags);
 		}
@@ -1181,7 +1178,6 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 
 		for (const LogFile& file : logs) {
 			Version end = getPartitionedLogsContinuousEndVersion(logs, file.beginVersion);
-std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
 			if (end > file.beginVersion) {
 				// desc->minLogBegin = file.beginVersion;
 				// contiguousLogEnd is not inclusive, so +1 here.
@@ -1194,8 +1190,6 @@ std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
 	// Returns the end version such that [begin, end] is continuous.
 	// "logs" should be already sorted.
 	static Version getPartitionedLogsContinuousEndVersion(const std::vector<LogFile>& logs, Version begin) {
-std::cout << "getPartitionedLogsContinuousEndVersion begin:" << begin << "\n";
-for (auto file : logs) std::cout << "    " << file.toString() << "\n";
 		Version end = 0;
 
 		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
@@ -1210,15 +1204,12 @@ for (auto file : logs) std::cout << "    " << file.toString() << "\n";
 			}
 			end = std::max(end, logs[i].endVersion - 1);
 		}
-std::cout << "Init end: " << end << ", begin " << begin << "\n";
 
 		// check tag 0 is continuous in [begin, end] and create a map of ranges to tags
 		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
 		isContinuous(logs, tagIndices[0], begin, end, &tags);
 		if (tags.empty() || end <= begin) return 0;
 		end = std::min(end, tags.rbegin()->first.second);
-std::cout << "  Tag 0 end: " << end << "\n";
-for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " << v << "\n";
 
 		// for each range in tags, check all tags from 1 are continouous
 		Version lastEnd = begin;
@@ -1228,7 +1219,6 @@ for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " <
 				std::map<std::pair<Version, Version>, int> rangeTags;
 				isContinuous(logs, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
 				tagEnd = rangeTags.empty() ? 0 : std::min(tagEnd, rangeTags.rbegin()->first.second);
-std::cout << "  Tag " << i << " end: " << tagEnd << ", return end = "<< lastEnd << "\n";
 				if (tagEnd == 0) return lastEnd;
 			}
 			if (tagEnd < beginEnd.second) {
@@ -1238,7 +1228,6 @@ std::cout << "  Tag " << i << " end: " << tagEnd << ", return end = "<< lastEnd
 			lastEnd = beginEnd.second;
 		}
 
-std::cout << "Return end = " << end << "\n\n";
 		return end;
 	}
 

From ceb56cf49d9508ca632b46943a8e93f3d563719c Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 11 Mar 2020 20:47:54 -0700
Subject: [PATCH 043/176] Add done trigger so that backup progress can be set

Otherwise, when there is no mutations for the unfinished range, the empty file
may not be created when the worker is displaced, thus leaving holes in version
ranges.
---
 fdbserver/BackupWorker.actor.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 2e9a3d5a48..258051ed83 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -110,6 +110,7 @@ struct BackupData {
 
 	std::map<UID, PerBackupInfo> backups; // Backup UID to infos
 	AsyncTrigger changedTrigger;
+	AsyncTrigger doneTrigger;
 
 	CounterCollection cc;
 	Future<Void> logger;
@@ -383,7 +384,9 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 						const Version current = savedLogVersions[versionConfigs[i].getUid()];
 						if (prevVersions[i].get().present()) {
 							const Version prev = prevVersions[i].get().get();
-							ASSERT(prev <= current);
+							TraceEvent(SevWarn, "BackupWorkerVersionInverse", self->myId)
+							    .detail("Prev", prev)
+							    .detail("Current", current);
 						}
 						if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
 							TraceEvent("BackupWorkerSetVersion", self->myId)
@@ -609,6 +612,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 		}
 
 		// If transition into NOOP mode, should clear messages
+		if (!self->pulling) self->messages.clear();
 
 		if (popVersion > self->savedVersion) {
 			wait(saveProgress(self, popVersion));
@@ -621,7 +625,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 		}
 
 		if (!self->pullFinished()) {
-			wait(uploadDelay);
+			wait(uploadDelay || self->doneTrigger.onTrigger());
 		}
 	}
 }
@@ -661,6 +665,7 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		TraceEvent("BackupWorkerGot", self->myId).suppressFor(1.0).detail("V", tagAt);
 		if (self->pullFinished()) {
 			self->eraseMessagesAfterEndVersion();
+			self->doneTrigger.trigger();
 			TraceEvent("BackupWorkerFinishPull", self->myId)
 			    .detail("Tag", self->tag.toString())
 			    .detail("VersionGot", tagAt)

From 14b5925276a2c8186ccf11b850a78b00e2899400 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 12 Mar 2020 10:28:10 -0700
Subject: [PATCH 044/176] Allow overlapped versions in partitioned logs

The overlapping can only happens between two generations, where the known
committed version to recovery version is copied from old generation to the new
generation. Within a generation, there is no overlap.

The fix here is related to the calculation of continuous version ranges,
allowing the overlap to happen.
---
 fdbclient/BackupContainer.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 6181fbb0fc..29027eff3a 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1085,7 +1085,7 @@ public:
 	// For a list of log files specified by their indices (of the same tag),
 	// returns if they are continous in the range [begin, end]. If "tags" is not
 	// nullptr, then it will be populated with [begin, end] -> tags, where next
-	// pair's begin == previous pair's end + 1. On return, the last pair's end
+	// pair's begin <= previous pair's end + 1. On return, the last pair's end
 	// version (inclusive) gives the continuous range from begin.
 	static bool isContinuous(const std::vector<LogFile>& files, const std::vector<int>& indices, Version begin,
 	                         Version end, std::map<std::pair<Version, Version>, int>* tags) {
@@ -1104,7 +1104,7 @@ public:
 				} else {
 					continue;
 				}
-			} else if (lastEnd != file.beginVersion) {
+			} else if (lastEnd < file.beginVersion) {
 				if (tags != nullptr) {
 					tags->emplace(std::make_pair(lastBegin, lastEnd - 1), lastTags);
 				}

From d5250084bd84f6bec4f618d544c3986b3ab9cd43 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 12 Mar 2020 14:38:40 -0700
Subject: [PATCH 045/176] Fix a time gap for monitoring backup keys

Backup worker starts by check if there are backup keys and then runs
monitorBackupKeyOrPullData() loop, which does the check again. The second check
can be delayed, which causes the loop to perform NOOP pops. The fix removes
this second check and uses the result of the first check to decide what to do
in the loop.
---
 fdbserver/BackupProgress.actor.cpp |  2 +-
 fdbserver/BackupWorker.actor.cpp   | 98 +++++++++++++++---------------
 fdbserver/RestoreMaster.actor.cpp  |  5 +-
 3 files changed, 55 insertions(+), 50 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index af236f86fa..1378980939 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -86,7 +86,7 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 				}
 				if (savedMore > 0) {
 					// The logRouterTags are the same
-					ASSERT(info.logRouterTags == epochTags[rit->first]);
+					// ASSERT(info.logRouterTags == epochTags[rit->first]);
 
 					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 258051ed83..cd7c00cd52 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -256,6 +256,23 @@ struct BackupData {
 		}
 		return true;
 	}
+
+	ACTOR static Future<Version> _getMinKnownCommittedVersion(BackupData* self) {
+		loop {
+			GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
+			                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
+			choose {
+				when(wait(self->cx->onMasterProxiesChanged())) {}
+				when(GetReadVersionReply reply = wait(loadBalance(self->cx->getMasterProxies(false),
+				                                                  &MasterProxyInterface::getConsistentReadVersion,
+				                                                  request, self->cx->taskID))) {
+					return reply.version;
+				}
+			}
+		}
+	}
+
+	Future<Version> getMinKnownCommittedVersion() { return _getMinKnownCommittedVersion(this); }
 };
 
 // Monitors "backupStartedKey". If "started" is true, wait until the key is set;
@@ -677,63 +694,48 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 	}
 }
 
-ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
+ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self, bool keyPresent) {
 	state Future<Void> pullFinished = Void();
-	state Future<bool> started;
-	state Future<GetReadVersionReply> replyFuture = Never();
 
 	loop {
-		started = monitorBackupStartedKeyChanges(self, true, true);
-		loop choose {
-			when(bool present = wait(started)) {
-				replyFuture = Never();
-				break;
+		state Future<bool> present = monitorBackupStartedKeyChanges(self, !keyPresent, /*watch=*/true);
+		if (keyPresent) {
+			pullFinished = pullAsyncData(self);
+			self->pulling = true;
+			wait(success(present) || pullFinished);
+			if (pullFinished.isReady()) {
+				self->pulling = false;
+				return Void(); // backup is done for some old epoch.
 			}
-			when(wait(self->cx->onMasterProxiesChanged() ||
-			          delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
-				GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
-				                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
-				replyFuture = loadBalance(self->cx->getMasterProxies(false),
-				                          &MasterProxyInterface::getConsistentReadVersion, request, self->cx->taskID);
-			}
-			when(GetReadVersionReply reply = wait(replyFuture)) {
-				replyFuture = Never();
-				self->savedVersion = std::max(reply.version, self->savedVersion);
-				self->minKnownCommittedVersion = std::max(reply.version, self->minKnownCommittedVersion);
-				TraceEvent("BackupWorkerNoopPop", self->myId).detail("SavedVersion", self->savedVersion);
-				self->pop(); // Pop while the worker is in this NOOP state.
-			}
-		}
 
-		Future<bool> stopped = monitorBackupStartedKeyChanges(self, false, true);
-		pullFinished = pullAsyncData(self);
-		self->pulling = true;
-		wait(success(stopped) || pullFinished);
-		if (pullFinished.isReady()) {
+			// Even though the snapshot is done, mutation logs may not be written
+			// out yet. We need to make sure mutations up to this point is written.
+			Version currentVersion = wait(self->getMinKnownCommittedVersion());
+			wait(self->pulledVersion.whenAtLeast(currentVersion));
+			pullFinished = Future<Void>(); // cancels pullAsyncData()
 			self->pulling = false;
-			return Void(); // backup is done for some old epoch.
-		}
+			TraceEvent("BackupWorkerPaused", self->myId);
+		} else {
+			// Backup key is not present, enter this NOOP POP mode.
+			state Future<Version> committedVersion = self->getMinKnownCommittedVersion();
 
-		// Even though the snapshot is done, mutation logs may not be written
-		// out yet. We need to make usre mutations up to this point is written.
-		state Version currentVersion;
-		loop {
-			GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
-			                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
-			choose {
-				when(wait(self->cx->onMasterProxiesChanged())) {}
-				when(GetReadVersionReply reply = wait(loadBalance(self->cx->getMasterProxies(false),
-				                                                  &MasterProxyInterface::getConsistentReadVersion,
-				                                                  request, self->cx->taskID))) {
-					currentVersion = reply.version;
-					break;
+			loop choose {
+				when(wait(success(present))) { break; }
+				when(wait(success(committedVersion) || delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
+					if (committedVersion.isReady()) {
+						self->savedVersion = std::max(committedVersion.get(), self->savedVersion);
+						self->minKnownCommittedVersion =
+						    std::max(committedVersion.get(), self->minKnownCommittedVersion);
+						TraceEvent("BackupWorkerNoopPop", self->myId).detail("SavedVersion", self->savedVersion);
+						self->pop(); // Pop while the worker is in this NOOP state.
+						committedVersion = Never();
+					} else {
+						committedVersion = self->getMinKnownCommittedVersion();
+					}
 				}
 			}
 		}
-		wait(self->pulledVersion.whenAtLeast(currentVersion));
-		pullFinished = Future<Void>(); // cancels pullAsyncData()
-		self->pulling = false;
-		TraceEvent("BackupWorkerPaused", self->myId);
+		keyPresent = !keyPresent;
 	}
 }
 
@@ -784,7 +786,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		bool present = wait(monitorBackupStartedKeyChanges(&self, true, false));
 		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present);
 
-		pull = monitorBackupKeyOrPullData(&self);
+		pull = monitorBackupKeyOrPullData(&self, present);
 		done = uploadData(&self);
 
 		loop choose {
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 37fe740860..8992062be8 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -686,7 +686,10 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 	if (request.targetVersion == invalidVersion && desc.maxRestorableVersion.present()) {
 		request.targetVersion = desc.maxRestorableVersion.get();
 	}
-	TraceEvent("FastRestore").detail("TargetVersion", request.targetVersion).detail("BackupDesc", desc.toString());
+
+	if (g_network->isSimulated()) {
+		std::cout << "Restore to version: " << request.targetVersion << "\nBackupDesc: \n" << desc.toString() << "\n\n";
+	}
 
 	Optional<RestorableFileSet> restorable =
 	    wait(SERVER_KNOBS->FASTRESTORE_USE_PARTITIONED_LOGS ? bc->getPartitionedRestoreSet(request.targetVersion)

From a855e871e003e157f645e4dd9f5ac12e802e2790 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 12 Mar 2020 15:30:07 -0700
Subject: [PATCH 046/176] Fix duplicate file removal for subset version ranges

Partitioned logs can have strict subset version ranges, which was not properly
handled -- we used to assume overlapping only happens for the same begin
version.
---
 fdbclient/BackupContainer.actor.cpp | 14 ++++++++++----
 fdbclient/BackupContainer.h         |  2 +-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 29027eff3a..7ab40e2113 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1156,11 +1156,13 @@ public:
 	// If a log file's progress is not saved, a new log file will be generated
 	// with the same begin version. So we can have a file that contains a subset
 	// of contents in another log file.
-	// PRE-CONDITION: logs are already sorted.
+	// PRE-CONDITION: logs are already sorted by (tagId, beginVersion, endVersion).
 	static std::vector<LogFile> filterDuplicates(const std::vector<LogFile>& logs) {
 		std::vector<LogFile> filtered;
 		int i = 0;
 		for (int j = 1; j < logs.size(); j++) {
+			if (logs[j].isSubset(logs[i])) continue;
+
 			if (!logs[i].isSubset(logs[j])) {
 				filtered.push_back(logs[i]);
 			}
@@ -1196,9 +1198,13 @@ public:
 		for (int i = 0; i < logs.size(); i++) {
 			ASSERT(logs[i].tagId >= 0 && logs[i].tagId < logs[i].totalTags);
 			auto& indices = tagIndices[logs[i].tagId];
-			// filter out if indices.back() is subset of files[i]
-			if (!indices.empty() && logs[indices.back()].isSubset(logs[i])) {
-				indices.back() = i;
+			// filter out if indices.back() is subset of files[i] or vice versa
+			if (!indices.empty()) {
+				if (logs[indices.back()].isSubset(logs[i])) {
+					indices.back() = i;
+				} else if (!logs[i].isSubset(logs[indices.back()])) {
+					indices.push_back(i);
+				}
 			} else {
 				indices.push_back(i);
 			}
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 4bf144c07e..3b1f5de5bf 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -85,7 +85,7 @@ struct LogFile {
 	// Returns if this log file contains a subset of content of the given file
 	// by comparing version range and tag ID.
 	bool isSubset(const LogFile& rhs) const {
-		return beginVersion == rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
+		return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
 	}
 
 	std::string toString() const {

From c3dd5931134bbabca6d096f74708c36299a72223 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 12 Mar 2020 20:51:10 -0700
Subject: [PATCH 047/176] Updates lastest backup worker progress after all
 previous epochs are done

If workers for previous epochs are still ongoing, we may end up with a
container that miss mutations in previous epochs. So the update only happens
after there are only current epoch's backup workers.
---
 fdbserver/BackupWorker.actor.cpp | 155 ++++++++++++++++---------------
 1 file changed, 81 insertions(+), 74 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index cd7c00cd52..8452d20d43 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -328,95 +328,102 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 // log saved version > snapshot version.
 ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 	loop {
-		wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger());
-		if (self->backups.empty()) {
-			continue;
+		while (self->backups.empty() || !self->logSystem.get()) {
+			wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger() ||
+			     self->logSystem.onChange());
 		}
 
 		// check all workers have started by checking their progress is larger
 		// than the backup's start version.
-		state Reference<BackupProgress> progress(new BackupProgress(self->myId, {}));
+		state Reference<BackupProgress> progress(
+		    new BackupProgress(self->myId, self->logSystem.get()->getOldEpochTagsVersionsInfo()));
 		wait(getBackupProgress(self->cx, self->myId, progress));
 		std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
+		std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
+		    progress->getUnfinishedBackup();
+		bool finishedPreviousEpochs =
+		    toRecruit.empty() || std::get<0>(toRecruit.begin()->first) == self->recruitedEpoch;
 
 		state std::vector<UID> ready;
 		state std::map<UID, Version> savedLogVersions;
-		if (tagVersions.size() == self->logSystem.get()->getLogRouterTags()) {
-			// Check every version is larger than backup's startVersion
-			for (auto& [uid, info] : self->backups) {
-				if (info.allWorkerStarted) {
-					// update update progress so far
-					Version v = std::numeric_limits<Version>::max();
-					for (const auto [tag, version] : tagVersions) {
-						v = std::min(v, version);
-					}
-					savedLogVersions.emplace(uid, v);
-					continue;
+		if (tagVersions.size() != self->logSystem.get()->getLogRouterTags()) {
+			continue;
+		}
+
+		// Check every version is larger than backup's startVersion
+		for (auto& [uid, info] : self->backups) {
+			if (info.allWorkerStarted && finishedPreviousEpochs) {
+				// update update progress so far
+				Version v = std::numeric_limits<Version>::max();
+				for (const auto [tag, version] : tagVersions) {
+					v = std::min(v, version);
 				}
-				bool saved = true;
-				for (const std::pair<Tag, Version> tv : tagVersions) {
-					if (tv.second < info.startVersion) {
-						saved = false;
-						break;
-					}
-				}
-				if (saved) {
-					ready.push_back(uid);
-					info.allWorkerStarted = true;
+				savedLogVersions.emplace(uid, v);
+				continue;
+			}
+			bool saved = true;
+			for (const std::pair<Tag, Version> tv : tagVersions) {
+				if (tv.second < info.startVersion) {
+					saved = false;
+					break;
 				}
 			}
-			if (ready.empty() && savedLogVersions.empty()) continue;
+			if (saved) {
+				ready.push_back(uid);
+				info.allWorkerStarted = true;
+			}
+		}
+		if (ready.empty() && savedLogVersions.empty()) continue;
 
-			// Set "allWorkerStarted" key for ready backups
-			loop {
-				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
-				try {
-					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+		// Set "allWorkerStarted" and "latestBackupWorkerSavedVersion" key for backups
+		loop {
+			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
+			try {
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-					state std::vector<Future<Optional<Value>>> readyValues;
-					state std::vector<BackupConfig> configs;
-					for (UID uid : ready) {
-						configs.emplace_back(uid);
-						readyValues.push_back(tr->get(configs.back().allWorkerStarted().key));
-					}
-
-					state std::vector<Future<Optional<Version>>> prevVersions;
-					state std::vector<BackupConfig> versionConfigs;
-					for (const auto [uid, version] : savedLogVersions) {
-						versionConfigs.emplace_back(uid);
-						prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
-					}
-
-					wait(waitForAll(readyValues) && waitForAll(prevVersions));
-
-					for (int i = 0; i < readyValues.size(); i++) {
-						if (!readyValues[i].get().present()) {
-							configs[i].allWorkerStarted().set(tr, true);
-							TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", ready[i].toString());
-						}
-					}
-
-					for (int i = 0; i < prevVersions.size(); i++) {
-						const Version current = savedLogVersions[versionConfigs[i].getUid()];
-						if (prevVersions[i].get().present()) {
-							const Version prev = prevVersions[i].get().get();
-							TraceEvent(SevWarn, "BackupWorkerVersionInverse", self->myId)
-							    .detail("Prev", prev)
-							    .detail("Current", current);
-						}
-						if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
-							TraceEvent("BackupWorkerSetVersion", self->myId)
-							    .detail("BackupID", versionConfigs[i].getUid())
-							    .detail("Version", current);
-							versionConfigs[i].latestBackupWorkerSavedVersion().set(tr, current);
-						}
-					}
-					wait(tr->commit());
-					break;
-				} catch (Error& e) {
-					wait(tr->onError(e));
+				state std::vector<Future<Optional<Value>>> readyValues;
+				state std::vector<BackupConfig> configs;
+				for (UID uid : ready) {
+					configs.emplace_back(uid);
+					readyValues.push_back(tr->get(configs.back().allWorkerStarted().key));
 				}
+
+				state std::vector<Future<Optional<Version>>> prevVersions;
+				state std::vector<BackupConfig> versionConfigs;
+				for (const auto [uid, version] : savedLogVersions) {
+					versionConfigs.emplace_back(uid);
+					prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
+				}
+
+				wait(waitForAll(readyValues) && waitForAll(prevVersions));
+
+				for (int i = 0; i < readyValues.size(); i++) {
+					if (!readyValues[i].get().present()) {
+						configs[i].allWorkerStarted().set(tr, true);
+						TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", ready[i].toString());
+					}
+				}
+
+				for (int i = 0; i < prevVersions.size(); i++) {
+					const Version current = savedLogVersions[versionConfigs[i].getUid()];
+					if (prevVersions[i].get().present()) {
+						const Version prev = prevVersions[i].get().get();
+						TraceEvent(SevWarn, "BackupWorkerVersionInverse", self->myId)
+						    .detail("Prev", prev)
+						    .detail("Current", current);
+					}
+					if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
+						TraceEvent("BackupWorkerSetVersion", self->myId)
+						    .detail("BackupID", versionConfigs[i].getUid())
+						    .detail("Version", current);
+						versionConfigs[i].latestBackupWorkerSavedVersion().set(tr, current);
+					}
+				}
+				wait(tr->commit());
+				break;
+			} catch (Error& e) {
+				wait(tr->onError(e));
 			}
 		}
 	}

From 3bb12bc8444348281e9bb4c0ea9599cf8f4a071c Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 13 Mar 2020 18:44:15 -0700
Subject: [PATCH 048/176] Fix decode bug of missing mutations

After reading a new block, all mutations are sorted by version again, which
can invalidate previously tuple. As a result, the decoded file will miss some
of the mutations.
---
 fdbbackup/FileDecoder.actor.cpp | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 62716a562d..ef074adfac 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -245,19 +245,18 @@ struct DecodeProgress {
 		ASSERT(!self->finished());
 
 		loop {
-			state std::tuple<Arena, Version, int32_t, StringRef> tuple = self->keyValues[0];
+			if (self->keyValues.size() == 1) {
+				// Try to decode another block when only one left
+				wait(readAndDecodeFile(self));
+			}
 
+			auto& tuple = self->keyValues[0];
 			ASSERT(std::get<2>(tuple) == 0); // first part number must be 0.
 
 			// decode next versions, check if they are continuous parts
-			state int idx = 1; // next kv pair in "keyValues"
-			state int bufSize = std::get<3>(tuple).size();
-			state int lastPart = 0;
-			loop {
-				// Try to decode another block if needed
-				if (idx == self->keyValues.size()) {
-					wait(readAndDecodeFile(self));
-				}
+			int idx = 1; // next kv pair in "keyValues"
+			int bufSize = std::get<3>(tuple).size();
+			for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
 				if (idx == self->keyValues.size()) break;
 
 				auto next_tuple = self->keyValues[idx];
@@ -270,8 +269,6 @@ struct DecodeProgress {
 					throw restore_corrupted_data();
 				}
 				bufSize += std::get<3>(next_tuple).size();
-				idx++;
-				lastPart++;
 			}
 
 			VersionedMutations m;

From d1ef6f1225852c6c2dafd469d716f6d6a0f700fa Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 14 Mar 2020 09:42:42 -0700
Subject: [PATCH 049/176] Fix missing mutations in splitMutation

When a range mutation is larger than the last split point, this mutation can
become missing in the RestoreLoader, which is fixed in this commit.
---
 fdbserver/RestoreLoader.actor.cpp | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 42fd83bf53..1b2f48925f 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -463,11 +463,25 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 				              nodeIDs.contents());
 				ASSERT(mvector.size() == nodeIDs.size());
 
+				if (debugMutation("RestoreLoader", commitVersion.version, kvm)) {
+					TraceEvent e("DebugSplit");
+					int i = 0;
+					for (auto& [key, uid] : *pRangeToApplier) {
+						e.detail(format("Range%d", i).c_str(), printable(key))
+						    .detail(format("UID%d", i).c_str(), uid.toString());
+						i++;
+					}
+				}
 				for (splitMutationIndex = 0; splitMutationIndex < mvector.size(); splitMutationIndex++) {
 					MutationRef mutation = mvector[splitMutationIndex];
 					UID applierID = nodeIDs[splitMutationIndex];
 					// printf("SPLITTED MUTATION: %d: mutation:%s applierID:%s\n", splitMutationIndex,
 					// mutation.toString().c_str(), applierID.toString().c_str());
+					if (debugMutation("RestoreLoader", commitVersion.version, mutation)) {
+						TraceEvent("SplittedMutation")
+						    .detail("Version", commitVersion.toString())
+						    .detail("Mutation", mutation.toString());
+					}
 					applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
 					applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
 					applierMutationsSize[applierID] += mutation.expectedSize();
@@ -522,8 +536,14 @@ void splitMutation(std::map<Key, UID>* pRangeToApplier, MutationRef m, Arena& mv
 	ASSERT(mvector.empty());
 	ASSERT(nodeIDs.empty());
 	// key range [m->param1, m->param2)
-	std::map<Standalone<KeyRef>, UID>::iterator itlow, itup; // we will return [itlow, itup)
+	std::map<Key, UID>::iterator itlow, itup; // we will return [itlow, itup)
 	itlow = pRangeToApplier->lower_bound(m.param1); // lower_bound returns the iterator that is >= m.param1
+	if (itlow == pRangeToApplier->end()) {
+		--itlow;
+		mvector.push_back_deep(mvector_arena, m);
+		nodeIDs.push_back(nodeIDs_arena, itlow->second);
+		return;
+	}
 	if (itlow->first > m.param1) {
 		if (itlow != pRangeToApplier->begin()) {
 			--itlow;
@@ -533,7 +553,7 @@ void splitMutation(std::map<Key, UID>* pRangeToApplier, MutationRef m, Arena& mv
 	itup = pRangeToApplier->upper_bound(m.param2); // return rmap::end if no key is after m.param2.
 	ASSERT(itup == pRangeToApplier->end() || itup->first > m.param2);
 
-	std::map<Standalone<KeyRef>, UID>::iterator itApplier;
+	std::map<Key, UID>::iterator itApplier;
 	while (itlow != itup) {
 		Standalone<MutationRef> curm; // current mutation
 		curm.type = m.type;
@@ -776,7 +796,6 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 		    .detail("CommitVersion", version)
 		    .detail("ParsedMutationKV", m.toString());
 
-		ASSERT_WE_THINK(kvOps.find(msgVersion) != kvOps.end());
 		it.first->second.push_back_deep(it.first->second.arena(), m);
 		// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
 		if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {

From 0fb9e943f289b66d9a94950af1ea58f45ab23064 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 14 Mar 2020 15:54:47 -0700
Subject: [PATCH 050/176] Small code refactor

---
 fdbclient/BackupContainer.actor.cpp |  5 ++---
 fdbserver/RestoreApplier.actor.cpp  |  1 -
 fdbserver/RestoreLoader.actor.cpp   | 23 +++++++++--------------
 fdbserver/RestoreMaster.actor.cpp   |  2 --
 4 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 7ab40e2113..e6a7c62680 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1266,9 +1266,8 @@ public:
 			if (partitioned) {
 				// sort by tag ID so that filterDuplicates works.
 				std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) {
-					return a.tagId == b.tagId ? (a.beginVersion == b.beginVersion ? a.endVersion < b.endVersion
-					                                                              : a.beginVersion < b.beginVersion)
-					                          : (a.tagId < b.tagId);
+					return std::tie(a.tagId, a.beginVersion, a.endVersion) <
+					       std::tie(b.tagId, b.beginVersion, b.endVersion);
 				});
 
 				// Remove duplicated log files that can happen for old epochs.
diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index fb31fea375..65deb246ff 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -139,7 +139,6 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMu
 			const MutationRef& mutation = req.mutations[mIndex];
 			const LogMessageVersion mutationVersion(commitVersion, req.subs[mIndex]);
 			TraceEvent(SevFRMutationInfo, "FastRestoreApplierPhaseReceiveMutations", self->id())
-			    .detail("ApplierNode", self->id())
 			    .detail("RestoreAsset", req.asset.toString())
 			    .detail("Version", mutationVersion.toString())
 			    .detail("Index", mIndex)
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 1b2f48925f..c21434e6cf 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -777,20 +777,19 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 
 	// Now data only contains the kv mutation within restoreRange
 	VectorRef<KeyValueRef> data = blockData.slice(rangeStart, rangeEnd);
-	int start = 0;
-	int end = data.size();
 
-	// Convert KV in data into mutations in kvOps
-	for (int i = start; i < end; ++i) {
+	// Note we give INT_MAX as the sub sequence number to override any log mutations.
+	const LogMessageVersion msgVersion(version, std::numeric_limits<int32_t>::max());
+
+	// Convert KV in data into SET mutations of different keys in kvOps
+	for (const KeyValueRef& kv : data) {
 		// NOTE: The KV pairs in range files are the real KV pairs in original DB.
 		// Should NOT add prefix or remove surfix for the backup data!
-		MutationRef m(MutationRef::Type::SetValue, data[i].key,
-		              data[i].value); // ASSUME: all operation in range file is set.
+		MutationRef m(MutationRef::Type::SetValue, kv.key,
+		              kv.value); // ASSUME: all operation in range file is set.
 		cc->loadedRangeBytes += m.totalSize();
 
 		// We cache all kv operations into kvOps, and apply all kv operations later in one place
-		// Note we give INT_MAX as the sub sequence number to override any log mutations.
-		const LogMessageVersion msgVersion(version, std::numeric_limits<int32_t>::max());
 		auto it = kvOps.insert(std::make_pair(msgVersion, MutationsVec()));
 		TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
 		    .detail("CommitVersion", version)
@@ -831,13 +830,9 @@ ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(NotifiedVersion* pPro
 	wait(pProcessedFileOffset->whenAtLeast(asset.offset));
 
 	if (pProcessedFileOffset->get() == asset.offset) {
-		int start = 0;
-		int end = data.size();
-		for (int i = start; i < end; ++i) {
-			// Key k = data[i].key.withPrefix(mutationLogPrefix);
-			// ValueRef v = data[i].value;
+		for (const KeyValueRef& kv : data) {
 			// Concatenate the backuped param1 and param2 (KV) at the same version.
-			concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, data[i].key, data[i].value, asset);
+			concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, kv.key, kv.value, asset);
 		}
 		pProcessedFileOffset->set(asset.offset + asset.len);
 	}
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 8992062be8..9b1551e242 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -616,14 +616,12 @@ void splitKeyRangeForAppliers(Reference<MasterBatchData> batchData,
 	}
 
 	std::set<Key>::iterator splitter = keyrangeSplitter.begin();
-	int i = 0;
 	batchData->rangeToApplier.clear();
 	for (auto& applier : appliersInterf) {
 		if (splitter == keyrangeSplitter.end()) {
 			break; // Not all appliers will be used
 		}
 		batchData->rangeToApplier[*splitter] = applier.first;
-		i++;
 		splitter++;
 	}
 	ASSERT(batchData->rangeToApplier.size() > 0);

From b697e46b19ff9f8a458ae8eeca7ab36edfda0036 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 15 Mar 2020 21:46:38 -0700
Subject: [PATCH 051/176] Fix duplicated mutation in StagingKey

For some reason I am not sure why, there can be duplicated mutations added to
StagingKey, which needs to be filtered out. Otherwise, atomic operations can
result in corrupted data in database.
---
 fdbserver/RestoreApplier.actor.h | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index a50c7f346f..1c398cfac8 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -64,6 +64,8 @@ struct StagingKey {
 			TraceEvent("FastRestoreApplierStagingKeyMutationAtSameVersion")
 			    .detail("Version", newVersion.toString())
 			    .detail("NewMutation", m.toString())
+			    .detail("Key", printable(key))
+			    .detail("Value", printable(val))
 			    .detail("ExistingKeyType", typeString[type]);
 			if (m.type == MutationRef::SetValue) {
 				if (type == MutationRef::SetValue) {
@@ -91,8 +93,13 @@ struct StagingKey {
 				    .detail("NewMutation", m.toString())
 				    .detail("ExistingKeyType", typeString[type])
 				    .detail("ExitingKeyValue", val);
+			} else {
+				ASSERT(false); // Can't be true same key, same version, different mutation
 			}
+
+			return;
 		}
+
 		// newVersion can be smaller than version as different loaders can send
 		// mutations out of order.
 		if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
@@ -107,9 +114,13 @@ struct StagingKey {
 			if (it == pendingMutations.end()) {
 				bool inserted;
 				std::tie(it, inserted) = pendingMutations.emplace(newVersion, MutationsVec());
+				// TODO: Do we really need deep copy?
+				it->second.push_back_deep(it->second.arena(), m);
+			} else {
+				// Duplicated mutation ignored.
+				MutationRef& m1 = *(it->second.begin());
+				ASSERT(m1.type == m.type && m1.param1 == m.param1 && m1.param2 == m.param2);
 			}
-			// TODO: Do we really need deep copy?
-			it->second.push_back_deep(it->second.arena(), m);
 		}
 	}
 
@@ -126,8 +137,7 @@ struct StagingKey {
 		}
 		if (lb->first == version) {
 			// Sanity check mutations at version are either atomicOps which can be ignored or the same value as buffered
-			for (int i = 0; i < lb->second.size(); i++) {
-				MutationRef m = lb->second[i];
+			for (const MutationRef& m : lb->second) {
 				if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
 					if (std::tie(type, key, val) != std::tie(m.type, m.param1, m.param2)) {
 						TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnhandledSituation")
@@ -138,12 +148,9 @@ struct StagingKey {
 					}
 				}
 			}
+			lb++;
 		}
-		while (lb != pendingMutations.end()) {
-			if (lb->first == version) {
-				lb++;
-				continue;
-			}
+		for (; lb != pendingMutations.end(); lb++) {
 			for (auto& mutation : lb->second) {
 				if (type == MutationRef::CompareAndClear) { // Special atomicOp
 					Arena arena;
@@ -162,16 +169,15 @@ struct StagingKey {
 				} else if (mutation.type == MutationRef::SetValue || mutation.type == MutationRef::ClearRange) {
 					type = MutationRef::SetValue; // Precomputed result should be set to DB.
 					TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet")
-					    .detail("Type", typeString[mutation.type])
+					    .detail("MutationType", typeString[mutation.type])
 					    .detail("Version", lb->first.toString());
 				} else {
 					TraceEvent(SevWarnAlways, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation")
-					    .detail("Type", typeString[mutation.type])
+					    .detail("MutationType", typeString[mutation.type])
 					    .detail("Version", lb->first.toString());
 				}
 			}
 			version = lb->first;
-			lb++;
 		}
 	}
 

From 9b11bd8ee41c4a12df85115d04ce5e005dacdd38 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 16 Mar 2020 18:20:02 -0700
Subject: [PATCH 052/176] Batch sending all mutations of a version from
 RestoreLoader

This optimization is to reduce the number of messages sent from loader to
applier, which was unintentionally done when introducing sub sequence numbers
for mutations.
---
 fdbserver/RestoreLoader.actor.cpp | 73 ++++++++++++++++++-------------
 1 file changed, 42 insertions(+), 31 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index c21434e6cf..1a9931ef91 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -436,17 +436,17 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 	splitMutationIndex = 0;
 	kvCount = 0;
 
+	// applierMutationsBuffer is the mutation vector to be sent to each applier
+	// applierMutationsSize is buffered mutation vector size for each applier
+	state std::map<UID, MutationsVec> applierMutationsBuffer;
+	state std::map<UID, SubSequenceVec> applierSubsBuffer;
+	state std::map<UID, double> applierMutationsSize;
+	for (auto& applierID : applierIDs) {
+		applierMutationsBuffer[applierID] = MutationsVec();
+		applierSubsBuffer[applierID] = SubSequenceVec();
+		applierMutationsSize[applierID] = 0.0;
+	}
 	for (kvOp = kvOps.begin(); kvOp != kvOps.end(); kvOp++) {
-		// applierMutationsBuffer is the mutation vector to be sent to each applier
-		// applierMutationsSize is buffered mutation vector size for each applier
-		std::map<UID, MutationsVec> applierMutationsBuffer;
-		std::map<UID, SubSequenceVec> applierSubsBuffer;
-		std::map<UID, double> applierMutationsSize;
-		for (auto& applierID : applierIDs) {
-			applierMutationsBuffer[applierID] = MutationsVec();
-			applierSubsBuffer[applierID] = SubSequenceVec();
-			applierMutationsSize[applierID] = 0.0;
-		}
 		const LogMessageVersion& commitVersion = kvOp->first;
 		ASSERT(commitVersion.version >= asset.beginVersion);
 		ASSERT(commitVersion.version <= asset.endVersion); // endVersion is an empty commit to ensure progress
@@ -493,35 +493,46 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 				--itlow; // make sure itlow->first <= m.param1
 				ASSERT(itlow->first <= kvm.param1);
 				UID applierID = itlow->second;
-				// printf("KV--Applier: K:%s ApplierID:%s\n", kvm.param1.toString().c_str(),
-				// applierID.toString().c_str());
 				kvCount++;
 
+				if (debugMutation("RestoreLoader", commitVersion.version, kvm)) {
+					TraceEvent("SendMutation")
+					    .detail("Applier", applierID)
+					    .detail("Version", commitVersion.toString())
+					    .detail("Mutation", kvm.toString());
+				}
 				applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), kvm);
 				applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
 				applierMutationsSize[applierID] += kvm.expectedSize();
 			}
-		} // Mutations at the same version
+		} // Mutations at the same LogMessageVersion
 
-		// TODO: Sanity check each asset has been received exactly once!
-		// Send the mutations to appliers for each version
-		for (const UID& applierID : applierIDs) {
-			requests.emplace_back(applierID, RestoreSendVersionedMutationsRequest(
-			                                     batchIndex, asset, prevVersion, commitVersion.version, isRangeFile,
-			                                     applierMutationsBuffer[applierID], applierSubsBuffer[applierID]));
+		// Batch same Version's mutations in one request. We could batch more by
+		// changing the version comparison below.
+		auto next = std::next(kvOp, 1);
+		if (next == kvOps.end() || commitVersion.version < next->first.version) {
+			// TODO: Sanity check each asset has been received exactly once!
+			// Send the mutations to appliers for each version
+			for (const UID& applierID : applierIDs) {
+				requests.emplace_back(applierID, RestoreSendVersionedMutationsRequest(
+				                                     batchIndex, asset, prevVersion, commitVersion.version, isRangeFile,
+				                                     applierMutationsBuffer[applierID], applierSubsBuffer[applierID]));
+			}
+			TraceEvent(SevDebug, "FastRestore_SendMutationToApplier")
+			    .detail("PrevVersion", prevVersion)
+			    .detail("CommitVersion", commitVersion.toString())
+			    .detail("RestoreAsset", asset.toString());
+			ASSERT(prevVersion < commitVersion.version);
+			prevVersion = commitVersion.version;
+			wait(sendBatchRequests(&RestoreApplierInterface::sendMutationVector, *pApplierInterfaces, requests,
+			                       TaskPriority::RestoreLoaderSendMutations));
+			requests.clear();
+			for (auto& applierID : applierIDs) {
+				applierMutationsBuffer[applierID] = MutationsVec();
+				applierSubsBuffer[applierID] = SubSequenceVec();
+				applierMutationsSize[applierID] = 0.0;
+			}
 		}
-		TraceEvent(SevDebug, "FastRestore_SendMutationToApplier")
-		    .detail("PrevVersion", prevVersion)
-		    .detail("CommitVersion", commitVersion.toString())
-		    .detail("RestoreAsset", asset.toString());
-		ASSERT(prevVersion <= commitVersion.version);
-		prevVersion = commitVersion.version;
-		// Tracking this request can be spammy
-		wait(sendBatchRequests(&RestoreApplierInterface::sendMutationVector, *pApplierInterfaces, requests,
-		                       TaskPriority::RestoreLoaderSendMutations,
-		                       SERVER_KNOBS->FASTRESTORE_TRACK_LOADER_SEND_REQUESTS));
-
-		requests.clear();
 	} // all versions of mutations in the same file
 
 	TraceEvent("FastRestore").detail("LoaderSendMutationOnAppliers", kvCount);

From 3513bbefe6967899225749098821a6ad165bf252 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 16 Mar 2020 18:22:24 -0700
Subject: [PATCH 053/176] StagingKey uses mutation instead of a vector of
 mutations for each log version

Because each log version contains commit version and subsequence number, each
key can only have one mutation for its log version. This simplifies
StagingKey::add() a lot.
---
 fdbserver/RestoreApplier.actor.cpp |  12 ++-
 fdbserver/RestoreApplier.actor.h   | 126 +++++++++++------------------
 2 files changed, 53 insertions(+), 85 deletions(-)

diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index 65deb246ff..ac2cee020e 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -202,8 +202,8 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
     std::map<Key, std::map<Key, StagingKey>::iterator> incompleteStagingKeys, Database cx, UID applierID) {
 	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 	state std::vector<Future<Optional<Value>>> fValues;
-	state int i = 0;
 	state int retries = 0;
+
 	TraceEvent("FastRestoreApplierGetAndComputeStagingKeysStart", applierID)
 	    .detail("GetKeys", incompleteStagingKeys.size());
 	loop {
@@ -228,7 +228,7 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 	}
 
 	ASSERT(fValues.size() == incompleteStagingKeys.size());
-	i = 0;
+	int i = 0;
 	for (auto& key : incompleteStagingKeys) {
 		if (!fValues[i].get().present()) {
 			TraceEvent(SevWarnAlways, "FastRestoreApplierGetAndComputeStagingKeysUnhandledError")
@@ -237,11 +237,9 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 			    .detail("PendingMutations", key.second->second.pendingMutations.size())
 			    .detail("StagingKeyType", (int)key.second->second.type);
 			for (auto& vm : key.second->second.pendingMutations) {
-				for (auto& m : vm.second) {
-					TraceEvent(SevWarnAlways, "FastRestoreApplierGetAndComputeStagingKeysUnhandledError")
-					    .detail("PendingMutationVersion", vm.first.toString())
-					    .detail("PendingMutation", m.toString());
-				}
+				TraceEvent(SevWarnAlways, "FastRestoreApplierGetAndComputeStagingKeysUnhandledError")
+				    .detail("PendingMutationVersion", vm.first.toString())
+				    .detail("PendingMutation", vm.second.toString());
 			}
 			key.second->second.precomputeResult();
 			i++;
diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index 1c398cfac8..66cf075bf6 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -52,7 +52,7 @@ struct StagingKey {
 	Value val;
 	MutationRef::Type type; // set or clear
 	LogMessageVersion version; // largest version of set or clear for the key
-	std::map<LogMessageVersion, MutationsVec> pendingMutations; // mutations not set or clear type
+	std::map<LogMessageVersion, Standalone<MutationRef>> pendingMutations; // mutations not set or clear type
 
 	explicit StagingKey() : version(0), type(MutationRef::MAX_ATOMIC_OP) {}
 
@@ -60,43 +60,15 @@ struct StagingKey {
 	// Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set
 	void add(const MutationRef& m, LogMessageVersion newVersion) {
 		ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue);
+		if (debugMutation("StagingKeyAdd", newVersion.version, m)) {
+			TraceEvent("StagingKeyAdd")
+			    .detail("Version", version.toString())
+			    .detail("NewVersion", newVersion.toString())
+			    .detail("Mutation", m.toString());
+		}
 		if (version == newVersion) { // Sanity check
-			TraceEvent("FastRestoreApplierStagingKeyMutationAtSameVersion")
-			    .detail("Version", newVersion.toString())
-			    .detail("NewMutation", m.toString())
-			    .detail("Key", printable(key))
-			    .detail("Value", printable(val))
-			    .detail("ExistingKeyType", typeString[type]);
-			if (m.type == MutationRef::SetValue) {
-				if (type == MutationRef::SetValue) {
-					if (m.param2 != val) {
-						TraceEvent(SevError, "FastRestoreApplierStagingKeyMutationAtSameVersionUnhandled")
-						    .detail("Version", newVersion.toString())
-						    .detail("NewMutation", m.toString())
-						    .detail("ExistingKeyType", typeString[type])
-						    .detail("ExitingKeyValue", val)
-						    .detail("Investigate",
-						            "Why would backup have two sets with different value at same version");
-					} // else {} Backup has duplicate set at the same version
-				} else {
-					TraceEvent(SevWarnAlways, "FastRestoreApplierStagingKeyMutationAtSameVersionOverride")
-					    .detail("Version", newVersion.toString())
-					    .detail("NewMutation", m.toString())
-					    .detail("ExistingKeyType", typeString[type])
-					    .detail("ExitingKeyValue", val);
-					type = (MutationRef::Type)m.type;
-					val = m.param2;
-				}
-			} else if (m.type == MutationRef::ClearRange) {
-				TraceEvent(SevWarnAlways, "FastRestoreApplierStagingKeyMutationAtSameVersionSkipped")
-				    .detail("Version", newVersion.toString())
-				    .detail("NewMutation", m.toString())
-				    .detail("ExistingKeyType", typeString[type])
-				    .detail("ExitingKeyValue", val);
-			} else {
-				ASSERT(false); // Can't be true same key, same version, different mutation
-			}
-
+			TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m.toString());
+			ASSERT(type == m.type && key == m.param1 && val == m.param2);
 			return;
 		}
 
@@ -112,14 +84,14 @@ struct StagingKey {
 		} else {
 			auto it = pendingMutations.find(newVersion);
 			if (it == pendingMutations.end()) {
-				bool inserted;
-				std::tie(it, inserted) = pendingMutations.emplace(newVersion, MutationsVec());
-				// TODO: Do we really need deep copy?
-				it->second.push_back_deep(it->second.arena(), m);
+				pendingMutations.emplace(newVersion, m);
 			} else {
 				// Duplicated mutation ignored.
-				MutationRef& m1 = *(it->second.begin());
-				ASSERT(m1.type == m.type && m1.param1 == m.param1 && m1.param2 == m.param2);
+				TraceEvent("SameVersion")
+				    .detail("Version", version.toString())
+				    .detail("Mutation", m.toString())
+				    .detail("NewVersion", newVersion.toString());
+				ASSERT(it->second.type == m.type && it->second.param1 == m.param1 && it->second.param2 == m.param2);
 			}
 		}
 	}
@@ -130,52 +102,50 @@ struct StagingKey {
 		    .detail("Key", key)
 		    .detail("Version", version.toString())
 		    .detail("LargestPendingVersion",
-		            (pendingMutations.empty() ? "-1" : pendingMutations.rbegin()->first.toString()));
-		std::map<LogMessageVersion, MutationsVec>::iterator lb = pendingMutations.lower_bound(version);
+		            (pendingMutations.empty() ? "[none]" : pendingMutations.rbegin()->first.toString()));
+		std::map<LogMessageVersion, Standalone<MutationRef>>::iterator lb = pendingMutations.lower_bound(version);
 		if (lb == pendingMutations.end()) {
 			return;
 		}
 		if (lb->first == version) {
 			// Sanity check mutations at version are either atomicOps which can be ignored or the same value as buffered
-			for (const MutationRef& m : lb->second) {
-				if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
-					if (std::tie(type, key, val) != std::tie(m.type, m.param1, m.param2)) {
-						TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnhandledSituation")
-						    .detail("BufferedType", typeString[type])
-						    .detail("PendingType", typeString[m.type])
-						    .detail("BufferedVal", val.toString())
-						    .detail("PendingVal", m.param2.toString());
-					}
+			MutationRef m = lb->second;
+			if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
+				if (std::tie(type, key, val) != std::tie(m.type, m.param1, m.param2)) {
+					TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnhandledSituation")
+					    .detail("BufferedType", typeString[type])
+					    .detail("PendingType", typeString[m.type])
+					    .detail("BufferedVal", val.toString())
+					    .detail("PendingVal", m.param2.toString());
 				}
 			}
 			lb++;
 		}
 		for (; lb != pendingMutations.end(); lb++) {
-			for (auto& mutation : lb->second) {
-				if (type == MutationRef::CompareAndClear) { // Special atomicOp
-					Arena arena;
-					Optional<ValueRef> retVal = doCompareAndClear(val, mutation.param2, arena);
-					if (!retVal.present()) {
-						val = key;
-						type = MutationRef::ClearRange;
-					} // else no-op
-				} else if (isAtomicOp((MutationRef::Type)mutation.type)) {
-					Optional<StringRef> inputVal;
-					if (hasBaseValue()) {
-						inputVal = val;
-					}
-					val = applyAtomicOp(inputVal, mutation.param2, (MutationRef::Type)mutation.type);
-					type = MutationRef::SetValue; // Precomputed result should be set to DB.
-				} else if (mutation.type == MutationRef::SetValue || mutation.type == MutationRef::ClearRange) {
-					type = MutationRef::SetValue; // Precomputed result should be set to DB.
-					TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet")
-					    .detail("MutationType", typeString[mutation.type])
-					    .detail("Version", lb->first.toString());
-				} else {
-					TraceEvent(SevWarnAlways, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation")
-					    .detail("MutationType", typeString[mutation.type])
-					    .detail("Version", lb->first.toString());
+			MutationRef mutation = lb->second;
+			if (type == MutationRef::CompareAndClear) { // Special atomicOp
+				Arena arena;
+				Optional<ValueRef> retVal = doCompareAndClear(val, mutation.param2, arena);
+				if (!retVal.present()) {
+					val = key;
+					type = MutationRef::ClearRange;
+				} // else no-op
+			} else if (isAtomicOp((MutationRef::Type)mutation.type)) {
+				Optional<StringRef> inputVal;
+				if (hasBaseValue()) {
+					inputVal = val;
 				}
+				val = applyAtomicOp(inputVal, mutation.param2, (MutationRef::Type)mutation.type);
+				type = MutationRef::SetValue; // Precomputed result should be set to DB.
+			} else if (mutation.type == MutationRef::SetValue || mutation.type == MutationRef::ClearRange) {
+				type = MutationRef::SetValue; // Precomputed result should be set to DB.
+				TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet")
+				    .detail("MutationType", typeString[mutation.type])
+				    .detail("Version", lb->first.toString());
+			} else {
+				TraceEvent(SevWarnAlways, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation")
+				    .detail("MutationType", typeString[mutation.type])
+				    .detail("Version", lb->first.toString());
 			}
 			version = lb->first;
 		}

From 19f6394dc99746b822d9e7418636dc52837a1794 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 17 Mar 2020 14:45:07 -0700
Subject: [PATCH 054/176] Fix oldest backup epoch for backup workers

The oldest backup epoch is piggybacked in LogSystemConfig from master to
cluster controller and then to all workers. Previously, this epoch is set
to the current master epoch, which is wrong.
---
 fdbserver/BackupWorker.actor.cpp            | 19 +++++++++++++------
 fdbserver/ClusterController.actor.cpp       | 13 +++++++++++--
 fdbserver/TagPartitionedLogSystem.actor.cpp |  4 +++-
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 8452d20d43..ca7be4a40f 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -72,6 +72,7 @@ struct BackupData {
 	const Optional<Version> endVersion; // old epoch's end version (inclusive), or empty for current epoch
 	const LogEpoch recruitedEpoch;
 	const LogEpoch backupEpoch;
+	LogEpoch oldestBackupEpoch = 0;
 	Version minKnownCommittedVersion;
 	Version savedVersion;
 	AsyncVar<Reference<ILogSystem>> logSystem;
@@ -169,13 +170,12 @@ struct BackupData {
 	}
 
 	void pop() {
-		const LogEpoch oldest = logSystem.get()->getOldestBackupEpoch();
-		if (backupEpoch > oldest) {
+		if (backupEpoch > oldestBackupEpoch) {
 			// Defer pop if old epoch hasn't finished popping yet.
 			TraceEvent("BackupWorkerPopDeferred", myId)
 			    .suppressFor(1.0)
 			    .detail("BackupEpoch", backupEpoch)
-			    .detail("OldestEpoch", oldest)
+			    .detail("OldestEpoch", oldestBackupEpoch)
 			    .detail("Version", savedVersion);
 			return;
 		}
@@ -549,6 +549,14 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		MutationRef m;
 		if (!message.isBackupMessage(&m)) continue;
 
+		if (debugMutation("addMutation", message.version.version, m)) {
+			TraceEvent("BackupWorkerDebug", self->myId)
+			    .detail("Version", message.version.toString())
+			    .detail("Mutation", m.toString())
+			    .detail("KCV", self->minKnownCommittedVersion)
+			    .detail("SavedVersion", self->savedVersion);
+		}
+
 		std::vector<Future<Void>> adds;
 		if (m.type != MutationRef::Type::ClearRange) {
 			for (int index : keyRangeMap[m.param1]) {
@@ -801,15 +809,14 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 				dbInfoChange = db->onChange();
 				Reference<ILogSystem> ls = ILogSystem::fromServerDBInfo(self.myId, db->get(), true);
 				bool hasPseudoLocality = ls.isValid() && ls->hasPseudoLocality(tagLocalityBackup);
-				LogEpoch oldestBackupEpoch = 0;
 				if (hasPseudoLocality) {
 					self.logSystem.set(ls);
 					self.pop();
-					oldestBackupEpoch = ls->getOldestBackupEpoch();
+					self.oldestBackupEpoch = std::max(self.oldestBackupEpoch, ls->getOldestBackupEpoch());
 				}
 				TraceEvent("BackupWorkerLogSystem", self.myId)
 				    .detail("HasBackupLocality", hasPseudoLocality)
-				    .detail("OldestBackupEpoch", oldestBackupEpoch)
+				    .detail("OldestBackupEpoch", self.oldestBackupEpoch)
 				    .detail("Tag", self.tag.toString());
 			}
 			when(wait(done)) {
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 28648fbd10..fc00bccbf0 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2052,8 +2052,17 @@ ACTOR Future<Void> clusterRecruitRemoteFromConfiguration( ClusterControllerData*
 void clusterRegisterMaster( ClusterControllerData* self, RegisterMasterRequest const& req ) {
 	req.reply.send( Void() );
 
-	TraceEvent("MasterRegistrationReceived", self->id).detail("MasterId", req.id).detail("Master", req.mi.toString()).detail("Tlogs", describe(req.logSystemConfig.tLogs)).detail("Resolvers", req.resolvers.size())
-		.detail("RecoveryState", (int)req.recoveryState).detail("RegistrationCount", req.registrationCount).detail("Proxies", req.proxies.size()).detail("RecoveryCount", req.recoveryCount).detail("Stalled", req.recoveryStalled);
+	TraceEvent("MasterRegistrationReceived", self->id)
+	    .detail("MasterId", req.id)
+	    .detail("Master", req.mi.toString())
+	    .detail("Tlogs", describe(req.logSystemConfig.tLogs))
+	    .detail("Resolvers", req.resolvers.size())
+	    .detail("RecoveryState", (int)req.recoveryState)
+	    .detail("RegistrationCount", req.registrationCount)
+	    .detail("Proxies", req.proxies.size())
+	    .detail("RecoveryCount", req.recoveryCount)
+	    .detail("Stalled", req.recoveryStalled)
+	    .detail("OldestBackupEpoch", req.logSystemConfig.oldestBackupEpoch);
 
 	//make sure the request comes from an active database
 	auto db = &self->db;
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 66c4db462a..3eed2b2de5 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -202,7 +202,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 	TagPartitionedLogSystem(UID dbgid, LocalityData locality, LogEpoch e,
 	                        Optional<PromiseStream<Future<Void>>> addActor = Optional<PromiseStream<Future<Void>>>())
 	  : dbgid(dbgid), logSystemType(LogSystemType::empty), expectedLogSets(0), logRouterTags(0), txsTags(0),
-	    repopulateRegionAntiQuorum(0), epoch(e), oldestBackupEpoch(e), recoveryCompleteWrittenToCoreState(false),
+	    repopulateRegionAntiQuorum(0), epoch(e), oldestBackupEpoch(0), recoveryCompleteWrittenToCoreState(false),
 	    locality(locality), remoteLogsWrittenToCoreState(false), hasRemoteServers(false), stopped(false),
 	    addActor(addActor), popActors(false) {}
 
@@ -308,6 +308,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 
 		logSystem->logSystemType = lsConf.logSystemType;
+		logSystem->oldestBackupEpoch = lsConf.oldestBackupEpoch;
 		return logSystem;
 	}
 
@@ -1393,6 +1394,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 			}
 			logset->backupWorkers.push_back(worker);
 		}
+		TraceEvent("SetOldestBackupEpoch", dbgid).detail("Epoch", oldestBackupEpoch);
 		backupWorkerChanged.trigger();
 	}
 

From be8c9585c95c126f2504d29af57376ce47ec91f1 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 17 Mar 2020 19:30:49 -0700
Subject: [PATCH 055/176] Skip setting backupStartedKey if using old mutation
 logs

For old submitBackup(), where partitionedLog is false, do not set the
backupStartedKey in BackupConfig, which signals backup workers to skip these
backups.
---
 fdbclient/FileBackupAgent.actor.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 660bb6c526..14b16ee7dd 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -2405,7 +2405,12 @@ namespace fileBackup {
 
 					state Future<Optional<Value>> started = tr->get(backupStartedKey);
 					state Future<Optional<Value>> taskStarted = tr->get(config.allWorkerStarted().key);
-					wait(success(started) && success(taskStarted));
+					state Future<Optional<bool>> partitionedLog = config.partitionedLogEnabled().get(tr);
+					wait(success(started) && success(taskStarted) && success(partitionedLog));
+
+					if (!partitionedLog.get().present() || !partitionedLog.get().get()) {
+						return Void(); // Skip if not using partitioned logs
+					}
 
 					std::vector<std::pair<UID, Version>> ids;
 					if (started.get().present()) {

From 61f8cd25296a800853d8a13d16902c2a6daf8ac0 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 17 Mar 2020 21:35:44 -0700
Subject: [PATCH 056/176] Add an exitEarly flag for backup worker

If a backup worker is on an old epoch, it could exit early if either of the
following is true:
- there is no backups
- all backups starts a version >= the endVersion

If this flag is set, the backup worker exit without doing any work, which
signals the master to update oldest backup epoch.
---
 fdbserver/BackupWorker.actor.cpp | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index ca7be4a40f..4eeccbc95d 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -81,6 +81,7 @@ struct BackupData {
 	NotifiedVersion pulledVersion;
 	bool pulling = false;
 	bool stopped = false;
+	bool exitEarly = false; // If the worker is on an old epoch and all backups starts a version >= the endVersion
 
 	struct PerBackupInfo {
 		PerBackupInfo() = default;
@@ -135,7 +136,7 @@ struct BackupData {
 	}
 
 	bool allMessageSaved() const {
-		return (endVersion.present() && savedVersion >= endVersion.get()) || stopped;
+		return (endVersion.present() && savedVersion >= endVersion.get()) || stopped || exitEarly;
 	}
 
 	Version maxPopVersion() const {
@@ -288,21 +289,27 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				Optional<Value> value = wait(tr.get(backupStartedKey));
 				std::vector<std::pair<UID, Version>> uidVersions;
+				bool shouldExit = self->endVersion.present();
 				if (value.present()) {
 					uidVersions = decodeBackupStartedValue(value.get());
 					TraceEvent e("BackupWorkerGotStartKey", self->myId);
 					int i = 1;
-					for (auto uidVersion : uidVersions) {
-						e.detail(format("BackupID%d", i), uidVersion.first)
-						    .detail(format("Version%d", i), uidVersion.second);
+					for (auto [uid, version] : uidVersions) {
+						e.detail(format("BackupID%d", i), uid)
+						    .detail(format("Version%d", i), version);
 						i++;
+						if (shouldExit && version < self->endVersion.get()) {
+							shouldExit = false;
+						}
 					}
+					self->exitEarly = shouldExit;
 					self->onBackupChanges(uidVersions);
 					if (started || !watch) return true;
 				} else {
 					TraceEvent("BackupWorkerEmptyStartKey", self->myId);
 					self->onBackupChanges(uidVersions);
 
+					self->exitEarly = shouldExit;
 					if (!started || !watch) {
 						return false;
 					}
@@ -797,12 +804,12 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 
 		// Check if backup key is present to avoid race between this check and
 		// noop pop as well as upload data: pop or skip upload before knowing
-		// there are backup keys.
+		// there are backup keys. Set the "exitEarly" flag if needed.
 		bool present = wait(monitorBackupStartedKeyChanges(&self, true, false));
-		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present);
+		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present).detail("ExitEarly", self.exitEarly);
 
-		pull = monitorBackupKeyOrPullData(&self, present);
-		done = uploadData(&self);
+		pull = self.exitEarly ? Void() : monitorBackupKeyOrPullData(&self, present);
+		done = self.exitEarly ? Void() : uploadData(&self);
 
 		loop choose {
 			when(wait(dbInfoChange)) {

From 9a91bb2b9e9a509ad86c90c6e73a1791420ba2a9 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 18 Mar 2020 15:38:06 -0700
Subject: [PATCH 057/176] Add target version as the limit for version batches

If using partitioned logs, the mutations after the target version can be
included if this limit is not considered.
---
 fdbserver/RestoreMaster.actor.cpp | 20 ++++++++++++--------
 fdbserver/RestoreMaster.actor.h   |  6 +++---
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 9b1551e242..95f6f0a95e 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -36,8 +36,9 @@
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 ACTOR static Future<Void> clearDB(Database cx);
-ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
-                                             std::vector<RestoreFileFR>* logFiles, Database cx, RestoreRequest request);
+ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
+                                                std::vector<RestoreFileFR>* logFiles, Database cx,
+                                                RestoreRequest request);
 
 ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData> self, Database cx, RestoreRequest request);
 ACTOR static Future<Void> startProcessRestoreRequests(Reference<RestoreMasterData> self, Database cx);
@@ -276,7 +277,8 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 	self->initBackupContainer(request.url);
 
 	// Get all backup files' description and save them to files
-	wait(collectBackupFiles(self->bc, &rangeFiles, &logFiles, cx, request));
+	Version targetVersion = wait(collectBackupFiles(self->bc, &rangeFiles, &logFiles, cx, request));
+	ASSERT(targetVersion > 0);
 
 	std::sort(rangeFiles.begin(), rangeFiles.end());
 	std::sort(logFiles.begin(), logFiles.end(), [](RestoreFileFR const& f1, RestoreFileFR const& f2) -> bool {
@@ -284,7 +286,8 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 		       std::tie(f2.endVersion, f2.beginVersion, f2.fileIndex, f2.fileName);
 	});
 
-	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches); // Divide files into version batches
+	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches,
+	                          targetVersion); // Divide files into version batches
 	self->dumpVersionBatches(self->versionBatches);
 
 	state std::vector<Future<Void>> fBatches;
@@ -672,9 +675,10 @@ ACTOR static Future<Standalone<VectorRef<RestoreRequest>>> collectRestoreRequest
 }
 
 // Collect the backup files' description into output_files by reading the backupContainer bc.
-ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
-                                             std::vector<RestoreFileFR>* logFiles, Database cx,
-                                             RestoreRequest request) {
+// Returns the restore target version.
+ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
+                                                std::vector<RestoreFileFR>* logFiles, Database cx,
+                                                RestoreRequest request) {
 	state BackupDescription desc = wait(bc->describePartitionedBackup());
 
 	// Convert version to real time for operators to read the BackupDescription desc.
@@ -730,7 +734,7 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 	    .detail("BackupDesc", desc.toString())
 	    .detail("RangeFiles", rangeFiles->size())
 	    .detail("LogFiles", logFiles->size());
-	return Void();
+	return request.targetVersion;
 }
 
 ACTOR static Future<Void> clearDB(Database cx) {
diff --git a/fdbserver/RestoreMaster.actor.h b/fdbserver/RestoreMaster.actor.h
index 4a4520f75d..0731326f58 100644
--- a/fdbserver/RestoreMaster.actor.h
+++ b/fdbserver/RestoreMaster.actor.h
@@ -255,7 +255,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 	// Note: We do not allow a versionBatch size larger than the FASTRESTORE_VERSIONBATCH_MAX_BYTES because the range
 	// file size at a version depends on the number of backupAgents and its upper bound is hard to get.
 	void buildVersionBatches(const std::vector<RestoreFileFR>& rangeFiles, const std::vector<RestoreFileFR>& logFiles,
-	                         std::map<Version, VersionBatch>* versionBatches) {
+	                         std::map<Version, VersionBatch>* versionBatches, Version targetVersion) {
 		bool rewriteNextVersion = false;
 		int rangeIdx = 0;
 		int logIdx = 0; // Ensure each log file is included in version batch
@@ -340,7 +340,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 					vb.logFiles.insert(log);
 				}
 
-				vb.endVersion = nextVersion;
+				vb.endVersion = std::min(nextVersion, targetVersion + 1);
 				prevEndVersion = vb.endVersion;
 			} else {
 				if (vb.size < 1) {
@@ -376,7 +376,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 		}
 		// The last wip version batch has some files
 		if (vb.size > 0) {
-			vb.endVersion = nextVersion;
+			vb.endVersion = std::min(nextVersion, targetVersion + 1);
 			versionBatches->emplace(vb.beginVersion, vb);
 		}
 	}

From 5bf62c8f85d49261a8e87c2304a57aa83c7f757d Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 19 Mar 2020 10:08:19 -0700
Subject: [PATCH 058/176] Reduce a call to getLogSystemConfig()

---
 fdbserver/masterserver.actor.cpp | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index cda864a732..8f73d5ad6b 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -478,13 +478,20 @@ ACTOR Future<Void> updateRegistration( Reference<MasterData> self, Reference<ILo
 
 		trigger = self->registrationTrigger.onTrigger();
 
-		TraceEvent("MasterUpdateRegistration", self->dbgid).detail("RecoveryCount", self->cstate.myDBState.recoveryCount).detail("Logs", describe(logSystem->getLogSystemConfig().tLogs));
+		auto logSystemConfig = logSystem->getLogSystemConfig();
+		TraceEvent("MasterUpdateRegistration", self->dbgid)
+		    .detail("RecoveryCount", self->cstate.myDBState.recoveryCount)
+		    .detail("OldestBackupEpoch", logSystemConfig.oldestBackupEpoch)
+		    .detail("Logs", describe(logSystemConfig.tLogs));
 
 		if (!self->cstateUpdated.isSet()) {
-			wait(sendMasterRegistration(self.getPtr(), logSystem->getLogSystemConfig(), self->provisionalProxies, self->resolvers, self->cstate.myDBState.recoveryCount, self->cstate.prevDBState.getPriorCommittedLogServers() ));
+			wait(sendMasterRegistration(self.getPtr(), logSystemConfig, self->provisionalProxies, self->resolvers,
+			                            self->cstate.myDBState.recoveryCount,
+			                            self->cstate.prevDBState.getPriorCommittedLogServers()));
 		} else {
 			updateLogsKey = updateLogsValue(self, cx);
-			wait( sendMasterRegistration( self.getPtr(), logSystem->getLogSystemConfig(), self->proxies, self->resolvers, self->cstate.myDBState.recoveryCount, vector<UID>() ) );
+			wait(sendMasterRegistration(self.getPtr(), logSystemConfig, self->proxies, self->resolvers,
+			                            self->cstate.myDBState.recoveryCount, vector<UID>()));
 		}
 	}
 }

From 8bdda0fe041ffc190aba1db6e1950306c208ca81 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 19 Mar 2020 14:59:38 -0700
Subject: [PATCH 059/176] Backup Worker: Give a chance of saving progress
 before displaced

Move the exit loop after the saving of progress so that when doneTrigger is
active, we won't exit the loop immediately.
---
 fdbserver/BackupWorker.actor.cpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 4eeccbc95d..71af661932 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -420,7 +420,8 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 						    .detail("Prev", prev)
 						    .detail("Current", current);
 					}
-					if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
+					if (self->backupEpoch == self->oldestBackupEpoch &&
+					    (!prevVersions[i].get().present() || prevVersions[i].get().get() < current)) {
 						TraceEvent("BackupWorkerSetVersion", self->myId)
 						    .detail("BackupID", versionConfigs[i].getUid())
 						    .detail("Version", current);
@@ -615,11 +616,6 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 	state Version popVersion = invalidVersion;
 
 	loop {
-		if (self->allMessageSaved()) {
-			self->messages.clear();
-			return Void();
-		}
-
 		// FIXME: knobify the delay of 10s. This delay is sensitive, as it is the
 		// lag TLog might have. Changing to 20s may fail consistency check.
 		state Future<Void> uploadDelay = delay(10);
@@ -663,6 +659,11 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			self->pop();
 		}
 
+		if (self->allMessageSaved()) {
+			self->messages.clear();
+			return Void();
+		}
+
 		if (!self->pullFinished()) {
 			wait(uploadDelay || self->doneTrigger.onTrigger());
 		}
@@ -841,6 +842,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		if (e.code() == error_code_worker_removed) {
 			pull = Void(); // cancels pulling
 			self.stopped = true;
+			self.doneTrigger.trigger();
 			wait(done);
 		}
 		TraceEvent("BackupWorkerTerminated", self.myId).error(err, true);

From 3801e502885b5b3819a6dd47fefed8662b14dd0d Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 5 Feb 2020 14:23:54 -0800
Subject: [PATCH 060/176] Backup worker: enable 50% of time in simulation

Make this randomization a separate one.
---
 fdbserver/SimulatedCluster.actor.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 46ab8fb8ff..e548fa7db3 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -816,7 +816,6 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR
 		set_config(format("log_spill:=%d", logSpill));
 		int logVersion = deterministicRandom()->randomInt( TLogVersion::MIN_RECRUITABLE, TLogVersion::MAX_SUPPORTED+1 );
 		set_config(format("log_version:=%d", logVersion));
-		set_config("backup_worker_enabled:=1");
 	} else {
 		if (deterministicRandom()->random01() < 0.7)
 			set_config(format("log_version:=%d", TLogVersion::MAX_SUPPORTED));
@@ -824,6 +823,10 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR
 			set_config(format("log_spill:=%d", TLogSpillType::DEFAULT));
 	}
 
+	if (deterministicRandom()->random01() < 0.5) {
+		set_config("backup_worker_enabled:=1");
+	}
+
 	if(generateFearless || (datacenters == 2 && deterministicRandom()->random01() < 0.5)) {
 		//The kill region workload relies on the fact that all "0", "2", and "4" are all of the possible primary dcids.
 		StatusObject primaryObj;

From d3071409c5d5a72ce8b1b9ed6f1559d579bf86a8 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Mon, 3 Feb 2020 16:17:39 -0800
Subject: [PATCH 061/176] FastRestore:Add comment for integrating with new
 backup format

---
 fdbserver/RestoreLoader.actor.cpp   | 14 +++++++++++---
 fdbserver/RestoreMaster.actor.cpp   |  2 ++
 fdbserver/RestoreRoleCommon.actor.h |  2 ++
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 70eedf06f0..bf75158c4f 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -612,6 +612,11 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 }
 
 // Parsing the data blocks in a range file
+// kvOpsIter: saves the parsed versioned-mutations for the sepcific LoadingParam;
+// samplesIter: saves the sampled mutations from the parsed versioned-mutations;
+// bc: backup container to read the backup file
+// version: the version the parsed mutations should be at
+// asset: RestoreAsset about which backup data should be parsed
 ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
     std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
     std::map<LoadingParam, MutationsVec>::iterator samplesIter, LoaderCounters* cc, Reference<IBackupContainer> bc,
@@ -691,9 +696,12 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 	return Void();
 }
 
-// Parse data blocks in a log file into a vector of <string, string> pairs. Each pair.second contains the mutations at a
-// version encoded in pair.first Step 1: decodeLogFileBlock into <string, string> pairs Step 2: Concatenate the
-// pair.second of pairs with the same pair.first.
+// Parse data blocks in a log file into a vector of <string, string> pairs.
+// Each pair.second contains the mutations at a version encoded in pair.first;
+// Step 1: decodeLogFileBlock into <string, string> pairs;
+// Step 2: Concatenate the second of pairs with the same pair.first.
+// pProcessedFileOffset: ensure each data block is processed in order exactly once;
+// pMutationMap: concatenated mutation list string at the mutation's commit version
 ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(NotifiedVersion* pProcessedFileOffset,
                                                            SerializedMutationListMap* pMutationMap,
                                                            SerializedMutationPartMap* pMutationPartMap,
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 59a3c1d63c..a216ba08b4 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -276,6 +276,7 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 	self->initBackupContainer(request.url);
 
 	// Get all backup files' description and save them to files
+	// TODO for Jingyu: Verify all backup files in new backup are collected
 	wait(collectBackupFiles(self->bc, &rangeFiles, &logFiles, cx, request));
 
 	std::sort(rangeFiles.begin(), rangeFiles.end());
@@ -284,6 +285,7 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 		       std::tie(f2.endVersion, f2.beginVersion, f2.fileIndex, f2.fileName);
 	});
 
+	// TODO for Jingyu: Verify new backup files are grouped into correct version batches.
 	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches); // Divide files into version batches
 	self->dumpVersionBatches(self->versionBatches);
 
diff --git a/fdbserver/RestoreRoleCommon.actor.h b/fdbserver/RestoreRoleCommon.actor.h
index 531e652a7e..8679a5e0a2 100644
--- a/fdbserver/RestoreRoleCommon.actor.h
+++ b/fdbserver/RestoreRoleCommon.actor.h
@@ -51,6 +51,8 @@ struct RestoreMasterData;
 
 struct RestoreSimpleRequest;
 
+// VersionedMutationsMap: Key is the version of parsed backup mutations
+// Value MutationsVec is the vector of parsed backup mutations
 using VersionedMutationsMap = std::map<Version, MutationsVec>;
 
 ACTOR Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name);

From ec352c03c9fa971785eef1bb49e4b4490327561c Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 13 Feb 2020 15:40:39 -0800
Subject: [PATCH 062/176] Add partitioned logs to BackupContainer

---
 fdbclient/BackupContainer.actor.cpp | 112 +++++++++++++++++-----------
 1 file changed, 67 insertions(+), 45 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 53ddf397df..b097ee692d 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -240,9 +240,13 @@ std::string BackupDescription::toJSON() const {
  *     file written will be after the start version of the snapshot's execution.
  * 
  *   Log files are at file paths like
- *       /logs/.../log,startVersion,endVersion,blockSize
+ *       /plogs/...log,startVersion,endVersion,UID,blocksize,tagID
+ *       /logs/.../log,startVersion,endVersion,UID,blockSize
  *     where ... is a multi level path which sorts lexically into version order and results in approximately 1
- *     unique folder per day containing about 5,000 files.
+ *     unique folder per day containing about 5,000 files. Logs after 7.0 are stored in "plogs"
+ *     directory and are partitioned according to tagIDs (0, 1, 2, ...). Logs before 7.0 are
+ *     stored in "logs" directory and are not partitioned.
+ *
  *
  *   BACKWARD COMPATIBILITY
  *
@@ -329,18 +333,18 @@ public:
 	}
 
 	// The innermost folder covers 100,000 seconds (1e11 versions) which is 5,000 mutation log files at current settings.
-	static std::string logVersionFolderString(Version v, bool mlogs) {
-		return format("%s/%s/", (mlogs ? "mlogs" : "logs"), versionFolderString(v, 11).c_str());
+	static std::string logVersionFolderString(Version v, bool partitioned) {
+		return format("%s/%s/", (partitioned ? "plogs" : "logs"), versionFolderString(v, 11).c_str());
 	}
 
-	Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) override {
+	Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) final {
 		return writeFile(logVersionFolderString(beginVersion, false) +
 		                 format("log,%lld,%lld,%s,%d", beginVersion, endVersion,
 		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize));
 	}
 
 	Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
-	                                                  uint16_t tagId) override {
+	                                                  uint16_t tagId) final {
 		return writeFile(logVersionFolderString(beginVersion, true) +
 		                 format("log,%lld,%lld,%s,%d,%d", beginVersion, endVersion,
 		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize, tagId));
@@ -528,18 +532,19 @@ public:
 		return writeKeyspaceSnapshotFile_impl(Reference<BackupContainerFileSystem>::addRef(this), fileNames, totalBytes);
 	};
 
-	// List log files, unsorted, which contain data at any version >= beginVersion and <= targetVersion
-	Future<std::vector<LogFile>> listLogFiles(Version beginVersion = 0, Version targetVersion = std::numeric_limits<Version>::max()) {
-		// The first relevant log file could have a begin version less than beginVersion based on the knobs which determine log file range size,
-		// so start at an earlier version adjusted by how many versions a file could contain.
+	// List log files, unsorted, which contain data at any version >= beginVersion and <= targetVersion.
+	// "partitioned" flag indicates if new partitioned mutation logs or old logs should be listed.
+	Future<std::vector<LogFile>> listLogFiles(Version beginVersion, Version targetVersion, bool partitioned) {
+		// The first relevant log file could have a begin version less than beginVersion based on the knobs which
+		// determine log file range size, so start at an earlier version adjusted by how many versions a file could
+		// contain.
 		//
 		// Get the cleaned (without slashes) first and last folders that could contain relevant results.
-		bool mlogs = false; // tagged mutation logs
 		std::string firstPath = cleanFolderString(
 		    logVersionFolderString(std::max<Version>(0, beginVersion - CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES *
 		                                                                   CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE),
-		                           mlogs));
-		std::string lastPath = cleanFolderString(logVersionFolderString(targetVersion, mlogs));
+		                           partitioned));
+		std::string lastPath = cleanFolderString(logVersionFolderString(targetVersion, partitioned));
 
 		std::function<bool(std::string const &)> pathFilter = [=](const std::string &folderPath) {
 			// Remove slashes in the given folder path so that the '/' positions in the version folder string do not matter
@@ -549,7 +554,7 @@ public:
 				|| (cleaned > firstPath && cleaned < lastPath);
 		};
 
-		return map(listFiles("logs/", pathFilter), [=](const FilesAndSizesT &files) {
+		return map(listFiles((partitioned ? "plogs/" : "logs/"), pathFilter), [=](const FilesAndSizesT& files) {
 			std::vector<LogFile> results;
 			LogFile lf;
 			for(auto &f : files) {
@@ -636,11 +641,15 @@ public:
 	ACTOR static Future<BackupFileList> dumpFileList_impl(Reference<BackupContainerFileSystem> bc, Version begin, Version end) {
 		state Future<std::vector<RangeFile>> fRanges = bc->listRangeFiles(begin, end);
 		state Future<std::vector<KeyspaceSnapshotFile>> fSnapshots = bc->listKeyspaceSnapshots(begin, end);
-		state Future<std::vector<LogFile>> fLogs = bc->listLogFiles(begin, end);
+		state std::vector<LogFile> logs;
+		state std::vector<LogFile> pLogs;
 
-		wait(success(fRanges) && success(fSnapshots) && success(fLogs));
+		wait(success(fRanges) && success(fSnapshots) &&
+		     store(logs, bc->listLogFiles(begin, end, false)) &&
+		     store(pLogs, bc->listLogFiles(begin, end, true)));
+		logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
-		return BackupFileList({fRanges.get(), fLogs.get(), fSnapshots.get()});
+		return BackupFileList({ fRanges.get(), std::move(logs), fSnapshots.get() });
 	}
 
 	Future<BackupFileList> dumpFileList(Version begin, Version end) override {
@@ -767,7 +776,12 @@ public:
 		}
 
 		state std::vector<LogFile> logs;
-		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd)) && store(desc.snapshots, bc->listKeyspaceSnapshots()));
+		state std::vector<LogFile> pLogs;
+		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) &&
+		     store(pLogs, bc->listLogFiles(scanBegin, scanEnd, true)) &&
+		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
+		// FIXME: check partitioned logs & maybe enable the below line
+		// logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
 		// List logs in version order so log continuity can be analyzed
 		std::sort(logs.begin(), logs.end());
@@ -879,8 +893,10 @@ public:
 		state BackupDescription desc = wait(bc->describeBackup(false, expireEndVersion));
 
 		// Resolve relative versions using max log version
-		expireEndVersion =       resolveRelativeVersion(desc.maxLogEnd, expireEndVersion,       "ExpireEndVersion",       invalid_option_value());
-		restorableBeginVersion = resolveRelativeVersion(desc.maxLogEnd, restorableBeginVersion, "RestorableBeginVersion", invalid_option_value());
+		expireEndVersion =
+		    resolveRelativeVersion(desc.maxLogEnd, expireEndVersion, "ExpireEndVersion", invalid_option_value());
+		restorableBeginVersion = resolveRelativeVersion(desc.maxLogEnd, restorableBeginVersion,
+		                                                "RestorableBeginVersion", invalid_option_value());
 
 		// It would be impossible to have restorability to any version < expireEndVersion after expiring to that version
 		if(restorableBeginVersion < expireEndVersion)
@@ -921,13 +937,17 @@ public:
 			.detail("ScanBeginVersion", scanBegin);
 
 		state std::vector<LogFile> logs;
+		state std::vector<LogFile> pLogs; // partitioned mutation logs
 		state std::vector<RangeFile> ranges;
 
 		if(progress != nullptr) {
 			progress->step = "Listing files";
 		}
 		// Get log files or range files that contain any data at or before expireEndVersion
-		wait(store(logs, bc->listLogFiles(scanBegin, expireEndVersion - 1)) && store(ranges, bc->listRangeFiles(scanBegin, expireEndVersion - 1)));
+		wait(store(logs, bc->listLogFiles(scanBegin, expireEndVersion - 1, false)) &&
+		     store(pLogs, bc->listLogFiles(scanBegin, expireEndVersion - 1, true)) &&
+		     store(ranges, bc->listRangeFiles(scanBegin, expireEndVersion - 1)));
+		logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
 		// The new logBeginVersion will be taken from the last log file, if there is one
 		state Optional<Version> newLogBeginVersion;
@@ -1067,7 +1087,8 @@ public:
 				return Optional<RestorableFileSet>(restorable);
 			}
 
-			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion));
+			// FIXME: check if there are tagged logs. for each tag, there is no version gap.
+			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false));
 
 			// List logs in version order so log continuity can be analyzed
 			std::sort(logs.begin(), logs.end());
@@ -1098,7 +1119,7 @@ public:
 		return Optional<RestorableFileSet>();
 	}
 
-	Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) override {
+	Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) final {
 		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion);
 	}
 
@@ -1183,8 +1204,8 @@ public:
 
 class BackupContainerLocalDirectory : public BackupContainerFileSystem, ReferenceCounted<BackupContainerLocalDirectory> {
 public:
-	void addref() override { return ReferenceCounted<BackupContainerLocalDirectory>::addref(); }
-	void delref() override { return ReferenceCounted<BackupContainerLocalDirectory>::delref(); }
+	void addref() final { return ReferenceCounted<BackupContainerLocalDirectory>::addref(); }
+	void delref() final { return ReferenceCounted<BackupContainerLocalDirectory>::delref(); }
 
 	static std::string getURLFormat() { return "file://</path/to/base/dir/>"; }
 
@@ -1233,7 +1254,7 @@ public:
 		return results;
 	}
 
-	Future<Void> create() override {
+	Future<Void> create() final {
 		// Nothing should be done here because create() can be called by any process working with the container URL, such as fdbbackup.
 		// Since "local directory" containers are by definition local to the machine they are accessed from,
 		// the container's creation (in this case the creation of a directory) must be ensured prior to every file creation,
@@ -1243,11 +1264,11 @@ public:
 	}
 
 	// The container exists if the folder it resides in exists
-	Future<bool> exists() override {
+	Future<bool> exists() final {
 		return directoryExists(m_path);
 	}
 
-	Future<Reference<IAsyncFile>> readFile(std::string path) override {
+	Future<Reference<IAsyncFile>> readFile(std::string path) final {
 		int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED;
 		// Simulation does not properly handle opening the same file from multiple machines using a shared filesystem,
 		// so create a symbolic link to make each file opening appear to be unique.  This could also work in production
@@ -1272,10 +1293,10 @@ public:
 			int blockSize = 0;
 			// Extract block size from the filename, if present
 			size_t lastComma = path.find_last_of(',');
-			if(lastComma != path.npos) {
+			if (lastComma != path.npos) {
 				blockSize = atoi(path.substr(lastComma + 1).c_str());
 			}
-			if(blockSize <= 0) {
+			if (blockSize <= 0) {
 				blockSize = deterministicRandom()->randomInt(1e4, 1e6);
 			}
 			if(deterministicRandom()->random01() < .01) {
@@ -1324,7 +1345,7 @@ public:
 		std::string m_finalFullPath;
 	};
 
-	Future<Reference<IBackupFile>> writeFile(std::string path) override {
+	Future<Reference<IBackupFile>> writeFile(std::string path) final {
 		int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE;
 		std::string fullPath = joinPath(m_path, path);
 		platform::createDirectory(parentDirectory(fullPath));
@@ -1335,12 +1356,12 @@ public:
 		});
 	}
 
-	Future<Void> deleteFile(std::string path) override {
+	Future<Void> deleteFile(std::string path) final {
 		::deleteFile(joinPath(m_path, path));
 		return Void();
 	}
 
-	Future<FilesAndSizesT> listFiles(std::string path, std::function<bool(std::string const &)>) {
+	Future<FilesAndSizesT> listFiles(std::string path, std::function<bool(std::string const&)>) final {
 		FilesAndSizesT results;
 
 		std::vector<std::string> files;
@@ -1360,7 +1381,7 @@ public:
 		return results;
 	}
 
-	Future<Void> deleteContainer(int* pNumDeleted) override {
+	Future<Void> deleteContainer(int* pNumDeleted) final {
 		// In order to avoid deleting some random directory due to user error, first describe the backup
 		// and make sure it has something in it.
 		return map(describeBackup(false, invalidVersion), [=](BackupDescription const &desc) {
@@ -1420,8 +1441,8 @@ public:
 		}
 	}
 
-	void addref() override { return ReferenceCounted<BackupContainerBlobStore>::addref(); }
-	void delref() override { return ReferenceCounted<BackupContainerBlobStore>::delref(); }
+	void addref() final { return ReferenceCounted<BackupContainerBlobStore>::addref(); }
+	void delref() final { return ReferenceCounted<BackupContainerBlobStore>::delref(); }
 
 	static std::string getURLFormat() {
 		return BlobStoreEndpoint::getURLFormat(true) + " (Note: The 'bucket' parameter is required.)";
@@ -1429,7 +1450,7 @@ public:
 
 	virtual ~BackupContainerBlobStore() {}
 
-	Future<Reference<IAsyncFile>> readFile(std::string path) override {
+	Future<Reference<IAsyncFile>> readFile(std::string path) final {
 			return Reference<IAsyncFile>(
 				new AsyncFileReadAheadCache(
 					Reference<IAsyncFile>(new AsyncFileBlobStoreRead(m_bstore, m_bucket, dataPath(path))),
@@ -1466,17 +1487,18 @@ public:
 			return map(m_file->sync(), [=](Void _) { self->m_file.clear(); return Void(); });
 		}
 
-		void addref() override { return ReferenceCounted<BackupFile>::addref(); }
-		void delref() override { return ReferenceCounted<BackupFile>::delref(); }
+		void addref() final { return ReferenceCounted<BackupFile>::addref(); }
+		void delref() final { return ReferenceCounted<BackupFile>::delref(); }
+
 	private:
 		Reference<IAsyncFile> m_file;
 	};
 
-	Future<Reference<IBackupFile>> writeFile(std::string path) override {
+	Future<Reference<IBackupFile>> writeFile(std::string path) final {
 		return Reference<IBackupFile>(new BackupFile(path, Reference<IAsyncFile>(new AsyncFileBlobStoreWrite(m_bstore, m_bucket, dataPath(path)))));
 	}
 
-	Future<Void> deleteFile(std::string path) override {
+	Future<Void> deleteFile(std::string path) final {
 		return m_bstore->deleteObject(m_bucket, dataPath(path));
 	}
 
@@ -1498,7 +1520,7 @@ public:
 		return files;
 	}
 
-	Future<FilesAndSizesT> listFiles(std::string path, std::function<bool(std::string const &)> pathFilter) {
+	Future<FilesAndSizesT> listFiles(std::string path, std::function<bool(std::string const &)> pathFilter) final {
 		return listFiles_impl(Reference<BackupContainerBlobStore>::addRef(this), path, pathFilter);
 	}
 
@@ -1514,12 +1536,12 @@ public:
 		return Void();
 	}
 
-	Future<Void> create() override {
+	Future<Void> create() final {
 		return create_impl(Reference<BackupContainerBlobStore>::addRef(this));
 	}
 
 	// The container exists if the index entry in the blob bucket exists
-	Future<bool> exists() override {
+	Future<bool> exists() final {
 		return m_bstore->objectExists(m_bucket, indexEntry());
 	}
 
@@ -1539,7 +1561,7 @@ public:
 		return Void();
 	}
 
-	Future<Void> deleteContainer(int* pNumDeleted) override {
+	Future<Void> deleteContainer(int* pNumDeleted) final {
 		return deleteContainer_impl(Reference<BackupContainerBlobStore>::addRef(this), pNumDeleted);
 	}
 

From e15015ee6c72a2a1fe219e17b99b23ea12f1b052 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 14 Feb 2020 11:27:02 -0800
Subject: [PATCH 063/176] Add mutation log version names

I.e., BACKUP_AGENT_MLOG_VERSION for 2001 and PARTITIONED_MLOG_VERSION for 4110.
---
 fdbbackup/FileConverter.actor.cpp   |  3 +++
 fdbbackup/FileConverter.h           |  3 ---
 fdbbackup/FileDecoder.actor.cpp     |  2 +-
 fdbclient/BackupContainer.h         |  6 ++++++
 fdbclient/FileBackupAgent.actor.cpp | 12 ++++++------
 fdbserver/BackupWorker.actor.cpp    |  3 ++-
 fdbserver/RestoreCommon.actor.cpp   |  4 ++--
 fdbserver/RestoreLoader.actor.cpp   |  2 +-
 8 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp
index 46beea723b..f0bffa73e1 100644
--- a/fdbbackup/FileConverter.actor.cpp
+++ b/fdbbackup/FileConverter.actor.cpp
@@ -162,6 +162,9 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
 			Version msgVersion = invalidVersion;
 
 			try {
+				// Read block header
+				if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
+
 				while (1) {
 					// If eof reached or first key len bytes is 0xFF then end of block was reached.
 					if (reader.eof() || *reader.rptr == 0xFF) break;
diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index a342a41dd8..fc82e5dfb2 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -59,7 +59,4 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 
 }  // namespace file_converter
 
-// Mutation log version written by old FileBackupAgent
-static const uint32_t BACKUP_AGENT_MLOG_VERSION = 2001;
-
 #endif  // FDBBACKUP_FILECONVERTER_H
diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index ab4257885c..02b98e4825 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -290,7 +290,7 @@ struct DecodeProgress {
 		StringRefReader reader(block, restore_corrupted_data());
 
 		try {
-			// Read header, currently only decoding version 2001
+			// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
 			if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version();
 
 			// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 4d1ec5ecbe..7bc4ed70e5 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -62,6 +62,12 @@ protected:
 
 // Structures for various backup components
 
+// Mutation log version written by old FileBackupAgent
+static const uint32_t BACKUP_AGENT_MLOG_VERSION = 2001;
+
+// Mutation log version written by BackupWorker
+static const uint32_t PARTITIONED_MLOG_VERSION = 4110;
+
 struct LogFile {
 	Version beginVersion;
 	Version endVersion;
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 0eec26fa8a..8c58cbc162 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -612,7 +612,8 @@ namespace fileBackup {
 	struct LogFileWriter {
 		static const std::string &FFs;
 
-		LogFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0) : file(file), blockSize(blockSize), blockEnd(0), fileVersion(2001) {}
+		LogFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0)
+		  : file(file), blockSize(blockSize), blockEnd(0) {}
 
 		// Start a new block if needed, then write the key and value
 		ACTOR static Future<Void> writeKV_impl(LogFileWriter *self, Key k, Value v) {
@@ -629,8 +630,8 @@ namespace fileBackup {
 				// Set new blockEnd
 				self->blockEnd += self->blockSize;
 
-				// write Header
-				wait(self->file->append((uint8_t *)&self->fileVersion, sizeof(self->fileVersion)));
+				// write the block header
+				wait(self->file->append((uint8_t *)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION)));
 			}
 
 			wait(self->file->appendStringRefWithLen(k));
@@ -650,7 +651,6 @@ namespace fileBackup {
 
 	private:
 		int64_t blockEnd;
-		uint32_t fileVersion;
 	};
 
 	ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file, int64_t offset, int len) {
@@ -663,8 +663,8 @@ namespace fileBackup {
 		state StringRefReader reader(buf, restore_corrupted_data());
 
 		try {
-			// Read header, currently only decoding version 2001
-			if(reader.consume<int32_t>() != 2001)
+			// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
+			if(reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
 				throw restore_unsupported_file_version();
 
 			// Read k/v pairs.  Block ends either at end of last value exactly or with 0xFF as first key len byte.
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index f34dd89e3d..9cbbbc0659 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -373,7 +373,8 @@ ACTOR Future<Void> addMutation(Reference<IBackupFile> logFile, VersionedMessage
 		}
 
 		*blockEnd += blockSize;
-		// TODO: add block header
+		// write block Header
+		wait(logFile->append((uint8_t*)&PARTITIONED_MLOG_VERSION, sizeof(PARTITIONED_MLOG_VERSION)));
 	}
 
 	wait(logFile->append((void*)header.begin(), header.size()));
diff --git a/fdbserver/RestoreCommon.actor.cpp b/fdbserver/RestoreCommon.actor.cpp
index 0c336538da..7ea783c04a 100644
--- a/fdbserver/RestoreCommon.actor.cpp
+++ b/fdbserver/RestoreCommon.actor.cpp
@@ -396,8 +396,8 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IA
 	state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
 
 	try {
-		// Read header, currently only decoding version 2001
-		if (reader.consume<int32_t>() != 2001) throw restore_unsupported_file_version();
+		// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
+		if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version();
 
 		// Read k/v pairs.  Block ends either at end of last value exactly or with 0xFF as first key len byte.
 		while (1) {
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index bf75158c4f..22646b307a 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

From 88ad28e5761a25b6d2c4b5e9c2d1c2f4dd0ce3cd Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 17 Feb 2020 14:36:09 -0800
Subject: [PATCH 064/176] Integrate parallel restore with partitioned logs

In parallel restore, use new getPartitionedRestoreSet() to get a set containing
partitioned mutation logs. The loader uses a new parser to extract mutations
from partitioned logs.

TODO: fix unable to restore errors.
---
 fdbclient/BackupContainer.actor.cpp      |  14 ++-
 fdbclient/BackupContainer.h              |   5 +
 fdbclient/RestoreWorkerInterface.actor.h |  18 ++--
 fdbserver/RestoreCommon.actor.h          |  26 +++--
 fdbserver/RestoreLoader.actor.cpp        | 119 ++++++++++++++++++++---
 fdbserver/RestoreMaster.actor.cpp        |  15 +--
 fdbserver/RestoreMaster.actor.h          |   2 +-
 7 files changed, 153 insertions(+), 46 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index b097ee692d..98215aabcd 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1064,7 +1064,7 @@ public:
 		return expireData_impl(Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
 	}
 
-	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion) {
+	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion, bool partitioned) {
 		// Find the most recent keyrange snapshot to end at or before targetVersion
 		state Optional<KeyspaceSnapshotFile> snapshot;
 		std::vector<KeyspaceSnapshotFile> snapshots = wait(bc->listKeyspaceSnapshots());
@@ -1088,11 +1088,15 @@ public:
 			}
 
 			// FIXME: check if there are tagged logs. for each tag, there is no version gap.
-			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false));
+			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion, partitioned));
 
 			// List logs in version order so log continuity can be analyzed
 			std::sort(logs.begin(), logs.end());
 
+			// TODO(jingyu): for partitioned logs, the continuity checking should be based on
+			// epochs and versions, which should be saved in a metadata file by backup worker and
+			// thus is available here.
+
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
 			if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {
 				auto i = logs.begin();
@@ -1120,7 +1124,11 @@ public:
 	}
 
 	Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) final {
-		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion);
+		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion, false);
+	}
+
+	Future<Optional<RestorableFileSet>> getPartitionedRestoreSet(Version targetVersion) final {
+		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion, true);
 	}
 
 private:
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 7bc4ed70e5..9c4526e6f4 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -260,6 +260,11 @@ public:
 	// restore to given version is not possible.
 	virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) = 0;
 
+	// Get exactly the files necessary to restore to targetVersion. Returns non-present if
+	// restore to given version is not possible. This is intended for parallel
+	// restore in FDB 7.0, which reads partitioned mutation logs.
+	virtual Future<Optional<RestorableFileSet>> getPartitionedRestoreSet(Version targetVersion) = 0;
+
 	// Get an IBackupContainer based on a container spec string
 	static Reference<IBackupContainer> openContainer(std::string url);
 	static std::vector<std::string> getURLFormats();
diff --git a/fdbclient/RestoreWorkerInterface.actor.h b/fdbclient/RestoreWorkerInterface.actor.h
index 684a12c44e..84c2f603e8 100644
--- a/fdbclient/RestoreWorkerInterface.actor.h
+++ b/fdbclient/RestoreWorkerInterface.actor.h
@@ -209,6 +209,8 @@ struct RestoreAsset {
 	KeyRange range; // Only use mutations in range
 
 	int fileIndex;
+	// Partition ID for mutation log files, which is also encoded in the filename of mutation logs.
+	int partitionId = -1;
 	std::string filename;
 	int64_t offset;
 	int64_t len;
@@ -218,12 +220,12 @@ struct RestoreAsset {
 	RestoreAsset() = default;
 
 	bool operator==(const RestoreAsset& r) const {
-		return fileIndex == r.fileIndex && filename == r.filename && offset == r.offset && len == r.len &&
-		       beginVersion == r.beginVersion && endVersion == r.endVersion && range == r.range;
+		return beginVersion == r.beginVersion && endVersion == r.endVersion && range == r.range &&
+		       fileIndex == r.fileIndex && partitionId == r.partitionId && filename == r.filename &&
+		       offset == r.offset && len == r.len;
 	}
 	bool operator!=(const RestoreAsset& r) const {
-		return fileIndex != r.fileIndex || filename != r.filename || offset != r.offset || len != r.len ||
-		       beginVersion != r.beginVersion || endVersion != r.endVersion || range != r.range;
+		return !(*this == r);
 	}
 	bool operator<(const RestoreAsset& r) const {
 		return std::make_tuple(fileIndex, filename, offset, len, beginVersion, endVersion, range.begin, range.end) <
@@ -233,14 +235,14 @@ struct RestoreAsset {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, beginVersion, endVersion, range, filename, fileIndex, offset, len, uid);
+		serializer(ar, beginVersion, endVersion, range, filename, fileIndex, partitionId, offset, len, uid);
 	}
 
 	std::string toString() {
 		std::stringstream ss;
 		ss << "UID:" << uid.toString() << " begin:" << beginVersion << " end:" << endVersion
 		   << " range:" << range.toString() << " filename:" << filename << " fileIndex:" << fileIndex
-		   << " offset:" << offset << " len:" << len;
+		   << " partitionId:" << partitionId << " offset:" << offset << " len:" << len;
 		return ss.str();
 	}
 
@@ -269,6 +271,10 @@ struct LoadingParam {
 		return (isRangeFile < r.isRangeFile) || (isRangeFile == r.isRangeFile && asset < r.asset);
 	}
 
+	bool isPartitionedLog() const {
+		return !isRangeFile && asset.partitionId >= 0;
+	}
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, isRangeFile, url, rangeVersion, blockSize, asset);
diff --git a/fdbserver/RestoreCommon.actor.h b/fdbserver/RestoreCommon.actor.h
index 72d86d8d49..ea0e54837d 100644
--- a/fdbserver/RestoreCommon.actor.h
+++ b/fdbserver/RestoreCommon.actor.h
@@ -188,6 +188,7 @@ struct RestoreFileFR {
 	int64_t cursor; // The start block location to be restored. All blocks before cursor have been scheduled to load and
 	                // restore
 	int fileIndex; // index of backup file. Must be identical per file.
+	int partitionId = -1; // Partition ID (Log Router Tag ID) for mutation files.
 
 	Tuple pack() const {
 		return Tuple()
@@ -199,7 +200,8 @@ struct RestoreFileFR {
 		    .append(endVersion)
 		    .append(beginVersion)
 		    .append(cursor)
-		    .append(fileIndex);
+		    .append(fileIndex)
+		    .append(partitionId);
 	}
 	static RestoreFileFR unpack(Tuple const& t) {
 		RestoreFileFR r;
@@ -213,6 +215,7 @@ struct RestoreFileFR {
 		r.beginVersion = t.getInt(i++);
 		r.cursor = t.getInt(i++);
 		r.fileIndex = t.getInt(i++);
+		r.partitionId = t.getInt(i++);
 		return r;
 	}
 
@@ -225,18 +228,21 @@ struct RestoreFileFR {
 	  : version(invalidVersion), isRange(false), blockSize(0), fileSize(0), endVersion(invalidVersion),
 	    beginVersion(invalidVersion), cursor(0), fileIndex(0) {}
 
-	RestoreFileFR(Version version, std::string fileName, bool isRange, int64_t blockSize, int64_t fileSize,
-	              Version endVersion, Version beginVersion)
-	  : version(version), fileName(fileName), isRange(isRange), blockSize(blockSize), fileSize(fileSize),
-	    endVersion(endVersion), beginVersion(beginVersion), cursor(0), fileIndex(0) {}
+	explicit RestoreFileFR(const RangeFile& f)
+	  : version(f.version), fileName(f.fileName), isRange(true), blockSize(f.blockSize), fileSize(f.fileSize),
+	    endVersion(f.version), beginVersion(f.version), cursor(0), fileIndex(0) {}
+
+	explicit RestoreFileFR(const LogFile& f)
+	  : version(f.beginVersion), fileName(f.fileName), isRange(false), blockSize(f.blockSize), fileSize(f.fileSize),
+	    endVersion(f.endVersion), beginVersion(f.beginVersion), cursor(0), fileIndex(0), partitionId(f.tagId) {}
 
 	std::string toString() const {
 		std::stringstream ss;
-		ss << "version:" << std::to_string(version) << " fileName:" << fileName
-		   << " isRange:" << std::to_string(isRange) << " blockSize:" << std::to_string(blockSize)
-		   << " fileSize:" << std::to_string(fileSize) << " endVersion:" << std::to_string(endVersion)
-		   << " beginVersion:" << std::to_string(beginVersion) << " cursor:" << std::to_string(cursor)
-		   << " fileIndex:" << std::to_string(fileIndex);
+		ss << "version:" << version << " fileName:" << fileName
+		   << " isRange:" << isRange << " blockSize:" << blockSize
+		   << " fileSize:" << fileSize << " endVersion:" << endVersion
+		   << " beginVersion:" << beginVersion << " cursor:" << cursor
+		   << " fileIndex:" << fileIndex << " partitionId:" << partitionId;
 		return ss.str();
 	}
 };
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 22646b307a..79bfb057d3 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -23,6 +23,7 @@
 
 #include "fdbclient/BackupContainer.h"
 #include "fdbserver/RestoreLoader.actor.h"
+#include "fdbserver/RestoreRoleCommon.actor.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -140,6 +141,90 @@ void handleRestoreSysInfoRequest(const RestoreSysInfoRequest& req, Reference<Res
 	req.reply.send(RestoreCommonReply(self->id()));
 }
 
+// Parse a data block in a partitioned mutation log file and store mutations
+// into "kvOpsIter" and samples into "samplesIter".
+ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
+    NotifiedVersion* processedFileOffset, std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
+    std::map<LoadingParam, MutationsVec>::iterator samplesIter, Reference<IBackupContainer> bc, RestoreAsset asset) {
+	state Standalone<StringRef> buf = makeString(asset.len);
+	state Reference<IAsyncFile> file = wait(bc->readFile(asset.filename));
+	int rLen = wait(file->read(mutateString(buf), asset.len, asset.offset));
+	if (rLen != asset.len) throw restore_bad_read();
+
+	TraceEvent("FastRestore")
+	    .detail("DecodingLogFile", asset.filename)
+	    .detail("Offset", asset.offset)
+	    .detail("Length", asset.len);
+
+	// Ensure data blocks in the same file are processed in order
+	wait(processedFileOffset->whenAtLeast(asset.offset));
+	ASSERT(processedFileOffset->get() == asset.offset);
+
+	BackupStringRefReader reader(buf, restore_corrupted_data());
+	try {
+		// Read block header
+		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
+
+		Version lastVersion = invalidVersion;
+		VersionedMutationsMap& kvOps = kvOpsIter->second;
+		VersionedMutationsMap::iterator it = kvOps.end();
+		while (1) {
+			// If eof reached or first key len bytes is 0xFF then end of block was reached.
+			if (reader.eof() || *reader.rptr == 0xFF) break;
+
+			// Deserialize messages written in saveMutationsToFile().
+			Version msgVersion = bigEndian64(reader.consume<Version>());
+			uint32_t sub = bigEndian32(reader.consume<uint32_t>());
+			int msgSize = bigEndian32(reader.consume<int>());
+			const uint8_t* message = reader.consume(msgSize);
+
+			// Skip mutations out of the version range
+			if (!asset.isInVersionRange(msgVersion)) continue;
+
+			if (lastVersion != msgVersion) {
+				bool inserted;
+				std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec());
+				lastVersion = msgVersion;
+			}
+			ASSERT(it != kvOps.end());
+
+			ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(currentProtocolVersion));
+			MutationRef mutation;
+			rd >> mutation;
+
+			// Should this mutation be skipped?
+			if (mutation.param1 >= asset.range.end ||
+			    (isRangeMutation(mutation) && mutation.param2 < asset.range.begin) ||
+			    (!isRangeMutation(mutation) && mutation.param1 < asset.range.begin)) {
+				continue;
+			}
+			// Only apply mutation within the asset.range
+			if (isRangeMutation(mutation)) {
+				mutation.param1 = mutation.param1 >= asset.range.begin ? mutation.param1 : asset.range.begin;
+				mutation.param2 = mutation.param2 < asset.range.end ? mutation.param2 : asset.range.end;
+			}
+
+			TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
+			    .detail("CommitVersion", msgVersion)
+			    .detail("ParsedMutation", mutation.toString());
+			it->second.push_back_deep(it->second.arena(), mutation);
+			// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
+			if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {
+				samplesIter->second.push_back_deep(samplesIter->second.arena(), mutation);
+			}
+		}
+	} catch (Error& e) {
+		TraceEvent(SevWarn, "FileRestoreCorruptLogFileBlock")
+		    .error(e)
+		    .detail("Filename", file->getFilename())
+		    .detail("BlockOffset", asset.offset)
+		    .detail("BlockLen", asset.len);
+		throw;
+	}
+	processedFileOffset->set(asset.offset + asset.len);
+	return Void();
+}
+
 ACTOR Future<Void> _processLoadingParam(LoadingParam param, Reference<LoaderBatchData> batchData, UID loaderID,
                                         Reference<IBackupContainer> bc) {
 	// Temporary data structure for parsing log files into (version, <K, V, mutationType>)
@@ -155,15 +240,15 @@ ACTOR Future<Void> _processLoadingParam(LoadingParam param, Reference<LoaderBatc
 	// Q: How to record the  param's fields inside LoadingParam Refer to storageMetrics
 	TraceEvent("FastRestoreLoaderProcessLoadingParam", loaderID).detail("LoadingParam", param.toString());
 	ASSERT(param.blockSize > 0);
-	ASSERT(param.asset.offset % param.blockSize == 0); // Parse file must be at block bondary.
+	ASSERT(param.asset.offset % param.blockSize == 0); // Parse file must be at block boundary.
 	ASSERT(batchData->kvOpsPerLP.find(param) == batchData->kvOpsPerLP.end());
 
 	// NOTE: map's iterator is guaranteed to be stable, but pointer may not.
-	// state VersionedMutationsMap* kvOps = &self->kvOpsPerLP[param];
-	batchData->kvOpsPerLP.emplace(param, VersionedMutationsMap());
-	batchData->sampleMutations.emplace(param, MutationsVec());
-	kvOpsPerLPIter = batchData->kvOpsPerLP.find(param);
-	samplesIter = batchData->sampleMutations.find(param);
+	bool inserted;
+	std::tie(kvOpsPerLPIter, inserted) = batchData->kvOpsPerLP.emplace(param, VersionedMutationsMap());
+	ASSERT(inserted);
+	std::tie(samplesIter, inserted) = batchData->sampleMutations.emplace(param, MutationsVec());
+	ASSERT(inserted);
 
 	for (int64_t j = param.asset.offset; j < param.asset.len; j += param.blockSize) {
 		RestoreAsset subAsset = param.asset;
@@ -174,13 +259,18 @@ ACTOR Future<Void> _processLoadingParam(LoadingParam param, Reference<LoaderBatc
 			    kvOpsPerLPIter, samplesIter, &batchData->counters, bc, param.rangeVersion.get(), subAsset));
 		} else {
 			// TODO: Sanity check the log file's range is overlapped with the restored version range
-			fileParserFutures.push_back(
-			    _parseLogFileToMutationsOnLoader(&processedFileOffset, &mutationMap, &mutationPartMap, bc, subAsset));
+			if (param.isPartitionedLog()) {
+				fileParserFutures.push_back(_parsePartitionedLogFileOnLoader(&processedFileOffset, kvOpsPerLPIter,
+				                                                             samplesIter, bc, subAsset));
+			} else {
+				fileParserFutures.push_back(_parseLogFileToMutationsOnLoader(&processedFileOffset, &mutationMap,
+				                                                             &mutationPartMap, bc, subAsset));
+			}
 		}
 	}
 	wait(waitForAll(fileParserFutures));
 
-	if (!param.isRangeFile) {
+	if (!param.isRangeFile && !param.isPartitionedLog()) {
 		_parseSerializedMutation(kvOpsPerLPIter, &mutationMap, samplesIter, &batchData->counters, param.asset);
 	}
 
@@ -508,15 +598,15 @@ bool concatenateBackupMutationForLogFile(std::map<Standalone<StringRef>, Standal
 	// Use commitVersion as id
 	Standalone<StringRef> id = StringRef((uint8_t*)&commitVersion, sizeof(Version));
 
-	if (mutationMap.find(id) == mutationMap.end()) {
+	auto it = mutationMap.find(id);
+	if (it == mutationMap.end()) {
 		mutationMap.insert(std::make_pair(id, val_input));
 		if (part != 0) {
 			TraceEvent(SevError, "FastRestore").detail("FirstPartNotZero", part).detail("KeyInput", getHexString(key_input));
 		}
 		mutationPartMap.insert(std::make_pair(id, part));
 	} else { // Concatenate the val string with the same commitVersion
-		mutationMap[id] =
-		    mutationMap[id].contents().withSuffix(val_input.contents()); // Assign the new Areana to the map's value
+		it->second = it->second.contents().withSuffix(val_input.contents()); // Assign the new Areana to the map's value
 		if (part != (mutationPartMap[id] + 1)) {
 			// Check if the same range or log file has been processed more than once!
 			TraceEvent(SevError, "FastRestore")
@@ -722,14 +812,11 @@ ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(NotifiedVersion* pPro
 	if (pProcessedFileOffset->get() == asset.offset) {
 		int start = 0;
 		int end = data.size();
-		int numConcatenated = 0;
 		for (int i = start; i < end; ++i) {
 			// Key k = data[i].key.withPrefix(mutationLogPrefix);
 			// ValueRef v = data[i].value;
 			// Concatenate the backuped param1 and param2 (KV) at the same version.
-			bool concatenated =
-			    concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, data[i].key, data[i].value, asset);
-			numConcatenated += (concatenated ? 1 : 0);
+			concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, data[i].key, data[i].value, asset);
 		}
 		pProcessedFileOffset->set(asset.offset + asset.len);
 	}
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index a216ba08b4..51c168cc5f 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -276,7 +276,6 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 	self->initBackupContainer(request.url);
 
 	// Get all backup files' description and save them to files
-	// TODO for Jingyu: Verify all backup files in new backup are collected
 	wait(collectBackupFiles(self->bc, &rangeFiles, &logFiles, cx, request));
 
 	std::sort(rangeFiles.begin(), rangeFiles.end());
@@ -337,12 +336,7 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<MasterBatchData> batchDat
                                              Database cx, RestoreRequest request, VersionBatch versionBatch,
                                              bool isRangeFile) {
 	// set is internally sorted
-	std::set<RestoreFileFR>* files = nullptr;
-	if (isRangeFile) {
-		files = &versionBatch.rangeFiles;
-	} else {
-		files = &versionBatch.logFiles;
-	}
+	std::set<RestoreFileFR>* files = isRangeFile ? &versionBatch.rangeFiles : &versionBatch.logFiles;
 
 	TraceEvent("FastRestoreMasterPhaseLoadFilesStart")
 	    .detail("BatchIndex", batchIndex)
@@ -376,6 +370,7 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<MasterBatchData> batchDat
 		param.asset.uid = deterministicRandom()->randomUniqueID();
 		param.asset.filename = file.fileName;
 		param.asset.fileIndex = file.fileIndex;
+		param.asset.partitionId = file.partitionId;
 		param.asset.offset = 0;
 		param.asset.len = file.fileSize;
 		param.asset.range = request.range;
@@ -692,7 +687,7 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 		request.targetVersion = desc.maxRestorableVersion.get();
 	}
 
-	Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(request.targetVersion));
+	Optional<RestorableFileSet> restorable = wait(bc->getPartitionedRestoreSet(request.targetVersion));
 
 	if (!restorable.present()) {
 		TraceEvent(SevWarn, "FastRestoreMasterPhaseCollectBackupFiles").detail("NotRestorable", request.targetVersion);
@@ -709,7 +704,7 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 		if (f.fileSize <= 0) {
 			continue;
 		}
-		RestoreFileFR file(f.version, f.fileName, true, f.blockSize, f.fileSize, f.version, f.version);
+		RestoreFileFR file(f);
 		TraceEvent("FastRestoreMasterPhaseCollectBackupFiles").detail("RangeFileFR", file.toString());
 		uniqueRangeFiles.insert(file);
 	}
@@ -718,7 +713,7 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 		if (f.fileSize <= 0) {
 			continue;
 		}
-		RestoreFileFR file(f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion, f.beginVersion);
+		RestoreFileFR file(f);
 		TraceEvent("FastRestoreMasterPhaseCollectBackupFiles").detail("LogFileFR", file.toString());
 		logFiles->push_back(file);
 		uniqueLogFiles.insert(file);
diff --git a/fdbserver/RestoreMaster.actor.h b/fdbserver/RestoreMaster.actor.h
index 434e4d6f0d..fce748cc5b 100644
--- a/fdbserver/RestoreMaster.actor.h
+++ b/fdbserver/RestoreMaster.actor.h
@@ -251,7 +251,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 	// Split backup files into version batches, each of which has similar data size
 	// Input: sorted range files, sorted log files;
 	// Output: a set of version batches whose size is less than SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES
-	//    	   and each mutation in backup files is included in the version batches exactly once.
+	//         and each mutation in backup files is included in the version batches exactly once.
 	// Assumption 1: input files has no empty files;
 	// Assumption 2: range files at one version <= FASTRESTORE_VERSIONBATCH_MAX_BYTES.
 	// Note: We do not allow a versionBatch size larger than the FASTRESTORE_VERSIONBATCH_MAX_BYTES because the range

From 35aafefb896b7f025e75f0a754c22c1a82fc50e8 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 18 Feb 2020 13:21:29 -0800
Subject: [PATCH 065/176] Consolidate StringRefReader classes

Fix a compiler error of unused variable too.
---
 fdbclient/BackupAgent.actor.h       |  4 +++
 fdbserver/RestoreCommon.actor.cpp   | 48 +++++------------------------
 fdbserver/RestoreLoader.actor.cpp   | 10 +++---
 fdbserver/RestoreRoleCommon.actor.h | 47 ----------------------------
 4 files changed, 17 insertions(+), 92 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index ae6717c619..9ef90976d1 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -880,6 +880,10 @@ struct StringRefReader {
 	const int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
 	const uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
 
+	// Convert big Endian value (e.g., encoded in log file) into a littleEndian uint64_t value.
+	int64_t consumeNetworkInt64() { return (int64_t)bigEndian64((uint32_t)consume<int64_t>()); }
+	uint64_t consumeNetworkUInt64() { return bigEndian64(consume<uint64_t>()); }
+
 	bool eof() { return rptr == end; }
 
 	const uint8_t *rptr, *end;
diff --git a/fdbserver/RestoreCommon.actor.cpp b/fdbserver/RestoreCommon.actor.cpp
index 7ea783c04a..e5776e97d8 100644
--- a/fdbserver/RestoreCommon.actor.cpp
+++ b/fdbserver/RestoreCommon.actor.cpp
@@ -23,15 +23,15 @@
 
 #include "fdbserver/RestoreCommon.actor.h"
 
+// Backup agent header
+#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupContainer.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/MutationList.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
 
-// Backup agent header
-#include "fdbclient/BackupAgent.actor.h"
-//#include "FileBackupAgent.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/MutationList.h"
-#include "fdbclient/BackupContainer.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // Split RestoreConfigFR defined in FileBackupAgent.actor.cpp to declaration in Restore.actor.h and implementation in
@@ -296,38 +296,6 @@ std::string RestoreConfigFR::toString() {
 // The implementation of parallelFileRestore is copied from FileBackupAgent.actor.cpp
 // parallelFileRestore is copied from FileBackupAgent.actor.cpp for the same reason as RestoreConfigFR is copied
 namespace parallelFileRestore {
-// Helper class for reading restore data from a buffer and throwing the right errors.
-struct StringRefReader {
-	StringRefReader(StringRef s = StringRef(), Error e = Error()) : rptr(s.begin()), end(s.end()), failure_error(e) {}
-
-	// Return remainder of data as a StringRef
-	StringRef remainder() { return StringRef(rptr, end - rptr); }
-
-	// Return a pointer to len bytes at the current read position and advance read pos
-	const uint8_t* consume(unsigned int len) {
-		if (rptr == end && len != 0) throw end_of_stream();
-		const uint8_t* p = rptr;
-		rptr += len;
-		if (rptr > end) throw failure_error;
-		return p;
-	}
-
-	// Return a T from the current read position and advance read pos
-	template <typename T>
-	const T consume() {
-		return *(const T*)consume(sizeof(T));
-	}
-
-	// Functions for consuming big endian (network byte order) integers.
-	// Consumes a big endian number, swaps it to little endian, and returns it.
-	int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
-	uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
-
-	bool eof() { return rptr == end; }
-
-	const uint8_t *rptr, *end;
-	Error failure_error;
-};
 
 ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset,
                                                                       int len) {
@@ -336,7 +304,7 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
 	if (rLen != len) throw restore_bad_read();
 
 	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
-	state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
+	state StringRefReader reader(buf, restore_corrupted_data());
 
 	try {
 		// Read header, currently only decoding version 1001
@@ -393,7 +361,7 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IA
 	if (rLen != len) throw restore_bad_read();
 
 	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
-	state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
+	state StringRefReader reader(buf, restore_corrupted_data());
 
 	try {
 		// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 79bfb057d3..7eb3b0726c 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -160,7 +160,7 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 	wait(processedFileOffset->whenAtLeast(asset.offset));
 	ASSERT(processedFileOffset->get() == asset.offset);
 
-	BackupStringRefReader reader(buf, restore_corrupted_data());
+	StringRefReader reader(buf, restore_corrupted_data());
 	try {
 		// Read block header
 		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
@@ -174,7 +174,7 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 
 			// Deserialize messages written in saveMutationsToFile().
 			Version msgVersion = bigEndian64(reader.consume<Version>());
-			uint32_t sub = bigEndian32(reader.consume<uint32_t>());
+			bigEndian32(reader.consume<uint32_t>()); // subsequence number
 			int msgSize = bigEndian32(reader.consume<int>());
 			const uint8_t* message = reader.consume(msgSize);
 
@@ -576,7 +576,7 @@ bool concatenateBackupMutationForLogFile(std::map<Standalone<StringRef>, Standal
 	std::map<Standalone<StringRef>, uint32_t>& mutationPartMap = *pMutationPartMap;
 	const int key_prefix_len = sizeof(uint8_t) + sizeof(Version) + sizeof(uint32_t);
 
-	BackupStringRefReader readerKey(key_input, restore_corrupted_data()); // read key_input!
+	StringRefReader readerKey(key_input, restore_corrupted_data()); // read key_input!
 	int logRangeMutationFirstLength = key_input.size() - key_prefix_len;
 	bool concatenated = false;
 
@@ -646,13 +646,13 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 		StringRef k = m.first.contents();
 		StringRef val = m.second.contents();
 
-		BackupStringRefReader kReader(k, restore_corrupted_data());
+		StringRefReader kReader(k, restore_corrupted_data());
 		uint64_t commitVersion = kReader.consume<uint64_t>(); // Consume little Endian data
 		// We have already filter the commit not in [beginVersion, endVersion) when we concatenate kv pair in log file
 		ASSERT_WE_THINK(asset.isInVersionRange(commitVersion));
 		kvOps.insert(std::make_pair(commitVersion, MutationsVec()));
 
-		BackupStringRefReader vReader(val, restore_corrupted_data());
+		StringRefReader vReader(val, restore_corrupted_data());
 		vReader.consume<uint64_t>(); // Consume the includeVersion
 		// TODO(xumengpanda): verify the protocol version is compatible and raise error if needed
 
diff --git a/fdbserver/RestoreRoleCommon.actor.h b/fdbserver/RestoreRoleCommon.actor.h
index 8679a5e0a2..9ddbc3d82e 100644
--- a/fdbserver/RestoreRoleCommon.actor.h
+++ b/fdbserver/RestoreRoleCommon.actor.h
@@ -60,53 +60,6 @@ ACTOR Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id);
 ACTOR Future<Void> handleInitVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self);
 void handleFinishRestoreRequest(const RestoreFinishRequest& req, Reference<RestoreRoleData> self);
 
-// Helper class for reading restore data from a buffer and throwing the right errors.
-// This struct is mostly copied from StringRefReader. We add a sanity check in this struct.
-// We want to decouple code between fast restore and old restore. So we keep this duplicate struct
-struct BackupStringRefReader {
-	BackupStringRefReader(StringRef s = StringRef(), Error e = Error())
-	  : rptr(s.begin()), end(s.end()), failure_error(e), str_size(s.size()) {}
-
-	// Return remainder of data as a StringRef
-	StringRef remainder() { return StringRef(rptr, end - rptr); }
-
-	// Return a pointer to len bytes at the current read position and advance read pos
-	// Consume a little-Endian data. Since we only run on little-Endian machine, the data on storage is little Endian
-	const uint8_t* consume(unsigned int len) {
-		if (rptr == end && len != 0) throw end_of_stream();
-		const uint8_t* p = rptr;
-		rptr += len;
-		if (rptr > end) {
-			printf("[ERROR] BackupStringRefReader throw error! string length:%d\n", str_size);
-			printf("!!!!!!!!!!!![ERROR]!!!!!!!!!!!!!! Worker may die due to the error. Master will stuck when a worker "
-			       "die\n");
-			throw failure_error;
-		}
-		return p;
-	}
-
-	// Return a T from the current read position and advance read pos
-	template <typename T>
-	const T consume() {
-		return *(const T*)consume(sizeof(T));
-	}
-
-	// Functions for consuming big endian (network byte oselfer) integers.
-	// Consumes a big endian number, swaps it to little endian, and returns it.
-	int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
-	uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
-
-	// Convert big Endian value (e.g., encoded in log file) into a littleEndian uint64_t value.
-	int64_t consumeNetworkInt64() { return (int64_t)bigEndian64((uint32_t)consume<int64_t>()); }
-	uint64_t consumeNetworkUInt64() { return bigEndian64(consume<uint64_t>()); }
-
-	bool eof() { return rptr == end; }
-
-	const uint8_t *rptr, *end;
-	const int str_size;
-	Error failure_error;
-};
-
 class RoleVersionBatchState {
 public:
 	static const int INVALID = -1;

From 6b9b93314e201d00048f20ec3c426679689e611b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 19 Feb 2020 11:24:17 -0800
Subject: [PATCH 066/176] Check block padding is \0xff for new mutation logs

---
 fdbserver/RestoreLoader.actor.cpp |  5 +++++
 fdbserver/RestoreMaster.actor.h   | 14 ++++++--------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 7eb3b0726c..3f15aebac1 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -213,6 +213,11 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 				samplesIter->second.push_back_deep(samplesIter->second.arena(), mutation);
 			}
 		}
+
+		// Make sure any remaining bytes in the block are 0xFF
+		for (auto b : reader.remainder()) {
+			if (b != 0xFF) throw restore_corrupted_data_padding();
+		}
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "FileRestoreCorruptLogFileBlock")
 		    .error(e)
diff --git a/fdbserver/RestoreMaster.actor.h b/fdbserver/RestoreMaster.actor.h
index fce748cc5b..4a4520f75d 100644
--- a/fdbserver/RestoreMaster.actor.h
+++ b/fdbserver/RestoreMaster.actor.h
@@ -231,19 +231,17 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 			}
 			++rangeIdx;
 		}
-		int logIdx = 0;
 		std::vector<RestoreFileFR> retLogs;
 		// Scan all logFiles every time to avoid assumption on log files' version ranges.
 		// For example, we do not assume each version range only exists in one log file
-		while (logIdx < logFiles.size()) {
-			Version begin = std::max(prevVersion, logFiles[logIdx].beginVersion);
-			Version end = std::min(nextVersion, logFiles[logIdx].endVersion);
+		for (const auto& file : logFiles) {
+			Version begin = std::max(prevVersion, file.beginVersion);
+			Version end = std::min(nextVersion, file.endVersion);
 			if (begin < end) { // logIdx file overlap in [prevVersion, nextVersion)
-				double ratio = (end - begin) * 1.0 / (logFiles[logIdx].endVersion - logFiles[logIdx].beginVersion);
-				size += logFiles[logIdx].fileSize * ratio;
-				retLogs.push_back(logFiles[logIdx]);
+				double ratio = (end - begin) * 1.0 / (file.endVersion - file.beginVersion);
+				size += file.fileSize * ratio;
+				retLogs.push_back(file);
 			}
-			++logIdx;
 		}
 		return std::make_tuple(size, rangeIdx, retLogs);
 	}

From 940bea102afb235fe8558405091a75eae3f898fc Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 19 Feb 2020 15:50:12 -0800
Subject: [PATCH 067/176] Add a knob to switch mutation logs for parallel
 restore

Knob FASTRESTORE_USE_PARTITIONED_LOGS, default is true to enable partitioned
mutation logs. Otherwise, old mutation logs are used.
---
 fdbserver/Knobs.cpp               | 1 +
 fdbserver/Knobs.h                 | 1 +
 fdbserver/RestoreMaster.actor.cpp | 6 ++++--
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp
index c44b21181f..392de76c2c 100644
--- a/fdbserver/Knobs.cpp
+++ b/fdbserver/Knobs.cpp
@@ -569,6 +569,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs, bool isSimula
 	init( FASTRESTORE_APPLYING_PARALLELISM,               	     100 ); if( randomize ) { FASTRESTORE_APPLYING_PARALLELISM = deterministicRandom()->random01() * 10 + 1; }
 	init( FASTRESTORE_MONITOR_LEADER_DELAY,                        5 ); if( randomize ) { FASTRESTORE_MONITOR_LEADER_DELAY = deterministicRandom()->random01() * 100; }
 	init( FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS,                60 ); if( randomize && BUGGIFY ) { FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS = deterministicRandom()->random01() * 240 + 10; }
+	init( FASTRESTORE_USE_PARTITIONED_LOGS,                     true );
 	init( FASTRESTORE_TRACK_REQUEST_LATENCY,              	    true ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_REQUEST_LATENCY = false; }
 	init( FASTRESTORE_TRACK_LOADER_SEND_REQUESTS,              false ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_LOADER_SEND_REQUESTS = true; }
 	init( FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT,                 6144 ); if( randomize && BUGGIFY ) { FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT = 1; }
diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h
index 8b9b09ca24..f32ac22bc0 100644
--- a/fdbserver/Knobs.h
+++ b/fdbserver/Knobs.h
@@ -511,6 +511,7 @@ public:
 	int64_t FASTRESTORE_APPLYING_PARALLELISM; // number of outstanding txns writing to dest. DB
 	int64_t FASTRESTORE_MONITOR_LEADER_DELAY;
 	int64_t FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS;
+	bool FASTRESTORE_USE_PARTITIONED_LOGS;
 	bool FASTRESTORE_TRACK_REQUEST_LATENCY; // true to track reply latency of each request in a request batch
 	bool FASTRESTORE_TRACK_LOADER_SEND_REQUESTS; // track requests of load send mutations to appliers?
 	int64_t FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT; // threshold when pipelined actors should be delayed
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 51c168cc5f..2fba9204d2 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -284,7 +284,6 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 		       std::tie(f2.endVersion, f2.beginVersion, f2.fileIndex, f2.fileName);
 	});
 
-	// TODO for Jingyu: Verify new backup files are grouped into correct version batches.
 	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches); // Divide files into version batches
 	self->dumpVersionBatches(self->versionBatches);
 
@@ -686,8 +685,11 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 	if (request.targetVersion == invalidVersion && desc.maxRestorableVersion.present()) {
 		request.targetVersion = desc.maxRestorableVersion.get();
 	}
+	TraceEvent("FastRestore").detail("TargetVersion", request.targetVersion).detail("BackupDesc", desc.toString());
 
-	Optional<RestorableFileSet> restorable = wait(bc->getPartitionedRestoreSet(request.targetVersion));
+	Optional<RestorableFileSet> restorable =
+	    wait(SERVER_KNOBS->FASTRESTORE_USE_PARTITIONED_LOGS ? bc->getPartitionedRestoreSet(request.targetVersion)
+	                                                        : bc->getRestoreSet(request.targetVersion));
 
 	if (!restorable.present()) {
 		TraceEvent(SevWarn, "FastRestoreMasterPhaseCollectBackupFiles").detail("NotRestorable", request.targetVersion);

From 64859467e40980e75130e65023f51a7ac780531f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 20 Feb 2020 14:35:09 -0800
Subject: [PATCH 068/176] Return partitioned logs for RestorableFileSet

---
 fdbclient/BackupContainer.actor.cpp | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 98215aabcd..9397be7b1d 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1093,9 +1093,24 @@ public:
 			// List logs in version order so log continuity can be analyzed
 			std::sort(logs.begin(), logs.end());
 
-			// TODO(jingyu): for partitioned logs, the continuity checking should be based on
-			// epochs and versions, which should be saved in a metadata file by backup worker and
-			// thus is available here.
+			if (partitioned) {
+				// Remove duplicated log files that can happen for old epochs.
+				std::vector<LogFile> filtered;
+				int i = 0;
+				for (int j = 1; j < logs.size(); j++) {
+					if (!logs[i].sameContent(logs[j])) {
+						filtered.push_back(logs[i]);
+						i = j;
+					}
+				}
+				if (i < logs.size()) filtered.push_back(logs[i]);
+
+				// TODO(jingyu): for partitioned logs, the continuity checking should be based on
+				// epochs and versions, which should be saved in a metadata file by backup worker and
+				// thus is available here. For now, assume it's continuous.
+				restorable.logs.swap(filtered);
+				return Optional<RestorableFileSet>(restorable);
+			}
 
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
 			if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {

From fda6c086404bb189bb7c6b797c1fc4a82d1dd266 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 20 Feb 2020 16:28:27 -0800
Subject: [PATCH 069/176] Include a total number of tags in partition log file
 names

This is needed for BackupContainer to check partitioned mutation logs are
continuous, i.e., restorable to a version.
---
 fdbclient/BackupContainer.actor.cpp | 14 +++++++-------
 fdbclient/BackupContainer.h         |  3 ++-
 fdbserver/BackupProgress.actor.cpp  | 18 ++++++++++--------
 fdbserver/BackupProgress.actor.h    |  6 ++++--
 fdbserver/BackupWorker.actor.cpp    | 10 ++++++----
 fdbserver/WorkerInterface.actor.h   |  3 ++-
 fdbserver/masterserver.actor.cpp    | 11 +++++++----
 7 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 9397be7b1d..3b9ce2389e 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -344,10 +344,11 @@ public:
 	}
 
 	Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
-	                                                  uint16_t tagId) final {
+	                                                  uint16_t tagId, int totalTags) final {
 		return writeFile(logVersionFolderString(beginVersion, true) +
-		                 format("log,%lld,%lld,%s,%d,%d", beginVersion, endVersion,
-		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize, tagId));
+		                 format("log,%lld,%lld,%s,%d,%d-of-%d", beginVersion, endVersion,
+		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize, tagId,
+		                        totalTags));
 	}
 
 	Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) override {
@@ -398,8 +399,8 @@ public:
 		if(sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n", &f.beginVersion, &f.endVersion, &f.blockSize, &len) == 3 && len == name.size()) {
 			out = f;
 			return true;
-		} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u,%d%n", &f.beginVersion, &f.endVersion,
-		                  &f.blockSize, &f.tagId, &len) == 4 &&
+		} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u,%d-of-%d%n", &f.beginVersion,
+		                  &f.endVersion, &f.blockSize, &f.tagId, &f.totalTags, &len) == 5 &&
 		           len == name.size() && f.tagId >= 0) {
 			out = f;
 			return true;
@@ -488,7 +489,6 @@ public:
 	ACTOR static Future<Void> writeKeyspaceSnapshotFile_impl(Reference<BackupContainerFileSystem> bc, std::vector<std::string> fileNames, int64_t totalBytes) {
 		ASSERT(!fileNames.empty());
 
-
 		state Version minVer = std::numeric_limits<Version>::max();
 		state Version maxVer = 0;
 		state RangeFile rf;
@@ -528,7 +528,7 @@ public:
 		return Void();
 	}
 
-	Future<Void> writeKeyspaceSnapshotFile(std::vector<std::string> fileNames, int64_t totalBytes) override {
+	Future<Void> writeKeyspaceSnapshotFile(std::vector<std::string> fileNames, int64_t totalBytes) final {
 		return writeKeyspaceSnapshotFile_impl(Reference<BackupContainerFileSystem>::addRef(this), fileNames, totalBytes);
 	};
 
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 9c4526e6f4..437f6e3eaa 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -75,6 +75,7 @@ struct LogFile {
 	std::string fileName;
 	int64_t fileSize;
 	int tagId = -1; // Log router tag. Non-negative for new backup format.
+	int totalTags = -1; // Total number of log router tags.
 
 	// Order by beginVersion, break ties with endVersion
 	bool operator< (const LogFile &rhs) const {
@@ -220,7 +221,7 @@ public:
 
 	// Open a tagged log file for writing, where tagId is the log router tag's id.
 	virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
-	                                                          uint16_t tagId) = 0;
+	                                                          uint16_t tagId, int totalTags) = 0;
 
 	// Write a KeyspaceSnapshotFile of range file names representing a full non overlapping
 	// snapshot of the key ranges this backup is targeting.
diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 0e7ccbaaa9..5492db7aa8 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -37,8 +37,8 @@ void BackupProgress::addBackupStatus(const WorkerBackupStatus& status) {
 	}
 }
 
-std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> BackupProgress::getUnfinishedBackup() {
-	std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> toRecruit;
+std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgress::getUnfinishedBackup() {
+	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit;
 
 	if (!backupStartedValue.present()) return toRecruit; // No active backups
 
@@ -68,7 +68,7 @@ std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> BackupProgress::g
 			    .detail("EndVersion", info.epochEnd);
 		}
 		if (!tagVersions.empty()) {
-			toRecruit[{ epoch, info.epochEnd }] = tagVersions;
+			toRecruit[{ epoch, info.epochEnd, info.logRouterTags }] = tagVersions;
 		}
 	}
 	return toRecruit;
@@ -115,11 +115,12 @@ TEST_CASE("/BackupProgress/Unfinished") {
 	BackupProgress progress(UID(0, 0), epochInfos);
 	progress.setBackupStartedValue(Optional<Value>(LiteralStringRef("1")));
 
-	std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> unfinished = progress.getUnfinishedBackup();
+	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> unfinished = progress.getUnfinishedBackup();
 
 	ASSERT(unfinished.size() == 1);
-	for (const auto [epochVersion, tagVersion] : unfinished) {
-		ASSERT(epochVersion.first == epoch1 && epochVersion.second == end1);
+	for (const auto [epochVersionCount, tagVersion] : unfinished) {
+		ASSERT(std::get<0>(epochVersionCount) == epoch1 && std::get<1>(epochVersionCount) == end1 &&
+		       std::get<2>(epochVersionCount) == 1);
 		ASSERT(tagVersion.size() == 1 && tagVersion.begin()->first == tag1 && tagVersion.begin()->second == begin1);
 	}
 
@@ -128,8 +129,9 @@ TEST_CASE("/BackupProgress/Unfinished") {
 	progress.addBackupStatus(status1);
 	unfinished = progress.getUnfinishedBackup();
 	ASSERT(unfinished.size() == 1);
-	for (const auto [epochVersion, tagVersion] : unfinished) {
-		ASSERT(epochVersion.first == epoch1 && epochVersion.second == end1);
+	for (const auto [epochVersionCount, tagVersion] : unfinished) {
+		ASSERT(std::get<0>(epochVersionCount) == epoch1 && std::get<1>(epochVersionCount) == end1 &&
+		       std::get<2>(epochVersionCount) == 1);
 		ASSERT(tagVersion.size() == 1 && tagVersion.begin()->first == tag1 && tagVersion.begin()->second == saved1 + 1);
 	}
 
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index f12002dbfe..90e93fc95e 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -25,6 +25,8 @@
 #define FDBSERVER_BACKUPPROGRESS_ACTOR_H
 
 #include <map>
+#include <tuple>
+
 #include "fdbclient/FDBTypes.h"
 #include "fdbserver/LogSystem.h"
 #include "flow/Arena.h"
@@ -41,7 +43,7 @@ public:
 	// savedVersion is used.
 	void addBackupStatus(const WorkerBackupStatus& status);
 
-	// Returns a map of pair<Epoch, endVersion> : map<tag, savedVersion>, so that
+	// Returns a map of tuple<Epoch, endVersion, logRouterTags> : map<tag, savedVersion>, so that
 	// the backup range should be [savedVersion + 1, endVersion) for the "tag" of the "Epoch".
 	//
 	// Specifically, the backup ranges for each old epoch are:
@@ -49,7 +51,7 @@ public:
 	//        backup [epochBegin, endVersion)
 	//    else if savedVersion < endVersion - 1 = knownCommittedVersion
 	//        backup [savedVersion + 1, endVersion)
-	std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> getUnfinishedBackup();
+	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> getUnfinishedBackup();
 
 	// Set the value for "backupStartedKey"
 	void setBackupStartedValue(Optional<Value> value) {
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 9cbbbc0659..f2f4d675f8 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -67,6 +67,7 @@ struct VersionedMessage {
 struct BackupData {
 	const UID myId;
 	const Tag tag; // LogRouter tag for this worker, i.e., (-2, i)
+	const int totalTags; // Total log router tags
 	const Version startVersion;
 	const Optional<Version> endVersion; // old epoch's end version (inclusive), or empty for current epoch
 	const LogEpoch recruitedEpoch;
@@ -102,9 +103,9 @@ struct BackupData {
 	Future<Void> logger;
 
 	explicit BackupData(UID id, Reference<AsyncVar<ServerDBInfo>> db, const InitializeBackupRequest& req)
-	  : myId(id), tag(req.routerTag), startVersion(req.startVersion), endVersion(req.endVersion),
-	    recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch), minKnownCommittedVersion(invalidVersion),
-	    savedVersion(invalidVersion), cc("BackupWorker", myId.toString()) {
+	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
+	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
+	    minKnownCommittedVersion(invalidVersion), savedVersion(invalidVersion), cc("BackupWorker", myId.toString()) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
 		pullFinished.set(false);
 
@@ -417,7 +418,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 			it->second.lastSavedVersion = self->messages[0].getVersion();
 		}
 		logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile(
-		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id));
+		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));
 		it++;
 	}
 	if (activeUids.empty()) {
@@ -648,6 +649,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 
 	TraceEvent("BackupWorkerStart", self.myId)
 	    .detail("Tag", req.routerTag.toString())
+	    .detail("TotalTags", req.totalTags)
 	    .detail("StartVersion", req.startVersion)
 	    .detail("EndVersion", req.endVersion.present() ? req.endVersion.get() : -1)
 	    .detail("LogEpoch", req.recruitedEpoch)
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index ee613912a1..c8885cb4a0 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -170,6 +170,7 @@ struct InitializeBackupRequest {
 	LogEpoch backupEpoch; // The epoch the worker should work on. If different from the recruitedEpoch, then it refers
 	                      // to some previous epoch with unfinished work.
 	Tag routerTag;
+	int totalTags;
 	Version startVersion;
 	Optional<Version> endVersion;
 	ReplyPromise<struct InitializeBackupReply> reply;
@@ -179,7 +180,7 @@ struct InitializeBackupRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, reqId, recruitedEpoch, backupEpoch, routerTag, startVersion, endVersion, reply);
+		serializer(ar, reqId, recruitedEpoch, backupEpoch, routerTag, totalTags, startVersion, endVersion, reply);
 	}
 };
 
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index a556c2fec2..7acf67b72a 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1261,6 +1261,7 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 		req.recruitedEpoch = epoch;
 		req.backupEpoch = epoch;
 		req.routerTag = idsTags[i].second;
+		req.totalTags = logRouterTags;
 		req.startVersion = startVersion;
 		TraceEvent("BackupRecruitment", self->dbgid)
 		    .detail("BKID", req.reqId)
@@ -1275,17 +1276,19 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 	}
 
 	wait(gotProgress);
-	std::map<std::pair<LogEpoch, Version>, std::map<Tag, Version>> toRecruit = backupProgress->getUnfinishedBackup();
-	for (const auto& [epochVersion, tagVersions] : toRecruit) {
+	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
+	    backupProgress->getUnfinishedBackup();
+	for (const auto& [epochVersionCount, tagVersions] : toRecruit) {
 		for (const auto& [tag, version] : tagVersions) {
 			const auto& worker = self->backupWorkers[i % self->backupWorkers.size()];
 			i++;
 			InitializeBackupRequest req(deterministicRandom()->randomUniqueID());
 			req.recruitedEpoch = epoch;
-			req.backupEpoch = epochVersion.first;
+			req.backupEpoch = std::get<0>(epochVersionCount);
 			req.routerTag = tag;
+			req.totalTags = std::get<2>(epochVersionCount);
 			req.startVersion = version; // savedVersion + 1
-			req.endVersion = epochVersion.second - 1;
+			req.endVersion = std::get<1>(epochVersionCount) - 1;
 			TraceEvent("BackupRecruitment", self->dbgid)
 			    .detail("BKID", req.reqId)
 			    .detail("Tag", req.routerTag.toString())

From ab0b59b0c3d1d3c801724d49e1d80e39d818a824 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 21 Feb 2020 11:47:51 -0800
Subject: [PATCH 070/176] Add subsequence number to restore loader & applier

The subsequence number is needed so that mutations of the same commit version
number, but from different partitioned logs can be correctly reassembled in
order.

For old backup files, the sub number is always 0. For partitioned mutation
logs, the actual sub number is used. For range files, the sub number is always
0.
---
 fdbclient/RestoreWorkerInterface.actor.h | 10 +--
 fdbserver/RestoreApplier.actor.cpp       | 21 ++++---
 fdbserver/RestoreApplier.actor.h         | 56 ++++++++---------
 fdbserver/RestoreLoader.actor.cpp        | 78 ++++++++++++------------
 fdbserver/RestoreRoleCommon.actor.h      |  8 ++-
 fdbserver/RestoreUtil.h                  |  1 +
 6 files changed, 89 insertions(+), 85 deletions(-)

diff --git a/fdbclient/RestoreWorkerInterface.actor.h b/fdbclient/RestoreWorkerInterface.actor.h
index 84c2f603e8..e51359b82a 100644
--- a/fdbclient/RestoreWorkerInterface.actor.h
+++ b/fdbclient/RestoreWorkerInterface.actor.h
@@ -453,26 +453,28 @@ struct RestoreSendVersionedMutationsRequest : TimedRequest {
 	Version prevVersion, version; // version is the commitVersion of the mutation vector.
 	bool isRangeFile;
 	MutationsVec mutations; // All mutations at the same version parsed by one loader
+	SubSequenceVec subs; // Sub-sequence number for mutations
 
 	ReplyPromise<RestoreCommonReply> reply;
 
 	RestoreSendVersionedMutationsRequest() = default;
 	explicit RestoreSendVersionedMutationsRequest(int batchIndex, const RestoreAsset& asset, Version prevVersion,
-	                                              Version version, bool isRangeFile, MutationsVec mutations)
+	                                              Version version, bool isRangeFile, MutationsVec mutations,
+	                                              SubSequenceVec subs)
 	  : batchIndex(batchIndex), asset(asset), prevVersion(prevVersion), version(version), isRangeFile(isRangeFile),
-	    mutations(mutations) {}
+	    mutations(mutations), subs(subs) {}
 
 	std::string toString() {
 		std::stringstream ss;
 		ss << "VersionBatchIndex:" << batchIndex << "RestoreAsset:" << asset.toString()
 		   << " prevVersion:" << prevVersion << " version:" << version << " isRangeFile:" << isRangeFile
-		   << " mutations.size:" << mutations.size();
+		   << " mutations.size:" << mutations.size() << " subs.size:" << subs.size();
 		return ss.str();
 	}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, batchIndex, asset, prevVersion, version, isRangeFile, mutations, reply);
+		serializer(ar, batchIndex, asset, prevVersion, version, isRangeFile, mutations, subs, reply);
 	}
 };
 
diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index 6df9baec32..fb31fea375 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -101,7 +101,7 @@ ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int
 
 // The actor may be invovked multiple times and executed async.
 // No race condition as long as we do not wait or yield when operate the shared data.
-// Multiple such actors can run on different fileIDs, because mutations in different files belong to different versions;
+// Multiple such actors can run on different fileIDs;
 // Only one actor can process mutations from the same file
 ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMutationsRequest req,
                                                           Reference<RestoreApplierData> self) {
@@ -126,21 +126,22 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMu
 	state bool isDuplicated = true;
 	if (curFilePos.get() == req.prevVersion) {
 		isDuplicated = false;
-		Version commitVersion = req.version;
+		const Version commitVersion = req.version;
 		uint16_t numVersionStampedKV = 0;
-		MutationsVec mutations(req.mutations);
 		// Sanity check: mutations in range file is in [beginVersion, endVersion);
 		// mutations in log file is in [beginVersion, endVersion], both inclusive.
 		ASSERT_WE_THINK(commitVersion >= req.asset.beginVersion);
 		// Loader sends the endVersion to ensure all useful versions are sent
 		ASSERT_WE_THINK(commitVersion <= req.asset.endVersion);
+		ASSERT(req.mutations.size() == req.subs.size());
 
-		for (int mIndex = 0; mIndex < mutations.size(); mIndex++) {
-			MutationRef mutation = mutations[mIndex];
+		for (int mIndex = 0; mIndex < req.mutations.size(); mIndex++) {
+			const MutationRef& mutation = req.mutations[mIndex];
+			const LogMessageVersion mutationVersion(commitVersion, req.subs[mIndex]);
 			TraceEvent(SevFRMutationInfo, "FastRestoreApplierPhaseReceiveMutations", self->id())
 			    .detail("ApplierNode", self->id())
 			    .detail("RestoreAsset", req.asset.toString())
-			    .detail("Version", commitVersion)
+			    .detail("Version", mutationVersion.toString())
 			    .detail("Index", mIndex)
 			    .detail("MutationReceived", mutation.toString());
 			batchData->counters.receivedBytes += mutation.totalSize();
@@ -159,10 +160,10 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMu
 			// Note: Log and range mutations may be delivered out of order. Can we handle it?
 			if (mutation.type == MutationRef::SetVersionstampedKey ||
 			    mutation.type == MutationRef::SetVersionstampedValue) {
-				batchData->addVersionStampedKV(mutation, commitVersion, numVersionStampedKV);
+				batchData->addVersionStampedKV(mutation, mutationVersion, numVersionStampedKV);
 				numVersionStampedKV++;
 			} else {
-				batchData->addMutation(mutation, commitVersion);
+				batchData->addMutation(mutation, mutationVersion);
 			}
 		}
 		curFilePos.set(req.version);
@@ -239,7 +240,7 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 			for (auto& vm : key.second->second.pendingMutations) {
 				for (auto& m : vm.second) {
 					TraceEvent(SevWarnAlways, "FastRestoreApplierGetAndComputeStagingKeysUnhandledError")
-					    .detail("PendingMutationVersion", vm.first)
+					    .detail("PendingMutationVersion", vm.first.toString())
 					    .detail("PendingMutation", m.toString());
 				}
 			}
@@ -250,7 +251,7 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 			// The key's version ideally should be the most recently committed version.
 			// But as long as it is > 1 and less than the start version of the version batch, it is the same result.
 			MutationRef m(MutationRef::SetValue, key.first, fValues[i].get().get());
-			key.second->second.add(m, (Version)1);
+			key.second->second.add(m, LogMessageVersion(1));
 			key.second->second.precomputeResult();
 			i++;
 		}
diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index 72424eed62..9d8f6b60d8 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -51,14 +51,14 @@ struct StagingKey {
 	Key key; // TODO: Maybe not needed?
 	Value val;
 	MutationRef::Type type; // set or clear
-	Version version; // largest version of set or clear for the key
-	std::map<Version, MutationsVec> pendingMutations; // mutations not set or clear type
+	LogMessageVersion version; // largest version of set or clear for the key
+	std::map<LogMessageVersion, MutationsVec> pendingMutations; // mutations not set or clear type
 
 	explicit StagingKey() : version(0), type(MutationRef::MAX_ATOMIC_OP) {}
 
 	// Add mutation m at newVersion to stagingKey
 	// Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set
-	void add(const MutationRef& m, Version newVersion) {
+	void add(const MutationRef& m, LogMessageVersion newVersion) {
 		ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue);
 		if (version < newVersion) {
 			if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
@@ -76,14 +76,14 @@ struct StagingKey {
 			}
 		} else if (version == newVersion) { // Sanity check
 			TraceEvent("FastRestoreApplierStagingKeyMutationAtSameVersion")
-			    .detail("Version", newVersion)
+			    .detail("Version", newVersion.toString())
 			    .detail("NewMutation", m.toString())
 			    .detail("ExistingKeyType", typeString[type]);
 			if (m.type == MutationRef::SetValue) {
 				if (type == MutationRef::SetValue) {
 					if (m.param2 != val) {
 						TraceEvent(SevError, "FastRestoreApplierStagingKeyMutationAtSameVersionUnhandled")
-						    .detail("Version", newVersion)
+						    .detail("Version", newVersion.toString())
 						    .detail("NewMutation", m.toString())
 						    .detail("ExistingKeyType", typeString[type])
 						    .detail("ExitingKeyValue", val)
@@ -92,7 +92,7 @@ struct StagingKey {
 					} // else {} Backup has duplicate set at the same version
 				} else {
 					TraceEvent(SevWarnAlways, "FastRestoreApplierStagingKeyMutationAtSameVersionOverride")
-					    .detail("Version", newVersion)
+					    .detail("Version", newVersion.toString())
 					    .detail("NewMutation", m.toString())
 					    .detail("ExistingKeyType", typeString[type])
 					    .detail("ExitingKeyValue", val);
@@ -101,7 +101,7 @@ struct StagingKey {
 				}
 			} else if (m.type == MutationRef::ClearRange) {
 				TraceEvent(SevWarnAlways, "FastRestoreApplierStagingKeyMutationAtSameVersionSkipped")
-				    .detail("Version", newVersion)
+				    .detail("Version", newVersion.toString())
 				    .detail("NewMutation", m.toString())
 				    .detail("ExistingKeyType", typeString[type])
 				    .detail("ExitingKeyValue", val);
@@ -113,9 +113,10 @@ struct StagingKey {
 	void precomputeResult() {
 		TraceEvent(SevDebug, "FastRestoreApplierPrecomputeResult")
 		    .detail("Key", key)
-		    .detail("Version", version)
-		    .detail("LargestPendingVersion", (pendingMutations.empty() ? -1 : pendingMutations.rbegin()->first));
-		std::map<Version, MutationsVec>::iterator lb = pendingMutations.lower_bound(version);
+		    .detail("Version", version.toString())
+		    .detail("LargestPendingVersion",
+		            (pendingMutations.empty() ? "-1" : pendingMutations.rbegin()->first.toString()));
+		std::map<LogMessageVersion, MutationsVec>::iterator lb = pendingMutations.lower_bound(version);
 		if (lb == pendingMutations.end()) {
 			return;
 		}
@@ -158,11 +159,11 @@ struct StagingKey {
 					type = MutationRef::SetValue; // Precomputed result should be set to DB.
 					TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet")
 					    .detail("Type", typeString[mutation.type])
-					    .detail("Version", lb->first);
+					    .detail("Version", lb->first.toString());
 				} else {
 					TraceEvent(SevWarnAlways, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation")
 					    .detail("Type", typeString[mutation.type])
-					    .detail("Version", lb->first);
+					    .detail("Version", lb->first.toString());
 				}
 			}
 			version = lb->first;
@@ -172,10 +173,10 @@ struct StagingKey {
 
 	// Does the key has at least 1 set or clear mutation to get the base value
 	bool hasBaseValue() {
-		if (version > 0) {
+		if (version.version > 0) {
 			ASSERT(type == MutationRef::SetValue || type == MutationRef::ClearRange);
 		}
-		return version > 0;
+		return version.version > 0;
 	}
 
 	// Has all pendingMutations been pre-applied to the val?
@@ -191,9 +192,9 @@ struct StagingKey {
 // Range mutations should be applied both to the destination DB and to the StagingKeys
 struct StagingKeyRange {
 	Standalone<MutationRef> mutation;
-	Version version;
+	LogMessageVersion version;
 
-	explicit StagingKeyRange(MutationRef m, Version newVersion) : mutation(m), version(newVersion) {}
+	explicit StagingKeyRange(MutationRef m, LogMessageVersion newVersion) : mutation(m), version(newVersion) {}
 
 	bool operator<(const StagingKeyRange& rhs) const {
 		return std::tie(version, mutation.type, mutation.param1, mutation.param2) <
@@ -263,7 +264,7 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 	}
 	~ApplierBatchData() = default;
 
-	void addMutation(MutationRef m, Version ver) {
+	void addMutation(MutationRef m, LogMessageVersion ver) {
 		if (!isRangeMutation(m)) {
 			auto item = stagingKeys.emplace(m.param1, StagingKey());
 			item.first->second.add(m, ver);
@@ -272,20 +273,20 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 		}
 	}
 
-	void addVersionStampedKV(MutationRef m, Version ver, uint16_t numVersionStampedKV) {
+	void addVersionStampedKV(MutationRef m, LogMessageVersion ver, uint16_t numVersionStampedKV) {
 		if (m.type == MutationRef::SetVersionstampedKey) {
 			// Assume transactionNumber = 0 does not affect result
 			TraceEvent(SevDebug, "FastRestoreApplierAddMutation")
 			    .detail("MutationType", typeString[m.type])
 			    .detail("FakedTransactionNumber", numVersionStampedKV);
-			transformVersionstampMutation(m, &MutationRef::param1, ver, numVersionStampedKV);
+			transformVersionstampMutation(m, &MutationRef::param1, ver.version, numVersionStampedKV);
 			addMutation(m, ver);
 		} else if (m.type == MutationRef::SetVersionstampedValue) {
 			// Assume transactionNumber = 0 does not affect result
 			TraceEvent(SevDebug, "FastRestoreApplierAddMutation")
 			    .detail("MutationType", typeString[m.type])
 			    .detail("FakedTransactionNumber", numVersionStampedKV);
-			transformVersionstampMutation(m, &MutationRef::param2, ver, numVersionStampedKV);
+			transformVersionstampMutation(m, &MutationRef::param2, ver.version, numVersionStampedKV);
 			addMutation(m, ver);
 		} else {
 			ASSERT(false);
@@ -298,8 +299,8 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 			if (!stagingKey.second.hasPrecomputed()) {
 				TraceEvent("FastRestoreApplierAllKeysPrecomputedFalse")
 				    .detail("Key", stagingKey.first)
-				    .detail("BufferedVersion", stagingKey.second.version)
-				    .detail("MaxPendingVersion", stagingKey.second.pendingMutations.rbegin()->first);
+				    .detail("BufferedVersion", stagingKey.second.version.toString())
+				    .detail("MaxPendingVersion", stagingKey.second.pendingMutations.rbegin()->first.toString());
 				return false;
 			}
 		}
@@ -320,20 +321,17 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 	}
 
 	bool isKVOpsSorted() {
-		bool ret = true;
 		auto prev = kvOps.begin();
 		for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
 			if (prev->first > it->first) {
-				ret = false;
-				break;
+				return false;
 			}
 			prev = it;
 		}
-		return ret;
+		return true;
 	}
 
 	bool allOpsAreKnown() {
-		bool ret = true;
 		for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
 			for (auto m = it->second.begin(); m != it->second.end(); ++m) {
 				if (m->type == MutationRef::SetValue || m->type == MutationRef::ClearRange ||
@@ -341,11 +339,11 @@ struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
 					continue;
 				else {
 					TraceEvent(SevError, "FastRestore").detail("UnknownMutationType", m->type);
-					ret = false;
+					return false;
 				}
 			}
 		}
-		return ret;
+		return true;
 	}
 };
 
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 3f15aebac1..15dbb3e179 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -165,7 +165,6 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 		// Read block header
 		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
 
-		Version lastVersion = invalidVersion;
 		VersionedMutationsMap& kvOps = kvOpsIter->second;
 		VersionedMutationsMap::iterator it = kvOps.end();
 		while (1) {
@@ -173,20 +172,18 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 			if (reader.eof() || *reader.rptr == 0xFF) break;
 
 			// Deserialize messages written in saveMutationsToFile().
-			Version msgVersion = bigEndian64(reader.consume<Version>());
-			bigEndian32(reader.consume<uint32_t>()); // subsequence number
+			LogMessageVersion msgVersion;
+			msgVersion.version = bigEndian64(reader.consume<Version>());
+			msgVersion.sub = bigEndian32(reader.consume<uint32_t>());
 			int msgSize = bigEndian32(reader.consume<int>());
 			const uint8_t* message = reader.consume(msgSize);
 
 			// Skip mutations out of the version range
-			if (!asset.isInVersionRange(msgVersion)) continue;
+			if (!asset.isInVersionRange(msgVersion.version)) continue;
 
-			if (lastVersion != msgVersion) {
-				bool inserted;
-				std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec());
-				lastVersion = msgVersion;
-			}
-			ASSERT(it != kvOps.end());
+			bool inserted;
+			std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec());
+			ASSERT(inserted);
 
 			ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(currentProtocolVersion));
 			MutationRef mutation;
@@ -205,7 +202,7 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 			}
 
 			TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
-			    .detail("CommitVersion", msgVersion)
+			    .detail("CommitVersion", msgVersion.toString())
 			    .detail("ParsedMutation", mutation.toString());
 			it->second.push_back_deep(it->second.arena(), mutation);
 			// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
@@ -306,7 +303,6 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
 		    .detail("BatchIndex", req.batchIndex)
 		    .detail("ProcessLoadParam", req.param.toString());
 		ASSERT(batchData->sampleMutations.find(req.param) == batchData->sampleMutations.end());
-		batchData->processedFileParams[req.param] = Never(); // Ensure second exec. wait on _processLoadingParam()
 		batchData->processedFileParams[req.param] = _processLoadingParam(req.param, batchData, self->id(), self->bc);
 		isDuplicated = false;
 	} else {
@@ -314,8 +310,9 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
 		    .detail("BatchIndex", req.batchIndex)
 		    .detail("WaitOnProcessLoadParam", req.param.toString());
 	}
-	ASSERT(batchData->processedFileParams.find(req.param) != batchData->processedFileParams.end());
-	wait(batchData->processedFileParams[req.param]); // wait on the processing of the req.param.
+	auto it = batchData->processedFileParams.find(req.param);
+	ASSERT(it != batchData->processedFileParams.end());
+	wait(it->second); // wait on the processing of the req.param.
 
 	req.reply.send(RestoreLoadFileReply(req.param, batchData->sampleMutations[req.param], isDuplicated));
 	TraceEvent("FastRestoreLoaderPhaseLoadFileDone", self->id())
@@ -426,16 +423,15 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 	    .detail("RestoreAsset", asset.toString());
 
 	// There should be no mutation at asset.endVersion version because it is exclusive
-	if (kvOps.find(asset.endVersion) != kvOps.end()) {
+	if (kvOps.find(LogMessageVersion(asset.endVersion)) != kvOps.end()) {
 		TraceEvent(SevError, "FastRestoreLoaderSendMutationToApplier")
 		    .detail("BatchIndex", batchIndex)
 		    .detail("RestoreAsset", asset.toString())
 		    .detail("IsRangeFile", isRangeFile)
 		    .detail("Data loss at version", asset.endVersion);
-	}
-	// Ensure there is a mutation request sent at endVersion, so that applier can advance its notifiedVersion
-	if (kvOps.find(asset.endVersion) == kvOps.end()) {
-		kvOps[asset.endVersion] = MutationsVec(); // Empty mutation vector will be handled by applier
+	} else {
+		// Ensure there is a mutation request sent at endVersion, so that applier can advance its notifiedVersion
+		kvOps[LogMessageVersion(asset.endVersion)] = MutationsVec(); // Empty mutation vector will be handled by applier
 	}
 
 	splitMutationIndex = 0;
@@ -445,22 +441,24 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 		// applierMutationsBuffer is the mutation vector to be sent to each applier
 		// applierMutationsSize is buffered mutation vector size for each applier
 		std::map<UID, MutationsVec> applierMutationsBuffer;
+		std::map<UID, SubSequenceVec> applierSubsBuffer;
 		std::map<UID, double> applierMutationsSize;
 		for (auto& applierID : applierIDs) {
 			applierMutationsBuffer[applierID] = MutationsVec();
+			applierSubsBuffer[applierID] = SubSequenceVec();
 			applierMutationsSize[applierID] = 0.0;
 		}
-		Version commitVersion = kvOp->first;
-		if (!(commitVersion >= asset.beginVersion && commitVersion <= asset.endVersion)) { // Debug purpose
+		const LogMessageVersion& commitVersion = kvOp->first;
+		if (!(commitVersion.version >= asset.beginVersion &&
+		      commitVersion.version <= asset.endVersion)) { // Debug purpose
 			TraceEvent(SevError, "FastRestore_SendMutationsToApplier")
-			    .detail("CommitVersion", commitVersion)
+			    .detail("CommitVersion", commitVersion.version)
 			    .detail("RestoreAsset", asset.toString());
 		}
-		ASSERT(commitVersion >= asset.beginVersion);
-		ASSERT(commitVersion <= asset.endVersion); // endVersion is an empty commit to ensure progress
+		ASSERT(commitVersion.version >= asset.beginVersion);
+		ASSERT(commitVersion.version <= asset.endVersion); // endVersion is an empty commit to ensure progress
 
-		for (int mIndex = 0; mIndex < kvOp->second.size(); mIndex++) {
-			MutationRef kvm = kvOp->second[mIndex];
+		for (const MutationRef& kvm : kvOp->second) {
 			// Send the mutation to applier
 			if (isRangeMutation(kvm)) {
 				MutationsVec mvector;
@@ -478,6 +476,7 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 					// printf("SPLITTED MUTATION: %d: mutation:%s applierID:%s\n", splitMutationIndex,
 					// mutation.toString().c_str(), applierID.toString().c_str());
 					applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
+					applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
 					applierMutationsSize[applierID] += mutation.expectedSize();
 
 					kvCount++;
@@ -493,30 +492,30 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 				kvCount++;
 
 				applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
+				applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
 				applierMutationsSize[applierID] += mutation.expectedSize();
 			}
 		} // Mutations at the same version
 
 		// TODO: Sanity check each asset has been received exactly once!
 		// Send the mutations to appliers for each version
-		for (auto& applierID : applierIDs) {
-			requests.push_back(std::make_pair(
-			    applierID, RestoreSendVersionedMutationsRequest(batchIndex, asset, prevVersion, commitVersion,
-			                                                    isRangeFile, applierMutationsBuffer[applierID])));
+		for (const UID& applierID : applierIDs) {
+			requests.emplace_back(applierID, RestoreSendVersionedMutationsRequest(
+			                                     batchIndex, asset, prevVersion, commitVersion.version, isRangeFile,
+			                                     applierMutationsBuffer[applierID], applierSubsBuffer[applierID]));
 		}
 		TraceEvent(SevDebug, "FastRestore_SendMutationToApplier")
 		    .detail("PrevVersion", prevVersion)
-		    .detail("CommitVersion", commitVersion)
+		    .detail("CommitVersion", commitVersion.toString())
 		    .detail("RestoreAsset", asset.toString());
-		ASSERT(prevVersion < commitVersion);
-		prevVersion = commitVersion;
+		ASSERT(prevVersion <= commitVersion.version);
+		prevVersion = commitVersion.version;
 		// Tracking this request can be spammy
 		wait(sendBatchRequests(&RestoreApplierInterface::sendMutationVector, *pApplierInterfaces, requests,
 		                       TaskPriority::RestoreLoaderSendMutations,
 		                       SERVER_KNOBS->FASTRESTORE_TRACK_LOADER_SEND_REQUESTS));
 
 		requests.clear();
-
 	} // all versions of mutations in the same file
 
 	TraceEvent("FastRestore").detail("LoaderSendMutationOnAppliers", kvCount);
@@ -655,7 +654,8 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 		uint64_t commitVersion = kReader.consume<uint64_t>(); // Consume little Endian data
 		// We have already filter the commit not in [beginVersion, endVersion) when we concatenate kv pair in log file
 		ASSERT_WE_THINK(asset.isInVersionRange(commitVersion));
-		kvOps.insert(std::make_pair(commitVersion, MutationsVec()));
+		auto it = kvOps.insert(std::make_pair(LogMessageVersion(commitVersion), MutationsVec()));
+		ASSERT(it.second); // inserted is true
 
 		StringRefReader vReader(val, restore_corrupted_data());
 		vReader.consume<uint64_t>(); // Consume the includeVersion
@@ -695,7 +695,7 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 			TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
 			    .detail("CommitVersion", commitVersion)
 			    .detail("ParsedMutation", mutation.toString());
-			kvOps[commitVersion].push_back_deep(kvOps[commitVersion].arena(), mutation);
+			it.first->second.push_back_deep(it.first->second.arena(), mutation);
 			// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
 			if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {
 				samples.push_back_deep(samples.arena(), mutation);
@@ -774,13 +774,13 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 		cc->loadedRangeBytes += m.totalSize();
 
 		// We cache all kv operations into kvOps, and apply all kv operations later in one place
-		kvOps.insert(std::make_pair(version, MutationsVec()));
+		auto it = kvOps.insert(std::make_pair(LogMessageVersion(version), MutationsVec()));
 		TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
 		    .detail("CommitVersion", version)
 		    .detail("ParsedMutationKV", m.toString());
 
-		ASSERT_WE_THINK(kvOps.find(version) != kvOps.end());
-		kvOps[version].push_back_deep(kvOps[version].arena(), m);
+		ASSERT_WE_THINK(kvOps.find(LogMessageVersion(version)) != kvOps.end());
+		it.first->second.push_back_deep(it.first->second.arena(), m);
 		// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
 		if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {
 			cc->sampledRangeBytes += m.totalSize();
diff --git a/fdbserver/RestoreRoleCommon.actor.h b/fdbserver/RestoreRoleCommon.actor.h
index 9ddbc3d82e..cedbeb795c 100644
--- a/fdbserver/RestoreRoleCommon.actor.h
+++ b/fdbserver/RestoreRoleCommon.actor.h
@@ -51,9 +51,11 @@ struct RestoreMasterData;
 
 struct RestoreSimpleRequest;
 
-// VersionedMutationsMap: Key is the version of parsed backup mutations
-// Value MutationsVec is the vector of parsed backup mutations
-using VersionedMutationsMap = std::map<Version, MutationsVec>;
+// Key is the (version, subsequence) of parsed backup mutations.
+// Value MutationsVec is the vector of parsed backup mutations.
+// For old mutation logs, the subsequence number is always 0.
+// For partitioned mutation logs, each mutation has a unique LogMessageVersion.
+using VersionedMutationsMap = std::map<LogMessageVersion, MutationsVec>;
 
 ACTOR Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name);
 ACTOR Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id);
diff --git a/fdbserver/RestoreUtil.h b/fdbserver/RestoreUtil.h
index 3c1e1fa7d8..683d785fc2 100644
--- a/fdbserver/RestoreUtil.h
+++ b/fdbserver/RestoreUtil.h
@@ -39,6 +39,7 @@
 //#define SevFRMutationInfo SevInfo
 
 using MutationsVec = Standalone<VectorRef<MutationRef>>;
+using SubSequenceVec = Standalone<VectorRef<uint32_t>>;
 
 enum class RestoreRole { Invalid = 0, Master = 1, Loader, Applier };
 BINARY_SERIALIZABLE(RestoreRole);

From 659843ff515f0b5645da124845670d570d828f01 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 21 Feb 2020 14:07:46 -0800
Subject: [PATCH 071/176] Check partitioned log files are continuous for
 RestoreSet

The idea of checking is to use Tag 0 to find out ranges and their number of
tags. Then for each tag 1 and above, check versions are continuous.
---
 fdbclient/BackupContainer.actor.cpp | 143 +++++++++++++++++++++++++---
 1 file changed, 130 insertions(+), 13 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 3b9ce2389e..eed735eaa4 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -228,23 +228,24 @@ std::string BackupDescription::toJSON() const {
  *   Snapshot manifests (a complete set of files constituting a database snapshot for the backup's target ranges)
  *   are stored as JSON files at paths like
  *       /snapshots/snapshot,minVersion,maxVersion,totalBytes
- * 
+ *
  *   Key range files for snapshots are stored at paths like
  *       /kvranges/snapshot,startVersion/N/range,version,uid,blockSize
  *     where startVersion is the version at which the backup snapshot execution began and N is a number
- *     that is increased as key range files are generated over time (at varying rates) such that there 
+ *     that is increased as key range files are generated over time (at varying rates) such that there
  *     are around 5,000 key range files in each folder.
  *
- *     Note that startVersion will NOT correspond to the minVersion of a snapshot manifest because 
+ *     Note that startVersion will NOT correspond to the minVersion of a snapshot manifest because
  *     snapshot manifest min/max versions are based on the actual contained data and the first data
  *     file written will be after the start version of the snapshot's execution.
- * 
+ *
  *   Log files are at file paths like
- *       /plogs/...log,startVersion,endVersion,UID,blocksize,tagID
+ *       /plogs/...log,startVersion,endVersion,UID,blocksize,tagID-of-N
  *       /logs/.../log,startVersion,endVersion,UID,blockSize
  *     where ... is a multi level path which sorts lexically into version order and results in approximately 1
  *     unique folder per day containing about 5,000 files. Logs after 7.0 are stored in "plogs"
- *     directory and are partitioned according to tagIDs (0, 1, 2, ...). Logs before 7.0 are
+ *     directory and are partitioned according to tagIDs (0, 1, 2, ...) and the total number
+ *     partitions is N. Logs before 7.0 are
  *     stored in "logs" directory and are not partitioned.
  *
  *
@@ -252,8 +253,8 @@ std::string BackupDescription::toJSON() const {
  *
  *   Prior to FDB version 6.0.16, key range files were stored using a different folder scheme.  Newer versions
  *   still support this scheme for all restore and backup management operations but key range files generated
- *   by backup using version 6.0.16 or later use the scheme describe above.  
- * 
+ *   by backup using version 6.0.16 or later use the scheme describe above.
+ *
  *   The old format stored key range files at paths like
  *       /ranges/.../range,version,uid,blockSize
  *     where ... is a multi level path with sorts lexically into version order and results in up to approximately
@@ -1060,10 +1061,75 @@ public:
 	}
 
 	// Delete all data up to (but not including endVersion)
-	Future<Void> expireData(Version expireEndVersion, bool force, ExpireProgress *progress, Version restorableBeginVersion) override {
+	Future<Void> expireData(Version expireEndVersion, bool force, ExpireProgress* progress,
+	                        Version restorableBeginVersion) final {
 		return expireData_impl(Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
 	}
 
+	// For a list of log files specified by their indices (of the same tag),
+	// returns if they are continous in the range [begin, end].
+	static bool isContinuous(const std::vector<LogFile>& files, std::vector<int> indices, Version begin, Version end,
+	                         std::map<std::pair<Version, Version>, int>* tags) {
+		Version lastBegin = invalidVersion;
+		Version lastEnd = invalidVersion;
+		int lastTags = -1;
+
+		for (int idx : indices) {
+			const LogFile& file = files[idx];
+			if (lastEnd == invalidVersion) {
+				if (file.beginVersion > begin) return false;
+				if (file.endVersion > begin) {
+					lastBegin = begin;
+					lastTags = file.totalTags;
+				} else {
+					continue;
+				}
+			} else if (lastEnd != file.beginVersion) {
+				return false; // not continuous
+			}
+
+			if (lastTags != file.totalTags) {
+				if (tags != nullptr) {
+					tags->emplace(std::make_pair(lastBegin, file.beginVersion - 1), lastTags);
+				}
+				lastBegin = file.beginVersion;
+				lastTags = file.totalTags;
+			}
+			lastEnd = file.endVersion;
+			if (lastEnd > end) break;
+		}
+		if (lastBegin == invalidVersion || lastEnd <= end) return false; // not covering the range
+		if (tags != nullptr) {
+			tags->emplace(std::make_pair(lastBegin, end), lastTags);
+		}
+		return true;
+	}
+
+	// Returns true if logs are continuous in the range [begin, end].
+	// "files" should be pre-sorted according to version order.
+	static bool isPartitionedLogsContinuous(const std::vector<LogFile>& files, Version begin, Version end) {
+		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
+		for (int i = 0; i < files.size(); i++) {
+			ASSERT(files[i].tagId >= 0 && files[i].tagId < files[i].totalTags);
+			auto& indices = tagIndices[files[i].tagId];
+			indices.push_back(i);
+		}
+
+		// check tag 0 is continuous and create a map of ranges to tags
+		std::map<std::pair<Version, Version>, int> tags; // range [start, end) -> tags
+		if (!isContinuous(files, tagIndices[0], begin, end, &tags)) return false;
+
+		// for each range in tags, check all tags from 1 are continouous
+		for (const auto [beginEnd, count] : tags) {
+			for (int i = 1; i < count; i++) {
+				if (!isContinuous(files, tagIndices[i], beginEnd.first, beginEnd.second, nullptr)) {
+					return false;
+				}
+			}
+		}
+		return true;
+	}
+
 	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion, bool partitioned) {
 		// Find the most recent keyrange snapshot to end at or before targetVersion
 		state Optional<KeyspaceSnapshotFile> snapshot;
@@ -1105,11 +1171,11 @@ public:
 				}
 				if (i < logs.size()) filtered.push_back(logs[i]);
 
-				// TODO(jingyu): for partitioned logs, the continuity checking should be based on
-				// epochs and versions, which should be saved in a metadata file by backup worker and
-				// thus is available here. For now, assume it's continuous.
 				restorable.logs.swap(filtered);
-				return Optional<RestorableFileSet>(restorable);
+				if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) {
+					return Optional<RestorableFileSet>(restorable);
+				}
+				return Optional<RestorableFileSet>();
 			}
 
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
@@ -2008,3 +2074,54 @@ TEST_CASE("/backup/time") {
 
 	return Void();
 }
+
+TEST_CASE("/backup/continuous") {
+	std::vector<LogFile> files;
+
+	// [0, 100) 2 tags
+	files.push_back({ 0, 100, 10, "file1", 100, 0, 2 }); // Tag 0: 0-100
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
+
+	files.push_back({ 0, 100, 10, "file2", 200, 1, 2 }); // Tag 1: 0-100
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
+
+	// [100, 300) 3 tags
+	files.push_back({ 100, 200, 10, "file3", 200, 0, 3 }); // Tag 0: 100-200
+	files.push_back({ 100, 250, 10, "file4", 200, 1, 3 }); // Tag 1: 100-250
+	std::sort(files.begin(), files.end());
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150));
+
+	files.push_back({ 100, 300, 10, "file5", 200, 2, 3 }); // Tag 2: 100-300
+	std::sort(files.begin(), files.end());
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 200));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 10, 199));
+
+	files.push_back({ 250, 300, 10, "file6", 200, 0, 3 }); // Tag 0: 250-300, missing 200-250
+	std::sort(files.begin(), files.end());
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 240));
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+
+	files.push_back({ 250, 300, 10, "file7", 200, 1, 3 }); // Tag 1: 250-300
+	std::sort(files.begin(), files.end());
+	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+
+	files.push_back({ 200, 250, 10, "file8", 200, 0, 3 }); // Tag 0: 200-250
+	std::sort(files.begin(), files.end());
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 299));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+
+	// [300, 400) 1 tag
+	// files.push_back({200, 250, 10, "file9", 200, 0, 3}); // Tag 0: 200-250, duplicate file
+	files.push_back({ 300, 400, 10, "file10", 200, 0, 1 }); // Tag 1: 300-400
+	std::sort(files.begin(), files.end());
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 399));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 399));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 150, 399));
+	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 250, 399));
+
+	return Void();
+}
\ No newline at end of file

From 938a6f358dbded7f49e4e1df0e79d687f9e4a60b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 24 Feb 2020 16:53:57 -0800
Subject: [PATCH 072/176] Describe backup uses partitioned logs to find
 continuous end version

For partitioned logs, the continuous end version has to be done range by range,
where each range must contain continuous version for all tags.
---
 fdbclient/BackupContainer.actor.cpp | 119 +++++++++++++++++++++++-----
 1 file changed, 99 insertions(+), 20 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index eed735eaa4..7b7f27c14e 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -781,13 +781,21 @@ public:
 		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) &&
 		     store(pLogs, bc->listLogFiles(scanBegin, scanEnd, true)) &&
 		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
-		// FIXME: check partitioned logs & maybe enable the below line
-		// logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
 		// List logs in version order so log continuity can be analyzed
 		std::sort(logs.begin(), logs.end());
 
-		if(!logs.empty()) {
+		// Check partitioned logs
+		if (!pLogs.empty()) {
+			std::sort(pLogs.begin(), pLogs.end());
+			// If we didn't get log versions above then seed them using the first log file
+			if(!desc.contiguousLogEnd.present()) {
+				auto it = pLogs.begin();
+				desc.minLogBegin = it->beginVersion;
+				desc.contiguousLogEnd = it->endVersion;
+			}
+			desc.contiguousLogEnd.get() = getPartitionedLogsContinuousEndVersion(pLogs, scanBegin);
+		} else if (!logs.empty()) {
 			desc.maxLogEnd = logs.rbegin()->endVersion;
 
 			auto i = logs.begin();
@@ -875,7 +883,7 @@ public:
 	}
 
 	// Uses the virtual methods to describe the backup contents
-	Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) override {
+	Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) final {
 		return describeBackup_impl(Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride);
 	}
 
@@ -1067,15 +1075,20 @@ public:
 	}
 
 	// For a list of log files specified by their indices (of the same tag),
-	// returns if they are continous in the range [begin, end].
+	// returns if they are continous in the range [begin, end]. If "tags" is not
+	// nullptr, then it will be populated with [begin, end] -> tags, where next
+	// pair's begin == previous pair's end + 1. On return, the last pair's end
+	// version (inclusive) gives the continuous range from begin.
 	static bool isContinuous(const std::vector<LogFile>& files, std::vector<int> indices, Version begin, Version end,
 	                         std::map<std::pair<Version, Version>, int>* tags) {
 		Version lastBegin = invalidVersion;
 		Version lastEnd = invalidVersion;
 		int lastTags = -1;
 
+		ASSERT(tags == nullptr || tags->empty());
 		for (int idx : indices) {
 			const LogFile& file = files[idx];
+std::cout << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags" << lastTags << "\n";
 			if (lastEnd == invalidVersion) {
 				if (file.beginVersion > begin) return false;
 				if (file.endVersion > begin) {
@@ -1085,7 +1098,10 @@ public:
 					continue;
 				}
 			} else if (lastEnd != file.beginVersion) {
-				return false; // not continuous
+				if (tags != nullptr) {
+					tags->emplace(std::make_pair(lastBegin, lastEnd - 1), lastTags);
+				}
+				return false;
 			}
 
 			if (lastTags != file.totalTags) {
@@ -1098,11 +1114,11 @@ public:
 			lastEnd = file.endVersion;
 			if (lastEnd > end) break;
 		}
-		if (lastBegin == invalidVersion || lastEnd <= end) return false; // not covering the range
-		if (tags != nullptr) {
-			tags->emplace(std::make_pair(lastBegin, end), lastTags);
+std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags" << lastTags << "\n";
+		if (tags != nullptr && lastBegin != invalidVersion) {
+			tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags);
 		}
-		return true;
+		return lastBegin != invalidVersion && lastEnd > end;
 	}
 
 	// Returns true if logs are continuous in the range [begin, end].
@@ -1116,7 +1132,7 @@ public:
 		}
 
 		// check tag 0 is continuous and create a map of ranges to tags
-		std::map<std::pair<Version, Version>, int> tags; // range [start, end) -> tags
+		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
 		if (!isContinuous(files, tagIndices[0], begin, end, &tags)) return false;
 
 		// for each range in tags, check all tags from 1 are continouous
@@ -1130,6 +1146,67 @@ public:
 		return true;
 	}
 
+	// Returns log files that are not duplicated.
+	static std::vector<LogFile> filterDuplicates(std::vector<LogFile>& logs) {
+		std::sort(logs.begin(), logs.end());
+
+		std::vector<LogFile> filtered;
+		int i = 0;
+		for (int j = 1; j < logs.size(); j++) {
+			if (!logs[i].sameContent(logs[j])) {
+				filtered.push_back(logs[i]);
+				i = j;
+			}
+		}
+		if (i < logs.size()) filtered.push_back(logs[i]);
+		return filtered;
+	}
+
+	// Returns the end version such that [begin, end] is continuous.
+	static Version getPartitionedLogsContinuousEndVersion(std::vector<LogFile>& logs, Version begin) {
+		auto files = filterDuplicates(logs);
+for (auto file : files) std::cout << file.toString() << "\n";
+		Version end = 0;
+
+		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
+		for (int i = 0; i < files.size(); i++) {
+			ASSERT(files[i].tagId >= 0 && files[i].tagId < files[i].totalTags);
+			auto& indices = tagIndices[files[i].tagId];
+			indices.push_back(i);
+			end = files[i].endVersion - 1;
+		}
+std::cout << "Init end: " << end << "\n";
+
+		// check tag 0 is continuous in [begin, end] and create a map of ranges to tags
+		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
+		isContinuous(files, tagIndices[0], begin, end, &tags);
+		if (tags.empty() || end <= begin) return 0;
+		end = std::min(end, tags.rbegin()->first.second);
+std::cout << "Tag 0 end: " << end << "\n";
+for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " << v << "\n";
+
+		// for each range in tags, check all tags from 1 are continouous
+		Version lastEnd = begin;
+		for (const auto [beginEnd, count] : tags) {
+			Version tagEnd = end; // This range's minimum continous tag version
+			for (int i = 1; i < count; i++) {
+				std::map<std::pair<Version, Version>, int> rangeTags;
+				isContinuous(files, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
+				tagEnd = rangeTags.empty() ? 0 : std::min(tagEnd, rangeTags.rbegin()->first.second);
+std::cout << "Tag " << i << " end: " << tagEnd << "\n";
+				if (tagEnd == 0) return lastEnd;
+			}
+			if (tagEnd < beginEnd.second) {
+				end = tagEnd;
+				break;
+			}
+			lastEnd = beginEnd.second;
+		}
+
+std::cout << "Return end = " << end << "\n\n";
+		return end;
+	}
+
 	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion, bool partitioned) {
 		// Find the most recent keyrange snapshot to end at or before targetVersion
 		state Optional<KeyspaceSnapshotFile> snapshot;
@@ -1161,15 +1238,7 @@ public:
 
 			if (partitioned) {
 				// Remove duplicated log files that can happen for old epochs.
-				std::vector<LogFile> filtered;
-				int i = 0;
-				for (int j = 1; j < logs.size(); j++) {
-					if (!logs[i].sameContent(logs[j])) {
-						filtered.push_back(logs[i]);
-						i = j;
-					}
-				}
-				if (i < logs.size()) filtered.push_back(logs[i]);
+				std::vector<LogFile> filtered = filterDuplicates(logs);
 
 				restorable.logs.swap(filtered);
 				if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) {
@@ -2081,10 +2150,12 @@ TEST_CASE("/backup/continuous") {
 	// [0, 100) 2 tags
 	files.push_back({ 0, 100, 10, "file1", 100, 0, 2 }); // Tag 0: 0-100
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 0);
 
 	files.push_back({ 0, 100, 10, "file2", 200, 1, 2 }); // Tag 1: 0-100
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 99);
 
 	// [100, 300) 3 tags
 	files.push_back({ 100, 200, 10, "file3", 200, 0, 3 }); // Tag 0: 100-200
@@ -2093,17 +2164,21 @@ TEST_CASE("/backup/continuous") {
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 99);
 
 	files.push_back({ 100, 300, 10, "file5", 200, 2, 3 }); // Tag 2: 100-300
 	std::sort(files.begin(), files.end());
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 200));
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 10, 199));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 199);
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 100) == 199);
 
 	files.push_back({ 250, 300, 10, "file6", 200, 0, 3 }); // Tag 0: 250-300, missing 200-250
 	std::sort(files.begin(), files.end());
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 240));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 99) == 199);
 
 	files.push_back({ 250, 300, 10, "file7", 200, 1, 3 }); // Tag 1: 250-300
 	std::sort(files.begin(), files.end());
@@ -2113,6 +2188,7 @@ TEST_CASE("/backup/continuous") {
 	std::sort(files.begin(), files.end());
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 299));
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 150) == 299);
 
 	// [300, 400) 1 tag
 	// files.push_back({200, 250, 10, "file9", 200, 0, 3}); // Tag 0: 200-250, duplicate file
@@ -2122,6 +2198,9 @@ TEST_CASE("/backup/continuous") {
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 399));
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 150, 399));
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 250, 399));
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 399);
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 99) == 399);
+	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 250) == 399);
 
 	return Void();
 }
\ No newline at end of file

From 2eac17b553296bc992ffae49fafb5051e1ac4b7e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 24 Feb 2020 16:57:31 -0800
Subject: [PATCH 073/176] StagingKey can add out-of-order mutations

For partitioned logs, mutations of the same version may be sent to applier
out-of-order. If one loader advances to the next version, an applier may
receive later version mutations for different loaders. So, dropping of early
mutations is wrong.
---
 fdbserver/RestoreApplier.actor.h    | 36 ++++++++++++++-----------
 fdbserver/RestoreLoader.actor.cpp   | 41 +++++++++++++----------------
 fdbserver/RestoreRoleCommon.actor.h |  3 +++
 3 files changed, 41 insertions(+), 39 deletions(-)

diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index 9d8f6b60d8..a50c7f346f 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -60,21 +60,7 @@ struct StagingKey {
 	// Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set
 	void add(const MutationRef& m, LogMessageVersion newVersion) {
 		ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue);
-		if (version < newVersion) {
-			if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
-				key = m.param1;
-				val = m.param2;
-				type = (MutationRef::Type)m.type;
-				version = newVersion;
-			} else {
-				if (pendingMutations.find(newVersion) == pendingMutations.end()) {
-					pendingMutations.emplace(newVersion, MutationsVec());
-				}
-				// TODO: Do we really need deep copy?
-				MutationsVec& mutations = pendingMutations[newVersion];
-				mutations.push_back_deep(mutations.arena(), m);
-			}
-		} else if (version == newVersion) { // Sanity check
+		if (version == newVersion) { // Sanity check
 			TraceEvent("FastRestoreApplierStagingKeyMutationAtSameVersion")
 			    .detail("Version", newVersion.toString())
 			    .detail("NewMutation", m.toString())
@@ -106,7 +92,25 @@ struct StagingKey {
 				    .detail("ExistingKeyType", typeString[type])
 				    .detail("ExitingKeyValue", val);
 			}
-		} // else  input mutation is old and can be ignored
+		}
+		// newVersion can be smaller than version as different loaders can send
+		// mutations out of order.
+		if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
+			if (version < newVersion) {
+				key = m.param1;
+				val = m.param2;
+				type = (MutationRef::Type)m.type;
+				version = newVersion;
+			}
+		} else {
+			auto it = pendingMutations.find(newVersion);
+			if (it == pendingMutations.end()) {
+				bool inserted;
+				std::tie(it, inserted) = pendingMutations.emplace(newVersion, MutationsVec());
+			}
+			// TODO: Do we really need deep copy?
+			it->second.push_back_deep(it->second.arena(), m);
+		}
 	}
 
 	// Precompute the final value of the key.
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 15dbb3e179..109b25c43c 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -166,7 +166,6 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
 
 		VersionedMutationsMap& kvOps = kvOpsIter->second;
-		VersionedMutationsMap::iterator it = kvOps.end();
 		while (1) {
 			// If eof reached or first key len bytes is 0xFF then end of block was reached.
 			if (reader.eof() || *reader.rptr == 0xFF) break;
@@ -181,6 +180,7 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 			// Skip mutations out of the version range
 			if (!asset.isInVersionRange(msgVersion.version)) continue;
 
+			VersionedMutationsMap::iterator it;
 			bool inserted;
 			std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec());
 			ASSERT(inserted);
@@ -327,7 +327,6 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
 ACTOR Future<Void> handleSendMutationsRequest(RestoreSendMutationsToAppliersRequest req,
                                               Reference<RestoreLoaderData> self) {
 	state Reference<LoaderBatchData> batchData = self->batch[req.batchIndex];
-	state std::map<LoadingParam, VersionedMutationsMap>::iterator item = batchData->kvOpsPerLP.begin();
 	state Reference<LoaderBatchStatus> batchStatus = self->status[req.batchIndex];
 	state bool isDuplicated = true;
 
@@ -377,11 +376,11 @@ ACTOR Future<Void> handleSendMutationsRequest(RestoreSendMutationsToAppliersRequ
 	if (!isDuplicated) {
 		vector<Future<Void>> fSendMutations;
 		batchData->rangeToApplier = req.rangeToApplier;
-		for (; item != batchData->kvOpsPerLP.end(); item++) {
-			if (item->first.isRangeFile == req.useRangeFile) {
+		for (auto& [loadParam, kvOps] : batchData->kvOpsPerLP) {
+			if (loadParam.isRangeFile == req.useRangeFile) {
 				// Send the parsed mutation to applier who will apply the mutation to DB
-				fSendMutations.push_back(sendMutationsToApplier(&item->second, req.batchIndex, item->first.asset,
-				                                                item->first.isRangeFile, &batchData->rangeToApplier,
+				fSendMutations.push_back(sendMutationsToApplier(&kvOps, req.batchIndex, loadParam.asset,
+				                                                loadParam.isRangeFile, &batchData->rangeToApplier,
 				                                                &self->appliersInterf));
 			}
 		}
@@ -423,7 +422,7 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 	    .detail("RestoreAsset", asset.toString());
 
 	// There should be no mutation at asset.endVersion version because it is exclusive
-	if (kvOps.find(LogMessageVersion(asset.endVersion)) != kvOps.end()) {
+	if (kvOps.lower_bound(LogMessageVersion(asset.endVersion)) != kvOps.end()) {
 		TraceEvent(SevError, "FastRestoreLoaderSendMutationToApplier")
 		    .detail("BatchIndex", batchIndex)
 		    .detail("RestoreAsset", asset.toString())
@@ -449,12 +448,6 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 			applierMutationsSize[applierID] = 0.0;
 		}
 		const LogMessageVersion& commitVersion = kvOp->first;
-		if (!(commitVersion.version >= asset.beginVersion &&
-		      commitVersion.version <= asset.endVersion)) { // Debug purpose
-			TraceEvent(SevError, "FastRestore_SendMutationsToApplier")
-			    .detail("CommitVersion", commitVersion.version)
-			    .detail("RestoreAsset", asset.toString());
-		}
 		ASSERT(commitVersion.version >= asset.beginVersion);
 		ASSERT(commitVersion.version <= asset.endVersion); // endVersion is an empty commit to ensure progress
 
@@ -485,15 +478,14 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 				std::map<Key, UID>::iterator itlow = pRangeToApplier->upper_bound(kvm.param1);
 				--itlow; // make sure itlow->first <= m.param1
 				ASSERT(itlow->first <= kvm.param1);
-				MutationRef mutation = kvm;
 				UID applierID = itlow->second;
 				// printf("KV--Applier: K:%s ApplierID:%s\n", kvm.param1.toString().c_str(),
 				// applierID.toString().c_str());
 				kvCount++;
 
-				applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
+				applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), kvm);
 				applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
-				applierMutationsSize[applierID] += mutation.expectedSize();
+				applierMutationsSize[applierID] += kvm.expectedSize();
 			}
 		} // Mutations at the same version
 
@@ -606,20 +598,23 @@ bool concatenateBackupMutationForLogFile(std::map<Standalone<StringRef>, Standal
 	if (it == mutationMap.end()) {
 		mutationMap.insert(std::make_pair(id, val_input));
 		if (part != 0) {
-			TraceEvent(SevError, "FastRestore").detail("FirstPartNotZero", part).detail("KeyInput", getHexString(key_input));
+			TraceEvent(SevError, "FastRestore")
+			    .detail("FirstPartNotZero", part)
+			    .detail("KeyInput", getHexString(key_input));
 		}
 		mutationPartMap.insert(std::make_pair(id, part));
 	} else { // Concatenate the val string with the same commitVersion
 		it->second = it->second.contents().withSuffix(val_input.contents()); // Assign the new Areana to the map's value
-		if (part != (mutationPartMap[id] + 1)) {
+		auto& currentPart = mutationPartMap[id];
+		if (part != (currentPart + 1)) {
 			// Check if the same range or log file has been processed more than once!
 			TraceEvent(SevError, "FastRestore")
-				.detail("CurrentPart1", mutationPartMap[id])
-				.detail("CurrentPart2", part)
-				.detail("KeyInput", getHexString(key_input))
-				.detail("Hint", "Check if the same range or log file has been processed more than once");
+			    .detail("CurrentPart1", currentPart)
+			    .detail("CurrentPart2", part)
+			    .detail("KeyInput", getHexString(key_input))
+			    .detail("Hint", "Check if the same range or log file has been processed more than once");
 		}
-		mutationPartMap[id] = part;
+		currentPart = part;
 		concatenated = true;
 	}
 
diff --git a/fdbserver/RestoreRoleCommon.actor.h b/fdbserver/RestoreRoleCommon.actor.h
index cedbeb795c..5a3b30509a 100644
--- a/fdbserver/RestoreRoleCommon.actor.h
+++ b/fdbserver/RestoreRoleCommon.actor.h
@@ -55,6 +55,9 @@ struct RestoreSimpleRequest;
 // Value MutationsVec is the vector of parsed backup mutations.
 // For old mutation logs, the subsequence number is always 0.
 // For partitioned mutation logs, each mutation has a unique LogMessageVersion.
+// Note for partitioned logs, one LogMessageVersion can have multiple mutations,
+// because a clear mutation may be split into several smaller clear mutations by
+// backup workers.
 using VersionedMutationsMap = std::map<LogMessageVersion, MutationsVec>;
 
 ACTOR Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name);

From 1f95cba53ec211900bfc7593aa65dbc4f363c5a5 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 25 Feb 2020 16:37:25 -0800
Subject: [PATCH 074/176] Add describePartitionedBackup() for parallel restore

For partitioned logs, computing continuous log end version from min logs begin
version. Old backup test keeps using describeBackup() to be correctness clean.

Rename partitioned log file so that the last number is block size.
---
 fdbclient/BackupContainer.actor.cpp           | 125 +++++++++---------
 fdbclient/BackupContainer.h                   |   3 +
 fdbserver/RestoreMaster.actor.cpp             |   2 +-
 ...kupAndParallelRestoreCorrectness.actor.cpp |   4 +-
 4 files changed, 69 insertions(+), 65 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 7b7f27c14e..01d76bfb87 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -240,7 +240,7 @@ std::string BackupDescription::toJSON() const {
  *     file written will be after the start version of the snapshot's execution.
  *
  *   Log files are at file paths like
- *       /plogs/...log,startVersion,endVersion,UID,blocksize,tagID-of-N
+ *       /plogs/...log,startVersion,endVersion,UID,tagID-of-N,blocksize
  *       /logs/.../log,startVersion,endVersion,UID,blockSize
  *     where ... is a multi level path which sorts lexically into version order and results in approximately 1
  *     unique folder per day containing about 5,000 files. Logs after 7.0 are stored in "plogs"
@@ -347,9 +347,9 @@ public:
 	Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
 	                                                  uint16_t tagId, int totalTags) final {
 		return writeFile(logVersionFolderString(beginVersion, true) +
-		                 format("log,%lld,%lld,%s,%d,%d-of-%d", beginVersion, endVersion,
-		                        deterministicRandom()->randomUniqueID().toString().c_str(), blockSize, tagId,
-		                        totalTags));
+		                 format("log,%lld,%lld,%s,%d-of-%d,%d", beginVersion, endVersion,
+		                        deterministicRandom()->randomUniqueID().toString().c_str(), tagId, totalTags,
+		                        blockSize));
 	}
 
 	Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) override {
@@ -400,8 +400,8 @@ public:
 		if(sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n", &f.beginVersion, &f.endVersion, &f.blockSize, &len) == 3 && len == name.size()) {
 			out = f;
 			return true;
-		} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u,%d-of-%d%n", &f.beginVersion,
-		                  &f.endVersion, &f.blockSize, &f.tagId, &f.totalTags, &len) == 5 &&
+		} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%d-of-%d,%u%n", &f.beginVersion,
+		                  &f.endVersion, &f.tagId, &f.totalTags, &f.blockSize, &len) == 5 &&
 		           len == name.size() && f.tagId >= 0) {
 			out = f;
 			return true;
@@ -672,7 +672,27 @@ public:
 		return v;
 	}
 
-	ACTOR static Future<BackupDescription> describeBackup_impl(Reference<BackupContainerFileSystem> bc, bool deepScan, Version logStartVersionOverride) {
+	// Computes the continuous end version for non-partitioned mutation logs up to
+	// the "targetVersion". If "outLogs" is not nullptr, it will be updated with
+	// continuous log files. "*end" is updated with the continuous end version.
+	static void computeRestoreEndVersion(const std::vector<LogFile>& logs, std::vector<LogFile>* outLogs, Version* end,
+	                                     Version targetVersion) {
+		auto i = logs.begin();
+		if (outLogs != nullptr) outLogs->push_back(*i);
+
+		// Add logs to restorable logs set until continuity is broken OR we reach targetVersion
+		while (++i != logs.end()) {
+			if (i->beginVersion > *end || i->beginVersion > targetVersion) break;
+
+			// If the next link in the log chain is found, update the end
+			if (i->beginVersion == *end) {
+				if (outLogs != nullptr) outLogs->push_back(*i);
+				*end = i->endVersion;
+			}
+		}
+	}
+
+	ACTOR static Future<BackupDescription> describeBackup_impl(Reference<BackupContainerFileSystem> bc, bool deepScan, Version logStartVersionOverride, bool partitioned) {
 		state BackupDescription desc;
 		desc.url = bc->getURL();
 
@@ -690,8 +710,10 @@ public:
 		// from which to resolve the relative version.
 		// This could be handled more efficiently without recursion but it's tricky, this will do for now.
 		if(logStartVersionOverride != invalidVersion && logStartVersionOverride < 0) {
-			BackupDescription tmp = wait(bc->describeBackup(false, invalidVersion));
-			logStartVersionOverride = resolveRelativeVersion(tmp.maxLogEnd, logStartVersionOverride, "LogStartVersionOverride", invalid_option_value());
+			BackupDescription tmp = wait(partitioned ? bc->describePartitionedBackup(false, invalidVersion)
+			                                         : bc->describeBackup(false, invalidVersion));
+			logStartVersionOverride = resolveRelativeVersion(tmp.maxLogEnd, logStartVersionOverride,
+			                                                 "LogStartVersionOverride", invalid_option_value());
 		}
 
 		// Get metadata versions
@@ -777,45 +799,31 @@ public:
 		}
 
 		state std::vector<LogFile> logs;
-		state std::vector<LogFile> pLogs;
-		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) &&
-		     store(pLogs, bc->listLogFiles(scanBegin, scanEnd, true)) &&
+		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, partitioned)) &&
 		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
 
 		// List logs in version order so log continuity can be analyzed
 		std::sort(logs.begin(), logs.end());
 
-		// Check partitioned logs
-		if (!pLogs.empty()) {
-			std::sort(pLogs.begin(), pLogs.end());
+		// Find out contiguous log end version
+		if (partitioned) {
 			// If we didn't get log versions above then seed them using the first log file
-			if(!desc.contiguousLogEnd.present()) {
-				auto it = pLogs.begin();
-				desc.minLogBegin = it->beginVersion;
-				desc.contiguousLogEnd = it->endVersion;
+			if (!desc.contiguousLogEnd.present()) {
+				desc.minLogBegin = logs.begin()->beginVersion;
+				desc.contiguousLogEnd = logs.begin()->endVersion;
 			}
-			desc.contiguousLogEnd.get() = getPartitionedLogsContinuousEndVersion(pLogs, scanBegin);
+			// contiguousLogEnd is not inclusive, so +1 here.
+			desc.contiguousLogEnd.get() = getPartitionedLogsContinuousEndVersion(logs, desc.minLogBegin.get()) + 1;
 		} else if (!logs.empty()) {
 			desc.maxLogEnd = logs.rbegin()->endVersion;
 
-			auto i = logs.begin();
 			// If we didn't get log versions above then seed them using the first log file
 			if(!desc.contiguousLogEnd.present()) {
-				desc.minLogBegin = i->beginVersion;
-				desc.contiguousLogEnd = i->endVersion;
-				++i;
-			}
-			auto &end = desc.contiguousLogEnd.get();  // For convenience to make loop cleaner
-
-			// Advance until continuity is broken
-			while(i != logs.end()) {
-				if(i->beginVersion > end)
-					break;
-				// If the next link in the log chain is found, update the end
-				if(i->beginVersion == end)
-					end = i->endVersion;
-				++i;
+				desc.minLogBegin = logs.begin()->beginVersion;
+				desc.contiguousLogEnd = logs.begin()->endVersion;
 			}
+			Version& end = desc.contiguousLogEnd.get();
+			computeRestoreEndVersion(logs, nullptr, &end, std::numeric_limits<Version>::max());
 		}
 
 		// Only update stored contiguous log begin and end versions if we did NOT use a log start override.
@@ -884,7 +892,11 @@ public:
 
 	// Uses the virtual methods to describe the backup contents
 	Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) final {
-		return describeBackup_impl(Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride);
+		return describeBackup_impl(Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride, false);
+	}
+
+	Future<BackupDescription> describePartitionedBackup(bool deepScan, Version logStartVersionOverride) final {
+		return describeBackup_impl(Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride, true);
 	}
 
 	ACTOR static Future<Void> expireData_impl(Reference<BackupContainerFileSystem> bc, Version expireEndVersion, bool force, ExpireProgress *progress, Version restorableBeginVersion) {
@@ -1175,7 +1187,7 @@ for (auto file : files) std::cout << file.toString() << "\n";
 			indices.push_back(i);
 			end = files[i].endVersion - 1;
 		}
-std::cout << "Init end: " << end << "\n";
+std::cout << "Init end: " << end << ", begin " << begin << "\n";
 
 		// check tag 0 is continuous in [begin, end] and create a map of ranges to tags
 		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
@@ -1249,22 +1261,9 @@ std::cout << "Return end = " << end << "\n\n";
 
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
 			if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {
-				auto i = logs.begin();
-				Version end = i->endVersion;
-				restorable.logs.push_back(*i);
-
-				// Add logs to restorable logs set until continuity is broken OR we reach targetVersion
-				while(++i != logs.end()) {
-					if(i->beginVersion > end || i->beginVersion > targetVersion)
-						break;
-					// If the next link in the log chain is found, update the end
-					if(i->beginVersion == end) {
-						restorable.logs.push_back(*i);
-						end = i->endVersion;
-					}
-				}
-
-				if(end >= targetVersion) {
+				Version end = logs.begin()->endVersion;
+				computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion);
+				if (end >= targetVersion) {
 					return Optional<RestorableFileSet>(restorable);
 				}
 			}
@@ -1460,6 +1459,7 @@ public:
 			if(deterministicRandom()->random01() < .01) {
 				blockSize /= deterministicRandom()->randomInt(1, 3);
 			}
+			ASSERT(blockSize > 0);
 
 			return map(f, [=](Reference<IAsyncFile> fr) {
 				int readAhead = deterministicRandom()->randomInt(0, 3);
@@ -1609,15 +1609,16 @@ public:
 	virtual ~BackupContainerBlobStore() {}
 
 	Future<Reference<IAsyncFile>> readFile(std::string path) final {
-			return Reference<IAsyncFile>(
-				new AsyncFileReadAheadCache(
-					Reference<IAsyncFile>(new AsyncFileBlobStoreRead(m_bstore, m_bucket, dataPath(path))),
-					m_bstore->knobs.read_block_size,
-					m_bstore->knobs.read_ahead_blocks,
-					m_bstore->knobs.concurrent_reads_per_file,
-					m_bstore->knobs.read_cache_blocks_per_file
-				)
-			);
+		ASSERT(m_bstore->knobs.read_ahead_blocks > 0);
+		return Reference<IAsyncFile>(
+			new AsyncFileReadAheadCache(
+				Reference<IAsyncFile>(new AsyncFileBlobStoreRead(m_bstore, m_bucket, dataPath(path))),
+				m_bstore->knobs.read_block_size,
+				m_bstore->knobs.read_ahead_blocks,
+				m_bstore->knobs.concurrent_reads_per_file,
+				m_bstore->knobs.read_cache_blocks_per_file
+			)
+		);
 	}
 
 	ACTOR static Future<std::vector<std::string>> listURLs(Reference<BlobStoreEndpoint> bstore, std::string bucket) {
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 437f6e3eaa..3eba09f06f 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -255,6 +255,9 @@ public:
 	// be after deleting all data prior to logStartVersionOverride.
 	virtual Future<BackupDescription> describeBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0;
 
+	// The same as above, except using partitioned mutation logs.
+	virtual Future<BackupDescription> describePartitionedBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0;
+
 	virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0;
 
 	// Get exactly the files necessary to restore to targetVersion.  Returns non-present if
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 2fba9204d2..a4da897650 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -676,7 +676,7 @@ ACTOR static Future<Standalone<VectorRef<RestoreRequest>>> collectRestoreRequest
 ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
                                              std::vector<RestoreFileFR>* logFiles, Database cx,
                                              RestoreRequest request) {
-	state BackupDescription desc = wait(bc->describeBackup());
+	state BackupDescription desc = wait(bc->describePartitionedBackup());
 
 	// Convert version to real time for operators to read the BackupDescription desc.
 	wait(desc.resolveVersionTimes(cx));
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 2435d3f4e0..6a1fec7129 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -209,7 +209,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 
 					state bool restorable = false;
 					if(lastBackupContainer) {
-						state Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
+						state Future<BackupDescription> fdesc = lastBackupContainer->describePartitionedBackup();
 						wait(ready(fdesc));
 
 						if(!fdesc.isError()) {
@@ -430,7 +430,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				    .detail("BackupTag", printable(self->backupTag));
 
 				auto container = IBackupContainer::openContainer(lastBackupContainer->getURL());
-				BackupDescription desc = wait(container->describeBackup());
+				BackupDescription desc = wait(container->describePartitionedBackup());
 
 				state Version targetVersion = -1;
 				if (desc.maxRestorableVersion.present()) {

From 05b87cf288006648e2fd7ed112d2d3037f61ab6d Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 27 Feb 2020 14:04:19 -0800
Subject: [PATCH 075/176] Partitioned logs need to compute continuous begin
 Version

Because different tags may start at different versions, tag 0 can start at a
higher version. In this case, another tag's high version should be used as
the start version for continuous logs.
---
 fdbclient/BackupContainer.actor.cpp | 63 ++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 23 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 01d76bfb87..097feaf2af 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -799,6 +799,7 @@ public:
 		}
 
 		state std::vector<LogFile> logs;
+std::cout << "describe list: scanBegin:" << scanBegin << ", scanEnd:" << scanEnd << ", partitioned:" << partitioned << "\n";
 		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, partitioned)) &&
 		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
 
@@ -806,24 +807,20 @@ public:
 		std::sort(logs.begin(), logs.end());
 
 		// Find out contiguous log end version
-		if (partitioned) {
+		if (!logs.empty()) {
+			desc.maxLogEnd = logs.rbegin()->endVersion;
 			// If we didn't get log versions above then seed them using the first log file
 			if (!desc.contiguousLogEnd.present()) {
 				desc.minLogBegin = logs.begin()->beginVersion;
 				desc.contiguousLogEnd = logs.begin()->endVersion;
 			}
-			// contiguousLogEnd is not inclusive, so +1 here.
-			desc.contiguousLogEnd.get() = getPartitionedLogsContinuousEndVersion(logs, desc.minLogBegin.get()) + 1;
-		} else if (!logs.empty()) {
-			desc.maxLogEnd = logs.rbegin()->endVersion;
 
-			// If we didn't get log versions above then seed them using the first log file
-			if(!desc.contiguousLogEnd.present()) {
-				desc.minLogBegin = logs.begin()->beginVersion;
-				desc.contiguousLogEnd = logs.begin()->endVersion;
+			if (partitioned) {
+				determinePartitionedLogsBeginEnd(&desc, logs);
+			} else {
+				Version& end = desc.contiguousLogEnd.get();
+				computeRestoreEndVersion(logs, nullptr, &end, std::numeric_limits<Version>::max());
 			}
-			Version& end = desc.contiguousLogEnd.get();
-			computeRestoreEndVersion(logs, nullptr, &end, std::numeric_limits<Version>::max());
 		}
 
 		// Only update stored contiguous log begin and end versions if we did NOT use a log start override.
@@ -1091,8 +1088,8 @@ public:
 	// nullptr, then it will be populated with [begin, end] -> tags, where next
 	// pair's begin == previous pair's end + 1. On return, the last pair's end
 	// version (inclusive) gives the continuous range from begin.
-	static bool isContinuous(const std::vector<LogFile>& files, std::vector<int> indices, Version begin, Version end,
-	                         std::map<std::pair<Version, Version>, int>* tags) {
+	static bool isContinuous(const std::vector<LogFile>& files, const std::vector<int>& indices, Version begin,
+	                         Version end, std::map<std::pair<Version, Version>, int>* tags) {
 		Version lastBegin = invalidVersion;
 		Version lastEnd = invalidVersion;
 		int lastTags = -1;
@@ -1100,7 +1097,7 @@ public:
 		ASSERT(tags == nullptr || tags->empty());
 		for (int idx : indices) {
 			const LogFile& file = files[idx];
-std::cout << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags" << lastTags << "\n";
+std::cout << "  " << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags " << lastTags << "\n";
 			if (lastEnd == invalidVersion) {
 				if (file.beginVersion > begin) return false;
 				if (file.endVersion > begin) {
@@ -1126,7 +1123,7 @@ std::cout << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd "
 			lastEnd = file.endVersion;
 			if (lastEnd > end) break;
 		}
-std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags" << lastTags << "\n";
+std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags " << lastTags << "\n";
 		if (tags != nullptr && lastBegin != invalidVersion) {
 			tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags);
 		}
@@ -1159,9 +1156,8 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 	}
 
 	// Returns log files that are not duplicated.
-	static std::vector<LogFile> filterDuplicates(std::vector<LogFile>& logs) {
-		std::sort(logs.begin(), logs.end());
-
+	// PRE-CONDITION: logs are already sorted.
+	static std::vector<LogFile> filterDuplicates(const std::vector<LogFile>& logs) {
 		std::vector<LogFile> filtered;
 		int i = 0;
 		for (int j = 1; j < logs.size(); j++) {
@@ -1174,10 +1170,30 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 		return filtered;
 	}
 
+	// Analyze partitioned logs and set minLogBegin and contiguousLogEnd.
+	// For partitioned logs, different tags may start at different versions, so
+	// we need to find the "minLogBegin" version as well.
+	static void determinePartitionedLogsBeginEnd(BackupDescription* desc, const std::vector<LogFile>& logs) {
+		if (logs.empty()) return;
+
+		for (const LogFile& file : logs) {
+			Version end = getPartitionedLogsContinuousEndVersion(logs, file.beginVersion);
+std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
+			if (end > file.beginVersion) {
+				desc->minLogBegin = file.beginVersion;
+				// contiguousLogEnd is not inclusive, so +1 here.
+				desc->contiguousLogEnd.get() = end + 1;
+				return;
+			}
+		}
+	}
+
 	// Returns the end version such that [begin, end] is continuous.
-	static Version getPartitionedLogsContinuousEndVersion(std::vector<LogFile>& logs, Version begin) {
+	// "logs" should be already sorted.
+	static Version getPartitionedLogsContinuousEndVersion(const std::vector<LogFile>& logs, Version begin) {
 		auto files = filterDuplicates(logs);
-for (auto file : files) std::cout << file.toString() << "\n";
+std::cout << "getPartitionedLogsContinuousEndVersion begin:" << begin << "\n";
+for (auto file : files) std::cout << "    " << file.toString() << "\n";
 		Version end = 0;
 
 		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
@@ -1185,7 +1201,7 @@ for (auto file : files) std::cout << file.toString() << "\n";
 			ASSERT(files[i].tagId >= 0 && files[i].tagId < files[i].totalTags);
 			auto& indices = tagIndices[files[i].tagId];
 			indices.push_back(i);
-			end = files[i].endVersion - 1;
+			end = std::max(end, files[i].endVersion - 1);
 		}
 std::cout << "Init end: " << end << ", begin " << begin << "\n";
 
@@ -1194,7 +1210,7 @@ std::cout << "Init end: " << end << ", begin " << begin << "\n";
 		isContinuous(files, tagIndices[0], begin, end, &tags);
 		if (tags.empty() || end <= begin) return 0;
 		end = std::min(end, tags.rbegin()->first.second);
-std::cout << "Tag 0 end: " << end << "\n";
+std::cout << "  Tag 0 end: " << end << "\n";
 for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " << v << "\n";
 
 		// for each range in tags, check all tags from 1 are continouous
@@ -1205,7 +1221,7 @@ for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " <
 				std::map<std::pair<Version, Version>, int> rangeTags;
 				isContinuous(files, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
 				tagEnd = rangeTags.empty() ? 0 : std::min(tagEnd, rangeTags.rbegin()->first.second);
-std::cout << "Tag " << i << " end: " << tagEnd << "\n";
+std::cout << "  Tag " << i << " end: " << tagEnd << ", return end = "<< lastEnd << "\n";
 				if (tagEnd == 0) return lastEnd;
 			}
 			if (tagEnd < beginEnd.second) {
@@ -2154,6 +2170,7 @@ TEST_CASE("/backup/continuous") {
 	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 0);
 
 	files.push_back({ 0, 100, 10, "file2", 200, 1, 2 }); // Tag 1: 0-100
+	std::sort(files.begin(), files.end());
 	ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99));
 	ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100));
 	ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 99);

From 86edc1c9c8b56025f8e1740d387493d69496c0be Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 27 Feb 2020 19:51:12 -0800
Subject: [PATCH 076/176] Fix backup worker does NOOP pop before getting backup
 key

The NOOP pop cuases some mutation ranges being dropped by backup workers. As a
result, the backup is incomplete. Specifically, the wait of BACKUP_NOOP_POP_DELAY
blocks the monitoring of backup key actor.
---
 fdbserver/BackupWorker.actor.cpp | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index f2f4d675f8..9c1b7e6290 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -591,25 +591,25 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 
 ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
 	state Future<Void> started, pullFinished;
+	state Future<GetReadVersionReply> replyFuture = Never();
 
 	loop {
 		started = monitorBackupStartedKeyChanges(self, true);
-		loop {
-			GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
-			                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
-
-			choose {
-				when(wait(started)) { break; }
-				when(wait(self->cx->onMasterProxiesChanged())) {}
-				when(GetReadVersionReply reply = wait(loadBalance(self->cx->getMasterProxies(false),
-				                                                  &MasterProxyInterface::getConsistentReadVersion,
-				                                                  request, self->cx->taskID))) {
-					self->savedVersion = std::max(reply.version, self->savedVersion);
-					self->minKnownCommittedVersion = std::max(reply.version, self->minKnownCommittedVersion);
-					TraceEvent("BackupWorkerNoopPop", self->myId).detail("SavedVersion", self->savedVersion);
-					self->pop(); // Pop while the worker is in this NOOP state.
-					wait(delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID));
-				}
+		loop choose {
+			when(wait(started)) { break; }
+			when(wait(self->cx->onMasterProxiesChanged() ||
+			          delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
+				GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
+				                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
+				replyFuture = loadBalance(self->cx->getMasterProxies(false),
+				                          &MasterProxyInterface::getConsistentReadVersion, request, self->cx->taskID);
+			}
+			when(GetReadVersionReply reply = wait(replyFuture)) {
+				replyFuture = Never();
+				self->savedVersion = std::max(reply.version, self->savedVersion);
+				self->minKnownCommittedVersion = std::max(reply.version, self->minKnownCommittedVersion);
+				TraceEvent("BackupWorkerNoopPop", self->myId).detail("SavedVersion", self->savedVersion);
+				self->pop(); // Pop while the worker is in this NOOP state.
 			}
 		}
 

From c300a5c1b7d98b0cc657a6326e82d65f1b3565da Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 28 Feb 2020 14:11:14 -0800
Subject: [PATCH 077/176] Fix contract changes: backup worker generate
 continuous versions

Before we allow holes in version ranges in partitioned mutation logs. This
has been changed so that restore can easily figure out if database is
restorable. A specific problem is that if the backup worker didn't find any
mutations for an old epoch, the worker can just exit without generating a
log file, thus leaving holes in version ranges.

Another contract change is that if a backup key is set, then we must store
all mutations for that key, especially for the worker for the old epoch. As a
result, the worker must first check backup key, before pulling mutations and
uploading logs. Otherwise, we may lose mutations.

Finally, when a backup key is removed, the saving of mutations should be up to
the current version so that backup worker doesn't exit too early. I.e., avoid
the case saved mutation versions are less than the snapshot version taken.
---
 fdbserver/BackupWorker.actor.cpp | 142 ++++++++++++++++++++++---------
 1 file changed, 102 insertions(+), 40 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 9c1b7e6290..3681bd2fdb 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -78,15 +78,30 @@ struct BackupData {
 	Database cx;
 	std::vector<VersionedMessage> messages;
 	AsyncVar<bool> pullFinished;
+	NotifiedVersion pulledVersion;
 
 	struct PerBackupInfo {
 		PerBackupInfo() = default;
 		PerBackupInfo(BackupData* data, Version v) : self(data), startVersion(v) {}
 
-		bool isRunning() {
+		bool isReady() const {
+			return stopped || (container.isReady() && ranges.isReady());
+		}
+
+		bool isRunning() const {
 			return container.isReady() && ranges.isReady() && !stopped;
 		}
 
+		Future<Void> waitReady() {
+			if (stopped) return Void();
+			return _waitReady(this);
+		}
+
+		ACTOR static Future<Void> _waitReady(PerBackupInfo* info) {
+			wait(success(info->container) && success(info->ranges));
+			return Void();
+		}
+
 		BackupData* self = nullptr;
 		Version startVersion = invalidVersion;
 		Version lastSavedVersion = invalidVersion;
@@ -105,7 +120,8 @@ struct BackupData {
 	explicit BackupData(UID id, Reference<AsyncVar<ServerDBInfo>> db, const InitializeBackupRequest& req)
 	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
 	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
-	    minKnownCommittedVersion(invalidVersion), savedVersion(invalidVersion), cc("BackupWorker", myId.toString()) {
+	    minKnownCommittedVersion(invalidVersion), savedVersion(invalidVersion), cc("BackupWorker", myId.toString()),
+	    pulledVersion(0) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
 		pullFinished.set(false);
 
@@ -206,11 +222,38 @@ struct BackupData {
 		}
 		if (modified) changedTrigger.trigger();
 	}
+
+	ACTOR static Future<Void> _waitAllInfoReady(BackupData* self) {
+		std::vector<Future<Void>> all;
+		for (auto it = self->backups.begin(); it != self->backups.end(); ) {
+			if (it->second.stopped) {
+				TraceEvent("BackupWorkerRemoveStoppedContainer", self->myId).detail("BackupId", it->first);
+				it = self->backups.erase(it);
+				continue;
+			}
+			all.push_back(it->second.waitReady());
+			it++;
+		}
+		wait(waitForAll(all));
+		return Void();
+	}
+
+	Future<Void> waitAllInfoReady() {
+		return _waitAllInfoReady(this);
+	}
+
+	bool isAllInfoReady() const {
+		for (const auto& [uid, info] : backups) {
+			if (!info.isReady()) return false;
+		}
+		return true;
+	}
 };
 
 // Monitors "backupStartedKey". If "started" is true, wait until the key is set;
-// otherwise, wait until the key is cleared.
-ACTOR Future<Void> monitorBackupStartedKeyChanges(BackupData* self, bool started) {
+// otherwise, wait until the key is cleared. If "watch" is false, do not perform
+// the wait for key set/clear events. Returns if key present.
+ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started, bool watch) {
 	loop {
 		state ReadYourWritesTransaction tr(self->cx);
 
@@ -230,13 +273,13 @@ ACTOR Future<Void> monitorBackupStartedKeyChanges(BackupData* self, bool started
 						i++;
 					}
 					self->onBackupChanges(uidVersions);
-					if (started) return Void();
+					if (started || !watch) return true;
 				} else {
 					TraceEvent("BackupWorkerEmptyStartKey", self->myId);
 					self->onBackupChanges(uidVersions);
 
-					if (!started) {
-						return Void();
+					if (!started || !watch) {
+						return false;
 					}
 				}
 
@@ -396,36 +439,24 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 	state std::vector<Standalone<StringRef>> mutations;
 	state int idx;
 
-	for (auto it = self->backups.begin(); it != self->backups.end();) {
-		if (!it->second.isRunning()) {
-			if (it->second.stopped) {
-				TraceEvent("BackupWorkerRemoveStoppedContainer", self->myId).detail("BackupId", it->first);
-				it = self->backups.erase(it);
-			} else {
-				it++;
-			}
-			continue;
-		}
-		if (!it->second.container.get().present()) {
-			TraceEvent("BackupWorkerNoContainer", self->myId).detail("BackupId", it->first);
-			it = self->backups.erase(it);
-			continue;
-		}
+	// Make sure all backups are ready, otherwise mutations will be lost.
+	while (!self->isAllInfoReady()) {
+		wait(self->waitAllInfoReady());
+	}
+
+	for (auto it = self->backups.begin(); it != self->backups.end(); it++) {
+		ASSERT(it->second.container.get().present());
 		const int index = logFileFutures.size();
 		activeUids.insert(it->first);
 		self->insertRanges(keyRangeMap, it->second.ranges.get(), index);
 		if (it->second.lastSavedVersion == invalidVersion) {
-			it->second.lastSavedVersion = self->messages[0].getVersion();
+			it->second.lastSavedVersion = self->messages.empty() ? self->savedVersion : self->messages[0].getVersion();
 		}
 		logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile(
 		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));
-		it++;
-	}
-	if (activeUids.empty()) {
-		// stop early if there is no active backups
-		TraceEvent("BackupWorkerSkip", self->myId).detail("Count", numMsg);
-		return Void();
 	}
+	ASSERT(!activeUids.empty());
+
 	keyRangeMap.coalesce(allKeys);
 	wait(waitForAll(logFileFutures));
 
@@ -505,27 +536,29 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 
 		const Version maxPopVersion =
 		    self->endVersion.present() ? self->endVersion.get() : self->minKnownCommittedVersion;
+		state int numMsg = 0;
 		if (self->messages.empty()) {
 			// Even though messages is empty, we still want to advance popVersion.
 			popVersion = std::max(popVersion, maxPopVersion);
 		} else {
-			state int numMsg = 0;
 			for (const auto& message : self->messages) {
 				// message may be prefetched in peek; uncommitted message should not be uploaded.
 				if (message.getVersion() > maxPopVersion) break;
 				popVersion = std::max(popVersion, message.getVersion());
 				numMsg++;
 			}
-			if (numMsg > 0) {
-				wait(saveMutationsToFile(self, popVersion, numMsg));
-				self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);
-			}
 		}
 		if (self->pullFinished.get() && self->messages.empty()) {
 			// Advance popVersion to the endVersion to avoid gap between last
 			// message version and the endVersion.
 			popVersion = self->endVersion.get();
 		}
+		if (numMsg > 0 || self->endVersion.present()) {
+			// save an empty file for old epochs so that log file versions are continuous
+			TraceEvent("BackupWorkerSave", self->myId).detail("PopVersion", popVersion).detail("MsgQ", self->messages.size());
+			wait(saveMutationsToFile(self, popVersion, numMsg));
+			self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);
+		}
 
 		if (popVersion > self->savedVersion) {
 			wait(saveProgress(self, popVersion));
@@ -574,6 +607,7 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		}
 
 		tagAt = r->version().version;
+		self->pulledVersion = tagAt;
 		TraceEvent("BackupWorkerGot", self->myId).suppressFor(1.0).detail("V", tagAt);
 		if (self->endVersion.present() && tagAt > self->endVersion.get()) {
 			self->eraseMessagesAfterEndVersion();
@@ -590,13 +624,17 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 }
 
 ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
-	state Future<Void> started, pullFinished;
+	state Future<Void> pullFinished = Void();
+	state Future<bool> started;
 	state Future<GetReadVersionReply> replyFuture = Never();
 
 	loop {
-		started = monitorBackupStartedKeyChanges(self, true);
+		started = monitorBackupStartedKeyChanges(self, true, true);
 		loop choose {
-			when(wait(started)) { break; }
+			when(bool present = wait(started)) {
+				replyFuture = Never();
+				break;
+			}
 			when(wait(self->cx->onMasterProxiesChanged() ||
 			          delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
 				GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
@@ -613,10 +651,28 @@ ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
 			}
 		}
 
-		Future<Void> stopped = monitorBackupStartedKeyChanges(self, false);
+		Future<bool> stopped = monitorBackupStartedKeyChanges(self, false, true);
 		pullFinished = pullAsyncData(self);
-		wait(stopped || pullFinished);
+		wait(success(stopped) || pullFinished);
 		if (pullFinished.isReady()) return Void(); // backup is done for some old epoch.
+
+		// Even though the snapshot is done, mutation logs may not be written
+		// out yet. We need to make usre mutations up to this point is written.
+		state Version currentVersion;
+		loop {
+			GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
+			                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
+			choose {
+				when(wait(self->cx->onMasterProxiesChanged())) {}
+				when(GetReadVersionReply reply = wait(loadBalance(self->cx->getMasterProxies(false),
+				                                                  &MasterProxyInterface::getConsistentReadVersion,
+				                                                  request, self->cx->taskID))) {
+					currentVersion = reply.version;
+					break;
+				}
+			}
+		}
+		wait(self->pulledVersion.whenAtLeast(currentVersion));
 		pullFinished = Future<Void>(); // cancels pullAsyncData()
 		TraceEvent("BackupWorkerPaused", self->myId);
 	}
@@ -655,13 +711,19 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 	    .detail("LogEpoch", req.recruitedEpoch)
 	    .detail("BackupEpoch", req.backupEpoch);
 	try {
-		addActor.send(monitorBackupKeyOrPullData(&self));
 		addActor.send(checkRemoved(db, req.recruitedEpoch, &self));
 		addActor.send(waitFailureServer(interf.waitFailure.getFuture()));
 		if (req.recruitedEpoch == req.backupEpoch && req.routerTag.id == 0) {
 			addActor.send(monitorAllWorkerStarted(&self));
 		}
 
+		// Check if backup key is present to avoid race between this check and
+		// noop pop as well as upload data: pop or skip upload before knowing
+		// there are backup keys.
+		bool present = wait(monitorBackupStartedKeyChanges(&self, true, false));
+		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present);
+
+		addActor.send(monitorBackupKeyOrPullData(&self));
 		state Future<Void> done = uploadData(&self);
 
 		loop choose {

From 672ad7a8eac645a2a4c4688b49d88beac87ea4a8 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 28 Feb 2020 14:38:11 -0800
Subject: [PATCH 078/176] Fix: backup worker savedVersion init to begin version

Choosing invalidVersion is wrong, as the worker starts at beginVersion.
---
 fdbserver/BackupWorker.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 3681bd2fdb..a34399e99d 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -120,7 +120,7 @@ struct BackupData {
 	explicit BackupData(UID id, Reference<AsyncVar<ServerDBInfo>> db, const InitializeBackupRequest& req)
 	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
 	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
-	    minKnownCommittedVersion(invalidVersion), savedVersion(invalidVersion), cc("BackupWorker", myId.toString()),
+	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion), cc("BackupWorker", myId.toString()),
 	    pulledVersion(0) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
 		pullFinished.set(false);
@@ -450,7 +450,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		activeUids.insert(it->first);
 		self->insertRanges(keyRangeMap, it->second.ranges.get(), index);
 		if (it->second.lastSavedVersion == invalidVersion) {
-			it->second.lastSavedVersion = self->messages.empty() ? self->savedVersion : self->messages[0].getVersion();
+			it->second.lastSavedVersion = self->savedVersion;
 		}
 		logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile(
 		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));

From 00350dd3d846df76b3bce00d644dd18f12d3bd58 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 28 Feb 2020 16:05:34 -0800
Subject: [PATCH 079/176] Fix pulledVersion of backup worker

Not sure why, the cursor's version can be smaller than before.
---
 fdbserver/BackupWorker.actor.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index a34399e99d..73af1037ac 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -607,7 +607,9 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		}
 
 		tagAt = r->version().version;
-		self->pulledVersion = tagAt;
+		if (tagAt > self->pulledVersion.get()) {
+			self->pulledVersion.set(tagAt);
+		}
 		TraceEvent("BackupWorkerGot", self->myId).suppressFor(1.0).detail("V", tagAt);
 		if (self->endVersion.present() && tagAt > self->endVersion.get()) {
 			self->eraseMessagesAfterEndVersion();

From 1b159a3785b2b08c34aba31c5e26e69e125867a6 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 28 Feb 2020 17:14:18 -0800
Subject: [PATCH 080/176] Fix: backup worker ignores deleted container

---
 fdbserver/BackupWorker.actor.cpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 73af1037ac..5501e8cb36 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -88,10 +88,6 @@ struct BackupData {
 			return stopped || (container.isReady() && ranges.isReady());
 		}
 
-		bool isRunning() const {
-			return container.isReady() && ranges.isReady() && !stopped;
-		}
-
 		Future<Void> waitReady() {
 			if (stopped) return Void();
 			return _waitReady(this);
@@ -231,6 +227,7 @@ struct BackupData {
 				it = self->backups.erase(it);
 				continue;
 			}
+
 			all.push_back(it->second.waitReady());
 			it++;
 		}
@@ -444,8 +441,12 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		wait(self->waitAllInfoReady());
 	}
 
-	for (auto it = self->backups.begin(); it != self->backups.end(); it++) {
-		ASSERT(it->second.container.get().present());
+	for (auto it = self->backups.begin(); it != self->backups.end();) {
+		if (!it->second.container.get().present()) {
+			TraceEvent("BackupWorkerNoContainer", self->myId).detail("BackupId", it->first);
+			it = self->backups.erase(it);
+			continue;
+		}
 		const int index = logFileFutures.size();
 		activeUids.insert(it->first);
 		self->insertRanges(keyRangeMap, it->second.ranges.get(), index);
@@ -454,6 +455,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		}
 		logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile(
 		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));
+		it++;
 	}
 	ASSERT(!activeUids.empty());
 

From e3eb3beaafd8a25eddf1384708bc4a532781f27d Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 2 Mar 2020 13:29:42 -0800
Subject: [PATCH 081/176] Consider previously pulled version for pulling
 version

Saving files only happens if we are not pulling, i.e., not in NOOP mode.
---
 fdbserver/BackupWorker.actor.cpp | 56 +++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 5501e8cb36..bbbdf90d0c 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -77,8 +77,8 @@ struct BackupData {
 	AsyncVar<Reference<ILogSystem>> logSystem;
 	Database cx;
 	std::vector<VersionedMessage> messages;
-	AsyncVar<bool> pullFinished;
 	NotifiedVersion pulledVersion;
+	bool pulling = false;
 
 	struct PerBackupInfo {
 		PerBackupInfo() = default;
@@ -119,7 +119,6 @@ struct BackupData {
 	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion), cc("BackupWorker", myId.toString()),
 	    pulledVersion(0) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
-		pullFinished.set(false);
 
 		specialCounter(cc, "SavedVersion", [this]() { return this->savedVersion; });
 		specialCounter(cc, "MinKnownCommittedVersion", [this]() { return this->minKnownCommittedVersion; });
@@ -128,6 +127,18 @@ struct BackupData {
 		                       "BackupWorkerMetrics");
 	}
 
+	bool pullFinished() const {
+		return endVersion.present() && pulledVersion.get() > endVersion.get();
+	}
+
+	bool allMessageSaved() const {
+		return endVersion.present() && savedVersion >= endVersion.get();
+	}
+
+	Version maxPopVersion() const {
+		return endVersion.present() ? endVersion.get() : minKnownCommittedVersion;
+	}
+
 	// Inserts a backup's single range into rangeMap.
 	template <class T>
 	void insertRange(KeyRangeMap<std::set<T>>& keyRangeMap, KeyRangeRef range, T value) {
@@ -528,7 +539,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 	state Version popVersion = invalidVersion;
 
 	loop {
-		if (self->endVersion.present() && self->savedVersion >= self->endVersion.get()) {
+		if (self->allMessageSaved()) {
 			self->messages.clear();
 			return Void();
 		}
@@ -536,28 +547,25 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 		// Too large uploadDelay will delay popping tLog data for too long.
 		state Future<Void> uploadDelay = delay(SERVER_KNOBS->BACKUP_UPLOAD_DELAY);
 
-		const Version maxPopVersion =
-		    self->endVersion.present() ? self->endVersion.get() : self->minKnownCommittedVersion;
 		state int numMsg = 0;
+		Version lastPopVersion = popVersion;
 		if (self->messages.empty()) {
 			// Even though messages is empty, we still want to advance popVersion.
-			popVersion = std::max(popVersion, maxPopVersion);
+			if (!self->endVersion.present()) {
+				popVersion = std::max(popVersion, self->minKnownCommittedVersion);
+			} else if (self->pullFinished()) {
+				popVersion = self->endVersion.get();
+			}
 		} else {
 			for (const auto& message : self->messages) {
 				// message may be prefetched in peek; uncommitted message should not be uploaded.
-				if (message.getVersion() > maxPopVersion) break;
+				if (message.getVersion() > self->maxPopVersion()) break;
 				popVersion = std::max(popVersion, message.getVersion());
 				numMsg++;
 			}
 		}
-		if (self->pullFinished.get() && self->messages.empty()) {
-			// Advance popVersion to the endVersion to avoid gap between last
-			// message version and the endVersion.
-			popVersion = self->endVersion.get();
-		}
-		if (numMsg > 0 || self->endVersion.present()) {
+		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling)) {
 			// save an empty file for old epochs so that log file versions are continuous
-			TraceEvent("BackupWorkerSave", self->myId).detail("PopVersion", popVersion).detail("MsgQ", self->messages.size());
 			wait(saveMutationsToFile(self, popVersion, numMsg));
 			self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);
 		}
@@ -572,8 +580,8 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			self->pop();
 		}
 
-		if (!self->pullFinished.get()) {
-			wait(uploadDelay || self->pullFinished.onChange());
+		if (!self->pullFinished()) {
+			wait(uploadDelay);
 		}
 	}
 }
@@ -582,7 +590,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 ACTOR Future<Void> pullAsyncData(BackupData* self) {
 	state Future<Void> logSystemChange = Void();
 	state Reference<ILogSystem::IPeekCursor> r;
-	state Version tagAt = std::max(self->startVersion, self->savedVersion);
+	state Version tagAt = std::max(self->pulledVersion.get(), std::max(self->startVersion, self->savedVersion));
 
 	TraceEvent("BackupWorkerPull", self->myId);
 	loop {
@@ -609,18 +617,15 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		}
 
 		tagAt = r->version().version;
-		if (tagAt > self->pulledVersion.get()) {
-			self->pulledVersion.set(tagAt);
-		}
+		self->pulledVersion.set(tagAt);
 		TraceEvent("BackupWorkerGot", self->myId).suppressFor(1.0).detail("V", tagAt);
-		if (self->endVersion.present() && tagAt > self->endVersion.get()) {
+		if (self->pullFinished()) {
 			self->eraseMessagesAfterEndVersion();
 			TraceEvent("BackupWorkerFinishPull", self->myId)
 			    .detail("Tag", self->tag.toString())
 			    .detail("VersionGot", tagAt)
 			    .detail("EndVersion", self->endVersion.get())
 			    .detail("MsgQ", self->messages.size());
-			self->pullFinished.set(true);
 			return Void();
 		}
 		wait(yield());
@@ -657,8 +662,12 @@ ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
 
 		Future<bool> stopped = monitorBackupStartedKeyChanges(self, false, true);
 		pullFinished = pullAsyncData(self);
+		self->pulling = true;
 		wait(success(stopped) || pullFinished);
-		if (pullFinished.isReady()) return Void(); // backup is done for some old epoch.
+		if (pullFinished.isReady()) {
+			self->pulling = false;
+			return Void(); // backup is done for some old epoch.
+		}
 
 		// Even though the snapshot is done, mutation logs may not be written
 		// out yet. We need to make usre mutations up to this point is written.
@@ -678,6 +687,7 @@ ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
 		}
 		wait(self->pulledVersion.whenAtLeast(currentVersion));
 		pullFinished = Future<Void>(); // cancels pullAsyncData()
+		self->pulling = false;
 		TraceEvent("BackupWorkerPaused", self->myId);
 	}
 }

From 31f7108eab17f2f2ee441b32a64b4d48b99340af Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 3 Mar 2020 16:25:21 -0800
Subject: [PATCH 082/176] Handle partial recovery in BackupProgress

A partial recovery can result in empty epoch that copies previous epoch's
version range. In this case, getOldEpochTagsVersionsInfo() will not return
previous epoch's information. To correctly compute the start version for a
backup worker, we need to check previous epoch's saved version. If they are
larger than this epoch's begin version, use previously saved version as the
start version.
---
 fdbserver/BackupProgress.actor.cpp | 46 +++++++++++++++++++++++-------
 fdbserver/BackupProgress.actor.h   | 12 ++++++++
 2 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 5492db7aa8..cc7cadaaa0 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -37,6 +37,20 @@ void BackupProgress::addBackupStatus(const WorkerBackupStatus& status) {
 	}
 }
 
+void BackupProgress::updateTagVersions(std::map<Tag, Version>* tagVersions, std::set<Tag>* tags,
+                                       const std::map<Tag, Version>& progress, Version endVersion, LogEpoch epoch) {
+	for (const auto& [tag, savedVersion] : progress) {
+		tags->erase(tag);
+		if (savedVersion < endVersion - 1) {
+			tagVersions->insert({ tag, savedVersion + 1 });
+			TraceEvent("BW", dbgid)
+			    .detail("OldEpoch", epoch)
+			    .detail("Tag", tag.toString())
+			    .detail("BeginVersion", savedVersion + 1)
+			    .detail("EndVersion", endVersion);
+		}
+	}
+}
 std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgress::getUnfinishedBackup() {
 	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit;
 
@@ -45,20 +59,30 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 	for (const auto& [epoch, info] : epochInfos) {
 		std::set<Tag> tags = enumerateLogRouterTags(info.logRouterTags);
 		std::map<Tag, Version> tagVersions;
-		auto progressIt = progress.find(epoch);
-		if (progressIt != progress.end()) {
-			for (const auto& [tag, savedVersion] : progressIt->second) {
-				tags.erase(tag);
-				if (savedVersion < info.epochEnd - 1) {
-					tagVersions.insert({ tag, savedVersion + 1 });
-					TraceEvent("BW", dbgid)
-					    .detail("OldEpoch", epoch)
-					    .detail("Tag", tag.toString())
-					    .detail("BeginVersion", savedVersion + 1)
-					    .detail("EndVersion", info.epochEnd);
+		auto progressIt = progress.lower_bound(epoch);
+		if (progressIt != progress.end() && progressIt->first == epoch) {
+			updateTagVersions(&tagVersions, &tags, progressIt->second, info.epochEnd, epoch);
+		} else {
+			auto rit = findPreviousProgress(epoch);
+			if (rit != progress.rend()) {
+				// A partial recovery can result in empty epoch that copies previous
+				// epoch's version range. In this case, we should check previous
+				// epoch's savedVersion.
+				int savedMore = 0;
+				for (auto [tag, version] : rit->second) {
+					if (version > info.epochBegin) {
+						savedMore++;
+					}
+				}
+				if (savedMore > 1) {
+					ASSERT(savedMore == rit->second.size()); // all tags should saved more
+					ASSERT(savedMore == info.logRouterTags); // Smae number as logRouterTags
+
+					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
 			}
 		}
+
 		for (const Tag tag : tags) { // tags without progress data
 			tagVersions.insert({ tag, info.epochBegin });
 			TraceEvent("BW", dbgid)
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index 90e93fc95e..be7ed26e89 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -78,6 +78,18 @@ private:
 		return tags;
 	}
 
+	// For each tag in progress, the saved version is smaller than endVersion - 1,
+	// add {tag, savedVersion+1} to tagVersions and remove the tag from "tags".
+	void updateTagVersions(std::map<Tag, Version>* tagVersions, std::set<Tag>* tags,
+	                       const std::map<Tag, Version>& progress, Version endVersion, LogEpoch epoch);
+
+	std::map<LogEpoch, std::map<Tag, Version>>::reverse_iterator findPreviousProgress(LogEpoch epoch) {
+		for (auto it = progress.rbegin(); it != progress.rend(); ++it) {
+			if (it->first < epoch) return it;
+		}
+		return progress.rend();
+	}
+
 	const UID dbgid;
 
 	// Note this MUST be iterated in ascending order.

From b792d76d629dc03b8af8b092c3c01933e040b92d Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 3 Mar 2020 21:04:50 -0800
Subject: [PATCH 083/176] Fix version gap in old epoch's backup

When pull finished and message queue is empty, we should use end version as the
popVersion for backup files. Otherwise, there might be a version gap between
last message and end version.
---
 fdbserver/BackupWorker.actor.cpp | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index bbbdf90d0c..15e8427bd8 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -553,8 +553,6 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			// Even though messages is empty, we still want to advance popVersion.
 			if (!self->endVersion.present()) {
 				popVersion = std::max(popVersion, self->minKnownCommittedVersion);
-			} else if (self->pullFinished()) {
-				popVersion = self->endVersion.get();
 			}
 		} else {
 			for (const auto& message : self->messages) {
@@ -564,7 +562,13 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 				numMsg++;
 			}
 		}
-		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling)) {
+		if (self->messages.empty() && self->pullFinished()) {
+			popVersion = self->endVersion.get();
+		}
+		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling) || self->pullFinished()) {
+			TraceEvent("BackupWorkerSave", self->myId)
+			    .detail("Version", popVersion)
+			    .detail("MsgQ", self->messages.size());
 			// save an empty file for old epochs so that log file versions are continuous
 			wait(saveMutationsToFile(self, popVersion, numMsg));
 			self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);

From 696ce6aa82614f96acca3cb050e88eaa2eaf312a Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 3 Mar 2020 21:15:36 -0800
Subject: [PATCH 084/176] Fix compiling error of reverse iterators

MacOS and Windows compiler doesn't like the use of "!=" operator of
std::map::reverse_iterator.
---
 fdbserver/BackupProgress.actor.cpp | 9 ++++-----
 fdbserver/BackupProgress.actor.h   | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index cc7cadaaa0..a9fd2fb365 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -64,19 +64,18 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 			updateTagVersions(&tagVersions, &tags, progressIt->second, info.epochEnd, epoch);
 		} else {
 			auto rit = findPreviousProgress(epoch);
-			if (rit != progress.rend()) {
+			if (!(rit == progress.rend())) {
 				// A partial recovery can result in empty epoch that copies previous
 				// epoch's version range. In this case, we should check previous
 				// epoch's savedVersion.
 				int savedMore = 0;
 				for (auto [tag, version] : rit->second) {
-					if (version > info.epochBegin) {
+					if (version >= info.epochBegin) {
 						savedMore++;
 					}
 				}
-				if (savedMore > 1) {
-					ASSERT(savedMore == rit->second.size()); // all tags should saved more
-					ASSERT(savedMore == info.logRouterTags); // Smae number as logRouterTags
+				if (savedMore > 0) {
+					ASSERT(info.logRouterTags == rit->second.size()); // Same number as logRouterTags
 
 					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index be7ed26e89..f7eacbe180 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -84,7 +84,7 @@ private:
 	                       const std::map<Tag, Version>& progress, Version endVersion, LogEpoch epoch);
 
 	std::map<LogEpoch, std::map<Tag, Version>>::reverse_iterator findPreviousProgress(LogEpoch epoch) {
-		for (auto it = progress.rbegin(); it != progress.rend(); ++it) {
+		for (auto it = progress.rbegin(); !(it == progress.rend()); ++it) {
 			if (it->first < epoch) return it;
 		}
 		return progress.rend();

From 20df67ee6ab4f871ee5da523cd4a1fe1fa2422dd Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 4 Mar 2020 10:52:51 -0800
Subject: [PATCH 085/176] Filter partitioned logs with subset relationship

If a log file's progress is not saved, a new log file will be generated
with the same begin version. Then we can have a file that contains a subset
of contents in another log file. During restore, we should filter out files
that their contents are subset of other files.
---
 fdbbackup/FileConverter.actor.cpp   |  4 ++--
 fdbclient/BackupContainer.actor.cpp | 11 +++++++----
 fdbclient/BackupContainer.h         |  7 ++++---
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp
index f0bffa73e1..67cd4738b6 100644
--- a/fdbbackup/FileConverter.actor.cpp
+++ b/fdbbackup/FileConverter.actor.cpp
@@ -81,10 +81,10 @@ std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, Vers
 	std::vector<LogFile> sorted;
 	int i = 0;
 	for (int j = 1; j < filtered.size(); j++) {
-		if (!filtered[i].sameContent(filtered[j])) {
+		if (!filtered[i].isSubset(filtered[j])) {
 			sorted.push_back(filtered[i]);
-			i = j;
 		}
+		i = j;
 	}
 	if (i < filtered.size()) {
 		sorted.push_back(filtered[i]);
diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 097feaf2af..1f1e0ea281 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1155,16 +1155,19 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 		return true;
 	}
 
-	// Returns log files that are not duplicated.
+	// Returns log files that are not duplicated, or subset of another log.
+	// If a log file's progress is not saved, a new log file will be generated
+	// with the same begin version. So we can have a file that contains a subset
+	// of contents in another log file.
 	// PRE-CONDITION: logs are already sorted.
 	static std::vector<LogFile> filterDuplicates(const std::vector<LogFile>& logs) {
 		std::vector<LogFile> filtered;
 		int i = 0;
 		for (int j = 1; j < logs.size(); j++) {
-			if (!logs[i].sameContent(logs[j])) {
+			if (!logs[i].isSubset(logs[j])) {
 				filtered.push_back(logs[i]);
-				i = j;
 			}
+			i = j;
 		}
 		if (i < logs.size()) filtered.push_back(logs[i]);
 		return filtered;
@@ -1180,7 +1183,7 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 			Version end = getPartitionedLogsContinuousEndVersion(logs, file.beginVersion);
 std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
 			if (end > file.beginVersion) {
-				desc->minLogBegin = file.beginVersion;
+				// desc->minLogBegin = file.beginVersion;
 				// contiguousLogEnd is not inclusive, so +1 here.
 				desc->contiguousLogEnd.get() = end + 1;
 				return;
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 3eba09f06f..4bf144c07e 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -82,9 +82,10 @@ struct LogFile {
 		return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
 	}
 
-	// Returns if two log files have the same content by comparing version range and tag ID.
-	bool sameContent(const LogFile& rhs) const {
-		return beginVersion == rhs.beginVersion && endVersion == rhs.endVersion && tagId == rhs.tagId;
+	// Returns if this log file contains a subset of content of the given file
+	// by comparing version range and tag ID.
+	bool isSubset(const LogFile& rhs) const {
+		return beginVersion == rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
 	}
 
 	std::string toString() const {

From 5afc23a0e16f69d6b30db856cb544ed9c30ea07c Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 4 Mar 2020 12:32:06 -0800
Subject: [PATCH 086/176] Give a chance for backup worker to finish writing
 files

If a backup worker is cancelled, wait until it finishes writing files so that
we don't need to create these files in the next epoch.
---
 fdbserver/BackupWorker.actor.cpp | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 15e8427bd8..be1b21c853 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -79,6 +79,7 @@ struct BackupData {
 	std::vector<VersionedMessage> messages;
 	NotifiedVersion pulledVersion;
 	bool pulling = false;
+	bool stopped = false;
 
 	struct PerBackupInfo {
 		PerBackupInfo() = default;
@@ -132,7 +133,7 @@ struct BackupData {
 	}
 
 	bool allMessageSaved() const {
-		return endVersion.present() && savedVersion >= endVersion.get();
+		return (endVersion.present() && savedVersion >= endVersion.get()) || stopped;
 	}
 
 	Version maxPopVersion() const {
@@ -562,7 +563,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 				numMsg++;
 			}
 		}
-		if (self->messages.empty() && self->pullFinished()) {
+		if (self->pullFinished()) {
 			popVersion = self->endVersion.get();
 		}
 		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling) || self->pullFinished()) {
@@ -720,6 +721,8 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 	state PromiseStream<Future<Void>> addActor;
 	state Future<Void> error = actorCollection(addActor.getFuture());
 	state Future<Void> dbInfoChange = Void();
+	state Future<Void> pull;
+	state Future<Void> done;
 
 	TraceEvent("BackupWorkerStart", self.myId)
 	    .detail("Tag", req.routerTag.toString())
@@ -741,8 +744,8 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		bool present = wait(monitorBackupStartedKeyChanges(&self, true, false));
 		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present);
 
-		addActor.send(monitorBackupKeyOrPullData(&self));
-		state Future<Void> done = uploadData(&self);
+		pull = monitorBackupKeyOrPullData(&self);
+		done = uploadData(&self);
 
 		loop choose {
 			when(wait(dbInfoChange)) {
@@ -771,9 +774,15 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 			when(wait(error)) {}
 		}
 	} catch (Error& e) {
-		TraceEvent("BackupWorkerTerminated", self.myId).error(e, true);
-		if (e.code() != error_code_actor_cancelled && e.code() != error_code_worker_removed) {
-			throw;
+		state Error err = e;
+		if (e.code() == error_code_worker_removed) {
+			pull = Void(); // cancels pulling
+			self.stopped = true;
+			wait(done);
+		}
+		TraceEvent("BackupWorkerTerminated", self.myId).error(err, true);
+		if (err.code() != error_code_actor_cancelled && err.code() != error_code_worker_removed) {
+			throw err;
 		}
 	}
 	return Void();

From 5ce9fc0e4cf592facdd6599c2bdc7ba36669b9ab Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 4 Mar 2020 14:07:42 -0800
Subject: [PATCH 087/176] Partitioned logs should be filtered after sorting by
 tag IDs

The default sorting by begin and end version doesn't work with duplicates
removal, as tags are also compared.
---
 fdbclient/BackupContainer.actor.cpp | 33 ++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 1f1e0ea281..e1a51ccf63 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1194,23 +1194,27 @@ std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
 	// Returns the end version such that [begin, end] is continuous.
 	// "logs" should be already sorted.
 	static Version getPartitionedLogsContinuousEndVersion(const std::vector<LogFile>& logs, Version begin) {
-		auto files = filterDuplicates(logs);
 std::cout << "getPartitionedLogsContinuousEndVersion begin:" << begin << "\n";
-for (auto file : files) std::cout << "    " << file.toString() << "\n";
+for (auto file : logs) std::cout << "    " << file.toString() << "\n";
 		Version end = 0;
 
 		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
-		for (int i = 0; i < files.size(); i++) {
-			ASSERT(files[i].tagId >= 0 && files[i].tagId < files[i].totalTags);
-			auto& indices = tagIndices[files[i].tagId];
-			indices.push_back(i);
-			end = std::max(end, files[i].endVersion - 1);
+		for (int i = 0; i < logs.size(); i++) {
+			ASSERT(logs[i].tagId >= 0 && logs[i].tagId < logs[i].totalTags);
+			auto& indices = tagIndices[logs[i].tagId];
+			// filter out if indices.back() is subset of files[i]
+			if (!indices.empty() && logs[indices.back()].isSubset(logs[i])) {
+				indices.back() = i;
+			} else {
+				indices.push_back(i);
+			}
+			end = std::max(end, logs[i].endVersion - 1);
 		}
 std::cout << "Init end: " << end << ", begin " << begin << "\n";
 
 		// check tag 0 is continuous in [begin, end] and create a map of ranges to tags
 		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
-		isContinuous(files, tagIndices[0], begin, end, &tags);
+		isContinuous(logs, tagIndices[0], begin, end, &tags);
 		if (tags.empty() || end <= begin) return 0;
 		end = std::min(end, tags.rbegin()->first.second);
 std::cout << "  Tag 0 end: " << end << "\n";
@@ -1222,7 +1226,7 @@ for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " <
 			Version tagEnd = end; // This range's minimum continous tag version
 			for (int i = 1; i < count; i++) {
 				std::map<std::pair<Version, Version>, int> rangeTags;
-				isContinuous(files, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
+				isContinuous(logs, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
 				tagEnd = rangeTags.empty() ? 0 : std::min(tagEnd, rangeTags.rbegin()->first.second);
 std::cout << "  Tag " << i << " end: " << tagEnd << ", return end = "<< lastEnd << "\n";
 				if (tagEnd == 0) return lastEnd;
@@ -1264,10 +1268,12 @@ std::cout << "Return end = " << end << "\n\n";
 			// FIXME: check if there are tagged logs. for each tag, there is no version gap.
 			state std::vector<LogFile> logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion, partitioned));
 
-			// List logs in version order so log continuity can be analyzed
-			std::sort(logs.begin(), logs.end());
-
 			if (partitioned) {
+				// sort by tag ID so that filterDuplicates works.
+				std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) {
+					return a.tagId < b.tagId || a.beginVersion < b.beginVersion || a.endVersion < b.endVersion;
+				});
+
 				// Remove duplicated log files that can happen for old epochs.
 				std::vector<LogFile> filtered = filterDuplicates(logs);
 
@@ -1278,6 +1284,9 @@ std::cout << "Return end = " << end << "\n\n";
 				return Optional<RestorableFileSet>();
 			}
 
+			// List logs in version order so log continuity can be analyzed
+			std::sort(logs.begin(), logs.end());
+
 			// If there are logs and the first one starts at or before the snapshot begin version then proceed
 			if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {
 				Version end = logs.begin()->endVersion;

From fe6b4a43980147b729957169801447f78b0fdc2e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 4 Mar 2020 16:27:24 -0800
Subject: [PATCH 088/176] Some correctness fixes

---
 fdbclient/BackupContainer.actor.cpp | 6 +++++-
 fdbserver/BackupProgress.actor.cpp  | 3 ++-
 fdbserver/BackupWorker.actor.cpp    | 6 ++++--
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index e1a51ccf63..69e0aa3378 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1271,13 +1271,17 @@ std::cout << "Return end = " << end << "\n\n";
 			if (partitioned) {
 				// sort by tag ID so that filterDuplicates works.
 				std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) {
-					return a.tagId < b.tagId || a.beginVersion < b.beginVersion || a.endVersion < b.endVersion;
+					return a.tagId == b.tagId ? (a.beginVersion == b.beginVersion ? a.endVersion < b.endVersion
+					                                                              : a.beginVersion < b.beginVersion)
+					                          : (a.tagId < b.tagId);
 				});
 
 				// Remove duplicated log files that can happen for old epochs.
 				std::vector<LogFile> filtered = filterDuplicates(logs);
 
 				restorable.logs.swap(filtered);
+				// sort by version order again for continuous analysis
+				std::sort(restorable.logs.begin(), restorable.logs.end());
 				if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) {
 					return Optional<RestorableFileSet>(restorable);
 				}
diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index a9fd2fb365..8a42be686c 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -75,7 +75,8 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 					}
 				}
 				if (savedMore > 0) {
-					ASSERT(info.logRouterTags == rit->second.size()); // Same number as logRouterTags
+					// TODO: check the logRouterTags are the same
+					// ASSERT(info.logRouterTags == rit->second.size());
 
 					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index be1b21c853..99d0dfce04 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -454,7 +454,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 	}
 
 	for (auto it = self->backups.begin(); it != self->backups.end();) {
-		if (!it->second.container.get().present()) {
+		if (it->second.stopped || !it->second.container.get().present()) {
 			TraceEvent("BackupWorkerNoContainer", self->myId).detail("BackupId", it->first);
 			it = self->backups.erase(it);
 			continue;
@@ -566,7 +566,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 		if (self->pullFinished()) {
 			popVersion = self->endVersion.get();
 		}
-		if (numMsg > 0 || (popVersion > lastPopVersion && self->pulling) || self->pullFinished()) {
+		if (((numMsg > 0 || popVersion > lastPopVersion) && self->pulling) || self->pullFinished()) {
 			TraceEvent("BackupWorkerSave", self->myId)
 			    .detail("Version", popVersion)
 			    .detail("MsgQ", self->messages.size());
@@ -575,6 +575,8 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg);
 		}
 
+		// If transition into NOOP mode, should clear messages
+
 		if (popVersion > self->savedVersion) {
 			wait(saveProgress(self, popVersion));
 			TraceEvent("BackupWorkerSavedProgress", self->myId)

From 80d3fa12226e85b0f262074ac7b33cb5cd8ae34d Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 5 Mar 2020 11:34:37 -0800
Subject: [PATCH 089/176] Add delay for master to recruit backup workers

This delay is to ensure old epoch's backup workers can save their progress in
the database. Otherwise, the new master could attempts to recruit backup
workers for the old epoch on version ranges that have already been popped. As
a result, the logs will lose data.
---
 fdbserver/BackupWorker.actor.cpp | 3 ++-
 fdbserver/Knobs.cpp              | 1 +
 fdbserver/Knobs.h                | 1 +
 fdbserver/masterserver.actor.cpp | 3 +++
 4 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 99d0dfce04..282ed01289 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -375,6 +375,8 @@ ACTOR Future<Void> saveProgress(BackupData* self, Version backupVersion) {
 
 	loop {
 		try {
+			// It's critical to save progress immediately so that after a master
+			// recovery, the new master can know the progress so far.
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -469,7 +471,6 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));
 		it++;
 	}
-	ASSERT(!activeUids.empty());
 
 	keyRangeMap.coalesce(allKeys);
 	wait(waitForAll(logFileFutures));
diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp
index 392de76c2c..3e57063935 100644
--- a/fdbserver/Knobs.cpp
+++ b/fdbserver/Knobs.cpp
@@ -355,6 +355,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs, bool isSimula
 	init( PROVISIONAL_START_DELAY,                               1.0 );
 	init( PROVISIONAL_MAX_DELAY,                                60.0 );
 	init( PROVISIONAL_DELAY_GROWTH,                              1.5 );
+	init( SECONDS_BEFORE_RECRUIT_BACKUP_WORKER,                  4.0 );
 
 	// Resolver
 	init( SAMPLE_OFFSET_PER_KEY,                                 100 );
diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h
index f32ac22bc0..afc165e77e 100644
--- a/fdbserver/Knobs.h
+++ b/fdbserver/Knobs.h
@@ -292,6 +292,7 @@ public:
 	double PROVISIONAL_START_DELAY;
 	double PROVISIONAL_DELAY_GROWTH;
 	double PROVISIONAL_MAX_DELAY;
+	double SECONDS_BEFORE_RECRUIT_BACKUP_WORKER;
 
 	// Resolver
 	int64_t KEY_BYTES_PER_SAMPLE;
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 7acf67b72a..943d1afb40 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1241,6 +1241,9 @@ ACTOR Future<Void> configurationMonitor(Reference<MasterData> self, Database cx)
 ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Database cx) {
 	ASSERT(self->backupWorkers.size() > 0);
 
+	// Avoid race between a backup worker's save progress and the reads below.
+	wait(delay(SERVER_KNOBS->SECONDS_BEFORE_RECRUIT_BACKUP_WORKER));
+
 	state LogEpoch epoch = self->cstate.myDBState.recoveryCount;
 	state Reference<BackupProgress> backupProgress(
 	    new BackupProgress(self->dbgid, self->logSystem->getOldEpochTagsVersionsInfo()));

From e9287407d6598fb7735a11d2fef5ef33ce0dc125 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 6 Mar 2020 11:58:10 -0800
Subject: [PATCH 090/176] Backup worker updates latest log versions in
 BackupConfig

If backup worker is enabled, the current epoch's worker of tag (-2,0) will be
responsible for monitoring the backup progress of all workers and update the
BackupConfig with the latest saved log version, which is the minimum version
of all tags.

This change has been incorporated in the getLatestRestorableVersion() so that
it is transparent to clients.
---
 fdbclient/BackupAgent.actor.h       | 20 +++++++++--
 fdbclient/FileBackupAgent.actor.cpp |  6 +++-
 fdbserver/BackupWorker.actor.cpp    | 53 ++++++++++++++++++++++++-----
 3 files changed, 66 insertions(+), 13 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 9ef90976d1..896cd32509 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -787,6 +787,16 @@ public:
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
 	}
 
+	// Set to true if backup worker is enabled.
+	KeyBackedProperty<bool> backupWorkerEnabled() {
+		return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	}
+
+	// Latest version for which all prior versions have saved by backup workers.
+	KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
+		return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	}
+
 	// Stop differntial logging if already started or don't start after completing KV ranges
 	KeyBackedProperty<bool> stopWhenDone() {
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
@@ -816,10 +826,14 @@ public:
 		tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 		auto lastLog = latestLogEndVersion().get(tr);
 		auto firstSnapshot = firstSnapshotEndVersion().get(tr);
-		return map(success(lastLog) && success(firstSnapshot), [=](Void) -> Optional<Version> {
+		auto enabled = backupWorkerEnabled().get(tr);
+		auto workerVersion = latestBackupWorkerSavedVersion().get(tr);
+		return map(success(lastLog) && success(firstSnapshot) && success(enabled) && success(workerVersion), [=](Void) -> Optional<Version> {
 			// The latest log greater than the oldest snapshot is the restorable version
-			if(lastLog.get().present() && firstSnapshot.get().present() && lastLog.get().get() > firstSnapshot.get().get()) {
-				return std::max(lastLog.get().get() - 1, firstSnapshot.get().get());
+			Optional<Version> logVersion =
+			    enabled.get().present() && enabled.get().get() ? workerVersion.get() : lastLog.get();
+			if (logVersion.present() && firstSnapshot.get().present() && logVersion.get() > firstSnapshot.get().get()) {
+				return std::max(logVersion.get() - 1, firstSnapshot.get().get());
 			}
 			return {};
 		});
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 8c58cbc162..66f584e085 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -2388,7 +2388,8 @@ namespace fileBackup {
 
 			// Check if backup worker is enabled
 			DatabaseConfiguration dbConfig = wait(getDatabaseConfiguration(cx));
-			if (!dbConfig.backupWorkerEnabled) {
+			state bool backupWorkerEnabled = dbConfig.backupWorkerEnabled;
+			if (!backupWorkerEnabled) {
 				wait(success(changeConfig(cx, "backup_worker_enabled:=1", true)));
 			}
 
@@ -2420,6 +2421,9 @@ namespace fileBackup {
 					}
 
 					tr->set(backupStartedKey, encodeBackupStartedValue(ids));
+					if (backupWorkerEnabled) {
+						config.backupWorkerEnabled().set(tr, true);
+					}
 
 					// The task may be restarted. Set the watch if started key has NOT been set.
 					if (!taskStarted.get().present()) {
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 282ed01289..3583232692 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -307,7 +307,10 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 // set the "allWorkerStarted" key of the BackupConfig to true, which in turn
 // unblocks StartFullBackupTaskFunc::_execute. Note only worker with Tag (-2,0)
 // runs this actor so that the key is set by one process.
-ACTOR Future<Void> monitorAllWorkerStarted(BackupData* self) {
+// Additionally, this actor updates the saved version for each BackupConfig in
+// the system space so that the client can know if a backup is restorable --
+// log saved version > snapshot version.
+ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 	loop {
 		wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger());
 		if (self->backups.empty()) {
@@ -321,23 +324,32 @@ ACTOR Future<Void> monitorAllWorkerStarted(BackupData* self) {
 		std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
 
 		state std::vector<UID> ready;
+		state std::map<UID, Version> savedLogVersions;
 		if (tagVersions.size() == self->logSystem.get()->getLogRouterTags()) {
 			// Check every version is larger than backup's startVersion
-			for (auto& uidInfo : self->backups) {
-				if (uidInfo.second.allWorkerStarted) continue;
+			for (auto& [uid, info] : self->backups) {
+				if (info.allWorkerStarted) {
+					// update update progress so far
+					Version v = std::numeric_limits<Version>::max();
+					for (const auto [tag, version] : tagVersions) {
+						v = std::min(v, version);
+					}
+					savedLogVersions.emplace(uid, v);
+					continue;
+				}
 				bool saved = true;
 				for (const std::pair<Tag, Version> tv : tagVersions) {
-					if (tv.second < uidInfo.second.startVersion) {
+					if (tv.second < info.startVersion) {
 						saved = false;
 						break;
 					}
 				}
 				if (saved) {
-					ready.push_back(uidInfo.first);
-					uidInfo.second.allWorkerStarted = true;
+					ready.push_back(uid);
+					info.allWorkerStarted = true;
 				}
 			}
-			if (ready.empty()) continue;
+			if (ready.empty() && savedLogVersions.empty()) continue;
 
 			// Set "allWorkerStarted" key for ready backups
 			loop {
@@ -352,13 +364,36 @@ ACTOR Future<Void> monitorAllWorkerStarted(BackupData* self) {
 						configs.emplace_back(uid);
 						readyValues.push_back(tr->get(configs.back().allWorkerStarted().key));
 					}
-					wait(waitForAll(readyValues));
+
+					state std::vector<Future<Optional<Version>>> prevVersions;
+					state std::vector<BackupConfig> versionConfigs;
+					for (const auto [uid, version] : savedLogVersions) {
+						versionConfigs.emplace_back(uid);
+						prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
+					}
+
+					wait(waitForAll(readyValues) && waitForAll(prevVersions));
+
 					for (int i = 0; i < readyValues.size(); i++) {
 						if (!readyValues[i].get().present()) {
 							configs[i].allWorkerStarted().set(tr, true);
 							TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", ready[i].toString());
 						}
 					}
+
+					for (int i = 0; i < prevVersions.size(); i++) {
+						const Version current = savedLogVersions[versionConfigs[i].getUid()];
+						if (prevVersions[i].get().present()) {
+							const Version prev = prevVersions[i].get().get();
+							ASSERT(prev <= current);
+						}
+						if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
+							TraceEvent("BackupWorkerSetVersion", self->myId)
+							    .detail("BackupID", versionConfigs[i].getUid())
+							    .detail("Version", current);
+							versionConfigs[i].latestBackupWorkerSavedVersion().set(tr, current);
+						}
+					}
 					wait(tr->commit());
 					break;
 				} catch (Error& e) {
@@ -738,7 +773,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		addActor.send(checkRemoved(db, req.recruitedEpoch, &self));
 		addActor.send(waitFailureServer(interf.waitFailure.getFuture()));
 		if (req.recruitedEpoch == req.backupEpoch && req.routerTag.id == 0) {
-			addActor.send(monitorAllWorkerStarted(&self));
+			addActor.send(monitorAllWorkerProgress(&self));
 		}
 
 		// Check if backup key is present to avoid race between this check and

From 38def426f485e7b9a14e1302e990815092da6382 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 6 Mar 2020 15:40:06 -0800
Subject: [PATCH 091/176] Add a flag to submitBackup for partitioned log

This is to distinguish with old workloads so that they can work in simulation.
---
 fdbclient/BackupAgent.actor.h                 | 29 ++++++++++++++-----
 fdbclient/FileBackupAgent.actor.cpp           | 14 +++++++--
 ...kupAndParallelRestoreCorrectness.actor.cpp |  9 +++---
 3 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 896cd32509..7c84fa0121 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -308,9 +308,16 @@ public:
 
 	/** BACKUP METHODS **/
 
-	Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true);
-	Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true) {
-		return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone); });
+	Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds,
+	                          std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                          bool stopWhenDone = true, bool partitionedLog = false);
+	Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName,
+	                          Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true,
+	                          bool partitionedLog = false) {
+		return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
+			return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone,
+			                    partitionedLog);
+		});
 	}
 
 	Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
@@ -792,6 +799,11 @@ public:
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
 	}
 
+	// Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
+	KeyBackedProperty<bool> partitionedLogEnabled() {
+		return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	}
+
 	// Latest version for which all prior versions have saved by backup workers.
 	KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
@@ -826,12 +838,15 @@ public:
 		tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 		auto lastLog = latestLogEndVersion().get(tr);
 		auto firstSnapshot = firstSnapshotEndVersion().get(tr);
-		auto enabled = backupWorkerEnabled().get(tr);
+		auto workerEnabled = backupWorkerEnabled().get(tr);
+		auto plogEnabled = partitionedLogEnabled().get(tr);
 		auto workerVersion = latestBackupWorkerSavedVersion().get(tr);
-		return map(success(lastLog) && success(firstSnapshot) && success(enabled) && success(workerVersion), [=](Void) -> Optional<Version> {
+		return map(success(lastLog) && success(firstSnapshot) && success(workerEnabled) && success(plogEnabled) && success(workerVersion), [=](Void) -> Optional<Version> {
 			// The latest log greater than the oldest snapshot is the restorable version
-			Optional<Version> logVersion =
-			    enabled.get().present() && enabled.get().get() ? workerVersion.get() : lastLog.get();
+			Optional<Version> logVersion = workerEnabled.get().present() && workerEnabled.get().get() &&
+			                                       plogEnabled.get().present() && plogEnabled.get().get()
+			                                   ? workerVersion.get()
+			                                   : lastLog.get();
 			if (logVersion.present() && firstSnapshot.get().present() && logVersion.get() > firstSnapshot.get().get()) {
 				return std::max(logVersion.get() - 1, firstSnapshot.get().get());
 			}
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 66f584e085..660bb6c526 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3599,7 +3599,10 @@ public:
 		}
 	}
 
-	ACTOR static Future<Void> submitBackup(FileBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone) {
+	ACTOR static Future<Void> submitBackup(FileBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr,
+	                                       Key outContainer, int snapshotIntervalSeconds, std::string tagName,
+	                                       Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone,
+	                                       bool partitionedLog) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
@@ -3700,6 +3703,7 @@ public:
 		config.stopWhenDone().set(tr, stopWhenDone);
 		config.backupRanges().set(tr, normalizedRanges);
 		config.snapshotIntervalSeconds().set(tr, snapshotIntervalSeconds);
+		config.partitionedLogEnabled().set(tr, partitionedLog);
 
 		Key taskKey = wait(fileBackup::StartFullBackupTaskFunc::addTask(tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal()));
 
@@ -4444,8 +4448,12 @@ Future<ERestoreState> FileBackupAgent::waitRestore(Database cx, Key tagName, boo
 	return FileBackupAgentImpl::waitRestore(cx, tagName, verbose);
 };
 
-Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone) {
-	return FileBackupAgentImpl::submitBackup(this, tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone);
+Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer,
+                                           int snapshotIntervalSeconds, std::string tagName,
+                                           Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone,
+                                           bool partitionedLog) {
+	return FileBackupAgentImpl::submitBackup(this, tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges,
+	                                         stopWhenDone, partitionedLog);
 }
 
 Future<Void> FileBackupAgent::discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName){
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 6a1fec7129..1461419c34 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -180,7 +180,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 
 		try {
 			wait(backupAgent->submitBackup(cx, StringRef(backupContainer), deterministicRandom()->randomInt(0, 100),
-			                               tag.toString(), backupRanges, stopDifferentialDelay ? false : true));
+			                               tag.toString(), backupRanges, stopDifferentialDelay ? false : true,
+			                               /*partitionedLog=*/true));
 		} catch (Error& e) {
 			TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) throw;
@@ -395,9 +396,9 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 			if (!self->locked && BUGGIFY) {
 				TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
 				try {
-					extraBackup = backupAgent.submitBackup(cx, LiteralStringRef("file://simfdb/backups/"),
-					                                       deterministicRandom()->randomInt(0, 100),
-					                                       self->backupTag.toString(), self->backupRanges, true);
+					extraBackup = backupAgent.submitBackup(
+					    cx, LiteralStringRef("file://simfdb/backups/"), deterministicRandom()->randomInt(0, 100),
+					    self->backupTag.toString(), self->backupRanges, true, /*partitionedLog=*/true);
 				} catch (Error& e) {
 					TraceEvent("BARW_SubmitBackup2Exception", randomID)
 					    .error(e)

From 12ed8ad53657dec31eb4499f7c3b1af5fd452eef Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 8 Mar 2020 20:50:32 -0700
Subject: [PATCH 092/176] Fix backup worker start version when logset start
 version is lower

The start version of tlog set can be smaller than the last epoch's end version.
In this case, set backup worker's start version as last epoch's end version to
avoid overlapping of version ranges among backup workers.
---
 fdbserver/LogSystem.h                       | 3 ++-
 fdbserver/TagPartitionedLogSystem.actor.cpp | 6 ++++--
 fdbserver/masterserver.actor.cpp            | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h
index dad55b047f..7d8c79faba 100644
--- a/fdbserver/LogSystem.h
+++ b/fdbserver/LogSystem.h
@@ -721,7 +721,8 @@ struct ILogSystem {
 		// Call only on an ILogSystem obtained from recoverAndEndEpoch()
 		// Returns the first unreadable version number of the recovered epoch (i.e. message version numbers < (get_end(), 0) will be readable)
 
-	virtual Version getStartVersion() const = 0; // Returns the start version of current epoch.
+	// Returns the start version of current epoch for backup workers.
+	virtual Version getBackupStartVersion() const = 0;
 
 	struct EpochTagsVersionsInfo {
 		int32_t logRouterTags; // Number of log router tags.
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 07dd64c30e..146430cbee 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -191,6 +191,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 	Optional<Version> recoverAt;
 	Optional<Version> recoveredAt;
 	Version knownCommittedVersion;
+	Version backupStartVersion = invalidVersion; // max(tLogs[0].startVersion, previous epochEnd).
 	LocalityData locality;
 	std::map< std::pair<UID, Tag>, std::pair<Version, Version> > outstandingPops;  // For each currently running popFromLog actor, (log server #, tag)->popped version
 	Optional<PromiseStream<Future<Void>>> addActor;
@@ -1350,9 +1351,9 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 
 	int getLogRouterTags() const override { return logRouterTags; }
 
-	Version getStartVersion() const override {
+	Version getBackupStartVersion() const final {
 		ASSERT(tLogs.size() > 0);
-		return tLogs[0]->startVersion;
+		return backupStartVersion;
 	}
 
 	std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo> getOldEpochTagsVersionsInfo() const override {
@@ -2214,6 +2215,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		logSystem->oldLogData.insert(logSystem->oldLogData.end(), oldLogSystem->oldLogData.begin(), oldLogSystem->oldLogData.end());
 
 		logSystem->tLogs[0]->startVersion = oldLogSystem->knownCommittedVersion + 1;
+		logSystem->backupStartVersion = oldLogSystem->knownCommittedVersion + 1;
 		state int lockNum = 0;
 		while(lockNum < oldLogSystem->lockResults.size()) {
 			if(oldLogSystem->lockResults[lockNum].logSet->locality == primaryLocality) {
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 943d1afb40..cda864a732 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1256,7 +1256,7 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 		idsTags.emplace_back(deterministicRandom()->randomUniqueID(), Tag(tagLocalityLogRouter, i));
 	}
 
-	const Version startVersion = self->logSystem->getStartVersion();
+	const Version startVersion = self->logSystem->getBackupStartVersion();
 	state int i = 0;
 	for (; i < logRouterTags; i++) {
 		const auto& worker = self->backupWorkers[i % self->backupWorkers.size()];

From d8731a179658b264e48287e53caa4957f3c307e6 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 9 Mar 2020 10:17:38 -0700
Subject: [PATCH 093/176] Refactor to use std::find_if for more concise code

---
 fdbserver/BackupProgress.actor.cpp | 5 ++++-
 fdbserver/BackupProgress.actor.h   | 7 -------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 8a42be686c..c5e263e877 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -51,6 +51,7 @@ void BackupProgress::updateTagVersions(std::map<Tag, Version>* tagVersions, std:
 		}
 	}
 }
+
 std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgress::getUnfinishedBackup() {
 	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit;
 
@@ -63,7 +64,9 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 		if (progressIt != progress.end() && progressIt->first == epoch) {
 			updateTagVersions(&tagVersions, &tags, progressIt->second, info.epochEnd, epoch);
 		} else {
-			auto rit = findPreviousProgress(epoch);
+			auto rit =
+			    std::find_if(progress.rbegin(), progress.rend(),
+			                 [=](const std::pair<LogEpoch, std::map<Tag, Version>>& p) { return p.first < epoch; });
 			if (!(rit == progress.rend())) {
 				// A partial recovery can result in empty epoch that copies previous
 				// epoch's version range. In this case, we should check previous
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index f7eacbe180..3237fae6a0 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -83,13 +83,6 @@ private:
 	void updateTagVersions(std::map<Tag, Version>* tagVersions, std::set<Tag>* tags,
 	                       const std::map<Tag, Version>& progress, Version endVersion, LogEpoch epoch);
 
-	std::map<LogEpoch, std::map<Tag, Version>>::reverse_iterator findPreviousProgress(LogEpoch epoch) {
-		for (auto it = progress.rbegin(); !(it == progress.rend()); ++it) {
-			if (it->first < epoch) return it;
-		}
-		return progress.rend();
-	}
-
 	const UID dbgid;
 
 	// Note this MUST be iterated in ascending order.

From c59b0844a96d3ee540bdce3408c020dc72703225 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 9 Mar 2020 15:33:15 -0700
Subject: [PATCH 094/176] Add total number of tags to WorkerBackupStatus

This allows the backup worker to check the number of tags.
---
 fdbclient/FDBTypes.h               |  5 +++--
 fdbserver/BackupProgress.actor.cpp | 18 +++++++++++++-----
 fdbserver/BackupProgress.actor.h   |  4 ++++
 fdbserver/BackupWorker.actor.cpp   |  2 +-
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 6111b23138..6a4d1f73c4 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -978,13 +978,14 @@ struct WorkerBackupStatus {
 	LogEpoch epoch;
 	Version version;
 	Tag tag;
+	int32_t totalTags;
 
 	WorkerBackupStatus() : epoch(0), version(invalidVersion) {}
-	WorkerBackupStatus(LogEpoch e, Version v, Tag t) : epoch(e), version(v), tag(t) {}
+	WorkerBackupStatus(LogEpoch e, Version v, Tag t, int32_t total) : epoch(e), version(v), tag(t), totalTags(total) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, epoch, version, tag);
+		serializer(ar, epoch, version, tag, totalTags);
 	}
 };
 
diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index c5e263e877..1cb061af58 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -35,6 +35,13 @@ void BackupProgress::addBackupStatus(const WorkerBackupStatus& status) {
 	} else {
 		it.insert(lb, { status.tag, status.version });
 	}
+
+	auto tagIt = epochTags.find(status.epoch);
+	if (tagIt == epochTags.end()) {
+		epochTags.insert({ status.epoch, status.totalTags });
+	} else {
+		ASSERT(status.totalTags == tagIt->second);
+	}
 }
 
 void BackupProgress::updateTagVersions(std::map<Tag, Version>* tagVersions, std::set<Tag>* tags,
@@ -78,8 +85,8 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 					}
 				}
 				if (savedMore > 0) {
-					// TODO: check the logRouterTags are the same
-					// ASSERT(info.logRouterTags == rit->second.size());
+					// The logRouterTags are the same
+					ASSERT(info.logRouterTags == epochTags[rit->first]);
 
 					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
@@ -124,7 +131,8 @@ ACTOR Future<Void> getBackupProgress(Database cx, UID dbgid, Reference<BackupPro
 				    .detail("W", workerID)
 				    .detail("Epoch", status.epoch)
 				    .detail("Version", status.version)
-				    .detail("Tag", status.tag.toString());
+				    .detail("Tag", status.tag.toString())
+				    .detail("TotalTags", status.totalTags);
 			}
 			return Void();
 		} catch (Error& e) {
@@ -151,8 +159,8 @@ TEST_CASE("/BackupProgress/Unfinished") {
 		ASSERT(tagVersion.size() == 1 && tagVersion.begin()->first == tag1 && tagVersion.begin()->second == begin1);
 	}
 
-	const int saved1 = 50;
-	WorkerBackupStatus status1(epoch1, saved1, tag1);
+	const int saved1 = 50, totalTags = 1;
+	WorkerBackupStatus status1(epoch1, saved1, tag1, totalTags);
 	progress.addBackupStatus(status1);
 	unfinished = progress.getUnfinishedBackup();
 	ASSERT(unfinished.size() == 1);
diff --git a/fdbserver/BackupProgress.actor.h b/fdbserver/BackupProgress.actor.h
index 3237fae6a0..d17d2c9a15 100644
--- a/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/BackupProgress.actor.h
@@ -93,6 +93,10 @@ private:
 	// the gap. "progress" MUST be iterated in ascending order.
 	std::map<LogEpoch, std::map<Tag, Version>> progress;
 
+	// LogRouterTags for each epoch obtained by decoding backup progress from
+	// the system keyspace.
+	std::map<LogEpoch, int32_t> epochTags;
+
 	// Value of the "backupStartedKey".
 	Optional<Value> backupStartedValue;
 };
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 3583232692..831ce9608c 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -416,7 +416,7 @@ ACTOR Future<Void> saveProgress(BackupData* self, Version backupVersion) {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			WorkerBackupStatus status(self->backupEpoch, backupVersion, self->tag);
+			WorkerBackupStatus status(self->backupEpoch, backupVersion, self->tag, self->totalTags);
 			tr.set(key, backupProgressValue(status));
 			tr.addReadConflictRange(singleKeyRange(key));
 			wait(tr.commit());

From d82432da3c08887950ac13f697b0908b0ce533b9 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 9 Mar 2020 15:35:52 -0700
Subject: [PATCH 095/176] Fix wrong end version for restore loader

The restore cannot exceed the target version of the restore request. Otherwise,
the version restored is larger than the requested version.
---
 fdbserver/RestoreMaster.actor.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index a4da897650..e76d8eff05 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -314,8 +314,7 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 		TraceEvent("FastRestoreMasterDispatchVersionBatches")
 		    .detail("BatchIndex", batchIndex)
 		    .detail("BatchSize", versionBatch->size)
-		    .detail("RunningVersionBatches", self->runningVersionBatches.get())
-		    .detail("Start", now());
+		    .detail("RunningVersionBatches", self->runningVersionBatches.get());
 		self->batch[batchIndex] = Reference<MasterBatchData>(new MasterBatchData());
 		self->batchStatus[batchIndex] = Reference<MasterBatchStatus>(new MasterBatchStatus());
 		fBatches.push_back(distributeWorkloadPerVersionBatch(self, batchIndex, cx, request, *versionBatch));
@@ -374,7 +373,8 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<MasterBatchData> batchDat
 		param.asset.len = file.fileSize;
 		param.asset.range = request.range;
 		param.asset.beginVersion = versionBatch.beginVersion;
-		param.asset.endVersion = versionBatch.endVersion;
+		param.asset.endVersion =
+		    isRangeFile ? versionBatch.endVersion : std::min(versionBatch.endVersion, request.targetVersion + 1);
 
 		TraceEvent("FastRestoreMasterPhaseLoadFiles")
 		    .detail("BatchIndex", batchIndex)

From d0a24dd20d4860d3779029ee5cdcdc1fa0022317 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 10 Mar 2020 15:45:57 -0700
Subject: [PATCH 096/176] Decode out of order mutations in old mutation logs

In the old mutation logs, a version's mutations are serialized as a buffer.
Then the buffer is split into smaller chunks, e.g., 10000 bytes each. When
writting chunks to the final mutation log file, these chunks can be flushed
out of order. For instance, the (version, chunck_part) can be in the order of
(3, 0), (4, 0), (3, 1). As a result, the decoder must read forward to find all
chunks of data for a version.

Another complication is that the files are organized into blocks, where (3, 1)
can be in a subsequent block. This change checks the value size for each
version, if the size is smaller than the right size, the decoder will look
for the missing chucks in the next block.
---
 fdbbackup/FileConverter.h       |   2 +
 fdbbackup/FileDecoder.actor.cpp | 140 ++++++++++++++++++++++----------
 2 files changed, 100 insertions(+), 42 deletions(-)

diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index fc82e5dfb2..e01566b889 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -31,6 +31,7 @@ namespace file_converter {
 enum {
 	OPT_CONTAINER,
 	OPT_BEGIN_VERSION,
+	OPT_CRASHONERROR,
 	OPT_END_VERSION,
 	OPT_TRACE,
 	OPT_TRACE_DIR,
@@ -44,6 +45,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 	                                        { OPT_CONTAINER, "--container", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "-b", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "--begin", SO_REQ_SEP },
+	                                        { OPT_CRASHONERROR, "--crash", SO_NONE },
 	                                        { OPT_END_VERSION, "-e", SO_REQ_SEP },
 	                                        { OPT_END_VERSION, "--end", SO_REQ_SEP },
 	                                        { OPT_TRACE, "--log", SO_NONE },
diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 02b98e4825..62716a562d 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -30,12 +30,15 @@
 #include "flow/serialize.h"
 #include "flow/actorcompiler.h" // has to be last include
 
+extern bool g_crashOnError;
+
 namespace file_converter {
 
 void printDecodeUsage() {
 	std::cout << "\n"
 	             "  -r, --container   Container URL.\n"
 	             "  -i, --input FILE  Log file to be decoded.\n"
+	             "  --crash           Crash on serious error.\n"
 	             "\n";
 	return;
 }
@@ -89,6 +92,10 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
 			param->container_url = args->OptionArg();
 			break;
 
+		case OPT_CRASHONERROR:
+			g_crashOnError = true;
+			break;
+
 		case OPT_INPUT_FILE:
 			param->file = args->OptionArg();
 			break;
@@ -161,7 +168,13 @@ std::vector<MutationRef> decode_value(const StringRef& value) {
 
 	reader.consume<uint64_t>(); // Consume the includeVersion
 	uint32_t val_length = reader.consume<uint32_t>();
-	ASSERT(val_length == value.size() - sizeof(uint64_t) - sizeof(uint32_t));
+	if (val_length != value.size() - sizeof(uint64_t) - sizeof(uint32_t)) {
+		TraceEvent("ValueError")
+		    .detail("ValueLen", val_length)
+		    .detail("ValueSize", value.size())
+		    .detail("Value", printable(value));
+		ASSERT(false);
+	}
 
 	std::vector<MutationRef> mutations;
 	while (1) {
@@ -217,54 +230,74 @@ struct DecodeProgress {
 
 	// The following are private APIs:
 
+	// Returns true if value contains complete data.
+	bool isValueComplete(StringRef value) {
+		StringRefReader reader(value, restore_corrupted_data());
+
+		reader.consume<uint64_t>(); // Consume the includeVersion
+		uint32_t val_length = reader.consume<uint32_t>();
+		return val_length == value.size() - sizeof(uint64_t) - sizeof(uint32_t);
+	}
+
 	// PRECONDITION: finished() must return false before calling this function.
 	// Returns the next batch of mutations along with the arena backing it.
 	ACTOR static Future<VersionedMutations> getNextBatchImpl(DecodeProgress* self) {
 		ASSERT(!self->finished());
 
-		state std::pair<Arena, KeyValueRef> arena_kv = self->keyValues[0];
-
-		// decode this batch's version
-		state std::pair<Version, int32_t> version_part = decode_key(arena_kv.second.key);
-		ASSERT(version_part.second == 0); // first part number must be 0.
-
-		// decode next versions, check if they are continuous parts
-		state int idx = 1; // next kv pair in "keyValues"
-		state int bufSize = arena_kv.second.value.size();
-		state int lastPart = 0;
 		loop {
-			// Try to decode another block if needed
-			if (idx == self->keyValues.size()) {
-				wait(readAndDecodeFile(self));
+			state std::tuple<Arena, Version, int32_t, StringRef> tuple = self->keyValues[0];
+
+			ASSERT(std::get<2>(tuple) == 0); // first part number must be 0.
+
+			// decode next versions, check if they are continuous parts
+			state int idx = 1; // next kv pair in "keyValues"
+			state int bufSize = std::get<3>(tuple).size();
+			state int lastPart = 0;
+			loop {
+				// Try to decode another block if needed
+				if (idx == self->keyValues.size()) {
+					wait(readAndDecodeFile(self));
+				}
+				if (idx == self->keyValues.size()) break;
+
+				auto next_tuple = self->keyValues[idx];
+				if (std::get<1>(tuple) != std::get<1>(next_tuple)) {
+					break;
+				}
+
+				if (lastPart + 1 != std::get<2>(next_tuple)) {
+					TraceEvent("DecodeError").detail("Part1", lastPart).detail("Part2", std::get<2>(next_tuple));
+					throw restore_corrupted_data();
+				}
+				bufSize += std::get<3>(next_tuple).size();
+				idx++;
+				lastPart++;
 			}
-			if (idx == self->keyValues.size()) break;
 
-			std::pair<Version, int32_t> next_version_part = decode_key(self->keyValues[idx].second.key);
-			if (version_part.first != next_version_part.first) break;
-
-			if (lastPart + 1 != next_version_part.second) {
-				TraceEvent("DecodeError").detail("Part1", lastPart).detail("Part2", next_version_part.second);
+			VersionedMutations m;
+			m.version = std::get<1>(tuple);
+			TraceEvent("Decode").detail("Version", m.version).detail("Idx", idx).detail("Q", self->keyValues.size());
+			StringRef value = std::get<3>(tuple);
+			if (idx > 1) {
+				// Stitch parts into one and then decode one by one
+				Standalone<StringRef> buf = self->combineValues(idx, bufSize);
+				value = buf;
+				m.arena = buf.arena();
+			} else {
+				m.arena = std::get<0>(tuple);
+			}
+			if (self->isValueComplete(value)) {
+				m.mutations = decode_value(value);
+				self->keyValues.erase(self->keyValues.begin(), self->keyValues.begin() + idx);
+				return m;
+			} else if (!self->eof) {
+				// Read one more block, hopefully the missing part of the value can be found.
+				wait(readAndDecodeFile(self));
+			} else {
+				TraceEvent(SevError, "MissingValue").detail("Version", m.version);
 				throw restore_corrupted_data();
 			}
-			bufSize += self->keyValues[idx].second.value.size();
-			idx++;
-			lastPart++;
 		}
-
-		VersionedMutations m;
-		m.version = version_part.first;
-		if (idx > 1) {
-			// Stitch parts into one and then decode one by one
-			Standalone<StringRef> buf = self->combineValues(idx, bufSize);
-			m.mutations = decode_value(buf);
-			m.arena = buf.arena();
-		} else {
-			m.mutations = decode_value(arena_kv.second.value);
-			m.arena = arena_kv.first;
-		}
-		self->keyValues.erase(self->keyValues.begin(), self->keyValues.begin() + idx);
-
-		return m;
 	}
 
 	// Returns a buffer which stitches first "idx" values into one.
@@ -275,7 +308,7 @@ struct DecodeProgress {
 		Standalone<StringRef> buf = makeString(len);
 		int n = 0;
 		for (int i = 0; i < idx; i++) {
-			const auto& value = keyValues[i].second.value;
+			const auto& value = std::get<3>(keyValues[i]);
 			memcpy(mutateString(buf) + n, value.begin(), value.size());
 			n += value.size();
 		}
@@ -301,9 +334,16 @@ struct DecodeProgress {
 				// Read key and value.  If anything throws then there is a problem.
 				uint32_t kLen = reader.consumeNetworkUInt32();
 				const uint8_t* k = reader.consume(kLen);
+				std::pair<Version, int32_t> version_part = decode_key(StringRef(k, kLen));
 				uint32_t vLen = reader.consumeNetworkUInt32();
 				const uint8_t* v = reader.consume(vLen);
-				keyValues.emplace_back(buf.arena(), KeyValueRef(StringRef(k, kLen), StringRef(v, vLen)));
+				TraceEvent("Block")
+				    .detail("KeySize", kLen)
+				    .detail("valueSize", vLen)
+				    .detail("Offset", reader.rptr - buf.begin())
+				    .detail("Version", version_part.first)
+				    .detail("Part", version_part.second);
+				keyValues.emplace_back(buf.arena(), version_part.first, version_part.second, StringRef(v, vLen));
 			}
 
 			// Make sure any remaining bytes in the block are 0xFF
@@ -311,6 +351,15 @@ struct DecodeProgress {
 				if (b != 0xFF) throw restore_corrupted_data_padding();
 			}
 
+			// The (version, part) in a block can be out of order, i.e., (3, 0)
+			// can be followed by (4, 0), and then (3, 1). So we need to sort them
+			// first by version, and then by part number.
+			std::sort(keyValues.begin(), keyValues.end(),
+			          [](const std::tuple<Arena, Version, int32_t, StringRef>& a,
+			             const std::tuple<Arena, Version, int32_t, StringRef>& b) {
+				          return std::get<1>(a) == std::get<1>(b) ? std::get<2>(a) < std::get<2>(b)
+				                                                  : std::get<1>(a) < std::get<1>(b);
+			          });
 			return;
 		} catch (Error& e) {
 			TraceEvent(SevWarn, "CorruptBlock").error(e).detail("Offset", reader.rptr - buf.begin());
@@ -360,14 +409,21 @@ struct DecodeProgress {
 	Reference<IAsyncFile> fd;
 	int64_t offset = 0;
 	bool eof = false;
-	// Key value pairs and their memory arenas.
-	std::vector<std::pair<Arena, KeyValueRef>> keyValues;
+	// A (version, part_number)'s mutations and memory arena.
+	std::vector<std::tuple<Arena, Version, int32_t, StringRef>> keyValues;
 };
 
 ACTOR Future<Void> decode_logs(DecodeParams params) {
 	state Reference<IBackupContainer> container = IBackupContainer::openContainer(params.container_url);
 
 	state BackupFileList listing = wait(container->dumpFileList());
+	// remove partitioned logs
+	listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(),
+	                                  [](const LogFile& file) {
+		                                  std::string prefix("plogs/");
+		                                  return file.fileName.substr(0, prefix.size()) == prefix;
+	                                  }),
+	                   listing.logs.end());
 	std::sort(listing.logs.begin(), listing.logs.end());
 	TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size());
 

From ca1a4ef9fdb50081df2b727337b6740b5f4044cf Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 10 Mar 2020 16:14:35 -0700
Subject: [PATCH 097/176] Ignore mutation logs of size 0 in converter

---
 fdbbackup/FileConverter.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp
index 67cd4738b6..006b311f87 100644
--- a/fdbbackup/FileConverter.actor.cpp
+++ b/fdbbackup/FileConverter.actor.cpp
@@ -68,7 +68,7 @@ void printLogFiles(std::string msg, const std::vector<LogFile>& files) {
 std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, Version begin, Version end) {
 	std::vector<LogFile> filtered;
 	for (const auto& file : files) {
-		if (file.beginVersion <= end && file.endVersion >= begin && file.tagId >= 0) {
+		if (file.beginVersion <= end && file.endVersion >= begin && file.tagId >= 0 && file.fileSize > 0) {
 			filtered.push_back(file);
 		}
 	}
@@ -76,7 +76,7 @@ std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, Vers
 
 	// Remove duplicates. This is because backup workers may store the log for
 	// old epochs successfully, but do not update the progress before another
-	// recovery happened.	As a result, next epoch will retry and creates
+	// recovery happened. As a result, next epoch will retry and creates
 	// duplicated log files.
 	std::vector<LogFile> sorted;
 	int i = 0;

From 6fb7316185a88559c348a0898aa49d37b2de5600 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 10 Mar 2020 19:42:36 -0700
Subject: [PATCH 098/176] Fix asset end version if request.targetVersion is -1

---
 fdbserver/RestoreMaster.actor.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index e76d8eff05..37fe740860 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -373,8 +373,9 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<MasterBatchData> batchDat
 		param.asset.len = file.fileSize;
 		param.asset.range = request.range;
 		param.asset.beginVersion = versionBatch.beginVersion;
-		param.asset.endVersion =
-		    isRangeFile ? versionBatch.endVersion : std::min(versionBatch.endVersion, request.targetVersion + 1);
+		param.asset.endVersion = (isRangeFile || request.targetVersion == -1)
+		                             ? versionBatch.endVersion
+		                             : std::min(versionBatch.endVersion, request.targetVersion + 1);
 
 		TraceEvent("FastRestoreMasterPhaseLoadFiles")
 		    .detail("BatchIndex", batchIndex)

From 273c086b0fd1f72d9e6812e90f629c7c8a368355 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 10 Mar 2020 20:05:11 -0700
Subject: [PATCH 099/176] Fix MacOS compiling error

clang doesn't allow capture references, so use copy for lambda's capture list.
---
 fdbserver/BackupProgress.actor.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 1cb061af58..af236f86fa 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -71,9 +71,9 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 		if (progressIt != progress.end() && progressIt->first == epoch) {
 			updateTagVersions(&tagVersions, &tags, progressIt->second, info.epochEnd, epoch);
 		} else {
-			auto rit =
-			    std::find_if(progress.rbegin(), progress.rend(),
-			                 [=](const std::pair<LogEpoch, std::map<Tag, Version>>& p) { return p.first < epoch; });
+			auto rit = std::find_if(
+			    progress.rbegin(), progress.rend(),
+			    [epoch = epoch](const std::pair<LogEpoch, std::map<Tag, Version>>& p) { return p.first < epoch; });
 			if (!(rit == progress.rend())) {
 				// A partial recovery can result in empty epoch that copies previous
 				// epoch's version range. In this case, we should check previous

From fe51ba3d162e67592f45e8b77f7f21db39133d69 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 11 Mar 2020 15:39:09 -0700
Subject: [PATCH 100/176] Give maximum subsequence number for snapshot
 mutations

This is needed so that mutations in partitioned logs are applied first and
snapshot mutations are applied later for the same commit version.
---
 fdbserver/RestoreLoader.actor.cpp | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 109b25c43c..42fd83bf53 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -172,9 +172,9 @@ ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
 
 			// Deserialize messages written in saveMutationsToFile().
 			LogMessageVersion msgVersion;
-			msgVersion.version = bigEndian64(reader.consume<Version>());
-			msgVersion.sub = bigEndian32(reader.consume<uint32_t>());
-			int msgSize = bigEndian32(reader.consume<int>());
+			msgVersion.version = reader.consumeNetworkUInt64();
+			msgVersion.sub = reader.consumeNetworkUInt32();
+			int msgSize = reader.consumeNetworkInt32();
 			const uint8_t* message = reader.consume(msgSize);
 
 			// Skip mutations out of the version range
@@ -769,12 +769,14 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 		cc->loadedRangeBytes += m.totalSize();
 
 		// We cache all kv operations into kvOps, and apply all kv operations later in one place
-		auto it = kvOps.insert(std::make_pair(LogMessageVersion(version), MutationsVec()));
+		// Note we give INT_MAX as the sub sequence number to override any log mutations.
+		const LogMessageVersion msgVersion(version, std::numeric_limits<int32_t>::max());
+		auto it = kvOps.insert(std::make_pair(msgVersion, MutationsVec()));
 		TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
 		    .detail("CommitVersion", version)
 		    .detail("ParsedMutationKV", m.toString());
 
-		ASSERT_WE_THINK(kvOps.find(LogMessageVersion(version)) != kvOps.end());
+		ASSERT_WE_THINK(kvOps.find(msgVersion) != kvOps.end());
 		it.first->second.push_back_deep(it.first->second.arena(), m);
 		// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
 		if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {

From 4f4ce93f8c38df209c8c21cd07ff890587c5ead1 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 11 Mar 2020 15:45:44 -0700
Subject: [PATCH 101/176] Remove debug print out

---
 fdbclient/BackupContainer.actor.cpp | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 69e0aa3378..6181fbb0fc 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -799,7 +799,6 @@ public:
 		}
 
 		state std::vector<LogFile> logs;
-std::cout << "describe list: scanBegin:" << scanBegin << ", scanEnd:" << scanEnd << ", partitioned:" << partitioned << "\n";
 		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, partitioned)) &&
 		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
 
@@ -1097,7 +1096,6 @@ std::cout << "describe list: scanBegin:" << scanBegin << ", scanEnd:" << scanEnd
 		ASSERT(tags == nullptr || tags->empty());
 		for (int idx : indices) {
 			const LogFile& file = files[idx];
-std::cout << "  " << file.toString() << " " << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags " << lastTags << "\n";
 			if (lastEnd == invalidVersion) {
 				if (file.beginVersion > begin) return false;
 				if (file.endVersion > begin) {
@@ -1123,7 +1121,6 @@ std::cout << "  " << file.toString() << " " << "lastBegin " << lastBegin << ", l
 			lastEnd = file.endVersion;
 			if (lastEnd > end) break;
 		}
-std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " << end << ", lastTags " << lastTags << "\n";
 		if (tags != nullptr && lastBegin != invalidVersion) {
 			tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags);
 		}
@@ -1181,7 +1178,6 @@ std::cout << "lastBegin " << lastBegin << ", lastEnd " << lastEnd << ", end " <<
 
 		for (const LogFile& file : logs) {
 			Version end = getPartitionedLogsContinuousEndVersion(logs, file.beginVersion);
-std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
 			if (end > file.beginVersion) {
 				// desc->minLogBegin = file.beginVersion;
 				// contiguousLogEnd is not inclusive, so +1 here.
@@ -1194,8 +1190,6 @@ std::cout << "    determine " << file.toString() << " , end " << end << "\n\n";
 	// Returns the end version such that [begin, end] is continuous.
 	// "logs" should be already sorted.
 	static Version getPartitionedLogsContinuousEndVersion(const std::vector<LogFile>& logs, Version begin) {
-std::cout << "getPartitionedLogsContinuousEndVersion begin:" << begin << "\n";
-for (auto file : logs) std::cout << "    " << file.toString() << "\n";
 		Version end = 0;
 
 		std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
@@ -1210,15 +1204,12 @@ for (auto file : logs) std::cout << "    " << file.toString() << "\n";
 			}
 			end = std::max(end, logs[i].endVersion - 1);
 		}
-std::cout << "Init end: " << end << ", begin " << begin << "\n";
 
 		// check tag 0 is continuous in [begin, end] and create a map of ranges to tags
 		std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> tags
 		isContinuous(logs, tagIndices[0], begin, end, &tags);
 		if (tags.empty() || end <= begin) return 0;
 		end = std::min(end, tags.rbegin()->first.second);
-std::cout << "  Tag 0 end: " << end << "\n";
-for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " << v << "\n";
 
 		// for each range in tags, check all tags from 1 are continouous
 		Version lastEnd = begin;
@@ -1228,7 +1219,6 @@ for (auto [p, v] : tags) std::cout<<"[" << p.first << ", " << p.second << "] " <
 				std::map<std::pair<Version, Version>, int> rangeTags;
 				isContinuous(logs, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags);
 				tagEnd = rangeTags.empty() ? 0 : std::min(tagEnd, rangeTags.rbegin()->first.second);
-std::cout << "  Tag " << i << " end: " << tagEnd << ", return end = "<< lastEnd << "\n";
 				if (tagEnd == 0) return lastEnd;
 			}
 			if (tagEnd < beginEnd.second) {
@@ -1238,7 +1228,6 @@ std::cout << "  Tag " << i << " end: " << tagEnd << ", return end = "<< lastEnd
 			lastEnd = beginEnd.second;
 		}
 
-std::cout << "Return end = " << end << "\n\n";
 		return end;
 	}
 

From fa7c8d8bb385c957434627bf478f3bb9b70a5ed4 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 11 Mar 2020 20:47:54 -0700
Subject: [PATCH 102/176] Add done trigger so that backup progress can be set

Otherwise, when there is no mutations for the unfinished range, the empty file
may not be created when the worker is displaced, thus leaving holes in version
ranges.
---
 fdbserver/BackupWorker.actor.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 831ce9608c..651065d586 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -110,6 +110,7 @@ struct BackupData {
 
 	std::map<UID, PerBackupInfo> backups; // Backup UID to infos
 	AsyncTrigger changedTrigger;
+	AsyncTrigger doneTrigger;
 
 	CounterCollection cc;
 	Future<Void> logger;
@@ -385,7 +386,9 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 						const Version current = savedLogVersions[versionConfigs[i].getUid()];
 						if (prevVersions[i].get().present()) {
 							const Version prev = prevVersions[i].get().get();
-							ASSERT(prev <= current);
+							TraceEvent(SevWarn, "BackupWorkerVersionInverse", self->myId)
+							    .detail("Prev", prev)
+							    .detail("Current", current);
 						}
 						if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
 							TraceEvent("BackupWorkerSetVersion", self->myId)
@@ -612,6 +615,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 		}
 
 		// If transition into NOOP mode, should clear messages
+		if (!self->pulling) self->messages.clear();
 
 		if (popVersion > self->savedVersion) {
 			wait(saveProgress(self, popVersion));
@@ -624,7 +628,7 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 		}
 
 		if (!self->pullFinished()) {
-			wait(uploadDelay);
+			wait(uploadDelay || self->doneTrigger.onTrigger());
 		}
 	}
 }
@@ -664,6 +668,7 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		TraceEvent("BackupWorkerGot", self->myId).suppressFor(1.0).detail("V", tagAt);
 		if (self->pullFinished()) {
 			self->eraseMessagesAfterEndVersion();
+			self->doneTrigger.trigger();
 			TraceEvent("BackupWorkerFinishPull", self->myId)
 			    .detail("Tag", self->tag.toString())
 			    .detail("VersionGot", tagAt)

From c63493c34fe8cb65a4b6cb1a4dc2fb38588f142b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 12 Mar 2020 10:28:10 -0700
Subject: [PATCH 103/176] Allow overlapped versions in partitioned logs

The overlapping can only happens between two generations, where the known
committed version to recovery version is copied from old generation to the new
generation. Within a generation, there is no overlap.

The fix here is related to the calculation of continuous version ranges,
allowing the overlap to happen.
---
 fdbclient/BackupContainer.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 6181fbb0fc..29027eff3a 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1085,7 +1085,7 @@ public:
 	// For a list of log files specified by their indices (of the same tag),
 	// returns if they are continous in the range [begin, end]. If "tags" is not
 	// nullptr, then it will be populated with [begin, end] -> tags, where next
-	// pair's begin == previous pair's end + 1. On return, the last pair's end
+	// pair's begin <= previous pair's end + 1. On return, the last pair's end
 	// version (inclusive) gives the continuous range from begin.
 	static bool isContinuous(const std::vector<LogFile>& files, const std::vector<int>& indices, Version begin,
 	                         Version end, std::map<std::pair<Version, Version>, int>* tags) {
@@ -1104,7 +1104,7 @@ public:
 				} else {
 					continue;
 				}
-			} else if (lastEnd != file.beginVersion) {
+			} else if (lastEnd < file.beginVersion) {
 				if (tags != nullptr) {
 					tags->emplace(std::make_pair(lastBegin, lastEnd - 1), lastTags);
 				}

From 1a1f572f299e343e7cc2b806ba8f5683090c798b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 12 Mar 2020 14:38:40 -0700
Subject: [PATCH 104/176] Fix a time gap for monitoring backup keys

Backup worker starts by check if there are backup keys and then runs
monitorBackupKeyOrPullData() loop, which does the check again. The second check
can be delayed, which causes the loop to perform NOOP pops. The fix removes
this second check and uses the result of the first check to decide what to do
in the loop.
---
 fdbserver/BackupProgress.actor.cpp |  2 +-
 fdbserver/BackupWorker.actor.cpp   | 98 +++++++++++++++---------------
 fdbserver/RestoreMaster.actor.cpp  |  5 +-
 3 files changed, 55 insertions(+), 50 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index af236f86fa..1378980939 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -86,7 +86,7 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 				}
 				if (savedMore > 0) {
 					// The logRouterTags are the same
-					ASSERT(info.logRouterTags == epochTags[rit->first]);
+					// ASSERT(info.logRouterTags == epochTags[rit->first]);
 
 					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, epoch);
 				}
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 651065d586..0fa314bc4c 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -258,6 +258,23 @@ struct BackupData {
 		}
 		return true;
 	}
+
+	ACTOR static Future<Version> _getMinKnownCommittedVersion(BackupData* self) {
+		loop {
+			GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
+			                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
+			choose {
+				when(wait(self->cx->onMasterProxiesChanged())) {}
+				when(GetReadVersionReply reply = wait(loadBalance(self->cx->getMasterProxies(false),
+				                                                  &MasterProxyInterface::getConsistentReadVersion,
+				                                                  request, self->cx->taskID))) {
+					return reply.version;
+				}
+			}
+		}
+	}
+
+	Future<Version> getMinKnownCommittedVersion() { return _getMinKnownCommittedVersion(this); }
 };
 
 // Monitors "backupStartedKey". If "started" is true, wait until the key is set;
@@ -680,63 +697,48 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 	}
 }
 
-ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self) {
+ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self, bool keyPresent) {
 	state Future<Void> pullFinished = Void();
-	state Future<bool> started;
-	state Future<GetReadVersionReply> replyFuture = Never();
 
 	loop {
-		started = monitorBackupStartedKeyChanges(self, true, true);
-		loop choose {
-			when(bool present = wait(started)) {
-				replyFuture = Never();
-				break;
+		state Future<bool> present = monitorBackupStartedKeyChanges(self, !keyPresent, /*watch=*/true);
+		if (keyPresent) {
+			pullFinished = pullAsyncData(self);
+			self->pulling = true;
+			wait(success(present) || pullFinished);
+			if (pullFinished.isReady()) {
+				self->pulling = false;
+				return Void(); // backup is done for some old epoch.
 			}
-			when(wait(self->cx->onMasterProxiesChanged() ||
-			          delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
-				GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
-				                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
-				replyFuture = loadBalance(self->cx->getMasterProxies(false),
-				                          &MasterProxyInterface::getConsistentReadVersion, request, self->cx->taskID);
-			}
-			when(GetReadVersionReply reply = wait(replyFuture)) {
-				replyFuture = Never();
-				self->savedVersion = std::max(reply.version, self->savedVersion);
-				self->minKnownCommittedVersion = std::max(reply.version, self->minKnownCommittedVersion);
-				TraceEvent("BackupWorkerNoopPop", self->myId).detail("SavedVersion", self->savedVersion);
-				self->pop(); // Pop while the worker is in this NOOP state.
-			}
-		}
 
-		Future<bool> stopped = monitorBackupStartedKeyChanges(self, false, true);
-		pullFinished = pullAsyncData(self);
-		self->pulling = true;
-		wait(success(stopped) || pullFinished);
-		if (pullFinished.isReady()) {
+			// Even though the snapshot is done, mutation logs may not be written
+			// out yet. We need to make sure mutations up to this point is written.
+			Version currentVersion = wait(self->getMinKnownCommittedVersion());
+			wait(self->pulledVersion.whenAtLeast(currentVersion));
+			pullFinished = Future<Void>(); // cancels pullAsyncData()
 			self->pulling = false;
-			return Void(); // backup is done for some old epoch.
-		}
+			TraceEvent("BackupWorkerPaused", self->myId);
+		} else {
+			// Backup key is not present, enter this NOOP POP mode.
+			state Future<Version> committedVersion = self->getMinKnownCommittedVersion();
 
-		// Even though the snapshot is done, mutation logs may not be written
-		// out yet. We need to make usre mutations up to this point is written.
-		state Version currentVersion;
-		loop {
-			GetReadVersionRequest request(1, GetReadVersionRequest::PRIORITY_DEFAULT |
-			                                     GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
-			choose {
-				when(wait(self->cx->onMasterProxiesChanged())) {}
-				when(GetReadVersionReply reply = wait(loadBalance(self->cx->getMasterProxies(false),
-				                                                  &MasterProxyInterface::getConsistentReadVersion,
-				                                                  request, self->cx->taskID))) {
-					currentVersion = reply.version;
-					break;
+			loop choose {
+				when(wait(success(present))) { break; }
+				when(wait(success(committedVersion) || delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
+					if (committedVersion.isReady()) {
+						self->savedVersion = std::max(committedVersion.get(), self->savedVersion);
+						self->minKnownCommittedVersion =
+						    std::max(committedVersion.get(), self->minKnownCommittedVersion);
+						TraceEvent("BackupWorkerNoopPop", self->myId).detail("SavedVersion", self->savedVersion);
+						self->pop(); // Pop while the worker is in this NOOP state.
+						committedVersion = Never();
+					} else {
+						committedVersion = self->getMinKnownCommittedVersion();
+					}
 				}
 			}
 		}
-		wait(self->pulledVersion.whenAtLeast(currentVersion));
-		pullFinished = Future<Void>(); // cancels pullAsyncData()
-		self->pulling = false;
-		TraceEvent("BackupWorkerPaused", self->myId);
+		keyPresent = !keyPresent;
 	}
 }
 
@@ -787,7 +789,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		bool present = wait(monitorBackupStartedKeyChanges(&self, true, false));
 		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present);
 
-		pull = monitorBackupKeyOrPullData(&self);
+		pull = monitorBackupKeyOrPullData(&self, present);
 		done = uploadData(&self);
 
 		loop choose {
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 37fe740860..8992062be8 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -686,7 +686,10 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 	if (request.targetVersion == invalidVersion && desc.maxRestorableVersion.present()) {
 		request.targetVersion = desc.maxRestorableVersion.get();
 	}
-	TraceEvent("FastRestore").detail("TargetVersion", request.targetVersion).detail("BackupDesc", desc.toString());
+
+	if (g_network->isSimulated()) {
+		std::cout << "Restore to version: " << request.targetVersion << "\nBackupDesc: \n" << desc.toString() << "\n\n";
+	}
 
 	Optional<RestorableFileSet> restorable =
 	    wait(SERVER_KNOBS->FASTRESTORE_USE_PARTITIONED_LOGS ? bc->getPartitionedRestoreSet(request.targetVersion)

From 4c75c61f39748bc5716ef127fb8a5c000ecbf61e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 12 Mar 2020 15:30:07 -0700
Subject: [PATCH 105/176] Fix duplicate file removal for subset version ranges

Partitioned logs can have strict subset version ranges, which was not properly
handled -- we used to assume overlapping only happens for the same begin
version.
---
 fdbclient/BackupContainer.actor.cpp | 14 ++++++++++----
 fdbclient/BackupContainer.h         |  2 +-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 29027eff3a..7ab40e2113 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1156,11 +1156,13 @@ public:
 	// If a log file's progress is not saved, a new log file will be generated
 	// with the same begin version. So we can have a file that contains a subset
 	// of contents in another log file.
-	// PRE-CONDITION: logs are already sorted.
+	// PRE-CONDITION: logs are already sorted by (tagId, beginVersion, endVersion).
 	static std::vector<LogFile> filterDuplicates(const std::vector<LogFile>& logs) {
 		std::vector<LogFile> filtered;
 		int i = 0;
 		for (int j = 1; j < logs.size(); j++) {
+			if (logs[j].isSubset(logs[i])) continue;
+
 			if (!logs[i].isSubset(logs[j])) {
 				filtered.push_back(logs[i]);
 			}
@@ -1196,9 +1198,13 @@ public:
 		for (int i = 0; i < logs.size(); i++) {
 			ASSERT(logs[i].tagId >= 0 && logs[i].tagId < logs[i].totalTags);
 			auto& indices = tagIndices[logs[i].tagId];
-			// filter out if indices.back() is subset of files[i]
-			if (!indices.empty() && logs[indices.back()].isSubset(logs[i])) {
-				indices.back() = i;
+			// filter out if indices.back() is subset of files[i] or vice versa
+			if (!indices.empty()) {
+				if (logs[indices.back()].isSubset(logs[i])) {
+					indices.back() = i;
+				} else if (!logs[i].isSubset(logs[indices.back()])) {
+					indices.push_back(i);
+				}
 			} else {
 				indices.push_back(i);
 			}
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 4bf144c07e..3b1f5de5bf 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -85,7 +85,7 @@ struct LogFile {
 	// Returns if this log file contains a subset of content of the given file
 	// by comparing version range and tag ID.
 	bool isSubset(const LogFile& rhs) const {
-		return beginVersion == rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
+		return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
 	}
 
 	std::string toString() const {

From 9ea549ba7d460df1b526a1e351ae6cbad5f32761 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 12 Mar 2020 20:51:10 -0700
Subject: [PATCH 106/176] Updates lastest backup worker progress after all
 previous epochs are done

If workers for previous epochs are still ongoing, we may end up with a
container that miss mutations in previous epochs. So the update only happens
after there are only current epoch's backup workers.
---
 fdbserver/BackupWorker.actor.cpp | 155 ++++++++++++++++---------------
 1 file changed, 81 insertions(+), 74 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 0fa314bc4c..c2587eb905 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -330,95 +330,102 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 // log saved version > snapshot version.
 ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 	loop {
-		wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger());
-		if (self->backups.empty()) {
-			continue;
+		while (self->backups.empty() || !self->logSystem.get()) {
+			wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger() ||
+			     self->logSystem.onChange());
 		}
 
 		// check all workers have started by checking their progress is larger
 		// than the backup's start version.
-		state Reference<BackupProgress> progress(new BackupProgress(self->myId, {}));
+		state Reference<BackupProgress> progress(
+		    new BackupProgress(self->myId, self->logSystem.get()->getOldEpochTagsVersionsInfo()));
 		wait(getBackupProgress(self->cx, self->myId, progress));
 		std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
+		std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
+		    progress->getUnfinishedBackup();
+		bool finishedPreviousEpochs =
+		    toRecruit.empty() || std::get<0>(toRecruit.begin()->first) == self->recruitedEpoch;
 
 		state std::vector<UID> ready;
 		state std::map<UID, Version> savedLogVersions;
-		if (tagVersions.size() == self->logSystem.get()->getLogRouterTags()) {
-			// Check every version is larger than backup's startVersion
-			for (auto& [uid, info] : self->backups) {
-				if (info.allWorkerStarted) {
-					// update update progress so far
-					Version v = std::numeric_limits<Version>::max();
-					for (const auto [tag, version] : tagVersions) {
-						v = std::min(v, version);
-					}
-					savedLogVersions.emplace(uid, v);
-					continue;
+		if (tagVersions.size() != self->logSystem.get()->getLogRouterTags()) {
+			continue;
+		}
+
+		// Check every version is larger than backup's startVersion
+		for (auto& [uid, info] : self->backups) {
+			if (info.allWorkerStarted && finishedPreviousEpochs) {
+				// update update progress so far
+				Version v = std::numeric_limits<Version>::max();
+				for (const auto [tag, version] : tagVersions) {
+					v = std::min(v, version);
 				}
-				bool saved = true;
-				for (const std::pair<Tag, Version> tv : tagVersions) {
-					if (tv.second < info.startVersion) {
-						saved = false;
-						break;
-					}
-				}
-				if (saved) {
-					ready.push_back(uid);
-					info.allWorkerStarted = true;
+				savedLogVersions.emplace(uid, v);
+				continue;
+			}
+			bool saved = true;
+			for (const std::pair<Tag, Version> tv : tagVersions) {
+				if (tv.second < info.startVersion) {
+					saved = false;
+					break;
 				}
 			}
-			if (ready.empty() && savedLogVersions.empty()) continue;
+			if (saved) {
+				ready.push_back(uid);
+				info.allWorkerStarted = true;
+			}
+		}
+		if (ready.empty() && savedLogVersions.empty()) continue;
 
-			// Set "allWorkerStarted" key for ready backups
-			loop {
-				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
-				try {
-					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+		// Set "allWorkerStarted" and "latestBackupWorkerSavedVersion" key for backups
+		loop {
+			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
+			try {
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-					state std::vector<Future<Optional<Value>>> readyValues;
-					state std::vector<BackupConfig> configs;
-					for (UID uid : ready) {
-						configs.emplace_back(uid);
-						readyValues.push_back(tr->get(configs.back().allWorkerStarted().key));
-					}
-
-					state std::vector<Future<Optional<Version>>> prevVersions;
-					state std::vector<BackupConfig> versionConfigs;
-					for (const auto [uid, version] : savedLogVersions) {
-						versionConfigs.emplace_back(uid);
-						prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
-					}
-
-					wait(waitForAll(readyValues) && waitForAll(prevVersions));
-
-					for (int i = 0; i < readyValues.size(); i++) {
-						if (!readyValues[i].get().present()) {
-							configs[i].allWorkerStarted().set(tr, true);
-							TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", ready[i].toString());
-						}
-					}
-
-					for (int i = 0; i < prevVersions.size(); i++) {
-						const Version current = savedLogVersions[versionConfigs[i].getUid()];
-						if (prevVersions[i].get().present()) {
-							const Version prev = prevVersions[i].get().get();
-							TraceEvent(SevWarn, "BackupWorkerVersionInverse", self->myId)
-							    .detail("Prev", prev)
-							    .detail("Current", current);
-						}
-						if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
-							TraceEvent("BackupWorkerSetVersion", self->myId)
-							    .detail("BackupID", versionConfigs[i].getUid())
-							    .detail("Version", current);
-							versionConfigs[i].latestBackupWorkerSavedVersion().set(tr, current);
-						}
-					}
-					wait(tr->commit());
-					break;
-				} catch (Error& e) {
-					wait(tr->onError(e));
+				state std::vector<Future<Optional<Value>>> readyValues;
+				state std::vector<BackupConfig> configs;
+				for (UID uid : ready) {
+					configs.emplace_back(uid);
+					readyValues.push_back(tr->get(configs.back().allWorkerStarted().key));
 				}
+
+				state std::vector<Future<Optional<Version>>> prevVersions;
+				state std::vector<BackupConfig> versionConfigs;
+				for (const auto [uid, version] : savedLogVersions) {
+					versionConfigs.emplace_back(uid);
+					prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
+				}
+
+				wait(waitForAll(readyValues) && waitForAll(prevVersions));
+
+				for (int i = 0; i < readyValues.size(); i++) {
+					if (!readyValues[i].get().present()) {
+						configs[i].allWorkerStarted().set(tr, true);
+						TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", ready[i].toString());
+					}
+				}
+
+				for (int i = 0; i < prevVersions.size(); i++) {
+					const Version current = savedLogVersions[versionConfigs[i].getUid()];
+					if (prevVersions[i].get().present()) {
+						const Version prev = prevVersions[i].get().get();
+						TraceEvent(SevWarn, "BackupWorkerVersionInverse", self->myId)
+						    .detail("Prev", prev)
+						    .detail("Current", current);
+					}
+					if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
+						TraceEvent("BackupWorkerSetVersion", self->myId)
+						    .detail("BackupID", versionConfigs[i].getUid())
+						    .detail("Version", current);
+						versionConfigs[i].latestBackupWorkerSavedVersion().set(tr, current);
+					}
+				}
+				wait(tr->commit());
+				break;
+			} catch (Error& e) {
+				wait(tr->onError(e));
 			}
 		}
 	}

From b18f192831e685a575564b11a8faf9a5f40304ea Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 13 Mar 2020 18:44:15 -0700
Subject: [PATCH 107/176] Fix decode bug of missing mutations

After reading a new block, all mutations are sorted by version again, which
can invalidate previously tuple. As a result, the decoded file will miss some
of the mutations.
---
 fdbbackup/FileDecoder.actor.cpp | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 62716a562d..ef074adfac 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -245,19 +245,18 @@ struct DecodeProgress {
 		ASSERT(!self->finished());
 
 		loop {
-			state std::tuple<Arena, Version, int32_t, StringRef> tuple = self->keyValues[0];
+			if (self->keyValues.size() == 1) {
+				// Try to decode another block when only one left
+				wait(readAndDecodeFile(self));
+			}
 
+			auto& tuple = self->keyValues[0];
 			ASSERT(std::get<2>(tuple) == 0); // first part number must be 0.
 
 			// decode next versions, check if they are continuous parts
-			state int idx = 1; // next kv pair in "keyValues"
-			state int bufSize = std::get<3>(tuple).size();
-			state int lastPart = 0;
-			loop {
-				// Try to decode another block if needed
-				if (idx == self->keyValues.size()) {
-					wait(readAndDecodeFile(self));
-				}
+			int idx = 1; // next kv pair in "keyValues"
+			int bufSize = std::get<3>(tuple).size();
+			for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
 				if (idx == self->keyValues.size()) break;
 
 				auto next_tuple = self->keyValues[idx];
@@ -270,8 +269,6 @@ struct DecodeProgress {
 					throw restore_corrupted_data();
 				}
 				bufSize += std::get<3>(next_tuple).size();
-				idx++;
-				lastPart++;
 			}
 
 			VersionedMutations m;

From e40f937d3a0d923382dd8bc9bffa701f97b026c2 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 14 Mar 2020 09:42:42 -0700
Subject: [PATCH 108/176] Fix missing mutations in splitMutation

When a range mutation is larger than the last split point, this mutation can
become missing in the RestoreLoader, which is fixed in this commit.
---
 fdbserver/RestoreLoader.actor.cpp | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 42fd83bf53..1b2f48925f 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -463,11 +463,25 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 				              nodeIDs.contents());
 				ASSERT(mvector.size() == nodeIDs.size());
 
+				if (debugMutation("RestoreLoader", commitVersion.version, kvm)) {
+					TraceEvent e("DebugSplit");
+					int i = 0;
+					for (auto& [key, uid] : *pRangeToApplier) {
+						e.detail(format("Range%d", i).c_str(), printable(key))
+						    .detail(format("UID%d", i).c_str(), uid.toString());
+						i++;
+					}
+				}
 				for (splitMutationIndex = 0; splitMutationIndex < mvector.size(); splitMutationIndex++) {
 					MutationRef mutation = mvector[splitMutationIndex];
 					UID applierID = nodeIDs[splitMutationIndex];
 					// printf("SPLITTED MUTATION: %d: mutation:%s applierID:%s\n", splitMutationIndex,
 					// mutation.toString().c_str(), applierID.toString().c_str());
+					if (debugMutation("RestoreLoader", commitVersion.version, mutation)) {
+						TraceEvent("SplittedMutation")
+						    .detail("Version", commitVersion.toString())
+						    .detail("Mutation", mutation.toString());
+					}
 					applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
 					applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
 					applierMutationsSize[applierID] += mutation.expectedSize();
@@ -522,8 +536,14 @@ void splitMutation(std::map<Key, UID>* pRangeToApplier, MutationRef m, Arena& mv
 	ASSERT(mvector.empty());
 	ASSERT(nodeIDs.empty());
 	// key range [m->param1, m->param2)
-	std::map<Standalone<KeyRef>, UID>::iterator itlow, itup; // we will return [itlow, itup)
+	std::map<Key, UID>::iterator itlow, itup; // we will return [itlow, itup)
 	itlow = pRangeToApplier->lower_bound(m.param1); // lower_bound returns the iterator that is >= m.param1
+	if (itlow == pRangeToApplier->end()) {
+		--itlow;
+		mvector.push_back_deep(mvector_arena, m);
+		nodeIDs.push_back(nodeIDs_arena, itlow->second);
+		return;
+	}
 	if (itlow->first > m.param1) {
 		if (itlow != pRangeToApplier->begin()) {
 			--itlow;
@@ -533,7 +553,7 @@ void splitMutation(std::map<Key, UID>* pRangeToApplier, MutationRef m, Arena& mv
 	itup = pRangeToApplier->upper_bound(m.param2); // return rmap::end if no key is after m.param2.
 	ASSERT(itup == pRangeToApplier->end() || itup->first > m.param2);
 
-	std::map<Standalone<KeyRef>, UID>::iterator itApplier;
+	std::map<Key, UID>::iterator itApplier;
 	while (itlow != itup) {
 		Standalone<MutationRef> curm; // current mutation
 		curm.type = m.type;
@@ -776,7 +796,6 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 		    .detail("CommitVersion", version)
 		    .detail("ParsedMutationKV", m.toString());
 
-		ASSERT_WE_THINK(kvOps.find(msgVersion) != kvOps.end());
 		it.first->second.push_back_deep(it.first->second.arena(), m);
 		// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
 		if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {

From 799f0b4b0e09c443e3f02bfa80e5267048e4f2e4 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 14 Mar 2020 15:54:47 -0700
Subject: [PATCH 109/176] Small code refactor

---
 fdbclient/BackupContainer.actor.cpp |  5 ++---
 fdbserver/RestoreApplier.actor.cpp  |  1 -
 fdbserver/RestoreLoader.actor.cpp   | 23 +++++++++--------------
 fdbserver/RestoreMaster.actor.cpp   |  2 --
 4 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 7ab40e2113..e6a7c62680 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -1266,9 +1266,8 @@ public:
 			if (partitioned) {
 				// sort by tag ID so that filterDuplicates works.
 				std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) {
-					return a.tagId == b.tagId ? (a.beginVersion == b.beginVersion ? a.endVersion < b.endVersion
-					                                                              : a.beginVersion < b.beginVersion)
-					                          : (a.tagId < b.tagId);
+					return std::tie(a.tagId, a.beginVersion, a.endVersion) <
+					       std::tie(b.tagId, b.beginVersion, b.endVersion);
 				});
 
 				// Remove duplicated log files that can happen for old epochs.
diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index fb31fea375..65deb246ff 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -139,7 +139,6 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMu
 			const MutationRef& mutation = req.mutations[mIndex];
 			const LogMessageVersion mutationVersion(commitVersion, req.subs[mIndex]);
 			TraceEvent(SevFRMutationInfo, "FastRestoreApplierPhaseReceiveMutations", self->id())
-			    .detail("ApplierNode", self->id())
 			    .detail("RestoreAsset", req.asset.toString())
 			    .detail("Version", mutationVersion.toString())
 			    .detail("Index", mIndex)
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 1b2f48925f..c21434e6cf 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -777,20 +777,19 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
 
 	// Now data only contains the kv mutation within restoreRange
 	VectorRef<KeyValueRef> data = blockData.slice(rangeStart, rangeEnd);
-	int start = 0;
-	int end = data.size();
 
-	// Convert KV in data into mutations in kvOps
-	for (int i = start; i < end; ++i) {
+	// Note we give INT_MAX as the sub sequence number to override any log mutations.
+	const LogMessageVersion msgVersion(version, std::numeric_limits<int32_t>::max());
+
+	// Convert KV in data into SET mutations of different keys in kvOps
+	for (const KeyValueRef& kv : data) {
 		// NOTE: The KV pairs in range files are the real KV pairs in original DB.
 		// Should NOT add prefix or remove surfix for the backup data!
-		MutationRef m(MutationRef::Type::SetValue, data[i].key,
-		              data[i].value); // ASSUME: all operation in range file is set.
+		MutationRef m(MutationRef::Type::SetValue, kv.key,
+		              kv.value); // ASSUME: all operation in range file is set.
 		cc->loadedRangeBytes += m.totalSize();
 
 		// We cache all kv operations into kvOps, and apply all kv operations later in one place
-		// Note we give INT_MAX as the sub sequence number to override any log mutations.
-		const LogMessageVersion msgVersion(version, std::numeric_limits<int32_t>::max());
 		auto it = kvOps.insert(std::make_pair(msgVersion, MutationsVec()));
 		TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
 		    .detail("CommitVersion", version)
@@ -831,13 +830,9 @@ ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(NotifiedVersion* pPro
 	wait(pProcessedFileOffset->whenAtLeast(asset.offset));
 
 	if (pProcessedFileOffset->get() == asset.offset) {
-		int start = 0;
-		int end = data.size();
-		for (int i = start; i < end; ++i) {
-			// Key k = data[i].key.withPrefix(mutationLogPrefix);
-			// ValueRef v = data[i].value;
+		for (const KeyValueRef& kv : data) {
 			// Concatenate the backuped param1 and param2 (KV) at the same version.
-			concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, data[i].key, data[i].value, asset);
+			concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, kv.key, kv.value, asset);
 		}
 		pProcessedFileOffset->set(asset.offset + asset.len);
 	}
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 8992062be8..9b1551e242 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -616,14 +616,12 @@ void splitKeyRangeForAppliers(Reference<MasterBatchData> batchData,
 	}
 
 	std::set<Key>::iterator splitter = keyrangeSplitter.begin();
-	int i = 0;
 	batchData->rangeToApplier.clear();
 	for (auto& applier : appliersInterf) {
 		if (splitter == keyrangeSplitter.end()) {
 			break; // Not all appliers will be used
 		}
 		batchData->rangeToApplier[*splitter] = applier.first;
-		i++;
 		splitter++;
 	}
 	ASSERT(batchData->rangeToApplier.size() > 0);

From 4065ca2a65d8385e07762bca2adac62cc1938b4e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 15 Mar 2020 21:46:38 -0700
Subject: [PATCH 110/176] Fix duplicated mutation in StagingKey

For some reason I am not sure why, there can be duplicated mutations added to
StagingKey, which needs to be filtered out. Otherwise, atomic operations can
result in corrupted data in database.
---
 fdbserver/RestoreApplier.actor.h | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index a50c7f346f..1c398cfac8 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -64,6 +64,8 @@ struct StagingKey {
 			TraceEvent("FastRestoreApplierStagingKeyMutationAtSameVersion")
 			    .detail("Version", newVersion.toString())
 			    .detail("NewMutation", m.toString())
+			    .detail("Key", printable(key))
+			    .detail("Value", printable(val))
 			    .detail("ExistingKeyType", typeString[type]);
 			if (m.type == MutationRef::SetValue) {
 				if (type == MutationRef::SetValue) {
@@ -91,8 +93,13 @@ struct StagingKey {
 				    .detail("NewMutation", m.toString())
 				    .detail("ExistingKeyType", typeString[type])
 				    .detail("ExitingKeyValue", val);
+			} else {
+				ASSERT(false); // Can't be true same key, same version, different mutation
 			}
+
+			return;
 		}
+
 		// newVersion can be smaller than version as different loaders can send
 		// mutations out of order.
 		if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
@@ -107,9 +114,13 @@ struct StagingKey {
 			if (it == pendingMutations.end()) {
 				bool inserted;
 				std::tie(it, inserted) = pendingMutations.emplace(newVersion, MutationsVec());
+				// TODO: Do we really need deep copy?
+				it->second.push_back_deep(it->second.arena(), m);
+			} else {
+				// Duplicated mutation ignored.
+				MutationRef& m1 = *(it->second.begin());
+				ASSERT(m1.type == m.type && m1.param1 == m.param1 && m1.param2 == m.param2);
 			}
-			// TODO: Do we really need deep copy?
-			it->second.push_back_deep(it->second.arena(), m);
 		}
 	}
 
@@ -126,8 +137,7 @@ struct StagingKey {
 		}
 		if (lb->first == version) {
 			// Sanity check mutations at version are either atomicOps which can be ignored or the same value as buffered
-			for (int i = 0; i < lb->second.size(); i++) {
-				MutationRef m = lb->second[i];
+			for (const MutationRef& m : lb->second) {
 				if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
 					if (std::tie(type, key, val) != std::tie(m.type, m.param1, m.param2)) {
 						TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnhandledSituation")
@@ -138,12 +148,9 @@ struct StagingKey {
 					}
 				}
 			}
+			lb++;
 		}
-		while (lb != pendingMutations.end()) {
-			if (lb->first == version) {
-				lb++;
-				continue;
-			}
+		for (; lb != pendingMutations.end(); lb++) {
 			for (auto& mutation : lb->second) {
 				if (type == MutationRef::CompareAndClear) { // Special atomicOp
 					Arena arena;
@@ -162,16 +169,15 @@ struct StagingKey {
 				} else if (mutation.type == MutationRef::SetValue || mutation.type == MutationRef::ClearRange) {
 					type = MutationRef::SetValue; // Precomputed result should be set to DB.
 					TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet")
-					    .detail("Type", typeString[mutation.type])
+					    .detail("MutationType", typeString[mutation.type])
 					    .detail("Version", lb->first.toString());
 				} else {
 					TraceEvent(SevWarnAlways, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation")
-					    .detail("Type", typeString[mutation.type])
+					    .detail("MutationType", typeString[mutation.type])
 					    .detail("Version", lb->first.toString());
 				}
 			}
 			version = lb->first;
-			lb++;
 		}
 	}
 

From 4bdb32be142cb2ef3b602746b9e55cac607706ed Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 16 Mar 2020 18:20:02 -0700
Subject: [PATCH 111/176] Batch sending all mutations of a version from
 RestoreLoader

This optimization is to reduce the number of messages sent from loader to
applier, which was unintentionally done when introducing sub sequence numbers
for mutations.
---
 fdbserver/RestoreLoader.actor.cpp | 73 ++++++++++++++++++-------------
 1 file changed, 42 insertions(+), 31 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index c21434e6cf..1a9931ef91 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -436,17 +436,17 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 	splitMutationIndex = 0;
 	kvCount = 0;
 
+	// applierMutationsBuffer is the mutation vector to be sent to each applier
+	// applierMutationsSize is buffered mutation vector size for each applier
+	state std::map<UID, MutationsVec> applierMutationsBuffer;
+	state std::map<UID, SubSequenceVec> applierSubsBuffer;
+	state std::map<UID, double> applierMutationsSize;
+	for (auto& applierID : applierIDs) {
+		applierMutationsBuffer[applierID] = MutationsVec();
+		applierSubsBuffer[applierID] = SubSequenceVec();
+		applierMutationsSize[applierID] = 0.0;
+	}
 	for (kvOp = kvOps.begin(); kvOp != kvOps.end(); kvOp++) {
-		// applierMutationsBuffer is the mutation vector to be sent to each applier
-		// applierMutationsSize is buffered mutation vector size for each applier
-		std::map<UID, MutationsVec> applierMutationsBuffer;
-		std::map<UID, SubSequenceVec> applierSubsBuffer;
-		std::map<UID, double> applierMutationsSize;
-		for (auto& applierID : applierIDs) {
-			applierMutationsBuffer[applierID] = MutationsVec();
-			applierSubsBuffer[applierID] = SubSequenceVec();
-			applierMutationsSize[applierID] = 0.0;
-		}
 		const LogMessageVersion& commitVersion = kvOp->first;
 		ASSERT(commitVersion.version >= asset.beginVersion);
 		ASSERT(commitVersion.version <= asset.endVersion); // endVersion is an empty commit to ensure progress
@@ -493,35 +493,46 @@ ACTOR Future<Void> sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat
 				--itlow; // make sure itlow->first <= m.param1
 				ASSERT(itlow->first <= kvm.param1);
 				UID applierID = itlow->second;
-				// printf("KV--Applier: K:%s ApplierID:%s\n", kvm.param1.toString().c_str(),
-				// applierID.toString().c_str());
 				kvCount++;
 
+				if (debugMutation("RestoreLoader", commitVersion.version, kvm)) {
+					TraceEvent("SendMutation")
+					    .detail("Applier", applierID)
+					    .detail("Version", commitVersion.toString())
+					    .detail("Mutation", kvm.toString());
+				}
 				applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), kvm);
 				applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub);
 				applierMutationsSize[applierID] += kvm.expectedSize();
 			}
-		} // Mutations at the same version
+		} // Mutations at the same LogMessageVersion
 
-		// TODO: Sanity check each asset has been received exactly once!
-		// Send the mutations to appliers for each version
-		for (const UID& applierID : applierIDs) {
-			requests.emplace_back(applierID, RestoreSendVersionedMutationsRequest(
-			                                     batchIndex, asset, prevVersion, commitVersion.version, isRangeFile,
-			                                     applierMutationsBuffer[applierID], applierSubsBuffer[applierID]));
+		// Batch same Version's mutations in one request. We could batch more by
+		// changing the version comparison below.
+		auto next = std::next(kvOp, 1);
+		if (next == kvOps.end() || commitVersion.version < next->first.version) {
+			// TODO: Sanity check each asset has been received exactly once!
+			// Send the mutations to appliers for each version
+			for (const UID& applierID : applierIDs) {
+				requests.emplace_back(applierID, RestoreSendVersionedMutationsRequest(
+				                                     batchIndex, asset, prevVersion, commitVersion.version, isRangeFile,
+				                                     applierMutationsBuffer[applierID], applierSubsBuffer[applierID]));
+			}
+			TraceEvent(SevDebug, "FastRestore_SendMutationToApplier")
+			    .detail("PrevVersion", prevVersion)
+			    .detail("CommitVersion", commitVersion.toString())
+			    .detail("RestoreAsset", asset.toString());
+			ASSERT(prevVersion < commitVersion.version);
+			prevVersion = commitVersion.version;
+			wait(sendBatchRequests(&RestoreApplierInterface::sendMutationVector, *pApplierInterfaces, requests,
+			                       TaskPriority::RestoreLoaderSendMutations));
+			requests.clear();
+			for (auto& applierID : applierIDs) {
+				applierMutationsBuffer[applierID] = MutationsVec();
+				applierSubsBuffer[applierID] = SubSequenceVec();
+				applierMutationsSize[applierID] = 0.0;
+			}
 		}
-		TraceEvent(SevDebug, "FastRestore_SendMutationToApplier")
-		    .detail("PrevVersion", prevVersion)
-		    .detail("CommitVersion", commitVersion.toString())
-		    .detail("RestoreAsset", asset.toString());
-		ASSERT(prevVersion <= commitVersion.version);
-		prevVersion = commitVersion.version;
-		// Tracking this request can be spammy
-		wait(sendBatchRequests(&RestoreApplierInterface::sendMutationVector, *pApplierInterfaces, requests,
-		                       TaskPriority::RestoreLoaderSendMutations,
-		                       SERVER_KNOBS->FASTRESTORE_TRACK_LOADER_SEND_REQUESTS));
-
-		requests.clear();
 	} // all versions of mutations in the same file
 
 	TraceEvent("FastRestore").detail("LoaderSendMutationOnAppliers", kvCount);

From fea6155714bec684c34645080350dcb6442c3412 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 16 Mar 2020 18:22:24 -0700
Subject: [PATCH 112/176] StagingKey uses mutation instead of a vector of
 mutations for each log version

Because each log version contains commit version and subsequence number, each
key can only have one mutation for its log version. This simplifies
StagingKey::add() a lot.
---
 fdbserver/RestoreApplier.actor.cpp |  12 ++-
 fdbserver/RestoreApplier.actor.h   | 126 +++++++++++------------------
 2 files changed, 53 insertions(+), 85 deletions(-)

diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index 65deb246ff..ac2cee020e 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -202,8 +202,8 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
     std::map<Key, std::map<Key, StagingKey>::iterator> incompleteStagingKeys, Database cx, UID applierID) {
 	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 	state std::vector<Future<Optional<Value>>> fValues;
-	state int i = 0;
 	state int retries = 0;
+
 	TraceEvent("FastRestoreApplierGetAndComputeStagingKeysStart", applierID)
 	    .detail("GetKeys", incompleteStagingKeys.size());
 	loop {
@@ -228,7 +228,7 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 	}
 
 	ASSERT(fValues.size() == incompleteStagingKeys.size());
-	i = 0;
+	int i = 0;
 	for (auto& key : incompleteStagingKeys) {
 		if (!fValues[i].get().present()) {
 			TraceEvent(SevWarnAlways, "FastRestoreApplierGetAndComputeStagingKeysUnhandledError")
@@ -237,11 +237,9 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 			    .detail("PendingMutations", key.second->second.pendingMutations.size())
 			    .detail("StagingKeyType", (int)key.second->second.type);
 			for (auto& vm : key.second->second.pendingMutations) {
-				for (auto& m : vm.second) {
-					TraceEvent(SevWarnAlways, "FastRestoreApplierGetAndComputeStagingKeysUnhandledError")
-					    .detail("PendingMutationVersion", vm.first.toString())
-					    .detail("PendingMutation", m.toString());
-				}
+				TraceEvent(SevWarnAlways, "FastRestoreApplierGetAndComputeStagingKeysUnhandledError")
+				    .detail("PendingMutationVersion", vm.first.toString())
+				    .detail("PendingMutation", vm.second.toString());
 			}
 			key.second->second.precomputeResult();
 			i++;
diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index 1c398cfac8..66cf075bf6 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -52,7 +52,7 @@ struct StagingKey {
 	Value val;
 	MutationRef::Type type; // set or clear
 	LogMessageVersion version; // largest version of set or clear for the key
-	std::map<LogMessageVersion, MutationsVec> pendingMutations; // mutations not set or clear type
+	std::map<LogMessageVersion, Standalone<MutationRef>> pendingMutations; // mutations not set or clear type
 
 	explicit StagingKey() : version(0), type(MutationRef::MAX_ATOMIC_OP) {}
 
@@ -60,43 +60,15 @@ struct StagingKey {
 	// Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set
 	void add(const MutationRef& m, LogMessageVersion newVersion) {
 		ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue);
+		if (debugMutation("StagingKeyAdd", newVersion.version, m)) {
+			TraceEvent("StagingKeyAdd")
+			    .detail("Version", version.toString())
+			    .detail("NewVersion", newVersion.toString())
+			    .detail("Mutation", m.toString());
+		}
 		if (version == newVersion) { // Sanity check
-			TraceEvent("FastRestoreApplierStagingKeyMutationAtSameVersion")
-			    .detail("Version", newVersion.toString())
-			    .detail("NewMutation", m.toString())
-			    .detail("Key", printable(key))
-			    .detail("Value", printable(val))
-			    .detail("ExistingKeyType", typeString[type]);
-			if (m.type == MutationRef::SetValue) {
-				if (type == MutationRef::SetValue) {
-					if (m.param2 != val) {
-						TraceEvent(SevError, "FastRestoreApplierStagingKeyMutationAtSameVersionUnhandled")
-						    .detail("Version", newVersion.toString())
-						    .detail("NewMutation", m.toString())
-						    .detail("ExistingKeyType", typeString[type])
-						    .detail("ExitingKeyValue", val)
-						    .detail("Investigate",
-						            "Why would backup have two sets with different value at same version");
-					} // else {} Backup has duplicate set at the same version
-				} else {
-					TraceEvent(SevWarnAlways, "FastRestoreApplierStagingKeyMutationAtSameVersionOverride")
-					    .detail("Version", newVersion.toString())
-					    .detail("NewMutation", m.toString())
-					    .detail("ExistingKeyType", typeString[type])
-					    .detail("ExitingKeyValue", val);
-					type = (MutationRef::Type)m.type;
-					val = m.param2;
-				}
-			} else if (m.type == MutationRef::ClearRange) {
-				TraceEvent(SevWarnAlways, "FastRestoreApplierStagingKeyMutationAtSameVersionSkipped")
-				    .detail("Version", newVersion.toString())
-				    .detail("NewMutation", m.toString())
-				    .detail("ExistingKeyType", typeString[type])
-				    .detail("ExitingKeyValue", val);
-			} else {
-				ASSERT(false); // Can't be true same key, same version, different mutation
-			}
-
+			TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m.toString());
+			ASSERT(type == m.type && key == m.param1 && val == m.param2);
 			return;
 		}
 
@@ -112,14 +84,14 @@ struct StagingKey {
 		} else {
 			auto it = pendingMutations.find(newVersion);
 			if (it == pendingMutations.end()) {
-				bool inserted;
-				std::tie(it, inserted) = pendingMutations.emplace(newVersion, MutationsVec());
-				// TODO: Do we really need deep copy?
-				it->second.push_back_deep(it->second.arena(), m);
+				pendingMutations.emplace(newVersion, m);
 			} else {
 				// Duplicated mutation ignored.
-				MutationRef& m1 = *(it->second.begin());
-				ASSERT(m1.type == m.type && m1.param1 == m.param1 && m1.param2 == m.param2);
+				TraceEvent("SameVersion")
+				    .detail("Version", version.toString())
+				    .detail("Mutation", m.toString())
+				    .detail("NewVersion", newVersion.toString());
+				ASSERT(it->second.type == m.type && it->second.param1 == m.param1 && it->second.param2 == m.param2);
 			}
 		}
 	}
@@ -130,52 +102,50 @@ struct StagingKey {
 		    .detail("Key", key)
 		    .detail("Version", version.toString())
 		    .detail("LargestPendingVersion",
-		            (pendingMutations.empty() ? "-1" : pendingMutations.rbegin()->first.toString()));
-		std::map<LogMessageVersion, MutationsVec>::iterator lb = pendingMutations.lower_bound(version);
+		            (pendingMutations.empty() ? "[none]" : pendingMutations.rbegin()->first.toString()));
+		std::map<LogMessageVersion, Standalone<MutationRef>>::iterator lb = pendingMutations.lower_bound(version);
 		if (lb == pendingMutations.end()) {
 			return;
 		}
 		if (lb->first == version) {
 			// Sanity check mutations at version are either atomicOps which can be ignored or the same value as buffered
-			for (const MutationRef& m : lb->second) {
-				if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
-					if (std::tie(type, key, val) != std::tie(m.type, m.param1, m.param2)) {
-						TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnhandledSituation")
-						    .detail("BufferedType", typeString[type])
-						    .detail("PendingType", typeString[m.type])
-						    .detail("BufferedVal", val.toString())
-						    .detail("PendingVal", m.param2.toString());
-					}
+			MutationRef m = lb->second;
+			if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
+				if (std::tie(type, key, val) != std::tie(m.type, m.param1, m.param2)) {
+					TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnhandledSituation")
+					    .detail("BufferedType", typeString[type])
+					    .detail("PendingType", typeString[m.type])
+					    .detail("BufferedVal", val.toString())
+					    .detail("PendingVal", m.param2.toString());
 				}
 			}
 			lb++;
 		}
 		for (; lb != pendingMutations.end(); lb++) {
-			for (auto& mutation : lb->second) {
-				if (type == MutationRef::CompareAndClear) { // Special atomicOp
-					Arena arena;
-					Optional<ValueRef> retVal = doCompareAndClear(val, mutation.param2, arena);
-					if (!retVal.present()) {
-						val = key;
-						type = MutationRef::ClearRange;
-					} // else no-op
-				} else if (isAtomicOp((MutationRef::Type)mutation.type)) {
-					Optional<StringRef> inputVal;
-					if (hasBaseValue()) {
-						inputVal = val;
-					}
-					val = applyAtomicOp(inputVal, mutation.param2, (MutationRef::Type)mutation.type);
-					type = MutationRef::SetValue; // Precomputed result should be set to DB.
-				} else if (mutation.type == MutationRef::SetValue || mutation.type == MutationRef::ClearRange) {
-					type = MutationRef::SetValue; // Precomputed result should be set to DB.
-					TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet")
-					    .detail("MutationType", typeString[mutation.type])
-					    .detail("Version", lb->first.toString());
-				} else {
-					TraceEvent(SevWarnAlways, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation")
-					    .detail("MutationType", typeString[mutation.type])
-					    .detail("Version", lb->first.toString());
+			MutationRef mutation = lb->second;
+			if (type == MutationRef::CompareAndClear) { // Special atomicOp
+				Arena arena;
+				Optional<ValueRef> retVal = doCompareAndClear(val, mutation.param2, arena);
+				if (!retVal.present()) {
+					val = key;
+					type = MutationRef::ClearRange;
+				} // else no-op
+			} else if (isAtomicOp((MutationRef::Type)mutation.type)) {
+				Optional<StringRef> inputVal;
+				if (hasBaseValue()) {
+					inputVal = val;
 				}
+				val = applyAtomicOp(inputVal, mutation.param2, (MutationRef::Type)mutation.type);
+				type = MutationRef::SetValue; // Precomputed result should be set to DB.
+			} else if (mutation.type == MutationRef::SetValue || mutation.type == MutationRef::ClearRange) {
+				type = MutationRef::SetValue; // Precomputed result should be set to DB.
+				TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet")
+				    .detail("MutationType", typeString[mutation.type])
+				    .detail("Version", lb->first.toString());
+			} else {
+				TraceEvent(SevWarnAlways, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation")
+				    .detail("MutationType", typeString[mutation.type])
+				    .detail("Version", lb->first.toString());
 			}
 			version = lb->first;
 		}

From 5b36dcaad566bd055976d7d83dab5d26f0e0f855 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 17 Mar 2020 14:45:07 -0700
Subject: [PATCH 113/176] Fix oldest backup epoch for backup workers

The oldest backup epoch is piggybacked in LogSystemConfig from master to
cluster controller and then to all workers. Previously, this epoch is set
to the current master epoch, which is wrong.
---
 fdbserver/BackupWorker.actor.cpp            | 19 +++++++++++++------
 fdbserver/ClusterController.actor.cpp       | 13 +++++++++++--
 fdbserver/TagPartitionedLogSystem.actor.cpp |  4 +++-
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index c2587eb905..177aa41e65 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -72,6 +72,7 @@ struct BackupData {
 	const Optional<Version> endVersion; // old epoch's end version (inclusive), or empty for current epoch
 	const LogEpoch recruitedEpoch;
 	const LogEpoch backupEpoch;
+	LogEpoch oldestBackupEpoch = 0;
 	Version minKnownCommittedVersion;
 	Version savedVersion;
 	AsyncVar<Reference<ILogSystem>> logSystem;
@@ -169,13 +170,12 @@ struct BackupData {
 	}
 
 	void pop() {
-		const LogEpoch oldest = logSystem.get()->getOldestBackupEpoch();
-		if (backupEpoch > oldest) {
+		if (backupEpoch > oldestBackupEpoch) {
 			// Defer pop if old epoch hasn't finished popping yet.
 			TraceEvent("BackupWorkerPopDeferred", myId)
 			    .suppressFor(1.0)
 			    .detail("BackupEpoch", backupEpoch)
-			    .detail("OldestEpoch", oldest)
+			    .detail("OldestEpoch", oldestBackupEpoch)
 			    .detail("Version", savedVersion);
 			return;
 		}
@@ -552,6 +552,14 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		MutationRef m;
 		if (!message.isBackupMessage(&m)) continue;
 
+		if (debugMutation("addMutation", message.version.version, m)) {
+			TraceEvent("BackupWorkerDebug", self->myId)
+			    .detail("Version", message.version.toString())
+			    .detail("Mutation", m.toString())
+			    .detail("KCV", self->minKnownCommittedVersion)
+			    .detail("SavedVersion", self->savedVersion);
+		}
+
 		std::vector<Future<Void>> adds;
 		if (m.type != MutationRef::Type::ClearRange) {
 			for (int index : keyRangeMap[m.param1]) {
@@ -804,15 +812,14 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 				dbInfoChange = db->onChange();
 				Reference<ILogSystem> ls = ILogSystem::fromServerDBInfo(self.myId, db->get(), true);
 				bool hasPseudoLocality = ls.isValid() && ls->hasPseudoLocality(tagLocalityBackup);
-				LogEpoch oldestBackupEpoch = 0;
 				if (hasPseudoLocality) {
 					self.logSystem.set(ls);
 					self.pop();
-					oldestBackupEpoch = ls->getOldestBackupEpoch();
+					self.oldestBackupEpoch = std::max(self.oldestBackupEpoch, ls->getOldestBackupEpoch());
 				}
 				TraceEvent("BackupWorkerLogSystem", self.myId)
 				    .detail("HasBackupLocality", hasPseudoLocality)
-				    .detail("OldestBackupEpoch", oldestBackupEpoch)
+				    .detail("OldestBackupEpoch", self.oldestBackupEpoch)
 				    .detail("Tag", self.tag.toString());
 			}
 			when(wait(done)) {
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 28648fbd10..fc00bccbf0 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2052,8 +2052,17 @@ ACTOR Future<Void> clusterRecruitRemoteFromConfiguration( ClusterControllerData*
 void clusterRegisterMaster( ClusterControllerData* self, RegisterMasterRequest const& req ) {
 	req.reply.send( Void() );
 
-	TraceEvent("MasterRegistrationReceived", self->id).detail("MasterId", req.id).detail("Master", req.mi.toString()).detail("Tlogs", describe(req.logSystemConfig.tLogs)).detail("Resolvers", req.resolvers.size())
-		.detail("RecoveryState", (int)req.recoveryState).detail("RegistrationCount", req.registrationCount).detail("Proxies", req.proxies.size()).detail("RecoveryCount", req.recoveryCount).detail("Stalled", req.recoveryStalled);
+	TraceEvent("MasterRegistrationReceived", self->id)
+	    .detail("MasterId", req.id)
+	    .detail("Master", req.mi.toString())
+	    .detail("Tlogs", describe(req.logSystemConfig.tLogs))
+	    .detail("Resolvers", req.resolvers.size())
+	    .detail("RecoveryState", (int)req.recoveryState)
+	    .detail("RegistrationCount", req.registrationCount)
+	    .detail("Proxies", req.proxies.size())
+	    .detail("RecoveryCount", req.recoveryCount)
+	    .detail("Stalled", req.recoveryStalled)
+	    .detail("OldestBackupEpoch", req.logSystemConfig.oldestBackupEpoch);
 
 	//make sure the request comes from an active database
 	auto db = &self->db;
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 146430cbee..9e94c2e5a1 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -202,7 +202,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 	TagPartitionedLogSystem(UID dbgid, LocalityData locality, LogEpoch e,
 	                        Optional<PromiseStream<Future<Void>>> addActor = Optional<PromiseStream<Future<Void>>>())
 	  : dbgid(dbgid), logSystemType(LogSystemType::empty), expectedLogSets(0), logRouterTags(0), txsTags(0),
-	    repopulateRegionAntiQuorum(0), epoch(e), oldestBackupEpoch(e), recoveryCompleteWrittenToCoreState(false),
+	    repopulateRegionAntiQuorum(0), epoch(e), oldestBackupEpoch(0), recoveryCompleteWrittenToCoreState(false),
 	    locality(locality), remoteLogsWrittenToCoreState(false), hasRemoteServers(false), stopped(false),
 	    addActor(addActor), popActors(false) {}
 
@@ -309,6 +309,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 
 		logSystem->logSystemType = lsConf.logSystemType;
+		logSystem->oldestBackupEpoch = lsConf.oldestBackupEpoch;
 		return logSystem;
 	}
 
@@ -1394,6 +1395,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 			}
 			logset->backupWorkers.push_back(worker);
 		}
+		TraceEvent("SetOldestBackupEpoch", dbgid).detail("Epoch", oldestBackupEpoch);
 		backupWorkerChanged.trigger();
 	}
 

From e1737fc644ea040a4a9ab279b1e543e0a4a32b22 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 17 Mar 2020 19:30:49 -0700
Subject: [PATCH 114/176] Skip setting backupStartedKey if using old mutation
 logs

For old submitBackup(), where partitionedLog is false, do not set the
backupStartedKey in BackupConfig, which signals backup workers to skip these
backups.
---
 fdbclient/FileBackupAgent.actor.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 660bb6c526..14b16ee7dd 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -2405,7 +2405,12 @@ namespace fileBackup {
 
 					state Future<Optional<Value>> started = tr->get(backupStartedKey);
 					state Future<Optional<Value>> taskStarted = tr->get(config.allWorkerStarted().key);
-					wait(success(started) && success(taskStarted));
+					state Future<Optional<bool>> partitionedLog = config.partitionedLogEnabled().get(tr);
+					wait(success(started) && success(taskStarted) && success(partitionedLog));
+
+					if (!partitionedLog.get().present() || !partitionedLog.get().get()) {
+						return Void(); // Skip if not using partitioned logs
+					}
 
 					std::vector<std::pair<UID, Version>> ids;
 					if (started.get().present()) {

From 08173951bcbb36d75e4c4e0127e2483de4930bf7 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 17 Mar 2020 21:35:44 -0700
Subject: [PATCH 115/176] Add an exitEarly flag for backup worker

If a backup worker is on an old epoch, it could exit early if either of the
following is true:
- there is no backups
- all backups starts a version >= the endVersion

If this flag is set, the backup worker exit without doing any work, which
signals the master to update oldest backup epoch.
---
 fdbserver/BackupWorker.actor.cpp | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 177aa41e65..7c516e2e0a 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -81,6 +81,7 @@ struct BackupData {
 	NotifiedVersion pulledVersion;
 	bool pulling = false;
 	bool stopped = false;
+	bool exitEarly = false; // If the worker is on an old epoch and all backups starts a version >= the endVersion
 
 	struct PerBackupInfo {
 		PerBackupInfo() = default;
@@ -135,7 +136,7 @@ struct BackupData {
 	}
 
 	bool allMessageSaved() const {
-		return (endVersion.present() && savedVersion >= endVersion.get()) || stopped;
+		return (endVersion.present() && savedVersion >= endVersion.get()) || stopped || exitEarly;
 	}
 
 	Version maxPopVersion() const {
@@ -290,21 +291,27 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				Optional<Value> value = wait(tr.get(backupStartedKey));
 				std::vector<std::pair<UID, Version>> uidVersions;
+				bool shouldExit = self->endVersion.present();
 				if (value.present()) {
 					uidVersions = decodeBackupStartedValue(value.get());
 					TraceEvent e("BackupWorkerGotStartKey", self->myId);
 					int i = 1;
-					for (auto uidVersion : uidVersions) {
-						e.detail(format("BackupID%d", i), uidVersion.first)
-						    .detail(format("Version%d", i), uidVersion.second);
+					for (auto [uid, version] : uidVersions) {
+						e.detail(format("BackupID%d", i), uid)
+						    .detail(format("Version%d", i), version);
 						i++;
+						if (shouldExit && version < self->endVersion.get()) {
+							shouldExit = false;
+						}
 					}
+					self->exitEarly = shouldExit;
 					self->onBackupChanges(uidVersions);
 					if (started || !watch) return true;
 				} else {
 					TraceEvent("BackupWorkerEmptyStartKey", self->myId);
 					self->onBackupChanges(uidVersions);
 
+					self->exitEarly = shouldExit;
 					if (!started || !watch) {
 						return false;
 					}
@@ -800,12 +807,12 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 
 		// Check if backup key is present to avoid race between this check and
 		// noop pop as well as upload data: pop or skip upload before knowing
-		// there are backup keys.
+		// there are backup keys. Set the "exitEarly" flag if needed.
 		bool present = wait(monitorBackupStartedKeyChanges(&self, true, false));
-		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present);
+		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present).detail("ExitEarly", self.exitEarly);
 
-		pull = monitorBackupKeyOrPullData(&self, present);
-		done = uploadData(&self);
+		pull = self.exitEarly ? Void() : monitorBackupKeyOrPullData(&self, present);
+		done = self.exitEarly ? Void() : uploadData(&self);
 
 		loop choose {
 			when(wait(dbInfoChange)) {

From 6b0d2923e7cdf8e06db21e428d3cc4abfe199f66 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 18 Mar 2020 15:38:06 -0700
Subject: [PATCH 116/176] Add target version as the limit for version batches

If using partitioned logs, the mutations after the target version can be
included if this limit is not considered.
---
 fdbserver/RestoreMaster.actor.cpp | 20 ++++++++++++--------
 fdbserver/RestoreMaster.actor.h   |  6 +++---
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 9b1551e242..95f6f0a95e 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -36,8 +36,9 @@
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 ACTOR static Future<Void> clearDB(Database cx);
-ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
-                                             std::vector<RestoreFileFR>* logFiles, Database cx, RestoreRequest request);
+ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
+                                                std::vector<RestoreFileFR>* logFiles, Database cx,
+                                                RestoreRequest request);
 
 ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData> self, Database cx, RestoreRequest request);
 ACTOR static Future<Void> startProcessRestoreRequests(Reference<RestoreMasterData> self, Database cx);
@@ -276,7 +277,8 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 	self->initBackupContainer(request.url);
 
 	// Get all backup files' description and save them to files
-	wait(collectBackupFiles(self->bc, &rangeFiles, &logFiles, cx, request));
+	Version targetVersion = wait(collectBackupFiles(self->bc, &rangeFiles, &logFiles, cx, request));
+	ASSERT(targetVersion > 0);
 
 	std::sort(rangeFiles.begin(), rangeFiles.end());
 	std::sort(logFiles.begin(), logFiles.end(), [](RestoreFileFR const& f1, RestoreFileFR const& f2) -> bool {
@@ -284,7 +286,8 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 		       std::tie(f2.endVersion, f2.beginVersion, f2.fileIndex, f2.fileName);
 	});
 
-	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches); // Divide files into version batches
+	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches,
+	                          targetVersion); // Divide files into version batches
 	self->dumpVersionBatches(self->versionBatches);
 
 	state std::vector<Future<Void>> fBatches;
@@ -672,9 +675,10 @@ ACTOR static Future<Standalone<VectorRef<RestoreRequest>>> collectRestoreRequest
 }
 
 // Collect the backup files' description into output_files by reading the backupContainer bc.
-ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
-                                             std::vector<RestoreFileFR>* logFiles, Database cx,
-                                             RestoreRequest request) {
+// Returns the restore target version.
+ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
+                                                std::vector<RestoreFileFR>* logFiles, Database cx,
+                                                RestoreRequest request) {
 	state BackupDescription desc = wait(bc->describePartitionedBackup());
 
 	// Convert version to real time for operators to read the BackupDescription desc.
@@ -730,7 +734,7 @@ ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std
 	    .detail("BackupDesc", desc.toString())
 	    .detail("RangeFiles", rangeFiles->size())
 	    .detail("LogFiles", logFiles->size());
-	return Void();
+	return request.targetVersion;
 }
 
 ACTOR static Future<Void> clearDB(Database cx) {
diff --git a/fdbserver/RestoreMaster.actor.h b/fdbserver/RestoreMaster.actor.h
index 4a4520f75d..0731326f58 100644
--- a/fdbserver/RestoreMaster.actor.h
+++ b/fdbserver/RestoreMaster.actor.h
@@ -255,7 +255,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 	// Note: We do not allow a versionBatch size larger than the FASTRESTORE_VERSIONBATCH_MAX_BYTES because the range
 	// file size at a version depends on the number of backupAgents and its upper bound is hard to get.
 	void buildVersionBatches(const std::vector<RestoreFileFR>& rangeFiles, const std::vector<RestoreFileFR>& logFiles,
-	                         std::map<Version, VersionBatch>* versionBatches) {
+	                         std::map<Version, VersionBatch>* versionBatches, Version targetVersion) {
 		bool rewriteNextVersion = false;
 		int rangeIdx = 0;
 		int logIdx = 0; // Ensure each log file is included in version batch
@@ -340,7 +340,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 					vb.logFiles.insert(log);
 				}
 
-				vb.endVersion = nextVersion;
+				vb.endVersion = std::min(nextVersion, targetVersion + 1);
 				prevEndVersion = vb.endVersion;
 			} else {
 				if (vb.size < 1) {
@@ -376,7 +376,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
 		}
 		// The last wip version batch has some files
 		if (vb.size > 0) {
-			vb.endVersion = nextVersion;
+			vb.endVersion = std::min(nextVersion, targetVersion + 1);
 			versionBatches->emplace(vb.beginVersion, vb);
 		}
 	}

From 5359528132210feaa3f6bebd2b4be900984c1447 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 19 Mar 2020 10:08:19 -0700
Subject: [PATCH 117/176] Reduce a call to getLogSystemConfig()

---
 fdbserver/masterserver.actor.cpp | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index cda864a732..8f73d5ad6b 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -478,13 +478,20 @@ ACTOR Future<Void> updateRegistration( Reference<MasterData> self, Reference<ILo
 
 		trigger = self->registrationTrigger.onTrigger();
 
-		TraceEvent("MasterUpdateRegistration", self->dbgid).detail("RecoveryCount", self->cstate.myDBState.recoveryCount).detail("Logs", describe(logSystem->getLogSystemConfig().tLogs));
+		auto logSystemConfig = logSystem->getLogSystemConfig();
+		TraceEvent("MasterUpdateRegistration", self->dbgid)
+		    .detail("RecoveryCount", self->cstate.myDBState.recoveryCount)
+		    .detail("OldestBackupEpoch", logSystemConfig.oldestBackupEpoch)
+		    .detail("Logs", describe(logSystemConfig.tLogs));
 
 		if (!self->cstateUpdated.isSet()) {
-			wait(sendMasterRegistration(self.getPtr(), logSystem->getLogSystemConfig(), self->provisionalProxies, self->resolvers, self->cstate.myDBState.recoveryCount, self->cstate.prevDBState.getPriorCommittedLogServers() ));
+			wait(sendMasterRegistration(self.getPtr(), logSystemConfig, self->provisionalProxies, self->resolvers,
+			                            self->cstate.myDBState.recoveryCount,
+			                            self->cstate.prevDBState.getPriorCommittedLogServers()));
 		} else {
 			updateLogsKey = updateLogsValue(self, cx);
-			wait( sendMasterRegistration( self.getPtr(), logSystem->getLogSystemConfig(), self->proxies, self->resolvers, self->cstate.myDBState.recoveryCount, vector<UID>() ) );
+			wait(sendMasterRegistration(self.getPtr(), logSystemConfig, self->proxies, self->resolvers,
+			                            self->cstate.myDBState.recoveryCount, vector<UID>()));
 		}
 	}
 }

From 9d6de758a7ad191bf1a08a21aee70e87211f23e6 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 19 Mar 2020 14:59:38 -0700
Subject: [PATCH 118/176] Backup Worker: Give a chance of saving progress
 before displaced

Move the exit loop after the saving of progress so that when doneTrigger is
active, we won't exit the loop immediately.
---
 fdbserver/BackupWorker.actor.cpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 7c516e2e0a..150e3665c1 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -422,7 +422,8 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 						    .detail("Prev", prev)
 						    .detail("Current", current);
 					}
-					if (!prevVersions[i].get().present() || prevVersions[i].get().get() < current) {
+					if (self->backupEpoch == self->oldestBackupEpoch &&
+					    (!prevVersions[i].get().present() || prevVersions[i].get().get() < current)) {
 						TraceEvent("BackupWorkerSetVersion", self->myId)
 						    .detail("BackupID", versionConfigs[i].getUid())
 						    .detail("Version", current);
@@ -618,11 +619,6 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 	state Version popVersion = invalidVersion;
 
 	loop {
-		if (self->allMessageSaved()) {
-			self->messages.clear();
-			return Void();
-		}
-
 		// Too large uploadDelay will delay popping tLog data for too long.
 		state Future<Void> uploadDelay = delay(SERVER_KNOBS->BACKUP_UPLOAD_DELAY);
 
@@ -666,6 +662,11 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			self->pop();
 		}
 
+		if (self->allMessageSaved()) {
+			self->messages.clear();
+			return Void();
+		}
+
 		if (!self->pullFinished()) {
 			wait(uploadDelay || self->doneTrigger.onTrigger());
 		}
@@ -844,6 +845,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		if (e.code() == error_code_worker_removed) {
 			pull = Void(); // cancels pulling
 			self.stopped = true;
+			self.doneTrigger.trigger();
 			wait(done);
 		}
 		TraceEvent("BackupWorkerTerminated", self.myId).error(err, true);

From 4a499a3c971bde1d4b6f9b9ef273fd39d4a30653 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 19 Mar 2020 21:35:08 -0700
Subject: [PATCH 119/176] Remove backup worker's first and last pop

The first pop of current epoch can pop old epoch's data before they are saved.
The last pop of a stopped backup worker should be skipped so that after
recovery, the data is still accessible in case the last epoch's progress saving
transaction is delayed.
---
 fdbserver/BackupWorker.actor.cpp | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 150e3665c1..b48fb27f12 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -171,8 +171,12 @@ struct BackupData {
 	}
 
 	void pop() {
-		if (backupEpoch > oldestBackupEpoch) {
+		if (backupEpoch > oldestBackupEpoch || stopped) {
 			// Defer pop if old epoch hasn't finished popping yet.
+			// If stopped because of displacement, do NOT pop as the progress may
+			// not be saved in a timely fashion. As a result, next epoch may still
+			// need to read mutations in the version range. Let the next epoch's
+			// worker do the pop instead.
 			TraceEvent("BackupWorkerPopDeferred", myId)
 			    .suppressFor(1.0)
 			    .detail("BackupEpoch", backupEpoch)
@@ -180,8 +184,7 @@ struct BackupData {
 			    .detail("Version", savedVersion);
 			return;
 		}
-		// ASSERT will be fixed in PR#2642
-		// ASSERT_WE_THINK(backupEpoch == oldest);
+		ASSERT_WE_THINK(backupEpoch == oldestBackupEpoch);
 		const Tag popTag = logSystem.get()->getPseudoPopTag(tag, ProcessClass::BackupClass);
 		logSystem.get()->pop(savedVersion, popTag);
 	}
@@ -822,7 +825,6 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 				bool hasPseudoLocality = ls.isValid() && ls->hasPseudoLocality(tagLocalityBackup);
 				if (hasPseudoLocality) {
 					self.logSystem.set(ls);
-					self.pop();
 					self.oldestBackupEpoch = std::max(self.oldestBackupEpoch, ls->getOldestBackupEpoch());
 				}
 				TraceEvent("BackupWorkerLogSystem", self.myId)

From 0fe2810425454a07c7321568bdd18c1623a15380 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 20 Mar 2020 11:25:41 -0700
Subject: [PATCH 120/176] Fix repeated backup progress checking in backup
 worker

The delay is not used, which caused repeated progress checking in worker 0.
---
 fdbserver/BackupWorker.actor.cpp | 133 ++++++++++++++++---------------
 1 file changed, 69 insertions(+), 64 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index b48fb27f12..311dc2f607 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -331,6 +331,63 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 	}
 }
 
+// Set "allWorkerStarted" and "latestBackupWorkerSavedVersion" key for backups
+ACTOR Future<Void> setBackupKeys(BackupData* self, std::vector<UID> ready, std::map<UID, Version> savedLogVersions) {
+	loop {
+		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+
+			state std::vector<Future<Optional<Value>>> readyValues;
+			state std::vector<BackupConfig> configs;
+			for (UID uid : ready) {
+				configs.emplace_back(uid);
+				readyValues.push_back(tr->get(configs.back().allWorkerStarted().key));
+			}
+
+			state std::vector<Future<Optional<Version>>> prevVersions;
+			state std::vector<BackupConfig> versionConfigs;
+			for (const auto [uid, version] : savedLogVersions) {
+				versionConfigs.emplace_back(uid);
+				prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
+			}
+
+			wait(waitForAll(readyValues) && waitForAll(prevVersions));
+
+			for (int i = 0; i < readyValues.size(); i++) {
+				if (!readyValues[i].get().present()) {
+					configs[i].allWorkerStarted().set(tr, true);
+					TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", ready[i].toString());
+				}
+			}
+
+			for (int i = 0; i < prevVersions.size(); i++) {
+				const Version current = savedLogVersions[versionConfigs[i].getUid()];
+				if (prevVersions[i].get().present()) {
+					const Version prev = prevVersions[i].get().get();
+					if (prev > current) {
+						TraceEvent(SevWarn, "BackupWorkerVersionInverse", self->myId)
+						    .detail("Prev", prev)
+						    .detail("Current", current);
+					}
+				}
+				if (self->backupEpoch == self->oldestBackupEpoch &&
+				    (!prevVersions[i].get().present() || prevVersions[i].get().get() < current)) {
+					TraceEvent("BackupWorkerSetVersion", self->myId)
+					    .detail("BackupID", versionConfigs[i].getUid())
+					    .detail("Version", current);
+					versionConfigs[i].latestBackupWorkerSavedVersion().set(tr, current);
+				}
+			}
+			wait(tr->commit());
+			return Void();
+		} catch (Error& e) {
+			wait(tr->onError(e));
+		}
+	}
+}
+
 // Monitor all backup worker in the recruited epoch has been started. If so,
 // set the "allWorkerStarted" key of the BackupConfig to true, which in turn
 // unblocks StartFullBackupTaskFunc::_execute. Note only worker with Tag (-2,0)
@@ -339,26 +396,24 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 // the system space so that the client can know if a backup is restorable --
 // log saved version > snapshot version.
 ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
+	state Future<Void> interval;
+
 	loop {
+		interval = delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0);
 		while (self->backups.empty() || !self->logSystem.get()) {
-			wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger() ||
-			     self->logSystem.onChange());
+			wait(self->changedTrigger.onTrigger() || self->logSystem.onChange());
 		}
 
 		// check all workers have started by checking their progress is larger
 		// than the backup's start version.
-		state Reference<BackupProgress> progress(
-		    new BackupProgress(self->myId, self->logSystem.get()->getOldEpochTagsVersionsInfo()));
+		state Reference<BackupProgress> progress(new BackupProgress(self->myId, {}));
 		wait(getBackupProgress(self->cx, self->myId, progress));
-		std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
-		std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
-		    progress->getUnfinishedBackup();
-		bool finishedPreviousEpochs =
-		    toRecruit.empty() || std::get<0>(toRecruit.begin()->first) == self->recruitedEpoch;
-
+		state std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
+		state bool finishedPreviousEpochs = self->recruitedEpoch == self->oldestBackupEpoch;
 		state std::vector<UID> ready;
 		state std::map<UID, Version> savedLogVersions;
 		if (tagVersions.size() != self->logSystem.get()->getLogRouterTags()) {
+			wait(interval);
 			continue;
 		}
 
@@ -374,7 +429,7 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 				continue;
 			}
 			bool saved = true;
-			for (const std::pair<Tag, Version> tv : tagVersions) {
+			for (const std::pair<Tag, Version>& tv : tagVersions) {
 				if (tv.second < info.startVersion) {
 					saved = false;
 					break;
@@ -385,60 +440,10 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 				info.allWorkerStarted = true;
 			}
 		}
-		if (ready.empty() && savedLogVersions.empty()) continue;
+		Future<Void> setKeys =
+		    ready.empty() && savedLogVersions.empty() ? Void() : setBackupKeys(self, ready, savedLogVersions);
 
-		// Set "allWorkerStarted" and "latestBackupWorkerSavedVersion" key for backups
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
-			try {
-				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-
-				state std::vector<Future<Optional<Value>>> readyValues;
-				state std::vector<BackupConfig> configs;
-				for (UID uid : ready) {
-					configs.emplace_back(uid);
-					readyValues.push_back(tr->get(configs.back().allWorkerStarted().key));
-				}
-
-				state std::vector<Future<Optional<Version>>> prevVersions;
-				state std::vector<BackupConfig> versionConfigs;
-				for (const auto [uid, version] : savedLogVersions) {
-					versionConfigs.emplace_back(uid);
-					prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
-				}
-
-				wait(waitForAll(readyValues) && waitForAll(prevVersions));
-
-				for (int i = 0; i < readyValues.size(); i++) {
-					if (!readyValues[i].get().present()) {
-						configs[i].allWorkerStarted().set(tr, true);
-						TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", ready[i].toString());
-					}
-				}
-
-				for (int i = 0; i < prevVersions.size(); i++) {
-					const Version current = savedLogVersions[versionConfigs[i].getUid()];
-					if (prevVersions[i].get().present()) {
-						const Version prev = prevVersions[i].get().get();
-						TraceEvent(SevWarn, "BackupWorkerVersionInverse", self->myId)
-						    .detail("Prev", prev)
-						    .detail("Current", current);
-					}
-					if (self->backupEpoch == self->oldestBackupEpoch &&
-					    (!prevVersions[i].get().present() || prevVersions[i].get().get() < current)) {
-						TraceEvent("BackupWorkerSetVersion", self->myId)
-						    .detail("BackupID", versionConfigs[i].getUid())
-						    .detail("Version", current);
-						versionConfigs[i].latestBackupWorkerSavedVersion().set(tr, current);
-					}
-				}
-				wait(tr->commit());
-				break;
-			} catch (Error& e) {
-				wait(tr->onError(e));
-			}
-		}
+		wait(interval && setKeys);
 	}
 }
 

From 818072f3cbf646bbeab0e9ee07b1981845aff9c6 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 20 Mar 2020 13:58:20 -0700
Subject: [PATCH 121/176] Set oldest backup epoch if not recruiting backup
 workers

Since tlog is not kept until backup worker has pulled mutations from it, the
old tlogs can only be displaced after oldest backup epoch equals current epoch.
So if master is not recruiting backup workers, it should set the oldest backup
epoch as the current epoch.
---
 fdbserver/LogSystem.h                       | 1 +
 fdbserver/TagPartitionedLogSystem.actor.cpp | 9 +++++++--
 fdbserver/masterserver.actor.cpp            | 2 ++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h
index 7d8c79faba..f7b07e3666 100644
--- a/fdbserver/LogSystem.h
+++ b/fdbserver/LogSystem.h
@@ -785,6 +785,7 @@ struct ILogSystem {
 	virtual bool removeBackupWorker(const BackupWorkerDoneRequest& req) = 0;
 
 	virtual LogEpoch getOldestBackupEpoch() const = 0;
+	virtual void setOldestBackupEpoch(LogEpoch epoch) = 0;
 };
 
 struct LengthPrefixedStringRef {
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 9e94c2e5a1..c21010d910 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -360,7 +360,10 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 
 		newState.oldTLogData.clear();
-		if(!recoveryComplete.isValid() || !recoveryComplete.isReady() || (repopulateRegionAntiQuorum == 0 && (!remoteRecoveryComplete.isValid() || !remoteRecoveryComplete.isReady()))) {
+		if (!recoveryComplete.isValid() || !recoveryComplete.isReady() ||
+		    (repopulateRegionAntiQuorum == 0 &&
+		     (!remoteRecoveryComplete.isValid() || !remoteRecoveryComplete.isReady())) ||
+		    epoch != oldestBackupEpoch) {
 			for (const auto& oldData : oldLogData) {
 				newState.oldTLogData.emplace_back(oldData);
 				TraceEvent("BWToCore")
@@ -1430,7 +1433,9 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return removed;
 	}
 
-	LogEpoch getOldestBackupEpoch() const override { return oldestBackupEpoch; }
+	LogEpoch getOldestBackupEpoch() const final { return oldestBackupEpoch; }
+
+	void setOldestBackupEpoch(LogEpoch epoch) final { oldestBackupEpoch = epoch; }
 
 	ACTOR static Future<Void> monitorLog(Reference<AsyncVar<OptionalInterface<TLogInterface>>> logServer, Reference<AsyncVar<bool>> failed) {
 		state Future<Void> waitFailure;
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 8f73d5ad6b..63cc24b898 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1561,6 +1561,8 @@ ACTOR Future<Void> masterCore( Reference<MasterData> self ) {
 	self->addActor.send(configurationMonitor(self, cx));
 	if (self->configuration.backupWorkerEnabled) {
 		self->addActor.send(recruitBackupWorkers(self, cx));
+	} else {
+		self->logSystem->setOldestBackupEpoch(self->cstate.myDBState.recoveryCount);
 	}
 
 	wait( Future<Void>(Never()) );

From 0eacf1cdab678f449c3886e362fa15a193fab841 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 20 Mar 2020 20:09:32 -0700
Subject: [PATCH 122/176] trackTlogRecovery listens on backup worker change
 events

Old TLogs can only be removed when backup workers no long need them (i.e., the
oldest backup epoch == current epoch). As a result, the core state changes need
include backup worker changes, which updates the oldest backup epoch.
---
 fdbserver/TagPartitionedLogSystem.actor.cpp | 86 +++++++++++----------
 1 file changed, 45 insertions(+), 41 deletions(-)

diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index c21010d910..d0fb0eb532 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -206,17 +206,17 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 	    locality(locality), remoteLogsWrittenToCoreState(false), hasRemoteServers(false), stopped(false),
 	    addActor(addActor), popActors(false) {}
 
-	void stopRejoins() override { rejoins = Future<Void>(); }
+	void stopRejoins() final { rejoins = Future<Void>(); }
 
-	void addref() override {
+	void addref() final {
 		ReferenceCounted<TagPartitionedLogSystem>::addref();
 	}
 
-	void delref() override {
+	void delref() final {
 		ReferenceCounted<TagPartitionedLogSystem>::delref();
 	}
 
-	std::string describe() override {
+	std::string describe() final {
 		std::string result;
 		for( int i = 0; i < tLogs.size(); i++ ) {
 			result += format("%d: ", i);
@@ -227,7 +227,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return result;
 	}
 
-	UID getDebugID() override { return dbgid; }
+	UID getDebugID() final { return dbgid; }
 
 	void addPseudoLocality(int8_t locality) {
 		ASSERT(locality < 0);
@@ -237,7 +237,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	Tag getPseudoPopTag(Tag tag, ProcessClass::ClassType type) override {
+	Tag getPseudoPopTag(Tag tag, ProcessClass::ClassType type) final {
 		switch (type) {
 		case ProcessClass::LogRouterClass:
 			if (tag.locality == tagLocalityLogRouter) {
@@ -259,10 +259,10 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return tag;
 	}
 
-	bool hasPseudoLocality(int8_t locality) override { return pseudoLocalities.count(locality) > 0; }
+	bool hasPseudoLocality(int8_t locality) final { return pseudoLocalities.count(locality) > 0; }
 
 	// Return the min version of all pseudoLocalities, i.e., logRouter and backupTag
-	Version popPseudoLocalityTag(Tag tag, Version upTo) override {
+	Version popPseudoLocalityTag(Tag tag, Version upTo) final {
 		ASSERT(isPseudoLocality(tag.locality) && hasPseudoLocality(tag.locality));
 
 		Version& localityVersion = pseudoLocalityPopVersion[tag];
@@ -338,7 +338,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return logSystem;
 	}
 
-	void toCoreState(DBCoreState& newState) override {
+	void toCoreState(DBCoreState& newState) final {
 		if( recoveryComplete.isValid() && recoveryComplete.isError() )
 			throw recoveryComplete.getError();
 
@@ -376,11 +376,11 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		newState.logSystemType = logSystemType;
 	}
 
-	bool remoteStorageRecovered() override {
+	bool remoteStorageRecovered() final {
 		return remoteRecoveryComplete.isValid() && remoteRecoveryComplete.isReady();
 	}
 
-	Future<Void> onCoreStateChanged() override {
+	Future<Void> onCoreStateChanged() final {
 		std::vector<Future<Void>> changes;
 		changes.push_back(Never());
 		if(recoveryComplete.isValid() && !recoveryComplete.isReady()) {
@@ -392,10 +392,11 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		if(remoteRecoveryComplete.isValid() && !remoteRecoveryComplete.isReady()) {
 			changes.push_back(remoteRecoveryComplete);
 		}
+		changes.push_back(backupWorkerChanged.onTrigger()); // changes to oldestBackupEpoch
 		return waitForAny(changes);
 	}
 
-	void coreStateWritten(DBCoreState const& newState) override {
+	void coreStateWritten(DBCoreState const& newState) final {
 		if (!newState.oldTLogData.size()) {
 			recoveryCompleteWrittenToCoreState.set(true);
 		}
@@ -408,7 +409,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	Future<Void> onError() override {
+	Future<Void> onError() final {
 		return onError_internal(this);
 	}
 
@@ -483,7 +484,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 	}
 
 	Future<Version> push(Version prevVersion, Version version, Version knownCommittedVersion,
-	                     Version minKnownCommittedVersion, LogPushData& data, Optional<UID> debugID) override {
+	                     Version minKnownCommittedVersion, LogPushData& data, Optional<UID> debugID) final {
 		// FIXME: Randomize request order as in LegacyLogSystem?
 		vector<Future<Void>> quorumResults;
 		vector<Future<TLogCommitReply>> allReplies;
@@ -655,7 +656,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	Reference<IPeekCursor> peek( UID dbgid, Version begin, Optional<Version> end, Tag tag, bool parallelGetMore ) override {
+	Reference<IPeekCursor> peek( UID dbgid, Version begin, Optional<Version> end, Tag tag, bool parallelGetMore ) final {
 		if(!tLogs.size()) {
 			TraceEvent("TLogPeekNoLogSets", dbgid).detail("Tag", tag.toString()).detail("Begin", begin);
 			return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) );
@@ -668,7 +669,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	Reference<IPeekCursor> peek(UID dbgid, Version begin, Optional<Version> end, std::vector<Tag> tags, bool parallelGetMore) override {
+	Reference<IPeekCursor> peek(UID dbgid, Version begin, Optional<Version> end, std::vector<Tag> tags, bool parallelGetMore) final {
 		if(tags.empty()) {
 			TraceEvent("TLogPeekNoTags", dbgid).detail("Begin", begin);
 			return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), invalidTag, begin, getPeekEnd(), false, false ) );
@@ -793,7 +794,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	Reference<IPeekCursor> peekTxs(UID dbgid, Version begin, int8_t peekLocality, Version localEnd, bool canDiscardPopped) override {
+	Reference<IPeekCursor> peekTxs(UID dbgid, Version begin, int8_t peekLocality, Version localEnd, bool canDiscardPopped) final {
 		Version end = getEnd();
 		if(!tLogs.size()) {
 			TraceEvent("TLogPeekTxsNoLogs", dbgid);
@@ -875,7 +876,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	Reference<IPeekCursor> peekSingle(UID dbgid, Version begin, Tag tag, std::vector<std::pair<Version,Tag>> history) override {
+	Reference<IPeekCursor> peekSingle(UID dbgid, Version begin, Tag tag, std::vector<std::pair<Version,Tag>> history) final {
 		while(history.size() && begin >= history.back().first) {
 			history.pop_back();
 		}
@@ -900,7 +901,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	Reference<IPeekCursor> peekLogRouter(UID dbgid, Version begin, Tag tag) override {
+	Reference<IPeekCursor> peekLogRouter(UID dbgid, Version begin, Tag tag) final {
 		bool found = false;
 		for (const auto& log : tLogs) {
 			found = log->hasLogRouter(dbgid) || log->hasBackupWorker(dbgid);
@@ -999,7 +1000,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) );
 	}
 
-	Version getKnownCommittedVersion() override {
+	Version getKnownCommittedVersion() final {
 		Version result = invalidVersion;
 		for(auto& it : lockResults) {
 			auto versions = TagPartitionedLogSystem::getDurableVersion(dbgid, it);
@@ -1010,7 +1011,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return result;
 	}
 
-	Future<Void> onKnownCommittedVersionChange() override {
+	Future<Void> onKnownCommittedVersionChange() final {
 		std::vector<Future<Void>> result;
 		for(auto& it : lockResults) {
 			result.push_back(TagPartitionedLogSystem::getDurableVersionChanged(it));
@@ -1051,7 +1052,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	void popTxs(Version upTo, int8_t popLocality) override {
+	void popTxs(Version upTo, int8_t popLocality) final {
 		if( getTLogVersion() < TLogVersion::V4 ) {
 			pop(upTo, txsTag, 0, popLocality);
 		} else {
@@ -1061,7 +1062,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	void pop(Version upTo, Tag tag, Version durableKnownCommittedVersion, int8_t popLocality) override {
+	void pop(Version upTo, Tag tag, Version durableKnownCommittedVersion, int8_t popLocality) final {
 		if (upTo <= 0) return;
 		if( tag.locality == tagLocalityRemoteLog) {
 			popLogRouter(upTo, tag, durableKnownCommittedVersion, popLocality);
@@ -1161,7 +1162,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return maxPopped;
 	}
 
-	Future<Version> getTxsPoppedVersion() override {
+	Future<Version> getTxsPoppedVersion() final {
 		return getPoppedTxs(this);
 	}
 
@@ -1213,7 +1214,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 	}
 
 	// Returns success after confirming that pushes in the current epoch are still possible
-	Future<Void> confirmEpochLive(Optional<UID> debugID) override {
+	Future<Void> confirmEpochLive(Optional<UID> debugID) final {
 		vector<Future<Void>> quorumResults;
 		for(auto& it : tLogs) {
 			if(it->isLocal && it->logServers.size()) {
@@ -1224,7 +1225,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return waitForAll(quorumResults);
 	}
 
-	Future<Void> endEpoch() override {
+	Future<Void> endEpoch() final {
 		std::vector<Future<Void>> lockResults;
 		for( auto& logSet : tLogs ) {
 			for( auto& log : logSet->logServers ) {
@@ -1241,11 +1242,11 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 	                                       DatabaseConfiguration const& config, LogEpoch recoveryCount,
 	                                       int8_t primaryLocality, int8_t remoteLocality,
 	                                       std::vector<Tag> const& allTags,
-	                                       Reference<AsyncVar<bool>> const& recruitmentStalled) override {
+	                                       Reference<AsyncVar<bool>> const& recruitmentStalled) final {
 		return newEpoch( Reference<TagPartitionedLogSystem>::addRef(this), recr, fRemoteWorkers, config, recoveryCount, primaryLocality, remoteLocality, allTags, recruitmentStalled );
 	}
 
-	LogSystemConfig getLogSystemConfig() override {
+	LogSystemConfig getLogSystemConfig() final {
 		LogSystemConfig logSystemConfig(epoch);
 		logSystemConfig.logSystemType = logSystemType;
 		logSystemConfig.expectedLogSets = expectedLogSets;
@@ -1270,7 +1271,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return logSystemConfig;
 	}
 
-	Standalone<StringRef> getLogsValue() override {
+	Standalone<StringRef> getLogsValue() final {
 		vector<std::pair<UID, NetworkAddress>> logs;
 		vector<std::pair<UID, NetworkAddress>> oldLogs;
 		for(auto& t : tLogs) {
@@ -1292,7 +1293,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return logsValue( logs, oldLogs );
 	}
 
-	Future<Void> onLogSystemConfigChange() override {
+	Future<Void> onLogSystemConfigChange() final {
 		std::vector<Future<Void>> changes;
 		changes.push_back(logSystemConfigChanged.onTrigger());
 		for(auto& t : tLogs) {
@@ -1315,7 +1316,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return waitForAny(changes);
 	}
 
-	Version getEnd() override {
+	Version getEnd() final {
 		ASSERT( recoverAt.present() );
 		return recoverAt.get() + 1;
 	}
@@ -1327,7 +1328,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 			return std::numeric_limits<Version>::max();
 	}
 
-	void getPushLocations(VectorRef<Tag> tags, std::vector<int>& locations, bool allLocations) override {
+	void getPushLocations(VectorRef<Tag> tags, std::vector<int>& locations, bool allLocations) final {
 		int locationOffset = 0;
 		for(auto& log : tLogs) {
 			if(log->isLocal && log->logServers.size()) {
@@ -1337,30 +1338,30 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	bool hasRemoteLogs() const override {
+	bool hasRemoteLogs() const final {
 		return logRouterTags > 0 || pseudoLocalities.size() > 0;
 	}
 
-	Tag getRandomRouterTag() const override {
+	Tag getRandomRouterTag() const final {
 		return Tag(tagLocalityLogRouter, deterministicRandom()->randomInt(0, logRouterTags));
 	}
 
-	Tag getRandomTxsTag() const override {
+	Tag getRandomTxsTag() const final {
 		return Tag(tagLocalityTxs, deterministicRandom()->randomInt(0, txsTags));
 	}
 
-	TLogVersion getTLogVersion() const override {
+	TLogVersion getTLogVersion() const final {
 		return tLogs[0]->tLogVersion;
 	}
 
-	int getLogRouterTags() const override { return logRouterTags; }
+	int getLogRouterTags() const final { return logRouterTags; }
 
 	Version getBackupStartVersion() const final {
 		ASSERT(tLogs.size() > 0);
 		return backupStartVersion;
 	}
 
-	std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo> getOldEpochTagsVersionsInfo() const override {
+	std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo> getOldEpochTagsVersionsInfo() const final {
 		std::map<LogEpoch, EpochTagsVersionsInfo> epochInfos;
 		for (const auto& old : oldLogData) {
 			epochInfos.insert(
@@ -1381,7 +1382,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		return Reference<LogSet>(nullptr);
 	}
 
-	void setBackupWorkers(const std::vector<InitializeBackupReply>& replies) override {
+	void setBackupWorkers(const std::vector<InitializeBackupReply>& replies) final {
 		ASSERT(tLogs.size() > 0);
 
 		Reference<LogSet> logset = tLogs[0];  // Master recruits this epoch's worker first.
@@ -1402,7 +1403,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		backupWorkerChanged.trigger();
 	}
 
-	bool removeBackupWorker(const BackupWorkerDoneRequest& req) override {
+	bool removeBackupWorker(const BackupWorkerDoneRequest& req) final {
 		bool removed = false;
 		Reference<LogSet> logset = getEpochLogSet(req.backupEpoch);
 		if (logset.isValid()) {
@@ -1435,7 +1436,10 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 
 	LogEpoch getOldestBackupEpoch() const final { return oldestBackupEpoch; }
 
-	void setOldestBackupEpoch(LogEpoch epoch) final { oldestBackupEpoch = epoch; }
+	void setOldestBackupEpoch(LogEpoch epoch) final {
+		oldestBackupEpoch = epoch;
+		backupWorkerChanged.trigger();
+	}
 
 	ACTOR static Future<Void> monitorLog(Reference<AsyncVar<OptionalInterface<TLogInterface>>> logServer, Reference<AsyncVar<bool>> failed) {
 		state Future<Void> waitFailure;

From 97702d91c8ff0d834e6703e416faf00b51544d68 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 21 Mar 2020 13:44:02 -0700
Subject: [PATCH 123/176] Skip recruiting backup workers for older epochs
 before min backup version

When master starts recruiting backup workers, if there is no active backup job
or the min version of the backup job is greater than old epoch's end version,
then these old epochs can be skipped.
---
 fdbserver/RestoreMaster.actor.cpp |  8 ++++--
 fdbserver/masterserver.actor.cpp  | 48 ++++++++++++++++++++++++++++---
 2 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 95f6f0a95e..f42980c73e 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -286,8 +286,8 @@ ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData>
 		       std::tie(f2.endVersion, f2.beginVersion, f2.fileIndex, f2.fileName);
 	});
 
-	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches,
-	                          targetVersion); // Divide files into version batches
+	// Divide files into version batches.
+	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches, targetVersion);
 	self->dumpVersionBatches(self->versionBatches);
 
 	state std::vector<Future<Void>> fBatches;
@@ -683,12 +683,14 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
 
 	// Convert version to real time for operators to read the BackupDescription desc.
 	wait(desc.resolveVersionTimes(cx));
-	TraceEvent("FastRestoreMasterPhaseCollectBackupFilesStart").detail("BackupDesc", desc.toString());
 
 	if (request.targetVersion == invalidVersion && desc.maxRestorableVersion.present()) {
 		request.targetVersion = desc.maxRestorableVersion.get();
 	}
 
+	TraceEvent("FastRestoreMasterPhaseCollectBackupFilesStart")
+	    .detail("TargetVersion", request.targetVersion)
+	    .detail("BackupDesc", desc.toString());
 	if (g_network->isSimulated()) {
 		std::cout << "Restore to version: " << request.targetVersion << "\nBackupDesc: \n" << desc.toString() << "\n\n";
 	}
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 63cc24b898..6d848b6a4d 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1245,6 +1245,35 @@ ACTOR Future<Void> configurationMonitor(Reference<MasterData> self, Database cx)
 	}
 }
 
+ACTOR static Future<Optional<Version>> getMinBackupVersion(Reference<MasterData> self, Database cx) {
+	loop {
+		state ReadYourWritesTransaction tr(cx);
+
+		try {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<Value> value = wait(tr.get(backupStartedKey));
+			Optional<Version> minVersion;
+			if (value.present()) {
+				auto uidVersions = decodeBackupStartedValue(value.get());
+				TraceEvent e("GotBackupStartKey", self->dbgid);
+				int i = 1;
+				for (auto [uid, version] : uidVersions) {
+					e.detail(format("BackupID%d", i), uid).detail(format("Version%d", i), version);
+					i++;
+					minVersion = minVersion.present() ? std::min(version, minVersion.get()) : version;
+				}
+			} else {
+				TraceEvent("EmptyBackupStartKey", self->dbgid);
+			}
+			return minVersion;
+
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
 ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Database cx) {
 	ASSERT(self->backupWorkers.size() > 0);
 
@@ -1274,7 +1303,7 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 		req.totalTags = logRouterTags;
 		req.startVersion = startVersion;
 		TraceEvent("BackupRecruitment", self->dbgid)
-		    .detail("BKID", req.reqId)
+		    .detail("RequestID", req.reqId)
 		    .detail("Tag", req.routerTag.toString())
 		    .detail("Epoch", epoch)
 		    .detail("BackupEpoch", epoch)
@@ -1285,10 +1314,21 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 		                    master_backup_worker_failed()));
 	}
 
-	wait(gotProgress);
+	state Future<Optional<Version>> fMinVersion = getMinBackupVersion(self, cx);
+	wait(gotProgress && success(fMinVersion));
+
 	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
 	    backupProgress->getUnfinishedBackup();
 	for (const auto& [epochVersionCount, tagVersions] : toRecruit) {
+		const Version oldEpochEnd = std::get<1>(epochVersionCount);
+		if (!fMinVersion.get().present() || fMinVersion.get().get() >= oldEpochEnd) {
+			TraceEvent("SkipBackupRecruitment", self->dbgid)
+			    .detail("MinVersion", fMinVersion.get().get())
+			    .detail("Epoch", epoch)
+			    .detail("OldEpoch", std::get<0>(epochVersionCount))
+			    .detail("OldEpochEnd", oldEpochEnd);
+			continue;
+		}
 		for (const auto& [tag, version] : tagVersions) {
 			const auto& worker = self->backupWorkers[i % self->backupWorkers.size()];
 			i++;
@@ -1298,9 +1338,9 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 			req.routerTag = tag;
 			req.totalTags = std::get<2>(epochVersionCount);
 			req.startVersion = version; // savedVersion + 1
-			req.endVersion = std::get<1>(epochVersionCount) - 1;
+			req.endVersion = oldEpochEnd - 1;
 			TraceEvent("BackupRecruitment", self->dbgid)
-			    .detail("BKID", req.reqId)
+			    .detail("RequestID", req.reqId)
 			    .detail("Tag", req.routerTag.toString())
 			    .detail("Epoch", epoch)
 			    .detail("BackupEpoch", req.backupEpoch)

From 44c19969508f5d82c299547149876a4db2f49073 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 22 Mar 2020 16:31:39 -0700
Subject: [PATCH 124/176] Change all worker started to be set after all workers
 updated a key

Previously, all worker started is set to be when saved log versions are higher.
However, saving the versions can be wrong, as the worker is not guaranteed to
write to the right container. For instance, if the watch is triggered later,
then mutation logs are written to previous containers. So we need to ensure the
right container is ready -- all workers have acknowledged seeing the container.
---
 fdbclient/BackupAgent.actor.h    |   5 ++
 fdbserver/BackupWorker.actor.cpp | 147 ++++++++++++++++++++-----------
 2 files changed, 103 insertions(+), 49 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 7c84fa0121..779bb09bb0 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -794,6 +794,11 @@ public:
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
 	}
 
+	// Each backup worker adds its (epoch, tag.id) to this property.
+	KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() {
+		return configSpace.pack(LiteralStringRef(__FUNCTION__));
+	}
+
 	// Set to true if backup worker is enabled.
 	KeyBackedProperty<bool> backupWorkerEnabled() {
 		return configSpace.pack(LiteralStringRef(__FUNCTION__));
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 311dc2f607..7c890ea41f 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -85,7 +85,14 @@ struct BackupData {
 
 	struct PerBackupInfo {
 		PerBackupInfo() = default;
-		PerBackupInfo(BackupData* data, Version v) : self(data), startVersion(v) {}
+		PerBackupInfo(BackupData* data, UID uid, Version v) : self(data), startVersion(v) {
+			// Open the container and get key ranges
+			BackupConfig config(uid);
+			container = config.backupContainer().get(data->cx);
+			ranges = config.backupRanges().get(data->cx);
+			updateWorker = _updateStartedWorkers(this, data, uid);
+			TraceEvent("BackupWorkerAddJob", data->myId).detail("BackupID", uid).detail("Version", v);
+		}
 
 		bool isReady() const {
 			return stopped || (container.isReady() && ranges.isReady());
@@ -101,12 +108,78 @@ struct BackupData {
 			return Void();
 		}
 
+		// Update the number of backup workers in the BackupConfig. Each worker
+		// writes (epoch, tag.id) into the key. Worker 0 monitors the key and once
+		// all workers have updated the key, this backup is considered as started
+		// (i.e., the "submitBackup" call is successful). Worker 0 then sets
+		// the "allWorkerStarted" flag.
+		ACTOR static Future<Void> _updateStartedWorkers(PerBackupInfo* info, BackupData* self, UID uid) {
+			state BackupConfig config(uid);
+			state Future<Void> watchFuture;
+			state bool updated = false; // worker 0 has updated
+			state bool firstWorker = info->self->tag.id == 0;
+			state bool allUpdated = false;
+			state Optional<std::vector<std::pair<int64_t, int64_t>>> workers;
+
+			loop {
+				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
+				try {
+					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+
+					Optional<std::vector<std::pair<int64_t, int64_t>>> tmp =
+					    wait(config.startedBackupWorkers().get(tr));
+					workers = tmp;
+					if (!updated) {
+						if (workers.present()) {
+							workers.get().emplace_back(self->recruitedEpoch, (int64_t)self->tag.id);
+						} else {
+							std::vector<std::pair<int64_t, int64_t>> v(1, { self->recruitedEpoch, self->tag.id });
+							workers = Optional<std::vector<std::pair<int64_t, int64_t>>>(v);
+						}
+					}
+					if (firstWorker) {
+						std::vector<std::pair<int64_t, int64_t>>& v = workers.get();
+						v.erase(std::remove_if(v.begin(), v.end(),
+						                       [epoch = self->recruitedEpoch](const std::pair<LogEpoch, int32_t>& p) {
+							                       return p.first != epoch;
+						                       }),
+						        v.end());
+						if (self->totalTags == v.size()) {
+							config.allWorkerStarted().set(tr, true);
+							allUpdated = true;
+						} else {
+							// monitor all workers' updates
+							watchFuture = tr->watch(config.startedBackupWorkers().key);
+						}
+						config.startedBackupWorkers().set(tr, workers.get());
+						wait(tr->commit());
+
+						updated = true; // Only set to true after commit.
+						if (allUpdated) {
+							break;
+						}
+						wait(watchFuture);
+					} else {
+						config.startedBackupWorkers().set(tr, workers.get());
+						wait(tr->commit());
+						break;
+					}
+				} catch (Error& e) {
+					wait(tr->onError(e));
+					allUpdated = false;
+				}
+			}
+			TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", uid.toString());
+			return Void();
+		}
+
 		BackupData* self = nullptr;
 		Version startVersion = invalidVersion;
 		Version lastSavedVersion = invalidVersion;
 		Future<Optional<Reference<IBackupContainer>>> container;
 		Future<Optional<std::vector<KeyRange>>> ranges; // Key ranges of this backup
-		bool allWorkerStarted = false; // Only worker with Tag(-2,0) uses & sets this field
+		Future<Void> updateWorker;
 		bool stopped = false; // Is the backup stopped?
 	};
 
@@ -210,18 +283,11 @@ struct BackupData {
 		}
 
 		bool modified = false;
-		for (const auto uidVersion : uidVersions) {
-			const UID uid = uidVersion.first;
-
+		for (const auto [uid, version] : uidVersions) {
 			auto it = backups.find(uid);
 			if (it == backups.end()) {
 				modified = true;
-				auto inserted = backups.emplace(uid, BackupData::PerBackupInfo(this, uidVersion.second));
-
-				// Open the container and get key ranges
-				BackupConfig config(uid);
-				inserted.first->second.container = config.backupContainer().get(cx);
-				inserted.first->second.ranges = config.backupRanges().get(cx);
+				backups.emplace(uid, BackupData::PerBackupInfo(this, uid, version));
 			} else {
 				stopList.erase(uid);
 			}
@@ -331,38 +397,28 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 	}
 }
 
-// Set "allWorkerStarted" and "latestBackupWorkerSavedVersion" key for backups
-ACTOR Future<Void> setBackupKeys(BackupData* self, std::vector<UID> ready, std::map<UID, Version> savedLogVersions) {
+// Set "latestBackupWorkerSavedVersion" key for backups
+ACTOR Future<Void> setBackupKeys(BackupData* self, std::map<UID, Version> savedLogVersions) {
 	loop {
 		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			state std::vector<Future<Optional<Value>>> readyValues;
-			state std::vector<BackupConfig> configs;
-			for (UID uid : ready) {
-				configs.emplace_back(uid);
-				readyValues.push_back(tr->get(configs.back().allWorkerStarted().key));
-			}
-
 			state std::vector<Future<Optional<Version>>> prevVersions;
 			state std::vector<BackupConfig> versionConfigs;
+			state std::vector<Future<Optional<bool>>> allWorkersReady;
 			for (const auto [uid, version] : savedLogVersions) {
 				versionConfigs.emplace_back(uid);
 				prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
+				allWorkersReady.push_back(versionConfigs.back().allWorkerStarted().get(tr));
 			}
 
-			wait(waitForAll(readyValues) && waitForAll(prevVersions));
-
-			for (int i = 0; i < readyValues.size(); i++) {
-				if (!readyValues[i].get().present()) {
-					configs[i].allWorkerStarted().set(tr, true);
-					TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", ready[i].toString());
-				}
-			}
+			wait(waitForAll(prevVersions) && waitForAll(allWorkersReady));
 
 			for (int i = 0; i < prevVersions.size(); i++) {
+				if (!allWorkersReady[i].get().present() || !allWorkersReady[i].get().get()) continue;
+
 				const Version current = savedLogVersions[versionConfigs[i].getUid()];
 				if (prevVersions[i].get().present()) {
 					const Version prev = prevVersions[i].get().get();
@@ -410,38 +466,29 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 		wait(getBackupProgress(self->cx, self->myId, progress));
 		state std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
 		state bool finishedPreviousEpochs = self->recruitedEpoch == self->oldestBackupEpoch;
-		state std::vector<UID> ready;
 		state std::map<UID, Version> savedLogVersions;
-		if (tagVersions.size() != self->logSystem.get()->getLogRouterTags()) {
+		if (tagVersions.size() != self->totalTags) {
 			wait(interval);
 			continue;
 		}
 
 		// Check every version is larger than backup's startVersion
 		for (auto& [uid, info] : self->backups) {
-			if (info.allWorkerStarted && finishedPreviousEpochs) {
+			TraceEvent("BackupWorkerSavedBackupVersion", self->myId)
+			    .detail("BackupID", uid.toString())
+			    .detail("Done", finishedPreviousEpochs);
+			if (finishedPreviousEpochs) {
 				// update update progress so far
 				Version v = std::numeric_limits<Version>::max();
 				for (const auto [tag, version] : tagVersions) {
 					v = std::min(v, version);
 				}
 				savedLogVersions.emplace(uid, v);
-				continue;
-			}
-			bool saved = true;
-			for (const std::pair<Tag, Version>& tv : tagVersions) {
-				if (tv.second < info.startVersion) {
-					saved = false;
-					break;
-				}
-			}
-			if (saved) {
-				ready.push_back(uid);
-				info.allWorkerStarted = true;
+				TraceEvent("BackupWorkerSavedBackupVersion", self->myId).detail("BackupID", uid).detail("Version", v);
 			}
 		}
-		Future<Void> setKeys =
-		    ready.empty() && savedLogVersions.empty() ? Void() : setBackupKeys(self, ready, savedLogVersions);
+		TraceEvent("BackupWorkerSavedBackupVersion", self->myId).detail("Size", savedLogVersions.size());
+		Future<Void> setKeys = savedLogVersions.empty() ? Void() : setBackupKeys(self, savedLogVersions);
 
 		wait(interval && setKeys);
 	}
@@ -523,7 +570,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 	state std::vector<Future<Reference<IBackupFile>>> logFileFutures;
 	state std::vector<Reference<IBackupFile>> logFiles;
 	state std::vector<int64_t> blockEnds;
-	state std::set<UID> activeUids; // active Backups' UIDs
+	state std::vector<UID> activeUids; // active Backups' UIDs
 	state KeyRangeMap<std::set<int>> keyRangeMap; // range to index in logFileFutures, logFiles, & blockEnds
 	state std::vector<Standalone<StringRef>> mutations;
 	state int idx;
@@ -540,7 +587,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 			continue;
 		}
 		const int index = logFileFutures.size();
-		activeUids.insert(it->first);
+		activeUids.push_back(it->first);
 		self->insertRanges(keyRangeMap, it->second.ranges.get(), index);
 		if (it->second.lastSavedVersion == invalidVersion) {
 			it->second.lastSavedVersion = self->savedVersion;
@@ -556,10 +603,12 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 	std::transform(logFileFutures.begin(), logFileFutures.end(), std::back_inserter(logFiles),
 	               [](const Future<Reference<IBackupFile>>& f) { return f.get(); });
 
-	for (const auto& file : logFiles) {
+	ASSERT(activeUids.size() == logFiles.size());
+	for (int i = 0; i < logFiles.size(); i++) {
 		TraceEvent("OpenMutationFile", self->myId)
+		    .detail("BackupID", activeUids[i])
 		    .detail("TagId", self->tag.id)
-		    .detail("File", file->getFileName());
+		    .detail("File", logFiles[i]->getFileName());
 	}
 
 	blockEnds = std::vector<int64_t>(logFiles.size(), 0);

From 33ea027f842b73c5ce93397aff7b641ae15f487e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 22 Mar 2020 18:19:26 -0700
Subject: [PATCH 125/176] Make sure only current epoch's backup workers update
 all workers

So that backup workers from old epochs don't mess with the list of all workers.
---
 fdbserver/BackupWorker.actor.cpp | 37 ++++++++++++++++----------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 7c890ea41f..8bb52a388e 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -90,7 +90,10 @@ struct BackupData {
 			BackupConfig config(uid);
 			container = config.backupContainer().get(data->cx);
 			ranges = config.backupRanges().get(data->cx);
-			updateWorker = _updateStartedWorkers(this, data, uid);
+			if (self->backupEpoch == self->recruitedEpoch) {
+				// Only current epoch's worker update the number of backup workers.
+				updateWorker = _updateStartedWorkers(this, data, uid);
+			}
 			TraceEvent("BackupWorkerAddJob", data->myId).detail("BackupID", uid).detail("Version", v);
 		}
 
@@ -112,11 +115,12 @@ struct BackupData {
 		// writes (epoch, tag.id) into the key. Worker 0 monitors the key and once
 		// all workers have updated the key, this backup is considered as started
 		// (i.e., the "submitBackup" call is successful). Worker 0 then sets
-		// the "allWorkerStarted" flag.
+		// the "allWorkerStarted" flag, which in turn unblocks
+		// StartFullBackupTaskFunc::_execute.
 		ACTOR static Future<Void> _updateStartedWorkers(PerBackupInfo* info, BackupData* self, UID uid) {
 			state BackupConfig config(uid);
 			state Future<Void> watchFuture;
-			state bool updated = false; // worker 0 has updated
+			state bool updated = false;
 			state bool firstWorker = info->self->tag.id == 0;
 			state bool allUpdated = false;
 			state Optional<std::vector<std::pair<int64_t, int64_t>>> workers;
@@ -141,7 +145,7 @@ struct BackupData {
 					if (firstWorker) {
 						std::vector<std::pair<int64_t, int64_t>>& v = workers.get();
 						v.erase(std::remove_if(v.begin(), v.end(),
-						                       [epoch = self->recruitedEpoch](const std::pair<LogEpoch, int32_t>& p) {
+						                       [epoch = self->recruitedEpoch](const std::pair<int64_t, int64_t>& p) {
 							                       return p.first != epoch;
 						                       }),
 						        v.end());
@@ -152,7 +156,10 @@ struct BackupData {
 							// monitor all workers' updates
 							watchFuture = tr->watch(config.startedBackupWorkers().key);
 						}
-						config.startedBackupWorkers().set(tr, workers.get());
+						ASSERT(workers.present() && workers.get().size() > 0);
+						if (!updated) {
+							config.startedBackupWorkers().set(tr, workers.get());
+						}
 						wait(tr->commit());
 
 						updated = true; // Only set to true after commit.
@@ -161,6 +168,7 @@ struct BackupData {
 						}
 						wait(watchFuture);
 					} else {
+						ASSERT(workers.present() && workers.get().size() > 0);
 						config.startedBackupWorkers().set(tr, workers.get());
 						wait(tr->commit());
 						break;
@@ -170,7 +178,7 @@ struct BackupData {
 					allUpdated = false;
 				}
 			}
-			TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", uid.toString());
+			TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", uid).detail("TagId", self->tag.id);
 			return Void();
 		}
 
@@ -444,14 +452,11 @@ ACTOR Future<Void> setBackupKeys(BackupData* self, std::map<UID, Version> savedL
 	}
 }
 
-// Monitor all backup worker in the recruited epoch has been started. If so,
-// set the "allWorkerStarted" key of the BackupConfig to true, which in turn
-// unblocks StartFullBackupTaskFunc::_execute. Note only worker with Tag (-2,0)
-// runs this actor so that the key is set by one process.
-// Additionally, this actor updates the saved version for each BackupConfig in
-// the system space so that the client can know if a backup is restorable --
+// Note only worker with Tag (-2,0) runs this actor so that the latest saved
+// version key is set by one process, which is stored in each BackupConfig in
+// the system space. The client can know if a backup is restorable by checking
 // log saved version > snapshot version.
-ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
+ACTOR Future<Void> monitorBackupProgress(BackupData* self) {
 	state Future<Void> interval;
 
 	loop {
@@ -474,9 +479,6 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 
 		// Check every version is larger than backup's startVersion
 		for (auto& [uid, info] : self->backups) {
-			TraceEvent("BackupWorkerSavedBackupVersion", self->myId)
-			    .detail("BackupID", uid.toString())
-			    .detail("Done", finishedPreviousEpochs);
 			if (finishedPreviousEpochs) {
 				// update update progress so far
 				Version v = std::numeric_limits<Version>::max();
@@ -487,7 +489,6 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
 				TraceEvent("BackupWorkerSavedBackupVersion", self->myId).detail("BackupID", uid).detail("Version", v);
 			}
 		}
-		TraceEvent("BackupWorkerSavedBackupVersion", self->myId).detail("Size", savedLogVersions.size());
 		Future<Void> setKeys = savedLogVersions.empty() ? Void() : setBackupKeys(self, savedLogVersions);
 
 		wait(interval && setKeys);
@@ -860,7 +861,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		addActor.send(checkRemoved(db, req.recruitedEpoch, &self));
 		addActor.send(waitFailureServer(interf.waitFailure.getFuture()));
 		if (req.recruitedEpoch == req.backupEpoch && req.routerTag.id == 0) {
-			addActor.send(monitorAllWorkerProgress(&self));
+			addActor.send(monitorBackupProgress(&self));
 		}
 
 		// Check if backup key is present to avoid race between this check and

From 1552653f1cf869f6db0c5695499c916a9321fe83 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 22 Mar 2020 21:08:11 -0700
Subject: [PATCH 126/176] Backup Worker: Cancel the actor when container is
 stopped

---
 fdbserver/BackupWorker.actor.cpp | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 8bb52a388e..97efe9f97e 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -97,6 +97,11 @@ struct BackupData {
 			TraceEvent("BackupWorkerAddJob", data->myId).detail("BackupID", uid).detail("Version", v);
 		}
 
+		void stop() {
+			stopped = true;
+			updateWorker = Void(); // cancel actors
+		}
+
 		bool isReady() const {
 			return stopped || (container.isReady() && ranges.isReady());
 		}
@@ -143,6 +148,7 @@ struct BackupData {
 						}
 					}
 					if (firstWorker) {
+						ASSERT(workers.present() && workers.get().size() > 0);
 						std::vector<std::pair<int64_t, int64_t>>& v = workers.get();
 						v.erase(std::remove_if(v.begin(), v.end(),
 						                       [epoch = self->recruitedEpoch](const std::pair<int64_t, int64_t>& p) {
@@ -160,6 +166,9 @@ struct BackupData {
 						if (!updated) {
 							config.startedBackupWorkers().set(tr, workers.get());
 						}
+						for (auto p : workers.get()) {
+							TraceEvent("BackupWorkerDebug", self->myId).detail("Epoch", p.first).detail("TagID", p.second);
+						}
 						wait(tr->commit());
 
 						updated = true; // Only set to true after commit.
@@ -304,7 +313,7 @@ struct BackupData {
 		for (UID uid : stopList) {
 			auto it = backups.find(uid);
 			ASSERT(it != backups.end());
-			it->second.stopped = true;
+			it->second.stop();
 			modified = true;
 		}
 		if (modified) changedTrigger.trigger();

From 658504bc66031eff4754e7094987745462d4a703 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 10:22:24 -0700
Subject: [PATCH 127/176] Add a cache to handle repeated delivery of backup
 recruitment messages

---
 fdbserver/worker.actor.cpp | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index a3cfc9ce30..802d0f8967 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -906,6 +906,7 @@ ACTOR Future<Void> workerServer(
 	// here is no, so that when running with log_version==3, all files should say V=3.
 	state std::map<SharedLogsKey, SharedLogsValue> sharedLogs;
 	state Reference<AsyncVar<UID>> activeSharedTLog(new AsyncVar<UID>());
+	state WorkerCache<InitializeBackupReply> backupWorkerCache;
 
 	state std::string coordFolder = abspath(_coordFolder);
 
@@ -1164,17 +1165,23 @@ ACTOR Future<Void> workerServer(
 				req.reply.send(recruited);
 			}
 			when (InitializeBackupRequest req = waitNext(interf.backup.getFuture())) {
-				BackupInterface recruited(locality);
-				recruited.initEndpoints();
+				if (!backupWorkerCache.exists(req.reqId)) {
+					BackupInterface recruited(locality);
+					recruited.initEndpoints();
 
-				startRole(Role::BACKUP, recruited.id(), interf.id());
-				DUMPTOKEN(recruited.waitFailure);
+					startRole(Role::BACKUP, recruited.id(), interf.id());
+					DUMPTOKEN(recruited.waitFailure);
 
-				Future<Void> backupProcess = backupWorker(recruited, req, dbInfo);
-				errorForwarders.add(forwardError(errors, Role::BACKUP, recruited.id(), backupProcess));
-				TraceEvent("BackupInitRequest", req.reqId).detail("BackupId", recruited.id());
-				InitializeBackupReply reply(recruited, req.backupEpoch);
-				req.reply.send(reply);
+					ReplyPromise<InitializeBackupReply> backupReady = req.reply;
+					backupWorkerCache.set(req.reqId, backupReady.getFuture());
+					Future<Void> backupProcess = backupWorker(recruited, req, dbInfo);
+					errorForwarders.add(forwardError(errors, Role::BACKUP, recruited.id(), backupProcess));
+					TraceEvent("BackupInitRequest", req.reqId).detail("BackupId", recruited.id());
+					InitializeBackupReply reply(recruited, req.backupEpoch);
+					backupReady.send(reply);
+				} else {
+					forwardPromise(req.reply, backupWorkerCache.get(req.reqId));
+				}
 			}
 			when( InitializeTLogRequest req = waitNext(interf.tLog.getFuture()) ) {
 				// For now, there's a one-to-one mapping of spill type to TLogVersion.

From a8c2acdba0b9a10eda53b88934089826614bc730 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 10:44:26 -0700
Subject: [PATCH 128/176] Count the unique number of tags in
 startedBackupWorkers

---
 fdbserver/BackupWorker.actor.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 97efe9f97e..a344f55272 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -155,7 +155,11 @@ struct BackupData {
 							                       return p.first != epoch;
 						                       }),
 						        v.end());
-						if (self->totalTags == v.size()) {
+						std::set<int64_t> tags;
+						for (auto p : v) {
+							tags.insert(p.second);
+						}
+						if (self->totalTags == tags.size()) {
 							config.allWorkerStarted().set(tr, true);
 							allUpdated = true;
 						} else {

From 3f31ebf659abde431e6e31f06b2034cd9d3d377b Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Fri, 20 Mar 2020 18:39:51 -0700
Subject: [PATCH 129/176] New backup:Revise event name and explain code

---
 fdbclient/FDBTypes.h                          |  3 +++
 fdbserver/BackupProgress.actor.cpp            |  2 +-
 fdbserver/BackupWorker.actor.cpp              |  5 ++--
 fdbserver/LogSystem.h                         |  2 +-
 fdbserver/RestoreApplier.actor.h              |  1 +
 fdbserver/TagPartitionedLogSystem.actor.cpp   | 27 ++++++++++++-------
 fdbserver/masterserver.actor.cpp              | 10 +++----
 ...kupAndParallelRestoreCorrectness.actor.cpp |  2 ++
 8 files changed, 34 insertions(+), 18 deletions(-)

diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 6a4d1f73c4..3f1e8ffb51 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -695,6 +695,9 @@ struct TLogVersion {
 		UNSET = 0,
 		// Everything between BEGIN and END should be densely packed, so that we
 		// can iterate over them easily.
+		// V3 was the introduction of spill by reference;
+		// V4 changed how data gets written to satellite TLogs so that we can peek from them;
+		// V5 merged reference and value spilling
 		// V1 = 1,  // 4.6 is dispatched to via 6.0
 		V2 = 2, // 6.0
 		V3 = 3, // 6.1
diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 1378980939..5fe90113b2 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -128,7 +128,7 @@ ACTOR Future<Void> getBackupProgress(Database cx, UID dbgid, Reference<BackupPro
 				const WorkerBackupStatus status = decodeBackupProgressValue(it.value);
 				bStatus->addBackupStatus(status);
 				TraceEvent("GotBackupProgress", dbgid)
-				    .detail("W", workerID)
+				    .detail("BackupWorker", workerID)
 				    .detail("Epoch", status.epoch)
 				    .detail("Version", status.version)
 				    .detail("Tag", status.tag.toString())
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 71af661932..752cf49157 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -71,8 +71,8 @@ struct BackupData {
 	const Version startVersion;
 	const Optional<Version> endVersion; // old epoch's end version (inclusive), or empty for current epoch
 	const LogEpoch recruitedEpoch;
-	const LogEpoch backupEpoch;
-	LogEpoch oldestBackupEpoch = 0;
+	const LogEpoch backupEpoch; // most recent active epoch whose tLogs are receiving mutations
+	LogEpoch oldestBackupEpoch = 0; // oldest epoch that still has data on tLogs for backup to pull
 	Version minKnownCommittedVersion;
 	Version savedVersion;
 	AsyncVar<Reference<ILogSystem>> logSystem;
@@ -820,6 +820,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 				if (hasPseudoLocality) {
 					self.logSystem.set(ls);
 					self.pop();
+					// Q: When will self.oldestBackupEpoch > ls->getOldestBackupEpoch()
 					self.oldestBackupEpoch = std::max(self.oldestBackupEpoch, ls->getOldestBackupEpoch());
 				}
 				TraceEvent("BackupWorkerLogSystem", self.myId)
diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h
index 7d8c79faba..b844019ee8 100644
--- a/fdbserver/LogSystem.h
+++ b/fdbserver/LogSystem.h
@@ -36,7 +36,7 @@ struct DBCoreState;
 struct TLogSet;
 struct CoreTLogSet;
 
-// The set of tLog servers and logRouters for a log tag
+// The set of tLog servers, logRouters and backupWorkers for a log tag
 class LogSet : NonCopyable, public ReferenceCounted<LogSet> {
 public:
 	std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> logServers;
diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index 66cf075bf6..fe779d5678 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -97,6 +97,7 @@ struct StagingKey {
 	}
 
 	// Precompute the final value of the key.
+	// TODO: Look at the last LogMessageVersion, if it set or clear, we can ignore the rest of versions.
 	void precomputeResult() {
 		TraceEvent(SevDebug, "FastRestoreApplierPrecomputeResult")
 		    .detail("Key", key)
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 3eed2b2de5..778f94c184 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -41,6 +41,7 @@ ACTOR Future<Version> minVersionWhenReady(Future<Void> f, std::vector<Future<TLo
 	return minVersion;
 }
 
+// TagPartitionedLogSystem info in old epoch
 struct OldLogData {
 	std::vector<Reference<LogSet>> tLogs;
 	int32_t logRouterTags;
@@ -165,7 +166,7 @@ OldTLogCoreData::OldTLogCoreData(const OldLogData& oldData)
 struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogSystem> {
 	const UID dbgid;
 	LogSystemType logSystemType;
-	std::vector<Reference<LogSet>> tLogs;
+	std::vector<Reference<LogSet>> tLogs; // LogSets in different locations: primary, remote or satellite
 	int expectedLogSets;
 	int logRouterTags;
 	int txsTags;
@@ -196,7 +197,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 	std::map< std::pair<UID, Tag>, std::pair<Version, Version> > outstandingPops;  // For each currently running popFromLog actor, (log server #, tag)->popped version
 	Optional<PromiseStream<Future<Void>>> addActor;
 	ActorCollection popActors;
-	std::vector<OldLogData> oldLogData;
+	std::vector<OldLogData> oldLogData; // each element has the log info. in one old epoch.
 	AsyncTrigger logSystemConfigChanged;
 
 	TagPartitionedLogSystem(UID dbgid, LocalityData locality, LogEpoch e,
@@ -1059,24 +1060,32 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 
 	void pop(Version upTo, Tag tag, Version durableKnownCommittedVersion, int8_t popLocality) override {
 		if (upTo <= 0) return;
-		if( tag.locality == tagLocalityRemoteLog) {
+		if (tag.locality == tagLocalityRemoteLog) {
 			popLogRouter(upTo, tag, durableKnownCommittedVersion, popLocality);
 			return;
 		}
-		for(auto& t : tLogs) {
-			if(t->locality == tagLocalitySpecial || t->locality == tag.locality || tag.locality == tagLocalityUpgraded || (tag.locality < 0 && ((popLocality == tagLocalityInvalid) == t->isLocal))) {
+		for (auto& t : tLogs) {
+			if (t->locality == tagLocalitySpecial || t->locality == tag.locality ||
+			    tag.locality == tagLocalityUpgraded ||
+			    (tag.locality < 0 && ((popLocality == tagLocalityInvalid) == t->isLocal))) {
 				for(auto& log : t->logServers) {
 					Version prev = outstandingPops[std::make_pair(log->get().id(),tag)].first;
-					if (prev < upTo)
+					if (prev < upTo) {
+						// update pop version for popFromLog actor
 						outstandingPops[std::make_pair(log->get().id(),tag)] = std::make_pair(upTo, durableKnownCommittedVersion);
-					if (prev == 0)
+					}
+					if (prev == 0) {
+						// pop tag from log upto version defined in outstandingPops[].first
 						popActors.add( popFromLog( this, log, tag, 1.0 ) ); //< FIXME: knob
+					}
 				}
 			}
 		}
 	}
 
-	ACTOR static Future<Void> popFromLog( TagPartitionedLogSystem* self, Reference<AsyncVar<OptionalInterface<TLogInterface>>> log, Tag tag, double time ) {
+	ACTOR static Future<Void> popFromLog(TagPartitionedLogSystem* self,
+	                                     Reference<AsyncVar<OptionalInterface<TLogInterface>>> log, Tag tag,
+	                                     double time) {
 		state Version last = 0;
 		loop {
 			wait( delay(time, TaskPriority::TLogPop) );
@@ -2363,7 +2372,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 					}
 				}
 			}
-			
+
 			for( int i = 0; i < recr.satelliteTLogs.size(); i++ ) {
 				InitializeTLogRequest &req = sreqs[i];
 				req.recruitmentID = logSystem->recruitmentID;
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 8f73d5ad6b..250f0f5506 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1288,19 +1288,19 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 	wait(gotProgress);
 	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
 	    backupProgress->getUnfinishedBackup();
-	for (const auto& [epochVersionCount, tagVersions] : toRecruit) {
+	for (const auto& [epochVersionTags, tagVersions] : toRecruit) {
 		for (const auto& [tag, version] : tagVersions) {
 			const auto& worker = self->backupWorkers[i % self->backupWorkers.size()];
 			i++;
 			InitializeBackupRequest req(deterministicRandom()->randomUniqueID());
 			req.recruitedEpoch = epoch;
-			req.backupEpoch = std::get<0>(epochVersionCount);
+			req.backupEpoch = std::get<0>(epochVersionTags);
 			req.routerTag = tag;
-			req.totalTags = std::get<2>(epochVersionCount);
+			req.totalTags = std::get<2>(epochVersionTags);
 			req.startVersion = version; // savedVersion + 1
-			req.endVersion = std::get<1>(epochVersionCount) - 1;
+			req.endVersion = std::get<1>(epochVersionTags) - 1;
 			TraceEvent("BackupRecruitment", self->dbgid)
-			    .detail("BKID", req.reqId)
+			    .detail("BackupWorker", req.reqId)
 			    .detail("Tag", req.routerTag.toString())
 			    .detail("Epoch", epoch)
 			    .detail("BackupEpoch", req.backupEpoch)
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 1461419c34..07803dacb3 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -440,6 +440,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 					} else if (deterministicRandom()->random01() < 0.1) {
 						targetVersion = desc.maxRestorableVersion.get();
 					} else if (deterministicRandom()->random01() < 0.5) {
+						ASSERT_WE_THINK(desc.minRestorableVersion.get() <= desc.contiguousLogEnd.get());
+						ASSERT_WE_THINK(desc.contiguousLogEnd.get() <= desc.maxRestorableVersion.get());
 						targetVersion = deterministicRandom()->randomInt64(desc.minRestorableVersion.get(),
 						                                                   desc.contiguousLogEnd.get());
 					}

From f0f4e42a4cb8a8ebebd599b0c611c2d7d962b194 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 12:47:42 -0700
Subject: [PATCH 130/176] Add removal for backupWorkerCache

---
 fdbserver/worker.actor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 802d0f8967..b22015e296 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -1175,6 +1175,7 @@ ACTOR Future<Void> workerServer(
 					ReplyPromise<InitializeBackupReply> backupReady = req.reply;
 					backupWorkerCache.set(req.reqId, backupReady.getFuture());
 					Future<Void> backupProcess = backupWorker(recruited, req, dbInfo);
+					backupProcess = storageCache.removeOnReady(req.reqId, backupProcess);
 					errorForwarders.add(forwardError(errors, Role::BACKUP, recruited.id(), backupProcess));
 					TraceEvent("BackupInitRequest", req.reqId).detail("BackupId", recruited.id());
 					InitializeBackupReply reply(recruited, req.backupEpoch);

From be67ab4d6a42ded2c3c529369652a9c0eebc428f Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Mon, 23 Mar 2020 12:53:40 -0700
Subject: [PATCH 131/176] Correct comment based on review

---
 fdbserver/BackupWorker.actor.cpp | 4 ++--
 fdbserver/masterserver.actor.cpp | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 752cf49157..bb2a073a08 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -70,8 +70,8 @@ struct BackupData {
 	const int totalTags; // Total log router tags
 	const Version startVersion;
 	const Optional<Version> endVersion; // old epoch's end version (inclusive), or empty for current epoch
-	const LogEpoch recruitedEpoch;
-	const LogEpoch backupEpoch; // most recent active epoch whose tLogs are receiving mutations
+	const LogEpoch recruitedEpoch; // current epoch whose tLogs are receiving mutations
+	const LogEpoch backupEpoch; // the epoch workers should pull mutations
 	LogEpoch oldestBackupEpoch = 0; // oldest epoch that still has data on tLogs for backup to pull
 	Version minKnownCommittedVersion;
 	Version savedVersion;
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 250f0f5506..10e7fb1d3f 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1300,7 +1300,7 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 			req.startVersion = version; // savedVersion + 1
 			req.endVersion = std::get<1>(epochVersionTags) - 1;
 			TraceEvent("BackupRecruitment", self->dbgid)
-			    .detail("BackupWorker", req.reqId)
+			    .detail("BKID", req.reqId)
 			    .detail("Tag", req.routerTag.toString())
 			    .detail("Epoch", epoch)
 			    .detail("BackupEpoch", req.backupEpoch)

From fd7643c322f318a4cbadd3de58f2bb25d73dd9a9 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 13:45:48 -0700
Subject: [PATCH 132/176] Remove a variable

---
 fdbserver/BackupWorker.actor.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 4c6c4b1ff3..0c300c7b64 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -483,7 +483,6 @@ ACTOR Future<Void> monitorBackupProgress(BackupData* self) {
 		state Reference<BackupProgress> progress(new BackupProgress(self->myId, {}));
 		wait(getBackupProgress(self->cx, self->myId, progress));
 		state std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
-		state bool finishedPreviousEpochs = self->recruitedEpoch == self->oldestBackupEpoch;
 		state std::map<UID, Version> savedLogVersions;
 		if (tagVersions.size() != self->totalTags) {
 			wait(interval);
@@ -492,8 +491,8 @@ ACTOR Future<Void> monitorBackupProgress(BackupData* self) {
 
 		// Check every version is larger than backup's startVersion
 		for (auto& [uid, info] : self->backups) {
-			if (finishedPreviousEpochs) {
-				// update update progress so far
+			if (self->recruitedEpoch == self->oldestBackupEpoch) {
+				// update update progress so far if previous epochs are done
 				Version v = std::numeric_limits<Version>::max();
 				for (const auto [tag, version] : tagVersions) {
 					v = std::min(v, version);

From 196127fb92483160ec111ac3948a7b338fb55352 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 14:15:36 -0700
Subject: [PATCH 133/176] Address review comments

---
 fdbserver/RestoreApplier.actor.cpp | 12 +++++++-----
 fdbserver/RestoreApplier.actor.h   |  2 +-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index ac2cee020e..2c86fa6560 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -100,9 +100,11 @@ ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int
 }
 
 // The actor may be invovked multiple times and executed async.
-// No race condition as long as we do not wait or yield when operate the shared data.
-// Multiple such actors can run on different fileIDs;
-// Only one actor can process mutations from the same file
+// No race condition as long as we do not wait or yield when operate the shared
+// data. Multiple such actors can run on different fileIDs.
+// Different files may contain mutations of the same commit versions, but with
+// different subsequence number.
+// Only one actor can process mutations from the same file.
 ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMutationsRequest req,
                                                           Reference<RestoreApplierData> self) {
 	state Reference<ApplierBatchData> batchData = self->batch[req.batchIndex];
@@ -130,9 +132,9 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMu
 		uint16_t numVersionStampedKV = 0;
 		// Sanity check: mutations in range file is in [beginVersion, endVersion);
 		// mutations in log file is in [beginVersion, endVersion], both inclusive.
-		ASSERT_WE_THINK(commitVersion >= req.asset.beginVersion);
+		ASSERT(commitVersion >= req.asset.beginVersion);
 		// Loader sends the endVersion to ensure all useful versions are sent
-		ASSERT_WE_THINK(commitVersion <= req.asset.endVersion);
+		ASSERT(commitVersion <= req.asset.endVersion);
 		ASSERT(req.mutations.size() == req.subs.size());
 
 		for (int mIndex = 0; mIndex < req.mutations.size(); mIndex++) {
diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index fe779d5678..47b596963e 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -67,8 +67,8 @@ struct StagingKey {
 			    .detail("Mutation", m.toString());
 		}
 		if (version == newVersion) { // Sanity check
-			TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m.toString());
 			ASSERT(type == m.type && key == m.param1 && val == m.param2);
+			TraceEvent(SevError, "SameVersion").detail("Version", version.toString()).detail("Mutation", m.toString());
 			return;
 		}
 

From dd9084527785eabf3e121329eaec4a6f199cfc78 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 14:49:05 -0700
Subject: [PATCH 134/176] Fix assert failure

Should be backup's contiguousLogEnd > maxRestorableVersion.
---
 .../workloads/BackupAndParallelRestoreCorrectness.actor.cpp     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 07803dacb3..71c2ca8c7f 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -441,7 +441,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 						targetVersion = desc.maxRestorableVersion.get();
 					} else if (deterministicRandom()->random01() < 0.5) {
 						ASSERT_WE_THINK(desc.minRestorableVersion.get() <= desc.contiguousLogEnd.get());
-						ASSERT_WE_THINK(desc.contiguousLogEnd.get() <= desc.maxRestorableVersion.get());
+						ASSERT_WE_THINK(desc.contiguousLogEnd.get() > desc.maxRestorableVersion.get());
 						targetVersion = deterministicRandom()->randomInt64(desc.minRestorableVersion.get(),
 						                                                   desc.contiguousLogEnd.get());
 					}

From f1d7fbafb4b444493da3bd97e0203b3ece39bf13 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 18:48:06 -0700
Subject: [PATCH 135/176] Stop actors for displaced backup workers

If the worker is displaced, it should not update backup containers.
---
 fdbserver/BackupWorker.actor.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 0c300c7b64..e0ebd3b03d 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -283,6 +283,14 @@ struct BackupData {
 		logSystem.get()->pop(savedVersion, popTag);
 	}
 
+	void stop() {
+		stopped = true;
+		for (auto& [uid, info] : backups) {
+			info.stop();
+		}
+		doneTrigger.trigger();
+	}
+
 	void eraseMessagesAfterEndVersion() {
 		ASSERT(endVersion.present());
 		const Version ver = endVersion.get();
@@ -913,8 +921,7 @@ ACTOR Future<Void> backupWorker(BackupInterface interf, InitializeBackupRequest
 		state Error err = e;
 		if (e.code() == error_code_worker_removed) {
 			pull = Void(); // cancels pulling
-			self.stopped = true;
-			self.doneTrigger.trigger();
+			self.stop();
 			wait(done);
 		}
 		TraceEvent("BackupWorkerTerminated", self.myId).error(err, true);

From 243d078596c6fdd497755584548c30d839685ebd Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 20:44:31 -0700
Subject: [PATCH 136/176] Fix off by one error

Epoch end version is saved version + 1, so need +1 for minBackupVersion.
---
 fdbserver/masterserver.actor.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index efecb537bd..20eeac132a 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1316,12 +1316,13 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 
 	state Future<Optional<Version>> fMinVersion = getMinBackupVersion(self, cx);
 	wait(gotProgress && success(fMinVersion));
+	TraceEvent("MinBackupVersion", self->dbgid).detail("Version", fMinVersion.get().present() ? fMinVersion.get() : -1);
 
 	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
 	    backupProgress->getUnfinishedBackup();
 	for (const auto& [epochVersionTags, tagVersions] : toRecruit) {
 		const Version oldEpochEnd = std::get<1>(epochVersionTags);
-		if (!fMinVersion.get().present() || fMinVersion.get().get() >= oldEpochEnd) {
+		if (!fMinVersion.get().present() || fMinVersion.get().get() + 1 >= oldEpochEnd) {
 			TraceEvent("SkipBackupRecruitment", self->dbgid)
 			    .detail("MinVersion", fMinVersion.get().get())
 			    .detail("Epoch", epoch)

From 82a17907769c9f9bfbf35faadb1ad78751cc8b2f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 21:11:25 -0700
Subject: [PATCH 137/176] Fix backup worker crash due to aborted backup job

If a backup job is aborted, the "startedBackupWorkers" key can be cleared, thus
triggering the assertion failure.
---
 fdbserver/BackupWorker.actor.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index e0ebd3b03d..9c6102d8c6 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -148,6 +148,10 @@ struct BackupData {
 						}
 					}
 					if (firstWorker) {
+						if (!workers.present()) {
+							TraceEvent("BackupWorkerDetectAbortedJob", self->myId).detail("BackupID", uid);
+							return Void();
+						}
 						ASSERT(workers.present() && workers.get().size() > 0);
 						std::vector<std::pair<int64_t, int64_t>>& v = workers.get();
 						v.erase(std::remove_if(v.begin(), v.end(),

From 1155304cd5079595c1ff1d88c9ae8d74fcd6a1a3 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 21:39:40 -0700
Subject: [PATCH 138/176] Remove a spurious assertion

It's possible that there is a gap between backup's contiguousLogEnd and snapshot
version.
---
 .../workloads/BackupAndParallelRestoreCorrectness.actor.cpp  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 71c2ca8c7f..5900043dbd 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -441,7 +441,10 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 						targetVersion = desc.maxRestorableVersion.get();
 					} else if (deterministicRandom()->random01() < 0.5) {
 						ASSERT_WE_THINK(desc.minRestorableVersion.get() <= desc.contiguousLogEnd.get());
-						ASSERT_WE_THINK(desc.contiguousLogEnd.get() > desc.maxRestorableVersion.get());
+						// This assertion can fail when contiguousLogEnd < maxRestorableVersion and
+						// the snapshot version > contiguousLogEnd. I.e., there is a gap between
+						// contiguousLogEnd and snapshot version.
+						// ASSERT_WE_THINK(desc.contiguousLogEnd.get() > desc.maxRestorableVersion.get());
 						targetVersion = deterministicRandom()->randomInt64(desc.minRestorableVersion.get(),
 						                                                   desc.contiguousLogEnd.get());
 					}

From a3058e7d969ad7ca06264a4c1a64cfd605658b6e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 23 Mar 2020 22:04:10 -0700
Subject: [PATCH 139/176] Fix incorrectly marking a backup job as stopped

This causes missing version ranges for mutation logs.
---
 fdbserver/BackupWorker.actor.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 9c6102d8c6..436ae4ee15 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -102,6 +102,8 @@ struct BackupData {
 			updateWorker = Void(); // cancel actors
 		}
 
+		void cancelUpdater() { updateWorker = Void(); }
+
 		bool isReady() const {
 			return stopped || (container.isReady() && ranges.isReady());
 		}
@@ -290,7 +292,10 @@ struct BackupData {
 	void stop() {
 		stopped = true;
 		for (auto& [uid, info] : backups) {
-			info.stop();
+			// Cancel the actor. Because container is valid, CANNOT set the
+			// "stop" flag that will block writing mutation files in
+			// saveMutationsToFile().
+			info.cancelUpdater();
 		}
 		doneTrigger.trigger();
 	}

From 7831bec2b07e4826cbe8ff6eac6eaa29b577ec94 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 24 Mar 2020 10:54:12 -0700
Subject: [PATCH 140/176] Address review comments on trace events

---
 fdbbackup/FileDecoder.actor.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index ef074adfac..4a9fb393f5 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -30,6 +30,8 @@
 #include "flow/serialize.h"
 #include "flow/actorcompiler.h" // has to be last include
 
+#define SevDecodeInfo SevVerbose
+
 extern bool g_crashOnError;
 
 namespace file_converter {
@@ -169,11 +171,10 @@ std::vector<MutationRef> decode_value(const StringRef& value) {
 	reader.consume<uint64_t>(); // Consume the includeVersion
 	uint32_t val_length = reader.consume<uint32_t>();
 	if (val_length != value.size() - sizeof(uint64_t) - sizeof(uint32_t)) {
-		TraceEvent("ValueError")
+		TraceEvent(SevError, "ValueError")
 		    .detail("ValueLen", val_length)
 		    .detail("ValueSize", value.size())
 		    .detail("Value", printable(value));
-		ASSERT(false);
 	}
 
 	std::vector<MutationRef> mutations;
@@ -334,7 +335,7 @@ struct DecodeProgress {
 				std::pair<Version, int32_t> version_part = decode_key(StringRef(k, kLen));
 				uint32_t vLen = reader.consumeNetworkUInt32();
 				const uint8_t* v = reader.consume(vLen);
-				TraceEvent("Block")
+				TraceEvent(SevDecodeInfo, "Block")
 				    .detail("KeySize", kLen)
 				    .detail("valueSize", vLen)
 				    .detail("Offset", reader.rptr - buf.begin())

From 4e477f5489711d452c351827ad6afb5b68b71950 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 13:59:35 -0700
Subject: [PATCH 141/176] Copy atomicRestore to atomicParallelRestore

---
 fdbclient/FileBackupAgent.actor.cpp | 102 ++++++++++++++++++++++++++++
 1 file changed, 102 insertions(+)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 0eec26fa8a..562915aa86 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -4410,6 +4410,108 @@ public:
 		Version ver = wait( restore(backupAgent, cx, cx, tagName, KeyRef(bc->getURL()), ranges, true, -1, true, addPrefix, removePrefix, true, randomUid) );
 		return ver;
 	}
+
+	// Similar to atomicRestore, only used in simulation test.
+	// locks the database before discontinuing the backup and that same lock is then used while doing the restore.
+	//the tagname of the backup must be the same as the restore.
+	ACTOR static Future<Version> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
+		state Reference<ReadYourWritesTransaction> ryw_tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
+		state BackupConfig backupConfig;
+		loop {
+			try {
+				ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+				state KeyBackedTag tag = makeBackupTag(tagName.toString());
+				UidAndAbortedFlagT uidFlag = wait(tag.getOrThrow(ryw_tr));
+				backupConfig = BackupConfig(uidFlag.first);
+				state EBackupState status = wait(backupConfig.stateEnum().getOrThrow(ryw_tr));
+
+				if (status != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL ) {
+					throw backup_duplicate();
+				}
+
+				break;
+			} catch( Error &e ) {
+				wait( ryw_tr->onError(e) );
+			}
+		}
+
+		//Lock src, record commit version
+		state Transaction tr(cx);
+		state Version commitVersion;
+		state UID randomUid = deterministicRandom()->randomUniqueID();
+		loop {
+			try {
+				// We must get a commit version so add a conflict range that won't likely cause conflicts
+				// but will ensure that the transaction is actually submitted.
+				tr.addWriteConflictRange(backupConfig.snapshotRangeDispatchMap().space.range());
+				wait( lockDatabase(&tr, randomUid) );
+				wait(tr.commit());
+				commitVersion = tr.getCommittedVersion();
+				TraceEvent("AS_Locked").detail("CommitVer", commitVersion);
+				break;
+			} catch( Error &e ) {
+				wait(tr.onError(e));
+			}
+		}
+
+		ryw_tr->reset();
+		loop {
+			try {
+				Optional<Version> restoreVersion = wait( backupConfig.getLatestRestorableVersion(ryw_tr) );
+				if(restoreVersion.present() && restoreVersion.get() >= commitVersion) {
+					TraceEvent("AS_RestoreVersion").detail("RestoreVer", restoreVersion.get());
+					break;
+				} else {
+					ryw_tr->reset();
+					wait(delay(0.2));
+				}
+			} catch( Error &e ) {
+				wait( ryw_tr->onError(e) );
+			}
+		}
+
+		ryw_tr->reset();
+		loop {
+			try {
+				wait( discontinueBackup(backupAgent, ryw_tr, tagName) );
+				wait( ryw_tr->commit() );
+				TraceEvent("AS_DiscontinuedBackup");
+				break;
+			} catch( Error &e ) {
+				if(e.code() == error_code_backup_unneeded || e.code() == error_code_backup_duplicate){
+					break;
+				}
+				wait( ryw_tr->onError(e) );
+			}
+		}
+
+		wait(success( waitBackup(backupAgent, cx, tagName.toString(), true) ));
+		TraceEvent("AS_BackupStopped");
+
+		ryw_tr->reset();
+		loop {
+			try {
+				ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+				for (auto &range : ranges) {
+					ryw_tr->addReadConflictRange(range);
+					ryw_tr->clear(range);
+				}
+				wait( ryw_tr->commit() );
+				TraceEvent("AS_ClearedRange");
+				break;
+			} catch( Error &e ) {
+				wait( ryw_tr->onError(e) );
+			}
+		}
+
+		Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx));
+
+		TraceEvent("AS_StartRestore");
+		Version ver = wait( restore(backupAgent, cx, cx, tagName, KeyRef(bc->getURL()), ranges, true, -1, true, addPrefix, removePrefix, true, randomUid) );
+		return ver;
+	}
 };
 
 const std::string BackupAgentBase::defaultTagName = "default";

From 5584884c12f52dadc7678b0fd13e6bc9e45efcb1 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 14:15:15 -0700
Subject: [PATCH 142/176] Refactor parallelRestoreFinish function into
 FileBackupAgent

---
 fdbclient/BackupAgent.actor.h                 |  3 ++
 fdbclient/FileBackupAgent.actor.cpp           | 31 +++++++++++++++++++
 ...kupAndParallelRestoreCorrectness.actor.cpp | 27 +---------------
 3 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index ae6717c619..cbced5702d 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -275,6 +275,9 @@ public:
 	enum ERestoreState { UNITIALIZED = 0, QUEUED = 1, STARTING = 2, RUNNING = 3, COMPLETED = 4, ABORTED = 5 };
 	static StringRef restoreStateText(ERestoreState id);
 
+	// parallel restore
+	Future<Version> parallelRestoreFinish(Database cx);
+
 	// restore() will
 	//   - make sure that url is readable and appears to be a complete backup
 	//   - make sure the requested TargetVersion is valid
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 562915aa86..96e0efd8b9 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -4518,6 +4518,37 @@ const std::string BackupAgentBase::defaultTagName = "default";
 const int BackupAgentBase::logHeaderSize = 12;
 const int FileBackupAgent::dataFooterSize = 20;
 
+// Return if parallel restore has finished
+Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx) {
+	state bool restoreDone = false;
+	state Future<Void> watchForRestoreRequestDone;
+	state ReadYourWritesTransaction tr(cx);
+	loop {
+		try {
+			if (restoreDone) break;
+			tr.reset();
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<Value> restoreRequestDoneKeyValue = wait(tr.get(restoreRequestDoneKey));
+			// Restore may finish before restoreAgent waits on the restore finish event.
+			if (restoreRequestDoneKeyValue.present()) {
+				restoreDone = true; // In case commit clears the key but in unknown_state
+				tr.clear(restoreRequestDoneKey);
+				wait(tr.commit());
+				break;
+			} else {
+				watchForRestoreRequestDone = tr.watch(restoreRequestDoneKey);
+				wait(tr.commit());
+				wait(watchForRestoreRequestDone);
+				break;
+			}
+		} catch (Error& e) {
+			wait(tr2.onError(e));
+		}
+	}
+	return Void();
+}
+
 Future<Version> FileBackupAgent::restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete, Version targetVersion, bool verbose, Key addPrefix, Key removePrefix, bool lockDB) {
 	return FileBackupAgentImpl::restore(this, cx, cxOrig, tagName, url, ranges, waitForComplete, targetVersion, verbose, addPrefix, removePrefix, lockDB, deterministicRandom()->randomUniqueID());
 }
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 2435d3f4e0..143b8bb5e8 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -505,32 +505,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 
 				// We should wait on all restore before proceeds
 				TraceEvent("FastRestore").detail("BackupAndParallelRestore", "WaitForRestoreToFinish");
-				restoreDone = false;
-				state Future<Void> watchForRestoreRequestDone;
-				loop {
-					try {
-						if (restoreDone) break;
-						tr2.reset();
-						tr2.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-						tr2.setOption(FDBTransactionOptions::LOCK_AWARE);
-						Optional<Value> restoreRequestDoneKeyValue = wait(tr2.get(restoreRequestDoneKey));
-						// Restore may finish before restoreAgent waits on the restore finish event.
-						if (restoreRequestDoneKeyValue.present()) {
-							restoreDone = true; // In case commit clears the key but in unknown_state
-							tr2.clear(restoreRequestDoneKey);
-							wait(tr2.commit());
-							break;
-						} else {
-							watchForRestoreRequestDone = tr2.watch(restoreRequestDoneKey);
-							wait(tr2.commit());
-							wait(watchForRestoreRequestDone);
-							break;
-						}
-					} catch (Error& e) {
-						wait(tr2.onError(e));
-					}
-				}
-
+				wait(backupAgent.parallelRestoreFinish(cx));
 				TraceEvent("FastRestore").detail("BackupAndParallelRestore", "RestoreFinished");
 
 				for (auto& restore : restores) {

From 81f7181c9eb2af6d8a6e9183e80be19f4541559d Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 14:35:03 -0700
Subject: [PATCH 143/176] Refactor submitParallelRestore function into
 FileBackupAgent

---
 fdbclient/BackupAgent.actor.h                 |  4 ++-
 fdbclient/FileBackupAgent.actor.cpp           | 31 ++++++++++++++++-
 ...kupAndParallelRestoreCorrectness.actor.cpp | 33 +++----------------
 3 files changed, 37 insertions(+), 31 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index cbced5702d..79d4b0f7be 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -276,7 +276,9 @@ public:
 	static StringRef restoreStateText(ERestoreState id);
 
 	// parallel restore
-	Future<Version> parallelRestoreFinish(Database cx);
+	Future<Void> parallelRestoreFinish(Database cx);
+	Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                                   KeyRef bcUrl, Version targetVersion, bool locked);
 
 	// restore() will
 	//   - make sure that url is readable and appears to be a complete backup
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 96e0efd8b9..46f4389b4d 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -4543,12 +4543,41 @@ Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx) {
 				break;
 			}
 		} catch (Error& e) {
-			wait(tr2.onError(e));
+			wait(tr.onError(e));
 		}
 	}
 	return Void();
 }
 
+Future<Void> FileBackupAgent::submitParallelRestore(Database cx, Key backupTag,
+                                                    Standalone<VectorRef<KeyRangeRef>> backupRanges, KeyRef bcUrl,
+                                                    Version targetVersion, bool locked) {
+	loop {
+		state ReadYourWritesTransaction tr(cx);
+		state int restoreIndex = 0;
+		tr.reset();
+		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		try {
+			// Note: we always lock DB here in case DB is modified at the bacupRanges boundary.
+			for (restoreIndex = 0; restoreIndex < backupRanges.size(); restoreIndex++) {
+				auto range = backupRanges[restoreIndex];
+				Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
+				// Register the request request in DB, which will be picked up by restore worker leader
+				struct RestoreRequest restoreRequest(restoreIndex, restoreTag, bcUrl, true, targetVersion, true, range,
+				                                     Key(), Key(), locked, deterministicRandom()->randomUniqueID());
+				tr.set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
+			}
+			tr.set(restoreRequestTriggerKey,
+			       restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), backupRanges.size()));
+			wait(tr.commit()); // Trigger restore
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
 Future<Version> FileBackupAgent::restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete, Version targetVersion, bool verbose, Key addPrefix, Key removePrefix, bool lockDB) {
 	return FileBackupAgentImpl::restore(this, cx, cxOrig, tagName, url, ranges, waitForComplete, targetVersion, verbose, addPrefix, removePrefix, lockDB, deterministicRandom()->randomUniqueID());
 }
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 143b8bb5e8..400db8c1b9 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -447,35 +447,10 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				state std::vector<Future<Version>> restores;
 				state std::vector<Standalone<StringRef>> restoreTags;
 
-				// Restore each range by calling backupAgent.restore()
+				// Submit parallel restore requests
 				TraceEvent("FastRestore").detail("PrepareRestores", self->backupRanges.size());
-				loop {
-					state ReadYourWritesTransaction tr1(cx);
-					tr1.reset();
-					tr1.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					tr1.setOption(FDBTransactionOptions::LOCK_AWARE);
-					try {
-						// Note: we always lock DB here in case DB is modified at the bacupRanges boundary.
-						for (restoreIndex = 0; restoreIndex < self->backupRanges.size(); restoreIndex++) {
-							auto range = self->backupRanges[restoreIndex];
-							Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" +
-							                                 std::to_string(restoreIndex));
-							restoreTags.push_back(restoreTag);
-							// Register the request request in DB, which will be picked up by restore worker leader
-							struct RestoreRequest restoreRequest(
-							    restoreIndex, restoreTag, KeyRef(lastBackupContainer->getURL()), true, targetVersion,
-							    true, range, Key(), Key(), self->locked, deterministicRandom()->randomUniqueID());
-							tr1.set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
-						}
-						tr1.set(restoreRequestTriggerKey,
-						        restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(),
-						                                   self->backupRanges.size()));
-						wait(tr1.commit()); // Trigger restore
-						break;
-					} catch (Error& e) {
-						wait(tr1.onError(e));
-					}
-				};
+				wait(backupAgent.submitParallelRestore(cx, self->backupTag, self->backupRanges,
+				                           KeyRef(lastBackupContainer->getURL()), targetVersion, self->locked));
 				TraceEvent("FastRestore").detail("TriggerRestore", "Setting up restoreRequestTriggerKey");
 
 				// Sometimes kill and restart the restore
@@ -503,7 +478,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 					}
 				}
 
-				// We should wait on all restore before proceeds
+				// Wait for parallel restore to finish before we can proceed
 				TraceEvent("FastRestore").detail("BackupAndParallelRestore", "WaitForRestoreToFinish");
 				wait(backupAgent.parallelRestoreFinish(cx));
 				TraceEvent("FastRestore").detail("BackupAndParallelRestore", "RestoreFinished");

From 01921c39532688bbc72cbf2f76ac1bab693fba60 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 14:56:53 -0700
Subject: [PATCH 144/176] Move FileBackupAgent parallel restore impl to
 FileBackupAgentImpl

For consistency with existing code, and
state variable can only occur in ACTOR.
---
 fdbclient/FileBackupAgent.actor.cpp | 114 +++++++++++++++-------------
 1 file changed, 63 insertions(+), 51 deletions(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 46f4389b4d..85bfb35e7c 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3548,6 +3548,67 @@ class FileBackupAgentImpl {
 public:
 	static const int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8;
 
+	// Parallel restore
+	ACTOR static Future<Void> parallelRestoreFinish(Database cx) {
+		state ReadYourWritesTransaction tr(cx);
+		state Future<Void> watchForRestoreRequestDone;
+		state bool restoreDone = false;
+		loop {
+			try {
+				if (restoreDone) break;
+				tr.reset();
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				Optional<Value> restoreRequestDoneKeyValue = wait(tr.get(restoreRequestDoneKey));
+				// Restore may finish before restoreAgent waits on the restore finish event.
+				if (restoreRequestDoneKeyValue.present()) {
+					restoreDone = true; // In case commit clears the key but in unknown_state
+					tr.clear(restoreRequestDoneKey);
+					wait(tr.commit());
+					break;
+				} else {
+					watchForRestoreRequestDone = tr.watch(restoreRequestDoneKey);
+					wait(tr.commit());
+					wait(watchForRestoreRequestDone);
+					break;
+				}
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+		return Void();
+	}
+
+	ACTOR static Future<Void> submitParallelRestore(Database cx, Key backupTag,
+														Standalone<VectorRef<KeyRangeRef>> backupRanges, KeyRef bcUrl,
+														Version targetVersion, bool locked) {
+		state ReadYourWritesTransaction tr(cx);
+		state int restoreIndex = 0;
+		loop {
+			tr.reset();
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			try {
+				// Note: we always lock DB here in case DB is modified at the bacupRanges boundary.
+				for (restoreIndex = 0; restoreIndex < backupRanges.size(); restoreIndex++) {
+					auto range = backupRanges[restoreIndex];
+					Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
+					// Register the request request in DB, which will be picked up by restore worker leader
+					struct RestoreRequest restoreRequest(restoreIndex, restoreTag, bcUrl, true, targetVersion, true, range,
+														Key(), Key(), locked, deterministicRandom()->randomUniqueID());
+					tr.set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
+				}
+				tr.set(restoreRequestTriggerKey,
+					restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), backupRanges.size()));
+				wait(tr.commit()); // Trigger restore
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+		return Void();
+	}
+
 	// This method will return the final status of the backup at tag, and return the URL that was used on the tag
 	// when that status value was read.
 	ACTOR static Future<int> waitBackup(FileBackupAgent* backupAgent, Database cx, std::string tagName, bool stopWhenDone, Reference<IBackupContainer> *pContainer = nullptr, UID *pUID = nullptr) {
@@ -4520,62 +4581,13 @@ const int FileBackupAgent::dataFooterSize = 20;
 
 // Return if parallel restore has finished
 Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx) {
-	state bool restoreDone = false;
-	state Future<Void> watchForRestoreRequestDone;
-	state ReadYourWritesTransaction tr(cx);
-	loop {
-		try {
-			if (restoreDone) break;
-			tr.reset();
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> restoreRequestDoneKeyValue = wait(tr.get(restoreRequestDoneKey));
-			// Restore may finish before restoreAgent waits on the restore finish event.
-			if (restoreRequestDoneKeyValue.present()) {
-				restoreDone = true; // In case commit clears the key but in unknown_state
-				tr.clear(restoreRequestDoneKey);
-				wait(tr.commit());
-				break;
-			} else {
-				watchForRestoreRequestDone = tr.watch(restoreRequestDoneKey);
-				wait(tr.commit());
-				wait(watchForRestoreRequestDone);
-				break;
-			}
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-	return Void();
+	return FileBackupAgentImpl::parallelRestoreFinish(cx);
 }
 
 Future<Void> FileBackupAgent::submitParallelRestore(Database cx, Key backupTag,
                                                     Standalone<VectorRef<KeyRangeRef>> backupRanges, KeyRef bcUrl,
                                                     Version targetVersion, bool locked) {
-	loop {
-		state ReadYourWritesTransaction tr(cx);
-		state int restoreIndex = 0;
-		tr.reset();
-		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-		try {
-			// Note: we always lock DB here in case DB is modified at the bacupRanges boundary.
-			for (restoreIndex = 0; restoreIndex < backupRanges.size(); restoreIndex++) {
-				auto range = backupRanges[restoreIndex];
-				Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
-				// Register the request request in DB, which will be picked up by restore worker leader
-				struct RestoreRequest restoreRequest(restoreIndex, restoreTag, bcUrl, true, targetVersion, true, range,
-				                                     Key(), Key(), locked, deterministicRandom()->randomUniqueID());
-				tr.set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
-			}
-			tr.set(restoreRequestTriggerKey,
-			       restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), backupRanges.size()));
-			wait(tr.commit()); // Trigger restore
-			break;
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
+    return FileBackupAgentImpl::submitParallelRestore(cx, backupTag, backupRanges, bcUrl, targetVersion, locked);
 }
 
 Future<Version> FileBackupAgent::restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete, Version targetVersion, bool verbose, Key addPrefix, Key removePrefix, bool lockDB) {

From b17392931678348a15c7cd6188a446ea0d3d1c96 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 15:51:40 -0700
Subject: [PATCH 145/176] Add atomicParallelRestore to AtomicRestore workload

---
 fdbclient/FileBackupAgent.actor.cpp         |  8 +++--
 fdbserver/workloads/AtomicRestore.actor.cpp | 40 +++++++++++----------
 2 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 85bfb35e7c..6f4e885865 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -4569,8 +4569,12 @@ public:
 
 		Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx));
 
-		TraceEvent("AS_StartRestore");
-		Version ver = wait( restore(backupAgent, cx, cx, tagName, KeyRef(bc->getURL()), ranges, true, -1, true, addPrefix, removePrefix, true, randomUid) );
+		TraceEvent("AtomicParallelRestoreStartRestore");
+		Version targetVersion = -1;
+		bool locked = true;
+		wait(submitParallelRestore(cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, locked));
+		TraceEvent("AtomicParallelRestoreWaitForRestoreFinish");
+		wait(parallelRestoreFinish(cx));
 		return ver;
 	}
 };
diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.actor.cpp
index 4537970f7a..95c3eb2357 100644
--- a/fdbserver/workloads/AtomicRestore.actor.cpp
+++ b/fdbserver/workloads/AtomicRestore.actor.cpp
@@ -79,26 +79,30 @@ struct AtomicRestoreWorkload : TestWorkload {
 		wait( delay(self->restoreAfter * deterministicRandom()->random01()) );
 		TraceEvent("AtomicRestore_RestoreStart");
 
-		loop {
-			std::vector<Future<Version>> restores;
-			if (deterministicRandom()->random01() < 0.5) {
-				for (auto &range : self->backupRanges)
-					restores.push_back(backupAgent.atomicRestore(cx, BackupAgentBase::getDefaultTag(), range, StringRef(), StringRef()));
+		if (deterministicRandom()->random01() < 0.5 && BUGGIFY) { // New fast parallel restore
+			TraceEvent(SevWarnAlways, "AtomicParallelRestore");
+			wait(backupAgent.atomicParallelRestore(cx, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef()));
+		} else { // Old style restore
+			loop {
+				std::vector<Future<Version>> restores;
+				if (deterministicRandom()->random01() < 0.5) {
+					for (auto& range : self->backupRanges)
+						restores.push_back(backupAgent.atomicRestore(cx, BackupAgentBase::getDefaultTag(), range,
+						                                             StringRef(), StringRef()));
+				} else {
+					restores.push_back(backupAgent.atomicRestore(cx, BackupAgentBase::getDefaultTag(),
+					                                             self->backupRanges, StringRef(), StringRef()));
+				}
+				try {
+					wait(waitForAll(restores));
+					break;
+				} catch (Error& e) {
+					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) throw;
+				}
+				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
 			}
-			else {
-				restores.push_back(backupAgent.atomicRestore(cx, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef()));
-			}
-			try {
-				wait(waitForAll(restores));
-				break;
-			}
-			catch (Error& e) {
-				if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
-					throw;
-			}
-			wait( delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY) );
 		}
-		
+
 		// SOMEDAY: Remove after backup agents can exist quiescently
 		if (g_simulator.backupAgents == ISimulator::BackupToFile) {
 			g_simulator.backupAgents = ISimulator::NoBackupAgents;

From ffb0a439ead9881c26a9bdf1bcc9bd34850ee977 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 16:05:09 -0700
Subject: [PATCH 146/176] atomicParallelRestore should not return a version

---
 fdbclient/FileBackupAgent.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 6f4e885865..d8655daa1c 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -4475,7 +4475,7 @@ public:
 	// Similar to atomicRestore, only used in simulation test.
 	// locks the database before discontinuing the backup and that same lock is then used while doing the restore.
 	//the tagname of the backup must be the same as the restore.
-	ACTOR static Future<Version> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
+	ACTOR static Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
 		state Reference<ReadYourWritesTransaction> ryw_tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
 		state BackupConfig backupConfig;
 		loop {
@@ -4575,7 +4575,7 @@ public:
 		wait(submitParallelRestore(cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, locked));
 		TraceEvent("AtomicParallelRestoreWaitForRestoreFinish");
 		wait(parallelRestoreFinish(cx));
-		return ver;
+		return Void();
 	}
 };
 

From 80d62f3cb8956746231af32a0e22e7213a5d1bfc Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 16:28:08 -0700
Subject: [PATCH 147/176] Fix:Add atomicParallelRestore to header

---
 fdbclient/BackupAgent.actor.h       | 1 +
 fdbclient/FileBackupAgent.actor.cpp | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 79d4b0f7be..4f74d5606b 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -279,6 +279,7 @@ public:
 	Future<Void> parallelRestoreFinish(Database cx);
 	Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                                   KeyRef bcUrl, Version targetVersion, bool locked);
+	Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix);
 
 	// restore() will
 	//   - make sure that url is readable and appears to be a complete backup
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index d8655daa1c..092667e558 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -4475,7 +4475,7 @@ public:
 	// Similar to atomicRestore, only used in simulation test.
 	// locks the database before discontinuing the backup and that same lock is then used while doing the restore.
 	//the tagname of the backup must be the same as the restore.
-	ACTOR static Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
+	ACTOR Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
 		state Reference<ReadYourWritesTransaction> ryw_tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
 		state BackupConfig backupConfig;
 		loop {

From 241c2703c81f14dee0fd01ad4c9b3d92c5c94131 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 16:44:17 -0700
Subject: [PATCH 148/176] Fix atomicParallelRestore interface

---
 fdbclient/BackupAgent.actor.h       | 2 +-
 fdbclient/FileBackupAgent.actor.cpp | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 4f74d5606b..ac47c2cdf9 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -279,7 +279,7 @@ public:
 	Future<Void> parallelRestoreFinish(Database cx);
 	Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                                   KeyRef bcUrl, Version targetVersion, bool locked);
-	Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix);
+	Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix);
 
 	// restore() will
 	//   - make sure that url is readable and appears to be a complete backup
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 092667e558..431e4f30fa 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -4450,6 +4450,7 @@ public:
 
 		ryw_tr->reset();
 		loop {
+
 			try {
 				ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -4475,7 +4476,7 @@ public:
 	// Similar to atomicRestore, only used in simulation test.
 	// locks the database before discontinuing the backup and that same lock is then used while doing the restore.
 	//the tagname of the backup must be the same as the restore.
-	ACTOR Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
+	ACTOR static Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
 		state Reference<ReadYourWritesTransaction> ryw_tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
 		state BackupConfig backupConfig;
 		loop {
@@ -4594,6 +4595,10 @@ Future<Void> FileBackupAgent::submitParallelRestore(Database cx, Key backupTag,
     return FileBackupAgentImpl::submitParallelRestore(cx, backupTag, backupRanges, bcUrl, targetVersion, locked);
 }
 
+Future<Void> FileBackupAgent::atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
+	return FileBackupAgentImpl::atomicParallelRestore(this, cx, tagName, ranges, addPrefix, removePrefix);
+}
+
 Future<Version> FileBackupAgent::restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete, Version targetVersion, bool verbose, Key addPrefix, Key removePrefix, bool lockDB) {
 	return FileBackupAgentImpl::restore(this, cx, cxOrig, tagName, url, ranges, waitForComplete, targetVersion, verbose, addPrefix, removePrefix, lockDB, deterministicRandom()->randomUniqueID());
 }

From d185359422cfd4f9b77897a82795a5dd2eb4be61 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 17:36:17 -0700
Subject: [PATCH 149/176] ApiCorrectnessAtomicRestore test: Increase timeout
 value

---
 tests/slow/ApiCorrectnessAtomicRestore.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/slow/ApiCorrectnessAtomicRestore.txt b/tests/slow/ApiCorrectnessAtomicRestore.txt
index 4e1f6b612b..58cc9d6371 100644
--- a/tests/slow/ApiCorrectnessAtomicRestore.txt
+++ b/tests/slow/ApiCorrectnessAtomicRestore.txt
@@ -26,3 +26,6 @@ startAfter=10.0
 restoreAfter=50.0
 clearAfterTest=false
 simBackupAgents=BackupToFile
+
+;timeout is in seconds
+timeout=360000
\ No newline at end of file

From edcbeb8992d01298c8396d26914c36efb442c6df Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 24 Mar 2020 18:22:20 -0700
Subject: [PATCH 150/176] Address review comments

Move transaction object outside of the loop and rename trace events.
---
 fdbserver/BackupProgress.actor.cpp | 4 ++--
 fdbserver/BackupWorker.actor.cpp   | 7 +++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 5fe90113b2..0142a7263a 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -50,7 +50,7 @@ void BackupProgress::updateTagVersions(std::map<Tag, Version>* tagVersions, std:
 		tags->erase(tag);
 		if (savedVersion < endVersion - 1) {
 			tagVersions->insert({ tag, savedVersion + 1 });
-			TraceEvent("BW", dbgid)
+			TraceEvent("BackupRange", dbgid)
 			    .detail("OldEpoch", epoch)
 			    .detail("Tag", tag.toString())
 			    .detail("BeginVersion", savedVersion + 1)
@@ -95,7 +95,7 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 
 		for (const Tag tag : tags) { // tags without progress data
 			tagVersions.insert({ tag, info.epochBegin });
-			TraceEvent("BW", dbgid)
+			TraceEvent("BackupRange", dbgid)
 			    .detail("OldEpoch", epoch)
 			    .detail("Tag", tag.toString())
 			    .detail("BeginVersion", info.epochBegin)
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 436ae4ee15..f814e2c812 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -131,12 +131,13 @@ struct BackupData {
 			state bool firstWorker = info->self->tag.id == 0;
 			state bool allUpdated = false;
 			state Optional<std::vector<std::pair<int64_t, int64_t>>> workers;
+			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
 
 			loop {
-				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
 				try {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+					tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
 					Optional<std::vector<std::pair<int64_t, int64_t>>> tmp =
 					    wait(config.startedBackupWorkers().get(tr));
@@ -437,11 +438,13 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
 
 // Set "latestBackupWorkerSavedVersion" key for backups
 ACTOR Future<Void> setBackupKeys(BackupData* self, std::map<UID, Version> savedLogVersions) {
+	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
+
 	loop {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
 			state std::vector<Future<Optional<Version>>> prevVersions;
 			state std::vector<BackupConfig> versionConfigs;

From 669916467e3004b2b614619c84ace7daca12cf47 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 24 Mar 2020 20:14:37 -0700
Subject: [PATCH 151/176] Add missing transaction reset call

---
 fdbserver/BackupWorker.actor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index f814e2c812..d04c83530e 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -187,6 +187,7 @@ struct BackupData {
 							break;
 						}
 						wait(watchFuture);
+						tr->reset();
 					} else {
 						ASSERT(workers.present() && workers.get().size() > 0);
 						config.startedBackupWorkers().set(tr, workers.get());

From 6a8d6ddb8ec8569f986ee2a54383578502f03ad3 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 22:29:26 -0700
Subject: [PATCH 152/176] Introduce
 ParallelRestoreApiCorrectnessAtomicRestore.txt test

This covers ApiCorrectnessTest as workload for parallel restore.
---
 fdbserver/workloads/AtomicRestore.actor.cpp   |  4 ++-
 tests/slow/ApiCorrectnessAtomicRestore.txt    |  5 +--
 ...llelRestoreApiCorrectnessAtomicRestore.txt | 36 +++++++++++++++++++
 3 files changed, 40 insertions(+), 5 deletions(-)
 create mode 100644 tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt

diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.actor.cpp
index 95c3eb2357..984b6d8ef0 100644
--- a/fdbserver/workloads/AtomicRestore.actor.cpp
+++ b/fdbserver/workloads/AtomicRestore.actor.cpp
@@ -27,6 +27,7 @@
 //A workload which test the correctness of backup and restore process
 struct AtomicRestoreWorkload : TestWorkload {
 	double startAfter, restoreAfter;
+	bool fastRestore; // true: use fast restore, false: use old style restore
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
 
 	AtomicRestoreWorkload(WorkloadContext const& wcx)
@@ -34,6 +35,7 @@ struct AtomicRestoreWorkload : TestWorkload {
 
 		startAfter = getOption(options, LiteralStringRef("startAfter"), 10.0);
 		restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 20.0);
+		fastRestore = getOption(options, LiteralStringRef("fastRestore"), false);
 		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
 	}
 
@@ -79,7 +81,7 @@ struct AtomicRestoreWorkload : TestWorkload {
 		wait( delay(self->restoreAfter * deterministicRandom()->random01()) );
 		TraceEvent("AtomicRestore_RestoreStart");
 
-		if (deterministicRandom()->random01() < 0.5 && BUGGIFY) { // New fast parallel restore
+		if (self->fastRestore) { // New fast parallel restore
 			TraceEvent(SevWarnAlways, "AtomicParallelRestore");
 			wait(backupAgent.atomicParallelRestore(cx, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef()));
 		} else { // Old style restore
diff --git a/tests/slow/ApiCorrectnessAtomicRestore.txt b/tests/slow/ApiCorrectnessAtomicRestore.txt
index 58cc9d6371..9bdab100a7 100644
--- a/tests/slow/ApiCorrectnessAtomicRestore.txt
+++ b/tests/slow/ApiCorrectnessAtomicRestore.txt
@@ -25,7 +25,4 @@ testName=AtomicRestore
 startAfter=10.0
 restoreAfter=50.0
 clearAfterTest=false
-simBackupAgents=BackupToFile
-
-;timeout is in seconds
-timeout=360000
\ No newline at end of file
+simBackupAgents=BackupToFile
\ No newline at end of file
diff --git a/tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt b/tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt
new file mode 100644
index 0000000000..bcb6594fab
--- /dev/null
+++ b/tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt
@@ -0,0 +1,36 @@
+testTitle=ApiCorrectnessTest
+testName=ApiCorrectness
+runSetup=true
+clearAfterTest=true
+numKeys=5000
+onlyLowerCase=true
+shortKeysRatio=0.5
+minShortKeyLength=1
+maxShortKeyLength=3
+minLongKeyLength=1
+maxLongKeyLength=128
+minValueLength=1
+maxValueLength=1000
+numGets=1000
+numGetRanges=100
+numGetRangeSelectors=100
+numGetKeys=100
+numClears=100
+numClearRanges=10
+maxTransactionBytes=500000
+randomTestDuration=60
+timeout=2100
+
+testName=AtomicRestore
+startAfter=10.0
+restoreAfter=50.0
+clearAfterTest=false
+simBackupAgents=BackupToFile
+fastRestore=true
+
+; Each testName=RunRestoreWorkerWorkload creates a restore worker
+; We need at least 3 restore workers: master, loader, and applier
+testName=RunRestoreWorkerWorkload
+
+;timeout is in seconds
+timeout=360000
\ No newline at end of file

From e59becdec8529e00f73370845d5be902cbae4d8a Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 22:36:36 -0700
Subject: [PATCH 153/176] Refactor atomicRestore for atomicParallelRestore

---
 fdbclient/FileBackupAgent.actor.cpp | 121 +++++-----------------------
 1 file changed, 18 insertions(+), 103 deletions(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 431e4f30fa..e5eba0c591 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -4373,7 +4373,9 @@ public:
 
 	//used for correctness only, locks the database before discontinuing the backup and that same lock is then used while doing the restore.
 	//the tagname of the backup must be the same as the restore.
-	ACTOR static Future<Version> atomicRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
+	ACTOR static Future<Version> atomicRestore(FileBackupAgent* backupAgent, Database cx, Key tagName,
+	                                           Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix,
+	                                           Key removePrefix, bool fastRestore = false) {
 		state Reference<ReadYourWritesTransaction> ryw_tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
 		state BackupConfig backupConfig;
 		loop {
@@ -4468,114 +4470,27 @@ public:
 
 		Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx));
 
-		TraceEvent("AS_StartRestore");
-		Version ver = wait( restore(backupAgent, cx, cx, tagName, KeyRef(bc->getURL()), ranges, true, -1, true, addPrefix, removePrefix, true, randomUid) );
-		return ver;
+		if (fastRestore) {
+			TraceEvent("AtomicParallelRestoreStartRestore");
+			Version targetVersion = -1;
+			bool locked = true;
+			wait(submitParallelRestore(cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, locked));
+			TraceEvent("AtomicParallelRestoreWaitForRestoreFinish");
+			wait(parallelRestoreFinish(cx));
+			return -1;
+		} else {
+			TraceEvent("AS_StartRestore");
+			Version ver = wait(restore(backupAgent, cx, cx, tagName, KeyRef(bc->getURL()), ranges, true, -1, true,
+			                           addPrefix, removePrefix, true, randomUid));
+			return ver;
+		}
 	}
 
 	// Similar to atomicRestore, only used in simulation test.
 	// locks the database before discontinuing the backup and that same lock is then used while doing the restore.
 	//the tagname of the backup must be the same as the restore.
 	ACTOR static Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
-		state Reference<ReadYourWritesTransaction> ryw_tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
-		state BackupConfig backupConfig;
-		loop {
-			try {
-				ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-				state KeyBackedTag tag = makeBackupTag(tagName.toString());
-				UidAndAbortedFlagT uidFlag = wait(tag.getOrThrow(ryw_tr));
-				backupConfig = BackupConfig(uidFlag.first);
-				state EBackupState status = wait(backupConfig.stateEnum().getOrThrow(ryw_tr));
-
-				if (status != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL ) {
-					throw backup_duplicate();
-				}
-
-				break;
-			} catch( Error &e ) {
-				wait( ryw_tr->onError(e) );
-			}
-		}
-
-		//Lock src, record commit version
-		state Transaction tr(cx);
-		state Version commitVersion;
-		state UID randomUid = deterministicRandom()->randomUniqueID();
-		loop {
-			try {
-				// We must get a commit version so add a conflict range that won't likely cause conflicts
-				// but will ensure that the transaction is actually submitted.
-				tr.addWriteConflictRange(backupConfig.snapshotRangeDispatchMap().space.range());
-				wait( lockDatabase(&tr, randomUid) );
-				wait(tr.commit());
-				commitVersion = tr.getCommittedVersion();
-				TraceEvent("AS_Locked").detail("CommitVer", commitVersion);
-				break;
-			} catch( Error &e ) {
-				wait(tr.onError(e));
-			}
-		}
-
-		ryw_tr->reset();
-		loop {
-			try {
-				Optional<Version> restoreVersion = wait( backupConfig.getLatestRestorableVersion(ryw_tr) );
-				if(restoreVersion.present() && restoreVersion.get() >= commitVersion) {
-					TraceEvent("AS_RestoreVersion").detail("RestoreVer", restoreVersion.get());
-					break;
-				} else {
-					ryw_tr->reset();
-					wait(delay(0.2));
-				}
-			} catch( Error &e ) {
-				wait( ryw_tr->onError(e) );
-			}
-		}
-
-		ryw_tr->reset();
-		loop {
-			try {
-				wait( discontinueBackup(backupAgent, ryw_tr, tagName) );
-				wait( ryw_tr->commit() );
-				TraceEvent("AS_DiscontinuedBackup");
-				break;
-			} catch( Error &e ) {
-				if(e.code() == error_code_backup_unneeded || e.code() == error_code_backup_duplicate){
-					break;
-				}
-				wait( ryw_tr->onError(e) );
-			}
-		}
-
-		wait(success( waitBackup(backupAgent, cx, tagName.toString(), true) ));
-		TraceEvent("AS_BackupStopped");
-
-		ryw_tr->reset();
-		loop {
-			try {
-				ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-				for (auto &range : ranges) {
-					ryw_tr->addReadConflictRange(range);
-					ryw_tr->clear(range);
-				}
-				wait( ryw_tr->commit() );
-				TraceEvent("AS_ClearedRange");
-				break;
-			} catch( Error &e ) {
-				wait( ryw_tr->onError(e) );
-			}
-		}
-
-		Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx));
-
-		TraceEvent("AtomicParallelRestoreStartRestore");
-		Version targetVersion = -1;
-		bool locked = true;
-		wait(submitParallelRestore(cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, locked));
-		TraceEvent("AtomicParallelRestoreWaitForRestoreFinish");
-		wait(parallelRestoreFinish(cx));
+		Version ver = wait(atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, true));
 		return Void();
 	}
 };

From ca8966a28bf819d2ce724e84d4942873f0da3422 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 23:07:04 -0700
Subject: [PATCH 154/176] Move lockDB into submitRestore request from restore
 worker

AtomicRestore needs to lock DB before we start the restore worker.
So we cannot lock DB in restore worker with a different randomUID.
---
 fdbclient/BackupAgent.actor.h                 |  2 +-
 fdbclient/FileBackupAgent.actor.cpp           | 37 ++++++++++++++++---
 fdbserver/RestoreMaster.actor.cpp             | 24 +-----------
 ...kupAndParallelRestoreCorrectness.actor.cpp |  4 +-
 4 files changed, 35 insertions(+), 32 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index ac47c2cdf9..20abb828ea 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -278,7 +278,7 @@ public:
 	// parallel restore
 	Future<Void> parallelRestoreFinish(Database cx);
 	Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                   KeyRef bcUrl, Version targetVersion, bool locked);
+	                                   KeyRef bcUrl, Version targetVersion, bool lockDB, UID randomUID);
 	Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix);
 
 	// restore() will
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index e5eba0c591..670869dd22 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3581,9 +3581,34 @@ public:
 
 	ACTOR static Future<Void> submitParallelRestore(Database cx, Key backupTag,
 														Standalone<VectorRef<KeyRangeRef>> backupRanges, KeyRef bcUrl,
-														Version targetVersion, bool locked) {
+														Version targetVersion, bool lockDB, UID randomUID) {
 		state ReadYourWritesTransaction tr(cx);
 		state int restoreIndex = 0;
+		state int numTries = 0;
+		// lock DB for restore
+		loop {
+			try {
+				if (lockDB) {
+					wait(lockDatabase(cx, randomUID));
+				}
+				tr.reset();
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				wait(checkDatabaseLock(Reference<ReadYourWritesTransaction>(&tr), randomUID));
+
+				TraceEvent("FastRestoreMasterProcessRestoreRequests").detail("DBIsLocked", randomUID);
+				break;
+			} catch (Error& e) {
+				TraceEvent("FastRestoreMasterProcessRestoreRequests").detail("CheckLockError", e.what());
+				TraceEvent(numTries > 50 ? SevError : SevWarnAlways, "FastRestoreMayFail")
+					.detail("Reason", "DB is not properly locked")
+					.detail("ExpectedLockID", randomUID);
+				numTries++;
+				wait(delay(5.0));
+			}
+		}
+
+		// set up restore request
 		loop {
 			tr.reset();
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -3595,7 +3620,7 @@ public:
 					Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
 					// Register the request request in DB, which will be picked up by restore worker leader
 					struct RestoreRequest restoreRequest(restoreIndex, restoreTag, bcUrl, true, targetVersion, true, range,
-														Key(), Key(), locked, deterministicRandom()->randomUniqueID());
+														Key(), Key(), lockDB, deterministicRandom()->randomUniqueID());
 					tr.set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
 				}
 				tr.set(restoreRequestTriggerKey,
@@ -4473,8 +4498,8 @@ public:
 		if (fastRestore) {
 			TraceEvent("AtomicParallelRestoreStartRestore");
 			Version targetVersion = -1;
-			bool locked = true;
-			wait(submitParallelRestore(cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, locked));
+			bool lockDB = true;
+			wait(submitParallelRestore(cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, lockDB, randomUid));
 			TraceEvent("AtomicParallelRestoreWaitForRestoreFinish");
 			wait(parallelRestoreFinish(cx));
 			return -1;
@@ -4506,8 +4531,8 @@ Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx) {
 
 Future<Void> FileBackupAgent::submitParallelRestore(Database cx, Key backupTag,
                                                     Standalone<VectorRef<KeyRangeRef>> backupRanges, KeyRef bcUrl,
-                                                    Version targetVersion, bool locked) {
-    return FileBackupAgentImpl::submitParallelRestore(cx, backupTag, backupRanges, bcUrl, targetVersion, locked);
+                                                    Version targetVersion, bool lockDB, UID randomUID) {
+    return FileBackupAgentImpl::submitParallelRestore(cx, backupTag, backupRanges, bcUrl, targetVersion, lockDB, randomUID);
 }
 
 Future<Void> FileBackupAgent::atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 59a3c1d63c..c89ce7a530 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -186,29 +186,7 @@ ACTOR Future<Void> startProcessRestoreRequests(Reference<RestoreMasterData> self
 
 	TraceEvent("FastRestoreMasterWaitOnRestoreRequests", self->id());
 
-	// lock DB for restore
-	numTries = 0;
-	loop {
-		try {
-			wait(lockDatabase(cx, randomUID));
-			state Reference<ReadYourWritesTransaction> tr =
-			    Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
-			tr->reset();
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			wait(checkDatabaseLock(tr, randomUID));
-			TraceEvent("FastRestoreMasterProcessRestoreRequests", self->id()).detail("DBIsLocked", randomUID);
-			break;
-		} catch (Error& e) {
-			TraceEvent("FastRestoreMasterProcessRestoreRequests", self->id()).detail("CheckLockError", e.what());
-			TraceEvent(numTries > 50 ? SevError : SevWarnAlways, "FastRestoreMayFail")
-			    .detail("Reason", "DB is not properly locked")
-			    .detail("ExpectedLockID", randomUID);
-			numTries++;
-			wait(delay(5.0));
-		}
-	}
-
+	// DB has been locked where restore request is submitted
 	wait(clearDB(cx));
 
 	// Step: Perform the restore requests
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 400db8c1b9..31410125f8 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -450,11 +450,11 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				// Submit parallel restore requests
 				TraceEvent("FastRestore").detail("PrepareRestores", self->backupRanges.size());
 				wait(backupAgent.submitParallelRestore(cx, self->backupTag, self->backupRanges,
-				                           KeyRef(lastBackupContainer->getURL()), targetVersion, self->locked));
+				                           KeyRef(lastBackupContainer->getURL()), targetVersion, self->locked, randomID));
 				TraceEvent("FastRestore").detail("TriggerRestore", "Setting up restoreRequestTriggerKey");
 
 				// Sometimes kill and restart the restore
-				// In real cluster, aborting a restore needs: 
+				// In real cluster, aborting a restore needs:
 				// (1) kill restore cluster; (2) clear dest. DB restore system keyspace.
 				// TODO: Consider gracefully abort a restore and restart.
 				if (BUGGIFY && TEST_ABORT_FASTRESTORE) {

From 00fb4c1a354c67351d45355c8673a516e518ece3 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 24 Mar 2020 23:37:11 -0700
Subject: [PATCH 155/176] Fix an off by one error

Backup worker's saved version should start from its startVersion - 1, i.e.,
the startVersion is not saved yet. Otherwise, if the version range is just
the startVersion itself and there is no data, then the range [startVersion,
startVersion + 1) will be missing. This causes non-continuous partitioned logs.
---
 fdbserver/BackupWorker.actor.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index d04c83530e..3cfc1662d3 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -222,8 +222,8 @@ struct BackupData {
 	explicit BackupData(UID id, Reference<AsyncVar<ServerDBInfo>> db, const InitializeBackupRequest& req)
 	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
 	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
-	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion), cc("BackupWorker", myId.toString()),
-	    pulledVersion(0) {
+	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion - 1),
+	    cc("BackupWorker", myId.toString()), pulledVersion(0) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true);
 
 		specialCounter(cc, "SavedVersion", [this]() { return this->savedVersion; });
@@ -624,7 +624,8 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		activeUids.push_back(it->first);
 		self->insertRanges(keyRangeMap, it->second.ranges.get(), index);
 		if (it->second.lastSavedVersion == invalidVersion) {
-			it->second.lastSavedVersion = self->savedVersion;
+			it->second.lastSavedVersion =
+			    self->savedVersion > self->startVersion ? self->savedVersion : self->startVersion;
 		}
 		logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile(
 		    it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags));

From 130b91c820cd8cb17b53d1bc8688931efcc2cbd6 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 24 Mar 2020 23:52:34 -0700
Subject: [PATCH 156/176] Fix segmentation fault in submitParallelRestore

---
 fdbclient/FileBackupAgent.actor.cpp | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 670869dd22..29b4e49819 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3582,7 +3582,7 @@ public:
 	ACTOR static Future<Void> submitParallelRestore(Database cx, Key backupTag,
 														Standalone<VectorRef<KeyRangeRef>> backupRanges, KeyRef bcUrl,
 														Version targetVersion, bool lockDB, UID randomUID) {
-		state ReadYourWritesTransaction tr(cx);
+		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 		state int restoreIndex = 0;
 		state int numTries = 0;
 		// lock DB for restore
@@ -3591,15 +3591,15 @@ public:
 				if (lockDB) {
 					wait(lockDatabase(cx, randomUID));
 				}
-				tr.reset();
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				wait(checkDatabaseLock(Reference<ReadYourWritesTransaction>(&tr), randomUID));
+				tr->reset();
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+				wait(checkDatabaseLock(tr, randomUID));
 
-				TraceEvent("FastRestoreMasterProcessRestoreRequests").detail("DBIsLocked", randomUID);
+				TraceEvent("FastRestoreMasterSubmitRestoreRequests").detail("DBIsLocked", randomUID);
 				break;
 			} catch (Error& e) {
-				TraceEvent("FastRestoreMasterProcessRestoreRequests").detail("CheckLockError", e.what());
+				TraceEvent("FastRestoreMasterSubmitRestoreRequests").detail("CheckLockError", e.what());
 				TraceEvent(numTries > 50 ? SevError : SevWarnAlways, "FastRestoreMayFail")
 					.detail("Reason", "DB is not properly locked")
 					.detail("ExpectedLockID", randomUID);
@@ -3610,9 +3610,9 @@ public:
 
 		// set up restore request
 		loop {
-			tr.reset();
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr->reset();
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			try {
 				// Note: we always lock DB here in case DB is modified at the bacupRanges boundary.
 				for (restoreIndex = 0; restoreIndex < backupRanges.size(); restoreIndex++) {
@@ -3621,14 +3621,14 @@ public:
 					// Register the request request in DB, which will be picked up by restore worker leader
 					struct RestoreRequest restoreRequest(restoreIndex, restoreTag, bcUrl, true, targetVersion, true, range,
 														Key(), Key(), lockDB, deterministicRandom()->randomUniqueID());
-					tr.set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
+					tr->set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
 				}
-				tr.set(restoreRequestTriggerKey,
+				tr->set(restoreRequestTriggerKey,
 					restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), backupRanges.size()));
-				wait(tr.commit()); // Trigger restore
+				wait(tr->commit()); // Trigger restore
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				wait(tr->onError(e));
 			}
 		}
 		return Void();

From e2f317a0da54b07d65ac41ecc6634cc92bb9a67f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 25 Mar 2020 09:18:49 -0700
Subject: [PATCH 157/176] Fix a crash failure

---
 fdbserver/masterserver.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 20eeac132a..3dc883636b 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1324,7 +1324,7 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<MasterData> self, Datab
 		const Version oldEpochEnd = std::get<1>(epochVersionTags);
 		if (!fMinVersion.get().present() || fMinVersion.get().get() + 1 >= oldEpochEnd) {
 			TraceEvent("SkipBackupRecruitment", self->dbgid)
-			    .detail("MinVersion", fMinVersion.get().get())
+			    .detail("MinVersion", fMinVersion.get().present() ? fMinVersion.get() : -1)
 			    .detail("Epoch", epoch)
 			    .detail("OldEpoch", std::get<0>(epochVersionTags))
 			    .detail("OldEpochEnd", oldEpochEnd);

From 472f7bdd320f3506af4adbc22ec2c2c2ec76d57e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 25 Mar 2020 11:03:05 -0700
Subject: [PATCH 158/176] Rename a trace event to avoid confusion

Change from BackupRange to BackupVersionRange.
---
 fdbserver/BackupProgress.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp
index 0142a7263a..998c153ce7 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/BackupProgress.actor.cpp
@@ -50,7 +50,7 @@ void BackupProgress::updateTagVersions(std::map<Tag, Version>* tagVersions, std:
 		tags->erase(tag);
 		if (savedVersion < endVersion - 1) {
 			tagVersions->insert({ tag, savedVersion + 1 });
-			TraceEvent("BackupRange", dbgid)
+			TraceEvent("BackupVersionRange", dbgid)
 			    .detail("OldEpoch", epoch)
 			    .detail("Tag", tag.toString())
 			    .detail("BeginVersion", savedVersion + 1)
@@ -95,7 +95,7 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 
 		for (const Tag tag : tags) { // tags without progress data
 			tagVersions.insert({ tag, info.epochBegin });
-			TraceEvent("BackupRange", dbgid)
+			TraceEvent("BackupVersionRange", dbgid)
 			    .detail("OldEpoch", epoch)
 			    .detail("Tag", tag.toString())
 			    .detail("BeginVersion", info.epochBegin)

From 120272f0256cf46fe496bbb3f6a81d1a4e853b7f Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Wed, 25 Mar 2020 10:35:15 -0700
Subject: [PATCH 159/176] Change unlockDB from RestoreMaster to Agent

---
 fdbclient/BackupAgent.actor.h                 |  2 +-
 fdbclient/FileBackupAgent.actor.cpp           | 30 +++++++++++++++----
 fdbserver/RestoreMaster.actor.cpp             | 16 ----------
 ...kupAndParallelRestoreCorrectness.actor.cpp |  2 +-
 4 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 20abb828ea..13be056426 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -276,7 +276,7 @@ public:
 	static StringRef restoreStateText(ERestoreState id);
 
 	// parallel restore
-	Future<Void> parallelRestoreFinish(Database cx);
+	Future<Void> parallelRestoreFinish(Database cx, UID randomUID);
 	Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                                   KeyRef bcUrl, Version targetVersion, bool lockDB, UID randomUID);
 	Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix);
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 29b4e49819..6513eea102 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3549,10 +3549,11 @@ public:
 	static const int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8;
 
 	// Parallel restore
-	ACTOR static Future<Void> parallelRestoreFinish(Database cx) {
+	ACTOR static Future<Void> parallelRestoreFinish(Database cx, UID randomUID) {
 		state ReadYourWritesTransaction tr(cx);
 		state Future<Void> watchForRestoreRequestDone;
 		state bool restoreDone = false;
+		TraceEvent("FastRestoreAgentWaitForRestoreToFinish").detail("DBLock", randomUID);
 		loop {
 			try {
 				if (restoreDone) break;
@@ -3576,6 +3577,23 @@ public:
 				wait(tr.onError(e));
 			}
 		}
+		TraceEvent("FastRestoreAgentRestoreFinished").detail("UnlockDBStart", randomUID);
+		try {
+			wait(unlockDatabase(cx, randomUID));
+		} catch (Error& e) {
+			if (e.code() == error_code_operation_cancelled) { // Should only happen in simulation
+				TraceEvent(SevWarnAlways, "FastRestoreAgentOnCancelingActor")
+				    .detail("DBLock", randomUID)
+				    .detail("ManualCheck", "Is DB locked");
+			} else {
+				TraceEvent(SevError, "FastRestoreAgentUnlockDBFailed")
+				    .detail("DBLock", randomUID)
+				    .detail("ErrorCode", e.code())
+				    .detail("Error", e.what());
+				ASSERT_WE_THINK(false); // This unlockDatabase should always succeed, we think.
+			}
+		}
+		TraceEvent("FastRestoreAgentRestoreFinished").detail("UnlockDBFinish", randomUID);
 		return Void();
 	}
 
@@ -3596,10 +3614,10 @@ public:
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				wait(checkDatabaseLock(tr, randomUID));
 
-				TraceEvent("FastRestoreMasterSubmitRestoreRequests").detail("DBIsLocked", randomUID);
+				TraceEvent("FastRestoreAgentSubmitRestoreRequests").detail("DBIsLocked", randomUID);
 				break;
 			} catch (Error& e) {
-				TraceEvent("FastRestoreMasterSubmitRestoreRequests").detail("CheckLockError", e.what());
+				TraceEvent("FastRestoreAgentSubmitRestoreRequests").detail("CheckLockError", e.what());
 				TraceEvent(numTries > 50 ? SevError : SevWarnAlways, "FastRestoreMayFail")
 					.detail("Reason", "DB is not properly locked")
 					.detail("ExpectedLockID", randomUID);
@@ -4501,7 +4519,7 @@ public:
 			bool lockDB = true;
 			wait(submitParallelRestore(cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, lockDB, randomUid));
 			TraceEvent("AtomicParallelRestoreWaitForRestoreFinish");
-			wait(parallelRestoreFinish(cx));
+			wait(parallelRestoreFinish(cx, randomUid));
 			return -1;
 		} else {
 			TraceEvent("AS_StartRestore");
@@ -4525,8 +4543,8 @@ const int BackupAgentBase::logHeaderSize = 12;
 const int FileBackupAgent::dataFooterSize = 20;
 
 // Return if parallel restore has finished
-Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx) {
-	return FileBackupAgentImpl::parallelRestoreFinish(cx);
+Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID) {
+	return FileBackupAgentImpl::parallelRestoreFinish(cx, randomUID);
 }
 
 Future<Void> FileBackupAgent::submitParallelRestore(Database cx, Key backupTag,
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index c89ce7a530..02ab739e7f 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -214,22 +214,6 @@ ACTOR Future<Void> startProcessRestoreRequests(Reference<RestoreMasterData> self
 	// Step: Notify all restore requests have been handled by cleaning up the restore keys
 	wait(signalRestoreCompleted(self, cx));
 
-	try {
-		wait(unlockDatabase(cx, randomUID));
-	} catch (Error& e) {
-		if (e.code() == error_code_operation_cancelled) { // Should only happen in simulation
-			TraceEvent(SevWarnAlways, "FastRestoreMasterOnCancelingActor", self->id())
-			    .detail("DBLock", randomUID)
-			    .detail("ManualCheck", "Is DB locked");
-		} else {
-			TraceEvent(SevError, "FastRestoreMasterUnlockDBFailed", self->id())
-			    .detail("DBLock", randomUID)
-			    .detail("ErrorCode", e.code())
-			    .detail("Error", e.what());
-			ASSERT_WE_THINK(false); // This unlockDatabase should always succeed, we think.
-		}
-	}
-
 	TraceEvent("FastRestoreMasterRestoreCompleted", self->id());
 
 	return Void();
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 31410125f8..496838ba3f 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -480,7 +480,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 
 				// Wait for parallel restore to finish before we can proceed
 				TraceEvent("FastRestore").detail("BackupAndParallelRestore", "WaitForRestoreToFinish");
-				wait(backupAgent.parallelRestoreFinish(cx));
+				wait(backupAgent.parallelRestoreFinish(cx, randomID));
 				TraceEvent("FastRestore").detail("BackupAndParallelRestore", "RestoreFinished");
 
 				for (auto& restore : restores) {

From 1ba11dc74bba75132dbdb8fa8c0c33046057e9d7 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Wed, 25 Mar 2020 11:20:17 -0700
Subject: [PATCH 160/176] Apply clang format

---
 fdbclient/BackupAgent.actor.h                 |  3 ++-
 fdbclient/FileBackupAgent.actor.cpp           | 27 +++++++++++--------
 fdbserver/workloads/AtomicRestore.actor.cpp   |  3 ++-
 ...kupAndParallelRestoreCorrectness.actor.cpp |  3 ++-
 4 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 13be056426..4724429d2d 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -279,7 +279,8 @@ public:
 	Future<Void> parallelRestoreFinish(Database cx, UID randomUID);
 	Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                                   KeyRef bcUrl, Version targetVersion, bool lockDB, UID randomUID);
-	Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix);
+	Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges,
+	                                   Key addPrefix, Key removePrefix);
 
 	// restore() will
 	//   - make sure that url is readable and appears to be a complete backup
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 6513eea102..4ddbf15780 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3598,8 +3598,8 @@ public:
 	}
 
 	ACTOR static Future<Void> submitParallelRestore(Database cx, Key backupTag,
-														Standalone<VectorRef<KeyRangeRef>> backupRanges, KeyRef bcUrl,
-														Version targetVersion, bool lockDB, UID randomUID) {
+	                                                Standalone<VectorRef<KeyRangeRef>> backupRanges, KeyRef bcUrl,
+	                                                Version targetVersion, bool lockDB, UID randomUID) {
 		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 		state int restoreIndex = 0;
 		state int numTries = 0;
@@ -3619,8 +3619,8 @@ public:
 			} catch (Error& e) {
 				TraceEvent("FastRestoreAgentSubmitRestoreRequests").detail("CheckLockError", e.what());
 				TraceEvent(numTries > 50 ? SevError : SevWarnAlways, "FastRestoreMayFail")
-					.detail("Reason", "DB is not properly locked")
-					.detail("ExpectedLockID", randomUID);
+				    .detail("Reason", "DB is not properly locked")
+				    .detail("ExpectedLockID", randomUID);
 				numTries++;
 				wait(delay(5.0));
 			}
@@ -3637,12 +3637,13 @@ public:
 					auto range = backupRanges[restoreIndex];
 					Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
 					// Register the request request in DB, which will be picked up by restore worker leader
-					struct RestoreRequest restoreRequest(restoreIndex, restoreTag, bcUrl, true, targetVersion, true, range,
-														Key(), Key(), lockDB, deterministicRandom()->randomUniqueID());
+					struct RestoreRequest restoreRequest(restoreIndex, restoreTag, bcUrl, true, targetVersion, true,
+					                                     range, Key(), Key(), lockDB,
+					                                     deterministicRandom()->randomUniqueID());
 					tr->set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
 				}
 				tr->set(restoreRequestTriggerKey,
-					restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), backupRanges.size()));
+				        restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), backupRanges.size()));
 				wait(tr->commit()); // Trigger restore
 				break;
 			} catch (Error& e) {
@@ -4531,8 +4532,10 @@ public:
 
 	// Similar to atomicRestore, only used in simulation test.
 	// locks the database before discontinuing the backup and that same lock is then used while doing the restore.
-	//the tagname of the backup must be the same as the restore.
-	ACTOR static Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
+	// the tagname of the backup must be the same as the restore.
+	ACTOR static Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName,
+	                                                Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix,
+	                                                Key removePrefix) {
 		Version ver = wait(atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, true));
 		return Void();
 	}
@@ -4550,10 +4553,12 @@ Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID)
 Future<Void> FileBackupAgent::submitParallelRestore(Database cx, Key backupTag,
                                                     Standalone<VectorRef<KeyRangeRef>> backupRanges, KeyRef bcUrl,
                                                     Version targetVersion, bool lockDB, UID randomUID) {
-    return FileBackupAgentImpl::submitParallelRestore(cx, backupTag, backupRanges, bcUrl, targetVersion, lockDB, randomUID);
+	return FileBackupAgentImpl::submitParallelRestore(cx, backupTag, backupRanges, bcUrl, targetVersion, lockDB,
+	                                                  randomUID);
 }
 
-Future<Void> FileBackupAgent::atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
+Future<Void> FileBackupAgent::atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges,
+                                                    Key addPrefix, Key removePrefix) {
 	return FileBackupAgentImpl::atomicParallelRestore(this, cx, tagName, ranges, addPrefix, removePrefix);
 }
 
diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.actor.cpp
index 984b6d8ef0..7a7d8d21e2 100644
--- a/fdbserver/workloads/AtomicRestore.actor.cpp
+++ b/fdbserver/workloads/AtomicRestore.actor.cpp
@@ -83,7 +83,8 @@ struct AtomicRestoreWorkload : TestWorkload {
 
 		if (self->fastRestore) { // New fast parallel restore
 			TraceEvent(SevWarnAlways, "AtomicParallelRestore");
-			wait(backupAgent.atomicParallelRestore(cx, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef()));
+			wait(backupAgent.atomicParallelRestore(cx, BackupAgentBase::getDefaultTag(), self->backupRanges,
+			                                       StringRef(), StringRef()));
 		} else { // Old style restore
 			loop {
 				std::vector<Future<Version>> restores;
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 496838ba3f..e9b9edc095 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -450,7 +450,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				// Submit parallel restore requests
 				TraceEvent("FastRestore").detail("PrepareRestores", self->backupRanges.size());
 				wait(backupAgent.submitParallelRestore(cx, self->backupTag, self->backupRanges,
-				                           KeyRef(lastBackupContainer->getURL()), targetVersion, self->locked, randomID));
+				                                       KeyRef(lastBackupContainer->getURL()), targetVersion,
+				                                       self->locked, randomID));
 				TraceEvent("FastRestore").detail("TriggerRestore", "Setting up restoreRequestTriggerKey");
 
 				// Sometimes kill and restart the restore

From f240d393f2a04614522657c19749a69c0006e5d8 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Wed, 25 Mar 2020 11:23:19 -0700
Subject: [PATCH 161/176] Add ParallelRestoreApiCorrectnessAtomicRestore.txt
 into ctests

---
 tests/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 4d3b7c5e33..86ac74c8f7 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -210,6 +210,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES slow/ParallelRestoreCorrectnessAtomicOpTinyData.txt)
   add_fdb_test(TEST_FILES slow/ParallelRestoreCorrectnessCycle.txt)
   add_fdb_test(TEST_FILES slow/ParallelRestoreCorrectnessMultiCycles.txt)
+  add_fdb_test(TEST_FILES slow/ParallelRestoreApiCorrectnessAtomicRestore.txt)
   # Note that status tests are not deterministic.
   add_fdb_test(TEST_FILES status/invalid_proc_addresses.txt)
   add_fdb_test(TEST_FILES status/local_6_machine_no_replicas_remain.txt)

From 7d5ed532153f3a0809767a2968f07732ca647a3f Mon Sep 17 00:00:00 2001
From: tclinken <trevorclinkenbeard@gmail.com>
Date: Wed, 25 Mar 2020 13:27:56 -0700
Subject: [PATCH 162/176] Allow trace log group to be set after database is
 created

---
 fdbclient/NativeAPI.actor.cpp | 17 ++++++++++++-----
 flow/Trace.cpp                |  6 ++++++
 flow/Trace.h                  |  1 +
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 273eb0d993..38836c09f0 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -842,7 +842,7 @@ const UniqueOrderedOptionList<FDBTransactionOptions>& Database::getTransactionDe
 
 void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value) {
 	switch(option) {
-		// SOMEDAY: If the network is already started, should these three throw an error?
+		// SOMEDAY: If the network is already started, should these four throw an error?
 		case FDBNetworkOptions::TRACE_ENABLE:
 			networkOptions.traceDirectory = value.present() ? value.get().toString() : "";
 			break;
@@ -854,10 +854,6 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
 			validateOptionValue(value, true);
 			networkOptions.traceMaxLogsSize = extractIntOption(value, 0, std::numeric_limits<int64_t>::max());
 			break;
-		case FDBNetworkOptions::TRACE_LOG_GROUP:
-			if(value.present())
-				networkOptions.traceLogGroup = value.get().toString();
-			break;
 		case FDBNetworkOptions::TRACE_FORMAT:
 			validateOptionValue(value, true);
 			networkOptions.traceFormat = value.get().toString();
@@ -866,6 +862,17 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
 				throw invalid_option_value();
 			}
 			break;
+
+		case FDBNetworkOptions::TRACE_LOG_GROUP:
+			if(value.present()) {
+				if (traceFileIsOpen()) {
+					setTraceLogGroup(value.get().toString());
+				}
+				else {
+					networkOptions.traceLogGroup = value.get().toString();
+				}
+			}
+			break;
 		case FDBNetworkOptions::TRACE_CLOCK_SOURCE:
 			validateOptionValue(value, true);
 			networkOptions.traceClockSource = value.get().toString();
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index 4ce0b96e8a..00053493f8 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -550,6 +550,8 @@ public:
 		}
 	}
 
+	void setLogGroup(const std::string& logGroup) { this->logGroup = logGroup; }
+
 	Future<Void> pingWriterThread() {
 		auto ping = new WriterThread::Ping;
 		auto f = ping->ack.getFuture();
@@ -750,6 +752,10 @@ void removeTraceRole(std::string role) {
 	g_traceLog.removeRole(role);
 }
 
+void setTraceLogGroup(const std::string& logGroup) {
+	g_traceLog.setLogGroup(logGroup);
+}
+
 TraceEvent::TraceEvent() : initialized(true), enabled(false), logged(true) {}
 
 TraceEvent::TraceEvent(TraceEvent &&ev) {
diff --git a/flow/Trace.h b/flow/Trace.h
index 547b4fa653..0335f04535 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -608,6 +608,7 @@ bool validateTraceClockSource(std::string source);
 void addTraceRole(std::string role);
 void removeTraceRole(std::string role);
 void retriveTraceLogIssues(std::set<std::string>& out);
+void setTraceLogGroup(const std::string& role);
 template <class T>
 struct Future;
 struct Void;

From 495afe2e0b14cc3a8bdfd780da1c9fa772c15e3c Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Wed, 25 Mar 2020 12:56:09 -0700
Subject: [PATCH 163/176] Improve how to wati for restore to finish

Remove default parameter for atomicRestore as suggested in review.
---
 fdbbackup/backup.actor.cpp          | 10 +++++++---
 fdbclient/FileBackupAgent.actor.cpp |  9 +++++----
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 07e0f9887c..5197fc0bee 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -3879,7 +3879,7 @@ ACTOR static Future<FileBackupAgent::ERestoreState> waitFastRestore(Database cx,
 	// We should wait on all restore to finish before proceeds
 	TraceEvent("FastRestore").detail("Progress", "WaitForRestoreToFinish");
 	state ReadYourWritesTransaction tr(cx);
-	state Future<Void> watchForRestoreRequestDone;
+	state Future<Void> fRestoreRequestDone;
 	state bool restoreRequestDone = false;
 
 	loop {
@@ -3887,6 +3887,7 @@ ACTOR static Future<FileBackupAgent::ERestoreState> waitFastRestore(Database cx,
 			tr.reset();
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			// In case restoreRequestDoneKey is already set before we set watch on it
 			Optional<Value> restoreRequestDoneKeyValue = wait(tr.get(restoreRequestDoneKey));
 			if (restoreRequestDoneKeyValue.present()) {
@@ -3894,9 +3895,12 @@ ACTOR static Future<FileBackupAgent::ERestoreState> waitFastRestore(Database cx,
 				tr.clear(restoreRequestDoneKey);
 				wait(tr.commit());
 				break;
-			} else {
-				watchForRestoreRequestDone = tr.watch(restoreRequestDoneKey);
+			} else if (!restoreRequestDone) {
+				fRestoreRequestDone = tr.watch(restoreRequestDoneKey);
 				wait(tr.commit());
+				wait(fRestoreRequestDone);
+			} else {
+				break;
 			}
 			// The clear transaction may fail in uncertain state, which may already clear the restoreRequestDoneKey
 			if (restoreRequestDone) break;
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 4ddbf15780..09efb46001 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3556,10 +3556,10 @@ public:
 		TraceEvent("FastRestoreAgentWaitForRestoreToFinish").detail("DBLock", randomUID);
 		loop {
 			try {
-				if (restoreDone) break;
 				tr.reset();
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				Optional<Value> restoreRequestDoneKeyValue = wait(tr.get(restoreRequestDoneKey));
 				// Restore may finish before restoreAgent waits on the restore finish event.
 				if (restoreRequestDoneKeyValue.present()) {
@@ -3567,10 +3567,11 @@ public:
 					tr.clear(restoreRequestDoneKey);
 					wait(tr.commit());
 					break;
-				} else {
+				} else if (!restoreDone) {
 					watchForRestoreRequestDone = tr.watch(restoreRequestDoneKey);
 					wait(tr.commit());
 					wait(watchForRestoreRequestDone);
+				} else {
 					break;
 				}
 			} catch (Error& e) {
@@ -4419,7 +4420,7 @@ public:
 	//the tagname of the backup must be the same as the restore.
 	ACTOR static Future<Version> atomicRestore(FileBackupAgent* backupAgent, Database cx, Key tagName,
 	                                           Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix,
-	                                           Key removePrefix, bool fastRestore = false) {
+	                                           Key removePrefix, bool fastRestore) {
 		state Reference<ReadYourWritesTransaction> ryw_tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
 		state BackupConfig backupConfig;
 		loop {
@@ -4567,7 +4568,7 @@ Future<Version> FileBackupAgent::restore(Database cx, Optional<Database> cxOrig,
 }
 
 Future<Version> FileBackupAgent::atomicRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix, Key removePrefix) {
-	return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix);
+	return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix, false);
 }
 
 Future<ERestoreState> FileBackupAgent::abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName) {

From a93f13cfd7220341cc6411cd2159382aff3e04fa Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Wed, 25 Mar 2020 15:19:46 -0700
Subject: [PATCH 164/176] Remove redundant restoreRequestDone break in
 backup.actor

---
 fdbbackup/backup.actor.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 5197fc0bee..f6a46c6267 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -3902,8 +3902,6 @@ ACTOR static Future<FileBackupAgent::ERestoreState> waitFastRestore(Database cx,
 			} else {
 				break;
 			}
-			// The clear transaction may fail in uncertain state, which may already clear the restoreRequestDoneKey
-			if (restoreRequestDone) break;
 		} catch (Error& e) {
 			wait(tr.onError(e));
 		}

From 0f57bf96859d820a723e7784635e39e0a5d8c358 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 25 Mar 2020 15:23:21 -0700
Subject: [PATCH 165/176] Remove a SevError event

The same mutation can be present in overlapping mutation logs. Thus we cannot
assert its absence. This can be caused for multiple reasons. One possibility
is that new TLogs can copy mutations from old generation TLogs; another one
is backup worker is recruited without knowning previously saved progress.
---
 fdbserver/RestoreApplier.actor.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h
index 47b596963e..2d2cf69d1c 100644
--- a/fdbserver/RestoreApplier.actor.h
+++ b/fdbserver/RestoreApplier.actor.h
@@ -66,9 +66,13 @@ struct StagingKey {
 			    .detail("NewVersion", newVersion.toString())
 			    .detail("Mutation", m.toString());
 		}
-		if (version == newVersion) { // Sanity check
+		if (version == newVersion) {
+			// This could happen because the same mutation can be present in
+			// overlapping mutation logs, because new TLogs can copy mutations
+			// from old generation TLogs (or backup worker is recruited without
+			// knowning previously saved progress).
 			ASSERT(type == m.type && key == m.param1 && val == m.param2);
-			TraceEvent(SevError, "SameVersion").detail("Version", version.toString()).detail("Mutation", m.toString());
+			TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m.toString());
 			return;
 		}
 

From baf0fe956c1b1de06a8c60e571ad70a21dc6c65b Mon Sep 17 00:00:00 2001
From: tclinken <trevorclinkenbeard@gmail.com>
Date: Thu, 26 Mar 2020 09:55:03 -0700
Subject: [PATCH 166/176] Take trace mutex in setLogGroup

---
 flow/Trace.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index 00053493f8..82e2d7f662 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -550,7 +550,10 @@ public:
 		}
 	}
 
-	void setLogGroup(const std::string& logGroup) { this->logGroup = logGroup; }
+	void setLogGroup(const std::string& logGroup) {
+		MutexHolder holder(mutex);
+		this->logGroup = logGroup;
+	}
 
 	Future<Void> pingWriterThread() {
 		auto ping = new WriterThread::Ping;

From cdb6bbfc8530c4c6cfebdaeef1c1fccbf95e00ab Mon Sep 17 00:00:00 2001
From: Andrew Noyes <anoyes34@gmail.com>
Date: Tue, 19 Nov 2019 16:29:02 -0800
Subject: [PATCH 167/176] Test watch outliving transaction

---
 fdbclient/NativeAPI.actor.cpp         |  1 +
 fdbserver/workloads/Watches.actor.cpp | 33 ++++++++++++++++++---------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 273eb0d993..0058b59661 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -2151,6 +2151,7 @@ ACTOR Future<Void> watch( Reference<Watch> watch, Database cx, Transaction *self
 	cx->addWatch();
 	try {
 		self->watches.push_back(watch);
+		self = nullptr; // This actor may outlive *self
 
 		choose {
 			// RYOW write to value that is being watched (if applicable)
diff --git a/fdbserver/workloads/Watches.actor.cpp b/fdbserver/workloads/Watches.actor.cpp
index acac709a78..966f1ffd99 100644
--- a/fdbserver/workloads/Watches.actor.cpp
+++ b/fdbserver/workloads/Watches.actor.cpp
@@ -129,38 +129,49 @@ struct WatchesWorkload : TestWorkload {
 		state Optional<Optional<Value>> lastValue;
 
 		loop {
-			state Transaction tr( cx );
 			loop {
+				state std::unique_ptr<Transaction> tr = std::make_unique<Transaction>(cx);
 				try {
-					state Future<Optional<Value>> setValueFuture = tr.get( setKey );
-					state Optional<Value> watchValue = wait( tr.get( watchKey ) );
+					state Future<Optional<Value>> setValueFuture = tr->get(setKey);
+					state Optional<Value> watchValue = wait(tr->get(watchKey));
 					Optional<Value> setValue = wait( setValueFuture );
 					
 					if( lastValue.present() && lastValue.get() == watchValue) {
-						TraceEvent(SevError, "WatcherTriggeredWithoutChanging").detail("WatchKey", printable( watchKey )).detail("SetKey", printable( setKey )).detail("WatchValue", printable( watchValue )).detail("SetValue", printable( setValue )).detail("ReadVersion", tr.getReadVersion().get());
+						TraceEvent(SevError, "WatcherTriggeredWithoutChanging")
+						    .detail("WatchKey", printable(watchKey))
+						    .detail("SetKey", printable(setKey))
+						    .detail("WatchValue", printable(watchValue))
+						    .detail("SetValue", printable(setValue))
+						    .detail("ReadVersion", tr->getReadVersion().get());
 					}
 
 					lastValue = Optional<Optional<Value>>();
 
 					if( watchValue != setValue ) {
 						if( watchValue.present() )
-							tr.set( setKey, watchValue.get() );
+							tr->set(setKey, watchValue.get());
 						else
-							tr.clear( setKey );
+							tr->clear(setKey);
 						//TraceEvent("WatcherSetStart").detail("Watch", printable(watchKey)).detail("Set", printable(setKey)).detail("Value", printable( watchValue ) );
-						wait( tr.commit() );
-						//TraceEvent("WatcherSetFinish").detail("Watch", printable(watchKey)).detail("Set", printable(setKey)).detail("Value", printable( watchValue ) ).detail("Ver", tr.getCommittedVersion());
+						wait(tr->commit());
+						//TraceEvent("WatcherSetFinish").detail("Watch", printable(watchKey)).detail("Set", printable(setKey)).detail("Value", printable( watchValue ) ).detail("Ver", tr->getCommittedVersion());
 					} else {
 						//TraceEvent("WatcherWatch").detail("Watch", printable(watchKey));
-						state Future<Void> watchFuture = tr.watch( Reference<Watch>( new Watch(watchKey, watchValue) ) );
-						wait( tr.commit() );
+						state Future<Void> watchFuture = tr->watch(Reference<Watch>(new Watch(watchKey, watchValue)));
+						wait(tr->commit());
+						if (BUGGIFY) {
+							// Make watch future outlive transaction
+							tr.reset();
+						}
 						wait( watchFuture );
 						if( watchValue.present() )
 							lastValue = watchValue;
 					}
 					break;
 				} catch( Error &e ) {
-					wait( tr.onError(e) );
+					if (tr != nullptr) {
+						wait(tr->onError(e));
+					}
 				}
 			}
 		}

From f6af6ec77749856b8ebb1ea9f167ae161cf61ab1 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <anoyes34@gmail.com>
Date: Thu, 26 Mar 2020 10:29:23 -0700
Subject: [PATCH 168/176] Avoid passing self to watch actor

---
 fdbclient/NativeAPI.actor.cpp | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 0058b59661..f1d129614d 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -2146,13 +2146,9 @@ void Watch::setWatch(Future<Void> watchFuture) {
 }
 
 //FIXME: This seems pretty horrible. Now a Database can't die until all of its watches do...
-ACTOR Future<Void> watch( Reference<Watch> watch, Database cx, Transaction *self ) {
-	state TransactionInfo info = self->info;
+ACTOR Future<Void> watch(Reference<Watch> watch, Database cx, TransactionInfo info) {
 	cx->addWatch();
 	try {
-		self->watches.push_back(watch);
-		self = nullptr; // This actor may outlive *self
-
 		choose {
 			// RYOW write to value that is being watched (if applicable)
 			// Errors
@@ -2191,7 +2187,9 @@ Future<Version> Transaction::getRawReadVersion() {
 
 Future< Void > Transaction::watch( Reference<Watch> watch ) {
 	++cx->transactionWatchRequests;
-	return ::watch(watch, cx, this);
+	cx->addWatch();
+	watches.push_back(watch);
+	return ::watch(watch, cx, info);
 }
 
 ACTOR Future<Standalone<VectorRef<const char*>>> getAddressesForKeyActor(Key key, Future<Version> ver, Database cx,

From 772ab70aeea6170342ce8249f3e364afc76b745a Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 25 Mar 2020 22:53:22 -0700
Subject: [PATCH 169/176] Add an option for fast restore to restore old backups

If "usePartitionedLogs" is set to false, then the workload uses old backups for
restore.
---
 fdbclient/BackupContainer.actor.cpp              | 12 ++++++++++++
 fdbclient/BackupContainer.h                      |  7 +++++++
 fdbserver/RestoreMaster.actor.cpp                |  8 ++++----
 ...BackupAndParallelRestoreCorrectness.actor.cpp | 16 +++++++++++-----
 4 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index e6a7c62680..34e3e753a3 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -657,6 +657,18 @@ public:
 		return dumpFileList_impl(Reference<BackupContainerFileSystem>::addRef(this), begin, end);
 	}
 
+	ACTOR static Future<bool> isPartitionedBackup_impl(Reference<BackupContainerFileSystem> bc) {
+		BackupFileList list = wait(bc->dumpFileList(0, std::numeric_limits<Version>::max()));
+		for (const auto& file : list.logs) {
+			if (file.isPartitionedLog()) return true;
+		}
+		return false;
+	}
+
+	Future<bool> isPartitionedBackup() final {
+		return isPartitionedBackup_impl(Reference<BackupContainerFileSystem>::addRef(this));
+	}
+
 	static Version resolveRelativeVersion(Optional<Version> max, Version v, const char *name, Error e) {
 		if(v == invalidVersion) {
 			TraceEvent(SevError, "BackupExpireInvalidVersion").detail(name, v);
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 3b1f5de5bf..d134d53887 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -88,6 +88,10 @@ struct LogFile {
 		return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
 	}
 
+	bool isPartitionedLog() const {
+		return tagId >= 0 && tagId < totalTags;
+	}
+
 	std::string toString() const {
 		std::stringstream ss;
 		ss << "beginVersion:" << std::to_string(beginVersion) << " endVersion:" << std::to_string(endVersion)
@@ -261,6 +265,9 @@ public:
 
 	virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0;
 
+	// If there are partitioned log files, then returns true; otherwise, returns false.
+	virtual Future<bool> isPartitionedBackup() = 0;
+
 	// Get exactly the files necessary to restore to targetVersion.  Returns non-present if
 	// restore to given version is not possible.
 	virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) = 0;
diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp
index 6bd7fe9506..2e1a5a998b 100644
--- a/fdbserver/RestoreMaster.actor.cpp
+++ b/fdbserver/RestoreMaster.actor.cpp
@@ -641,7 +641,8 @@ ACTOR static Future<Standalone<VectorRef<RestoreRequest>>> collectRestoreRequest
 ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
                                                 std::vector<RestoreFileFR>* logFiles, Database cx,
                                                 RestoreRequest request) {
-	state BackupDescription desc = wait(bc->describePartitionedBackup());
+	state bool partitioned = wait(bc->isPartitionedBackup());
+	state BackupDescription desc = wait(partitioned ? bc->describePartitionedBackup() : bc->describeBackup());
 
 	// Convert version to real time for operators to read the BackupDescription desc.
 	wait(desc.resolveVersionTimes(cx));
@@ -657,9 +658,8 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
 		std::cout << "Restore to version: " << request.targetVersion << "\nBackupDesc: \n" << desc.toString() << "\n\n";
 	}
 
-	Optional<RestorableFileSet> restorable =
-	    wait(SERVER_KNOBS->FASTRESTORE_USE_PARTITIONED_LOGS ? bc->getPartitionedRestoreSet(request.targetVersion)
-	                                                        : bc->getRestoreSet(request.targetVersion));
+	Optional<RestorableFileSet> restorable = wait(partitioned ? bc->getPartitionedRestoreSet(request.targetVersion)
+	                                                          : bc->getRestoreSet(request.targetVersion));
 
 	if (!restorable.present()) {
 		TraceEvent(SevWarn, "FastRestoreMasterPhaseCollectBackupFiles").detail("NotRestorable", request.targetVersion);
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index c9b5bcf0d9..49496d8750 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -21,6 +21,7 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
 #include "fdbclient/RestoreWorkerInterface.actor.h"
@@ -40,6 +41,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 	bool locked;
 	bool allowPauses;
 	bool shareLogRange;
+	bool usePartitionedLogs;
 
 	std::map<Standalone<KeyRef>, Standalone<ValueRef>> dbKVs;
 
@@ -67,6 +69,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 		agentRequest = getOption(options, LiteralStringRef("simBackupAgents"), true);
 		allowPauses = getOption(options, LiteralStringRef("allowPauses"), true);
 		shareLogRange = getOption(options, LiteralStringRef("shareLogRange"), false);
+		usePartitionedLogs = getOption(options, LiteralStringRef("usePartitionedLogs"), true);
 
 		KeyRef beginRange;
 		KeyRef endRange;
@@ -181,7 +184,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 		try {
 			wait(backupAgent->submitBackup(cx, StringRef(backupContainer), deterministicRandom()->randomInt(0, 100),
 			                               tag.toString(), backupRanges, stopDifferentialDelay ? false : true,
-			                               /*partitionedLog=*/true));
+			                               /*partitionedLog=*/self->usePartitionedLogs));
 		} catch (Error& e) {
 			TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) throw;
@@ -209,8 +212,10 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 					TraceEvent("BARW_DoBackupWaitForRestorable", randomID).detail("Tag", backupTag.tagName).detail("Result", resultWait);
 
 					state bool restorable = false;
-					if(lastBackupContainer) {
-						state Future<BackupDescription> fdesc = lastBackupContainer->describePartitionedBackup();
+					if (lastBackupContainer) {
+						state Future<BackupDescription> fdesc = self->usePartitionedLogs
+						                                            ? lastBackupContainer->describePartitionedBackup()
+						                                            : lastBackupContainer->describeBackup();
 						wait(ready(fdesc));
 
 						if(!fdesc.isError()) {
@@ -398,7 +403,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				try {
 					extraBackup = backupAgent.submitBackup(
 					    cx, LiteralStringRef("file://simfdb/backups/"), deterministicRandom()->randomInt(0, 100),
-					    self->backupTag.toString(), self->backupRanges, true, /*partitionedLog=*/true);
+					    self->backupTag.toString(), self->backupRanges, true, self->usePartitionedLogs);
 				} catch (Error& e) {
 					TraceEvent("BARW_SubmitBackup2Exception", randomID)
 					    .error(e)
@@ -431,7 +436,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				    .detail("BackupTag", printable(self->backupTag));
 
 				auto container = IBackupContainer::openContainer(lastBackupContainer->getURL());
-				BackupDescription desc = wait(container->describePartitionedBackup());
+				BackupDescription desc = wait(self->usePartitionedLogs ? container->describePartitionedBackup()
+				                                                       : container->describeBackup());
 
 				state Version targetVersion = -1;
 				if (desc.maxRestorableVersion.present()) {

From 40b17e1e9be496a4862665949fa6a68d5ae5c0d2 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 25 Mar 2020 23:04:35 -0700
Subject: [PATCH 170/176] Remove a no longer unused knob

---
 fdbserver/Knobs.cpp | 1 -
 fdbserver/Knobs.h   | 1 -
 2 files changed, 2 deletions(-)

diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp
index 3e57063935..a7f97190e6 100644
--- a/fdbserver/Knobs.cpp
+++ b/fdbserver/Knobs.cpp
@@ -570,7 +570,6 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs, bool isSimula
 	init( FASTRESTORE_APPLYING_PARALLELISM,               	     100 ); if( randomize ) { FASTRESTORE_APPLYING_PARALLELISM = deterministicRandom()->random01() * 10 + 1; }
 	init( FASTRESTORE_MONITOR_LEADER_DELAY,                        5 ); if( randomize ) { FASTRESTORE_MONITOR_LEADER_DELAY = deterministicRandom()->random01() * 100; }
 	init( FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS,                60 ); if( randomize && BUGGIFY ) { FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS = deterministicRandom()->random01() * 240 + 10; }
-	init( FASTRESTORE_USE_PARTITIONED_LOGS,                     true );
 	init( FASTRESTORE_TRACK_REQUEST_LATENCY,              	    true ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_REQUEST_LATENCY = false; }
 	init( FASTRESTORE_TRACK_LOADER_SEND_REQUESTS,              false ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_LOADER_SEND_REQUESTS = true; }
 	init( FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT,                 6144 ); if( randomize && BUGGIFY ) { FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT = 1; }
diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h
index afc165e77e..6703c53375 100644
--- a/fdbserver/Knobs.h
+++ b/fdbserver/Knobs.h
@@ -512,7 +512,6 @@ public:
 	int64_t FASTRESTORE_APPLYING_PARALLELISM; // number of outstanding txns writing to dest. DB
 	int64_t FASTRESTORE_MONITOR_LEADER_DELAY;
 	int64_t FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS;
-	bool FASTRESTORE_USE_PARTITIONED_LOGS;
 	bool FASTRESTORE_TRACK_REQUEST_LATENCY; // true to track reply latency of each request in a request batch
 	bool FASTRESTORE_TRACK_LOADER_SEND_REQUESTS; // track requests of load send mutations to appliers?
 	int64_t FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT; // threshold when pipelined actors should be delayed

From 99f4ef6e0ce488a737b19e18cc68cfa20c34c5cb Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 26 Mar 2020 10:03:11 -0700
Subject: [PATCH 171/176] Fix restore loader to handle mutation sub number

For old backup format, give them a sub sequence number starting from 0 for each
commit version.
---
 fdbserver/RestoreLoader.actor.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 1a9931ef91..fadda9e28c 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -680,8 +680,6 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 		uint64_t commitVersion = kReader.consume<uint64_t>(); // Consume little Endian data
 		// We have already filter the commit not in [beginVersion, endVersion) when we concatenate kv pair in log file
 		ASSERT_WE_THINK(asset.isInVersionRange(commitVersion));
-		auto it = kvOps.insert(std::make_pair(LogMessageVersion(commitVersion), MutationsVec()));
-		ASSERT(it.second); // inserted is true
 
 		StringRefReader vReader(val, restore_corrupted_data());
 		vReader.consume<uint64_t>(); // Consume the includeVersion
@@ -691,6 +689,7 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 		uint32_t val_length_decoded = vReader.consume<uint32_t>();
 		ASSERT(val_length_decoded == val.size() - sizeof(uint64_t) - sizeof(uint32_t));
 
+		int sub = 0;
 		while (1) {
 			// stop when reach the end of the string
 			if (vReader.eof()) { //|| *reader.rptr == 0xFF
@@ -721,7 +720,11 @@ void _parseSerializedMutation(std::map<LoadingParam, VersionedMutationsMap>::ite
 			TraceEvent(SevFRMutationInfo, "FastRestore_VerboseDebug")
 			    .detail("CommitVersion", commitVersion)
 			    .detail("ParsedMutation", mutation.toString());
+
+			auto it = kvOps.insert(std::make_pair(LogMessageVersion(commitVersion, sub++), MutationsVec()));
+			ASSERT(it.second); // inserted is true
 			it.first->second.push_back_deep(it.first->second.arena(), mutation);
+
 			// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
 			if (deterministicRandom()->random01() * 100 < SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT) {
 				samples.push_back_deep(samples.arena(), mutation);

From aca458cd96a4db7b79533a60e7d6e66000270fcb Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 26 Mar 2020 10:25:24 -0700
Subject: [PATCH 172/176] Set 50% chance to restore old backup files for fast
 restore

---
 .../workloads/BackupAndParallelRestoreCorrectness.actor.cpp    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index 49496d8750..e448f17a81 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -69,7 +69,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 		agentRequest = getOption(options, LiteralStringRef("simBackupAgents"), true);
 		allowPauses = getOption(options, LiteralStringRef("allowPauses"), true);
 		shareLogRange = getOption(options, LiteralStringRef("shareLogRange"), false);
-		usePartitionedLogs = getOption(options, LiteralStringRef("usePartitionedLogs"), true);
+		usePartitionedLogs = getOption(options, LiteralStringRef("usePartitionedLogs"),
+		                               deterministicRandom()->random01() < 0.5 ? true : false);
 
 		KeyRef beginRange;
 		KeyRef endRange;

From 6be913a430f736fd613e03c51fabd25fc6bbb8e6 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 26 Mar 2020 11:08:57 -0700
Subject: [PATCH 173/176] Add partitioned logs option to AtomicRestore workload

---
 fdbserver/workloads/AtomicRestore.actor.cpp            | 10 +++++++---
 .../BackupAndParallelRestoreCorrectness.actor.cpp      |  3 +--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.actor.cpp
index 7a7d8d21e2..bcab956916 100644
--- a/fdbserver/workloads/AtomicRestore.actor.cpp
+++ b/fdbserver/workloads/AtomicRestore.actor.cpp
@@ -29,6 +29,7 @@ struct AtomicRestoreWorkload : TestWorkload {
 	double startAfter, restoreAfter;
 	bool fastRestore; // true: use fast restore, false: use old style restore
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
+	bool usePartitionedLogs;
 
 	AtomicRestoreWorkload(WorkloadContext const& wcx)
 		: TestWorkload(wcx) {
@@ -37,6 +38,8 @@ struct AtomicRestoreWorkload : TestWorkload {
 		restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 20.0);
 		fastRestore = getOption(options, LiteralStringRef("fastRestore"), false);
 		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
+		usePartitionedLogs = getOption(options, LiteralStringRef("usePartitionedLogs"),
+		                               deterministicRandom()->random01() < 0.5 ? true : false);
 	}
 
 	virtual std::string description() {
@@ -68,9 +71,10 @@ struct AtomicRestoreWorkload : TestWorkload {
 
 		state std::string backupContainer = "file://simfdb/backups/";
 		try {
-			wait(backupAgent.submitBackup(cx, StringRef(backupContainer), deterministicRandom()->randomInt(0, 100), BackupAgentBase::getDefaultTagName(), self->backupRanges, false));
-		}
-		catch (Error& e) {
+			wait(backupAgent.submitBackup(cx, StringRef(backupContainer), deterministicRandom()->randomInt(0, 100),
+			                              BackupAgentBase::getDefaultTagName(), self->backupRanges, false,
+			                              self->usePartitionedLogs));
+		} catch (Error& e) {
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 				throw;
 		}
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
index e448f17a81..f9ab0bed51 100644
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
@@ -21,7 +21,6 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
-#include "fdbserver/Knobs.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
 #include "fdbclient/RestoreWorkerInterface.actor.h"
@@ -185,7 +184,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 		try {
 			wait(backupAgent->submitBackup(cx, StringRef(backupContainer), deterministicRandom()->randomInt(0, 100),
 			                               tag.toString(), backupRanges, stopDifferentialDelay ? false : true,
-			                               /*partitionedLog=*/self->usePartitionedLogs));
+			                               self->usePartitionedLogs));
 		} catch (Error& e) {
 			TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) throw;

From cffcac92ad8461a5122d6b3cac5337b5eb307403 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 26 Mar 2020 13:02:29 -0700
Subject: [PATCH 174/176] Disable partitioned log for atomic restore workload

This will be reenabled after fixing test failures related to it.
---
 tests/slow/ApiCorrectnessAtomicRestore.txt                | 3 ++-
 tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt | 1 +
 tests/slow/WriteDuringReadAtomicRestore.txt               | 3 ++-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/slow/ApiCorrectnessAtomicRestore.txt b/tests/slow/ApiCorrectnessAtomicRestore.txt
index 9bdab100a7..4d19ff8b0b 100644
--- a/tests/slow/ApiCorrectnessAtomicRestore.txt
+++ b/tests/slow/ApiCorrectnessAtomicRestore.txt
@@ -25,4 +25,5 @@ testName=AtomicRestore
 startAfter=10.0
 restoreAfter=50.0
 clearAfterTest=false
-simBackupAgents=BackupToFile
\ No newline at end of file
+simBackupAgents=BackupToFile
+usePartitionedLogs=false
diff --git a/tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt b/tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt
index bcb6594fab..bd78b8e0d8 100644
--- a/tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt
+++ b/tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt
@@ -27,6 +27,7 @@ restoreAfter=50.0
 clearAfterTest=false
 simBackupAgents=BackupToFile
 fastRestore=true
+usePartitionedLogs=false
 
 ; Each testName=RunRestoreWorkerWorkload creates a restore worker
 ; We need at least 3 restore workers: master, loader, and applier
diff --git a/tests/slow/WriteDuringReadAtomicRestore.txt b/tests/slow/WriteDuringReadAtomicRestore.txt
index 2486e9de83..de0cfce799 100644
--- a/tests/slow/WriteDuringReadAtomicRestore.txt
+++ b/tests/slow/WriteDuringReadAtomicRestore.txt
@@ -11,7 +11,8 @@ testTitle=WriteDuringReadTest
     restoreAfter=50.0
     clearAfterTest=false
     simBackupAgents=BackupToFile
-    
+    usePartitionedLogs=false
+
     testName=RandomClogging
     testDuration=60.0
 

From 9a9af7d8a8546ca6584941630abe8d143a3eb1c7 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 26 Mar 2020 13:57:31 -0700
Subject: [PATCH 175/176] Add more trace event details on partitioned log

---
 fdbclient/FileBackupAgent.actor.cpp         | 7 ++++---
 fdbserver/workloads/AtomicRestore.actor.cpp | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 641190a6c9..2f76bad44e 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3719,9 +3719,10 @@ public:
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
 
 		TraceEvent(SevInfo, "FBA_SubmitBackup")
-				.detail("TagName", tagName.c_str())
-				.detail("StopWhenDone", stopWhenDone)
-				.detail("OutContainer", outContainer.toString());
+		    .detail("TagName", tagName.c_str())
+		    .detail("StopWhenDone", stopWhenDone)
+		    .detail("UsePartitionedLog", partitionedLog)
+		    .detail("OutContainer", outContainer.toString());
 
 		state KeyBackedTag tag = makeBackupTag(tagName);
 		Optional<UidAndAbortedFlagT> uidAndAbortedFlag = wait(tag.get(tr));
diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.actor.cpp
index bcab956916..3bb64d4d75 100644
--- a/fdbserver/workloads/AtomicRestore.actor.cpp
+++ b/fdbserver/workloads/AtomicRestore.actor.cpp
@@ -67,7 +67,7 @@ struct AtomicRestoreWorkload : TestWorkload {
 		state FileBackupAgent backupAgent;
 
 		wait( delay(self->startAfter * deterministicRandom()->random01()) );
-		TraceEvent("AtomicRestore_Start");
+		TraceEvent("AtomicRestore_Start").detail("UsePartitionedLog", self->usePartitionedLogs);
 
 		state std::string backupContainer = "file://simfdb/backups/";
 		try {

From 96a7b4b459dd9da383f8b17c09636d0ed99f2941 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 26 Mar 2020 14:35:29 -0700
Subject: [PATCH 176/176] Disable ParallelRestoreApiCorrectnessAtomicRestore
 test

---
 tests/CMakeLists.txt                                            | 2 +-
 tests/{slow => }/ParallelRestoreApiCorrectnessAtomicRestore.txt | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tests/{slow => }/ParallelRestoreApiCorrectnessAtomicRestore.txt (100%)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 86ac74c8f7..d019c85a88 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -56,6 +56,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES KVStoreTestWrite.txt UNIT IGNORE)
   add_fdb_test(TEST_FILES KVStoreValueSize.txt UNIT IGNORE)
   add_fdb_test(TEST_FILES LayerStatusMerge.txt IGNORE)
+  add_fdb_test(TEST_FILES ParallelRestoreApiCorrectnessAtomicRestore.txt IGNORE)
   add_fdb_test(TEST_FILES PureNetwork.txt IGNORE)
   add_fdb_test(TEST_FILES RRW2500.txt IGNORE)
   add_fdb_test(TEST_FILES RandomRead.txt IGNORE)
@@ -210,7 +211,6 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES slow/ParallelRestoreCorrectnessAtomicOpTinyData.txt)
   add_fdb_test(TEST_FILES slow/ParallelRestoreCorrectnessCycle.txt)
   add_fdb_test(TEST_FILES slow/ParallelRestoreCorrectnessMultiCycles.txt)
-  add_fdb_test(TEST_FILES slow/ParallelRestoreApiCorrectnessAtomicRestore.txt)
   # Note that status tests are not deterministic.
   add_fdb_test(TEST_FILES status/invalid_proc_addresses.txt)
   add_fdb_test(TEST_FILES status/local_6_machine_no_replicas_remain.txt)
diff --git a/tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt b/tests/ParallelRestoreApiCorrectnessAtomicRestore.txt
similarity index 100%
rename from tests/slow/ParallelRestoreApiCorrectnessAtomicRestore.txt
rename to tests/ParallelRestoreApiCorrectnessAtomicRestore.txt