From fd7198d874ba0550c45e75c75e57a04fa65cd7b1 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sat, 29 Aug 2020 19:53:04 -0700 Subject: [PATCH 01/13] Extend backup container interface to support query restorable files set by key ranges --- fdbbackup/backup.actor.cpp | 138 +++++++++++++++++++++++++++- fdbclient/BackupContainer.actor.cpp | 80 +++++++++++----- fdbclient/BackupContainer.h | 8 +- fdbclient/FDBTypes.h | 9 ++ fdbclient/NativeAPI.actor.cpp | 6 ++ 5 files changed, 208 insertions(+), 33 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 0eac955518..9d2a204bdc 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -18,6 +18,7 @@ * limitations under the License. */ +#include "flow/Arena.h" #define BOOST_DATE_TIME_NO_LIB #include @@ -81,7 +82,22 @@ enum enumProgramExe { }; enum enumBackupType { - BACKUP_UNDEFINED=0, BACKUP_START, BACKUP_MODIFY, BACKUP_STATUS, BACKUP_ABORT, BACKUP_WAIT, BACKUP_DISCONTINUE, BACKUP_PAUSE, BACKUP_RESUME, BACKUP_EXPIRE, BACKUP_DELETE, BACKUP_DESCRIBE, BACKUP_LIST, BACKUP_DUMP, BACKUP_CLEANUP + BACKUP_UNDEFINED = 0, + BACKUP_START, + BACKUP_MODIFY, + BACKUP_STATUS, + BACKUP_ABORT, + BACKUP_WAIT, + BACKUP_DISCONTINUE, + BACKUP_PAUSE, + BACKUP_RESUME, + BACKUP_EXPIRE, + BACKUP_DELETE, + BACKUP_DESCRIBE, + BACKUP_LIST, + BACKUP_QUERY, + BACKUP_DUMP, + BACKUP_CLEANUP }; enum enumDBType { @@ -585,6 +601,38 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = { SO_END_OF_OPTIONS }; +CSimpleOpt::SOption g_rgBackupQueryOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "-rv", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "--restore_version", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, +#ifndef TLS_DISABLED + TLS_OPTION_FLAGS +#endif + SO_END_OF_OPTIONS +}; + // g_rgRestoreOptions is used by fdbrestore and fastrestore_tool CSimpleOpt::SOption g_rgRestoreOptions[] = { #ifdef _WIN32 @@ -918,13 +966,16 @@ void printBackupContainerInfo() { static void printBackupUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | list | cleanup) [OPTIONS]\n\n", exeBackup.toString().c_str()); + printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | " + "list | query | cleanup) [OPTIONS]\n\n", + exeBackup.toString().c_str()); printf(" -C CONNFILE The path of a file containing the connection string for the\n" " FoundationDB cluster. The default is first the value of the\n" " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" " then `%s'.\n", platform::getDefaultClusterFilePath().c_str()); printf(" -d, --destcontainer URL\n" - " The Backup container URL for start, modify, describe, expire, and delete operations.\n"); + " The Backup container URL for start, modify, describe, query, expire, and delete " + "operations.\n"); printBackupContainerInfo(); printf(" -b, --base_url BASEURL\n" " Base backup URL for list operations. This looks like a Backup URL but without a backup name.\n"); @@ -956,8 +1007,8 @@ static void printBackupUsage(bool devhelp) { " Specifies a UID to verify against the BackupUID of the running backup. If provided, the UID is verified in the same transaction\n" " which sets the new backup parameters (if the UID matches).\n"); printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); - printf(" -k KEYS List of key ranges to backup.\n" - " If not specified, the entire database will be backed up.\n"); + printf(" -k KEYS List of key ranges to backup or to filter the backup.\n" + " If not specified, the entire database will be backed up or no filter will be applied.\n"); printf(" --partitioned_log_experimental Starts with new type of backup system using partitioned logs.\n"); printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); printf(" --log Enables trace file logging for the CLI session.\n" @@ -1273,6 +1324,7 @@ enumBackupType getBackupType(std::string backupType) values["delete"] = BACKUP_DELETE; values["describe"] = BACKUP_DESCRIBE; values["list"] = BACKUP_LIST; + values["query"] = BACKUP_QUERY; values["dump"] = BACKUP_DUMP; values["modify"] = BACKUP_MODIFY; } @@ -2400,6 +2452,73 @@ ACTOR Future describeBackup(const char *name, std::string destinationConta return Void(); } +// If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for +// selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overriden to the version +// resolved to that timestamp. +ACTOR Future queryBackup(const char* name, std::string destinationContainer, + Standalone> keyRangesFilter, Version restoreVersion, + std::string originalClusterFile, std::string restoreTimestamp) { + // Resolve restoreTimestamp if given + if (!restoreTimestamp.empty()) { + if (originalClusterFile.empty()) { + printf("Error: an original cluster file must be given in order to resolve restore target timestamp '%s'\n", + restoreTimestamp.c_str()); + return Void(); + } + + if (!fileExists(originalClusterFile)) { + printf("Error: original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); + return Void(); + } + + Database origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST); + Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb)); + printf("Timestamp '%s' resolves to version %" PRId64 "\n", restoreTimestamp.c_str(), v); + restoreVersion = v; + } + + try { + state Reference bc = openBackupContainer(name, destinationContainer); + if (restoreVersion == invalidVersion) { + printf("Using the maximum restorable version for the specified key ranges.\n"); + BackupDescription desc = wait(bc->describeBackup()); + if (!desc.maxRestorableVersion.present()) { + printf("Error: the specified backup is not restorable to any version.\n"); + return Void(); + } + restoreVersion = desc.maxRestorableVersion.get(); + } else if (restoreVersion == latestVersion) { + printf("Using the minimum restorable version for the specified key ranges.\n"); + } else if (restoreVersion < 0) { + printf("Error: the specified restorable version is not valid."); + } + Optional fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter)); + if (fileSet.present()) { + printf("Key ranges filter: %s\n", keyRangesFilter.empty() ? "empty" : printable(keyRangesFilter).c_str()); + printf("Restoring to version: %" PRId64 "\n", fileSet.get().targetVersion); + printf("Range Files (file_name; file_size; key_range; version): \n"); + for (const auto& rangeFile : fileSet.get().ranges) { + ASSERT(fileSet.get().keyRanges.count(rangeFile.fileName)); + printf(" %s; %" PRId64 ", %s; %" PRId64 "\n", rangeFile.fileName.c_str(), rangeFile.fileSize, + fileSet.get().keyRanges.at(rangeFile.fileName).toString().c_str(), rangeFile.version); + } + printf("Log Files (file_name; file_size; begin_version; end_version): \n"); + for (const auto& log : fileSet.get().logs) { + printf(" %s; %" PRId64 "; %" PRId64 "; %" PRId64 "\n", log.fileName.c_str(), log.fileSize, + log.beginVersion, log.endVersion); + } + } else { + printf("No restorable files set found for specified key ranges.\n"); + } + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } + + return Void(); +} + ACTOR Future listBackup(std::string baseUrl) { try { std::vector containers = wait(IBackupContainer::listContainers(baseUrl)); @@ -2769,6 +2888,9 @@ int main(int argc, char* argv[]) { case BACKUP_LIST: args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupListOptions, SO_O_EXACT); break; + case BACKUP_QUERY: + args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupQueryOptions, SO_O_EXACT); + break; case BACKUP_MODIFY: args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupModifyOptions, SO_O_EXACT); break; @@ -3661,6 +3783,12 @@ int main(int argc, char* argv[]) { f = stopAfter( listBackup(baseUrl) ); break; + case BACKUP_QUERY: + initTraceFile(); + f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeys, restoreVersion, + restoreClusterFileOrig, restoreTimestamp)); + break; + case BACKUP_DUMP: initTraceFile(); f = stopAfter( dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd) ); diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a76f01b991..2edeb89b69 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -22,6 +22,7 @@ #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/JsonBuilder.h" +#include "flow/Arena.h" #include "flow/Trace.h" #include "flow/UnitTest.h" #include "flow/Hash3.h" @@ -1364,24 +1365,54 @@ public: return getSnapshotFileKeyRange_impl(Reference::addRef(this), file); } - ACTOR static Future> getRestoreSet_impl(Reference bc, Version targetVersion) { - // Find the most recent keyrange snapshot to end at or before targetVersion - state Optional snapshot; - std::vector snapshots = wait(bc->listKeyspaceSnapshots()); - for(auto const &s : snapshots) { - if(s.endVersion <= targetVersion) - snapshot = s; - } + ACTOR static Future> getRestoreSet_impl(Reference bc, + Version targetVersion, + VectorRef keyRangesFilter) { + // Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion. + state std::vector snapshots = wait(bc->listKeyspaceSnapshots()); + state int i = snapshots.size() - 1; + for (; i >= 0; i--) { + state KeyspaceSnapshotFile snapshot = snapshots[i]; + // The smallest version of filtered range files >= snapshot beginVersion > targetVersion + if (targetVersion >= 0 && snapshot.beginVersion > targetVersion) { + break; + } - if(snapshot.present()) { state RestorableFileSet restorable; - restorable.snapshot = snapshot.get(); - restorable.targetVersion = targetVersion; + state Version minKeyRangeVersion = MAX_VERSION; + state Version maxKeyRangeVersion = -1; std::pair, std::map> results = - wait(bc->readKeyspaceSnapshot(snapshot.get())); - restorable.ranges = std::move(results.first); - restorable.keyRanges = std::move(results.second); + wait(bc->readKeyspaceSnapshot(snapshot)); + + // Filter by keyRangesFilter. + if (keyRangesFilter.empty()) { + restorable.ranges = std::move(results.first); + restorable.keyRanges = std::move(results.second); + minKeyRangeVersion = snapshot.beginVersion; + maxKeyRangeVersion = snapshot.endVersion; + } else { + for (const auto& rangeFile : results.first) { + const auto& keyRange = results.second.at(rangeFile.fileName); + if (keyRange.intersects(keyRangesFilter)) { + restorable.ranges.push_back(rangeFile); + restorable.keyRanges[rangeFile.fileName] = keyRange; + minKeyRangeVersion = std::min(minKeyRangeVersion, rangeFile.version); + maxKeyRangeVersion = std::max(maxKeyRangeVersion, rangeFile.version); + } + } + // No range file match 'keyRangesFilter'. + if (restorable.ranges.empty()) { + continue; + } + } + if (targetVersion >= 0 && targetVersion < maxKeyRangeVersion) continue; + // 'latestVersion' represents using the minimum restorable version in a snapshot. + if (targetVersion == latestVersion) { + targetVersion = maxKeyRangeVersion; + } + restorable.targetVersion = targetVersion; + restorable.snapshot = snapshot; // TODO: Reenable the sanity check after TooManyFiles error is resolved if (false && g_network->isSimulated()) { // Sanity check key ranges @@ -1395,9 +1426,8 @@ public: } } - // No logs needed if there is a complete key space snapshot at the target version. - if (snapshot.get().beginVersion == snapshot.get().endVersion && - snapshot.get().endVersion == targetVersion) { + // No logs needed if there is a complete filtered key space snapshot at the target version. + if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == targetVersion) { restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; return Optional(restorable); } @@ -1405,8 +1435,8 @@ public: // FIXME: check if there are tagged logs. for each tag, there is no version gap. state std::vector logs; state std::vector plogs; - wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) && - store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true))); + wait(store(logs, bc->listLogFiles(minKeyRangeVersion, targetVersion, false)) && + store(plogs, bc->listLogFiles(minKeyRangeVersion, targetVersion, true))); if (plogs.size() > 0) { logs.swap(plogs); @@ -1418,12 +1448,11 @@ public: // Remove duplicated log files that can happen for old epochs. std::vector filtered = filterDuplicates(logs); - restorable.logs.swap(filtered); // sort by version order again for continuous analysis std::sort(restorable.logs.begin(), restorable.logs.end()); - if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) { - restorable.continuousBeginVersion = snapshot.get().beginVersion; + if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, targetVersion)) { + restorable.continuousBeginVersion = minKeyRangeVersion; restorable.continuousEndVersion = targetVersion + 1; // not inclusive return Optional(restorable); } @@ -1434,7 +1463,7 @@ public: std::sort(logs.begin(), logs.end()); // If there are logs and the first one starts at or before the snapshot begin version then proceed - if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) { + if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) { Version end = logs.begin()->endVersion; computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); if (end >= targetVersion) { @@ -1448,8 +1477,9 @@ public: return Optional(); } - Future> getRestoreSet(Version targetVersion) final { - return getRestoreSet_impl(Reference::addRef(this), targetVersion); + Future> getRestoreSet(Version targetVersion, + VectorRef keyRangesFilter) final { + return getRestoreSet_impl(Reference::addRef(this), targetVersion, keyRangesFilter); } private: diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h index 8ac79937dd..fdce885329 100644 --- a/fdbclient/BackupContainer.h +++ b/fdbclient/BackupContainer.h @@ -280,9 +280,11 @@ public: virtual Future dumpFileList(Version begin = 0, Version end = std::numeric_limits::max()) = 0; - // Get exactly the files necessary to restore to targetVersion. Returns non-present if - // restore to given version is not possible. - virtual Future> getRestoreSet(Version targetVersion) = 0; + // Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion. + // If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot. Returns non-present if + // restoring to the given version is not possible. + virtual Future> getRestoreSet(Version targetVersion, + VectorRef keyRangesFilter = {}) = 0; // Get an IBackupContainer based on a container spec string static Reference openContainer(std::string url); diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 117414923c..c859408f37 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -230,6 +230,7 @@ std::string describe( std::set const& items, int max_items = -1 ) { std::string printable( const StringRef& val ); std::string printable( const std::string& val ); std::string printable( const KeyRangeRef& range ); +std::string printable(const VectorRef& val); std::string printable( const VectorRef& val ); std::string printable( const VectorRef& val ); std::string printable( const KeyValueRef& val ); @@ -261,6 +262,14 @@ struct KeyRangeRef { bool contains( const KeyRef& key ) const { return begin <= key && key < end; } bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; } bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; } + bool intersects(const VectorRef& keysVec) const { + for (const auto& keys : keysVec) { + if (intersects(keys)) { + return true; + } + } + return false; + } bool empty() const { return begin == end; } bool singleKeyRange() const { return equalsKeyAfter(begin, end); } diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 715c93b6af..cbd9c42be3 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -148,6 +148,12 @@ std::string printable( const KeyRangeRef& range ) { return printable(range.begin) + " - " + printable(range.end); } +std::string printable(const VectorRef& val) { + std::string s; + for (int i = 0; i < val.size(); i++) s = s + printable(val[i]) + " "; + return s; +} + int unhex( char c ) { if (c >= '0' && c <= '9') return c-'0'; From 33aa10b4617da51ebc25de63063e93c93d6a6a34 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sat, 29 Aug 2020 20:10:45 -0700 Subject: [PATCH 02/13] Minor optimizations --- fdbbackup/backup.actor.cpp | 1 + fdbclient/BackupContainer.actor.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 9d2a204bdc..f3dfbe9eb4 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -2491,6 +2491,7 @@ ACTOR Future queryBackup(const char* name, std::string destinationContaine printf("Using the minimum restorable version for the specified key ranges.\n"); } else if (restoreVersion < 0) { printf("Error: the specified restorable version is not valid."); + return Void(); } Optional fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter)); if (fileSet.present()) { diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index 2edeb89b69..a7a3ad8b07 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1401,7 +1401,7 @@ public: maxKeyRangeVersion = std::max(maxKeyRangeVersion, rangeFile.version); } } - // No range file match 'keyRangesFilter'. + // No range file matches 'keyRangesFilter'. if (restorable.ranges.empty()) { continue; } From 30e27ba27b0d455ddd76694be30508a92ff74553 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sun, 30 Aug 2020 00:44:17 -0700 Subject: [PATCH 03/13] Add support for keys in CLI --- fdbbackup/backup.actor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index f3dfbe9eb4..52cc6921c4 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -610,6 +610,8 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = { { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "-rv", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "--restore_version", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, From b6c0299d0972e3f09e70db04027046cc507c37cb Mon Sep 17 00:00:00 2001 From: Young Liu Date: Mon, 31 Aug 2020 09:31:57 -0700 Subject: [PATCH 04/13] Add help message in backup CLI for added options --- fdbbackup/backup.actor.cpp | 15 ++++++++++++--- fdbclient/BackupContainer.actor.cpp | 11 +++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 52cc6921c4..74b6c0dd7c 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -605,11 +605,11 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = { #ifdef _WIN32 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, + { OPT_RESTORE_TIMESTAMP, "--query_restore_timestamp", SO_REQ_SEP }, { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "-rv", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "--restore_version", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "--query_restore_version", SO_REQ_SEP }, { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, @@ -991,6 +991,15 @@ static void printBackupUsage(bool devhelp) { printf(" --delete_before_days NUM_DAYS\n" " Another way to specify version cutoff for expire operations. Deletes data files containing no data at or after a\n" " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n"); + printf(" -qrv --query_restore_version VERSION\n" + " For query operations, set target version for restoring a backup. Set -1 for maximum " + "restorable version and -2 for minimum restorable version.\n"); + printf( + " --query_restore_timestamp\n" + " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", + BackupAgentBase::timeFormat().c_str()); + printf( + " and it will be converted to a version from that time using metadata in the cluster file.\n"); printf(" --restorable_after_timestamp DATETIME\n" " For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n"); printf(" --restorable_after_version VERSION\n" diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a7a3ad8b07..bffa45f3fc 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1372,9 +1372,8 @@ public: state std::vector snapshots = wait(bc->listKeyspaceSnapshots()); state int i = snapshots.size() - 1; for (; i >= 0; i--) { - state KeyspaceSnapshotFile snapshot = snapshots[i]; // The smallest version of filtered range files >= snapshot beginVersion > targetVersion - if (targetVersion >= 0 && snapshot.beginVersion > targetVersion) { + if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) { break; } @@ -1383,14 +1382,14 @@ public: state Version maxKeyRangeVersion = -1; std::pair, std::map> results = - wait(bc->readKeyspaceSnapshot(snapshot)); + wait(bc->readKeyspaceSnapshot(snapshots[i])); // Filter by keyRangesFilter. if (keyRangesFilter.empty()) { restorable.ranges = std::move(results.first); restorable.keyRanges = std::move(results.second); - minKeyRangeVersion = snapshot.beginVersion; - maxKeyRangeVersion = snapshot.endVersion; + minKeyRangeVersion = snapshots[i].beginVersion; + maxKeyRangeVersion = snapshots[i].endVersion; } else { for (const auto& rangeFile : results.first) { const auto& keyRange = results.second.at(rangeFile.fileName); @@ -1412,7 +1411,7 @@ public: targetVersion = maxKeyRangeVersion; } restorable.targetVersion = targetVersion; - restorable.snapshot = snapshot; + restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved if (false && g_network->isSimulated()) { // Sanity check key ranges From e9d1f1c9c8fbc64fee7d3c4b55c92d1cb6372daf Mon Sep 17 00:00:00 2001 From: Young Liu Date: Mon, 31 Aug 2020 09:43:11 -0700 Subject: [PATCH 05/13] change formatting --- fdbbackup/backup.actor.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 74b6c0dd7c..2a9f12fcfd 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -993,13 +993,10 @@ static void printBackupUsage(bool devhelp) { " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n"); printf(" -qrv --query_restore_version VERSION\n" " For query operations, set target version for restoring a backup. Set -1 for maximum " - "restorable version and -2 for minimum restorable version.\n"); - printf( - " --query_restore_timestamp\n" - " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", - BackupAgentBase::timeFormat().c_str()); - printf( - " and it will be converted to a version from that time using metadata in the cluster file.\n"); + " restorable version and -2 for minimum restorable version.\n"); + printf(" --query_restore_timestamp DATETIME\n" + " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str()); + printf(" and it will be converted to a version from that time using metadata in the cluster file.\n"); printf(" --restorable_after_timestamp DATETIME\n" " For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n"); printf(" --restorable_after_version VERSION\n" @@ -1018,7 +1015,7 @@ static void printBackupUsage(bool devhelp) { " Specifies a UID to verify against the BackupUID of the running backup. If provided, the UID is verified in the same transaction\n" " which sets the new backup parameters (if the UID matches).\n"); printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); - printf(" -k KEYS List of key ranges to backup or to filter the backup.\n" + printf(" -k KEYS List of key ranges to backup or to filter the backup in query operations.\n" " If not specified, the entire database will be backed up or no filter will be applied.\n"); printf(" --partitioned_log_experimental Starts with new type of backup system using partitioned logs.\n"); printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); From 1ad5e174585f9b02d1b0df97a11d36e4655bbdec Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sat, 5 Sep 2020 11:14:59 -0700 Subject: [PATCH 06/13] add support for comparing original and current impls --- fdbclient/BackupContainer.actor.cpp | 166 ++++++++++++++++-- ...kupAndParallelRestoreCorrectness.actor.cpp | 1 + flow/error_definitions.h | 1 + 3 files changed, 154 insertions(+), 14 deletions(-) diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index bffa45f3fc..a5365d7a35 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1365,16 +1365,106 @@ public: return getSnapshotFileKeyRange_impl(Reference::addRef(this), file); } + ACTOR static Future> getRestoreSet_impl_original( + Reference bc, Version targetVersion) { + // Find the most recent keyrange snapshot to end at or before targetVersion + state Optional snapshot; + std::vector snapshots = wait(bc->listKeyspaceSnapshots()); + // printf("old h1 %ld %ld\n", targetVersion, snapshots.size()); + for (auto const& s : snapshots) { + if (s.endVersion <= targetVersion) snapshot = s; + } + + if (snapshot.present()) { + // printf("h2 %ld %ld\n", snapshot.get().beginVersion, snapshot.get().endVersion); + state RestorableFileSet restorable; + restorable.snapshot = snapshot.get(); + restorable.targetVersion = targetVersion; + + std::pair, std::map> results = + wait(bc->readKeyspaceSnapshot(snapshot.get())); + restorable.ranges = std::move(results.first); + restorable.keyRanges = std::move(results.second); + // TODO: Reenable the sanity check after TooManyFiles error is resolved + if (false && g_network->isSimulated()) { + // Sanity check key ranges + state std::map::iterator rit; + for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) { + auto it = std::find_if(restorable.ranges.begin(), restorable.ranges.end(), + [file = rit->first](const RangeFile f) { return f.fileName == file; }); + ASSERT(it != restorable.ranges.end()); + KeyRange result = wait(bc->getSnapshotFileKeyRange(*it)); + ASSERT(rit->second.begin <= result.begin && rit->second.end >= result.end); + } + } + + // No logs needed if there is a complete key space snapshot at the target version. + if (snapshot.get().beginVersion == snapshot.get().endVersion && + snapshot.get().endVersion == targetVersion) { + restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; + return Optional(restorable); + } + + // FIXME: check if there are tagged logs. for each tag, there is no version gap. + state std::vector logs; + state std::vector plogs; + wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) && + store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true))); + + if (plogs.size() > 0) { + logs.swap(plogs); + // sort by tag ID so that filterDuplicates works. + std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) { + return std::tie(a.tagId, a.beginVersion, a.endVersion) < + std::tie(b.tagId, b.beginVersion, b.endVersion); + }); + + // Remove duplicated log files that can happen for old epochs. + std::vector filtered = filterDuplicates(logs); + + restorable.logs.swap(filtered); + // sort by version order again for continuous analysis + std::sort(restorable.logs.begin(), restorable.logs.end()); + if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) { + restorable.continuousBeginVersion = snapshot.get().beginVersion; + restorable.continuousEndVersion = targetVersion + 1; // not inclusive + return Optional(restorable); + } + return Optional(); + } + + // List logs in version order so log continuity can be analyzed + std::sort(logs.begin(), logs.end()); + + // printf("old backup used log begin version: %ld\n", logs.empty() ? -1 : logs.front().beginVersion); + // If there are logs and the first one starts at or before the snapshot begin version then proceed + if (!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) { + Version end = logs.begin()->endVersion; + computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); + if (end >= targetVersion) { + restorable.continuousBeginVersion = logs.begin()->beginVersion; + restorable.continuousEndVersion = end; + return Optional(restorable); + } + } + } + + return Optional(); + } + + // might have problem working with old backup ACTOR static Future> getRestoreSet_impl(Reference bc, Version targetVersion, VectorRef keyRangesFilter) { // Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion. state std::vector snapshots = wait(bc->listKeyspaceSnapshots()); state int i = snapshots.size() - 1; + // printf("new h1 %ld %ld %ld\n", targetVersion, keyRangesFilter.size(), snapshots.size()); for (; i >= 0; i--) { + // printf("h2 %ld %ld\n", snapshots[i].beginVersion, snapshots[i].endVersion); // The smallest version of filtered range files >= snapshot beginVersion > targetVersion if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) { - break; + continue; } state RestorableFileSet restorable; @@ -1384,12 +1474,18 @@ public: std::pair, std::map> results = wait(bc->readKeyspaceSnapshot(snapshots[i])); + // Old backup does not have metadata about key ranges and can not be filtered with key ranges. + if (keyRangesFilter.size() && results.second.empty() && !results.first.empty()) { + throw backup_not_filterable_with_key_ranges(); + } + // Filter by keyRangesFilter. if (keyRangesFilter.empty()) { restorable.ranges = std::move(results.first); restorable.keyRanges = std::move(results.second); minKeyRangeVersion = snapshots[i].beginVersion; maxKeyRangeVersion = snapshots[i].endVersion; + // printf("h3 %ld %ld %ld\n", minKeyRangeVersion, maxKeyRangeVersion, restorable.keyRanges.size()); } else { for (const auto& rangeFile : results.first) { const auto& keyRange = results.second.at(rangeFile.fileName); @@ -1405,12 +1501,14 @@ public: continue; } } - if (targetVersion >= 0 && targetVersion < maxKeyRangeVersion) continue; // 'latestVersion' represents using the minimum restorable version in a snapshot. - if (targetVersion == latestVersion) { - targetVersion = maxKeyRangeVersion; - } - restorable.targetVersion = targetVersion; + // if (targetVersion == latestVersion) { + // restorable.targetVersion = maxKeyRangeVersion; + // } else + restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; + + if (restorable.targetVersion < maxKeyRangeVersion) continue; + // restorable.targetVersion = targetVersion; restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved if (false && g_network->isSimulated()) { @@ -1426,7 +1524,7 @@ public: } // No logs needed if there is a complete filtered key space snapshot at the target version. - if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == targetVersion) { + if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == restorable.targetVersion) { restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; return Optional(restorable); } @@ -1434,8 +1532,8 @@ public: // FIXME: check if there are tagged logs. for each tag, there is no version gap. state std::vector logs; state std::vector plogs; - wait(store(logs, bc->listLogFiles(minKeyRangeVersion, targetVersion, false)) && - store(plogs, bc->listLogFiles(minKeyRangeVersion, targetVersion, true))); + wait(store(logs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, false)) && + store(plogs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, true))); if (plogs.size() > 0) { logs.swap(plogs); @@ -1450,9 +1548,9 @@ public: restorable.logs.swap(filtered); // sort by version order again for continuous analysis std::sort(restorable.logs.begin(), restorable.logs.end()); - if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, targetVersion)) { + if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, restorable.targetVersion)) { restorable.continuousBeginVersion = minKeyRangeVersion; - restorable.continuousEndVersion = targetVersion + 1; // not inclusive + restorable.continuousEndVersion = restorable.targetVersion + 1; // not inclusive return Optional(restorable); } return Optional(); @@ -1461,11 +1559,17 @@ public: // List logs in version order so log continuity can be analyzed std::sort(logs.begin(), logs.end()); + // printf("6.2 backup used: log begin version: %ld\n", logs.empty() ? -1 : logs.front().beginVersion); // If there are logs and the first one starts at or before the snapshot begin version then proceed if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) { + Version end = logs.begin()->endVersion; - computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); - if (end >= targetVersion) { + computeRestoreEndVersion(logs, &restorable.logs, &end, restorable.targetVersion); + + // printf("6.2 backup used: end_version: %ld target: %ld\n", end, restorable.targetVersion); + + if (end >= restorable.targetVersion) { + // printf("6.2 backup used finish\n"); restorable.continuousBeginVersion = logs.begin()->beginVersion; restorable.continuousEndVersion = end; return Optional(restorable); @@ -1476,9 +1580,43 @@ public: return Optional(); } + static void printRestorableSet(const RestorableFileSet& restorable) { + printf("restorable: begin %ld, end %ld, target %ld, ranges size %ld, logs size %ld, filename %s\n", + restorable.continuousBeginVersion, restorable.continuousEndVersion, restorable.targetVersion, + restorable.ranges.size(), restorable.logs.size(), restorable.snapshot.fileName.c_str()); + } + + ACTOR static Future> getRestoreSet_compare(Reference bc, + Version targetVersion, + VectorRef keyRangesFilter) { + + state Optional newResult = wait(getRestoreSet_impl(bc, targetVersion, keyRangesFilter)); + if (keyRangesFilter.empty() && targetVersion != latestVersion) { + state Optional oldResult = wait(getRestoreSet_impl_original(bc, targetVersion)); + // printf("comparing\n"); + // if (oldResult.present()) { + // // printf("old\n"); + // printRestorableSet(oldResult.get()); + // } + ASSERT(newResult.present() == oldResult.present()); + if (newResult.present()) { + // printf("new\n"); + // printRestorableSet(newResult.get()); + ASSERT(oldResult.get().continuousBeginVersion == oldResult.get().continuousBeginVersion); + ASSERT(oldResult.get().continuousEndVersion == oldResult.get().continuousEndVersion); + ASSERT(oldResult.get().targetVersion == oldResult.get().targetVersion); + ASSERT(oldResult.get().ranges.size() == oldResult.get().ranges.size()); + ASSERT(oldResult.get().keyRanges.size() == oldResult.get().keyRanges.size()); + ASSERT(oldResult.get().snapshot.fileName == oldResult.get().snapshot.fileName); + } + } + return newResult; + } + Future> getRestoreSet(Version targetVersion, VectorRef keyRangesFilter) final { - return getRestoreSet_impl(Reference::addRef(this), targetVersion, keyRangesFilter); + return getRestoreSet_compare(Reference::addRef(this), targetVersion, + keyRangesFilter); } private: diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp index af983a13ee..dfa20c95ec 100644 --- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp @@ -468,6 +468,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { .detail("LastBackupContainer", lastBackupContainer->getURL()) .detail("RestoreAfter", self->restoreAfter) .detail("BackupTag", printable(self->backupTag)); + // start restoring auto container = IBackupContainer::openContainer(lastBackupContainer->getURL()); BackupDescription desc = wait(container->describeBackup()); diff --git a/flow/error_definitions.h b/flow/error_definitions.h index ca8460548d..3baf2aaa2c 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -197,6 +197,7 @@ ERROR( backup_cannot_expire, 2316, "Cannot expire requested data from backup wit ERROR( backup_auth_missing, 2317, "Cannot find authentication details (such as a password or secret key) for the specified Backup Container URL") ERROR( backup_auth_unreadable, 2318, "Cannot read or parse one or more sources of authentication information for Backup Container URLs") ERROR( backup_does_not_exist, 2319, "Backup does not exist") +ERROR( backup_not_filterable_with_key_ranges, 2320, "Backup before 6.3 cannot be filtered with key ranges") ERROR( restore_invalid_version, 2361, "Invalid restore version") ERROR( restore_corrupted_data, 2362, "Corrupted backup data") ERROR( restore_missing_data, 2363, "Missing backup data") From 3728ed03ddfbab5093910f726d727880f2ec2cdb Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sat, 5 Sep 2020 18:55:09 -0700 Subject: [PATCH 07/13] Resolve comments --- fdbbackup/backup.actor.cpp | 128 +++++++++++++++++++------ fdbclient/BackupContainer.actor.cpp | 143 +--------------------------- flow/error_definitions.h | 1 + 3 files changed, 101 insertions(+), 171 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 2a9f12fcfd..efd88087c6 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -18,7 +18,9 @@ * limitations under the License. */ +#include "fdbclient/JsonBuilder.h" #include "flow/Arena.h" +#include "flow/Trace.h" #define BOOST_DATE_TIME_NO_LIB #include @@ -120,7 +122,7 @@ enum { OPT_USE_PARTITIONED_LOG, // Backup and Restore constants - OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, + OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, OPT_BACKUPKEYS_FILTER, // Backup Modify OPT_MOD_ACTIVE_INTERVAL, OPT_MOD_VERIFY_UID, @@ -610,8 +612,8 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = { { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "--query_restore_version", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILTER, "--keys", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, @@ -992,8 +994,8 @@ static void printBackupUsage(bool devhelp) { " Another way to specify version cutoff for expire operations. Deletes data files containing no data at or after a\n" " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n"); printf(" -qrv --query_restore_version VERSION\n" - " For query operations, set target version for restoring a backup. Set -1 for maximum " - " restorable version and -2 for minimum restorable version.\n"); + " For query operations, set target version for restoring a backup. Set -1 for maximum\n" + " restorable version (default) and -2 for minimum restorable version.\n"); printf(" --query_restore_timestamp DATETIME\n" " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str()); printf(" and it will be converted to a version from that time using metadata in the cluster file.\n"); @@ -2460,71 +2462,125 @@ ACTOR Future describeBackup(const char *name, std::string destinationConta return Void(); } +static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, std::string errorMessage) { + result["error"] = errorMessage; + printf("%s\n", result.getJson().c_str()); + TraceEvent("BackupQueryFailure").detail("OperationId", operationId).detail("Reason", errorMessage); +} + // If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for // selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overriden to the version // resolved to that timestamp. ACTOR Future queryBackup(const char* name, std::string destinationContainer, Standalone> keyRangesFilter, Version restoreVersion, - std::string originalClusterFile, std::string restoreTimestamp) { + std::string originalClusterFile, std::string restoreTimestamp, bool verbose) { + state UID operationId = deterministicRandom()->randomUniqueID(); + state JsonBuilderObject result; + state std::string errorMessage; + result["key_ranges_filter"] = printable(keyRangesFilter); + result["destination_container"] = destinationContainer; + + TraceEvent("BackupQueryStart") + .detail("OperationId", operationId) + .detail("DestinationContainer", destinationContainer) + .detail("KeyRangesFilter", printable(keyRangesFilter)) + .detail("SpecifiedRestoreVersion", restoreVersion) + .detail("RestoreTimestamp", restoreTimestamp) + .detail("BackupClusterFile", originalClusterFile); + // Resolve restoreTimestamp if given if (!restoreTimestamp.empty()) { if (originalClusterFile.empty()) { - printf("Error: an original cluster file must be given in order to resolve restore target timestamp '%s'\n", - restoreTimestamp.c_str()); + reportBackupQueryError( + operationId, result, + format("an original cluster file must be given in order to resolve restore target timestamp '%s'", + restoreTimestamp.c_str())); return Void(); } if (!fileExists(originalClusterFile)) { - printf("Error: original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); + reportBackupQueryError(operationId, result, + format("The specified original source database cluster file '%s' does not exist\n", + originalClusterFile.c_str())); return Void(); } Database origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST); Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb)); - printf("Timestamp '%s' resolves to version %" PRId64 "\n", restoreTimestamp.c_str(), v); + result["restore_timestamp"] = restoreTimestamp; + result["restore_timestamp_resolved_version"] = v; restoreVersion = v; } try { state Reference bc = openBackupContainer(name, destinationContainer); if (restoreVersion == invalidVersion) { - printf("Using the maximum restorable version for the specified key ranges.\n"); BackupDescription desc = wait(bc->describeBackup()); if (!desc.maxRestorableVersion.present()) { - printf("Error: the specified backup is not restorable to any version.\n"); + reportBackupQueryError(operationId, result, "the specified backup is not restorable to any version"); return Void(); } restoreVersion = desc.maxRestorableVersion.get(); - } else if (restoreVersion == latestVersion) { - printf("Using the minimum restorable version for the specified key ranges.\n"); - } else if (restoreVersion < 0) { - printf("Error: the specified restorable version is not valid."); + } else if (restoreVersion < 0 && restoreVersion != latestVersion) { + reportBackupQueryError(operationId, result, + errorMessage = + format("the specified restorable version %ld is not valid", restoreVersion)); return Void(); } Optional fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter)); if (fileSet.present()) { - printf("Key ranges filter: %s\n", keyRangesFilter.empty() ? "empty" : printable(keyRangesFilter).c_str()); - printf("Restoring to version: %" PRId64 "\n", fileSet.get().targetVersion); - printf("Range Files (file_name; file_size; key_range; version): \n"); + int64_t totalRangeFilesSize = 0, totalLogFilesSize = 0; + result["restore_version"] = fileSet.get().targetVersion; + JsonBuilderArray rangeFilesJson; + JsonBuilderArray logFilesJson; for (const auto& rangeFile : fileSet.get().ranges) { - ASSERT(fileSet.get().keyRanges.count(rangeFile.fileName)); - printf(" %s; %" PRId64 ", %s; %" PRId64 "\n", rangeFile.fileName.c_str(), rangeFile.fileSize, - fileSet.get().keyRanges.at(rangeFile.fileName).toString().c_str(), rangeFile.version); + JsonBuilderObject object; + object["file_name"] = rangeFile.fileName; + object["file_size"] = rangeFile.fileSize; + object["version"] = rangeFile.version; + object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0 + ? "none" + : fileSet.get().keyRanges.at(rangeFile.fileName).toString(); + rangeFilesJson.push_back(object); + totalRangeFilesSize += rangeFile.fileSize; } - printf("Log Files (file_name; file_size; begin_version; end_version): \n"); for (const auto& log : fileSet.get().logs) { - printf(" %s; %" PRId64 "; %" PRId64 "; %" PRId64 "\n", log.fileName.c_str(), log.fileSize, - log.beginVersion, log.endVersion); + JsonBuilderObject object; + object["file_name"] = log.fileName; + object["file_size"] = log.fileSize; + object["begin_version"] = log.beginVersion; + object["end_version"] = log.endVersion; + logFilesJson.push_back(object); + totalLogFilesSize += log.fileSize; } + + result["total_range_files_size"] = totalRangeFilesSize; + result["total_log_files_size"] = totalLogFilesSize; + + if (verbose) { + result["ranges"] = rangeFilesJson; + result["logs"] = logFilesJson; + } + + TraceEvent("BackupQueryReceivedRestorableFilesSet") + .detail("DestinationContainer", destinationContainer) + .detail("KeyRangesFilter", printable(keyRangesFilter)) + .detail("ActualRestoreVersion", fileSet.get().targetVersion) + .detail("NumRangeFiles", fileSet.get().ranges.size()) + .detail("NumLogFiles", fileSet.get().logs.size()) + .detail("RangeFilesBytes", totalRangeFilesSize) + .detail("LogFilesBytes", totalLogFilesSize); } else { - printf("No restorable files set found for specified key ranges.\n"); + reportBackupQueryError(operationId, result, "no restorable files set found for specified key ranges"); + return Void(); } + } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; + reportBackupQueryError(operationId, result, e.what()); + return Void(); } + printf("%s\n", result.getJson().c_str()); return Void(); } @@ -3039,6 +3095,7 @@ int main(int argc, char* argv[]) { std::string addPrefix; std::string removePrefix; Standalone> backupKeys; + Standalone> backupKeysFilter; int maxErrors = 20; Version restoreVersion = invalidVersion; std::string restoreTimestamp; @@ -3259,6 +3316,15 @@ int main(int argc, char* argv[]) { return FDB_EXIT_ERROR; } break; + case OPT_BACKUPKEYS_FILTER: + try { + addKeyRange(args->OptionArg(), backupKeysFilter); + } + catch (Error &) { + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; + } + break; case OPT_DESTCONTAINER: destinationContainer = args->OptionArg(); // If the url starts with '/' then prepend "file://" for backwards compatibility @@ -3794,8 +3860,8 @@ int main(int argc, char* argv[]) { case BACKUP_QUERY: initTraceFile(); - f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeys, restoreVersion, - restoreClusterFileOrig, restoreTimestamp)); + f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeysFilter, restoreVersion, + restoreClusterFileOrig, restoreTimestamp, !quietDisplay)); break; case BACKUP_DUMP: diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a5365d7a35..f5f5bce2ae 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1365,103 +1365,13 @@ public: return getSnapshotFileKeyRange_impl(Reference::addRef(this), file); } - ACTOR static Future> getRestoreSet_impl_original( - Reference bc, Version targetVersion) { - // Find the most recent keyrange snapshot to end at or before targetVersion - state Optional snapshot; - std::vector snapshots = wait(bc->listKeyspaceSnapshots()); - // printf("old h1 %ld %ld\n", targetVersion, snapshots.size()); - for (auto const& s : snapshots) { - if (s.endVersion <= targetVersion) snapshot = s; - } - - if (snapshot.present()) { - // printf("h2 %ld %ld\n", snapshot.get().beginVersion, snapshot.get().endVersion); - state RestorableFileSet restorable; - restorable.snapshot = snapshot.get(); - restorable.targetVersion = targetVersion; - - std::pair, std::map> results = - wait(bc->readKeyspaceSnapshot(snapshot.get())); - restorable.ranges = std::move(results.first); - restorable.keyRanges = std::move(results.second); - // TODO: Reenable the sanity check after TooManyFiles error is resolved - if (false && g_network->isSimulated()) { - // Sanity check key ranges - state std::map::iterator rit; - for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) { - auto it = std::find_if(restorable.ranges.begin(), restorable.ranges.end(), - [file = rit->first](const RangeFile f) { return f.fileName == file; }); - ASSERT(it != restorable.ranges.end()); - KeyRange result = wait(bc->getSnapshotFileKeyRange(*it)); - ASSERT(rit->second.begin <= result.begin && rit->second.end >= result.end); - } - } - - // No logs needed if there is a complete key space snapshot at the target version. - if (snapshot.get().beginVersion == snapshot.get().endVersion && - snapshot.get().endVersion == targetVersion) { - restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; - return Optional(restorable); - } - - // FIXME: check if there are tagged logs. for each tag, there is no version gap. - state std::vector logs; - state std::vector plogs; - wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) && - store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true))); - - if (plogs.size() > 0) { - logs.swap(plogs); - // sort by tag ID so that filterDuplicates works. - std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) { - return std::tie(a.tagId, a.beginVersion, a.endVersion) < - std::tie(b.tagId, b.beginVersion, b.endVersion); - }); - - // Remove duplicated log files that can happen for old epochs. - std::vector filtered = filterDuplicates(logs); - - restorable.logs.swap(filtered); - // sort by version order again for continuous analysis - std::sort(restorable.logs.begin(), restorable.logs.end()); - if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) { - restorable.continuousBeginVersion = snapshot.get().beginVersion; - restorable.continuousEndVersion = targetVersion + 1; // not inclusive - return Optional(restorable); - } - return Optional(); - } - - // List logs in version order so log continuity can be analyzed - std::sort(logs.begin(), logs.end()); - - // printf("old backup used log begin version: %ld\n", logs.empty() ? -1 : logs.front().beginVersion); - // If there are logs and the first one starts at or before the snapshot begin version then proceed - if (!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) { - Version end = logs.begin()->endVersion; - computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); - if (end >= targetVersion) { - restorable.continuousBeginVersion = logs.begin()->beginVersion; - restorable.continuousEndVersion = end; - return Optional(restorable); - } - } - } - - return Optional(); - } - - // might have problem working with old backup ACTOR static Future> getRestoreSet_impl(Reference bc, Version targetVersion, VectorRef keyRangesFilter) { // Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion. state std::vector snapshots = wait(bc->listKeyspaceSnapshots()); state int i = snapshots.size() - 1; - // printf("new h1 %ld %ld %ld\n", targetVersion, keyRangesFilter.size(), snapshots.size()); for (; i >= 0; i--) { - // printf("h2 %ld %ld\n", snapshots[i].beginVersion, snapshots[i].endVersion); // The smallest version of filtered range files >= snapshot beginVersion > targetVersion if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) { continue; @@ -1485,7 +1395,6 @@ public: restorable.keyRanges = std::move(results.second); minKeyRangeVersion = snapshots[i].beginVersion; maxKeyRangeVersion = snapshots[i].endVersion; - // printf("h3 %ld %ld %ld\n", minKeyRangeVersion, maxKeyRangeVersion, restorable.keyRanges.size()); } else { for (const auto& rangeFile : results.first) { const auto& keyRange = results.second.at(rangeFile.fileName); @@ -1498,17 +1407,13 @@ public: } // No range file matches 'keyRangesFilter'. if (restorable.ranges.empty()) { - continue; + throw backup_not_overlapped_with_keys_filter(); } } // 'latestVersion' represents using the minimum restorable version in a snapshot. - // if (targetVersion == latestVersion) { - // restorable.targetVersion = maxKeyRangeVersion; - // } else restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; - if (restorable.targetVersion < maxKeyRangeVersion) continue; - // restorable.targetVersion = targetVersion; + restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved if (false && g_network->isSimulated()) { @@ -1558,65 +1463,23 @@ public: // List logs in version order so log continuity can be analyzed std::sort(logs.begin(), logs.end()); - - // printf("6.2 backup used: log begin version: %ld\n", logs.empty() ? -1 : logs.front().beginVersion); // If there are logs and the first one starts at or before the snapshot begin version then proceed if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) { - Version end = logs.begin()->endVersion; computeRestoreEndVersion(logs, &restorable.logs, &end, restorable.targetVersion); - - // printf("6.2 backup used: end_version: %ld target: %ld\n", end, restorable.targetVersion); - if (end >= restorable.targetVersion) { - // printf("6.2 backup used finish\n"); restorable.continuousBeginVersion = logs.begin()->beginVersion; restorable.continuousEndVersion = end; return Optional(restorable); } } } - return Optional(); } - static void printRestorableSet(const RestorableFileSet& restorable) { - printf("restorable: begin %ld, end %ld, target %ld, ranges size %ld, logs size %ld, filename %s\n", - restorable.continuousBeginVersion, restorable.continuousEndVersion, restorable.targetVersion, - restorable.ranges.size(), restorable.logs.size(), restorable.snapshot.fileName.c_str()); - } - - ACTOR static Future> getRestoreSet_compare(Reference bc, - Version targetVersion, - VectorRef keyRangesFilter) { - - state Optional newResult = wait(getRestoreSet_impl(bc, targetVersion, keyRangesFilter)); - if (keyRangesFilter.empty() && targetVersion != latestVersion) { - state Optional oldResult = wait(getRestoreSet_impl_original(bc, targetVersion)); - // printf("comparing\n"); - // if (oldResult.present()) { - // // printf("old\n"); - // printRestorableSet(oldResult.get()); - // } - ASSERT(newResult.present() == oldResult.present()); - if (newResult.present()) { - // printf("new\n"); - // printRestorableSet(newResult.get()); - ASSERT(oldResult.get().continuousBeginVersion == oldResult.get().continuousBeginVersion); - ASSERT(oldResult.get().continuousEndVersion == oldResult.get().continuousEndVersion); - ASSERT(oldResult.get().targetVersion == oldResult.get().targetVersion); - ASSERT(oldResult.get().ranges.size() == oldResult.get().ranges.size()); - ASSERT(oldResult.get().keyRanges.size() == oldResult.get().keyRanges.size()); - ASSERT(oldResult.get().snapshot.fileName == oldResult.get().snapshot.fileName); - } - } - return newResult; - } - Future> getRestoreSet(Version targetVersion, VectorRef keyRangesFilter) final { - return getRestoreSet_compare(Reference::addRef(this), targetVersion, - keyRangesFilter); + return getRestoreSet_impl(Reference::addRef(this), targetVersion, keyRangesFilter); } private: diff --git a/flow/error_definitions.h b/flow/error_definitions.h index 3baf2aaa2c..6d9d803cc8 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -198,6 +198,7 @@ ERROR( backup_auth_missing, 2317, "Cannot find authentication details (such as a ERROR( backup_auth_unreadable, 2318, "Cannot read or parse one or more sources of authentication information for Backup Container URLs") ERROR( backup_does_not_exist, 2319, "Backup does not exist") ERROR( backup_not_filterable_with_key_ranges, 2320, "Backup before 6.3 cannot be filtered with key ranges") +ERROR( backup_not_overlapped_with_keys_filter, 2321, "Backup key ranges doesn't overlap with key ranges filter") ERROR( restore_invalid_version, 2361, "Invalid restore version") ERROR( restore_corrupted_data, 2362, "Corrupted backup data") ERROR( restore_missing_data, 2363, "Missing backup data") From f9a5b727a68d6813c464bcf608344cbaa8909db2 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Mon, 14 Sep 2020 16:28:01 -0700 Subject: [PATCH 08/13] Add comments questions and TODOs --- fdbbackup/backup.actor.cpp | 2 ++ fdbclient/BackupContainer.actor.cpp | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index efd88087c6..eebcd95420 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -2516,6 +2516,8 @@ ACTOR Future queryBackup(const char* name, std::string destinationContaine state Reference bc = openBackupContainer(name, destinationContainer); if (restoreVersion == invalidVersion) { BackupDescription desc = wait(bc->describeBackup()); + // TODO: If the keyRangeFilter is restorable but the normalKeys is not, maxRestorableVersion will not + // present, but we should still provide a restorable version for the keyRangeFilter if (!desc.maxRestorableVersion.present()) { reportBackupQueryError(operationId, result, "the specified backup is not restorable to any version"); return Void(); diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index f5f5bce2ae..c53a0eb884 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -245,7 +245,7 @@ std::string BackupDescription::toJSON() const { * file written will be after the start version of the snapshot's execution. * * Log files are at file paths like - * /plogs/...log,startVersion,endVersion,UID,tagID-of-N,blocksize + * /plogs/.../log,startVersion,endVersion,UID,tagID-of-N,blocksize * /logs/.../log,startVersion,endVersion,UID,blockSize * where ... is a multi level path which sorts lexically into version order and results in approximately 1 * unique folder per day containing about 5,000 files. Logs after FDB 6.3 are stored in "plogs" @@ -1411,8 +1411,9 @@ public: } } // 'latestVersion' represents using the minimum restorable version in a snapshot. + // MX: Isn't "latestVersion" the earliest restorable version? so below should be minKeyRangeVersion restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; - if (restorable.targetVersion < maxKeyRangeVersion) continue; + if (restorable.targetVersion < maxKeyRangeVersion) continue; // Q: Isn't this always true? restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved From 8224e17a08be94ac20ff16f95d1dc3a7dd4f4f32 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Tue, 15 Sep 2020 09:38:40 -0700 Subject: [PATCH 09/13] Integrate extended getRestoreSet API into fast restore The extended getRestoreSet provides a much smaller set of backup files for small keyrange restore; This commit integrate it into fast restore so that fast restore does not have to filter out unneeded backup files. --- fdbclient/BackupContainer.actor.cpp | 4 ++-- fdbserver/RestoreController.actor.cpp | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index c53a0eb884..ba0dcf3376 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1410,8 +1410,7 @@ public: throw backup_not_overlapped_with_keys_filter(); } } - // 'latestVersion' represents using the minimum restorable version in a snapshot. - // MX: Isn't "latestVersion" the earliest restorable version? so below should be minKeyRangeVersion + // 'latestVersion' represents using the maximum restorable version in a snapshot. restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; if (restorable.targetVersion < maxKeyRangeVersion) continue; // Q: Isn't this always true? @@ -1432,6 +1431,7 @@ public: // No logs needed if there is a complete filtered key space snapshot at the target version. if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == restorable.targetVersion) { restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; + // TODO: Add a Trace here return Optional(restorable); } diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp index d599711b95..c1bf57bb1c 100644 --- a/fdbserver/RestoreController.actor.cpp +++ b/fdbserver/RestoreController.actor.cpp @@ -747,7 +747,11 @@ ACTOR static Future collectBackupFiles(Reference bc, std::cout << "Restore to version: " << request.targetVersion << "\nBackupDesc: \n" << desc.toString() << "\n\n"; } - Optional restorable = wait(bc->getRestoreSet(request.targetVersion)); + // NOTE: If correctness fails and it is hard to fix, do not add restoreRanges in the current PR. + // We should open a new PR to fix correctness failures. + state Standalone restoreRanges; + restoreRanges.push_back(request.range); + Optional restorable = wait(bc->getRestoreSet(request.targetVersion, restoreRanges)); if (!restorable.present()) { TraceEvent(SevWarn, "FastRestoreControllerPhaseCollectBackupFiles") From d4e3e989578ec9142954b47ac75b25adee33a644 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Thu, 17 Sep 2020 17:56:36 -0700 Subject: [PATCH 10/13] Use new getRestoreSet in RestoreController --- fdbbackup/backup.actor.cpp | 2 +- fdbclient/BackupContainer.actor.cpp | 10 +++++++--- fdbserver/RestoreController.actor.cpp | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index eebcd95420..46a82d365f 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -2516,7 +2516,7 @@ ACTOR Future queryBackup(const char* name, std::string destinationContaine state Reference bc = openBackupContainer(name, destinationContainer); if (restoreVersion == invalidVersion) { BackupDescription desc = wait(bc->describeBackup()); - // TODO: If the keyRangeFilter is restorable but the normalKeys is not, maxRestorableVersion will not + // TODO: If the keyRangeFilter is restorable but the normalKeys is not, maxRestorableVersion will not be // present, but we should still provide a restorable version for the keyRangeFilter if (!desc.maxRestorableVersion.present()) { reportBackupQueryError(operationId, result, "the specified backup is not restorable to any version"); diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index d7fc0126f1..725d6a0741 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1435,9 +1435,10 @@ public: throw backup_not_overlapped_with_keys_filter(); } } - // 'latestVersion' represents using the maximum restorable version in a snapshot. + // 'latestVersion' represents using the minimum restorable version in a snapshot. restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; - if (restorable.targetVersion < maxKeyRangeVersion) continue; // Q: Isn't this always true? + // Any version < maxKeyRangeVersion is not restorable. + if (restorable.targetVersion < maxKeyRangeVersion) continue; restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved @@ -1456,7 +1457,10 @@ public: // No logs needed if there is a complete filtered key space snapshot at the target version. if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == restorable.targetVersion) { restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; - // TODO: Add a Trace here + TraceEvent("BackupContainerGetRestorableFilesWithoutLogs") + .detail("KeyRangeVersion", restorable.targetVersion) + .detail("NumberOfRangeFiles", restorable.ranges.size()) + .detail("KeyRangesFilter", printable(keyRangesFilter)); return Optional(restorable); } diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp index 099f275197..1617a204ae 100644 --- a/fdbserver/RestoreController.actor.cpp +++ b/fdbserver/RestoreController.actor.cpp @@ -750,8 +750,8 @@ ACTOR static Future collectBackupFiles(Reference bc, // NOTE: If correctness fails and it is hard to fix, do not add restoreRanges in the current PR. // We should open a new PR to fix correctness failures. - state Standalone restoreRanges; - restoreRanges.push_back(request.range); + state VectorRef restoreRanges; + restoreRanges.add(request.range); Optional restorable = wait(bc->getRestoreSet(request.targetVersion, restoreRanges)); if (!restorable.present()) { From 60ccaa2ae977d2c624723ebdae3dbf00e468895e Mon Sep 17 00:00:00 2001 From: Young Liu Date: Thu, 17 Sep 2020 17:58:58 -0700 Subject: [PATCH 11/13] Support finding max restorable version for key ranges when keyspace's max restorable version is unavailable --- fdbbackup/backup.actor.cpp | 20 +++++++++++++------- fdbserver/RestoreController.actor.cpp | 2 +- fdbserver/workloads/AtomicOps.actor.cpp | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 46a82d365f..6717978f27 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -20,6 +20,7 @@ #include "fdbclient/JsonBuilder.h" #include "flow/Arena.h" +#include "flow/Error.h" #include "flow/Trace.h" #define BOOST_DATE_TIME_NO_LIB #include @@ -2516,14 +2517,19 @@ ACTOR Future queryBackup(const char* name, std::string destinationContaine state Reference bc = openBackupContainer(name, destinationContainer); if (restoreVersion == invalidVersion) { BackupDescription desc = wait(bc->describeBackup()); - // TODO: If the keyRangeFilter is restorable but the normalKeys is not, maxRestorableVersion will not be - // present, but we should still provide a restorable version for the keyRangeFilter - if (!desc.maxRestorableVersion.present()) { - reportBackupQueryError(operationId, result, "the specified backup is not restorable to any version"); - return Void(); + if (desc.maxRestorableVersion.present()) { + restoreVersion = desc.maxRestorableVersion.get(); + // Use continuous log end version for the maximum restorable version for the key ranges. + } else if (keyRangesFilter.size() && desc.contiguousLogEnd.present()) { + restoreVersion = desc.contiguousLogEnd.get(); + } else { + reportBackupQueryError( + operationId, result, + errorMessage = format("the backup for the specified key ranges is not restorable to any version")); } - restoreVersion = desc.maxRestorableVersion.get(); - } else if (restoreVersion < 0 && restoreVersion != latestVersion) { + } + + if (restoreVersion < 0 && restoreVersion != latestVersion) { reportBackupQueryError(operationId, result, errorMessage = format("the specified restorable version %ld is not valid", restoreVersion)); diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp index 1617a204ae..ddc070a102 100644 --- a/fdbserver/RestoreController.actor.cpp +++ b/fdbserver/RestoreController.actor.cpp @@ -749,7 +749,7 @@ ACTOR static Future collectBackupFiles(Reference bc, } // NOTE: If correctness fails and it is hard to fix, do not add restoreRanges in the current PR. - // We should open a new PR to fix correctness failures. + // We should open a new PR to fix correctness failures. // state VectorRef restoreRanges; restoreRanges.add(request.range); Optional restorable = wait(bc->getRestoreSet(request.targetVersion, restoreRanges)); diff --git a/fdbserver/workloads/AtomicOps.actor.cpp b/fdbserver/workloads/AtomicOps.actor.cpp index 64af5c164f..1307e3c9ce 100644 --- a/fdbserver/workloads/AtomicOps.actor.cpp +++ b/fdbserver/workloads/AtomicOps.actor.cpp @@ -101,7 +101,7 @@ struct AtomicOpsWorkload : TestWorkload { // case 10: // TEST(true); // Testing atomic CompareAndClear Not supported yet // opType = MutationRef::CompareAndClear - // break; + // break; default: ASSERT(false); } From 6032a21d30fbe89bcb24de8180c7b882815f09ea Mon Sep 17 00:00:00 2001 From: Young Liu Date: Thu, 17 Sep 2020 18:02:23 -0700 Subject: [PATCH 12/13] Fix comment --- fdbserver/RestoreController.actor.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp index ddc070a102..4d6f79c292 100644 --- a/fdbserver/RestoreController.actor.cpp +++ b/fdbserver/RestoreController.actor.cpp @@ -748,8 +748,6 @@ ACTOR static Future collectBackupFiles(Reference bc, std::cout << "Restore to version: " << request.targetVersion << "\nBackupDesc: \n" << desc.toString() << "\n\n"; } - // NOTE: If correctness fails and it is hard to fix, do not add restoreRanges in the current PR. - // We should open a new PR to fix correctness failures. // state VectorRef restoreRanges; restoreRanges.add(request.range); Optional restorable = wait(bc->getRestoreSet(request.targetVersion, restoreRanges)); From 7ac443df511172cc1beaa31314adaf1a22939417 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Thu, 17 Sep 2020 18:17:33 -0700 Subject: [PATCH 13/13] Add release notes --- documentation/sphinx/source/release-notes/release-notes-630.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index ece1be18c2..4ef3558806 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -8,6 +8,7 @@ Release Notes ===== * Report missing old tlogs information when in recovery before storage servers are fully recovered. `(PR #3706) `_ +* Support query command in backup CLI that allows users to query restorable files by key ranges. `(PR #3703) `_ Features --------