Merge branch 'release-6.3' of github.com:apple/foundationdb into feature-apiverison-batch-error
This commit is contained in:
Evan Tschannen
commit
c99627a7ab
|
|
@ -78,8 +78,9 @@ type Subspace interface {
|
|||
// FoundationDB keys (corresponding to the prefix of this Subspace).
|
||||
fdb.KeyConvertible
|
||||
|
||||
// All Subspaces implement fdb.ExactRange and fdb.Range, and describe all
|
||||
// keys logically in this Subspace.
|
||||
// All Subspaces implement fdb.ExactRange and fdb.Range, and describe all
|
||||
// keys strictly within the subspace that encode tuples. Specifically,
|
||||
// this will include all keys in [prefix + '\x00', prefix + '\xff').
|
||||
fdb.ExactRange
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ Release Notes
|
|||
=====
|
||||
|
||||
* Report missing old tlogs information when in recovery before storage servers are fully recovered. `(PR #3706) <https://github.com/apple/foundationdb/pull/3706>`_
|
||||
* Support query command in backup CLI that allows users to query restorable files by key ranges. `(PR #3703) <https://github.com/apple/foundationdb/pull/3703>`_
|
||||
|
||||
Features
|
||||
--------
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "fdbclient/JsonBuilder.h"
|
||||
#include "flow/Arena.h"
|
||||
#include "flow/Error.h"
|
||||
#include "flow/Trace.h"
|
||||
#define BOOST_DATE_TIME_NO_LIB
|
||||
#include <boost/interprocess/managed_shared_memory.hpp>
|
||||
|
||||
|
|
@ -81,7 +85,22 @@ enum enumProgramExe {
|
|||
};
|
||||
|
||||
enum enumBackupType {
|
||||
BACKUP_UNDEFINED=0, BACKUP_START, BACKUP_MODIFY, BACKUP_STATUS, BACKUP_ABORT, BACKUP_WAIT, BACKUP_DISCONTINUE, BACKUP_PAUSE, BACKUP_RESUME, BACKUP_EXPIRE, BACKUP_DELETE, BACKUP_DESCRIBE, BACKUP_LIST, BACKUP_DUMP, BACKUP_CLEANUP
|
||||
BACKUP_UNDEFINED = 0,
|
||||
BACKUP_START,
|
||||
BACKUP_MODIFY,
|
||||
BACKUP_STATUS,
|
||||
BACKUP_ABORT,
|
||||
BACKUP_WAIT,
|
||||
BACKUP_DISCONTINUE,
|
||||
BACKUP_PAUSE,
|
||||
BACKUP_RESUME,
|
||||
BACKUP_EXPIRE,
|
||||
BACKUP_DELETE,
|
||||
BACKUP_DESCRIBE,
|
||||
BACKUP_LIST,
|
||||
BACKUP_QUERY,
|
||||
BACKUP_DUMP,
|
||||
BACKUP_CLEANUP
|
||||
};
|
||||
|
||||
enum enumDBType {
|
||||
|
|
@ -104,7 +123,7 @@ enum {
|
|||
OPT_USE_PARTITIONED_LOG,
|
||||
|
||||
// Backup and Restore constants
|
||||
OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE,
|
||||
OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, OPT_BACKUPKEYS_FILTER,
|
||||
|
||||
// Backup Modify
|
||||
OPT_MOD_ACTIVE_INTERVAL, OPT_MOD_VERIFY_UID,
|
||||
|
|
@ -585,6 +604,40 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = {
|
|||
SO_END_OF_OPTIONS
|
||||
};
|
||||
|
||||
CSimpleOpt::SOption g_rgBackupQueryOptions[] = {
|
||||
#ifdef _WIN32
|
||||
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
|
||||
#endif
|
||||
{ OPT_RESTORE_TIMESTAMP, "--query_restore_timestamp", SO_REQ_SEP },
|
||||
{ OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
|
||||
{ OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
|
||||
{ OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP },
|
||||
{ OPT_RESTORE_VERSION, "--query_restore_version", SO_REQ_SEP },
|
||||
{ OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP },
|
||||
{ OPT_BACKUPKEYS_FILTER, "--keys", SO_REQ_SEP },
|
||||
{ OPT_TRACE, "--log", SO_NONE },
|
||||
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
|
||||
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
|
||||
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
|
||||
{ OPT_QUIET, "-q", SO_NONE },
|
||||
{ OPT_QUIET, "--quiet", SO_NONE },
|
||||
{ OPT_VERSION, "-v", SO_NONE },
|
||||
{ OPT_VERSION, "--version", SO_NONE },
|
||||
{ OPT_CRASHONERROR, "--crash", SO_NONE },
|
||||
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
|
||||
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
|
||||
{ OPT_HELP, "-?", SO_NONE },
|
||||
{ OPT_HELP, "-h", SO_NONE },
|
||||
{ OPT_HELP, "--help", SO_NONE },
|
||||
{ OPT_DEVHELP, "--dev-help", SO_NONE },
|
||||
{ OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
|
||||
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
|
||||
#ifndef TLS_DISABLED
|
||||
TLS_OPTION_FLAGS
|
||||
#endif
|
||||
SO_END_OF_OPTIONS
|
||||
};
|
||||
|
||||
// g_rgRestoreOptions is used by fdbrestore and fastrestore_tool
|
||||
CSimpleOpt::SOption g_rgRestoreOptions[] = {
|
||||
#ifdef _WIN32
|
||||
|
|
@ -918,13 +971,16 @@ void printBackupContainerInfo() {
|
|||
|
||||
static void printBackupUsage(bool devhelp) {
|
||||
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
|
||||
printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | list | cleanup) [OPTIONS]\n\n", exeBackup.toString().c_str());
|
||||
printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | "
|
||||
"list | query | cleanup) [OPTIONS]\n\n",
|
||||
exeBackup.toString().c_str());
|
||||
printf(" -C CONNFILE The path of a file containing the connection string for the\n"
|
||||
" FoundationDB cluster. The default is first the value of the\n"
|
||||
" FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n"
|
||||
" then `%s'.\n", platform::getDefaultClusterFilePath().c_str());
|
||||
printf(" -d, --destcontainer URL\n"
|
||||
" The Backup container URL for start, modify, describe, expire, and delete operations.\n");
|
||||
" The Backup container URL for start, modify, describe, query, expire, and delete "
|
||||
"operations.\n");
|
||||
printBackupContainerInfo();
|
||||
printf(" -b, --base_url BASEURL\n"
|
||||
" Base backup URL for list operations. This looks like a Backup URL but without a backup name.\n");
|
||||
|
|
@ -938,6 +994,12 @@ static void printBackupUsage(bool devhelp) {
|
|||
printf(" --delete_before_days NUM_DAYS\n"
|
||||
" Another way to specify version cutoff for expire operations. Deletes data files containing no data at or after a\n"
|
||||
" version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n");
|
||||
printf(" -qrv --query_restore_version VERSION\n"
|
||||
" For query operations, set target version for restoring a backup. Set -1 for maximum\n"
|
||||
" restorable version (default) and -2 for minimum restorable version.\n");
|
||||
printf(" --query_restore_timestamp DATETIME\n"
|
||||
" For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str());
|
||||
printf(" and it will be converted to a version from that time using metadata in the cluster file.\n");
|
||||
printf(" --restorable_after_timestamp DATETIME\n"
|
||||
" For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n");
|
||||
printf(" --restorable_after_version VERSION\n"
|
||||
|
|
@ -956,8 +1018,8 @@ static void printBackupUsage(bool devhelp) {
|
|||
" Specifies a UID to verify against the BackupUID of the running backup. If provided, the UID is verified in the same transaction\n"
|
||||
" which sets the new backup parameters (if the UID matches).\n");
|
||||
printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n");
|
||||
printf(" -k KEYS List of key ranges to backup.\n"
|
||||
" If not specified, the entire database will be backed up.\n");
|
||||
printf(" -k KEYS List of key ranges to backup or to filter the backup in query operations.\n"
|
||||
" If not specified, the entire database will be backed up or no filter will be applied.\n");
|
||||
printf(" --partitioned_log_experimental Starts with new type of backup system using partitioned logs.\n");
|
||||
printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n");
|
||||
printf(" --log Enables trace file logging for the CLI session.\n"
|
||||
|
|
@ -1273,6 +1335,7 @@ enumBackupType getBackupType(std::string backupType)
|
|||
values["delete"] = BACKUP_DELETE;
|
||||
values["describe"] = BACKUP_DESCRIBE;
|
||||
values["list"] = BACKUP_LIST;
|
||||
values["query"] = BACKUP_QUERY;
|
||||
values["dump"] = BACKUP_DUMP;
|
||||
values["modify"] = BACKUP_MODIFY;
|
||||
}
|
||||
|
|
@ -2400,6 +2463,135 @@ ACTOR Future<Void> describeBackup(const char *name, std::string destinationConta
|
|||
return Void();
|
||||
}
|
||||
|
||||
static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, std::string errorMessage) {
|
||||
result["error"] = errorMessage;
|
||||
printf("%s\n", result.getJson().c_str());
|
||||
TraceEvent("BackupQueryFailure").detail("OperationId", operationId).detail("Reason", errorMessage);
|
||||
}
|
||||
|
||||
// If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for
|
||||
// selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overriden to the version
|
||||
// resolved to that timestamp.
|
||||
ACTOR Future<Void> queryBackup(const char* name, std::string destinationContainer,
|
||||
Standalone<VectorRef<KeyRangeRef>> keyRangesFilter, Version restoreVersion,
|
||||
std::string originalClusterFile, std::string restoreTimestamp, bool verbose) {
|
||||
state UID operationId = deterministicRandom()->randomUniqueID();
|
||||
state JsonBuilderObject result;
|
||||
state std::string errorMessage;
|
||||
result["key_ranges_filter"] = printable(keyRangesFilter);
|
||||
result["destination_container"] = destinationContainer;
|
||||
|
||||
TraceEvent("BackupQueryStart")
|
||||
.detail("OperationId", operationId)
|
||||
.detail("DestinationContainer", destinationContainer)
|
||||
.detail("KeyRangesFilter", printable(keyRangesFilter))
|
||||
.detail("SpecifiedRestoreVersion", restoreVersion)
|
||||
.detail("RestoreTimestamp", restoreTimestamp)
|
||||
.detail("BackupClusterFile", originalClusterFile);
|
||||
|
||||
// Resolve restoreTimestamp if given
|
||||
if (!restoreTimestamp.empty()) {
|
||||
if (originalClusterFile.empty()) {
|
||||
reportBackupQueryError(
|
||||
operationId, result,
|
||||
format("an original cluster file must be given in order to resolve restore target timestamp '%s'",
|
||||
restoreTimestamp.c_str()));
|
||||
return Void();
|
||||
}
|
||||
|
||||
if (!fileExists(originalClusterFile)) {
|
||||
reportBackupQueryError(operationId, result,
|
||||
format("The specified original source database cluster file '%s' does not exist\n",
|
||||
originalClusterFile.c_str()));
|
||||
return Void();
|
||||
}
|
||||
|
||||
Database origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST);
|
||||
Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb));
|
||||
result["restore_timestamp"] = restoreTimestamp;
|
||||
result["restore_timestamp_resolved_version"] = v;
|
||||
restoreVersion = v;
|
||||
}
|
||||
|
||||
try {
|
||||
state Reference<IBackupContainer> bc = openBackupContainer(name, destinationContainer);
|
||||
if (restoreVersion == invalidVersion) {
|
||||
BackupDescription desc = wait(bc->describeBackup());
|
||||
if (desc.maxRestorableVersion.present()) {
|
||||
restoreVersion = desc.maxRestorableVersion.get();
|
||||
// Use continuous log end version for the maximum restorable version for the key ranges.
|
||||
} else if (keyRangesFilter.size() && desc.contiguousLogEnd.present()) {
|
||||
restoreVersion = desc.contiguousLogEnd.get();
|
||||
} else {
|
||||
reportBackupQueryError(
|
||||
operationId, result,
|
||||
errorMessage = format("the backup for the specified key ranges is not restorable to any version"));
|
||||
}
|
||||
}
|
||||
|
||||
if (restoreVersion < 0 && restoreVersion != latestVersion) {
|
||||
reportBackupQueryError(operationId, result,
|
||||
errorMessage =
|
||||
format("the specified restorable version %ld is not valid", restoreVersion));
|
||||
return Void();
|
||||
}
|
||||
Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter));
|
||||
if (fileSet.present()) {
|
||||
int64_t totalRangeFilesSize = 0, totalLogFilesSize = 0;
|
||||
result["restore_version"] = fileSet.get().targetVersion;
|
||||
JsonBuilderArray rangeFilesJson;
|
||||
JsonBuilderArray logFilesJson;
|
||||
for (const auto& rangeFile : fileSet.get().ranges) {
|
||||
JsonBuilderObject object;
|
||||
object["file_name"] = rangeFile.fileName;
|
||||
object["file_size"] = rangeFile.fileSize;
|
||||
object["version"] = rangeFile.version;
|
||||
object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0
|
||||
? "none"
|
||||
: fileSet.get().keyRanges.at(rangeFile.fileName).toString();
|
||||
rangeFilesJson.push_back(object);
|
||||
totalRangeFilesSize += rangeFile.fileSize;
|
||||
}
|
||||
for (const auto& log : fileSet.get().logs) {
|
||||
JsonBuilderObject object;
|
||||
object["file_name"] = log.fileName;
|
||||
object["file_size"] = log.fileSize;
|
||||
object["begin_version"] = log.beginVersion;
|
||||
object["end_version"] = log.endVersion;
|
||||
logFilesJson.push_back(object);
|
||||
totalLogFilesSize += log.fileSize;
|
||||
}
|
||||
|
||||
result["total_range_files_size"] = totalRangeFilesSize;
|
||||
result["total_log_files_size"] = totalLogFilesSize;
|
||||
|
||||
if (verbose) {
|
||||
result["ranges"] = rangeFilesJson;
|
||||
result["logs"] = logFilesJson;
|
||||
}
|
||||
|
||||
TraceEvent("BackupQueryReceivedRestorableFilesSet")
|
||||
.detail("DestinationContainer", destinationContainer)
|
||||
.detail("KeyRangesFilter", printable(keyRangesFilter))
|
||||
.detail("ActualRestoreVersion", fileSet.get().targetVersion)
|
||||
.detail("NumRangeFiles", fileSet.get().ranges.size())
|
||||
.detail("NumLogFiles", fileSet.get().logs.size())
|
||||
.detail("RangeFilesBytes", totalRangeFilesSize)
|
||||
.detail("LogFilesBytes", totalLogFilesSize);
|
||||
} else {
|
||||
reportBackupQueryError(operationId, result, "no restorable files set found for specified key ranges");
|
||||
return Void();
|
||||
}
|
||||
|
||||
} catch (Error& e) {
|
||||
reportBackupQueryError(operationId, result, e.what());
|
||||
return Void();
|
||||
}
|
||||
|
||||
printf("%s\n", result.getJson().c_str());
|
||||
return Void();
|
||||
}
|
||||
|
||||
ACTOR Future<Void> listBackup(std::string baseUrl) {
|
||||
try {
|
||||
std::vector<std::string> containers = wait(IBackupContainer::listContainers(baseUrl));
|
||||
|
|
@ -2769,6 +2961,9 @@ int main(int argc, char* argv[]) {
|
|||
case BACKUP_LIST:
|
||||
args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupListOptions, SO_O_EXACT);
|
||||
break;
|
||||
case BACKUP_QUERY:
|
||||
args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupQueryOptions, SO_O_EXACT);
|
||||
break;
|
||||
case BACKUP_MODIFY:
|
||||
args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupModifyOptions, SO_O_EXACT);
|
||||
break;
|
||||
|
|
@ -2908,6 +3103,7 @@ int main(int argc, char* argv[]) {
|
|||
std::string addPrefix;
|
||||
std::string removePrefix;
|
||||
Standalone<VectorRef<KeyRangeRef>> backupKeys;
|
||||
Standalone<VectorRef<KeyRangeRef>> backupKeysFilter;
|
||||
int maxErrors = 20;
|
||||
Version restoreVersion = invalidVersion;
|
||||
std::string restoreTimestamp;
|
||||
|
|
@ -3128,6 +3324,15 @@ int main(int argc, char* argv[]) {
|
|||
return FDB_EXIT_ERROR;
|
||||
}
|
||||
break;
|
||||
case OPT_BACKUPKEYS_FILTER:
|
||||
try {
|
||||
addKeyRange(args->OptionArg(), backupKeysFilter);
|
||||
}
|
||||
catch (Error &) {
|
||||
printHelpTeaser(argv[0]);
|
||||
return FDB_EXIT_ERROR;
|
||||
}
|
||||
break;
|
||||
case OPT_DESTCONTAINER:
|
||||
destinationContainer = args->OptionArg();
|
||||
// If the url starts with '/' then prepend "file://" for backwards compatibility
|
||||
|
|
@ -3661,6 +3866,12 @@ int main(int argc, char* argv[]) {
|
|||
f = stopAfter( listBackup(baseUrl) );
|
||||
break;
|
||||
|
||||
case BACKUP_QUERY:
|
||||
initTraceFile();
|
||||
f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeysFilter, restoreVersion,
|
||||
restoreClusterFileOrig, restoreTimestamp, !quietDisplay));
|
||||
break;
|
||||
|
||||
case BACKUP_DUMP:
|
||||
initTraceFile();
|
||||
f = stopAfter( dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd) );
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
#include "fdbclient/BackupAgent.actor.h"
|
||||
#include "fdbclient/FDBTypes.h"
|
||||
#include "fdbclient/JsonBuilder.h"
|
||||
#include "flow/Arena.h"
|
||||
#include "flow/Trace.h"
|
||||
#include "flow/UnitTest.h"
|
||||
#include "flow/Hash3.h"
|
||||
|
|
@ -244,7 +245,7 @@ std::string BackupDescription::toJSON() const {
|
|||
* file written will be after the start version of the snapshot's execution.
|
||||
*
|
||||
* Log files are at file paths like
|
||||
* /plogs/...log,startVersion,endVersion,UID,tagID-of-N,blocksize
|
||||
* /plogs/.../log,startVersion,endVersion,UID,tagID-of-N,blocksize
|
||||
* /logs/.../log,startVersion,endVersion,UID,blockSize
|
||||
* where ... is a multi level path which sorts lexically into version order and results in approximately 1
|
||||
* unique folder per day containing about 5,000 files. Logs after FDB 6.3 are stored in "plogs"
|
||||
|
|
@ -1389,24 +1390,57 @@ public:
|
|||
return getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem>::addRef(this), file);
|
||||
}
|
||||
|
||||
ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion) {
|
||||
// Find the most recent keyrange snapshot to end at or before targetVersion
|
||||
state Optional<KeyspaceSnapshotFile> snapshot;
|
||||
std::vector<KeyspaceSnapshotFile> snapshots = wait(bc->listKeyspaceSnapshots());
|
||||
for(auto const &s : snapshots) {
|
||||
if(s.endVersion <= targetVersion)
|
||||
snapshot = s;
|
||||
}
|
||||
ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc,
|
||||
Version targetVersion,
|
||||
VectorRef<KeyRangeRef> keyRangesFilter) {
|
||||
// Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion.
|
||||
state std::vector<KeyspaceSnapshotFile> snapshots = wait(bc->listKeyspaceSnapshots());
|
||||
state int i = snapshots.size() - 1;
|
||||
for (; i >= 0; i--) {
|
||||
// The smallest version of filtered range files >= snapshot beginVersion > targetVersion
|
||||
if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if(snapshot.present()) {
|
||||
state RestorableFileSet restorable;
|
||||
restorable.snapshot = snapshot.get();
|
||||
restorable.targetVersion = targetVersion;
|
||||
state Version minKeyRangeVersion = MAX_VERSION;
|
||||
state Version maxKeyRangeVersion = -1;
|
||||
|
||||
std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>> results =
|
||||
wait(bc->readKeyspaceSnapshot(snapshot.get()));
|
||||
restorable.ranges = std::move(results.first);
|
||||
restorable.keyRanges = std::move(results.second);
|
||||
wait(bc->readKeyspaceSnapshot(snapshots[i]));
|
||||
|
||||
// Old backup does not have metadata about key ranges and can not be filtered with key ranges.
|
||||
if (keyRangesFilter.size() && results.second.empty() && !results.first.empty()) {
|
||||
throw backup_not_filterable_with_key_ranges();
|
||||
}
|
||||
|
||||
// Filter by keyRangesFilter.
|
||||
if (keyRangesFilter.empty()) {
|
||||
restorable.ranges = std::move(results.first);
|
||||
restorable.keyRanges = std::move(results.second);
|
||||
minKeyRangeVersion = snapshots[i].beginVersion;
|
||||
maxKeyRangeVersion = snapshots[i].endVersion;
|
||||
} else {
|
||||
for (const auto& rangeFile : results.first) {
|
||||
const auto& keyRange = results.second.at(rangeFile.fileName);
|
||||
if (keyRange.intersects(keyRangesFilter)) {
|
||||
restorable.ranges.push_back(rangeFile);
|
||||
restorable.keyRanges[rangeFile.fileName] = keyRange;
|
||||
minKeyRangeVersion = std::min(minKeyRangeVersion, rangeFile.version);
|
||||
maxKeyRangeVersion = std::max(maxKeyRangeVersion, rangeFile.version);
|
||||
}
|
||||
}
|
||||
// No range file matches 'keyRangesFilter'.
|
||||
if (restorable.ranges.empty()) {
|
||||
throw backup_not_overlapped_with_keys_filter();
|
||||
}
|
||||
}
|
||||
// 'latestVersion' represents using the minimum restorable version in a snapshot.
|
||||
restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion;
|
||||
// Any version < maxKeyRangeVersion is not restorable.
|
||||
if (restorable.targetVersion < maxKeyRangeVersion) continue;
|
||||
|
||||
restorable.snapshot = snapshots[i];
|
||||
// TODO: Reenable the sanity check after TooManyFiles error is resolved
|
||||
if (false && g_network->isSimulated()) {
|
||||
// Sanity check key ranges
|
||||
|
|
@ -1420,18 +1454,21 @@ public:
|
|||
}
|
||||
}
|
||||
|
||||
// No logs needed if there is a complete key space snapshot at the target version.
|
||||
if (snapshot.get().beginVersion == snapshot.get().endVersion &&
|
||||
snapshot.get().endVersion == targetVersion) {
|
||||
// No logs needed if there is a complete filtered key space snapshot at the target version.
|
||||
if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == restorable.targetVersion) {
|
||||
restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion;
|
||||
TraceEvent("BackupContainerGetRestorableFilesWithoutLogs")
|
||||
.detail("KeyRangeVersion", restorable.targetVersion)
|
||||
.detail("NumberOfRangeFiles", restorable.ranges.size())
|
||||
.detail("KeyRangesFilter", printable(keyRangesFilter));
|
||||
return Optional<RestorableFileSet>(restorable);
|
||||
}
|
||||
|
||||
// FIXME: check if there are tagged logs. for each tag, there is no version gap.
|
||||
state std::vector<LogFile> logs;
|
||||
state std::vector<LogFile> plogs;
|
||||
wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) &&
|
||||
store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true)));
|
||||
wait(store(logs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, false)) &&
|
||||
store(plogs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, true)));
|
||||
|
||||
if (plogs.size() > 0) {
|
||||
logs.swap(plogs);
|
||||
|
|
@ -1443,13 +1480,12 @@ public:
|
|||
|
||||
// Remove duplicated log files that can happen for old epochs.
|
||||
std::vector<LogFile> filtered = filterDuplicates(logs);
|
||||
|
||||
restorable.logs.swap(filtered);
|
||||
// sort by version order again for continuous analysis
|
||||
std::sort(restorable.logs.begin(), restorable.logs.end());
|
||||
if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) {
|
||||
restorable.continuousBeginVersion = snapshot.get().beginVersion;
|
||||
restorable.continuousEndVersion = targetVersion + 1; // not inclusive
|
||||
if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, restorable.targetVersion)) {
|
||||
restorable.continuousBeginVersion = minKeyRangeVersion;
|
||||
restorable.continuousEndVersion = restorable.targetVersion + 1; // not inclusive
|
||||
return Optional<RestorableFileSet>(restorable);
|
||||
}
|
||||
return Optional<RestorableFileSet>();
|
||||
|
|
@ -1457,24 +1493,23 @@ public:
|
|||
|
||||
// List logs in version order so log continuity can be analyzed
|
||||
std::sort(logs.begin(), logs.end());
|
||||
|
||||
// If there are logs and the first one starts at or before the snapshot begin version then proceed
|
||||
if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {
|
||||
if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) {
|
||||
Version end = logs.begin()->endVersion;
|
||||
computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion);
|
||||
if (end >= targetVersion) {
|
||||
computeRestoreEndVersion(logs, &restorable.logs, &end, restorable.targetVersion);
|
||||
if (end >= restorable.targetVersion) {
|
||||
restorable.continuousBeginVersion = logs.begin()->beginVersion;
|
||||
restorable.continuousEndVersion = end;
|
||||
return Optional<RestorableFileSet>(restorable);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Optional<RestorableFileSet>();
|
||||
}
|
||||
|
||||
Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) final {
|
||||
return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion);
|
||||
Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
|
||||
VectorRef<KeyRangeRef> keyRangesFilter) final {
|
||||
return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion, keyRangesFilter);
|
||||
}
|
||||
|
||||
private:
|
||||
|
|
|
|||
|
|
@ -280,9 +280,11 @@ public:
|
|||
|
||||
virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0;
|
||||
|
||||
// Get exactly the files necessary to restore to targetVersion. Returns non-present if
|
||||
// restore to given version is not possible.
|
||||
virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion) = 0;
|
||||
// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
|
||||
// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot. Returns non-present if
|
||||
// restoring to the given version is not possible.
|
||||
virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
|
||||
VectorRef<KeyRangeRef> keyRangesFilter = {}) = 0;
|
||||
|
||||
// Get an IBackupContainer based on a container spec string
|
||||
static Reference<IBackupContainer> openContainer(std::string url);
|
||||
|
|
|
|||
|
|
@ -230,6 +230,7 @@ std::string describe( std::set<T> const& items, int max_items = -1 ) {
|
|||
std::string printable( const StringRef& val );
|
||||
std::string printable( const std::string& val );
|
||||
std::string printable( const KeyRangeRef& range );
|
||||
std::string printable(const VectorRef<KeyRangeRef>& val);
|
||||
std::string printable( const VectorRef<StringRef>& val );
|
||||
std::string printable( const VectorRef<KeyValueRef>& val );
|
||||
std::string printable( const KeyValueRef& val );
|
||||
|
|
@ -261,6 +262,14 @@ struct KeyRangeRef {
|
|||
bool contains( const KeyRef& key ) const { return begin <= key && key < end; }
|
||||
bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; }
|
||||
bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; }
|
||||
bool intersects(const VectorRef<KeyRangeRef>& keysVec) const {
|
||||
for (const auto& keys : keysVec) {
|
||||
if (intersects(keys)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
bool empty() const { return begin == end; }
|
||||
bool singleKeyRange() const { return equalsKeyAfter(begin, end); }
|
||||
|
||||
|
|
|
|||
|
|
@ -148,6 +148,12 @@ std::string printable( const KeyRangeRef& range ) {
|
|||
return printable(range.begin) + " - " + printable(range.end);
|
||||
}
|
||||
|
||||
std::string printable(const VectorRef<KeyRangeRef>& val) {
|
||||
std::string s;
|
||||
for (int i = 0; i < val.size(); i++) s = s + printable(val[i]) + " ";
|
||||
return s;
|
||||
}
|
||||
|
||||
int unhex( char c ) {
|
||||
if (c >= '0' && c <= '9')
|
||||
return c-'0';
|
||||
|
|
|
|||
|
|
@ -748,7 +748,9 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
|
|||
std::cout << "Restore to version: " << request.targetVersion << "\nBackupDesc: \n" << desc.toString() << "\n\n";
|
||||
}
|
||||
|
||||
Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(request.targetVersion));
|
||||
state VectorRef<KeyRangeRef> restoreRanges;
|
||||
restoreRanges.add(request.range);
|
||||
Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(request.targetVersion, restoreRanges));
|
||||
|
||||
if (!restorable.present()) {
|
||||
TraceEvent(SevWarn, "FastRestoreControllerPhaseCollectBackupFiles")
|
||||
|
|
|
|||
|
|
@ -93,12 +93,15 @@ struct AtomicOpsWorkload : TestWorkload {
|
|||
case 8:
|
||||
TEST(true); // Testing atomic MinV2
|
||||
opType = MutationRef::MinV2;
|
||||
break;
|
||||
case 9:
|
||||
TEST(true); // Testing atomic AndV2
|
||||
opType = MutationRef::AndV2;
|
||||
break;
|
||||
// case 10:
|
||||
// TEST(true); // Testing atomic CompareAndClear Not supported yet
|
||||
// opType = MutationRef::CompareAndClear
|
||||
// break;
|
||||
default:
|
||||
ASSERT(false);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -468,6 +468,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
|
|||
.detail("LastBackupContainer", lastBackupContainer->getURL())
|
||||
.detail("RestoreAfter", self->restoreAfter)
|
||||
.detail("BackupTag", printable(self->backupTag));
|
||||
// start restoring
|
||||
|
||||
auto container = IBackupContainer::openContainer(lastBackupContainer->getURL());
|
||||
BackupDescription desc = wait(container->describeBackup());
|
||||
|
|
|
|||
|
|
@ -197,6 +197,8 @@ ERROR( backup_cannot_expire, 2316, "Cannot expire requested data from backup wit
|
|||
ERROR( backup_auth_missing, 2317, "Cannot find authentication details (such as a password or secret key) for the specified Backup Container URL")
|
||||
ERROR( backup_auth_unreadable, 2318, "Cannot read or parse one or more sources of authentication information for Backup Container URLs")
|
||||
ERROR( backup_does_not_exist, 2319, "Backup does not exist")
|
||||
ERROR( backup_not_filterable_with_key_ranges, 2320, "Backup before 6.3 cannot be filtered with key ranges")
|
||||
ERROR( backup_not_overlapped_with_keys_filter, 2321, "Backup key ranges doesn't overlap with key ranges filter")
|
||||
ERROR( restore_invalid_version, 2361, "Invalid restore version")
|
||||
ERROR( restore_corrupted_data, 2362, "Corrupted backup data")
|
||||
ERROR( restore_missing_data, 2363, "Missing backup data")
|
||||
|
|
|
|||
Loading…
Reference in New Issue