wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4620 from RenxuanW/renxuan/first-pr 

Control backup's initial snapshot interval via backup cmd argument.
This commit is contained in:
Steve Atherton 2021-04-19 23:59:02 -07:00 committed by GitHub
parent 3974d6337d 41104040ad
commit 75425b5a24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 52 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
documentation/sphinx/source/backups.rst
View File

@ -244,6 +244,9 @@ The ``start`` subcommand is used to start a backup. If there is already a backu
``-s <DURATION>`` or ``--snapshot_interval <DURATION>``
Specifies the duration, in seconds, of the inconsistent snapshots written to the backup in continuous mode. The default is 864000 which is 10 days.
``--initial_snapshot_interval <DURATION>``
Specifies the duration, in seconds, of the first inconsistent snapshot written to the backup. The default is 0, which means as fast as possible.
``--partitioned_log_experimental``
Specifies the backup uses the partitioned mutation logs generated by backup workers. Since FDB version 6.3, this option is experimental and requires using fast restore for restoring the database from the generated files. The default is to use non-partitioned mutation logs generated by backup agents.

10
fdbbackup/backup.actor.cpp
View File

@ -106,6 +106,7 @@ enum {
// Backup constants
OPT_DESTCONTAINER,
OPT_SNAPSHOTINTERVAL,
OPT_INITIAL_SNAPSHOT_INTERVAL,
OPT_ERRORLIMIT,
OPT_NOSTOPWHENDONE,
OPT_EXPIRE_BEFORE_VERSION,
@ -233,6 +234,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
{ OPT_USE_PARTITIONED_LOG, "--partitioned_log_experimental", SO_NONE },
{ OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP },
{ OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP },
{ OPT_INITIAL_SNAPSHOT_INTERVAL, "--initial_snapshot_interval", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
{ OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
@ -1880,6 +1882,7 @@ ACTOR Future<Void> submitDBBackup(Database src,
ACTOR Future<Void> submitBackup(Database db,
std::string url,
int initialSnapshotIntervalSeconds,
int snapshotIntervalSeconds,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
std::string tagName,
@ -1936,6 +1939,7 @@ ACTOR Future<Void> submitBackup(Database db,
else {
wait(backupAgent.submitBackup(db,
KeyRef(url),
initialSnapshotIntervalSeconds,
snapshotIntervalSeconds,
tagName,
backupRanges,
@ -3213,6 +3217,8 @@ int main(int argc, char* argv[]) {
std::string destinationContainer;
bool describeDeep = false;
bool describeTimestamps = false;
int initialSnapshotIntervalSeconds =
0; // The initial snapshot has a desired duration of 0, meaning go as fast as possible.
int snapshotIntervalSeconds = CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC;
std::string clusterFile;
std::string sourceClusterFile;
@ -3467,6 +3473,7 @@ int main(int argc, char* argv[]) {
modifyOptions.destURL = destinationContainer;
break;
case OPT_SNAPSHOTINTERVAL:
case OPT_INITIAL_SNAPSHOT_INTERVAL:
case OPT_MOD_ACTIVE_INTERVAL: {
const char* a = args->OptionArg();
int seconds;
@ -3478,6 +3485,8 @@ int main(int argc, char* argv[]) {
if (optId == OPT_SNAPSHOTINTERVAL) {
snapshotIntervalSeconds = seconds;
modifyOptions.snapshotIntervalSeconds = seconds;
} else if (optId == OPT_INITIAL_SNAPSHOT_INTERVAL) {
initialSnapshotIntervalSeconds = seconds;
} else if (optId == OPT_MOD_ACTIVE_INTERVAL) {
modifyOptions.activeSnapshotIntervalSeconds = seconds;
}
@ -3837,6 +3846,7 @@ int main(int argc, char* argv[]) {
openBackupContainer(argv[0], destinationContainer);
f = stopAfter(submitBackup(db,
destinationContainer,
initialSnapshotIntervalSeconds,
snapshotIntervalSeconds,
backupKeys,
tagName,

14
fdbclient/BackupAgent.actor.h
View File

@ -357,6 +357,7 @@ public:
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Key outContainer,
int initialSnapshotIntervalSeconds,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
@ -365,6 +366,7 @@ public:
bool incrementalBackupOnly = false);
Future<Void> submitBackup(Database cx,
Key outContainer,
int initialSnapshotIntervalSeconds,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
@ -374,6 +376,7 @@ public:
return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(tr,
outContainer,
initialSnapshotIntervalSeconds,
snapshotIntervalSeconds,
tagName,
backupRanges,
@ -404,7 +407,8 @@ public:
Future<std::string> getStatus(Database cx, bool showErrors, std::string tagName);
Future<std::string> getStatusJSON(Database cx, std::string tagName);
Future<Optional<Version>> getLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName,
Future<Optional<Version>> getLastRestorable(Reference<ReadYourWritesTransaction> tr,
Key tagName,
bool snapshot = false);
void setLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName, Version version);
@ -835,6 +839,11 @@ public:
typedef KeyBackedMap<Key, bool> RangeDispatchMapT;
RangeDispatchMapT snapshotRangeDispatchMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Interval to use for the first (initial) snapshot.
KeyBackedProperty<int64_t> initialSnapshotIntervalSeconds() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Interval to use for determining the target end version for new snapshots
KeyBackedProperty<int64_t> snapshotIntervalSeconds() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
@ -864,8 +873,9 @@ public:
Future<Version> beginVersion = tr->getReadVersion();
Future<int64_t> defaultInterval = 0;
if (intervalSeconds < 0)
if (intervalSeconds < 0) {
defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr);
}
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot
// config members

16
fdbclient/FileBackupAgent.actor.cpp
View File

@ -2777,9 +2777,9 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase {
state Reference<TaskFuture> backupFinished = futureBucket->future(tr);
// Initialize the initial snapshot and create tasks to continually write logs and snapshots
// The initial snapshot has a desired duration of 0, meaning go as fast as possible.
wait(config.initNewSnapshot(tr, 0));
// Initialize the initial snapshot and create tasks to continually write logs and snapshots.
state Optional<int64_t> initialSnapshotIntervalSeconds = wait(config.initialSnapshotIntervalSeconds().get(tr));
wait(config.initNewSnapshot(tr, initialSnapshotIntervalSeconds.orDefault(0)));
// Using priority 1 for both of these to at least start both tasks soon
// Do not add snapshot task if we only want the incremental backup
@ -4442,6 +4442,7 @@ public:
ACTOR static Future<Void> submitBackup(FileBackupAgent* backupAgent,
Reference<ReadYourWritesTransaction> tr,
Key outContainer,
int initialSnapshotIntervalSeconds,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
@ -4557,6 +4558,7 @@ public:
config.backupContainer().set(tr, bc);
config.stopWhenDone().set(tr, stopWhenDone);
config.backupRanges().set(tr, normalizedRanges);
config.initialSnapshotIntervalSeconds().set(tr, initialSnapshotIntervalSeconds);
config.snapshotIntervalSeconds().set(tr, snapshotIntervalSeconds);
config.partitionedLogEnabled().set(tr, partitionedLog);
config.incrementalBackupOnly().set(tr, incrementalBackupOnly);
@ -5183,7 +5185,8 @@ public:
}
ACTOR static Future<Optional<Version>> getLastRestorable(FileBackupAgent* backupAgent,
Reference<ReadYourWritesTransaction> tr, Key tagName,
Reference<ReadYourWritesTransaction> tr,
Key tagName,
bool snapshot) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -5544,6 +5547,7 @@ Future<ERestoreState> FileBackupAgent::waitRestore(Database cx, Key tagName, boo
Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction> tr,
Key outContainer,
int initialSnapshotIntervalSeconds,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
@ -5553,6 +5557,7 @@ Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction>
return FileBackupAgentImpl::submitBackup(this,
tr,
outContainer,
initialSnapshotIntervalSeconds,
snapshotIntervalSeconds,
tagName,
backupRanges,
@ -5577,7 +5582,8 @@ Future<std::string> FileBackupAgent::getStatusJSON(Database cx, std::string tagN
return FileBackupAgentImpl::getStatusJSON(this, cx, tagName);
}
Future<Optional<Version>> FileBackupAgent::getLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName,
Future<Optional<Version>> FileBackupAgent::getLastRestorable(Reference<ReadYourWritesTransaction> tr,
Key tagName,
bool snapshot) {
return FileBackupAgentImpl::getLastRestorable(this, tr, tagName, snapshot);
}

1
fdbserver/workloads/AtomicRestore.actor.cpp
View File

@ -93,6 +93,7 @@ struct AtomicRestoreWorkload : TestWorkload {
try {
wait(backupAgent.submitBackup(cx,
StringRef(backupContainer),
deterministicRandom()->randomInt(0, 60),
deterministicRandom()->randomInt(0, 100),
BackupAgentBase::getDefaultTagName(),
self->backupRanges,

2
fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
View File

@ -222,6 +222,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
try {
wait(backupAgent->submitBackup(cx,
StringRef(backupContainer),
deterministicRandom()->randomInt(0, 60),
deterministicRandom()->randomInt(0, 100),
tag.toString(),
backupRanges,
@ -477,6 +478,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
// the configuration to disable backup workers before restore.
extraBackup = backupAgent.submitBackup(cx,
LiteralStringRef("file://simfdb/backups/"),
deterministicRandom()->randomInt(0, 60),
deterministicRandom()->randomInt(0, 100),
self->backupTag.toString(),
self->backupRanges,

2
fdbserver/workloads/BackupCorrectness.actor.cpp
View File

@ -248,6 +248,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
try {
wait(backupAgent->submitBackup(cx,
StringRef(backupContainer),
deterministicRandom()->randomInt(0, 60),
deterministicRandom()->randomInt(0, 100),
tag.toString(),
backupRanges,
@ -497,6 +498,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
try {
extraBackup = backupAgent.submitBackup(cx,
LiteralStringRef("file://simfdb/backups/"),
deterministicRandom()->randomInt(0, 60),
deterministicRandom()->randomInt(0, 100),
self->backupTag.toString(),
self->backupRanges,

9
fdbserver/workloads/BackupToBlob.actor.cpp
View File

@ -29,6 +29,7 @@ struct BackupToBlobWorkload : TestWorkload {
double backupAfter;
Key backupTag;
Standalone<StringRef> backupURL;
int initSnapshotInterval = 0;
int snapshotInterval = 100000;
static constexpr const char* DESCRIPTION = "BackupToBlob";
@ -59,8 +60,12 @@ struct BackupToBlobWorkload : TestWorkload {
backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
wait(delay(self->backupAfter));
wait(backupAgent.submitBackup(
cx, self->backupURL, self->snapshotInterval, self->backupTag.toString(), backupRanges));
wait(backupAgent.submitBackup(cx,
self->backupURL,
self->initSnapshotInterval,
self->snapshotInterval,
self->backupTag.toString(),
backupRanges));
EBackupState backupStatus = wait(backupAgent.waitBackup(cx, self->backupTag.toString(), true));
TraceEvent("BackupToBlob_BackupStatus").detail("Status", BackupAgentBase::getStateText(backupStatus));
return Void();

2
fdbserver/workloads/IncrementalBackup.actor.cpp
View File

@ -151,7 +151,7 @@ struct IncrementalBackupWorkload : TestWorkload {
TraceEvent("IBackupSubmitAttempt");
try {
wait(self->backupAgent.submitBackup(
cx, self->backupDir, 1e8, self->tag.toString(), backupRanges, false, false, true));
cx, self->backupDir, 0, 1e8, self->tag.toString(), backupRanges, false, false, true));
} catch (Error& e) {
TraceEvent("IBackupSubmitError").error(e);
if (e.code() != error_code_backup_duplicate) {

3
fdbserver/workloads/SubmitBackup.actor.cpp
View File

@ -33,6 +33,7 @@ struct SubmitBackupWorkload final : TestWorkload {
Standalone<StringRef> backupDir;
Standalone<StringRef> tag;
double delayFor;
int initSnapshotInterval;
int snapshotInterval;
bool stopWhenDone;
bool incremental;
@ -41,6 +42,7 @@ struct SubmitBackupWorkload final : TestWorkload {
backupDir = getOption(options, LiteralStringRef("backupDir"), LiteralStringRef("file://simfdb/backups/"));
tag = getOption(options, LiteralStringRef("tag"), LiteralStringRef("default"));
delayFor = getOption(options, LiteralStringRef("delayFor"), 10.0);
initSnapshotInterval = getOption(options, LiteralStringRef("initSnapshotInterval"), 0);
snapshotInterval = getOption(options, LiteralStringRef("snapshotInterval"), 1e8);
stopWhenDone = getOption(options, LiteralStringRef("stopWhenDone"), true);
incremental = getOption(options, LiteralStringRef("incremental"), false);
@ -55,6 +57,7 @@ struct SubmitBackupWorkload final : TestWorkload {
try {
wait(self->backupAgent.submitBackup(cx,
self->backupDir,
self->initSnapshotInterval,
self->snapshotInterval,
self->tag.toString(),
backupRanges,