2017-05-26 04:48:44 +08:00
|
|
|
/*
|
2019-02-18 07:19:05 +08:00
|
|
|
* BackupAgent.actor.h
|
2017-05-26 04:48:44 +08:00
|
|
|
*
|
|
|
|
* This source file is part of the FoundationDB open source project
|
|
|
|
*
|
|
|
|
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
|
2018-02-22 02:25:11 +08:00
|
|
|
*
|
2017-05-26 04:48:44 +08:00
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
2018-02-22 02:25:11 +08:00
|
|
|
*
|
2017-05-26 04:48:44 +08:00
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
2018-02-22 02:25:11 +08:00
|
|
|
*
|
2017-05-26 04:48:44 +08:00
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#pragma once
|
2019-02-18 07:19:05 +08:00
|
|
|
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_G_H)
|
|
|
|
#define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H
|
2019-02-18 13:59:55 +08:00
|
|
|
#include "fdbclient/BackupAgent.actor.g.h"
|
2019-02-18 07:19:05 +08:00
|
|
|
#elif !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_H)
|
|
|
|
#define FDBCLIENT_BACKUP_AGENT_ACTOR_H
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
#include "flow/flow.h"
|
2019-02-18 07:41:16 +08:00
|
|
|
#include "fdbclient/NativeAPI.actor.h"
|
2018-10-20 01:30:13 +08:00
|
|
|
#include "fdbclient/TaskBucket.h"
|
|
|
|
#include "fdbclient/Notified.h"
|
|
|
|
#include "fdbrpc/IAsyncFile.h"
|
|
|
|
#include "fdbclient/KeyBackedTypes.h"
|
2017-05-26 04:48:44 +08:00
|
|
|
#include <ctime>
|
|
|
|
#include <climits>
|
2018-10-20 01:30:13 +08:00
|
|
|
#include "fdbclient/BackupContainer.h"
|
2019-02-18 07:19:05 +08:00
|
|
|
#include "flow/actorcompiler.h" // has to be last include
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
class BackupAgentBase : NonCopyable {
|
|
|
|
public:
|
2019-03-07 14:34:25 +08:00
|
|
|
// Time formatter for anything backup or restore related
|
2019-03-20 16:18:37 +08:00
|
|
|
static std::string formatTime(int64_t epochs);
|
|
|
|
static int64_t parseTime(std::string timestamp);
|
2019-03-07 14:34:25 +08:00
|
|
|
|
2019-03-11 07:00:01 +08:00
|
|
|
static std::string timeFormat() {
|
|
|
|
return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM";
|
|
|
|
}
|
|
|
|
|
2017-05-26 04:48:44 +08:00
|
|
|
// Type of program being executed
|
|
|
|
enum enumActionResult {
|
|
|
|
RESULT_SUCCESSFUL = 0, RESULT_ERRORED = 1, RESULT_DUPLICATE = 2, RESULT_UNNEEDED = 3
|
|
|
|
};
|
|
|
|
|
|
|
|
enum enumState {
|
2019-03-11 07:00:01 +08:00
|
|
|
STATE_ERRORED = 0, STATE_SUBMITTED = 1, STATE_RUNNING = 2, STATE_RUNNING_DIFFERENTIAL = 3, STATE_COMPLETED = 4, STATE_NEVERRAN = 5, STATE_ABORTED = 6, STATE_PARTIALLY_ABORTED = 7
|
2017-05-26 04:48:44 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
static const Key keyFolderId;
|
|
|
|
static const Key keyBeginVersion;
|
|
|
|
static const Key keyEndVersion;
|
2018-02-21 05:22:31 +08:00
|
|
|
static const Key keyPrevBeginVersion;
|
2017-05-26 04:48:44 +08:00
|
|
|
static const Key keyConfigBackupTag;
|
|
|
|
static const Key keyConfigLogUid;
|
|
|
|
static const Key keyConfigBackupRanges;
|
|
|
|
static const Key keyConfigStopWhenDoneKey;
|
|
|
|
static const Key keyStateStatus;
|
|
|
|
static const Key keyStateStop;
|
|
|
|
static const Key keyLastUid;
|
|
|
|
static const Key keyBeginKey;
|
|
|
|
static const Key keyEndKey;
|
2018-04-27 08:24:40 +08:00
|
|
|
static const Key keyDrVersion;
|
2018-02-21 05:22:31 +08:00
|
|
|
static const Key destUid;
|
2018-03-14 02:21:24 +08:00
|
|
|
static const Key backupStartVersion;
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
static const Key keyTagName;
|
|
|
|
static const Key keyStates;
|
|
|
|
static const Key keyConfig;
|
|
|
|
static const Key keyErrors;
|
|
|
|
static const Key keyRanges;
|
|
|
|
static const Key keyTasks;
|
|
|
|
static const Key keyFutures;
|
|
|
|
static const Key keySourceStates;
|
|
|
|
static const Key keySourceTagName;
|
|
|
|
|
|
|
|
static const int logHeaderSize;
|
|
|
|
|
|
|
|
// Convert the status text to an enumerated value
|
|
|
|
static enumState getState(std::string stateText)
|
|
|
|
{
|
|
|
|
enumState enState = STATE_ERRORED;
|
|
|
|
|
|
|
|
if (stateText.empty()) {
|
|
|
|
enState = STATE_NEVERRAN;
|
|
|
|
}
|
|
|
|
|
|
|
|
else if (!stateText.compare("has been submitted")) {
|
|
|
|
enState = STATE_SUBMITTED;
|
|
|
|
}
|
|
|
|
|
|
|
|
else if (!stateText.compare("has been started")) {
|
2019-03-11 07:00:01 +08:00
|
|
|
enState = STATE_RUNNING;
|
2017-05-26 04:48:44 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
else if (!stateText.compare("is differential")) {
|
2019-03-11 07:00:01 +08:00
|
|
|
enState = STATE_RUNNING_DIFFERENTIAL;
|
2017-05-26 04:48:44 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
else if (!stateText.compare("has been completed")) {
|
|
|
|
enState = STATE_COMPLETED;
|
|
|
|
}
|
|
|
|
|
|
|
|
else if (!stateText.compare("has been aborted")) {
|
|
|
|
enState = STATE_ABORTED;
|
|
|
|
}
|
|
|
|
|
|
|
|
else if (!stateText.compare("has been partially aborted")) {
|
|
|
|
enState = STATE_PARTIALLY_ABORTED;
|
|
|
|
}
|
|
|
|
|
|
|
|
return enState;
|
|
|
|
}
|
|
|
|
|
2019-03-11 07:00:01 +08:00
|
|
|
// Convert the status enum to a text description
|
2017-05-26 04:48:44 +08:00
|
|
|
static const char* getStateText(enumState enState)
|
|
|
|
{
|
|
|
|
const char* stateText;
|
|
|
|
|
|
|
|
switch (enState)
|
|
|
|
{
|
|
|
|
case STATE_ERRORED:
|
|
|
|
stateText = "has errored";
|
|
|
|
break;
|
|
|
|
case STATE_NEVERRAN:
|
|
|
|
stateText = "has never been started";
|
|
|
|
break;
|
|
|
|
case STATE_SUBMITTED:
|
|
|
|
stateText = "has been submitted";
|
|
|
|
break;
|
2019-03-11 07:00:01 +08:00
|
|
|
case STATE_RUNNING:
|
2017-05-26 04:48:44 +08:00
|
|
|
stateText = "has been started";
|
|
|
|
break;
|
2019-03-11 07:00:01 +08:00
|
|
|
case STATE_RUNNING_DIFFERENTIAL:
|
2017-05-26 04:48:44 +08:00
|
|
|
stateText = "is differential";
|
|
|
|
break;
|
|
|
|
case STATE_COMPLETED:
|
|
|
|
stateText = "has been completed";
|
|
|
|
break;
|
|
|
|
case STATE_ABORTED:
|
|
|
|
stateText = "has been aborted";
|
|
|
|
break;
|
|
|
|
case STATE_PARTIALLY_ABORTED:
|
|
|
|
stateText = "has been partially aborted";
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
stateText = "<undefined>";
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return stateText;
|
|
|
|
}
|
|
|
|
|
2019-03-11 07:00:01 +08:00
|
|
|
// Convert the status enum to a name
|
|
|
|
static const char* getStateName(enumState enState)
|
|
|
|
{
|
|
|
|
const char* s;
|
|
|
|
|
|
|
|
switch (enState)
|
|
|
|
{
|
|
|
|
case STATE_ERRORED:
|
|
|
|
s = "Errored";
|
|
|
|
break;
|
|
|
|
case STATE_NEVERRAN:
|
|
|
|
s = "NeverRan";
|
|
|
|
break;
|
|
|
|
case STATE_SUBMITTED:
|
|
|
|
s = "Submitted";
|
|
|
|
break;
|
|
|
|
case STATE_RUNNING:
|
|
|
|
s = "Running";
|
|
|
|
break;
|
|
|
|
case STATE_RUNNING_DIFFERENTIAL:
|
|
|
|
s = "RunningDifferentially";
|
|
|
|
break;
|
|
|
|
case STATE_COMPLETED:
|
|
|
|
s = "Completed";
|
|
|
|
break;
|
|
|
|
case STATE_ABORTED:
|
|
|
|
s = "Aborted";
|
|
|
|
break;
|
|
|
|
case STATE_PARTIALLY_ABORTED:
|
|
|
|
s = "Aborting";
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
s = "<undefined>";
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2017-05-26 04:48:44 +08:00
|
|
|
// Determine if the specified state is runnable
|
|
|
|
static bool isRunnable(enumState enState)
|
|
|
|
{
|
|
|
|
bool isRunnable = false;
|
|
|
|
|
|
|
|
switch (enState)
|
|
|
|
{
|
|
|
|
case STATE_SUBMITTED:
|
2019-03-11 07:00:01 +08:00
|
|
|
case STATE_RUNNING:
|
|
|
|
case STATE_RUNNING_DIFFERENTIAL:
|
2017-05-26 04:48:44 +08:00
|
|
|
case STATE_PARTIALLY_ABORTED:
|
|
|
|
isRunnable = true;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return isRunnable;
|
|
|
|
}
|
|
|
|
|
2017-10-13 07:12:42 +08:00
|
|
|
static const KeyRef getDefaultTag() {
|
2017-09-06 02:38:40 +08:00
|
|
|
return StringRef(defaultTagName);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const std::string getDefaultTagName() {
|
|
|
|
return defaultTagName;
|
2017-05-26 04:48:44 +08:00
|
|
|
}
|
|
|
|
|
2019-03-07 14:34:25 +08:00
|
|
|
// This is only used for automatic backup name generation
|
2017-05-26 04:48:44 +08:00
|
|
|
static Standalone<StringRef> getCurrentTime() {
|
|
|
|
double t = now();
|
|
|
|
time_t curTime = t;
|
|
|
|
char buffer[128];
|
|
|
|
struct tm* timeinfo;
|
|
|
|
timeinfo = localtime(&curTime);
|
|
|
|
strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo);
|
|
|
|
|
|
|
|
std::string time(buffer);
|
|
|
|
return StringRef(time + format(".%06d", (int)(1e6*(t - curTime))));
|
|
|
|
}
|
|
|
|
|
|
|
|
protected:
|
2017-09-06 02:38:40 +08:00
|
|
|
static const std::string defaultTagName;
|
2017-05-26 04:48:44 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
class FileBackupAgent : public BackupAgentBase {
|
|
|
|
public:
|
|
|
|
FileBackupAgent();
|
|
|
|
|
2019-01-26 08:49:59 +08:00
|
|
|
FileBackupAgent( FileBackupAgent&& r ) BOOST_NOEXCEPT :
|
2017-05-26 04:48:44 +08:00
|
|
|
subspace( std::move(r.subspace) ),
|
|
|
|
config( std::move(r.config) ),
|
|
|
|
lastRestorable( std::move(r.lastRestorable) ),
|
|
|
|
taskBucket( std::move(r.taskBucket) ),
|
2017-09-02 04:50:38 +08:00
|
|
|
futureBucket( std::move(r.futureBucket) ) {}
|
2017-05-26 04:48:44 +08:00
|
|
|
|
2019-01-26 08:49:59 +08:00
|
|
|
void operator=( FileBackupAgent&& r ) BOOST_NOEXCEPT {
|
2017-05-26 04:48:44 +08:00
|
|
|
subspace = std::move(r.subspace);
|
|
|
|
config = std::move(r.config);
|
|
|
|
lastRestorable = std::move(r.lastRestorable),
|
|
|
|
taskBucket = std::move(r.taskBucket);
|
|
|
|
futureBucket = std::move(r.futureBucket);
|
|
|
|
}
|
|
|
|
|
2017-08-31 09:05:50 +08:00
|
|
|
KeyBackedProperty<Key> lastBackupTimestamp() {
|
|
|
|
return config.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-05-26 04:48:44 +08:00
|
|
|
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) {
|
|
|
|
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
|
|
|
|
}
|
|
|
|
|
|
|
|
/** RESTORE **/
|
2017-12-21 08:54:57 +08:00
|
|
|
|
2017-05-26 04:48:44 +08:00
|
|
|
enum ERestoreState { UNITIALIZED = 0, QUEUED = 1, STARTING = 2, RUNNING = 3, COMPLETED = 4, ABORTED = 5 };
|
|
|
|
static StringRef restoreStateText(ERestoreState id);
|
|
|
|
|
|
|
|
// restore() will
|
|
|
|
// - make sure that url is readable and appears to be a complete backup
|
|
|
|
// - make sure the requested TargetVersion is valid
|
|
|
|
// - submit a restore on the given tagName
|
|
|
|
// - Optionally wait for the restore's completion. Will restore_error if restore fails or is aborted.
|
|
|
|
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable version for the given url.
|
2019-03-06 18:45:55 +08:00
|
|
|
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete = true, Version targetVersion = -1, bool verbose = true, Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true);
|
|
|
|
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, bool waitForComplete = true, Version targetVersion = -1, bool verbose = true, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true) {
|
2019-02-26 10:00:28 +08:00
|
|
|
Standalone<VectorRef<KeyRangeRef>> rangeRef;
|
|
|
|
rangeRef.push_back_deep(rangeRef.arena(), range);
|
2019-03-06 18:45:55 +08:00
|
|
|
return restore(cx, cxOrig, tagName, url, rangeRef, waitForComplete, targetVersion, verbose, addPrefix, removePrefix, lockDB);
|
2019-02-26 10:00:28 +08:00
|
|
|
}
|
|
|
|
Future<Version> atomicRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix = Key(), Key removePrefix = Key());
|
|
|
|
Future<Version> atomicRestore(Database cx, Key tagName, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key()) {
|
|
|
|
Standalone<VectorRef<KeyRangeRef>> rangeRef;
|
|
|
|
rangeRef.push_back_deep(rangeRef.arena(), range);
|
|
|
|
return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
|
|
|
|
}
|
2017-05-26 04:48:44 +08:00
|
|
|
// Tries to abort the restore for a tag. Returns the final (stable) state of the tag.
|
|
|
|
Future<ERestoreState> abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName);
|
2017-12-05 09:21:43 +08:00
|
|
|
Future<ERestoreState> abortRestore(Database cx, Key tagName);
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
// Waits for a restore tag to reach a final (stable) state.
|
|
|
|
Future<ERestoreState> waitRestore(Database cx, Key tagName, bool verbose);
|
|
|
|
|
|
|
|
// Get a string describing the status of a tag
|
|
|
|
Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName);
|
|
|
|
Future<std::string> restoreStatus(Database cx, Key tagName) {
|
|
|
|
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return restoreStatus(tr, tagName); });
|
|
|
|
}
|
|
|
|
|
|
|
|
/** BACKUP METHODS **/
|
2017-12-21 08:54:57 +08:00
|
|
|
|
2017-12-14 17:44:38 +08:00
|
|
|
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true);
|
|
|
|
Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true) {
|
|
|
|
return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone); });
|
2017-05-26 04:48:44 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
|
|
|
|
Future<Void> discontinueBackup(Database cx, Key tagName) {
|
|
|
|
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
|
|
|
|
}
|
|
|
|
|
2017-12-21 08:54:57 +08:00
|
|
|
// Terminate an ongoing backup, without waiting for the backup to finish.
|
|
|
|
// Preconditions:
|
|
|
|
// A backup is running with the tag of `tagName`.
|
|
|
|
// Otherwise `backup_unneeded` will be thrown indicating that the backup never existed or already finished.
|
|
|
|
// Postconditions:
|
|
|
|
// No more tasks will be spawned to backup ranges of the database.
|
|
|
|
// logRangesRange and backupLogKeys will be cleared for this backup.
|
2017-09-06 02:38:40 +08:00
|
|
|
Future<Void> abortBackup(Reference<ReadYourWritesTransaction> tr, std::string tagName);
|
|
|
|
Future<Void> abortBackup(Database cx, std::string tagName) {
|
2017-05-26 04:48:44 +08:00
|
|
|
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return abortBackup(tr, tagName); });
|
|
|
|
}
|
|
|
|
|
2018-02-17 11:38:31 +08:00
|
|
|
Future<std::string> getStatus(Database cx, bool showErrors, std::string tagName);
|
2019-03-07 13:32:46 +08:00
|
|
|
Future<std::string> getStatusJSON(Database cx, std::string tagName);
|
2017-05-26 04:48:44 +08:00
|
|
|
|
2019-12-19 16:29:35 +08:00
|
|
|
Future<Version> getLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot = false);
|
2017-11-15 15:33:17 +08:00
|
|
|
void setLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName, Version version);
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
|
|
|
|
// will return when the backup directory is restorable.
|
2019-01-09 10:12:15 +08:00
|
|
|
Future<int> waitBackup(Database cx, std::string tagName, bool stopWhenDone = true, Reference<IBackupContainer> *pContainer = nullptr, UID *pUID = nullptr);
|
2017-05-26 04:48:44 +08:00
|
|
|
|
2017-10-28 05:06:15 +08:00
|
|
|
static const Key keyLastRestorable;
|
|
|
|
|
2017-05-26 04:48:44 +08:00
|
|
|
Future<int64_t> getTaskCount(Reference<ReadYourWritesTransaction> tr) { return taskBucket->getTaskCount(tr); }
|
|
|
|
Future<int64_t> getTaskCount(Database cx) { return taskBucket->getTaskCount(cx); }
|
|
|
|
Future<Void> watchTaskCount(Reference<ReadYourWritesTransaction> tr) { return taskBucket->watchTaskCount(tr); }
|
|
|
|
|
|
|
|
Future<bool> checkActive(Database cx) { return taskBucket->checkActive(cx); }
|
|
|
|
|
|
|
|
friend class FileBackupAgentImpl;
|
|
|
|
static const int dataFooterSize;
|
|
|
|
|
|
|
|
Subspace subspace;
|
|
|
|
Subspace config;
|
|
|
|
Subspace lastRestorable;
|
|
|
|
|
|
|
|
Reference<TaskBucket> taskBucket;
|
|
|
|
Reference<FutureBucket> futureBucket;
|
|
|
|
};
|
|
|
|
|
|
|
|
class DatabaseBackupAgent : public BackupAgentBase {
|
|
|
|
public:
|
|
|
|
DatabaseBackupAgent();
|
|
|
|
explicit DatabaseBackupAgent(Database src);
|
|
|
|
|
2019-01-26 08:49:59 +08:00
|
|
|
DatabaseBackupAgent( DatabaseBackupAgent&& r ) BOOST_NOEXCEPT :
|
2017-05-26 04:48:44 +08:00
|
|
|
subspace( std::move(r.subspace) ),
|
|
|
|
states( std::move(r.states) ),
|
|
|
|
config( std::move(r.config) ),
|
|
|
|
errors( std::move(r.errors) ),
|
|
|
|
ranges( std::move(r.ranges) ),
|
|
|
|
tagNames( std::move(r.tagNames) ),
|
|
|
|
taskBucket( std::move(r.taskBucket) ),
|
|
|
|
futureBucket( std::move(r.futureBucket) ),
|
|
|
|
sourceStates( std::move(r.sourceStates) ),
|
|
|
|
sourceTagNames( std::move(r.sourceTagNames) ) {}
|
|
|
|
|
2019-01-26 08:49:59 +08:00
|
|
|
void operator=( DatabaseBackupAgent&& r ) BOOST_NOEXCEPT {
|
2017-05-26 04:48:44 +08:00
|
|
|
subspace = std::move(r.subspace);
|
|
|
|
states = std::move(r.states);
|
|
|
|
config = std::move(r.config);
|
|
|
|
errors = std::move(r.errors);
|
|
|
|
ranges = std::move(r.ranges);
|
|
|
|
tagNames = std::move(r.tagNames);
|
|
|
|
taskBucket = std::move(r.taskBucket);
|
|
|
|
futureBucket = std::move(r.futureBucket);
|
|
|
|
sourceStates = std::move(r.sourceStates);
|
|
|
|
sourceTagNames = std::move(r.sourceTagNames);
|
|
|
|
}
|
|
|
|
|
|
|
|
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) {
|
|
|
|
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
|
|
|
|
}
|
|
|
|
|
2019-03-05 07:58:36 +08:00
|
|
|
Future<Void> atomicSwitchover(Database dest, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key addPrefix, Key removePrefix, bool forceAction=false);
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
|
|
|
|
Future<Void> unlockBackup(Database cx, Key tagName) {
|
|
|
|
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return unlockBackup(tr, tagName); });
|
|
|
|
}
|
|
|
|
|
|
|
|
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false);
|
|
|
|
Future<Void> submitBackup(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false) {
|
|
|
|
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync); });
|
|
|
|
}
|
|
|
|
|
|
|
|
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
|
|
|
|
Future<Void> discontinueBackup(Database cx, Key tagName) {
|
|
|
|
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
|
|
|
|
}
|
|
|
|
|
2018-04-27 08:24:40 +08:00
|
|
|
Future<Void> abortBackup(Database cx, Key tagName, bool partial = false, bool abortOldBackup = false);
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
Future<std::string> getStatus(Database cx, int errorLimit, Key tagName);
|
|
|
|
|
2019-12-19 16:29:35 +08:00
|
|
|
Future<int> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
|
2017-05-26 04:48:44 +08:00
|
|
|
Future<int> getStateValue(Database cx, UID logUid) {
|
|
|
|
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getStateValue(tr, logUid); });
|
|
|
|
}
|
|
|
|
|
2019-12-19 16:29:35 +08:00
|
|
|
Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
|
2018-02-21 05:22:31 +08:00
|
|
|
Future<UID> getDestUid(Database cx, UID logUid) {
|
|
|
|
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getDestUid(tr, logUid); });
|
|
|
|
}
|
|
|
|
|
2019-12-19 16:29:35 +08:00
|
|
|
Future<UID> getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot = false);
|
2017-05-26 04:48:44 +08:00
|
|
|
Future<UID> getLogUid(Database cx, Key tagName) {
|
|
|
|
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getLogUid(tr, tagName); });
|
|
|
|
}
|
|
|
|
|
2019-12-19 16:29:35 +08:00
|
|
|
Future<int64_t> getRangeBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
|
|
|
|
Future<int64_t> getLogBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
|
|
|
|
// will return when the backup directory is restorable.
|
|
|
|
Future<int> waitBackup(Database cx, Key tagName, bool stopWhenDone = true);
|
|
|
|
Future<int> waitSubmitted(Database cx, Key tagName);
|
2018-04-27 08:24:40 +08:00
|
|
|
Future<Void> waitUpgradeToLatestDrVersion(Database cx, Key tagName);
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
static const Key keyAddPrefix;
|
|
|
|
static const Key keyRemovePrefix;
|
|
|
|
static const Key keyRangeVersions;
|
|
|
|
static const Key keyCopyStop;
|
|
|
|
static const Key keyDatabasesInSync;
|
2018-04-27 08:24:40 +08:00
|
|
|
static const int LATEST_DR_VERSION;
|
2017-05-26 04:48:44 +08:00
|
|
|
|
|
|
|
Future<int64_t> getTaskCount(Reference<ReadYourWritesTransaction> tr) { return taskBucket->getTaskCount(tr); }
|
|
|
|
Future<int64_t> getTaskCount(Database cx) { return taskBucket->getTaskCount(cx); }
|
|
|
|
Future<Void> watchTaskCount(Reference<ReadYourWritesTransaction> tr) { return taskBucket->watchTaskCount(tr); }
|
|
|
|
|
|
|
|
Future<bool> checkActive(Database cx) { return taskBucket->checkActive(cx); }
|
|
|
|
|
|
|
|
friend class DatabaseBackupAgentImpl;
|
|
|
|
|
|
|
|
Subspace subspace;
|
|
|
|
Subspace states;
|
|
|
|
Subspace config;
|
|
|
|
Subspace errors;
|
|
|
|
Subspace ranges;
|
|
|
|
Subspace tagNames;
|
|
|
|
Subspace sourceStates;
|
|
|
|
Subspace sourceTagNames;
|
|
|
|
|
|
|
|
Reference<TaskBucket> taskBucket;
|
|
|
|
Reference<FutureBucket> futureBucket;
|
|
|
|
};
|
|
|
|
|
|
|
|
typedef std::pair<Standalone<RangeResultRef>, Version> RangeResultWithVersion;
|
|
|
|
|
|
|
|
struct RCGroup {
|
|
|
|
Standalone<RangeResultRef> items;
|
|
|
|
Version version;
|
|
|
|
uint64_t groupKey;
|
|
|
|
|
|
|
|
RCGroup() : version(-1), groupKey(ULLONG_MAX) {};
|
|
|
|
|
|
|
|
template <class Ar>
|
|
|
|
void serialize(Ar& ar) {
|
2018-12-29 02:49:26 +08:00
|
|
|
serializer(ar, items, version, groupKey);
|
2017-05-26 04:48:44 +08:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key);
|
|
|
|
Version getVersionFromString(std::string const& value);
|
2018-02-21 05:22:31 +08:00
|
|
|
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
|
2017-05-26 04:48:44 +08:00
|
|
|
Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid);
|
2019-09-28 09:32:27 +08:00
|
|
|
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion = Optional<Version>(), bool checkBackupUid = false, Version backupUid = 0);
|
2017-05-26 04:48:44 +08:00
|
|
|
Key getApplyKey( Version version, Key backupUid );
|
2019-09-13 06:40:14 +08:00
|
|
|
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key);
|
2017-05-26 04:48:44 +08:00
|
|
|
Standalone<VectorRef<MutationRef>> decodeBackupLogValue(StringRef value);
|
|
|
|
void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int64_t& mutationSize, StringRef value, StringRef addPrefix = StringRef(), StringRef removePrefix = StringRef());
|
|
|
|
Future<Void> logError(Database cx, Key keyErrors, const std::string& message);
|
|
|
|
Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message);
|
|
|
|
Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr);
|
2019-02-18 07:19:05 +08:00
|
|
|
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock,
|
|
|
|
KeyRangeRef range, bool terminator = true, bool systemAccess = false,
|
|
|
|
bool lockAware = false);
|
|
|
|
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active,
|
|
|
|
Reference<FlowLock> lock, KeyRangeRef range,
|
|
|
|
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy, bool terminator = true,
|
|
|
|
bool systemAccess = false, bool lockAware = false);
|
|
|
|
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion,
|
|
|
|
Version* endVersion, RequestStream<CommitTransactionRequest> commit,
|
|
|
|
NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion);
|
2019-09-28 09:32:27 +08:00
|
|
|
ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData);
|
2017-05-26 04:48:44 +08:00
|
|
|
|
2017-09-07 00:46:27 +08:00
|
|
|
typedef BackupAgentBase::enumState EBackupState;
|
|
|
|
template<> inline Tuple Codec<EBackupState>::pack(EBackupState const &val) { return Tuple().append(val); }
|
|
|
|
template<> inline EBackupState Codec<EBackupState>::unpack(Tuple const &val) { return (EBackupState)val.getInt(0); }
|
|
|
|
|
|
|
|
// Key backed tags are a single-key slice of the TagUidMap, defined below.
|
|
|
|
// The Value type of the key is a UidAndAbortedFlagT which is a pair of {UID, aborted_flag}
|
|
|
|
// All tasks on the UID will have a validation key/value that requires aborted_flag to be
|
|
|
|
// false, so changing that value, such as changing the UID or setting aborted_flag to true,
|
|
|
|
// will kill all of the active tasks on that backup/restore UID.
|
|
|
|
typedef std::pair<UID, bool> UidAndAbortedFlagT;
|
|
|
|
class KeyBackedTag : public KeyBackedProperty<UidAndAbortedFlagT> {
|
|
|
|
public:
|
|
|
|
KeyBackedTag() : KeyBackedProperty(StringRef()) {}
|
|
|
|
KeyBackedTag(std::string tagName, StringRef tagMapPrefix);
|
|
|
|
|
|
|
|
Future<Void> cancel(Reference<ReadYourWritesTransaction> tr) {
|
|
|
|
std::string tag = tagName;
|
|
|
|
Key _tagMapPrefix = tagMapPrefix;
|
|
|
|
return map(get(tr), [tag, _tagMapPrefix, tr](Optional<UidAndAbortedFlagT> up) -> Void {
|
|
|
|
if (up.present()) {
|
|
|
|
// Set aborted flag to true
|
|
|
|
up.get().second = true;
|
|
|
|
KeyBackedTag(tag, _tagMapPrefix).set(tr, up.get());
|
|
|
|
}
|
|
|
|
return Void();
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string tagName;
|
|
|
|
Key tagMapPrefix;
|
|
|
|
};
|
|
|
|
|
|
|
|
typedef KeyBackedMap<std::string, UidAndAbortedFlagT> TagMap;
|
|
|
|
// Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace.
|
|
|
|
class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
|
|
|
|
public:
|
|
|
|
TagUidMap(const StringRef & prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
|
|
|
|
|
2019-12-19 16:29:35 +08:00
|
|
|
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap, Reference<ReadYourWritesTransaction> tr, bool snapshot);
|
2017-09-07 00:46:27 +08:00
|
|
|
|
2019-12-19 16:29:35 +08:00
|
|
|
Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
|
|
|
|
return getAll_impl(this, tr, snapshot);
|
2017-09-07 00:46:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
Key prefix;
|
|
|
|
};
|
|
|
|
|
|
|
|
static inline KeyBackedTag makeRestoreTag(std::string tagName) {
|
|
|
|
return KeyBackedTag(tagName, fileRestorePrefixRange.begin);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline KeyBackedTag makeBackupTag(std::string tagName) {
|
|
|
|
return KeyBackedTag(tagName, fileBackupPrefixRange.begin);
|
|
|
|
}
|
|
|
|
|
2019-12-19 16:29:35 +08:00
|
|
|
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
|
|
|
|
return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot);
|
2017-09-07 00:46:27 +08:00
|
|
|
}
|
|
|
|
|
2019-12-19 16:29:35 +08:00
|
|
|
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
|
|
|
|
return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot);
|
2017-09-07 00:46:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
class KeyBackedConfig {
|
|
|
|
public:
|
|
|
|
static struct {
|
|
|
|
static TaskParam<UID> uid() {return LiteralStringRef(__FUNCTION__); }
|
|
|
|
} TaskParams;
|
|
|
|
|
|
|
|
KeyBackedConfig(StringRef prefix, UID uid = UID()) :
|
|
|
|
uid(uid),
|
|
|
|
prefix(prefix),
|
|
|
|
configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
|
|
|
|
|
|
|
|
KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {}
|
|
|
|
|
2018-02-21 05:22:31 +08:00
|
|
|
Future<Void> toTask(Reference<ReadYourWritesTransaction> tr, Reference<Task> task, bool setValidation = true) {
|
2017-09-07 00:46:27 +08:00
|
|
|
// Set the uid task parameter
|
|
|
|
TaskParams.uid().set(task, uid);
|
2018-02-21 05:22:31 +08:00
|
|
|
|
|
|
|
if (!setValidation) {
|
|
|
|
return Void();
|
|
|
|
}
|
|
|
|
|
2017-09-07 00:46:27 +08:00
|
|
|
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the restore uid.
|
|
|
|
// Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the validation key
|
|
|
|
// which TaskBucket will check, and its value must be this restore config's uid.
|
|
|
|
UID u = uid; // 'this' could be invalid in lambda
|
|
|
|
Key p = prefix;
|
|
|
|
return map(tag().get(tr), [u,p,task](Optional<std::string> const &tag) -> Void {
|
|
|
|
if(!tag.present())
|
|
|
|
throw restore_error();
|
|
|
|
// Validation contition is that the uidPair key must be exactly {u, false}
|
|
|
|
TaskBucket::setValidationCondition(task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({u, false}).pack());
|
|
|
|
return Void();
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
KeyBackedProperty<std::string> tag() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
|
|
|
UID getUid() { return uid; }
|
|
|
|
|
|
|
|
Key getUidAsKey() { return BinaryWriter::toValue(uid, Unversioned()); }
|
|
|
|
|
|
|
|
void clear(Reference<ReadYourWritesTransaction> tr) {
|
|
|
|
tr->clear(configSpace.range());
|
|
|
|
}
|
|
|
|
|
2017-09-09 07:09:18 +08:00
|
|
|
// lastError is a pair of error message and timestamp expressed as an int64_t
|
2018-02-17 11:38:31 +08:00
|
|
|
KeyBackedProperty<std::pair<std::string, Version>> lastError() {
|
2017-09-09 07:09:18 +08:00
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2018-02-17 11:38:31 +08:00
|
|
|
KeyBackedMap<int64_t, std::pair<std::string, Version>> lastErrorPerType() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Updates the error per type map and the last error property
|
|
|
|
Future<Void> updateErrorInfo(Database cx, Error e, std::string message) {
|
|
|
|
// Avoid capture of this ptr
|
|
|
|
auto © = *this;
|
|
|
|
|
|
|
|
return runRYWTransaction(cx, [=] (Reference<ReadYourWritesTransaction> tr) mutable {
|
|
|
|
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
|
|
|
|
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
|
|
|
|
|
|
|
|
return map(tr->getReadVersion(), [=] (Version v) mutable {
|
|
|
|
copy.lastError().set(tr, {message, v});
|
|
|
|
copy.lastErrorPerType().set(tr, e.code(), {message, v});
|
|
|
|
return Void();
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-09-07 00:46:27 +08:00
|
|
|
protected:
|
|
|
|
UID uid;
|
|
|
|
Key prefix;
|
|
|
|
Subspace configSpace;
|
|
|
|
};
|
|
|
|
|
2017-11-15 15:33:17 +08:00
|
|
|
template<> inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const &bc) {
|
|
|
|
return Tuple().append(StringRef(bc->getURL()));
|
|
|
|
}
|
|
|
|
template<> inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const &val) {
|
|
|
|
return IBackupContainer::openContainer(val.getString(0).toString());
|
|
|
|
}
|
|
|
|
|
2017-09-07 00:46:27 +08:00
|
|
|
class BackupConfig : public KeyBackedConfig {
|
|
|
|
public:
|
|
|
|
BackupConfig(UID uid = UID()) : KeyBackedConfig(fileBackupPrefixRange.begin, uid) {}
|
|
|
|
BackupConfig(Reference<Task> task) : KeyBackedConfig(fileBackupPrefixRange.begin, task) {}
|
|
|
|
|
|
|
|
// rangeFileMap maps a keyrange file's End to its Begin and Filename
|
2017-11-15 15:33:17 +08:00
|
|
|
struct RangeSlice {
|
|
|
|
Key begin;
|
|
|
|
Version version;
|
|
|
|
std::string fileName;
|
|
|
|
int64_t fileSize;
|
|
|
|
Tuple pack() const {
|
|
|
|
return Tuple().append(begin).append(version).append(StringRef(fileName)).append(fileSize);
|
|
|
|
}
|
|
|
|
static RangeSlice unpack(Tuple const &t) {
|
|
|
|
RangeSlice r;
|
|
|
|
int i = 0;
|
|
|
|
r.begin = t.getString(i++);
|
|
|
|
r.version = t.getInt(i++);
|
|
|
|
r.fileName = t.getString(i++).toString();
|
|
|
|
r.fileSize = t.getInt(i++);
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2017-12-18 06:29:57 +08:00
|
|
|
// Map of range end boundaries to info about the backup file written for that range.
|
2017-11-15 15:33:17 +08:00
|
|
|
typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT;
|
2017-12-18 06:29:57 +08:00
|
|
|
RangeFileMapT snapshotRangeFileMap() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2018-11-23 21:23:56 +08:00
|
|
|
// Number of kv range files that were both committed to persistent storage AND inserted into
|
|
|
|
// the snapshotRangeFileMap. Note that since insertions could replace 1 or more existing
|
|
|
|
// map entries this is not necessarily the number of entries currently in the map.
|
|
|
|
// This value exists to help with sizing of kv range folders for BackupContainers that
|
|
|
|
// require it.
|
|
|
|
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-12-18 06:29:57 +08:00
|
|
|
// Coalesced set of ranges already dispatched for writing.
|
|
|
|
typedef KeyBackedMap<Key, bool> RangeDispatchMapT;
|
|
|
|
RangeDispatchMapT snapshotRangeDispatchMap() {
|
2017-09-07 00:46:27 +08:00
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
2017-12-14 17:44:38 +08:00
|
|
|
}
|
|
|
|
|
2017-12-15 17:39:50 +08:00
|
|
|
// Interval to use for determining the target end version for new snapshots
|
2017-12-14 17:44:38 +08:00
|
|
|
KeyBackedProperty<int64_t> snapshotIntervalSeconds() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-12-15 17:39:50 +08:00
|
|
|
// When the current snapshot began
|
2017-12-14 17:44:38 +08:00
|
|
|
KeyBackedProperty<Version> snapshotBeginVersion() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-12-15 17:39:50 +08:00
|
|
|
// When the current snapshot is desired to end.
|
|
|
|
// This can be changed at runtime to speed up or slow down a snapshot
|
2017-12-14 17:44:38 +08:00
|
|
|
KeyBackedProperty<Version> snapshotTargetEndVersion() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-12-15 17:39:50 +08:00
|
|
|
KeyBackedProperty<int64_t> snapshotBatchSize() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-12-14 17:44:38 +08:00
|
|
|
KeyBackedProperty<Key> snapshotBatchFuture() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
|
|
|
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2019-03-11 16:25:51 +08:00
|
|
|
KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() {
|
2019-03-11 07:00:01 +08:00
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
|
|
|
KeyBackedProperty<Version> snapshotDispatchLastVersion() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-12-18 06:29:57 +08:00
|
|
|
Future<Void> initNewSnapshot(Reference<ReadYourWritesTransaction> tr, int64_t intervalSeconds = -1) {
|
|
|
|
BackupConfig © = *this; // Capture this by value instead of this ptr
|
|
|
|
|
|
|
|
Future<Version> beginVersion = tr->getReadVersion();
|
|
|
|
Future<int64_t> defaultInterval = 0;
|
|
|
|
if(intervalSeconds < 0)
|
|
|
|
defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr);
|
|
|
|
|
|
|
|
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot config members
|
|
|
|
return map(success(beginVersion) && success(defaultInterval), [=](Void) mutable {
|
|
|
|
copy.snapshotRangeFileMap().clear(tr);
|
|
|
|
copy.snapshotRangeDispatchMap().clear(tr);
|
|
|
|
copy.snapshotBatchSize().clear(tr);
|
|
|
|
copy.snapshotBatchFuture().clear(tr);
|
|
|
|
copy.snapshotBatchDispatchDoneKey().clear(tr);
|
|
|
|
|
|
|
|
if(intervalSeconds < 0)
|
|
|
|
intervalSeconds = defaultInterval.get();
|
|
|
|
Version endVersion = beginVersion.get() + intervalSeconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
|
|
|
|
|
|
|
|
copy.snapshotBeginVersion().set(tr, beginVersion.get());
|
|
|
|
copy.snapshotTargetEndVersion().set(tr, endVersion);
|
2018-11-23 21:23:56 +08:00
|
|
|
copy.snapshotRangeFileCount().set(tr, 0);
|
2019-03-11 07:00:01 +08:00
|
|
|
copy.snapshotDispatchLastVersion().clear(tr);
|
|
|
|
copy.snapshotDispatchLastShardsBehind().clear(tr);
|
2017-12-18 06:29:57 +08:00
|
|
|
|
|
|
|
return Void();
|
|
|
|
});
|
2017-09-07 00:46:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
KeyBackedBinaryValue<int64_t> rangeBytesWritten() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
|
|
|
KeyBackedBinaryValue<int64_t> logBytesWritten() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
|
|
|
KeyBackedProperty<EBackupState> stateEnum() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-11-15 15:33:17 +08:00
|
|
|
KeyBackedProperty<Reference<IBackupContainer>> backupContainer() {
|
2017-09-07 00:46:27 +08:00
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stop differntial logging if already started or don't start after completing KV ranges
|
|
|
|
KeyBackedProperty<bool> stopWhenDone() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-12-18 06:29:57 +08:00
|
|
|
// Latest version for which all prior versions have had their log copy tasks completed
|
|
|
|
KeyBackedProperty<Version> latestLogEndVersion() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
|
|
|
// The end version of the last complete snapshot
|
|
|
|
KeyBackedProperty<Version> latestSnapshotEndVersion() {
|
2017-09-07 00:46:27 +08:00
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-12-22 09:09:21 +08:00
|
|
|
// The end version of the first complete snapshot
|
|
|
|
KeyBackedProperty<Version> firstSnapshotEndVersion() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2018-02-21 05:22:31 +08:00
|
|
|
KeyBackedProperty<Key> destUidValue() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2017-12-18 06:29:57 +08:00
|
|
|
Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) {
|
2017-12-22 07:45:10 +08:00
|
|
|
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
|
|
|
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
|
2017-12-18 06:29:57 +08:00
|
|
|
auto lastLog = latestLogEndVersion().get(tr);
|
2017-12-22 09:09:21 +08:00
|
|
|
auto firstSnapshot = firstSnapshotEndVersion().get(tr);
|
|
|
|
return map(success(lastLog) && success(firstSnapshot), [=](Void) -> Optional<Version> {
|
|
|
|
// The latest log greater than the oldest snapshot is the restorable version
|
2018-09-22 02:48:28 +08:00
|
|
|
if(lastLog.get().present() && firstSnapshot.get().present() && lastLog.get().get() > firstSnapshot.get().get()) {
|
|
|
|
return std::max(lastLog.get().get() - 1, firstSnapshot.get().get());
|
2017-12-22 07:45:10 +08:00
|
|
|
}
|
2017-12-18 06:29:57 +08:00
|
|
|
return {};
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-09-07 00:46:27 +08:00
|
|
|
KeyBackedProperty<std::vector<KeyRange>> backupRanges() {
|
|
|
|
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
|
|
|
}
|
|
|
|
|
2018-02-21 05:22:31 +08:00
|
|
|
void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) {
|
|
|
|
Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin);
|
2018-03-14 02:21:24 +08:00
|
|
|
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())), logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
|
2017-09-07 00:46:27 +08:00
|
|
|
}
|
2017-09-09 07:09:18 +08:00
|
|
|
|
|
|
|
Future<Void> logError(Database cx, Error e, std::string details, void *taskInstance = nullptr) {
|
|
|
|
if(!uid.isValid()) {
|
|
|
|
TraceEvent(SevError, "FileBackupErrorNoUID").error(e).detail("Description", details);
|
|
|
|
return Void();
|
|
|
|
}
|
2017-11-19 20:34:28 +08:00
|
|
|
TraceEvent t(SevWarn, "FileBackupError");
|
|
|
|
t.error(e).detail("BackupUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance);
|
|
|
|
// These should not happen
|
|
|
|
if(e.code() == error_code_key_not_found)
|
|
|
|
t.backtrace();
|
2018-02-17 11:38:31 +08:00
|
|
|
|
2018-03-01 10:22:45 +08:00
|
|
|
return updateErrorInfo(cx, e, details);
|
2017-09-09 07:09:18 +08:00
|
|
|
}
|
2017-09-07 00:46:27 +08:00
|
|
|
};
|
2019-01-25 11:25:10 +08:00
|
|
|
|
2019-09-04 06:50:21 +08:00
|
|
|
ACTOR Future<Version> fastRestore(Database cx, Standalone<StringRef> tagName, Standalone<StringRef> url,
|
|
|
|
bool waitForComplete, long targetVersion, bool verbose, Standalone<KeyRangeRef> range,
|
|
|
|
Standalone<StringRef> addPrefix, Standalone<StringRef> removePrefix);
|
2019-01-25 11:25:10 +08:00
|
|
|
|
2019-10-26 07:01:09 +08:00
|
|
|
// Helper class for reading restore data from a buffer and throwing the right errors.
|
|
|
|
struct StringRefReader {
|
|
|
|
StringRefReader(StringRef s = StringRef(), Error e = Error()) : rptr(s.begin()), end(s.end()), failure_error(e) {}
|
|
|
|
|
|
|
|
// Return remainder of data as a StringRef
|
|
|
|
StringRef remainder() { return StringRef(rptr, end - rptr); }
|
|
|
|
|
|
|
|
// Return a pointer to len bytes at the current read position and advance read pos
|
|
|
|
const uint8_t* consume(unsigned int len) {
|
|
|
|
if (rptr == end && len != 0) throw end_of_stream();
|
|
|
|
const uint8_t* p = rptr;
|
|
|
|
rptr += len;
|
|
|
|
if (rptr > end) throw failure_error;
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return a T from the current read position and advance read pos
|
|
|
|
template <typename T>
|
|
|
|
const T consume() {
|
|
|
|
return *(const T*)consume(sizeof(T));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Functions for consuming big endian (network byte order) integers.
|
|
|
|
// Consumes a big endian number, swaps it to little endian, and returns it.
|
|
|
|
const int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
|
|
|
|
const uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
|
|
|
|
|
|
|
|
bool eof() { return rptr == end; }
|
|
|
|
|
|
|
|
const uint8_t *rptr, *end;
|
|
|
|
Error failure_error;
|
|
|
|
};
|
|
|
|
|
2019-06-27 04:33:19 +08:00
|
|
|
#include "flow/unactorcompiler.h"
|
2017-05-26 04:48:44 +08:00
|
|
|
#endif
|