wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FastRestore:Apply Clang-format

This commit is contained in:
Meng Xu 2019-08-01 17:00:13 -07:00
parent c2c2536de2
commit 3b54363780
34 changed files with 1797 additions and 1498 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

224
fdbbackup/backup.actor.cpp
View File

@ -74,7 +74,13 @@ using std::endl;
// Type of program being executed
enum enumProgramExe {
EXE_AGENT, EXE_BACKUP, EXE_RESTORE, EXE_FASTRESTORE_AGENT, EXE_DR_AGENT, EXE_DB_BACKUP, EXE_UNDEFINED
EXE_AGENT,
EXE_BACKUP,
EXE_RESTORE,
EXE_FASTRESTORE_AGENT,
EXE_DR_AGENT,
EXE_DB_BACKUP,
EXE_UNDEFINED
};
enum enumBackupType {
@ -1044,15 +1050,16 @@ static void printRestoreUsage(bool devhelp ) {
return;
}
static void printFastRestoreUsage(bool devhelp ) {
static void printFastRestoreUsage(bool devhelp) {
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
printf("Usage: %s (start | status | abort | wait) [OPTIONS]\n\n", exeRestore.toString().c_str());
//printf(" FOLDERS Paths to folders containing the backup files.\n");
// printf(" FOLDERS Paths to folders containing the backup files.\n");
printf("Options for all commands:\n\n");
printf(" -C CONNFILE The path of a file containing the connection string for the\n"
" FoundationDB cluster. The default is first the value of the\n"
" FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n"
" then `%s'.\n", platform::getDefaultClusterFilePath().c_str());
" FoundationDB cluster. The default is first the value of the\n"
" FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n"
" then `%s'.\n",
platform::getDefaultClusterFilePath().c_str());
printf(" -t TAGNAME The restore tag to act on. Default is 'default'\n");
printf(" --tagname TAGNAME\n\n");
printf(" Options for start:\n\n");
@ -1068,7 +1075,7 @@ static void printFastRestoreUsage(bool devhelp ) {
printf(" -h, --help Display this help and exit.\n");
printf("NOTE: Fast restore is still under development. The options may not be fully supported.\n");
if( devhelp ) {
if (devhelp) {
#ifdef _WIN32
printf(" -q Disable error dialog on crash.\n");
printf(" --parentpid PID\n");
@ -1077,7 +1084,7 @@ static void printFastRestoreUsage(bool devhelp ) {
}
printf("\n"
" KEYS FORMAT: \"<BEGINKEY> <ENDKEY>\" [...]\n");
" KEYS FORMAT: \"<BEGINKEY> <ENDKEY>\" [...]\n");
printf("\n");
puts(BlobCredentialInfo);
@ -1248,23 +1255,23 @@ enumProgramExe getProgramType(std::string programExe)
}
// Check if restore
else if ((programExe.length() >= exeFastRestoreAgent.size()) &&
(programExe.compare(programExe.length() - exeFastRestoreAgent.size(), exeFastRestoreAgent.size(), (const char*)exeFastRestoreAgent.begin()) == 0))
{
else if ((programExe.length() >= exeFastRestoreAgent.size()) &&
(programExe.compare(programExe.length() - exeFastRestoreAgent.size(), exeFastRestoreAgent.size(),
(const char*)exeFastRestoreAgent.begin()) == 0)) {
enProgramExe = EXE_FASTRESTORE_AGENT;
}
// Check if db agent
else if ((programExe.length() >= exeDatabaseAgent.size()) &&
(programExe.compare(programExe.length() - exeDatabaseAgent.size(), exeDatabaseAgent.size(), (const char*)exeDatabaseAgent.begin()) == 0))
{
else if ((programExe.length() >= exeDatabaseAgent.size()) &&
(programExe.compare(programExe.length() - exeDatabaseAgent.size(), exeDatabaseAgent.size(),
(const char*)exeDatabaseAgent.begin()) == 0)) {
enProgramExe = EXE_DR_AGENT;
}
// Check if db backup
else if ((programExe.length() >= exeDatabaseBackup.size()) &&
(programExe.compare(programExe.length() - exeDatabaseBackup.size(), exeDatabaseBackup.size(), (const char*)exeDatabaseBackup.begin()) == 0))
{
else if ((programExe.length() >= exeDatabaseBackup.size()) &&
(programExe.compare(programExe.length() - exeDatabaseBackup.size(), exeDatabaseBackup.size(),
(const char*)exeDatabaseBackup.begin()) == 0)) {
enProgramExe = EXE_DB_BACKUP;
}
@ -2162,13 +2169,15 @@ ACTOR Future<Void> runRestore(Database db, std::string originalClusterFile, std:
}
// Fast restore agent that kicks off the restore: send restore requests to restore workers.
ACTOR Future<Void> runFastRestoreAgent(Database db, std::string tagName, std::string container, Standalone<VectorRef<KeyRangeRef>> ranges, Version dbVersion, bool performRestore, bool verbose, bool waitForDone, std::string addPrefix, std::string removePrefix) {
try
{
ACTOR Future<Void> runFastRestoreAgent(Database db, std::string tagName, std::string container,
Standalone<VectorRef<KeyRangeRef>> ranges, Version dbVersion,
bool performRestore, bool verbose, bool waitForDone, std::string addPrefix,
std::string removePrefix) {
try {
state FileBackupAgent backupAgent;
state int64_t restoreVersion = -1;
if(ranges.size() > 1) {
if (ranges.size() > 1) {
fprintf(stderr, "Currently only a single restore range is supported!\n");
throw restore_error();
}
@ -2178,55 +2187,52 @@ ACTOR Future<Void> runFastRestoreAgent(Database db, std::string tagName, std::st
printf("[INFO] runFastRestoreAgent: num_ranges:%d restore_range:%s\n", ranges.size(), range.toString().c_str());
if (performRestore) {
if(dbVersion == invalidVersion) {
if (dbVersion == invalidVersion) {
BackupDescription desc = wait(IBackupContainer::openContainer(container)->describeBackup());
if(!desc.maxRestorableVersion.present()) {
if (!desc.maxRestorableVersion.present()) {
fprintf(stderr, "The specified backup is not restorable to any version.\n");
throw restore_error();
}
dbVersion = desc.maxRestorableVersion.get();
}
Version _restoreVersion = wait(fastRestore(db, KeyRef(tagName), KeyRef(container), waitForDone, dbVersion, verbose, range, KeyRef(addPrefix), KeyRef(removePrefix)));
Version _restoreVersion = wait(fastRestore(db, KeyRef(tagName), KeyRef(container), waitForDone, dbVersion,
verbose, range, KeyRef(addPrefix), KeyRef(removePrefix)));
restoreVersion = _restoreVersion;
}
else {
} else {
state Reference<IBackupContainer> bc = IBackupContainer::openContainer(container);
state BackupDescription description = wait(bc->describeBackup());
if(dbVersion <= 0) {
if (dbVersion <= 0) {
wait(description.resolveVersionTimes(db));
if(description.maxRestorableVersion.present())
if (description.maxRestorableVersion.present())
restoreVersion = description.maxRestorableVersion.get();
else {
fprintf(stderr, "Backup is not restorable\n");
throw restore_invalid_version();
}
}
else
} else
restoreVersion = dbVersion;
state Optional<RestorableFileSet> rset = wait(bc->getRestoreSet(restoreVersion));
if(!rset.present()) {
if (!rset.present()) {
fprintf(stderr, "Insufficient data to restore to version %lld\n", restoreVersion);
throw restore_invalid_version();
}
// Display the restore information, if requested
if (verbose) {
printf("[DRY RUN] Restoring backup to version: %lld\n", (long long) restoreVersion);
printf("[DRY RUN] Restoring backup to version: %lld\n", (long long)restoreVersion);
printf("%s\n", description.toString().c_str());
}
}
if(waitForDone && verbose) {
if (waitForDone && verbose) {
// If restore completed then report version restored
printf("Restored to version %lld%s\n", (long long) restoreVersion, (performRestore) ? "" : " (DRY RUN)");
printf("Restored to version %lld%s\n", (long long)restoreVersion, (performRestore) ? "" : " (DRY RUN)");
}
}
catch (Error& e) {
if(e.code() == error_code_actor_cancelled)
throw;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw;
fprintf(stderr, "ERROR: %s\n", e.what());
throw;
}
@ -2837,15 +2843,12 @@ int main(int argc, char* argv[]) {
}
// Get the restore operation type
restoreType = getRestoreType(argv[1]);
if(restoreType == RESTORE_UNKNOWN) {
if (restoreType == RESTORE_UNKNOWN) {
// Display help, if requested
if ((strcmp(argv[1], "-h") == 0) ||
(strcmp(argv[1], "--help") == 0) )
{
if ((strcmp(argv[1], "-h") == 0) || (strcmp(argv[1], "--help") == 0)) {
printFastRestoreUsage(false);
return FDB_EXIT_ERROR;
}
else {
} else {
fprintf(stderr, "ERROR: Unsupported restore command: '%s'\n", argv[1]);
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
@ -3276,7 +3279,8 @@ int main(int argc, char* argv[]) {
break;
case EXE_FASTRESTORE_AGENT:
fprintf(stderr, "ERROR: FDB Fast Restore Agent does not support argument value `%s'\n", args->File(argLoop));
fprintf(stderr, "ERROR: FDB Fast Restore Agent does not support argument value `%s'\n",
args->File(argLoop));
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
@ -3676,11 +3680,13 @@ int main(int argc, char* argv[]) {
f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) );
break;
case RESTORE_ABORT:
f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void {
printf("RESTORE_ABORT Tag: %s State: %s\n", tagName.c_str(), FileBackupAgent::restoreStateText(s).toString().c_str());
return Void();
}) );
break;
f = stopAfter(
map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void {
printf("RESTORE_ABORT Tag: %s State: %s\n", tagName.c_str(),
FileBackupAgent::restoreStateText(s).toString().c_str());
return Void();
}));
break;
case RESTORE_STATUS:
// If no tag is specifically provided then print all tag status, don't just use "default"
if(tagProvided)
@ -3696,38 +3702,37 @@ int main(int argc, char* argv[]) {
break;
case EXE_FASTRESTORE_AGENT:
// TODO: We have not implmented the code commented out in this case
if(!initCluster())
return FDB_EXIT_ERROR;
switch(restoreType) {
case RESTORE_START:
f = stopAfter( runFastRestoreAgent(db, tagName, restoreContainer, backupKeys, restoreVersion, !dryRun, !quietDisplay, waitForDone, addPrefix, removePrefix) );
break;
case RESTORE_WAIT:
printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n");
throw restore_error();
// f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) );
break;
case RESTORE_ABORT:
printf("[TODO][ERROR] FastRestore does not support RESTORE_ABORT yet!\n");
throw restore_error();
// f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void {
// printf("Tag: %s State: %s\n", tagName.c_str(), FileBackupAgent::restoreStateText(s).toString().c_str());
// return Void();
// }) );
break;
case RESTORE_STATUS:
printf("[TODO][ERROR] FastRestore does not support RESTORE_STATUS yet!\n");
throw restore_error();
// If no tag is specifically provided then print all tag status, don't just use "default"
if(tagProvided)
tag = tagName;
// f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void {
// printf("%s\n", s.c_str());
// return Void();
// }) );
break;
default:
throw restore_error();
if (!initCluster()) return FDB_EXIT_ERROR;
switch (restoreType) {
case RESTORE_START:
f = stopAfter(runFastRestoreAgent(db, tagName, restoreContainer, backupKeys, restoreVersion, !dryRun,
!quietDisplay, waitForDone, addPrefix, removePrefix));
break;
case RESTORE_WAIT:
printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n");
throw restore_error();
// f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) );
break;
case RESTORE_ABORT:
printf("[TODO][ERROR] FastRestore does not support RESTORE_ABORT yet!\n");
throw restore_error();
// f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)),
//[tagName](FileBackupAgent::ERestoreState s) -> Void { printf("Tag: %s State: %s\n", tagName.c_str(),
//FileBackupAgent::restoreStateText(s).toString().c_str()); return Void();
// }) );
break;
case RESTORE_STATUS:
printf("[TODO][ERROR] FastRestore does not support RESTORE_STATUS yet!\n");
throw restore_error();
// If no tag is specifically provided then print all tag status, don't just use "default"
if (tagProvided) tag = tagName;
// f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void {
// printf("%s\n", s.c_str());
// return Void();
// }) );
break;
default:
throw restore_error();
}
break;
case EXE_DR_AGENT:
@ -3856,20 +3861,20 @@ ACTOR static Future<FileBackupAgent::ERestoreState> waitFastRestore(Database cx,
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
// In case restoreRequestDoneKey is already set before we set watch on it
Optional<Value> restoreRequestDoneKeyValue = wait( tr.get(restoreRequestDoneKey) );
if ( restoreRequestDoneKeyValue.present() ) {
restoreRequestDone = true;
Optional<Value> restoreRequestDoneKeyValue = wait(tr.get(restoreRequestDoneKey));
if (restoreRequestDoneKeyValue.present()) {
restoreRequestDone = true;
tr.clear(restoreRequestDoneKey);
wait( tr.commit() );
wait(tr.commit());
break;
} else {
watch4RestoreRequestDone = tr.watch(restoreRequestDoneKey);
wait( tr.commit() );
wait(tr.commit());
}
// The clear transaction may fail in uncertain state, which may already clear the restoreRequestDoneKey
if ( restoreRequestDone ) break;
} catch( Error &e ) {
wait( tr.onError(e) );
if (restoreRequestDone) break;
} catch (Error& e) {
wait(tr.onError(e));
}
}
@ -3878,22 +3883,23 @@ ACTOR static Future<FileBackupAgent::ERestoreState> waitFastRestore(Database cx,
return FileBackupAgent::ERestoreState::COMPLETED;
}
ACTOR static Future<Version> _fastRestore(Database cx, Key tagName, Key url, bool waitForComplete, Version targetVersion, bool verbose, KeyRange range, Key addPrefix, Key removePrefix) {
ACTOR static Future<Version> _fastRestore(Database cx, Key tagName, Key url, bool waitForComplete,
Version targetVersion, bool verbose, KeyRange range, Key addPrefix,
Key removePrefix) {
state Reference<IBackupContainer> bc = IBackupContainer::openContainer(url.toString());
state BackupDescription desc = wait(bc->describeBackup());
wait(desc.resolveVersionTimes(cx));
if(targetVersion == invalidVersion && desc.maxRestorableVersion.present())
if (targetVersion == invalidVersion && desc.maxRestorableVersion.present())
targetVersion = desc.maxRestorableVersion.get();
Optional<RestorableFileSet> restoreSet = wait(bc->getRestoreSet(targetVersion));
TraceEvent("FastRestore").detail("BackupDesc", desc.toString()).detail("TargetVersion", targetVersion);
if(!restoreSet.present()) {
if (!restoreSet.present()) {
TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
.detail("BackupContainer", bc->getURL())
.detail("TargetVersion", targetVersion);
.detail("BackupContainer", bc->getURL())
.detail("TargetVersion", targetVersion);
throw restore_invalid_version();
}
@ -3907,29 +3913,33 @@ ACTOR static Future<Version> _fastRestore(Database cx, Key tagName, Key url, boo
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Standalone<StringRef> restoreTag(tagName.toString() + "_" + std::to_string(restoreIndex));
bool locked = true;
struct RestoreRequest restoreRequest(restoreIndex, restoreTag, KeyRef(bc->getURL()), true, targetVersion, true, range, Key(), Key(), locked, deterministicRandom()->randomUniqueID());
struct RestoreRequest restoreRequest(restoreIndex, restoreTag, KeyRef(bc->getURL()), true, targetVersion,
true, range, Key(), Key(), locked,
deterministicRandom()->randomUniqueID());
tr->set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
tr->set(restoreRequestTriggerKey, restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), 1)); //backupRanges.size = 1 because we only support restoring 1 range in real mode
// backupRanges.size = 1 because we only support restoring 1 range in real mode for now
tr->set(restoreRequestTriggerKey, restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(),1));
wait(tr->commit()); // Trigger fast restore
break;
} catch(Error &e) {
if(e.code() != error_code_restore_duplicate_tag) {
} catch (Error& e) {
if (e.code() != error_code_restore_duplicate_tag) {
wait(tr->onError(e));
}
}
}
if(waitForComplete) {
FileBackupAgent::ERestoreState finalState = wait(waitFastRestore(cx, tagName, verbose));
if(finalState != FileBackupAgent::ERestoreState::COMPLETED)
throw restore_error();
if (waitForComplete) {
FileBackupAgent::ERestoreState finalState = wait(waitFastRestore(cx, tagName, verbose));
if (finalState != FileBackupAgent::ERestoreState::COMPLETED) throw restore_error();
}
return targetVersion;
}
ACTOR Future<Version> fastRestore(Database cx, Standalone<StringRef> tagName, Standalone<StringRef> url, bool waitForComplete, long targetVersion, bool verbose, Standalone<KeyRangeRef> range, Standalone<StringRef> addPrefix, Standalone<StringRef> removePrefix) {
Version targetVersion = wait( _fastRestore(cx, tagName, url, waitForComplete, targetVersion, verbose, range, addPrefix, removePrefix) );
ACTOR Future<Version> fastRestore(Database cx, Standalone<StringRef> tagName, Standalone<StringRef> url,
bool waitForComplete, long targetVersion, bool verbose, Standalone<KeyRangeRef> range,
Standalone<StringRef> addPrefix, Standalone<StringRef> removePrefix) {
Version targetVersion =
wait(_fastRestore(cx, tagName, url, waitForComplete, targetVersion, verbose, range, addPrefix, removePrefix));
return targetVersion;
}

5
fdbclient/BackupAgent.actor.h
View File

@ -844,7 +844,10 @@ public:
}
};
Future<Version> fastRestore(Database const& cx, Standalone<StringRef> const& tagName, Standalone<StringRef> const& url, bool const& waitForComplete, long const& targetVersion, bool const& verbose, Standalone<KeyRangeRef> const& range, Standalone<StringRef> const& addPrefix, Standalone<StringRef> const& removePrefix);
Future<Version> fastRestore(Database const& cx, Standalone<StringRef> const& tagName, Standalone<StringRef> const& url,
bool const& waitForComplete, long const& targetVersion, bool const& verbose,
Standalone<KeyRangeRef> const& range, Standalone<StringRef> const& addPrefix,
Standalone<StringRef> const& removePrefix);
#include "flow/unactorcompiler.h"
#endif

3
fdbclient/BackupContainer.actor.cpp
View File

@ -1031,7 +1031,8 @@ public:
restorable.ranges = ranges;
// No logs needed if there is a complete key space snapshot at the target version.
if(snapshot.get().beginVersion == snapshot.get().endVersion && snapshot.get().endVersion == targetVersion) {
if (snapshot.get().beginVersion == snapshot.get().endVersion &&
snapshot.get().endVersion == targetVersion) {
return Optional<RestorableFileSet>(restorable);
}

9
fdbclient/BackupContainer.h
View File

@ -76,8 +76,9 @@ struct LogFile {
std::string toString() const {
std::string ret;
ret = "beginVersion:" + std::to_string(beginVersion) + " endVersion:" + std::to_string(endVersion)
+ " blockSize:" + std::to_string(blockSize) + " filename:" + fileName + " fileSize:" + std::to_string(fileSize);
ret = "beginVersion:" + std::to_string(beginVersion) + " endVersion:" + std::to_string(endVersion) +
" blockSize:" + std::to_string(blockSize) + " filename:" + fileName +
" fileSize:" + std::to_string(fileSize);
return ret;
}
};
@ -95,8 +96,8 @@ struct RangeFile {
std::string toString() const {
std::string ret;
ret = "version:" + std::to_string(version) + " blockSize:" + std::to_string(blockSize) + " fileName:" + fileName
+ " fileSize:" + std::to_string(fileSize);
ret = "version:" + std::to_string(version) + " blockSize:" + std::to_string(blockSize) +
" fileName:" + fileName + " fileSize:" + std::to_string(fileSize);
return ret;
}
};

9
fdbclient/FDBTypes.h
View File

@ -254,9 +254,7 @@ struct KeyRangeRef {
}
};
std::string toString() const {
return "Begin:" + begin.printable() + "End:" + end.printable();
}
std::string toString() const { return "Begin:" + begin.printable() + "End:" + end.printable(); }
};
template<>
@ -586,8 +584,9 @@ struct RangeResultRef : VectorRef<KeyValueRef> {
}
std::string toString() const {
return "more:" + std::to_string(more) + " readThrough:" + (readThrough.present() ? readThrough.get().toString() : "[unset]")
+ " readToBegin:" + std::to_string(readToBegin) + " readThroughEnd:" + std::to_string(readThroughEnd);
return "more:" + std::to_string(more) +
" readThrough:" + (readThrough.present() ? readThrough.get().toString() : "[unset]") +
" readToBegin:" + std::to_string(readToBegin) + " readThroughEnd:" + std::to_string(readThroughEnd);
}
};

3
fdbclient/FileBackupAgent.actor.cpp
View File

@ -3754,7 +3754,8 @@ public:
Optional<UidAndAbortedFlagT> current = wait(tag.get(tr));
if(!current.present()) {
if(verbose)
printf("waitRestore: Tag: %s State: %s\n", tagName.toString().c_str(), FileBackupAgent::restoreStateText(ERestoreState::UNITIALIZED).toString().c_str());
printf("waitRestore: Tag: %s State: %s\n", tagName.toString().c_str(),
FileBackupAgent::restoreStateText(ERestoreState::UNITIALIZED).toString().c_str());
return ERestoreState::UNITIALIZED;
}

5
fdbclient/ManagementAPI.actor.cpp
View File

@ -1749,7 +1749,7 @@ ACTOR Future<Void> checkDatabaseLock( Transaction* tr, UID id ) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> val = wait( tr->get(databaseLockedKey) );
if ( val.present() ) {
if (val.present()) {
printf("DB is locked at uid:%s\n", id.toString().c_str());
} else {
printf("DB is not locked!\n");
@ -1768,8 +1768,7 @@ ACTOR Future<Void> checkDatabaseLock( Reference<ReadYourWritesTransaction> tr, U
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> val = wait( tr->get(databaseLockedKey) );
if ( val.present() ) {
if (val.present()) {
printf("DB is locked at uid:%s\n", id.toString().c_str());
} else {
printf("DB is not locked!\n");

10
fdbclient/MutationList.h
View File

@ -29,18 +29,22 @@ struct MutationListRef {
// Represents an ordered, but not random-access, list of mutations that can be O(1) deserialized and
// quickly serialized, (forward) iterated or appended to.
// MutationListRef is a list of struct Blob
// Each blob has a struct Header following by the mutation's param1 and param2 content. The Header has the mutation's type and the length of param1 and param2
// Each blob has a struct Header following by the mutation's param1 and param2 content.
// The Header has the mutation's type and the length of param1 and param2
private:
struct Blob {
//StringRef data Format: |type|p1len|p2len|p1_content|p2_content|
// StringRef data Format: |type|p1len|p2len|p1_content|p2_content|
// |type|p1len|p2len| is the header; p1_content has p1len length; p2_content has p2len length
StringRef data;
Blob* next;
};
struct Header {
int type, p1len, p2len;
const uint8_t* p1begin() const { return (const uint8_t*)(this+1); } //(this+1) moves the pointer by Header size and get to the beginning of p1_content
const uint8_t* p1begin() const {
//(this+1) moves the pointer by Header size and get to the beginning of p1_content
return (const uint8_t*)(this + 1);
}
const uint8_t* p2begin() const { return (const uint8_t*)(this+1) + p1len; }
const uint8_t* end() const { return (const uint8_t*)(this+1) + p1len + p2len; }
};

41
fdbclient/SystemData.cpp
View File

@ -615,16 +615,13 @@ const KeyRangeRef restoreWorkersKeys(
);
const KeyRef restoreStatusKey = LiteralStringRef("\xff\x02/restoreStatus/");
const KeyRef restoreRequestTriggerKey = LiteralStringRef("\xff\x02/restoreRequestTrigger");
const KeyRef restoreRequestDoneKey = LiteralStringRef("\xff\x02/restoreRequestDone");
const KeyRangeRef restoreRequestKeys(
LiteralStringRef("\xff\x02/restoreRequests/"),
LiteralStringRef("\xff\x02/restoreRequests0")
);
const KeyRangeRef restoreRequestKeys(LiteralStringRef("\xff\x02/restoreRequests/"),
LiteralStringRef("\xff\x02/restoreRequests0"));
// Encode restore worker key for workerID
const Key restoreWorkerKeyFor( UID const& workerID ) {
const Key restoreWorkerKeyFor(UID const& workerID) {
BinaryWriter wr(Unversioned());
wr.serializeBytes( restoreWorkersKeys.begin );
wr << workerID;
@ -632,78 +629,78 @@ const Key restoreWorkerKeyFor( UID const& workerID ) {
}
// Encode restore agent value
const Value restoreWorkerInterfaceValue( RestoreWorkerInterface const& cmdInterf ) {
const Value restoreWorkerInterfaceValue(RestoreWorkerInterface const& cmdInterf) {
BinaryWriter wr(IncludeVersion());
wr << cmdInterf;
return wr.toValue();
}
RestoreWorkerInterface decodeRestoreWorkerInterfaceValue( ValueRef const& value ) {
RestoreWorkerInterface decodeRestoreWorkerInterfaceValue(ValueRef const& value) {
RestoreWorkerInterface s;
BinaryReader reader( value, IncludeVersion() );
BinaryReader reader(value, IncludeVersion());
reader >> s;
return s;
}
// Encode and decode restore request value
// restoreRequestTrigger key
const Value restoreRequestTriggerValue (UID randomID, int const numRequests) {
const Value restoreRequestTriggerValue(UID randomID, int const numRequests) {
BinaryWriter wr(IncludeVersion());
wr << numRequests;
wr << randomID;
return wr.toValue();
}
const int decodeRestoreRequestTriggerValue( ValueRef const& value ) {
const int decodeRestoreRequestTriggerValue(ValueRef const& value) {
int s;
UID randomID;
BinaryReader reader( value, IncludeVersion() );
BinaryReader reader(value, IncludeVersion());
reader >> s;
reader >> randomID;
return s;
}
// restoreRequestDone key
const Value restoreRequestDoneVersionValue (Version readVersion) {
const Value restoreRequestDoneVersionValue(Version readVersion) {
BinaryWriter wr(IncludeVersion());
wr << readVersion;
return wr.toValue();
}
Version decodeRestoreRequestDoneVersionValue( ValueRef const& value ) {
Version decodeRestoreRequestDoneVersionValue(ValueRef const& value) {
Version v;
BinaryReader reader( value, IncludeVersion() );
BinaryReader reader(value, IncludeVersion());
reader >> v;
return v;
}
const Key restoreRequestKeyFor( int const& index ) {
const Key restoreRequestKeyFor(int const& index) {
BinaryWriter wr(Unversioned());
wr.serializeBytes( restoreRequestKeys.begin );
wr.serializeBytes(restoreRequestKeys.begin);
wr << index;
return wr.toValue();
}
const Value restoreRequestValue( RestoreRequest const& request ) {
const Value restoreRequestValue(RestoreRequest const& request) {
BinaryWriter wr(IncludeVersion());
wr << request;
return wr.toValue();
}
RestoreRequest decodeRestoreRequestValue( ValueRef const& value ) {
RestoreRequest decodeRestoreRequestValue(ValueRef const& value) {
RestoreRequest s;
BinaryReader reader( value, IncludeVersion() );
BinaryReader reader(value, IncludeVersion());
reader >> s;
return s;
}
// TODO: Register restore performance data to restoreStatus key
const Key restoreStatusKeyFor ( StringRef statusType) {
const Key restoreStatusKeyFor(StringRef statusType) {
BinaryWriter wr(Unversioned());
wr.serializeBytes(restoreStatusKey);
wr << statusType;
return wr.toValue();
}
const Value restoreStatusValue( double const& val ) {
const Value restoreStatusValue(double const& val) {
BinaryWriter wr(IncludeVersion());
wr << StringRef(std::to_string(val));
return wr.toValue();

24
fdbclient/SystemData.h
View File

@ -287,18 +287,18 @@ extern const KeyRef restoreStatusKey; // To be used when we measure fast restore
extern const KeyRef restoreRequestTriggerKey;
extern const KeyRef restoreRequestDoneKey;
extern const KeyRangeRef restoreRequestKeys;
const Key restoreWorkerKeyFor( UID const& workerID );
const Value restoreWorkerInterfaceValue(RestoreWorkerInterface const& server );
RestoreWorkerInterface decodeRestoreWorkerInterfaceValue( ValueRef const& value );
const Value restoreRequestTriggerValue (UID randomUID, int const numRequests);
const int decodeRestoreRequestTriggerValue( ValueRef const& value );
const Value restoreRequestDoneVersionValue (Version readVersion);
Version decodeRestoreRequestDoneVersionValue( ValueRef const& value );
const Key restoreRequestKeyFor( int const& index );
const Value restoreRequestValue( RestoreRequest const& server );
RestoreRequest decodeRestoreRequestValue( ValueRef const& value );
const Key restoreStatusKeyFor( StringRef statusType);
const Value restoreStatusValue( double const& val );
const Key restoreWorkerKeyFor(UID const& workerID);
const Value restoreWorkerInterfaceValue(RestoreWorkerInterface const& server);
RestoreWorkerInterface decodeRestoreWorkerInterfaceValue(ValueRef const& value);
const Value restoreRequestTriggerValue(UID randomUID, int const numRequests);
const int decodeRestoreRequestTriggerValue(ValueRef const& value);
const Value restoreRequestDoneVersionValue(Version readVersion);
Version decodeRestoreRequestDoneVersionValue(ValueRef const& value);
const Key restoreRequestKeyFor(int const& index);
const Value restoreRequestValue(RestoreRequest const& server);
RestoreRequest decodeRestoreRequestValue(ValueRef const& value);
const Key restoreStatusKeyFor(StringRef statusType);
const Value restoreStatusValue(double const& val);
extern const KeyRef healthyZoneKey;
extern const StringRef ignoreSSFailuresZoneString;

29
fdbrpc/Locality.h
View File

@ -27,7 +27,24 @@
struct ProcessClass {
constexpr static FileIdentifier file_identifier = 6697257;
// This enum is stored in restartInfo.ini for upgrade tests, so be very careful about changing the existing items!
enum ClassType { UnsetClass, StorageClass, TransactionClass, ResolutionClass, TesterClass, ProxyClass, MasterClass, StatelessClass, LogClass, ClusterControllerClass, LogRouterClass, FastRestoreClass, DataDistributorClass, CoordinatorClass, RatekeeperClass, InvalidClass = -1 };
enum ClassType {
UnsetClass,
StorageClass,
TransactionClass,
ResolutionClass,
TesterClass,
ProxyClass,
MasterClass,
StatelessClass,
LogClass,
ClusterControllerClass,
LogRouterClass,
FastRestoreClass,
DataDistributorClass,
CoordinatorClass,
RatekeeperClass,
InvalidClass = -1
};
enum Fitness { BestFit, GoodFit, UnsetFit, OkayFit, WorstFit, ExcludeFit, NeverAssign }; //cannot be larger than 7 because of leader election mask
enum ClusterRole { Storage, TLog, Proxy, Master, Resolver, LogRouter, ClusterController, DataDistributor, Ratekeeper, NoRole };
@ -38,6 +55,7 @@ struct ProcessClass {
public:
ProcessClass() : _class( UnsetClass ), _source( CommandLineSource ) {}
ProcessClass( ClassType type, ClassSource source ) : _class( type ), _source( source ) {}
// clang-format off
explicit ProcessClass( std::string s, ClassSource source ) : _source( source ) {
if (s=="storage") _class = StorageClass;
else if (s=="transaction") _class = TransactionClass;
@ -50,7 +68,7 @@ public:
else if (s=="log") _class = LogClass;
else if (s=="router") _class = LogRouterClass;
else if (s=="cluster_controller") _class = ClusterControllerClass;
else if (s=="fast_restore") _class = FastRestoreClass;
else if (s == "fast_restore") _class = FastRestoreClass;
else if (s=="data_distributor") _class = DataDistributorClass;
else if (s=="coordinator") _class = CoordinatorClass;
else if (s=="ratekeeper") _class = RatekeeperClass;
@ -69,7 +87,7 @@ public:
else if (classStr=="log") _class = LogClass;
else if (classStr=="router") _class = LogRouterClass;
else if (classStr=="cluster_controller") _class = ClusterControllerClass;
else if (classStr=="fast_restore") _class = FastRestoreClass;
else if (classStr == "fast_restore") _class = FastRestoreClass;
else if (classStr=="data_distributor") _class = DataDistributorClass;
else if (classStr=="coordinator") _class = CoordinatorClass;
else if (classStr=="ratekeeper") _class = RatekeeperClass;
@ -103,13 +121,14 @@ public:
case LogClass: return "log";
case LogRouterClass: return "router";
case ClusterControllerClass: return "cluster_controller";
case FastRestoreClass: return "fast_restore";
case DataDistributorClass: return "data_distributor";
case FastRestoreClass: return "fast_restore";
case DataDistributorClass: return "data_distributor";
case CoordinatorClass: return "coordinator";
case RatekeeperClass: return "ratekeeper";
default: return "invalid";
}
}
// clang-format on
std::string sourceString() const {
switch (_source) {

124
fdbserver/RestoreApplier.actor.cpp
View File

@ -32,13 +32,16 @@
#include "fdbserver/RestoreRoleCommon.actor.h"
#include "fdbserver/RestoreApplier.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVectorVersionedRequest req, Reference<RestoreApplierData> self);
ACTOR static Future<Void> handleApplyToDBRequest(RestoreVersionBatchRequest req, Reference<RestoreApplierData> self, Database cx);
ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVectorVersionedRequest req,
Reference<RestoreApplierData> self);
ACTOR static Future<Void> handleApplyToDBRequest(RestoreVersionBatchRequest req, Reference<RestoreApplierData> self,
Database cx);
ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int nodeIndex, Database cx) {
state Reference<RestoreApplierData> self = Reference<RestoreApplierData>( new RestoreApplierData(applierInterf.id(), nodeIndex) );
state Reference<RestoreApplierData> self =
Reference<RestoreApplierData>(new RestoreApplierData(applierInterf.id(), nodeIndex));
state ActorCollection actors(false);
state Future<Void> exitRole = Never();
@ -46,42 +49,47 @@ ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int
loop {
double loopTopTime = now();
double elapsedTime = loopTopTime - lastLoopTopTime;
if( elapsedTime > 0.050 ) {
if (elapsedTime > 0.050) {
if (deterministicRandom()->random01() < 0.01)
TraceEvent(SevWarn, "SlowRestoreApplierLoopx100").detail("NodeDesc", self->describeNode()).detail("Elapsed", elapsedTime);
TraceEvent(SevWarn, "SlowRestoreApplierLoopx100")
.detail("NodeDesc", self->describeNode())
.detail("Elapsed", elapsedTime);
}
lastLoopTopTime = loopTopTime;
state std::string requestTypeStr = "[Init]";
try {
choose {
when ( RestoreSimpleRequest req = waitNext(applierInterf.heartbeat.getFuture()) ) {
when(RestoreSimpleRequest req = waitNext(applierInterf.heartbeat.getFuture())) {
requestTypeStr = "heartbeat";
actors.add(handleHeartbeat(req, applierInterf.id()));
}
when ( RestoreSendMutationVectorVersionedRequest req = waitNext(applierInterf.sendMutationVector.getFuture()) ) {
when(RestoreSendMutationVectorVersionedRequest req =
waitNext(applierInterf.sendMutationVector.getFuture())) {
requestTypeStr = "sendMutationVector";
actors.add( handleSendMutationVectorRequest(req, self) );
actors.add(handleSendMutationVectorRequest(req, self));
}
when ( RestoreVersionBatchRequest req = waitNext(applierInterf.applyToDB.getFuture()) ) {
when(RestoreVersionBatchRequest req = waitNext(applierInterf.applyToDB.getFuture())) {
requestTypeStr = "applyToDB";
actors.add( handleApplyToDBRequest(req, self, cx) );
actors.add(handleApplyToDBRequest(req, self, cx));
}
when ( RestoreVersionBatchRequest req = waitNext(applierInterf.initVersionBatch.getFuture()) ) {
when(RestoreVersionBatchRequest req = waitNext(applierInterf.initVersionBatch.getFuture())) {
requestTypeStr = "initVersionBatch";
actors.add(handleInitVersionBatchRequest(req, self));
}
when ( RestoreVersionBatchRequest req = waitNext(applierInterf.finishRestore.getFuture()) ) {
when(RestoreVersionBatchRequest req = waitNext(applierInterf.finishRestore.getFuture())) {
requestTypeStr = "finishRestore";
exitRole = handleFinishRestoreRequest(req, self);
exitRole = handleFinishRestoreRequest(req, self);
}
when ( wait(exitRole) ) {
when(wait(exitRole)) {
TraceEvent("FastRestore").detail("RestoreApplierCore", "ExitRole").detail("NodeID", self->id());
break;
}
}
} catch (Error &e) {
TraceEvent(SevWarn, "FastRestore").detail("RestoreLoaderError", e.what()).detail("RequestType", requestTypeStr);
} catch (Error& e) {
TraceEvent(SevWarn, "FastRestore")
.detail("RestoreLoaderError", e.what())
.detail("RequestType", requestTypeStr);
break;
}
}
@ -92,25 +100,29 @@ ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int
// The actor may be invovked multiple times and executed async.
// No race condition as long as we do not wait or yield when operate the shared data, it should be fine,
// because all actors run on 1 thread.
ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVectorVersionedRequest req, Reference<RestoreApplierData> self) {
ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVectorVersionedRequest req,
Reference<RestoreApplierData> self) {
state int numMutations = 0;
TraceEvent("FastRestore").detail("ApplierNode", self->id())
.detail("LogVersion", self->logVersion.get()).detail("RangeVersion", self->rangeVersion.get())
.detail("Request", req.toString());
TraceEvent("FastRestore")
.detail("ApplierNode", self->id())
.detail("LogVersion", self->logVersion.get())
.detail("RangeVersion", self->rangeVersion.get())
.detail("Request", req.toString());
if ( req.isRangeFile ) {
wait( self->rangeVersion.whenAtLeast(req.prevVersion) );
if (req.isRangeFile) {
wait(self->rangeVersion.whenAtLeast(req.prevVersion));
} else {
wait( self->logVersion.whenAtLeast(req.prevVersion) );
wait(self->logVersion.whenAtLeast(req.prevVersion));
}
if ( (req.isRangeFile && self->rangeVersion.get() == req.prevVersion) ||
(!req.isRangeFile && self->logVersion.get() == req.prevVersion) ) { // Not a duplicate (check relies on no waiting between here and self->version.set() below!)
// Not a duplicate (check relies on no waiting between here and self->version.set() below!)
if ((req.isRangeFile && self->rangeVersion.get() == req.prevVersion) ||
(!req.isRangeFile && self->logVersion.get() == req.prevVersion)) {
// Applier will cache the mutations at each version. Once receive all mutations, applier will apply them to DB
state Version commitVersion = req.version;
VectorRef<MutationRef> mutations(req.mutations);
if ( self->kvOps.find(commitVersion) == self->kvOps.end() ) {
if (self->kvOps.find(commitVersion) == self->kvOps.end()) {
self->kvOps.insert(std::make_pair(commitVersion, VectorRef<MutationRef>()));
}
state int mIndex = 0;
@ -121,7 +133,7 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVec
}
// Notify the same actor and unblock the request at the next version
if ( req.isRangeFile ) {
if (req.isRangeFile) {
self->rangeVersion.set(req.version);
} else {
self->logVersion.set(req.version);
@ -132,8 +144,8 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVec
return Void();
}
ACTOR Future<Void> applyToDB(Reference<RestoreApplierData> self, Database cx) {
state std::string typeStr = "";
ACTOR Future<Void> applyToDB(Reference<RestoreApplierData> self, Database cx) {
state std::string typeStr = "";
// Assume the process will not crash when it apply mutations to DB. The reply message can be lost though
if (self->kvOps.empty()) {
@ -144,15 +156,18 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVec
std::map<Version, Standalone<VectorRef<MutationRef>>>::iterator end = self->kvOps.end();
end--;
ASSERT_WE_THINK(end != self->kvOps.end());
TraceEvent("FastRestore").detail("ApplierApplyToDB", self->id()).detail("FromVersion", begin->first).detail("EndVersion", end->first);
TraceEvent("FastRestore")
.detail("ApplierApplyToDB", self->id())
.detail("FromVersion", begin->first)
.detail("EndVersion", end->first);
self->sanityCheckMutationOps();
state std::map<Version, Standalone<VectorRef<MutationRef>>>::iterator it = self->kvOps.begin();
state std::map<Version, Standalone<VectorRef<MutationRef>>>::iterator it = self->kvOps.begin();
state std::map<Version, Standalone<VectorRef<MutationRef>>>::iterator prevIt = it;
state int index = 0;
state int prevIndex = index;
state int count = 0;
state int count = 0;
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
state int numVersion = 0;
state double transactionSize = 0;
@ -163,32 +178,34 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVec
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
transactionSize = 0;
for ( ; it != self->kvOps.end(); ++it ) {
for (; it != self->kvOps.end(); ++it) {
numVersion++;
//TraceEvent("FastRestore").detail("Applier", self->id()).detail("ApplyKVsToDBVersion", it->first);
state MutationRef m;
for ( ; index < it->second.size(); ++index ) {
for (; index < it->second.size(); ++index) {
m = it->second[index];
if ( m.type >= MutationRef::Type::SetValue && m.type <= MutationRef::Type::MAX_ATOMIC_OP )
if (m.type >= MutationRef::Type::SetValue && m.type <= MutationRef::Type::MAX_ATOMIC_OP)
typeStr = typeString[m.type];
else {
TraceEvent(SevError, "FastRestore").detail("InvalidMutationType", m.type);
}
if ( m.type == MutationRef::SetValue ) {
if (m.type == MutationRef::SetValue) {
tr->set(m.param1, m.param2);
} else if ( m.type == MutationRef::ClearRange ) {
} else if (m.type == MutationRef::ClearRange) {
KeyRangeRef mutationRange(m.param1, m.param2);
tr->clear(mutationRange);
} else if ( isAtomicOp((MutationRef::Type) m.type) ) {
} else if (isAtomicOp((MutationRef::Type)m.type)) {
tr->atomicOp(m.param1, m.param2, m.type);
} else {
TraceEvent(SevError, "FastRestore").detail("UnhandledMutationType", m.type).detail("TypeName", typeStr);
TraceEvent(SevError, "FastRestore")
.detail("UnhandledMutationType", m.type)
.detail("TypeName", typeStr);
}
++count;
transactionSize += m.expectedSize();
if ( transactionSize >= opConfig.transactionBatchSizeThreshold ) { // commit per 1000 mutations
if (transactionSize >= opConfig.transactionBatchSizeThreshold) { // commit per 1000 mutations
wait(tr->commit());
tr->reset();
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -199,7 +216,7 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVec
}
}
if ( transactionSize > 0 ) { // the commit batch should NOT across versions
if (transactionSize > 0) { // the commit batch should NOT across versions
wait(tr->commit());
tr->reset();
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -215,7 +232,7 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVec
wait(tr->commit());
}
break;
} catch(Error &e) {
} catch (Error& e) {
wait(tr->onError(e));
it = prevIt;
index = prevIndex;
@ -223,20 +240,23 @@ ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendMutationVec
}
}
self->kvOps.clear();
self->kvOps.clear();
return Void();
}
return Void();
}
ACTOR static Future<Void> handleApplyToDBRequest(RestoreVersionBatchRequest req, Reference<RestoreApplierData> self, Database cx) {
TraceEvent("FastRestore").detail("ApplierApplyToDB", self->id()).detail("DBApplierPresent", self->dbApplier.present());
if ( !self->dbApplier.present() ) {
ACTOR static Future<Void> handleApplyToDBRequest(RestoreVersionBatchRequest req, Reference<RestoreApplierData> self,
Database cx) {
TraceEvent("FastRestore")
.detail("ApplierApplyToDB", self->id())
.detail("DBApplierPresent", self->dbApplier.present());
if (!self->dbApplier.present()) {
self->dbApplier = applyToDB(self, cx);
}
ASSERT(self->dbApplier.present());
wait( self->dbApplier.get() );
wait(self->dbApplier.get());
req.reply.send(RestoreCommonReply(self->id()));
return Void();

40
fdbserver/RestoreApplier.actor.h
View File

@ -22,10 +22,10 @@
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORE_APPLIER_G_H)
#define FDBSERVER_RESTORE_APPLIER_G_H
#include "fdbserver/RestoreApplier.actor.g.h"
#define FDBSERVER_RESTORE_APPLIER_G_H
#include "fdbserver/RestoreApplier.actor.g.h"
#elif !defined(FDBSERVER_RESTORE_APPLIER_H)
#define FDBSERVER_RESTORE_APPLIER_H
#define FDBSERVER_RESTORE_APPLIER_H
#include <sstream>
#include "flow/Stats.h"
@ -40,15 +40,16 @@
#include "flow/actorcompiler.h" // has to be last include
struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreApplierData> {
struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreApplierData> {
NotifiedVersion rangeVersion; // All requests of mutations in range file below this version has been processed
NotifiedVersion logVersion; // All requests of mutations in log file below this version has been processed
Optional<Future<Void>> dbApplier;
// range2Applier is in master and loader node. Loader node uses this to determine which applier a mutation should be sent
std::map<Standalone<KeyRef>, UID> range2Applier; // KeyRef is the inclusive lower bound of the key range the applier (UID) is responsible for
std::map<Standalone<KeyRef>, int> keyOpsCount; // The number of operations per key which is used to determine the key-range boundary for appliers
// range2Applier is in master and loader. Loader uses it to determine which applier a mutation should be sent
// KeyRef is the inclusive lower bound of the key range the applier (UID) is responsible for
std::map<Standalone<KeyRef>, UID> range2Applier;
// keyOpsCount is the number of operations per key that is used to determine the key-range boundary for appliers
std::map<Standalone<KeyRef>, int> keyOpsCount;
// For master applier to hold the lower bound of key ranges for each appliers
std::vector<Standalone<KeyRef>> keyRangeLowerBounds;
@ -67,7 +68,7 @@ struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreAppl
nodeIndex = assignedIndex;
// Q: Why do we need to initMetric?
//version.initMetric(LiteralStringRef("RestoreApplier.Version"), cc.id);
// version.initMetric(LiteralStringRef("RestoreApplier.Version"), cc.id);
role = RestoreRole::Applier;
}
@ -89,18 +90,17 @@ struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreAppl
}
void sanityCheckMutationOps() {
if (kvOps.empty())
return;
if (kvOps.empty()) return;
ASSERT_WE_THINK( isKVOpsSorted() );
ASSERT_WE_THINK( allOpsAreKnown() );
ASSERT_WE_THINK(isKVOpsSorted());
ASSERT_WE_THINK(allOpsAreKnown());
}
bool isKVOpsSorted() {
bool ret = true;
auto prev = kvOps.begin();
for ( auto it = kvOps.begin(); it != kvOps.end(); ++it ) {
if ( prev->first > it->first ) {
for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
if (prev->first > it->first) {
ret = false;
break;
}
@ -111,10 +111,10 @@ struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreAppl
bool allOpsAreKnown() {
bool ret = true;
for ( auto it = kvOps.begin(); it != kvOps.end(); ++it ) {
for ( auto m = it->second.begin(); m != it->second.end(); ++m ) {
if ( m->type == MutationRef::SetValue || m->type == MutationRef::ClearRange
|| isAtomicOp((MutationRef::Type) m->type) )
for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
for (auto m = it->second.begin(); m != it->second.end(); ++m) {
if (m->type == MutationRef::SetValue || m->type == MutationRef::ClearRange ||
isAtomicOp((MutationRef::Type)m->type))
continue;
else {
TraceEvent(SevError, "FastRestore").detail("UnknownMutationType", m->type);
@ -126,9 +126,7 @@ struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreAppl
}
};
ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int nodeIndex, Database cx);
#include "flow/unactorcompiler.h"
#endif

358
fdbserver/RestoreCommon.actor.cpp
View File

@ -35,10 +35,11 @@
// For convenience
typedef FileBackupAgent::ERestoreState ERestoreState;
template<> Tuple Codec<ERestoreState>::pack(ERestoreState const &val); // { return Tuple().append(val); }
template<> ERestoreState Codec<ERestoreState>::unpack(Tuple const &val); // { return (ERestoreState)val.getInt(0); }
template <> Tuple Codec<ERestoreState>::pack(ERestoreState const& val);
template <> ERestoreState Codec<ERestoreState>::unpack(Tuple const& val);
// Split RestoreConfig defined in FileBackupAgent.actor.cpp to declaration in Restore.actor.h and implementation in RestoreCommon.actor.cpp
// Split RestoreConfig defined in FileBackupAgent.actor.cpp to declaration in Restore.actor.h and implementation in
// RestoreCommon.actor.cpp
KeyBackedProperty<ERestoreState> RestoreConfig::stateEnum() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
@ -98,7 +99,8 @@ Future<std::vector<KeyRange>> RestoreConfig::getRestoreRangesOrDefault(Reference
return getRestoreRangesOrDefault_impl(this, tr);
}
ACTOR Future<std::vector<KeyRange>> RestoreConfig::getRestoreRangesOrDefault_impl(RestoreConfig *self, Reference<ReadYourWritesTransaction> tr) {
ACTOR Future<std::vector<KeyRange>> RestoreConfig::getRestoreRangesOrDefault_impl(
RestoreConfig* self, Reference<ReadYourWritesTransaction> tr) {
state std::vector<KeyRange> ranges = wait(self->restoreRanges().getD(tr));
if (ranges.empty()) {
state KeyRange range = wait(self->restoreRange().getD(tr));
@ -107,28 +109,25 @@ ACTOR Future<std::vector<KeyRange>> RestoreConfig::getRestoreRangesOrDefault_imp
return ranges;
}
KeyBackedSet<RestoreFile> RestoreConfig::fileSet() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
Future<bool> RestoreConfig::isRunnable(Reference<ReadYourWritesTransaction> tr) {
return map(stateEnum().getD(tr), [](ERestoreState s) -> bool { return s != ERestoreState::ABORTED
&& s != ERestoreState::COMPLETED
&& s != ERestoreState::UNITIALIZED;
return map(stateEnum().getD(tr), [](ERestoreState s) -> bool {
return s != ERestoreState::ABORTED && s != ERestoreState::COMPLETED && s != ERestoreState::UNITIALIZED;
});
}
Future<Void> RestoreConfig::logError(Database cx, Error e, std::string const &details, void *taskInstance) {
if(!uid.isValid()) {
Future<Void> RestoreConfig::logError(Database cx, Error e, std::string const& details, void* taskInstance) {
if (!uid.isValid()) {
TraceEvent(SevError, "FileRestoreErrorNoUID").error(e).detail("Description", details);
return Void();
}
TraceEvent t(SevWarn, "FileRestoreError");
t.error(e).detail("RestoreUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance);
// These should not happen
if(e.code() == error_code_key_not_found)
t.backtrace();
if (e.code() == error_code_key_not_found) t.backtrace();
return updateErrorInfo(cx, e, details);
}
@ -138,7 +137,7 @@ Key RestoreConfig::mutationLogPrefix() {
}
Key RestoreConfig::applyMutationsMapPrefix() {
return uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
return uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
}
ACTOR Future<int64_t> RestoreConfig::getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid) {
@ -147,8 +146,7 @@ ACTOR Future<int64_t> RestoreConfig::getApplyVersionLag_impl(Reference<ReadYourW
state Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), true);
wait(success(beginVal) && success(endVal));
if(!beginVal.get().present() || !endVal.get().present())
return 0;
if (!beginVal.get().present() || !endVal.get().present()) return 0;
Version beginVersion = BinaryReader::fromStringRef<Version>(beginVal.get().get(), Unversioned());
Version endVersion = BinaryReader::fromStringRef<Version>(endVal.get().get(), Unversioned());
@ -177,7 +175,7 @@ void RestoreConfig::initApplyMutations(Reference<ReadYourWritesTransaction> tr,
void RestoreConfig::clearApplyMutationsKeys(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
// Clear add/remove prefix keys
tr->clear(uidPrefixKey(applyMutationsAddPrefixRange.begin, uid));
tr->clear(uidPrefixKey(applyMutationsRemovePrefixRange.begin, uid));
@ -205,13 +203,14 @@ void RestoreConfig::setApplyEndVersion(Reference<ReadYourWritesTransaction> tr,
}
Future<Version> RestoreConfig::getApplyEndVersion(Reference<ReadYourWritesTransaction> tr) {
return map(tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid)), [=](Optional<Value> const &value) -> Version {
return map(tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid)), [=](Optional<Value> const& value) -> Version {
return value.present() ? BinaryReader::fromStringRef<Version>(value.get(), Unversioned()) : 0;
});
}
// Meng: Change RestoreConfig to Reference<RestoreConfig> because FastRestore pass the Reference<RestoreConfig> around
ACTOR Future<std::string> RestoreConfig::getProgress_impl(Reference<RestoreConfig> restore, Reference<ReadYourWritesTransaction> tr) {
ACTOR Future<std::string> RestoreConfig::getProgress_impl(Reference<RestoreConfig> restore,
Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -227,46 +226,42 @@ ACTOR Future<std::string> RestoreConfig::getProgress_impl(Reference<RestoreConfi
// restore might no longer be valid after the first wait so make sure it is not needed anymore.
state UID uid = restore->getUid();
wait(success(fileCount) && success(fileBlockCount) && success(fileBlocksDispatched) && success(fileBlocksFinished) && success(bytesWritten) && success(status) && success(lag) && success(tag) && success(lastError));
wait(success(fileCount) && success(fileBlockCount) && success(fileBlocksDispatched) &&
success(fileBlocksFinished) && success(bytesWritten) && success(status) && success(lag) && success(tag) &&
success(lastError));
std::string errstr = "None";
if(lastError.get().second != 0)
errstr = format("'%s' %llds ago.\n", lastError.get().first.c_str(), (tr->getReadVersion().get() - lastError.get().second) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND );
if (lastError.get().second != 0)
errstr = format("'%s' %llds ago.\n", lastError.get().first.c_str(),
(tr->getReadVersion().get() - lastError.get().second) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
TraceEvent("FileRestoreProgress")
.detail("RestoreUID", uid)
.detail("Tag", tag.get())
.detail("State", status.get().toString())
.detail("FileCount", fileCount.get())
.detail("FileBlocksFinished", fileBlocksFinished.get())
.detail("FileBlocksTotal", fileBlockCount.get())
.detail("FileBlocksInProgress", fileBlocksDispatched.get() - fileBlocksFinished.get())
.detail("BytesWritten", bytesWritten.get())
.detail("ApplyLag", lag.get())
.detail("TaskInstance", THIS_ADDR)
.backtrace();
.detail("RestoreUID", uid)
.detail("Tag", tag.get())
.detail("State", status.get().toString())
.detail("FileCount", fileCount.get())
.detail("FileBlocksFinished", fileBlocksFinished.get())
.detail("FileBlocksTotal", fileBlockCount.get())
.detail("FileBlocksInProgress", fileBlocksDispatched.get() - fileBlocksFinished.get())
.detail("BytesWritten", bytesWritten.get())
.detail("ApplyLag", lag.get())
.detail("TaskInstance", THIS_ADDR)
.backtrace();
return format("Tag: %s UID: %s State: %s Blocks: %lld/%lld BlocksInProgress: %lld Files: %lld BytesWritten: %lld ApplyVersionLag: %lld LastError: %s",
tag.get().c_str(),
uid.toString().c_str(),
status.get().toString().c_str(),
fileBlocksFinished.get(),
fileBlockCount.get(),
fileBlocksDispatched.get() - fileBlocksFinished.get(),
fileCount.get(),
bytesWritten.get(),
lag.get(),
errstr.c_str()
);
return format("Tag: %s UID: %s State: %s Blocks: %lld/%lld BlocksInProgress: %lld Files: %lld BytesWritten: "
"%lld ApplyVersionLag: %lld LastError: %s",
tag.get().c_str(), uid.toString().c_str(), status.get().toString().c_str(), fileBlocksFinished.get(),
fileBlockCount.get(), fileBlocksDispatched.get() - fileBlocksFinished.get(), fileCount.get(),
bytesWritten.get(), lag.get(), errstr.c_str());
}
Future<std::string> RestoreConfig::getProgress(Reference<ReadYourWritesTransaction> tr) {
Future<std::string> RestoreConfig::getProgress(Reference<ReadYourWritesTransaction> tr) {
Reference<RestoreConfig> restore = Reference<RestoreConfig>(this);
return getProgress_impl(restore, tr);
return getProgress_impl(restore, tr);
}
// Meng: Change RestoreConfig to Reference<RestoreConfig>
ACTOR Future<std::string> RestoreConfig::getFullStatus_impl(Reference<RestoreConfig> restore, Reference<ReadYourWritesTransaction> tr) {
ACTOR Future<std::string> RestoreConfig::getFullStatus_impl(Reference<RestoreConfig> restore,
Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -279,18 +274,16 @@ ACTOR Future<std::string> RestoreConfig::getFullStatus_impl(Reference<RestoreCon
// restore might no longer be valid after the first wait so make sure it is not needed anymore.
state UID uid = restore->getUid();
wait(success(ranges) && success(addPrefix) && success(removePrefix) && success(url) && success(restoreVersion) && success(progress));
wait(success(ranges) && success(addPrefix) && success(removePrefix) &&
success(url) && success(restoreVersion) && success(progress));
std::string returnStr;
returnStr = format("%s URL: %s", progress.get().c_str(), url.get().toString().c_str());
for (auto &range : ranges.get()) {
for (auto& range : ranges.get()) {
returnStr += format(" Range: '%s'-'%s'", printable(range.begin).c_str(), printable(range.end).c_str());
}
returnStr += format(" AddPrefix: '%s' RemovePrefix: '%s' Version: %lld",
printable(addPrefix.get()).c_str(),
printable(removePrefix.get()).c_str(),
restoreVersion.get()
);
returnStr += format(" AddPrefix: '%s' RemovePrefix: '%s' Version: %lld", printable(addPrefix.get()).c_str(),
printable(removePrefix.get()).c_str(), restoreVersion.get());
return returnStr;
}
Future<std::string> RestoreConfig::getFullStatus(Reference<ReadYourWritesTransaction> tr) {
@ -306,154 +299,143 @@ std::string RestoreConfig::toString() {
typedef RestoreConfig::RestoreFile RestoreFile;
// parallelFileRestore is copied from FileBackupAgent.actor.cpp for the same reason as RestoreConfig is copied
// The implementation of parallelFileRestore is copied from FileBackupAgent.actor.cpp
// The implementation of parallelFileRestore is copied from FileBackupAgent.actor.cpp
// parallelFileRestore is copied from FileBackupAgent.actor.cpp for the same reason as RestoreConfig is copied
namespace parallelFileRestore {
// Helper class for reading restore data from a buffer and throwing the right errors.
struct StringRefReader {
StringRefReader(StringRef s = StringRef(), Error e = Error()) : rptr(s.begin()), end(s.end()), failure_error(e) {}
// Helper class for reading restore data from a buffer and throwing the right errors.
struct StringRefReader {
StringRefReader(StringRef s = StringRef(), Error e = Error()) : rptr(s.begin()), end(s.end()), failure_error(e) {}
// Return remainder of data as a StringRef
StringRef remainder() {
return StringRef(rptr, end - rptr);
// Return remainder of data as a StringRef
StringRef remainder() { return StringRef(rptr, end - rptr); }
// Return a pointer to len bytes at the current read position and advance read pos
const uint8_t* consume(unsigned int len) {
if (rptr == end && len != 0) throw end_of_stream();
const uint8_t* p = rptr;
rptr += len;
if (rptr > end) throw failure_error;
return p;
}
// Return a T from the current read position and advance read pos
template <typename T>
const T consume() {
return *(const T*)consume(sizeof(T));
}
// Functions for consuming big endian (network byte order) integers.
// Consumes a big endian number, swaps it to little endian, and returns it.
const int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
const uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
bool eof() { return rptr == end; }
const uint8_t *rptr, *end;
Error failure_error;
};
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset,
int len) {
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(file->read(mutateString(buf), len, offset));
if (rLen != len) throw restore_bad_read();
Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
try {
// Read header, currently only decoding version 1001
if (reader.consume<int32_t>() != 1001) throw restore_unsupported_file_version();
// Read begin key, if this fails then block was invalid.
uint32_t kLen = reader.consumeNetworkUInt32();
const uint8_t* k = reader.consume(kLen);
results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
// Read kv pairs and end key
while (1) {
// Read a key.
kLen = reader.consumeNetworkUInt32();
k = reader.consume(kLen);
// If eof reached or first value len byte is 0xFF then a valid block end was reached.
if (reader.eof() || *reader.rptr == 0xFF) {
results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
break;
}
// Read a value, which must exist or the block is invalid
uint32_t vLen = reader.consumeNetworkUInt32();
const uint8_t* v = reader.consume(vLen);
results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
// If eof reached or first byte of next key len is 0xFF then a valid block end was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
}
// Return a pointer to len bytes at the current read position and advance read pos
const uint8_t * consume(unsigned int len) {
if(rptr == end && len != 0)
throw end_of_stream();
const uint8_t *p = rptr;
rptr += len;
if(rptr > end)
throw failure_error;
return p;
}
// Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder())
if (b != 0xFF) throw restore_corrupted_data_padding();
// Return a T from the current read position and advance read pos
template<typename T> const T consume() {
return *(const T *)consume(sizeof(T));
}
return results;
// Functions for consuming big endian (network byte order) integers.
// Consumes a big endian number, swaps it to little endian, and returns it.
const int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume< int32_t>());}
const uint32_t consumeNetworkUInt32() { return bigEndian32( consume<uint32_t>());}
} catch (Error& e) {
TraceEvent(SevWarn, "FileRestoreCorruptRangeFileBlock")
.error(e)
.detail("Filename", file->getFilename())
.detail("BlockOffset", offset)
.detail("BlockLen", len)
.detail("ErrorRelativeOffset", reader.rptr - buf.begin())
.detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset);
throw;
}
}
bool eof() { return rptr == end; }
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file, int64_t offset,
int len) {
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(file->read(mutateString(buf), len, offset));
if (rLen != len) throw restore_bad_read();
const uint8_t *rptr, *end;
Error failure_error;
};
Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
try {
// Read header, currently only decoding version 2001
if (reader.consume<int32_t>() != 2001) throw restore_unsupported_file_version();
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset, int len) {
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(file->read(mutateString(buf), len, offset));
if(rLen != len)
throw restore_bad_read();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
try {
// Read header, currently only decoding version 1001
if(reader.consume<int32_t>() != 1001)
throw restore_unsupported_file_version();
// Read begin key, if this fails then block was invalid.
// Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32();
const uint8_t *k = reader.consume(kLen);
results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
const uint8_t* k = reader.consume(kLen);
uint32_t vLen = reader.consumeNetworkUInt32();
const uint8_t* v = reader.consume(vLen);
// Read kv pairs and end key
while(1) {
// Read a key.
kLen = reader.consumeNetworkUInt32();
k = reader.consume(kLen);
// If eof reached or first value len byte is 0xFF then a valid block end was reached.
if(reader.eof() || *reader.rptr == 0xFF) {
results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
break;
}
// Read a value, which must exist or the block is invalid
uint32_t vLen = reader.consumeNetworkUInt32();
const uint8_t *v = reader.consume(vLen);
results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
// If eof reached or first byte of next key len is 0xFF then a valid block end was reached.
if(reader.eof() || *reader.rptr == 0xFF)
break;
}
// Make sure any remaining bytes in the block are 0xFF
for(auto b : reader.remainder())
if(b != 0xFF)
throw restore_corrupted_data_padding();
return results;
} catch(Error &e) {
TraceEvent(SevWarn, "FileRestoreCorruptRangeFileBlock")
.error(e)
.detail("Filename", file->getFilename())
.detail("BlockOffset", offset)
.detail("BlockLen", len)
.detail("ErrorRelativeOffset", reader.rptr - buf.begin())
.detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset);
throw;
results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
}
// Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder())
if (b != 0xFF) throw restore_corrupted_data_padding();
return results;
} catch (Error& e) {
TraceEvent(SevWarn, "FileRestoreCorruptLogFileBlock")
.error(e)
.detail("Filename", file->getFilename())
.detail("BlockOffset", offset)
.detail("BlockLen", len)
.detail("ErrorRelativeOffset", reader.rptr - buf.begin())
.detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset);
throw;
}
}
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file, int64_t offset, int len) {
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(file->read(mutateString(buf), len, offset));
if(rLen != len)
throw restore_bad_read();
Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
state parallelFileRestore::StringRefReader reader(buf, restore_corrupted_data());
try {
// Read header, currently only decoding version 2001
if(reader.consume<int32_t>() != 2001)
throw restore_unsupported_file_version();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while(1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if(reader.eof() || *reader.rptr == 0xFF)
break;
// Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32();
const uint8_t *k = reader.consume(kLen);
uint32_t vLen = reader.consumeNetworkUInt32();
const uint8_t *v = reader.consume(vLen);
results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
}
// Make sure any remaining bytes in the block are 0xFF
for(auto b : reader.remainder())
if(b != 0xFF)
throw restore_corrupted_data_padding();
return results;
} catch(Error &e) {
TraceEvent(SevWarn, "FileRestoreCorruptLogFileBlock")
.error(e)
.detail("Filename", file->getFilename())
.detail("BlockOffset", offset)
.detail("BlockLen", len)
.detail("ErrorRelativeOffset", reader.rptr - buf.begin())
.detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset);
throw;
}
}
}
} // namespace parallelFileRestore

117
fdbserver/RestoreCommon.actor.h
View File

@ -19,15 +19,15 @@
*/
// This file includes the code copied from the old restore in FDB 5.2
// The functions and structure declared in this file can be shared by
// The functions and structure declared in this file can be shared by
// the old restore and the new performant restore systems
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORECOMMON_ACTOR_G_H)
#define FDBSERVER_RESTORECOMMON_ACTOR_G_H
#include "fdbserver/RestoreCommon.actor.g.h"
#define FDBSERVER_RESTORECOMMON_ACTOR_G_H
#include "fdbserver/RestoreCommon.actor.g.h"
#elif !defined(FDBSERVER_RESTORECOMMON_ACTOR_H)
#define FDBSERVER_RESTORECOMMON_ACTOR_H
#define FDBSERVER_RESTORECOMMON_ACTOR_H
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
@ -38,23 +38,24 @@
#include "flow/actorcompiler.h" // has to be last include
// RestoreConfig copied from FileBackupAgent.actor.cpp
// We copy RestoreConfig instead of using (and potentially changing) it in place to avoid conflict with the existing code
// We copy RestoreConfig instead of using (and potentially changing) it in place to avoid conflict with the existing
// code
// TODO: Merge this RestoreConfig with the original RestoreConfig in FileBackupAgent.actor.cpp
typedef FileBackupAgent::ERestoreState ERestoreState;
struct RestoreFileFR;
// We copy RestoreConfig copied from FileBackupAgent.actor.cpp instead of using (and potentially changing) it in place to avoid conflict with the existing code
// Split RestoreConfig defined in FileBackupAgent.actor.cpp to declaration in Restore.actor.h and implementation in RestoreCommon.actor.cpp,
// so that we can use in both the existing restore and the new fast restore subsystems
// We use RestoreConfig as a Reference<RestoreConfig>, which leads to some non-functional changes in RestoreConfig
// We copy RestoreConfig copied from FileBackupAgent.actor.cpp instead of using (and potentially changing) it in place
// to avoid conflict with the existing code Split RestoreConfig defined in FileBackupAgent.actor.cpp to declaration in
// Restore.actor.h and implementation in RestoreCommon.actor.cpp, so that we can use in both the existing restore and
// the new fast restore subsystems We use RestoreConfig as a Reference<RestoreConfig>, which leads to some
// non-functional changes in RestoreConfig
class RestoreConfig : public KeyBackedConfig, public ReferenceCounted<RestoreConfig> {
public:
RestoreConfig(UID uid = UID()) : KeyBackedConfig(fileRestorePrefixRange.begin, uid) {}
RestoreConfig(Reference<Task> task) : KeyBackedConfig(fileRestorePrefixRange.begin, task) {}
KeyBackedProperty<ERestoreState> stateEnum();
KeyBackedProperty<ERestoreState> stateEnum();
Future<StringRef> stateText(Reference<ReadYourWritesTransaction> tr);
@ -92,29 +93,30 @@ public:
KeyBackedBinaryValue<int64_t> fileBlockCount();
Future<std::vector<KeyRange>> getRestoreRangesOrDefault(Reference<ReadYourWritesTransaction> tr);
ACTOR static Future<std::vector<KeyRange>> getRestoreRangesOrDefault_impl(RestoreConfig *self, Reference<ReadYourWritesTransaction> tr);
ACTOR static Future<std::vector<KeyRange>> getRestoreRangesOrDefault_impl(RestoreConfig* self,
Reference<ReadYourWritesTransaction> tr);
// Describes a file to load blocks from during restore. Ordered by version and then fileName to enable
// incrementally advancing through the map, saving the version and path of the next starting point.
struct RestoreFile {
Version version;
std::string fileName;
bool isRange; // false for log file
bool isRange; // false for log file
int64_t blockSize;
int64_t fileSize;
Version endVersion; // not meaningful for range files
Version endVersion; // not meaningful for range files
Tuple pack() const {
//fprintf(stderr, "Filename:%s\n", fileName.c_str());
// fprintf(stderr, "Filename:%s\n", fileName.c_str());
return Tuple()
.append(version)
.append(StringRef(fileName))
.append(isRange)
.append(fileSize)
.append(blockSize)
.append(endVersion);
.append(version)
.append(StringRef(fileName))
.append(isRange)
.append(fileSize)
.append(blockSize)
.append(endVersion);
}
static RestoreFile unpack(Tuple const &t) {
static RestoreFile unpack(Tuple const& t) {
RestoreFile r;
int i = 0;
r.version = t.getInt(i++);
@ -127,12 +129,12 @@ public:
}
};
//typedef KeyBackedSet<RestoreFile> FileSetT;
// typedef KeyBackedSet<RestoreFile> FileSetT;
KeyBackedSet<RestoreFile> fileSet();
Future<bool> isRunnable(Reference<ReadYourWritesTransaction> tr);
Future<Void> logError(Database cx, Error e, std::string const &details, void *taskInstance = nullptr);
Future<Void> logError(Database cx, Error e, std::string const& details, void* taskInstance = nullptr);
Key mutationLogPrefix();
@ -152,10 +154,12 @@ public:
Future<Version> getApplyEndVersion(Reference<ReadYourWritesTransaction> tr);
ACTOR static Future<std::string> getProgress_impl(Reference<RestoreConfig> restore, Reference<ReadYourWritesTransaction> tr);
ACTOR static Future<std::string> getProgress_impl(Reference<RestoreConfig> restore,
Reference<ReadYourWritesTransaction> tr);
Future<std::string> getProgress(Reference<ReadYourWritesTransaction> tr);
ACTOR static Future<std::string> getFullStatus_impl(Reference<RestoreConfig> restore, Reference<ReadYourWritesTransaction> tr);
ACTOR static Future<std::string> getFullStatus_impl(Reference<RestoreConfig> restore,
Reference<ReadYourWritesTransaction> tr);
Future<std::string> getFullStatus(Reference<ReadYourWritesTransaction> tr);
std::string toString(); // Added by Meng
@ -163,33 +167,34 @@ public:
typedef RestoreConfig::RestoreFile RestoreFile;
// Describes a file to load blocks from during restore. Ordered by version and then fileName to enable
// incrementally advancing through the map, saving the version and path of the next starting point.
// NOTE: The struct RestoreFileFR can NOT be named RestoreFile, because compiler will get confused in linking which RestoreFile should be used.
// If we use RestoreFile, the compilation can succeed, but weird segmentation fault will happen.
// NOTE: The struct RestoreFileFR can NOT be named RestoreFile, because compiler will get confused in linking which
// RestoreFile should be used. If we use RestoreFile, compilation succeeds, but weird segmentation fault will happen.
struct RestoreFileFR {
Version version;
std::string fileName;
bool isRange; // false for log file
bool isRange; // false for log file
int64_t blockSize;
int64_t fileSize;
Version endVersion; // not meaningful for range files
Version beginVersion; // range file's beginVersion == endVersion; log file contains mutations in version [beginVersion, endVersion)
int64_t cursor; //The start block location to be restored. All blocks before cursor have been scheduled to load and restore
Version endVersion; // not meaningful for range files
Version beginVersion; // range file's beginVersion == endVersion; log file contains mutations in version
// [beginVersion, endVersion)
int64_t cursor; // The start block location to be restored. All blocks before cursor have been scheduled to load and
// restore
Tuple pack() const {
return Tuple()
.append(version)
.append(StringRef(fileName))
.append(isRange)
.append(fileSize)
.append(blockSize)
.append(endVersion)
.append(beginVersion)
.append(cursor);
.append(version)
.append(StringRef(fileName))
.append(isRange)
.append(fileSize)
.append(blockSize)
.append(endVersion)
.append(beginVersion)
.append(cursor);
}
static RestoreFileFR unpack(Tuple const &t) {
static RestoreFileFR unpack(Tuple const& t) {
RestoreFileFR r;
int i = 0;
r.version = t.getInt(i++);
@ -205,25 +210,31 @@ struct RestoreFileFR {
bool operator<(const RestoreFileFR& rhs) const { return beginVersion < rhs.beginVersion; }
RestoreFileFR() : version(invalidVersion), isRange(false), blockSize(0), fileSize(0), endVersion(invalidVersion), beginVersion(invalidVersion), cursor(0) {}
RestoreFileFR(Version version, std::string fileName, bool isRange, int64_t blockSize, int64_t fileSize, Version endVersion, Version beginVersion) : version(version), fileName(fileName), isRange(isRange), blockSize(blockSize), fileSize(fileSize), endVersion(endVersion), beginVersion(beginVersion), cursor(0) {}
RestoreFileFR()
: version(invalidVersion), isRange(false), blockSize(0), fileSize(0), endVersion(invalidVersion),
beginVersion(invalidVersion), cursor(0) {}
RestoreFileFR(Version version, std::string fileName, bool isRange, int64_t blockSize, int64_t fileSize,
Version endVersion, Version beginVersion)
: version(version), fileName(fileName), isRange(isRange), blockSize(blockSize), fileSize(fileSize),
endVersion(endVersion), beginVersion(beginVersion), cursor(0) {}
std::string toString() const {
std::stringstream ss;
ss << "version:" << std::to_string(version) << " fileName:" << fileName << " isRange:" << std::to_string(isRange)
<< " blockSize:" << std::to_string(blockSize) << " fileSize:" << std::to_string(fileSize)
<< " endVersion:" << std::to_string(endVersion) << std::to_string(beginVersion)
<< " cursor:" << std::to_string(cursor);
ss << "version:" << std::to_string(version) << " fileName:" << fileName
<< " isRange:" << std::to_string(isRange) << " blockSize:" << std::to_string(blockSize)
<< " fileSize:" << std::to_string(fileSize) << " endVersion:" << std::to_string(endVersion)
<< std::to_string(beginVersion) << " cursor:" << std::to_string(cursor);
return ss.str();
}
};
namespace parallelFileRestore {
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset, int len);
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file, int64_t offset, int len);
}
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset,
int len);
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file, int64_t offset,
int len);
} // namespace parallelFileRestore
#include "flow/unactorcompiler.h"
#endif //FDBCLIENT_Restore_H
#endif // FDBCLIENT_Restore_H

419
fdbserver/RestoreLoader.actor.cpp
View File

@ -24,80 +24,93 @@
#include "fdbclient/BackupContainer.h"
#include "fdbserver/RestoreLoader.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
typedef std::map<Standalone<StringRef>, Standalone<StringRef>> SerializedMutationListMap; // Key is the signature/version of the mutation list, Value is the mutation list (or part of the mutation list)
// SerializedMutationListMap:
// Key is the signature/version of the mutation list, Value is the mutation list (or part of the mutation list)
typedef std::map<Standalone<StringRef>, Standalone<StringRef>> SerializedMutationListMap;
bool isRangeMutation(MutationRef m);
void splitMutation(Reference<RestoreLoaderData> self, MutationRef m, Arena& mvector_arena, VectorRef<MutationRef>& mvector, Arena& nodeIDs_arena, VectorRef<UID>& nodeIDs) ;
void _parseSerializedMutation(VersionedMutationsMap *kvOps, SerializedMutationListMap *mutationMap, bool isSampling = false);
void splitMutation(Reference<RestoreLoaderData> self, MutationRef m, Arena& mvector_arena,
VectorRef<MutationRef>& mvector, Arena& nodeIDs_arena, VectorRef<UID>& nodeIDs);
void _parseSerializedMutation(VersionedMutationsMap* kvOps, SerializedMutationListMap* mutationMap,
bool isSampling = false);
ACTOR Future<Void> handleRestoreSysInfoRequest(RestoreSysInfoRequest req, Reference<RestoreLoaderData> self);
ACTOR Future<Void> handleSetApplierKeyRangeVectorRequest(RestoreSetApplierKeyRangeVectorRequest req, Reference<RestoreLoaderData> self);
ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self, bool isSampling = false);
ACTOR Future<Void> sendMutationsToApplier(Reference<RestoreLoaderData> self, VersionedMutationsMap *kvOps, bool isRangeFile, Version startVersion, Version endVersion);
ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(SerializedMutationListMap *mutationMap,
std::map<Standalone<StringRef>, uint32_t> *mutationPartMap,
Reference<IBackupContainer> bc, Version version,
std::string fileName, int64_t readOffset, int64_t readLen,
KeyRange restoreRange, Key addPrefix, Key removePrefix,
Key mutationLogPrefix);
ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(VersionedMutationsMap *kvOps,
Reference<IBackupContainer> bc, Version version,
std::string fileName, int64_t readOffset_input, int64_t readLen_input,KeyRange restoreRange);
ACTOR Future<Void> handleSetApplierKeyRangeVectorRequest(RestoreSetApplierKeyRangeVectorRequest req,
Reference<RestoreLoaderData> self);
ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self,
bool isSampling = false);
ACTOR Future<Void> sendMutationsToApplier(Reference<RestoreLoaderData> self, VersionedMutationsMap* kvOps,
bool isRangeFile, Version startVersion, Version endVersion);
ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(SerializedMutationListMap* mutationMap,
std::map<Standalone<StringRef>, uint32_t>* mutationPartMap,
Reference<IBackupContainer> bc, Version version,
std::string fileName, int64_t readOffset, int64_t readLen,
KeyRange restoreRange, Key addPrefix, Key removePrefix,
Key mutationLogPrefix);
ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(VersionedMutationsMap* kvOps,
Reference<IBackupContainer> bc, Version version,
std::string fileName, int64_t readOffset_input,
int64_t readLen_input, KeyRange restoreRange);
ACTOR Future<Void> restoreLoaderCore(RestoreLoaderInterface loaderInterf, int nodeIndex, Database cx) {
state Reference<RestoreLoaderData> self = Reference<RestoreLoaderData>( new RestoreLoaderData(loaderInterf.id(), nodeIndex) );
state Reference<RestoreLoaderData> self =
Reference<RestoreLoaderData>(new RestoreLoaderData(loaderInterf.id(), nodeIndex));
state ActorCollection actors(false);
state Future<Void> exitRole = Never();
state double lastLoopTopTime;
loop {
double loopTopTime = now();
double elapsedTime = loopTopTime - lastLoopTopTime;
if( elapsedTime > 0.050 ) {
if (elapsedTime > 0.050) {
if (deterministicRandom()->random01() < 0.01)
TraceEvent(SevWarn, "SlowRestoreLoaderLoopx100").detail("NodeDesc", self->describeNode()).detail("Elapsed", elapsedTime);
TraceEvent(SevWarn, "SlowRestoreLoaderLoopx100")
.detail("NodeDesc", self->describeNode())
.detail("Elapsed", elapsedTime);
}
lastLoopTopTime = loopTopTime;
state std::string requestTypeStr = "[Init]";
try {
choose {
when ( RestoreSimpleRequest req = waitNext(loaderInterf.heartbeat.getFuture()) ) {
when(RestoreSimpleRequest req = waitNext(loaderInterf.heartbeat.getFuture())) {
requestTypeStr = "heartbeat";
actors.add(handleHeartbeat(req, loaderInterf.id()));
}
when ( RestoreSysInfoRequest req = waitNext(loaderInterf.updateRestoreSysInfo.getFuture()) ) {
when(RestoreSysInfoRequest req = waitNext(loaderInterf.updateRestoreSysInfo.getFuture())) {
requestTypeStr = "updateRestoreSysInfo";
actors.add( handleRestoreSysInfoRequest(req, self) );
actors.add(handleRestoreSysInfoRequest(req, self));
}
when ( RestoreSetApplierKeyRangeVectorRequest req = waitNext(loaderInterf.setApplierKeyRangeVectorRequest.getFuture()) ) {
when(RestoreSetApplierKeyRangeVectorRequest req =
waitNext(loaderInterf.setApplierKeyRangeVectorRequest.getFuture())) {
requestTypeStr = "setApplierKeyRangeVectorRequest";
actors.add(handleSetApplierKeyRangeVectorRequest(req, self));
}
when ( RestoreLoadFileRequest req = waitNext(loaderInterf.loadFile.getFuture()) ) {
when(RestoreLoadFileRequest req = waitNext(loaderInterf.loadFile.getFuture())) {
requestTypeStr = "loadFile";
self->initBackupContainer(req.param.url);
actors.add( handleLoadFileRequest(req, self, false) );
actors.add(handleLoadFileRequest(req, self, false));
}
when ( RestoreVersionBatchRequest req = waitNext(loaderInterf.initVersionBatch.getFuture()) ) {
when(RestoreVersionBatchRequest req = waitNext(loaderInterf.initVersionBatch.getFuture())) {
requestTypeStr = "initVersionBatch";
actors.add( handleInitVersionBatchRequest(req, self) );
actors.add(handleInitVersionBatchRequest(req, self));
}
when ( RestoreVersionBatchRequest req = waitNext(loaderInterf.finishRestore.getFuture()) ) {
when(RestoreVersionBatchRequest req = waitNext(loaderInterf.finishRestore.getFuture())) {
requestTypeStr = "finishRestore";
exitRole = handleFinishRestoreRequest(req, self);
}
when ( wait(exitRole) ) {
when(wait(exitRole)) {
TraceEvent("FastRestore").detail("RestoreLoaderCore", "ExitRole").detail("NodeID", self->id());
break;
}
}
} catch (Error &e) {
TraceEvent(SevWarn, "FastRestore").detail("RestoreLoaderError", e.what()).detail("RequestType", requestTypeStr);
} catch (Error& e) {
TraceEvent(SevWarn, "FastRestore")
.detail("RestoreLoaderError", e.what())
.detail("RequestType", requestTypeStr);
break;
}
}
@ -109,25 +122,25 @@ ACTOR Future<Void> restoreLoaderCore(RestoreLoaderInterface loaderInterf, int no
ACTOR Future<Void> handleRestoreSysInfoRequest(RestoreSysInfoRequest req, Reference<RestoreLoaderData> self) {
TraceEvent("FastRestore").detail("HandleRestoreSysInfoRequest", self->id());
ASSERT(self.isValid());
// The loader has received the appliers interfaces
if ( !self->appliersInterf.empty() ) {
if (!self->appliersInterf.empty()) {
req.reply.send(RestoreCommonReply(self->id()));
return Void();
}
self->appliersInterf = req.sysInfo.appliers;
req.reply.send(RestoreCommonReply(self->id()) );
req.reply.send(RestoreCommonReply(self->id()));
return Void();
}
ACTOR Future<Void> handleSetApplierKeyRangeVectorRequest(RestoreSetApplierKeyRangeVectorRequest req, Reference<RestoreLoaderData> self) {
ACTOR Future<Void> handleSetApplierKeyRangeVectorRequest(RestoreSetApplierKeyRangeVectorRequest req,
Reference<RestoreLoaderData> self) {
// Idempodent operation. OK to re-execute the duplicate cmd
if ( self->range2Applier.empty() ) {
if (self->range2Applier.empty()) {
self->range2Applier = req.range2Applier;
}
}
req.reply.send(RestoreCommonReply(self->id()));
return Void();
@ -136,13 +149,14 @@ ACTOR Future<Void> handleSetApplierKeyRangeVectorRequest(RestoreSetApplierKeyRan
ACTOR Future<Void> _processLoadingParam(LoadingParam param, Reference<RestoreLoaderData> self) {
// Q: How to record the param's fields inside LoadingParam Refer to storageMetrics
TraceEvent("FastRestore").detail("Loader", self->id()).detail("StartProcessLoadParam", param.toString());
ASSERT( param.blockSize > 0 );
ASSERT(param.blockSize > 0);
ASSERT(param.offset % param.blockSize == 0); // Parse file must be at block bondary.
// Temporary data structure for parsing range and log files into (version, <K, V, mutationType>)
// Must use StandAlone to save mutations, otherwise, the mutationref memory will be corrupted
state VersionedMutationsMap kvOps;
state SerializedMutationListMap mutationMap; // Key is the unique identifier for a batch of mutation logs at the same version
// mutationMap: Key is the unique identifier for a batch of mutation logs at the same version
state SerializedMutationListMap mutationMap;
state std::map<Standalone<StringRef>, uint32_t> mutationPartMap; // Sanity check the data parsing is correct
state std::vector<Future<Void>> fileParserFutures;
@ -152,56 +166,65 @@ ACTOR Future<Void> _processLoadingParam(LoadingParam param, Reference<RestoreLoa
for (j = param.offset; j < param.length; j += param.blockSize) {
readOffset = j;
readLen = std::min<int64_t>(param.blockSize, param.length - j);
if ( param.isRangeFile ) {
fileParserFutures.push_back( _parseRangeFileToMutationsOnLoader(&kvOps, self->bc, param.version, param.filename, readOffset, readLen, param.restoreRange) );
if (param.isRangeFile) {
fileParserFutures.push_back(_parseRangeFileToMutationsOnLoader(
&kvOps, self->bc, param.version, param.filename, readOffset, readLen, param.restoreRange));
} else {
fileParserFutures.push_back( _parseLogFileToMutationsOnLoader(&mutationMap, &mutationPartMap, self->bc, param.version, param.filename, readOffset, readLen, param.restoreRange, param.addPrefix, param.removePrefix, param.mutationLogPrefix) );
fileParserFutures.push_back(_parseLogFileToMutationsOnLoader(
&mutationMap, &mutationPartMap, self->bc, param.version, param.filename, readOffset, readLen,
param.restoreRange, param.addPrefix, param.removePrefix, param.mutationLogPrefix));
}
}
wait( waitForAll(fileParserFutures) );
if ( !param.isRangeFile ) {
wait(waitForAll(fileParserFutures));
if (!param.isRangeFile) {
_parseSerializedMutation(&kvOps, &mutationMap);
}
wait( sendMutationsToApplier(self, &kvOps, param.isRangeFile, param.prevVersion, param.endVersion) ); // Send the parsed mutation to applier who will apply the mutation to DB
// Send the parsed mutation to applier who will apply the mutation to DB
wait(sendMutationsToApplier(self, &kvOps, param.isRangeFile, param.prevVersion, param.endVersion));
TraceEvent("FastRestore").detail("Loader", self->id()).detail("FinishLoadingFile", param.filename);
return Void();
}
ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self, bool isSampling) {
if (self->processedFileParams.find(req.param) == self->processedFileParams.end()) {
ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self,
bool isSampling) {
if (self->processedFileParams.find(req.param) == self->processedFileParams.end()) {
TraceEvent("FastRestore").detail("Loader", self->id()).detail("ProcessLoadParam", req.param.toString());
self->processedFileParams[req.param] = Never();
self->processedFileParams[req.param] = _processLoadingParam(req.param, self);
self->processedFileParams[req.param] = _processLoadingParam(req.param, self);
}
ASSERT(self->processedFileParams.find(req.param) != self->processedFileParams.end());
wait(self->processedFileParams[req.param]); // wait on the processing of the req.param.
ASSERT(self->processedFileParams.find(req.param) != self->processedFileParams.end());
wait(self->processedFileParams[req.param]); // wait on the processing of the req.param.
req.reply.send(RestoreCommonReply(self->id()));
return Void();
}
// TODO: This function can be revised better
ACTOR Future<Void> sendMutationsToApplier(Reference<RestoreLoaderData> self,
VersionedMutationsMap *pkvOps,
bool isRangeFile, Version startVersion, Version endVersion) {
state VersionedMutationsMap &kvOps = *pkvOps;
ACTOR Future<Void> sendMutationsToApplier(Reference<RestoreLoaderData> self, VersionedMutationsMap* pkvOps,
bool isRangeFile, Version startVersion, Version endVersion) {
state VersionedMutationsMap& kvOps = *pkvOps;
state int kvCount = 0;
state int splitMutationIndex = 0;
TraceEvent("FastRestore").detail("SendMutationToApplier", self->id()).detail("IsRangeFile", isRangeFile)
.detail("StartVersion", startVersion).detail("EndVersion", endVersion);
TraceEvent("FastRestore")
.detail("SendMutationToApplier", self->id())
.detail("IsRangeFile", isRangeFile)
.detail("StartVersion", startVersion)
.detail("EndVersion", endVersion);
// Ensure there is a mutation request sent at endVersion, so that applier can advance its notifiedVersion
if ( kvOps.find(endVersion) == kvOps.end() ) {
if (kvOps.find(endVersion) == kvOps.end()) {
kvOps[endVersion] = VectorRef<MutationRef>(); // Empty mutation vector will be handled by applier
}
state std::map<UID, Standalone<VectorRef<MutationRef>>> applierMutationsBuffer; // The mutation vector to be sent to each applier
state std::map<UID, double> applierMutationsSize; // buffered mutation vector size for each applier
// applierMutationsBuffer is the mutation vector to be sent to each applier
// applierMutationsSize is buffered mutation vector size for each applier
state std::map<UID, Standalone<VectorRef<MutationRef>>> applierMutationsBuffer;
state std::map<UID, double> applierMutationsSize;
state Standalone<VectorRef<MutationRef>> mvector;
state Standalone<VectorRef<UID>> nodeIDs;
// Initialize the above two maps
@ -212,11 +235,11 @@ ACTOR Future<Void> sendMutationsToApplier(Reference<RestoreLoaderData> self,
splitMutationIndex = 0;
kvCount = 0;
state VersionedMutationsMap::iterator kvOp;
for ( kvOp = kvOps.begin(); kvOp != kvOps.end(); kvOp++) {
for (kvOp = kvOps.begin(); kvOp != kvOps.end(); kvOp++) {
applierMutationsBuffer.clear();
applierMutationsSize.clear();
for (auto &applierID : applierIDs) {
for (auto& applierID : applierIDs) {
applierMutationsBuffer[applierID] = Standalone<VectorRef<MutationRef>>(VectorRef<MutationRef>());
applierMutationsSize[applierID] = 0.0;
}
@ -226,7 +249,7 @@ ACTOR Future<Void> sendMutationsToApplier(Reference<RestoreLoaderData> self,
for (mIndex = 0; mIndex < kvOp->second.size(); mIndex++) {
kvm = kvOp->second[mIndex];
// Send the mutation to applier
if ( isRangeMutation(kvm) ) {
if (isRangeMutation(kvm)) {
// Because using a vector of mutations causes overhead, and the range mutation should happen rarely;
// We handle the range mutation and key mutation differently for the benefit of avoiding memory copy
mvector.pop_front(mvector.size());
@ -235,38 +258,42 @@ ACTOR Future<Void> sendMutationsToApplier(Reference<RestoreLoaderData> self,
splitMutation(self, kvm, mvector.arena(), mvector.contents(), nodeIDs.arena(), nodeIDs.contents());
ASSERT(mvector.size() == nodeIDs.size());
for (splitMutationIndex = 0; splitMutationIndex < mvector.size(); splitMutationIndex++ ) {
for (splitMutationIndex = 0; splitMutationIndex < mvector.size(); splitMutationIndex++) {
MutationRef mutation = mvector[splitMutationIndex];
UID applierID = nodeIDs[splitMutationIndex];
//printf("SPLITTED MUTATION: %d: mutation:%s applierID:%s\n", splitMutationIndex, mutation.toString().c_str(), applierID.toString().c_str());
applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation); // Q: Maybe push_back_deep()?
// printf("SPLITTED MUTATION: %d: mutation:%s applierID:%s\n", splitMutationIndex,
// mutation.toString().c_str(), applierID.toString().c_str());
applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
applierMutationsSize[applierID] += mutation.expectedSize();
kvCount++;
}
} else { // mutation operates on a particular key
std::map<Standalone<KeyRef>, UID>::iterator itlow = self->range2Applier.upper_bound(kvm.param1); // lower_bound returns the iterator that is > m.param1
std::map<Standalone<KeyRef>, UID>::iterator itlow = self->range2Applier.upper_bound(kvm.param1);
--itlow; // make sure itlow->first <= m.param1
ASSERT( itlow->first <= kvm.param1 );
ASSERT(itlow->first <= kvm.param1);
MutationRef mutation = kvm;
UID applierID = itlow->second;
//printf("KV--Applier: K:%s ApplierID:%s\n", kvm.param1.toString().c_str(), applierID.toString().c_str());
// printf("KV--Applier: K:%s ApplierID:%s\n", kvm.param1.toString().c_str(),
// applierID.toString().c_str());
kvCount++;
applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation); // Q: Maybe push_back_deep()?
applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation);
applierMutationsSize[applierID] += mutation.expectedSize();
}
} // Mutations at the same version
// Send the mutations to appliers for each version
for (auto &applierID : applierIDs) {
requests.push_back( std::make_pair(applierID, RestoreSendMutationVectorVersionedRequest(prevVersion, commitVersion, isRangeFile, applierMutationsBuffer[applierID])) );
for (auto& applierID : applierIDs) {
requests.push_back(std::make_pair(
applierID, RestoreSendMutationVectorVersionedRequest(prevVersion, commitVersion, isRangeFile,
applierMutationsBuffer[applierID])));
applierMutationsBuffer[applierID].pop_front(applierMutationsBuffer[applierID].size());
applierMutationsSize[applierID] = 0;
}
wait( sendBatchRequests(&RestoreApplierInterface::sendMutationVector, self->appliersInterf, requests) );
wait(sendBatchRequests(&RestoreApplierInterface::sendMutationVector, self->appliersInterf, requests));
requests.clear();
ASSERT( prevVersion < commitVersion );
ASSERT(prevVersion < commitVersion);
prevVersion = commitVersion;
} // all versions of mutations
@ -274,30 +301,31 @@ ACTOR Future<Void> sendMutationsToApplier(Reference<RestoreLoaderData> self,
return Void();
}
// TODO: Add a unit test for this function
void splitMutation(Reference<RestoreLoaderData> self, MutationRef m, Arena& mvector_arena, VectorRef<MutationRef>& mvector, Arena& nodeIDs_arena, VectorRef<UID>& nodeIDs) {
void splitMutation(Reference<RestoreLoaderData> self, MutationRef m, Arena& mvector_arena,
VectorRef<MutationRef>& mvector, Arena& nodeIDs_arena, VectorRef<UID>& nodeIDs) {
// mvector[i] should be mapped to nodeID[i]
ASSERT(mvector.empty());
ASSERT(nodeIDs.empty());
// key range [m->param1, m->param2)
std::map<Standalone<KeyRef>, UID>::iterator itlow, itup; //we will return [itlow, itup)
std::map<Standalone<KeyRef>, UID>::iterator itlow, itup; // we will return [itlow, itup)
itlow = self->range2Applier.lower_bound(m.param1); // lower_bound returns the iterator that is >= m.param1
if ( itlow->first > m.param1 ) {
if ( itlow != self->range2Applier.begin() ) {
if (itlow->first > m.param1) {
if (itlow != self->range2Applier.begin()) {
--itlow;
}
}
itup = self->range2Applier.upper_bound(m.param2); // upper_bound returns the iterator that is > m.param2; return rmap::end if no keys are considered to go after m.param2.
ASSERT( itup == self->range2Applier.end() || itup->first > m.param2 );
itup = self->range2Applier.upper_bound(m.param2); // return rmap::end if no key is after m.param2.
ASSERT(itup == self->range2Applier.end() || itup->first > m.param2);
std::map<Standalone<KeyRef>, UID>::iterator itApplier;
while (itlow != itup) {
Standalone<MutationRef> curm; //current mutation
Standalone<MutationRef> curm; // current mutation
curm.type = m.type;
// The first split mutation should starts with m.first. The later ones should start with the range2Applier boundary
if ( m.param1 > itlow->first ) {
// The first split mutation should starts with m.first.
// The later ones should start with the range2Applier boundary.
if (m.param1 > itlow->first) {
curm.param1 = m.param1;
} else {
curm.param1 = itlow->first;
@ -305,15 +333,15 @@ void splitMutation(Reference<RestoreLoaderData> self, MutationRef m, Arena& mve
itApplier = itlow;
itlow++;
if (itlow == itup) {
ASSERT( m.param2 <= normalKeys.end );
ASSERT(m.param2 <= normalKeys.end);
curm.param2 = m.param2;
} else if ( m.param2 < itlow->first ) {
} else if (m.param2 < itlow->first) {
UNREACHABLE();
curm.param2 = m.param2;
} else {
curm.param2 = itlow->first;
}
ASSERT( curm.param1 <= curm.param2 );
ASSERT(curm.param1 <= curm.param2);
mvector.push_back_deep(mvector_arena, curm);
nodeIDs.push_back(nodeIDs_arena, itApplier->second);
}
@ -321,47 +349,50 @@ void splitMutation(Reference<RestoreLoaderData> self, MutationRef m, Arena& mve
return;
}
// key_input format: [logRangeMutation.first][hash_value_of_commit_version:1B][bigEndian64(commitVersion)][bigEndian32(part)]
// key_input format:
// [logRangeMutation.first][hash_value_of_commit_version:1B][bigEndian64(commitVersion)][bigEndian32(part)]
// value_input: serialized binary of mutations at the same version
bool concatenateBackupMutationForLogFile(std::map<Standalone<StringRef>, Standalone<StringRef>> *pMutationMap,
std::map<Standalone<StringRef>, uint32_t> *pMutationPartMap,
Standalone<StringRef> key_input, Standalone<StringRef> val_input) {
SerializedMutationListMap &mutationMap = *pMutationMap;
std::map<Standalone<StringRef>, uint32_t> &mutationPartMap = *pMutationPartMap;
bool concatenateBackupMutationForLogFile(std::map<Standalone<StringRef>, Standalone<StringRef>>* pMutationMap,
std::map<Standalone<StringRef>, uint32_t>* pMutationPartMap,
Standalone<StringRef> key_input, Standalone<StringRef> val_input) {
SerializedMutationListMap& mutationMap = *pMutationMap;
std::map<Standalone<StringRef>, uint32_t>& mutationPartMap = *pMutationPartMap;
std::string prefix = "||\t";
std::stringstream ss;
StringRef val = val_input.contents();
StringRefReaderMX reader(val, restore_corrupted_data());
StringRefReaderMX readerKey(key_input, restore_corrupted_data()); //read key_input!
StringRefReaderMX readerKey(key_input, restore_corrupted_data()); // read key_input!
int logRangeMutationFirstLength = key_input.size() - 1 - 8 - 4;
bool concatenated = false;
ASSERT_WE_THINK( key_input.size() >= 1 + 8 + 4 );
ASSERT_WE_THINK(key_input.size() >= 1 + 8 + 4);
if ( logRangeMutationFirstLength > 0 ) {
readerKey.consume(logRangeMutationFirstLength); // Strip out the [logRangeMutation.first]; otherwise, the following readerKey.consume will produce wrong value
if (logRangeMutationFirstLength > 0) {
// Strip out the [logRangeMutation.first]; otherwise, the following readerKey.consume will produce wrong value
readerKey.consume(logRangeMutationFirstLength);
}
uint8_t hashValue = readerKey.consume<uint8_t>();
uint64_t commitVersion = readerKey.consumeNetworkUInt64(); // Convert big Endian value encoded in log file into a littleEndian uint64_t value, i.e., commitVersion
uint32_t part = readerKey.consumeNetworkUInt32(); //Consume big Endian value encoded in log file
//Use commitVersion as id
Standalone<StringRef> id = StringRef((uint8_t*) &commitVersion, 8);
uint64_t commitVersion = readerKey.consumeNetworkUInt64();
uint32_t part = readerKey.consumeNetworkUInt32();
// Use commitVersion as id
Standalone<StringRef> id = StringRef((uint8_t*)&commitVersion, 8);
if ( mutationMap.find(id) == mutationMap.end() ) {
if (mutationMap.find(id) == mutationMap.end()) {
mutationMap.insert(std::make_pair(id, val_input));
if ( part != 0 ) {
if (part != 0) {
fprintf(stderr, "[ERROR]!!! part:%d != 0 for key_input:%s\n", part, getHexString(key_input).c_str());
}
mutationPartMap.insert(std::make_pair(id, part));
} else { // concatenate the val string with the same commitVersion
mutationMap[id] = mutationMap[id].contents().withSuffix(val_input.contents()); //Assign the new Areana to the map's value
if ( part != (mutationPartMap[id] + 1) ) {
mutationMap[id] =
mutationMap[id].contents().withSuffix(val_input.contents()); // Assign the new Areana to the map's value
if (part != (mutationPartMap[id] + 1)) {
// Check if the same range or log file has been processed more than once!
fprintf(stderr, "[ERROR]!!! current part id:%d new part_direct:%d is not the next integer of key_input:%s\n", mutationPartMap[id], part, getHexString(key_input).c_str());
fprintf(stderr,
"[ERROR]!!! current part id:%d new part_direct:%d is not the next integer of key_input:%s\n",
mutationPartMap[id], part, getHexString(key_input).c_str());
printf("[HINT] Check if the same range or log file has been processed more than once!\n");
}
mutationPartMap[id] = part;
@ -376,25 +407,28 @@ bool isRangeMutation(MutationRef m) {
ASSERT(m.type != MutationRef::Type::DebugKeyRange);
return true;
} else {
ASSERT( m.type == MutationRef::Type::SetValue || isAtomicOp((MutationRef::Type) m.type) );
ASSERT(m.type == MutationRef::Type::SetValue || isAtomicOp((MutationRef::Type)m.type));
return false;
}
}
// Parse the kv pair (version, serialized_mutation), which are the results parsed from log file, into
// (version, <K, V, mutationType>) pair;
// Put the parsed versioned mutations into *pkvOps.
//
// Input key: [commitVersion_of_the_mutation_batch:uint64_t];
// Input value: [includeVersion:uint64_t][val_length:uint32_t][encoded_list_of_mutations], where
// includeVersion is the serialized version in the batch commit. It is not the commitVersion in Input key.
//
// val_length is always equal to (val.size() - 12); otherwise,
// we may not get the entire mutation list for the version encoded_list_of_mutations:
// [mutation1][mutation2]...[mutationk], where
// a mutation is encoded as [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][keyContent][valueContent]
void _parseSerializedMutation(VersionedMutationsMap* pkvOps, SerializedMutationListMap* pmutationMap, bool isSampling) {
VersionedMutationsMap& kvOps = *pkvOps;
SerializedMutationListMap& mutationMap = *pmutationMap;
// Parse the kv pair (version, serialized_mutation), which are the results parsed from log file, into (version, <K, V, mutationType>) pair
// Put the parsed versioned mutations into *pkvOps
// Input key: [commitVersion_of_the_mutation_batch:uint64_t]
// Input value: [includeVersion:uint64_t][val_length:uint32_t][encoded_list_of_mutations], where
// includeVersion is the serialized version in the batch commit. It is not the commitVersion in Input key.
// val_length is always equal to (val.size() - 12); otherwise, we may not get the entire mutation list for the version
// encoded_list_of_mutations: [mutation1][mutation2]...[mutationk], where
// a mutation is encoded as [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][keyContent][valueContent]
void _parseSerializedMutation(VersionedMutationsMap *pkvOps, SerializedMutationListMap *pmutationMap, bool isSampling) {
VersionedMutationsMap &kvOps = *pkvOps;
SerializedMutationListMap &mutationMap = *pmutationMap;
for ( auto& m : mutationMap ) {
for (auto& m : mutationMap) {
StringRef k = m.first.contents();
StringRef val = m.second.contents();
@ -404,76 +438,82 @@ bool isRangeMutation(MutationRef m) {
StringRefReaderMX vReader(val, restore_corrupted_data());
vReader.consume<uint64_t>(); // Consume the includeVersion
uint32_t val_length_decoded = vReader.consume<uint32_t>(); // Parse little endian value, confirmed it is correct!
ASSERT( val_length_decoded == val.size() - 12 ); // 12 is the length of [includeVersion:uint64_t][val_length:uint32_t]
uint32_t val_length_decoded =
vReader.consume<uint32_t>(); // Parse little endian value, confirmed it is correct!
ASSERT(val_length_decoded ==
val.size() - 12); // 12 is the length of [includeVersion:uint64_t][val_length:uint32_t]
while (1) {
// stop when reach the end of the string
if(vReader.eof() ) { //|| *reader.rptr == 0xFF
if (vReader.eof()) { //|| *reader.rptr == 0xFF
break;
}
uint32_t type = vReader.consume<uint32_t>();
uint32_t kLen = vReader.consume<uint32_t>();
uint32_t vLen = vReader.consume<uint32_t>();
const uint8_t *k = vReader.consume(kLen);
const uint8_t *v = vReader.consume(vLen);
const uint8_t* k = vReader.consume(kLen);
const uint8_t* v = vReader.consume(vLen);
MutationRef mutation((MutationRef::Type) type, KeyRef(k, kLen), KeyRef(v, vLen));
MutationRef mutation((MutationRef::Type)type, KeyRef(k, kLen), KeyRef(v, vLen));
kvOps[commitVersion].push_back_deep(kvOps[commitVersion].arena(), mutation);
ASSERT_WE_THINK( kLen >= 0 && kLen < val.size() );
ASSERT_WE_THINK( vLen >= 0 && vLen < val.size() );
ASSERT_WE_THINK(kLen >= 0 && kLen < val.size());
ASSERT_WE_THINK(vLen >= 0 && vLen < val.size());
}
}
}
// Parsing the data blocks in a range file
ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(VersionedMutationsMap *pkvOps,
Reference<IBackupContainer> bc, Version version,
std::string fileName, int64_t readOffset, int64_t readLen,
KeyRange restoreRange) {
state VersionedMutationsMap &kvOps = *pkvOps;
ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(VersionedMutationsMap* pkvOps,
Reference<IBackupContainer> bc, Version version,
std::string fileName, int64_t readOffset, int64_t readLen,
KeyRange restoreRange) {
state VersionedMutationsMap& kvOps = *pkvOps;
// The set of key value version is rangeFile.version. the key-value set in the same range file has the same version
Reference<IAsyncFile> inFile = wait(bc->readFile(fileName));
state Standalone<VectorRef<KeyValueRef>> blockData = wait(parallelFileRestore::decodeRangeFileBlock(inFile, readOffset, readLen));
// The set of key value version is rangeFile.version. the key-value set in the same range file has the same version
Reference<IAsyncFile> inFile = wait(bc->readFile(fileName));
state Standalone<VectorRef<KeyValueRef>> blockData =
wait(parallelFileRestore::decodeRangeFileBlock(inFile, readOffset, readLen));
TraceEvent("FastRestore").detail("DecodedRangeFile", fileName).detail("DataSize", blockData.contents().size());
// First and last key are the range for this file
state KeyRange fileRange = KeyRangeRef(blockData.front().key, blockData.back().key);
// First and last key are the range for this file
state KeyRange fileRange = KeyRangeRef(blockData.front().key, blockData.back().key);
// If fileRange doesn't intersect restore range then we're done.
if(!fileRange.intersects(restoreRange)) {
return Void();
}
// If fileRange doesn't intersect restore range then we're done.
if (!fileRange.intersects(restoreRange)) {
return Void();
}
// We know the file range intersects the restore range but there could still be keys outside the restore range.
// Find the subvector of kv pairs that intersect the restore range.
// Note that the first and last keys are just the range endpoints for this file. They are metadata, not the real data
int rangeStart = 1;
int rangeEnd = blockData.size() -1; // The rangeStart and rangeEnd is [,)
// We know the file range intersects the restore range but there could still be keys outside the restore range.
// Find the subvector of kv pairs that intersect the restore range.
// Note that the first and last keys are just the range endpoints for this file.
// They are metadata, not the real data.
int rangeStart = 1;
int rangeEnd = blockData.size() - 1; // The rangeStart and rangeEnd is [,)
// Slide start from begining, stop if something in range is found
// Slide start from begining, stop if something in range is found
// Move rangeStart and rangeEnd until they is within restoreRange
while(rangeStart < rangeEnd && !restoreRange.contains(blockData[rangeStart].key)) {
while (rangeStart < rangeEnd && !restoreRange.contains(blockData[rangeStart].key)) {
++rangeStart;
}
// Side end from back, stop if something at (rangeEnd-1) is found in range
while(rangeEnd > rangeStart && !restoreRange.contains(blockData[rangeEnd - 1].key)) {
// Side end from back, stop if something at (rangeEnd-1) is found in range
while (rangeEnd > rangeStart && !restoreRange.contains(blockData[rangeEnd - 1].key)) {
--rangeEnd;
}
// Now data only contains the kv mutation within restoreRange
state VectorRef<KeyValueRef> data = blockData.slice(rangeStart, rangeEnd);
state int start = 0;
state int end = data.size();
// Now data only contains the kv mutation within restoreRange
state VectorRef<KeyValueRef> data = blockData.slice(rangeStart, rangeEnd);
state int start = 0;
state int end = data.size();
// Convert KV in data into mutations in kvOps
for(int i = start; i < end; ++i) {
// NOTE: The KV pairs in range files are the real KV pairs in original DB.
for (int i = start; i < end; ++i) {
// NOTE: The KV pairs in range files are the real KV pairs in original DB.
// Should NOT removePrefix and addPrefix for the backup data!
// In other words, the following operation is wrong: data[i].key.removePrefix(removePrefix).withPrefix(addPrefix)
MutationRef m(MutationRef::Type::SetValue, data[i].key, data[i].value); //ASSUME: all operation in range file is set.
// In other words, the following operation is wrong:
// data[i].key.removePrefix(removePrefix).withPrefix(addPrefix)
MutationRef m(MutationRef::Type::SetValue, data[i].key,
data[i].value); // ASSUME: all operation in range file is set.
// We cache all kv operations into kvOps, and apply all kv operations later in one place
kvOps.insert(std::make_pair(version, VectorRef<MutationRef>()));
@ -483,32 +523,33 @@ ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(VersionedMutationsM
}
return Void();
}
}
// Parse data blocks in a log file into a vector of <string, string> pairs. Each pair.second contains the mutations at a version encoded in pair.first
// Step 1: decodeLogFileBlock into <string, string> pairs
// Step 2: Concatenate the pair.second of pairs with the same pair.first.
ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(std::map<Standalone<StringRef>, Standalone<StringRef>> *pMutationMap,
std::map<Standalone<StringRef>, uint32_t> *pMutationPartMap,
Reference<IBackupContainer> bc, Version version,
std::string fileName, int64_t readOffset, int64_t readLen,
KeyRange restoreRange, Key addPrefix, Key removePrefix,
Key mutationLogPrefix) {
state Reference<IAsyncFile> inFile = wait(bc->readFile(fileName));
// decodeLogFileBlock() must read block by block!
state Standalone<VectorRef<KeyValueRef>> data = wait(parallelFileRestore::decodeLogFileBlock(inFile, readOffset, readLen));
TraceEvent("FastRestore").detail("DecodedLogFile", fileName).detail("DataSize", data.contents().size());
// Parse data blocks in a log file into a vector of <string, string> pairs. Each pair.second contains the mutations at a
// version encoded in pair.first Step 1: decodeLogFileBlock into <string, string> pairs Step 2: Concatenate the
// pair.second of pairs with the same pair.first.
ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(
std::map<Standalone<StringRef>, Standalone<StringRef>>* pMutationMap,
std::map<Standalone<StringRef>, uint32_t>* pMutationPartMap, Reference<IBackupContainer> bc, Version version,
std::string fileName, int64_t readOffset, int64_t readLen, KeyRange restoreRange, Key addPrefix, Key removePrefix,
Key mutationLogPrefix) {
state Reference<IAsyncFile> inFile = wait(bc->readFile(fileName));
// decodeLogFileBlock() must read block by block!
state Standalone<VectorRef<KeyValueRef>> data =
wait(parallelFileRestore::decodeLogFileBlock(inFile, readOffset, readLen));
TraceEvent("FastRestore").detail("DecodedLogFile", fileName).detail("DataSize", data.contents().size());
state int start = 0;
state int end = data.size();
state int start = 0;
state int end = data.size();
state int numConcatenated = 0;
for(int i = start; i < end; ++i) {
for (int i = start; i < end; ++i) {
Key k = data[i].key.withPrefix(mutationLogPrefix);
ValueRef v = data[i].value;
// Concatenate the backuped param1 and param2 (KV) at the same version.
bool concatenated = concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, data[i].key, data[i].value);
numConcatenated += ( concatenated ? 1 : 0);
bool concatenated =
concatenateBackupMutationForLogFile(pMutationMap, pMutationPartMap, data[i].key, data[i].value);
numConcatenated += (concatenated ? 1 : 0);
}
return Void();
}
}

39
fdbserver/RestoreLoader.actor.h
View File

@ -22,10 +22,10 @@
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORE_LOADER_G_H)
#define FDBSERVER_RESTORE_LOADER_G_H
#include "fdbserver/RestoreLoader.actor.g.h"
#define FDBSERVER_RESTORE_LOADER_G_H
#include "fdbserver/RestoreLoader.actor.g.h"
#elif !defined(FDBSERVER_RESTORE_LOADER_H)
#define FDBSERVER_RESTORE_LOADER_H
#define FDBSERVER_RESTORE_LOADER_H
#include <sstream>
#include "flow/Stats.h"
@ -45,9 +45,11 @@
struct RestoreLoaderData : RestoreRoleData, public ReferenceCounted<RestoreLoaderData> {
std::map<LoadingParam, Future<Void>> processedFileParams;
// range2Applier is in master and loader node. Loader node uses this to determine which applier a mutation should be sent
std::map<Standalone<KeyRef>, UID> range2Applier; // KeyRef is the inclusive lower bound of the key range the applier (UID) is responsible for
std::map<Standalone<KeyRef>, int> keyOpsCount; // The number of operations per key which is used to determine the key-range boundary for appliers
// range2Applier is in master and loader. Loader uses this to determine which applier a mutation should be sent
// KeyRef is the inclusive lower bound of the key range the applier (UID) is responsible for
std::map<Standalone<KeyRef>, UID> range2Applier;
// keyOpsCount is the number of operations per key which is used to determine the key-range boundary for appliers
std::map<Standalone<KeyRef>, int> keyOpsCount;
int numSampledMutations; // The total number of mutations received from sampled data.
Reference<IBackupContainer> bc; // Backup container is used to read backup files
@ -66,12 +68,12 @@ struct RestoreLoaderData : RestoreRoleData, public ReferenceCounted<RestoreLoade
std::string describeNode() {
std::stringstream ss;
ss << "[Role: Loader] [NodeID:" << nodeID.toString().c_str()
<< "] [NodeIndex:" << std::to_string(nodeIndex) << "]";
ss << "[Role: Loader] [NodeID:" << nodeID.toString().c_str() << "] [NodeIndex:" << std::to_string(nodeIndex)
<< "]";
return ss.str();
}
void resetPerVersionBatch() {
void resetPerVersionBatch() {
TraceEvent("FastRestore").detail("ResetPerVersionBatchOnLoader", nodeID);
RestoreRoleData::resetPerVersionBatch();
range2Applier.clear();
@ -81,18 +83,18 @@ struct RestoreLoaderData : RestoreRoleData, public ReferenceCounted<RestoreLoade
}
// Only get the appliers that are responsible for a range
std::vector<UID> getWorkingApplierIDs() {
std::vector<UID> applierIDs;
for ( auto &applier : range2Applier ) {
applierIDs.push_back(applier.second);
}
std::vector<UID> getWorkingApplierIDs() {
std::vector<UID> applierIDs;
for (auto& applier : range2Applier) {
applierIDs.push_back(applier.second);
}
ASSERT( !applierIDs.empty() );
return applierIDs;
}
ASSERT(!applierIDs.empty());
return applierIDs;
}
void initBackupContainer(Key url) {
if ( bcUrl == url && bc.isValid() ) {
if (bcUrl == url && bc.isValid()) {
return;
}
bcUrl = url;
@ -100,7 +102,6 @@ struct RestoreLoaderData : RestoreRoleData, public ReferenceCounted<RestoreLoade
}
};
ACTOR Future<Void> restoreLoaderCore(RestoreLoaderInterface loaderInterf, int nodeIndex, Database cx);
#include "flow/unactorcompiler.h"

268
fdbserver/RestoreMaster.actor.cpp
View File

@ -33,17 +33,22 @@
#include "fdbserver/RestoreApplier.actor.h"
#include "fdbserver/RestoreLoader.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR static Future<Void> _clearDB(Database cx);
ACTOR static Future<Void> _collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* output_files, Database cx, RestoreRequest request);
ACTOR static Future<Void> _collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* output_files,
Database cx, RestoreRequest request);
ACTOR static Future<Version> processRestoreRequest(RestoreRequest request, Reference<RestoreMasterData> self, Database cx);
ACTOR static Future<Version> processRestoreRequest(RestoreRequest request, Reference<RestoreMasterData> self,
Database cx);
ACTOR static Future<Void> startProcessRestoreRequests(Reference<RestoreMasterData> self, Database cx);
ACTOR static Future<Void> distributeWorkloadPerVersionBatch(Reference<RestoreMasterData> self, Database cx, RestoreRequest request, VersionBatch versionBatch);
ACTOR static Future<Void> distributeWorkloadPerVersionBatch(Reference<RestoreMasterData> self, Database cx,
RestoreRequest request, VersionBatch versionBatch);
ACTOR static Future<Void> recruitRestoreRoles(Reference<RestoreWorkerData> masterWorker, Reference<RestoreMasterData> masterData);
ACTOR static Future<Void> distributeRestoreSysInfo(Reference<RestoreWorkerData> masterWorker, Reference<RestoreMasterData> masterData);
ACTOR static Future<Void> recruitRestoreRoles(Reference<RestoreWorkerData> masterWorker,
Reference<RestoreMasterData> masterData);
ACTOR static Future<Void> distributeRestoreSysInfo(Reference<RestoreWorkerData> masterWorker,
Reference<RestoreMasterData> masterData);
ACTOR static Future<Standalone<VectorRef<RestoreRequest>>> collectRestoreRequests(Database cx);
ACTOR static Future<Void> initializeVersionBatch(Reference<RestoreMasterData> self);
@ -53,57 +58,63 @@ ACTOR static Future<Void> notifyRestoreCompleted(Reference<RestoreMasterData> se
void dummySampleWorkload(Reference<RestoreMasterData> self);
ACTOR Future<Void> startRestoreMaster(Reference<RestoreWorkerData> masterWorker, Database cx) {
state Reference<RestoreMasterData> self = Reference<RestoreMasterData>(new RestoreMasterData());
// recruitRestoreRoles must come after masterWorker has finished collectWorkerInterface
wait( recruitRestoreRoles(masterWorker, self) );
wait(recruitRestoreRoles(masterWorker, self));
wait( distributeRestoreSysInfo(masterWorker, self) );
wait(distributeRestoreSysInfo(masterWorker, self));
wait( startProcessRestoreRequests(self, cx) );
wait(startProcessRestoreRequests(self, cx));
return Void();
}
// RestoreWorker that has restore master role: Recruite a role for each worker
ACTOR Future<Void> recruitRestoreRoles(Reference<RestoreWorkerData> masterWorker, Reference<RestoreMasterData> masterData) {
TraceEvent("FastRestore").detail("RecruitRestoreRoles", masterWorker->workerInterfaces.size())
.detail("NumLoaders", opConfig.num_loaders).detail("NumAppliers", opConfig.num_appliers);
ACTOR Future<Void> recruitRestoreRoles(Reference<RestoreWorkerData> masterWorker,
Reference<RestoreMasterData> masterData) {
TraceEvent("FastRestore")
.detail("RecruitRestoreRoles", masterWorker->workerInterfaces.size())
.detail("NumLoaders", opConfig.num_loaders)
.detail("NumAppliers", opConfig.num_appliers);
ASSERT(masterData->loadersInterf.empty() && masterData->appliersInterf.empty());
ASSERT( masterData.isValid() );
ASSERT( opConfig.num_loaders > 0 && opConfig.num_appliers > 0 );
ASSERT( opConfig.num_loaders + opConfig.num_appliers <= masterWorker->workerInterfaces.size() ); // We assign 1 role per worker for now
ASSERT(masterData.isValid());
ASSERT(opConfig.num_loaders > 0 && opConfig.num_appliers > 0);
// We assign 1 role per worker for now
ASSERT(opConfig.num_loaders + opConfig.num_appliers <= masterWorker->workerInterfaces.size());
// Assign a role to each worker
state int nodeIndex = 0;
state RestoreRole role;
std::map<UID, RestoreRecruitRoleRequest> requests;
for (auto &workerInterf : masterWorker->workerInterfaces) {
if ( nodeIndex >= 0 && nodeIndex < opConfig.num_appliers ) {
for (auto& workerInterf : masterWorker->workerInterfaces) {
if (nodeIndex >= 0 && nodeIndex < opConfig.num_appliers) {
// [0, numApplier) are appliers
role = RestoreRole::Applier;
} else if ( nodeIndex >= opConfig.num_appliers && nodeIndex < opConfig.num_loaders + opConfig.num_appliers ) {
} else if (nodeIndex >= opConfig.num_appliers && nodeIndex < opConfig.num_loaders + opConfig.num_appliers) {
// [numApplier, numApplier + numLoader) are loaders
role = RestoreRole::Loader;
} else {
break;
}
TraceEvent("FastRestore").detail("Role", getRoleStr(role)).detail("NodeIndex", nodeIndex).detail("WorkerNode", workerInterf.first);
TraceEvent("FastRestore")
.detail("Role", getRoleStr(role))
.detail("NodeIndex", nodeIndex)
.detail("WorkerNode", workerInterf.first);
requests[workerInterf.first] = RestoreRecruitRoleRequest(role, nodeIndex);
nodeIndex++;
}
state std::vector<RestoreRecruitRoleReply> replies;
wait( getBatchReplies(&RestoreWorkerInterface::recruitRole, masterWorker->workerInterfaces, requests, &replies) );
wait(getBatchReplies(&RestoreWorkerInterface::recruitRole, masterWorker->workerInterfaces, requests, &replies));
for (auto& reply : replies) {
if ( reply.role == RestoreRole::Applier ) {
if (reply.role == RestoreRole::Applier) {
ASSERT_WE_THINK(reply.applier.present());
masterData->appliersInterf[reply.applier.get().id()] = reply.applier.get();
} else if ( reply.role == RestoreRole::Loader ) {
} else if (reply.role == RestoreRole::Loader) {
ASSERT_WE_THINK(reply.loader.present());
masterData->loadersInterf[reply.loader.get().id()] = reply.loader.get();
} else {
@ -115,18 +126,19 @@ ACTOR Future<Void> recruitRestoreRoles(Reference<RestoreWorkerData> masterWorker
return Void();
}
ACTOR Future<Void> distributeRestoreSysInfo(Reference<RestoreWorkerData> masterWorker, Reference<RestoreMasterData> masterData) {
ASSERT( masterData.isValid() );
ASSERT( !masterData->loadersInterf.empty() );
ACTOR Future<Void> distributeRestoreSysInfo(Reference<RestoreWorkerData> masterWorker,
Reference<RestoreMasterData> masterData) {
ASSERT(masterData.isValid());
ASSERT(!masterData->loadersInterf.empty());
RestoreSysInfo sysInfo(masterData->appliersInterf);
std::vector<std::pair<UID, RestoreSysInfoRequest>> requests;
for (auto &loader : masterData->loadersInterf) {
requests.push_back( std::make_pair(loader.first, RestoreSysInfoRequest(sysInfo)) );
for (auto& loader : masterData->loadersInterf) {
requests.push_back(std::make_pair(loader.first, RestoreSysInfoRequest(sysInfo)));
}
TraceEvent("FastRestore").detail("DistributeRestoreSysInfoToLoaders", masterData->loadersInterf.size());
wait( sendBatchRequests(&RestoreLoaderInterface::updateRestoreSysInfo, masterData->loadersInterf, requests) );
wait(sendBatchRequests(&RestoreLoaderInterface::updateRestoreSysInfo, masterData->loadersInterf, requests));
return Void();
}
@ -142,31 +154,31 @@ ACTOR Future<Void> distributeRestoreSysInfo(Reference<RestoreWorkerData> masterW
ACTOR Future<Void> startProcessRestoreRequests(Reference<RestoreMasterData> self, Database cx) {
state UID randomUID = deterministicRandom()->randomUniqueID();
TraceEvent("FastRestore").detail("RestoreMaster", "WaitOnRestoreRequests");
state Standalone<VectorRef<RestoreRequest>> restoreRequests = wait( collectRestoreRequests(cx) );
state Standalone<VectorRef<RestoreRequest>> restoreRequests = wait(collectRestoreRequests(cx));
// lock DB for restore
wait( lockDatabase(cx,randomUID) );
wait( _clearDB(cx) );
wait(lockDatabase(cx, randomUID));
wait(_clearDB(cx));
// Step: Perform the restore requests
state int restoreIndex = 0;
try {
for ( restoreIndex = 0; restoreIndex < restoreRequests.size(); restoreIndex++ ) {
for (restoreIndex = 0; restoreIndex < restoreRequests.size(); restoreIndex++) {
RestoreRequest& request = restoreRequests[restoreIndex];
TraceEvent("FastRestore").detail("RestoreRequestInfo", request.toString());
Version ver = wait( processRestoreRequest(request, self, cx) );
Version ver = wait(processRestoreRequest(request, self, cx));
}
} catch(Error &e) {
} catch (Error& e) {
TraceEvent(SevError, "FastRestoreFailed").detail("RestoreRequest", restoreRequests[restoreIndex].toString());
}
// Step: Notify all restore requests have been handled by cleaning up the restore keys
wait( notifyRestoreCompleted(self, cx) );
wait(notifyRestoreCompleted(self, cx));
try {
wait( unlockDatabase(cx,randomUID) );
} catch(Error &e) {
TraceEvent(SevError, "UnlockDBFailed").detail("UID", randomUID.toString());
wait(unlockDatabase(cx, randomUID));
} catch (Error& e) {
TraceEvent(SevError, "UnlockDBFailed").detail("UID", randomUID.toString());
}
TraceEvent("FastRestore").detail("RestoreMasterComplete", self->id());
@ -174,31 +186,37 @@ ACTOR Future<Void> startProcessRestoreRequests(Reference<RestoreMasterData> self
return Void();
}
ACTOR static Future<Version> processRestoreRequest(RestoreRequest request, Reference<RestoreMasterData> self, Database cx) {
ACTOR static Future<Version> processRestoreRequest(RestoreRequest request, Reference<RestoreMasterData> self,
Database cx) {
state std::vector<RestoreFileFR> files;
state std::vector<RestoreFileFR> allFiles;
self->initBackupContainer(request.url);
wait( _collectBackupFiles(self->bc, &files, cx, request) ); // Get all backup files' description and save them to files
self->buildVersionBatches(files, self->versionBatches); // Divide files into version batches
wait(
_collectBackupFiles(self->bc, &files, cx, request)); // Get all backup files' description and save them to files
self->buildVersionBatches(files, self->versionBatches); // Divide files into version batches
state std::map<Version, VersionBatch>::iterator versionBatch;
for (versionBatch = self->versionBatches.begin(); versionBatch != self->versionBatches.end(); versionBatch++) {
wait( initializeVersionBatch(self) );
wait( distributeWorkloadPerVersionBatch(self, cx, request, versionBatch->second) );
wait(initializeVersionBatch(self));
wait(distributeWorkloadPerVersionBatch(self, cx, request, versionBatch->second));
}
TraceEvent("FastRestore").detail("RestoreToVersion", request.targetVersion);
TraceEvent("FastRestore").detail("RestoreToVersion", request.targetVersion);
return request.targetVersion;
}
ACTOR static Future<Void> loadFilesOnLoaders(Reference<RestoreMasterData> self, Database cx, RestoreRequest request, VersionBatch versionBatch, bool isRangeFile) {
TraceEvent("FastRestore").detail("FileTypeLoadedInVersionBatch", isRangeFile).detail("BeginVersion", versionBatch.beginVersion).detail("EndVersion", versionBatch.endVersion);
ACTOR static Future<Void> loadFilesOnLoaders(Reference<RestoreMasterData> self, Database cx, RestoreRequest request,
VersionBatch versionBatch, bool isRangeFile) {
TraceEvent("FastRestore")
.detail("FileTypeLoadedInVersionBatch", isRangeFile)
.detail("BeginVersion", versionBatch.beginVersion)
.detail("EndVersion", versionBatch.endVersion);
Key mutationLogPrefix;
std::vector<RestoreFileFR> *files;
if ( isRangeFile ) {
std::vector<RestoreFileFR>* files;
if (isRangeFile) {
files = &versionBatch.rangeFiles;
} else {
files = &versionBatch.logFiles;
@ -211,15 +229,16 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<RestoreMasterData> self,
Version prevVersion = versionBatch.beginVersion;
for (auto &file : *files) {
// NOTE: Cannot skip empty files because empty files, e.g., log file, still need to generate dummy mutation to drive applier's NotifiedVersion (e.g., logVersion and rangeVersion)
if ( loader == self->loadersInterf.end() ) {
for (auto& file : *files) {
// NOTE: Cannot skip empty files because empty files, e.g., log file, still need to generate dummy mutation to
// drive applier's NotifiedVersion (e.g., logVersion and rangeVersion)
if (loader == self->loadersInterf.end()) {
loader = self->loadersInterf.begin();
}
// Prepare loading
LoadingParam param;
param.url = request.url;
param.prevVersion = prevVersion;
param.prevVersion = prevVersion;
param.endVersion = file.isRange ? file.version : file.endVersion;
prevVersion = param.endVersion;
param.isRangeFile = file.isRange;
@ -232,43 +251,42 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<RestoreMasterData> self,
param.addPrefix = request.addPrefix;
param.removePrefix = request.removePrefix;
param.mutationLogPrefix = mutationLogPrefix;
ASSERT_WE_THINK( param.length >= 0 ); // we may load an empty file
ASSERT_WE_THINK( param.offset >= 0 );
ASSERT_WE_THINK( param.offset <= file.fileSize );
ASSERT_WE_THINK( param.prevVersion <= param.endVersion );
ASSERT_WE_THINK(param.length >= 0); // we may load an empty file
ASSERT_WE_THINK(param.offset >= 0);
ASSERT_WE_THINK(param.offset <= file.fileSize);
ASSERT_WE_THINK(param.prevVersion <= param.endVersion);
requests.push_back( std::make_pair(loader->first, RestoreLoadFileRequest(param)) );
requests.push_back(std::make_pair(loader->first, RestoreLoadFileRequest(param)));
// Log file to be loaded
TraceEvent("FastRestore").detail("LoadParam", param.toString())
.detail("LoaderID", loader->first.toString());
TraceEvent("FastRestore").detail("LoadParam", param.toString()).detail("LoaderID", loader->first.toString());
loader++;
}
// Wait on the batch of load files or log files
wait( sendBatchRequests(&RestoreLoaderInterface::loadFile, self->loadersInterf, requests) );
wait(sendBatchRequests(&RestoreLoaderInterface::loadFile, self->loadersInterf, requests));
return Void();
}
ACTOR static Future<Void> distributeWorkloadPerVersionBatch(Reference<RestoreMasterData> self, Database cx, RestoreRequest request, VersionBatch versionBatch) {
ASSERT( !versionBatch.isEmpty() );
ACTOR static Future<Void> distributeWorkloadPerVersionBatch(Reference<RestoreMasterData> self, Database cx,
RestoreRequest request, VersionBatch versionBatch) {
ASSERT(!versionBatch.isEmpty());
ASSERT(self->loadersInterf.size() > 0);
ASSERT(self->appliersInterf.size() > 0);
ASSERT( self->loadersInterf.size() > 0 );
ASSERT( self->appliersInterf.size() > 0 );
dummySampleWorkload(self);
wait( notifyLoaderAppliersKeyRange(self) );
wait(notifyLoaderAppliersKeyRange(self));
// Parse log files and send mutations to appliers before we parse range files
wait( loadFilesOnLoaders(self, cx, request, versionBatch, false) );
wait( loadFilesOnLoaders(self, cx, request, versionBatch, true) );
wait( notifyApplierToApplyMutations(self) );
wait(loadFilesOnLoaders(self, cx, request, versionBatch, false));
wait(loadFilesOnLoaders(self, cx, request, versionBatch, true));
wait(notifyApplierToApplyMutations(self));
return Void();
}
// Placehold for sample workload
// Produce the key-range for each applier
void dummySampleWorkload(Reference<RestoreMasterData> self) {
@ -279,10 +297,10 @@ void dummySampleWorkload(Reference<RestoreMasterData> self) {
for (i = 0; i < numAppliers - 1; i++) {
keyrangeSplitter.push_back(deterministicRandom()->randomUniqueID());
}
std::sort( keyrangeSplitter.begin(), keyrangeSplitter.end() );
std::sort(keyrangeSplitter.begin(), keyrangeSplitter.end());
i = 0;
for (auto& applier : self->appliersInterf) {
if ( i == 0 ) {
if (i == 0) {
self->range2Applier[normalKeys.begin] = applier.first;
} else {
self->range2Applier[StringRef(keyrangeSplitter[i].toString())] = applier.first;
@ -295,30 +313,31 @@ ACTOR static Future<Standalone<VectorRef<RestoreRequest>>> collectRestoreRequest
state Standalone<VectorRef<RestoreRequest>> restoreRequests;
state Future<Void> watch4RestoreRequest;
//wait for the restoreRequestTriggerKey to be set by the client/test workload
// wait for the restoreRequestTriggerKey to be set by the client/test workload
state ReadYourWritesTransaction tr(cx);
loop{
loop {
try {
tr.reset();
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
state Optional<Value> numRequests = wait(tr.get(restoreRequestTriggerKey));
if ( !numRequests.present() ) {
if (!numRequests.present()) {
watch4RestoreRequest = tr.watch(restoreRequestTriggerKey);
wait(tr.commit());
wait( watch4RestoreRequest );
wait(watch4RestoreRequest);
} else {
state Standalone<RangeResultRef> restoreRequestValues = wait(tr.getRange(restoreRequestKeys, CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> restoreRequestValues =
wait(tr.getRange(restoreRequestKeys, CLIENT_KNOBS->TOO_MANY));
ASSERT(!restoreRequestValues.more);
if(restoreRequestValues.size()) {
for ( auto &it : restoreRequestValues ) {
if (restoreRequestValues.size()) {
for (auto& it : restoreRequestValues) {
restoreRequests.push_back(restoreRequests.arena(), decodeRestoreRequestValue(it.value));
printf("Restore Request:%s\n", restoreRequests.back().toString().c_str());
}
}
break;
}
} catch(Error &e) {
} catch (Error& e) {
wait(tr.onError(e));
}
}
@ -327,20 +346,21 @@ ACTOR static Future<Standalone<VectorRef<RestoreRequest>>> collectRestoreRequest
}
// Collect the backup files' description into output_files by reading the backupContainer bc.
ACTOR static Future<Void> _collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* output_files, Database cx, RestoreRequest request) {
state std::vector<RestoreFileFR> &files = *output_files;
ACTOR static Future<Void> _collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* output_files,
Database cx, RestoreRequest request) {
state std::vector<RestoreFileFR>& files = *output_files;
state BackupDescription desc = wait(bc->describeBackup());
// TODO: Delete this and see if it works
wait(desc.resolveVersionTimes(cx));
printf("[INFO] Backup Description\n%s", desc.toString().c_str());
if(request.targetVersion == invalidVersion && desc.maxRestorableVersion.present())
if (request.targetVersion == invalidVersion && desc.maxRestorableVersion.present())
request.targetVersion = desc.maxRestorableVersion.get();
Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(request.targetVersion));
if(!restorable.present()) {
if (!restorable.present()) {
TraceEvent(SevWarn, "FastRestore").detail("NotRestorable", request.targetVersion);
throw restore_missing_data();
}
@ -350,45 +370,44 @@ ACTOR static Future<Void> _collectBackupFiles(Reference<IBackupContainer> bc, st
files.clear();
}
for(const RangeFile &f : restorable.get().ranges) {
TraceEvent("FastRestore").detail("RangeFile", f.toString());
for (const RangeFile& f : restorable.get().ranges) {
TraceEvent("FastRestore").detail("RangeFile", f.toString());
RestoreFileFR file(f.version, f.fileName, true, f.blockSize, f.fileSize, f.version, f.version);
files.push_back(file);
}
for(const LogFile &f : restorable.get().logs) {
TraceEvent("FastRestore").detail("LogFile", f.toString());
files.push_back(file);
}
for (const LogFile& f : restorable.get().logs) {
TraceEvent("FastRestore").detail("LogFile", f.toString());
RestoreFileFR file(f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion, f.beginVersion);
files.push_back(file);
}
}
return Void();
}
ACTOR static Future<Void> _clearDB(Database cx) {
wait( runRYWTransaction( cx, [](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->clear(normalKeys);
return Void();
}) );
wait(runRYWTransaction(cx, [](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->clear(normalKeys);
return Void();
}));
return Void();
}
ACTOR static Future<Void> initializeVersionBatch(Reference<RestoreMasterData> self) {
std::vector<std::pair<UID, RestoreVersionBatchRequest>> requests;
for (auto &applier : self->appliersInterf) {
requests.push_back( std::make_pair(applier.first, RestoreVersionBatchRequest(self->batchIndex)) );
for (auto& applier : self->appliersInterf) {
requests.push_back(std::make_pair(applier.first, RestoreVersionBatchRequest(self->batchIndex)));
}
wait( sendBatchRequests(&RestoreApplierInterface::initVersionBatch, self->appliersInterf, requests) );
wait(sendBatchRequests(&RestoreApplierInterface::initVersionBatch, self->appliersInterf, requests));
std::vector<std::pair<UID, RestoreVersionBatchRequest>> requests;
for (auto &loader : self->loadersInterf) {
requests.push_back( std::make_pair(loader.first, RestoreVersionBatchRequest(self->batchIndex)) );
for (auto& loader : self->loadersInterf) {
requests.push_back(std::make_pair(loader.first, RestoreVersionBatchRequest(self->batchIndex)));
}
wait( sendBatchRequests(&RestoreLoaderInterface::initVersionBatch, self->loadersInterf, requests) );
wait(sendBatchRequests(&RestoreLoaderInterface::initVersionBatch, self->loadersInterf, requests));
return Void();
}
@ -398,21 +417,21 @@ ACTOR static Future<Void> notifyApplierToApplyMutations(Reference<RestoreMasterD
// Prepare the applyToDB requests
std::vector<std::pair<UID, RestoreVersionBatchRequest>> requests;
for (auto& applier : self->appliersInterf) {
requests.push_back( std::make_pair(applier.first, RestoreVersionBatchRequest(self->batchIndex)) );
requests.push_back(std::make_pair(applier.first, RestoreVersionBatchRequest(self->batchIndex)));
}
wait( sendBatchRequests(&RestoreApplierInterface::applyToDB, self->appliersInterf, requests) );
wait(sendBatchRequests(&RestoreApplierInterface::applyToDB, self->appliersInterf, requests));
TraceEvent("FastRestore").detail("Master", self->id()).detail("ApplyToDB", "Completed");
return Void();
}
// Send the map of key-range to applier to each loader
ACTOR static Future<Void> notifyLoaderAppliersKeyRange(Reference<RestoreMasterData> self) {
ACTOR static Future<Void> notifyLoaderAppliersKeyRange(Reference<RestoreMasterData> self) {
std::vector<std::pair<UID, RestoreSetApplierKeyRangeVectorRequest>> requests;
for (auto& loader : self->loadersInterf) {
requests.push_back(std::make_pair(loader.first, RestoreSetApplierKeyRangeVectorRequest(self->range2Applier)) );
requests.push_back(std::make_pair(loader.first, RestoreSetApplierKeyRangeVectorRequest(self->range2Applier)));
}
wait( sendBatchRequests(&RestoreLoaderInterface::setApplierKeyRangeVectorRequest, self->loadersInterf, requests) );
wait(sendBatchRequests(&RestoreLoaderInterface::setApplierKeyRangeVectorRequest, self->loadersInterf, requests));
return Void();
}
@ -421,19 +440,20 @@ ACTOR static Future<Void> notifyLoaderAppliersKeyRange(Reference<RestoreMasterDa
// Register the restoreRequestDoneKey to signal the end of restore
ACTOR static Future<Void> notifyRestoreCompleted(Reference<RestoreMasterData> self, Database cx) {
std::vector<std::pair<UID, RestoreVersionBatchRequest>> requests;
for ( auto &loader : self->loadersInterf ) {
requests.push_back( std::make_pair(loader.first, RestoreVersionBatchRequest(self->batchIndex)) );
for (auto& loader : self->loadersInterf) {
requests.push_back(std::make_pair(loader.first, RestoreVersionBatchRequest(self->batchIndex)));
}
// A loader exits immediately after it receives the request. Master may not receive acks.
Future<Void> endLoaders = sendBatchRequests(&RestoreLoaderInterface::finishRestore, self->loadersInterf, requests);
requests.clear();
for ( auto &applier : self->appliersInterf ) {
requests.push_back( std::make_pair(applier.first, RestoreVersionBatchRequest(self->batchIndex)) );
for (auto& applier : self->appliersInterf) {
requests.push_back(std::make_pair(applier.first, RestoreVersionBatchRequest(self->batchIndex)));
}
Future<Void> endApplier = sendBatchRequests(&RestoreApplierInterface::finishRestore, self->appliersInterf, requests);
Future<Void> endApplier =
sendBatchRequests(&RestoreApplierInterface::finishRestore, self->appliersInterf, requests);
wait( delay(5.0) ); // Give some time for loaders and appliers to exit
wait(delay(5.0)); // Give some time for loaders and appliers to exit
// Notify tester that the restore has finished
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
@ -445,9 +465,9 @@ ACTOR static Future<Void> notifyRestoreCompleted(Reference<RestoreMasterData> se
tr->clear(restoreRequestKeys);
Version readVersion = wait(tr->getReadVersion());
tr->set(restoreRequestDoneKey, restoreRequestDoneVersionValue(readVersion));
wait( tr->commit() );
wait(tr->commit());
break;
} catch( Error &e ) {
} catch (Error& e) {
wait(tr->onError(e));
}
}

60
fdbserver/RestoreMaster.actor.h
View File

@ -22,10 +22,10 @@
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORE_MASTER_G_H)
#define FDBSERVER_RESTORE_MASTER_G_H
#include "fdbserver/RestoreMaster.actor.g.h"
#define FDBSERVER_RESTORE_MASTER_G_H
#include "fdbserver/RestoreMaster.actor.g.h"
#elif !defined(FDBSERVER_RESTORE_MASTER_H)
#define FDBSERVER_RESTORE_MASTER_H
#define FDBSERVER_RESTORE_MASTER_H
#include <sstream>
#include "flow/Stats.h"
@ -48,14 +48,13 @@ struct VersionBatch {
std::vector<RestoreFileFR> logFiles;
std::vector<RestoreFileFR> rangeFiles;
bool isEmpty() {
return logFiles.empty() && rangeFiles.empty();
}
bool isEmpty() { return logFiles.empty() && rangeFiles.empty(); }
};
struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMasterData> {
// range2Applier is in master and loader node. Loader node uses this to determine which applier a mutation should be sent
std::map<Standalone<KeyRef>, UID> range2Applier; // KeyRef is the inclusive lower bound of the key range the applier (UID) is responsible for
struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMasterData> {
// range2Applier is in master and loader node. Loader uses this to determine which applier a mutation should be sent.
// KeyRef is the inclusive lower bound of the key range the applier (UID) is responsible for
std::map<Standalone<KeyRef>, UID> range2Applier;
std::map<Version, VersionBatch> versionBatches; // key is the beginVersion of the version batch
int batchIndex;
@ -74,56 +73,57 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMast
std::string describeNode() {
std::stringstream ss;
ss << "Master versionBatch:" << batchIndex;
ss << "Master versionBatch:" << batchIndex;
return ss.str();
}
// Split allFiles into multiple versionBatches based on files' version
void buildVersionBatches(const std::vector<RestoreFileFR>& allFiles, std::map<Version, VersionBatch>& versionBatches) {
// A version batch includes a log file
// Because log file's verion range does not overlap, we use log file's version range as the version range of a version batch
// Create a version batch for a log file
void buildVersionBatches(const std::vector<RestoreFileFR>& allFiles,
std::map<Version, VersionBatch>& versionBatches) {
// A version batch includes a log file; Because log file's verion range does not overlap,
// we use log file's version range as the version range of a version batch.
Version beginVersion = 0;
Version maxVersion = 0;
for ( int i = 0; i < allFiles.size(); ++i ) {
if ( !allFiles[i].isRange ) {
ASSERT( versionBatches.find(allFiles[i].beginVersion) == versionBatches.end() );
for (int i = 0; i < allFiles.size(); ++i) {
if (!allFiles[i].isRange) {
ASSERT(versionBatches.find(allFiles[i].beginVersion) == versionBatches.end());
VersionBatch vb;
vb.beginVersion = beginVersion;
vb.endVersion = allFiles[i].endVersion;
versionBatches[vb.beginVersion] = vb; // We ensure the version range are continuous across version batches
versionBatches[vb.beginVersion] = vb; // Ensure continuous version range across version batches
beginVersion = allFiles[i].endVersion;
}
if ( maxVersion < allFiles[i].endVersion ) {
if (maxVersion < allFiles[i].endVersion) {
maxVersion = allFiles[i].endVersion;
}
}
// In case there is no log file
if ( versionBatches.empty() ) {
if (versionBatches.empty()) {
VersionBatch vb;
vb.beginVersion = 0;
vb.endVersion = maxVersion + 1; // version batch's endVersion is exclusive
versionBatches[vb.beginVersion] = vb; // We ensure the version range are continuous across version batches
}
// Put range and log files into its version batch
for ( int i = 0; i < allFiles.size(); ++i ) {
std::map<Version, VersionBatch>::iterator vbIter = versionBatches.upper_bound(allFiles[i].beginVersion); // vbiter's beginVersion > allFiles[i].beginVersion
for (int i = 0; i < allFiles.size(); ++i) {
// vbiter's beginVersion > allFiles[i].beginVersion.
std::map<Version, VersionBatch>::iterator vbIter = versionBatches.upper_bound(allFiles[i].beginVersion);
--vbIter;
ASSERT_WE_THINK( vbIter != versionBatches.end() );
if ( allFiles[i].isRange ) {
vbIter->second.rangeFiles.push_back(allFiles[i]);
ASSERT_WE_THINK(vbIter != versionBatches.end());
if (allFiles[i].isRange) {
vbIter->second.rangeFiles.push_back(allFiles[i]);
} else {
vbIter->second.logFiles.push_back(allFiles[i]);
}
}
printf("versionBatches.size:%d\n", versionBatches.size());
// Sanity check
for (auto &versionBatch : versionBatches) {
for ( auto &logFile : versionBatch.second.logFiles ) {
for (auto& versionBatch : versionBatches) {
for (auto& logFile : versionBatch.second.logFiles) {
ASSERT(logFile.beginVersion >= versionBatch.second.beginVersion);
ASSERT(logFile.endVersion <= versionBatch.second.endVersion);
}
for ( auto &rangeFile : versionBatch.second.rangeFiles ) {
for (auto& rangeFile : versionBatch.second.rangeFiles) {
ASSERT(rangeFile.beginVersion == rangeFile.endVersion);
ASSERT(rangeFile.beginVersion >= versionBatch.second.beginVersion);
ASSERT(rangeFile.endVersion < versionBatch.second.endVersion);
@ -133,13 +133,13 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMast
void logApplierKeyRange() {
TraceEvent("FastRestore").detail("ApplierKeyRangeNum", range2Applier.size());
for (auto &applier : range2Applier) {
for (auto& applier : range2Applier) {
TraceEvent("FastRestore").detail("KeyRangeLowerBound", applier.first).detail("Applier", applier.second);
}
}
void initBackupContainer(Key url) {
if ( bcUrl == url && bc.isValid() ) {
if (bcUrl == url && bc.isValid()) {
return;
}
printf("initBackupContainer, url:%s\n", url.toString().c_str());

53
fdbserver/RestoreRoleCommon.actor.cpp
View File

@ -29,60 +29,63 @@
#include "fdbserver/RestoreApplier.actor.h"
#include "fdbserver/RestoreMaster.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
class Database;
struct RestoreWorkerData;
// id is the id of the worker to be monitored
// id is the id of the worker to be monitored
// This actor is used for both restore loader and restore applier
ACTOR Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id) {
wait( delayJittered(5.0) ); // Random jitter reduces heat beat monitor's pressure
wait(delayJittered(5.0)); // Random jitter reduces heat beat monitor's pressure
req.reply.send(RestoreCommonReply(id));
return Void();
}
ACTOR Future<Void> handleFinishRestoreRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self) {
if ( self->versionBatchStart ) {
if (self->versionBatchStart) {
self->versionBatchStart = false;
}
TraceEvent("FastRestore").detail("FinishRestoreRequest", req.batchID)
.detail("Role", getRoleStr(self->role)).detail("Node", self->id());
req.reply.send( RestoreCommonReply(self->id()) );
return Void();
}
ACTOR Future<Void> handleInitVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self) {
if ( !self->versionBatchStart ) {
self->versionBatchStart = true;
self->resetPerVersionBatch();
}
TraceEvent("FastRestore").detail("InitVersionBatch", req.batchID)
.detail("Role", getRoleStr(self->role)).detail("Node", self->id());
TraceEvent("FastRestore")
.detail("FinishRestoreRequest", req.batchID)
.detail("Role", getRoleStr(self->role))
.detail("Node", self->id());
req.reply.send(RestoreCommonReply(self->id()));
return Void();
}
ACTOR Future<Void> handleInitVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self) {
if (!self->versionBatchStart) {
self->versionBatchStart = true;
self->resetPerVersionBatch();
}
TraceEvent("FastRestore")
.detail("InitVersionBatch", req.batchID)
.detail("Role", getRoleStr(self->role))
.detail("Node", self->id());
req.reply.send(RestoreCommonReply(self->id()));
return Void();
}
//-------Helper functions
std::string getHexString(StringRef input) {
std::stringstream ss;
for (int i = 0; i<input.size(); i++) {
if ( i % 4 == 0 )
ss << " ";
if ( i == 12 ) { //The end of 12bytes, which is the version size for value
for (int i = 0; i < input.size(); i++) {
if (i % 4 == 0) ss << " ";
if (i == 12) { // The end of 12bytes, which is the version size for value
ss << "|";
}
if ( i == (12 + 12) ) { //The end of version + header
if (i == (12 + 12)) { // The end of version + header
ss << "@";
}
ss << std::setfill('0') << std::setw(2) << std::hex << (int) input[i]; // [] operator moves the pointer in step of unit8
ss << std::setfill('0') << std::setw(2) << std::hex
<< (int)input[i]; // [] operator moves the pointer in step of unit8
}
return ss.str();
}

57
fdbserver/RestoreRoleCommon.actor.h
View File

@ -23,10 +23,10 @@
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RestoreRoleCommon_G_H)
#define FDBSERVER_RestoreRoleCommon_G_H
#include "fdbserver/RestoreRoleCommon.actor.g.h"
#define FDBSERVER_RestoreRoleCommon_G_H
#include "fdbserver/RestoreRoleCommon.actor.g.h"
#elif !defined(FDBSERVER_RestoreRoleCommon_H)
#define FDBSERVER_RestoreRoleCommon_H
#define FDBSERVER_RestoreRoleCommon_H
#include <sstream>
#include "flow/Stats.h"
@ -57,45 +57,45 @@ ACTOR Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id);
ACTOR Future<Void> handleInitVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self);
ACTOR Future<Void> handleFinishRestoreRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self);
// Helper class for reading restore data from a buffer and throwing the right errors.
// This struct is mostly copied from StringRefReader. We add a sanity check in this struct.
// TODO: Merge this struct with StringRefReader.
struct StringRefReaderMX {
StringRefReaderMX(StringRef s = StringRef(), Error e = Error()) : rptr(s.begin()), end(s.end()), failure_error(e), str_size(s.size()) {}
StringRefReaderMX(StringRef s = StringRef(), Error e = Error())
: rptr(s.begin()), end(s.end()), failure_error(e), str_size(s.size()) {}
// Return remainder of data as a StringRef
StringRef remainder() {
return StringRef(rptr, end - rptr);
}
StringRef remainder() { return StringRef(rptr, end - rptr); }
// Return a pointer to len bytes at the current read position and advance read pos
//Consume a little-Endian data. Since we only run on little-Endian machine, the data on storage is little Endian
const uint8_t * consume(unsigned int len) {
if(rptr == end && len != 0)
throw end_of_stream();
const uint8_t *p = rptr;
// Consume a little-Endian data. Since we only run on little-Endian machine, the data on storage is little Endian
const uint8_t* consume(unsigned int len) {
if (rptr == end && len != 0) throw end_of_stream();
const uint8_t* p = rptr;
rptr += len;
if(rptr > end) {
if (rptr > end) {
printf("[ERROR] StringRefReaderMX throw error! string length:%d\n", str_size);
printf("!!!!!!!!!!!![ERROR]!!!!!!!!!!!!!! Worker may die due to the error. Master will stuck when a worker die\n");
printf("!!!!!!!!!!!![ERROR]!!!!!!!!!!!!!! Worker may die due to the error. Master will stuck when a worker "
"die\n");
throw failure_error;
}
return p;
}
// Return a T from the current read position and advance read pos
template<typename T> const T consume() {
return *(const T *)consume(sizeof(T));
template <typename T>
const T consume() {
return *(const T*)consume(sizeof(T));
}
// Functions for consuming big endian (network byte oselfer) integers.
// Consumes a big endian number, swaps it to little endian, and returns it.
const int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume< int32_t>());}
const uint32_t consumeNetworkUInt32() { return bigEndian32( consume<uint32_t>());}
const int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
const uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
const int64_t consumeNetworkInt64() { return (int64_t)bigEndian64((uint32_t)consume< int64_t>());}
const uint64_t consumeNetworkUInt64() { return bigEndian64( consume<uint64_t>());}
// Convert big Endian value (e.g., encoded in log file) into a littleEndian uint64_t value.
const int64_t consumeNetworkInt64() { return (int64_t)bigEndian64((uint32_t)consume<int64_t>()); }
const uint64_t consumeNetworkUInt64() { return bigEndian64(consume<uint64_t>()); }
bool eof() { return rptr == end; }
@ -104,11 +104,10 @@ struct StringRefReaderMX {
Error failure_error;
};
struct RestoreRoleData : NonCopyable, public ReferenceCounted<RestoreRoleData> {
public:
struct RestoreRoleData : NonCopyable, public ReferenceCounted<RestoreRoleData> {
public:
RestoreRole role;
UID nodeID; //
UID nodeID;
int nodeIndex;
std::map<UID, RestoreLoaderInterface> loadersInterf;
@ -119,15 +118,13 @@ public:
uint32_t inProgressFlag = 0;
RestoreRoleData() : role(RestoreRole::Invalid) {};
RestoreRoleData() : role(RestoreRole::Invalid){};
~RestoreRoleData() {};
~RestoreRoleData(){};
UID id() const { return nodeID; }
void resetPerVersionBatch() {
inProgressFlag = 0;
}
void resetPerVersionBatch() { inProgressFlag = 0; }
void clearInterfaces() {
loadersInterf.clear();

8
fdbserver/RestoreUtil.actor.cpp
View File

@ -22,14 +22,14 @@
#include "fdbserver/RestoreUtil.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
const std::vector<std::string> RestoreRoleStr = {"Invalid", "Master", "Loader", "Applier"};
const std::vector<std::string> RestoreRoleStr = { "Invalid", "Master", "Loader", "Applier" };
int numRoles = RestoreRoleStr.size();
std::string getRoleStr(RestoreRole role) {
if ( (int) role >= numRoles || (int) role < 0) {
printf("[ERROR] role:%d is out of scope\n", (int) role);
if ((int)role >= numRoles || (int)role < 0) {
printf("[ERROR] role:%d is out of scope\n", (int)role);
return "[Unset]";
}
return RestoreRoleStr[(int)role];

18
fdbserver/RestoreUtil.h
View File

@ -34,8 +34,8 @@
#include <cstdint>
#include <cstdarg>
enum class RestoreRole {Invalid = 0, Master = 1, Loader, Applier};
BINARY_SERIALIZABLE( RestoreRole );
enum class RestoreRole { Invalid = 0, Master = 1, Loader, Applier };
BINARY_SERIALIZABLE(RestoreRole);
std::string getRoleStr(RestoreRole role);
extern const std::vector<std::string> RestoreRoleStr;
extern int numRoles;
@ -45,18 +45,18 @@ extern int numRoles;
struct FastRestoreOpConfig {
int num_loaders = 120;
int num_appliers = 40;
// transactionBatchSizeThreshold is used when applier applies multiple mutations in a transaction to DB
double transactionBatchSizeThreshold = 512; //512 in Bytes
// transactionBatchSizeThreshold is used when applier applies multiple mutations in a transaction to DB
double transactionBatchSizeThreshold = 512; // 512 in Bytes
};
extern FastRestoreOpConfig opConfig;
struct RestoreCommonReply {
constexpr static FileIdentifier file_identifier = 56140435;
UID id; // unique ID of the server who sends the reply
RestoreCommonReply() = default;
explicit RestoreCommonReply(UID id) : id(id) {}
std::string toString() const {
std::stringstream ss;
ss << "ServerNodeID:" << id.toString();
@ -76,8 +76,8 @@ struct RestoreSimpleRequest : TimedRequest {
RestoreSimpleRequest() = default;
template <class Ar>
void serialize( Ar& ar ) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, reply);
}
@ -88,4 +88,4 @@ struct RestoreSimpleRequest : TimedRequest {
}
};
#endif //FDBSERVER_RESTOREUTIL_ACTOR_H
#endif // FDBSERVER_RESTOREUTIL_ACTOR_H

167
fdbserver/RestoreWorker.actor.cpp
View File

@ -43,8 +43,7 @@
// #include "fdbserver/RestoreApplier.actor.h"
#include "fdbserver/RestoreMaster.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
FastRestoreOpConfig opConfig;
@ -57,37 +56,45 @@ struct RestoreWorkerData; // Only declare the struct exist but we cannot use its
void initRestoreWorkerConfig();
ACTOR Future<Void> handlerTerminateWorkerRequest(RestoreSimpleRequest req, Reference<RestoreWorkerData> self, RestoreWorkerInterface workerInterf, Database cx);
ACTOR Future<Void> handlerTerminateWorkerRequest(RestoreSimpleRequest req, Reference<RestoreWorkerData> self,
RestoreWorkerInterface workerInterf, Database cx);
ACTOR Future<Void> monitorWorkerLiveness(Reference<RestoreWorkerData> self);
ACTOR Future<Void> handleRecruitRoleRequest(RestoreRecruitRoleRequest req, Reference<RestoreWorkerData> self, ActorCollection *actors, Database cx);
ACTOR Future<Void> collectRestoreWorkerInterface(Reference<RestoreWorkerData> self, Database cx, int min_num_workers = 2);
ACTOR Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> leader, Database cx, RestoreWorkerInterface myWorkerInterf);
ACTOR Future<Void> startRestoreWorkerLeader(Reference<RestoreWorkerData> self, RestoreWorkerInterface workerInterf, Database cx);
template<> Tuple Codec<ERestoreState>::pack(ERestoreState const &val);
template<> ERestoreState Codec<ERestoreState>::unpack(Tuple const &val);
ACTOR Future<Void> handleRecruitRoleRequest(RestoreRecruitRoleRequest req, Reference<RestoreWorkerData> self,
ActorCollection* actors, Database cx);
ACTOR Future<Void> collectRestoreWorkerInterface(Reference<RestoreWorkerData> self, Database cx,
int min_num_workers = 2);
ACTOR Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> leader, Database cx,
RestoreWorkerInterface myWorkerInterf);
ACTOR Future<Void> startRestoreWorkerLeader(Reference<RestoreWorkerData> self, RestoreWorkerInterface workerInterf,
Database cx);
template <>
Tuple Codec<ERestoreState>::pack(ERestoreState const& val);
template <>
ERestoreState Codec<ERestoreState>::unpack(Tuple const& val);
// Remove the worker interface from restoreWorkerKey and remove its roles interfaces from their keys.
ACTOR Future<Void> handlerTerminateWorkerRequest(RestoreSimpleRequest req, Reference<RestoreWorkerData> self, RestoreWorkerInterface workerInterf, Database cx) {
wait( runRYWTransaction( cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
ACTOR Future<Void> handlerTerminateWorkerRequest(RestoreSimpleRequest req, Reference<RestoreWorkerData> self,
RestoreWorkerInterface workerInterf, Database cx) {
wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->clear(restoreWorkerKeyFor(workerInterf.id()));
return Void();
}) );
}));
TraceEvent("FastRestore").detail("HandleTerminateWorkerReq", self->id());
return Void();
}
}
// Assume only 1 role on a restore worker.
// Future: Multiple roles in a restore worker
ACTOR Future<Void> handleRecruitRoleRequest(RestoreRecruitRoleRequest req, Reference<RestoreWorkerData> self, ActorCollection *actors, Database cx) {
ACTOR Future<Void> handleRecruitRoleRequest(RestoreRecruitRoleRequest req, Reference<RestoreWorkerData> self,
ActorCollection* actors, Database cx) {
// Already recruited a role
if (self->loaderInterf.present()) {
ASSERT( req.role == RestoreRole::Loader );
ASSERT(req.role == RestoreRole::Loader);
req.reply.send(RestoreRecruitRoleReply(self->id(), RestoreRole::Loader, self->loaderInterf.get()));
return Void();
} else if (self->applierInterf.present()) {
@ -96,44 +103,44 @@ ACTOR Future<Void> handleRecruitRoleRequest(RestoreRecruitRoleRequest req, Refer
}
if (req.role == RestoreRole::Loader) {
ASSERT( !self->loaderInterf.present() );
ASSERT(!self->loaderInterf.present());
self->loaderInterf = RestoreLoaderInterface();
self->loaderInterf.get().initEndpoints();
RestoreLoaderInterface &recruited = self->loaderInterf.get();
RestoreLoaderInterface& recruited = self->loaderInterf.get();
DUMPTOKEN(recruited.setApplierKeyRangeVectorRequest);
DUMPTOKEN(recruited.initVersionBatch);
DUMPTOKEN(recruited.collectRestoreRoleInterfaces);
DUMPTOKEN(recruited.finishRestore);
actors->add( restoreLoaderCore(self->loaderInterf.get(), req.nodeIndex, cx) );
actors->add(restoreLoaderCore(self->loaderInterf.get(), req.nodeIndex, cx));
TraceEvent("FastRestore").detail("RecruitedLoaderNodeIndex", req.nodeIndex);
req.reply.send(RestoreRecruitRoleReply(self->id(), RestoreRole::Loader, self->loaderInterf.get()));
req.reply.send(RestoreRecruitRoleReply(self->id(), RestoreRole::Loader, self->loaderInterf.get()));
} else if (req.role == RestoreRole::Applier) {
ASSERT( !self->applierInterf.present() );
ASSERT(!self->applierInterf.present());
self->applierInterf = RestoreApplierInterface();
self->applierInterf.get().initEndpoints();
RestoreApplierInterface &recruited = self->applierInterf.get();
RestoreApplierInterface& recruited = self->applierInterf.get();
DUMPTOKEN(recruited.sendMutationVector);
DUMPTOKEN(recruited.applyToDB);
DUMPTOKEN(recruited.initVersionBatch);
DUMPTOKEN(recruited.collectRestoreRoleInterfaces);
DUMPTOKEN(recruited.finishRestore);
actors->add( restoreApplierCore(self->applierInterf.get(), req.nodeIndex, cx) );
actors->add(restoreApplierCore(self->applierInterf.get(), req.nodeIndex, cx));
TraceEvent("FastRestore").detail("RecruitedApplierNodeIndex", req.nodeIndex);
req.reply.send(RestoreRecruitRoleReply(self->id(), RestoreRole::Applier, self->applierInterf.get()));
req.reply.send(RestoreRecruitRoleReply(self->id(), RestoreRole::Applier, self->applierInterf.get()));
} else {
TraceEvent(SevError, "FastRestore").detail("HandleRecruitRoleRequest", "UnknownRole"); //.detail("Request", req.printable());
TraceEvent(SevError, "FastRestore")
.detail("HandleRecruitRoleRequest", "UnknownRole"); //.detail("Request", req.printable());
}
return Void();
}
// Read restoreWorkersKeys from DB to get each restore worker's restore workerInterface and set it to self->workerInterfaces
// This is done before we assign restore roles for restore workers
ACTOR Future<Void> collectRestoreWorkerInterface(Reference<RestoreWorkerData> self, Database cx, int min_num_workers) {
// Read restoreWorkersKeys from DB to get each restore worker's workerInterface and set it to self->workerInterfaces;
// This is done before we assign restore roles for restore workers.
ACTOR Future<Void> collectRestoreWorkerInterface(Reference<RestoreWorkerData> self, Database cx, int min_num_workers) {
state Transaction tr(cx);
state vector<RestoreWorkerInterface> agents; // agents is cmdsInterf
loop {
try {
self->workerInterfaces.clear();
@ -143,18 +150,19 @@ ACTOR Future<Void> handleRecruitRoleRequest(RestoreRecruitRoleRequest req, Refer
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
Standalone<RangeResultRef> agentValues = wait(tr.getRange(restoreWorkersKeys, CLIENT_KNOBS->TOO_MANY));
ASSERT(!agentValues.more);
// If agentValues.size() < min_num_workers, we should wait for coming workers to register their workerInterface before we read them once for all
if(agentValues.size() >= min_num_workers) {
for(auto& it : agentValues) {
// If agentValues.size() < min_num_workers, we should wait for coming workers to register their
// workerInterface before we read them once for all
if (agentValues.size() >= min_num_workers) {
for (auto& it : agentValues) {
agents.push_back(BinaryReader::fromStringRef<RestoreWorkerInterface>(it.value, IncludeVersion()));
// Save the RestoreWorkerInterface for the later operations
self->workerInterfaces.insert(std::make_pair(agents.back().id(), agents.back()));
}
break;
}
wait( delay(5.0) );
} catch( Error &e ) {
wait( tr.onError(e) );
wait(delay(5.0));
} catch (Error& e) {
wait(tr.onError(e));
}
}
ASSERT(agents.size() >= min_num_workers); // ASSUMPTION: We must have at least 1 loader and 1 applier
@ -162,12 +170,11 @@ ACTOR Future<Void> handleRecruitRoleRequest(RestoreRecruitRoleRequest req, Refer
TraceEvent("FastRestore").detail("CollectWorkerInterfaceNumWorkers", self->workerInterfaces.size());
return Void();
}
}
// Periodically send worker heartbeat to
ACTOR Future<Void> monitorWorkerLiveness(Reference<RestoreWorkerData> self) {
ASSERT( !self->workerInterfaces.empty() );
// Periodically send worker heartbeat to
ACTOR Future<Void> monitorWorkerLiveness(Reference<RestoreWorkerData> self) {
ASSERT(!self->workerInterfaces.empty());
state std::map<UID, RestoreWorkerInterface>::iterator workerInterf;
loop {
@ -175,34 +182,38 @@ ACTOR Future<Void> handleRecruitRoleRequest(RestoreRecruitRoleRequest req, Refer
for (auto& worker : self->workerInterfaces) {
requests.push_back(std::make_pair(worker.first, RestoreSimpleRequest()));
}
wait( sendBatchRequests(&RestoreWorkerInterface::heartbeat, self->workerInterfaces, requests) );
wait( delay(60.0) );
wait(sendBatchRequests(&RestoreWorkerInterface::heartbeat, self->workerInterfaces, requests));
wait(delay(60.0));
}
}
}
void initRestoreWorkerConfig() {
opConfig.num_loaders = g_network->isSimulated() ? 3 : opConfig.num_loaders;
opConfig.num_appliers = g_network->isSimulated() ? 3 : opConfig.num_appliers;
opConfig.transactionBatchSizeThreshold = g_network->isSimulated() ? 512 : opConfig.transactionBatchSizeThreshold; // Byte
TraceEvent("FastRestore").detail("InitOpConfig", "Result")
.detail("NumLoaders", opConfig.num_loaders).detail("NumAppliers", opConfig.num_appliers)
.detail("TxnBatchSize", opConfig.transactionBatchSizeThreshold);
opConfig.transactionBatchSizeThreshold =
g_network->isSimulated() ? 512 : opConfig.transactionBatchSizeThreshold; // Byte
TraceEvent("FastRestore")
.detail("InitOpConfig", "Result")
.detail("NumLoaders", opConfig.num_loaders)
.detail("NumAppliers", opConfig.num_appliers)
.detail("TxnBatchSize", opConfig.transactionBatchSizeThreshold);
}
// RestoreWorkerLeader is the worker that runs RestoreMaster role
ACTOR Future<Void> startRestoreWorkerLeader(Reference<RestoreWorkerData> self, RestoreWorkerInterface workerInterf, Database cx) {
ACTOR Future<Void> startRestoreWorkerLeader(Reference<RestoreWorkerData> self, RestoreWorkerInterface workerInterf,
Database cx) {
// We must wait for enough time to make sure all restore workers have registered their workerInterfaces into the DB
printf("[INFO][Master] NodeID:%s Restore master waits for agents to register their workerKeys\n",
workerInterf.id().toString().c_str());
wait( delay(10.0) );
workerInterf.id().toString().c_str());
wait(delay(10.0));
printf("[INFO][Master] NodeID:%s starts collect restore worker interfaces\n", workerInterf.id().toString().c_str());
wait( collectRestoreWorkerInterface(self, cx, opConfig.num_loaders + opConfig.num_appliers) );
wait(collectRestoreWorkerInterface(self, cx, opConfig.num_loaders + opConfig.num_appliers));
// TODO: Needs to keep this monitor's future. May use actorCollection
state Future<Void> workersFailureMonitor = monitorWorkerLiveness(self);
wait( startRestoreMaster(self, cx) );
wait(startRestoreMaster(self, cx));
return Void();
}
@ -211,39 +222,43 @@ ACTOR Future<Void> startRestoreWorker(Reference<RestoreWorkerData> self, Restore
state double lastLoopTopTime;
state ActorCollection actors(false); // Collect the main actor for each role
state Future<Void> exitRole = Never();
loop {
double loopTopTime = now();
double elapsedTime = loopTopTime - lastLoopTopTime;
if( elapsedTime > 0.050 ) {
if (elapsedTime > 0.050) {
if (deterministicRandom()->random01() < 0.01)
TraceEvent(SevWarn, "SlowRestoreWorkerLoopx100").detail("NodeDesc", self->describeNode()).detail("Elapsed", elapsedTime);
TraceEvent(SevWarn, "SlowRestoreWorkerLoopx100")
.detail("NodeDesc", self->describeNode())
.detail("Elapsed", elapsedTime);
}
lastLoopTopTime = loopTopTime;
state std::string requestTypeStr = "[Init]";
try {
choose {
when ( RestoreSimpleRequest req = waitNext(interf.heartbeat.getFuture()) ) {
when(RestoreSimpleRequest req = waitNext(interf.heartbeat.getFuture())) {
requestTypeStr = "heartbeat";
actors.add( handleHeartbeat(req, interf.id()) );
actors.add(handleHeartbeat(req, interf.id()));
}
when ( RestoreRecruitRoleRequest req = waitNext(interf.recruitRole.getFuture()) ) {
when(RestoreRecruitRoleRequest req = waitNext(interf.recruitRole.getFuture())) {
requestTypeStr = "recruitRole";
actors.add( handleRecruitRoleRequest(req, self, &actors, cx) );
actors.add(handleRecruitRoleRequest(req, self, &actors, cx));
}
when ( RestoreSimpleRequest req = waitNext(interf.terminateWorker.getFuture()) ) {
when(RestoreSimpleRequest req = waitNext(interf.terminateWorker.getFuture())) {
// Destroy the worker at the end of the restore
requestTypeStr = "terminateWorker";
exitRole = handlerTerminateWorkerRequest(req, self, interf, cx);
}
when ( wait(exitRole) ) {
when(wait(exitRole)) {
TraceEvent("FastRestore").detail("RestoreWorkerCore", "ExitRole").detail("NodeID", self->id());
break;
}
}
} catch (Error &e) {
TraceEvent(SevWarn, "FastRestore").detail("RestoreWorkerError", e.what()).detail("RequestType", requestTypeStr);
} catch (Error& e) {
TraceEvent(SevWarn, "FastRestore")
.detail("RestoreWorkerError", e.what())
.detail("RequestType", requestTypeStr);
break;
// if ( requestTypeStr.find("[Init]") != std::string::npos ) {
// TraceEvent(SevError, "FastRestore").detail("RestoreWorkerUnexpectedExit", "RequestType_Init");
@ -258,8 +273,8 @@ ACTOR Future<Void> startRestoreWorker(Reference<RestoreWorkerData> self, Restore
ACTOR Future<Void> _restoreWorker(Database cx, LocalityData locality) {
state ActorCollection actors(false);
state Future<Void> myWork = Never();
state Reference<AsyncVar<RestoreWorkerInterface>> leader = Reference<AsyncVar<RestoreWorkerInterface>>(
new AsyncVar<RestoreWorkerInterface>() );
state Reference<AsyncVar<RestoreWorkerInterface>> leader =
Reference<AsyncVar<RestoreWorkerInterface>>(new AsyncVar<RestoreWorkerInterface>());
state RestoreWorkerInterface myWorkerInterf;
myWorkerInterf.initEndpoints();
@ -267,7 +282,7 @@ ACTOR Future<Void> _restoreWorker(Database cx, LocalityData locality) {
self->workerID = myWorkerInterf.id();
initRestoreWorkerConfig();
wait( monitorleader(leader, cx, myWorkerInterf) );
wait(monitorleader(leader, cx, myWorkerInterf));
printf("Wait for leader\n");
wait(delay(1));
@ -283,12 +298,11 @@ ACTOR Future<Void> _restoreWorker(Database cx, LocalityData locality) {
return Void();
}
// RestoreMaster is the leader
ACTOR Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> leader, Database cx, RestoreWorkerInterface myWorkerInterf) {
state ReadYourWritesTransaction tr(cx);
//state Future<Void> leaderWatch;
ACTOR Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> leader, Database cx,
RestoreWorkerInterface myWorkerInterf) {
state ReadYourWritesTransaction tr(cx);
// state Future<Void> leaderWatch;
state RestoreWorkerInterface leaderInterf;
loop {
try {
@ -296,7 +310,7 @@ ACTOR Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> lea
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> leaderValue = wait(tr.get(restoreLeaderKey));
if(leaderValue.present()) {
if (leaderValue.present()) {
leaderInterf = BinaryReader::fromStringRef<RestoreWorkerInterface>(leaderValue.get(), IncludeVersion());
// Register my interface as an worker if I am not the leader
if (leaderInterf != myWorkerInterf) {
@ -307,11 +321,11 @@ ACTOR Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> lea
tr.set(restoreLeaderKey, BinaryWriter::toValue(myWorkerInterf, IncludeVersion()));
leaderInterf = myWorkerInterf;
}
wait( tr.commit() );
wait(tr.commit());
leader->set(leaderInterf);
break;
} catch( Error &e ) {
wait( tr.onError(e) );
} catch (Error& e) {
wait(tr.onError(e));
}
}
@ -323,4 +337,3 @@ ACTOR Future<Void> restoreWorker(Reference<ClusterConnectionFile> ccf, LocalityD
wait(_restoreWorker(cx, locality));
return Void();
}

73
fdbserver/RestoreWorker.actor.h Normal file
View File

@ -0,0 +1,73 @@
/*
* RestoreWorker.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTOREWORKER_G_H)
#define FDBSERVER_RESTOREWORKER_G_H
#include "fdbserver/RestoreWorker.actor.g.h"
#elif !defined(FDBSERVER_RESTOREWORKER_H)
#define FDBSERVER_RESTOREWORKER_H
#include "fdbclient/Tuple.h"
#include "flow/flow.h"
#include "flow/Stats.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/IAsyncFile.h"
#include <cstdint>
#include <cstdarg>
#include "fdbserver/RestoreUtil.h"
#include "fdbserver/RestoreWorkerInterface.actor.h"
#include "fdbserver/RestoreCommon.actor.h"
#include "fdbserver/RestoreRoleCommon.actor.h"
#include "fdbserver/RestoreLoader.actor.h"
#include "fdbserver/RestoreApplier.actor.h"
// Each restore worker (a process) is assigned for a role.
// MAYBE Later: We will support multiple restore roles on a worker
struct RestoreWorkerData : NonCopyable, public ReferenceCounted<RestoreWorkerData> {
UID workerID;
std::map<UID, RestoreWorkerInterface> workerInterfaces; // UID is worker's node id, RestoreWorkerInterface is worker's communication workerInterface
// Restore Roles
Optional<RestoreLoaderInterface> loaderInterf;
Optional<RestoreApplierInterface> applierInterf;
uint32_t inProgressFlag = 0; // To avoid race between duplicate message delivery that invokes the same actor multiple times
UID id() const { return workerID; };
RestoreWorkerData() = default;
~RestoreWorkerData() {
printf("[Exit] Worker:%s RestoreWorkerData is deleted\n", workerID.toString().c_str());
}
std::string describeNode() {
std::stringstream ss;
ss << "RestoreWorker workerID:" << workerID.toString();
return ss.str();
}
};
#endif //FDBSERVER_RESTOREWORKER_H

293
fdbserver/RestoreWorkerInterface.actor.h
View File

@ -21,14 +21,12 @@
// This file declare and define the interface for RestoreWorker and restore roles
// which are RestoreMaster, RestoreLoader, and RestoreApplier
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_G_H)
#define FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_G_H
#include "fdbserver/RestoreWorkerInterface.actor.g.h"
#define FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_G_H
#include "fdbserver/RestoreWorkerInterface.actor.g.h"
#elif !defined(FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_H)
#define FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_H
#define FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_H
#include <sstream>
#include "flow/Stats.h"
@ -43,7 +41,8 @@
#include "flow/actorcompiler.h" // has to be last include
#define DUMPTOKEN( name ) TraceEvent("DumpToken", recruited.id()).detail("Name", #name).detail("Token", name.getEndpoint().token)
#define DUMPTOKEN(name) \
TraceEvent("DumpToken", recruited.id()).detail("Name", #name).detail("Token", name.getEndpoint().token)
class RestoreConfig;
@ -57,7 +56,6 @@ struct RestoreSetApplierKeyRangeVectorRequest;
struct RestoreSysInfo;
struct RestoreApplierInterface;
// RestoreSysInfo includes information each (type of) restore roles should know.
// At this moment, it only include appliers. We keep the name for future extension.
// TODO: If it turns out this struct only has appliers in the final version, we will rename it to a more specific name, e.g., AppliersMap
@ -82,23 +80,23 @@ struct RestoreWorkerInterface {
RequestStream<RestoreRecruitRoleRequest> recruitRole;
RequestStream<RestoreSimpleRequest> terminateWorker;
bool operator == (RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator != (RestoreWorkerInterface const& r) const { return id() != r.id(); }
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
UID id() const { return interfID; } //cmd.getEndpoint().token;
UID id() const { return interfID; } // cmd.getEndpoint().token;
NetworkAddress address() const { return recruitRole.getEndpoint().addresses.address; }
void initEndpoints() {
heartbeat.getEndpoint( TaskPriority::LoadBalancedEndpoint );
recruitRole.getEndpoint( TaskPriority::LoadBalancedEndpoint );// Q: Why do we need this?
terminateWorker.getEndpoint( TaskPriority::LoadBalancedEndpoint );
heartbeat.getEndpoint(TaskPriority::LoadBalancedEndpoint);
recruitRole.getEndpoint(TaskPriority::LoadBalancedEndpoint); // Q: Why do we need this?
terminateWorker.getEndpoint(TaskPriority::LoadBalancedEndpoint);
interfID = deterministicRandom()->randomUniqueID();
}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, interfID, heartbeat, recruitRole, terminateWorker);
}
};
@ -108,11 +106,9 @@ struct RestoreRoleInterface {
UID nodeID;
RestoreRole role;
RestoreRoleInterface() {
role = RestoreRole::Invalid;
}
RestoreRoleInterface() { role = RestoreRole::Invalid; }
explicit RestoreRoleInterface(RestoreRoleInterface const& interf) : nodeID(interf.nodeID), role(interf.role) {};
explicit RestoreRoleInterface(RestoreRoleInterface const& interf) : nodeID(interf.nodeID), role(interf.role){};
UID id() const { return nodeID; }
@ -123,7 +119,7 @@ struct RestoreRoleInterface {
}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, nodeID, role);
}
};
@ -139,10 +135,10 @@ struct RestoreLoaderInterface : RestoreRoleInterface {
RequestStream<RestoreSimpleRequest> collectRestoreRoleInterfaces; // TODO: Change to collectRestoreRoleInterfaces
RequestStream<RestoreVersionBatchRequest> finishRestore;
bool operator == (RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator != (RestoreWorkerInterface const& r) const { return id() != r.id(); }
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
RestoreLoaderInterface () {
RestoreLoaderInterface() {
role = RestoreRole::Loader;
nodeID = deterministicRandom()->randomUniqueID();
}
@ -150,24 +146,22 @@ struct RestoreLoaderInterface : RestoreRoleInterface {
NetworkAddress address() const { return heartbeat.getEndpoint().addresses.address; }
void initEndpoints() {
heartbeat.getEndpoint( TaskPriority::LoadBalancedEndpoint );
updateRestoreSysInfo.getEndpoint( TaskPriority::LoadBalancedEndpoint );
setApplierKeyRangeVectorRequest.getEndpoint( TaskPriority::LoadBalancedEndpoint );
loadFile.getEndpoint( TaskPriority::LoadBalancedEndpoint );
initVersionBatch.getEndpoint( TaskPriority::LoadBalancedEndpoint );
collectRestoreRoleInterfaces.getEndpoint( TaskPriority::LoadBalancedEndpoint );
finishRestore.getEndpoint( TaskPriority::LoadBalancedEndpoint );
heartbeat.getEndpoint(TaskPriority::LoadBalancedEndpoint);
updateRestoreSysInfo.getEndpoint(TaskPriority::LoadBalancedEndpoint);
setApplierKeyRangeVectorRequest.getEndpoint(TaskPriority::LoadBalancedEndpoint);
loadFile.getEndpoint(TaskPriority::LoadBalancedEndpoint);
initVersionBatch.getEndpoint(TaskPriority::LoadBalancedEndpoint);
collectRestoreRoleInterfaces.getEndpoint(TaskPriority::LoadBalancedEndpoint);
finishRestore.getEndpoint(TaskPriority::LoadBalancedEndpoint);
}
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, * (RestoreRoleInterface*) this, heartbeat, updateRestoreSysInfo,
setApplierKeyRangeVectorRequest, loadFile,
initVersionBatch, collectRestoreRoleInterfaces, finishRestore);
void serialize(Ar& ar) {
serializer(ar, *(RestoreRoleInterface*)this, heartbeat, updateRestoreSysInfo, setApplierKeyRangeVectorRequest,
loadFile, initVersionBatch, collectRestoreRoleInterfaces, finishRestore);
}
};
struct RestoreApplierInterface : RestoreRoleInterface {
constexpr static FileIdentifier file_identifier = 54253048;
@ -178,9 +172,8 @@ struct RestoreApplierInterface : RestoreRoleInterface {
RequestStream<RestoreSimpleRequest> collectRestoreRoleInterfaces;
RequestStream<RestoreVersionBatchRequest> finishRestore;
bool operator == (RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator != (RestoreWorkerInterface const& r) const { return id() != r.id(); }
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
RestoreApplierInterface() {
role = RestoreRole::Applier;
@ -190,26 +183,25 @@ struct RestoreApplierInterface : RestoreRoleInterface {
NetworkAddress address() const { return heartbeat.getEndpoint().addresses.address; }
void initEndpoints() {
heartbeat.getEndpoint( TaskPriority::LoadBalancedEndpoint );
sendMutationVector.getEndpoint( TaskPriority::LoadBalancedEndpoint );
applyToDB.getEndpoint( TaskPriority::LoadBalancedEndpoint );
initVersionBatch.getEndpoint( TaskPriority::LoadBalancedEndpoint );
collectRestoreRoleInterfaces.getEndpoint( TaskPriority::LoadBalancedEndpoint );
finishRestore.getEndpoint( TaskPriority::LoadBalancedEndpoint );
heartbeat.getEndpoint(TaskPriority::LoadBalancedEndpoint);
sendMutationVector.getEndpoint(TaskPriority::LoadBalancedEndpoint);
applyToDB.getEndpoint(TaskPriority::LoadBalancedEndpoint);
initVersionBatch.getEndpoint(TaskPriority::LoadBalancedEndpoint);
collectRestoreRoleInterfaces.getEndpoint(TaskPriority::LoadBalancedEndpoint);
finishRestore.getEndpoint(TaskPriority::LoadBalancedEndpoint);
}
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, * (RestoreRoleInterface*) this, heartbeat,
sendMutationVector, applyToDB, initVersionBatch, collectRestoreRoleInterfaces, finishRestore);
void serialize(Ar& ar) {
serializer(ar, *(RestoreRoleInterface*)this, heartbeat, sendMutationVector, applyToDB, initVersionBatch,
collectRestoreRoleInterfaces, finishRestore);
}
std::string toString() {
return nodeID.toString();
}
std::string toString() { return nodeID.toString(); }
};
// TODO: MX: It is probably better to specify the (beginVersion, endVersion] for each loadingParam. beginVersion (endVersion) is the version the applier is before (after) it receives the request.
// TODO: It is probably better to specify the (beginVersion, endVersion] for each loadingParam.
// beginVersion (endVersion) is the version the applier is before (after) it receives the request.
struct LoadingParam {
constexpr static FileIdentifier file_identifier = 17023837;
@ -228,24 +220,25 @@ struct LoadingParam {
Key mutationLogPrefix;
// TODO: Compare all fields for loadingParam
bool operator == ( const LoadingParam& r ) const { return isRangeFile == r.isRangeFile && filename == r.filename; }
bool operator != ( const LoadingParam& r ) const { return isRangeFile != r.isRangeFile || filename != r.filename; }
bool operator < ( const LoadingParam& r ) const {
return (isRangeFile < r.isRangeFile) ||
(isRangeFile == r.isRangeFile && filename < r.filename);
bool operator==(const LoadingParam& r) const { return isRangeFile == r.isRangeFile && filename == r.filename; }
bool operator!=(const LoadingParam& r) const { return isRangeFile != r.isRangeFile || filename != r.filename; }
bool operator<(const LoadingParam& r) const {
return (isRangeFile < r.isRangeFile) || (isRangeFile == r.isRangeFile && filename < r.filename);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, isRangeFile, url, prevVersion, endVersion, version, filename, offset, length, blockSize, restoreRange, addPrefix, removePrefix, mutationLogPrefix);
serializer(ar, isRangeFile, url, prevVersion, endVersion, version, filename, offset, length, blockSize,
restoreRange, addPrefix, removePrefix, mutationLogPrefix);
}
std::string toString() {
std::stringstream str;
str << "isRangeFile:" << isRangeFile << "url:" << url.toString() << " prevVersion:" << prevVersion << " endVersion:" << endVersion << " version:" << version
<< " filename:" << filename << " offset:" << offset << " length:" << length << " blockSize:" << blockSize
<< " restoreRange:" << restoreRange.toString()
<< " addPrefix:" << addPrefix.toString() << " removePrefix:" << removePrefix.toString();
str << "isRangeFile:" << isRangeFile << "url:" << url.toString() << " prevVersion:" << prevVersion
<< " endVersion:" << endVersion << " version:" << version << " filename:" << filename
<< " offset:" << offset << " length:" << length << " blockSize:" << blockSize
<< " restoreRange:" << restoreRange.toString() << " addPrefix:" << addPrefix.toString()
<< " removePrefix:" << removePrefix.toString();
return str.str();
}
};
@ -259,11 +252,13 @@ struct RestoreRecruitRoleReply : TimedRequest {
Optional<RestoreApplierInterface> applier;
RestoreRecruitRoleReply() = default;
explicit RestoreRecruitRoleReply(UID id, RestoreRole role, RestoreLoaderInterface const& loader): id(id), role(role), loader(loader) {}
explicit RestoreRecruitRoleReply(UID id, RestoreRole role, RestoreApplierInterface const& applier): id(id), role(role), applier(applier) {}
explicit RestoreRecruitRoleReply(UID id, RestoreRole role, RestoreLoaderInterface const& loader)
: id(id), role(role), loader(loader) {}
explicit RestoreRecruitRoleReply(UID id, RestoreRole role, RestoreApplierInterface const& applier)
: id(id), role(role), applier(applier) {}
template <class Ar>
void serialize( Ar& ar ) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, id, role, loader, applier);
}
@ -271,12 +266,12 @@ struct RestoreRecruitRoleReply : TimedRequest {
std::stringstream ss;
ss << "roleInterf role:" << getRoleStr(role) << " replyID:" << id.toString();
if (loader.present()) {
ss << "loader:" << loader.get().toString();
ss << "loader:" << loader.get().toString();
}
if (applier.present()) {
ss << "applier:" << applier.get().toString();
}
return ss.str();
}
};
@ -289,23 +284,21 @@ struct RestoreRecruitRoleRequest : TimedRequest {
ReplyPromise<RestoreRecruitRoleReply> reply;
RestoreRecruitRoleRequest() : role(RestoreRole::Invalid) {}
explicit RestoreRecruitRoleRequest(RestoreRole role, int nodeIndex) : role(role), nodeIndex(nodeIndex){}
RestoreRecruitRoleRequest() : role(RestoreRole::Invalid) {}
explicit RestoreRecruitRoleRequest(RestoreRole role, int nodeIndex) : role(role), nodeIndex(nodeIndex) {}
template <class Ar>
void serialize( Ar& ar ) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, role, nodeIndex, reply);
}
std::string printable() {
std::stringstream ss;
ss << "RestoreRecruitRoleRequest Role:" << getRoleStr(role) << " NodeIndex:" << nodeIndex;
ss << "RestoreRecruitRoleRequest Role:" << getRoleStr(role) << " NodeIndex:" << nodeIndex;
return ss.str();
}
std::string toString() {
return printable();
}
std::string toString() { return printable(); }
};
struct RestoreSysInfoRequest : TimedRequest {
@ -325,12 +318,11 @@ struct RestoreSysInfoRequest : TimedRequest {
std::string toString() {
std::stringstream ss;
ss << "RestoreSysInfoRequest";
ss << "RestoreSysInfoRequest";
return ss.str();
}
};
// Sample_Range_File and Assign_Loader_Range_File, Assign_Loader_Log_File
struct RestoreLoadFileRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 26557364;
@ -342,14 +334,14 @@ struct RestoreLoadFileRequest : TimedRequest {
RestoreLoadFileRequest() = default;
explicit RestoreLoadFileRequest(LoadingParam param) : param(param) {}
template <class Ar>
void serialize( Ar& ar ) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, param, reply);
}
std::string toString() {
std::stringstream ss;
ss << "RestoreLoadFileRequest param:" << param.toString();
ss << "RestoreLoadFileRequest param:" << param.toString();
return ss.str();
}
};
@ -364,22 +356,23 @@ struct RestoreSendMutationVectorVersionedRequest : TimedRequest {
ReplyPromise<RestoreCommonReply> reply;
RestoreSendMutationVectorVersionedRequest() = default;
explicit RestoreSendMutationVectorVersionedRequest(Version prevVersion, Version version, bool isRangeFile, VectorRef<MutationRef> mutations) :
prevVersion(prevVersion), version(version), isRangeFile(isRangeFile), mutations(mutations) {}
explicit RestoreSendMutationVectorVersionedRequest(Version prevVersion, Version version, bool isRangeFile,
VectorRef<MutationRef> mutations)
: prevVersion(prevVersion), version(version), isRangeFile(isRangeFile), mutations(mutations) {}
std::string toString() {
std::stringstream ss;
ss << "prevVersion:" << prevVersion << " version:" << version << " isRangeFile:" << isRangeFile << " mutations.size:" << mutations.size();
ss << "prevVersion:" << prevVersion << " version:" << version << " isRangeFile:" << isRangeFile
<< " mutations.size:" << mutations.size();
return ss.str();
}
template <class Ar>
void serialize( Ar& ar ) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, prevVersion, version, isRangeFile, mutations, reply);
}
};
struct RestoreVersionBatchRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 13018413;
@ -390,14 +383,14 @@ struct RestoreVersionBatchRequest : TimedRequest {
RestoreVersionBatchRequest() = default;
explicit RestoreVersionBatchRequest(int batchID) : batchID(batchID) {}
template <class Ar>
void serialize( Ar& ar ) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, batchID, reply);
}
std::string toString() {
std::stringstream ss;
ss << "RestoreVersionBatchRequest BatchID:" << batchID;
ss << "RestoreVersionBatchRequest BatchID:" << batchID;
return ss.str();
}
};
@ -406,20 +399,21 @@ struct RestoreSetApplierKeyRangeVectorRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 92038306;
std::map<Standalone<KeyRef>, UID> range2Applier;
ReplyPromise<RestoreCommonReply> reply;
RestoreSetApplierKeyRangeVectorRequest() = default;
explicit RestoreSetApplierKeyRangeVectorRequest(std::map<Standalone<KeyRef>, UID> range2Applier) : range2Applier(range2Applier) {}
explicit RestoreSetApplierKeyRangeVectorRequest(std::map<Standalone<KeyRef>, UID> range2Applier)
: range2Applier(range2Applier) {}
template <class Ar>
void serialize( Ar& ar ) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, range2Applier, reply);
}
std::string toString() {
std::stringstream ss;
ss << "RestoreVersionBatchRequest range2ApplierSize:" << range2Applier.size();
ss << "RestoreVersionBatchRequest range2ApplierSize:" << range2Applier.size();
return ss.str();
}
};
@ -427,7 +421,7 @@ struct RestoreSetApplierKeyRangeVectorRequest : TimedRequest {
struct RestoreRequest {
constexpr static FileIdentifier file_identifier = 49589770;
//Database cx;
// Database cx;
int index;
Key tagName;
Key url;
@ -442,34 +436,36 @@ struct RestoreRequest {
int testData;
std::vector<int> restoreRequests;
//Key restoreTag;
// Key restoreTag;
ReplyPromise< struct RestoreCommonReply > reply;
ReplyPromise<struct RestoreCommonReply> reply;
RestoreRequest() : testData(0) {}
explicit RestoreRequest(int testData) : testData(testData) {}
explicit RestoreRequest(int testData, std::vector<int> &restoreRequests) : testData(testData), restoreRequests(restoreRequests) {}
explicit RestoreRequest(int testData, std::vector<int>& restoreRequests)
: testData(testData), restoreRequests(restoreRequests) {}
explicit RestoreRequest(const int index, const Key &tagName, const Key &url, bool waitForComplete, Version targetVersion, bool verbose,
const KeyRange &range, const Key &addPrefix, const Key &removePrefix, bool lockDB,
const UID &randomUid) : index(index), tagName(tagName), url(url), waitForComplete(waitForComplete),
targetVersion(targetVersion), verbose(verbose), range(range),
addPrefix(addPrefix), removePrefix(removePrefix), lockDB(lockDB),
randomUid(randomUid) {}
explicit RestoreRequest(const int index, const Key& tagName, const Key& url, bool waitForComplete,
Version targetVersion, bool verbose, const KeyRange& range, const Key& addPrefix,
const Key& removePrefix, bool lockDB, const UID& randomUid)
: index(index), tagName(tagName), url(url), waitForComplete(waitForComplete), targetVersion(targetVersion),
verbose(verbose), range(range), addPrefix(addPrefix), removePrefix(removePrefix), lockDB(lockDB),
randomUid(randomUid) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, index , tagName , url , waitForComplete , targetVersion , verbose , range , addPrefix , removePrefix , lockDB , randomUid ,
testData , restoreRequests , reply);
serializer(ar, index, tagName, url, waitForComplete, targetVersion, verbose, range, addPrefix, removePrefix,
lockDB, randomUid, testData, restoreRequests, reply);
}
//Q: Should I convert this toString() to a function to dump RestoreRequest to TraceEvent?
std::string toString() const {
std::stringstream ss;
ss << "index:" << std::to_string(index) << " tagName:" << tagName.contents().toString() << " url:" << url.contents().toString()
<< " waitForComplete:" << std::to_string(waitForComplete) << " targetVersion:" << std::to_string(targetVersion)
<< " verbose:" << std::to_string(verbose) << " range:" << range.toString() << " addPrefix:" << addPrefix.contents().toString()
<< " removePrefix:" << removePrefix.contents().toString() << " lockDB:" << std::to_string(lockDB) << " randomUid:" << randomUid.toString();
ss << "index:" << std::to_string(index) << " tagName:" << tagName.contents().toString()
<< " url:" << url.contents().toString() << " waitForComplete:" << std::to_string(waitForComplete)
<< " targetVersion:" << std::to_string(targetVersion) << " verbose:" << std::to_string(verbose)
<< " range:" << range.toString() << " addPrefix:" << addPrefix.contents().toString()
<< " removePrefix:" << removePrefix.contents().toString() << " lockDB:" << std::to_string(lockDB)
<< " randomUid:" << randomUid.toString();
return ss.str();
}
};
@ -480,81 +476,78 @@ std::string getRoleStr(RestoreRole role);
Future<Void> _restoreWorker(Database const& cx, LocalityData const& locality);
Future<Void> restoreWorker(Reference<ClusterConnectionFile> const& ccf, LocalityData const& locality);
// Send each request in requests via channel of the request's interface.
// Do not expect a meaningful reply
// The UID in a request is the UID of the interface to handle the request
ACTOR template <class Interface, class Request>
//Future< REPLY_TYPE(Request) >
Future<Void> sendBatchRequests(
RequestStream<Request> Interface::* channel,
std::map<UID, Interface> interfaces,
std::vector<std::pair<UID, Request>> requests) {
if ( requests.empty() ) {
Future<Void> sendBatchRequests(RequestStream<Request> Interface::*channel, std::map<UID, Interface> interfaces,
std::vector<std::pair<UID, Request>> requests) {
if (requests.empty()) {
return Void();
}
loop{
try {
loop {
try {
std::vector<Future<REPLY_TYPE(Request)>> cmdReplies;
for(auto& request : requests) {
RequestStream<Request> const* stream = & (interfaces[request.first].*channel);
cmdReplies.push_back( stream->getReply(request.second) );
for (auto& request : requests) {
RequestStream<Request> const* stream = &(interfaces[request.first].*channel);
cmdReplies.push_back(stream->getReply(request.second));
}
// Alex: Unless you want to do some action when it timeout multiple times, you should use timout. Otherwise, getReply will automatically keep retrying for you.
std::vector<REPLY_TYPE(Request)> reps = wait( timeoutError(getAll(cmdReplies), SERVER_KNOBS->FASTRESTORE_FAILURE_TIMEOUT) ); //tryGetReply. Use GetReply. // Alex: you probably do NOT need the timeoutError.
//wait( waitForAll(cmdReplies) ); //tryGetReply. Use GetReply. // Alex: you probably do NOT need the timeoutError.
// Alex: Unless you want to do some action when it timeout multiple times, you should use timout. Otherwise,
// getReply will automatically keep retrying for you.
// Alex: you probably do NOT need the timeoutError.
std::vector<REPLY_TYPE(Request)> reps = wait(
timeoutError(getAll(cmdReplies), SERVER_KNOBS->FASTRESTORE_FAILURE_TIMEOUT));
break;
} catch (Error &e) {
if ( e.code() == error_code_operation_cancelled ) break;
} catch (Error& e) {
if (e.code() == error_code_operation_cancelled) break;
fprintf(stdout, "sendBatchRequests Error code:%d, error message:%s\n", e.code(), e.what());
for (auto& request : requests ) {
TraceEvent(SevWarn, "FastRestore").detail("SendBatchRequests", requests.size())
.detail("RequestID", request.first).detail("Request", request.second.toString());
for (auto& request : requests) {
TraceEvent(SevWarn, "FastRestore")
.detail("SendBatchRequests", requests.size())
.detail("RequestID", request.first)
.detail("Request", request.second.toString());
}
}
}
return Void();
}
}
// Similar to sendBatchRequests except that the caller expect to process the reply.
// This actor can be combined with sendBatchRequests(...)
ACTOR template <class Interface, class Request>
//Future< REPLY_TYPE(Request) >
Future<Void> getBatchReplies(
RequestStream<Request> Interface::* channel,
std::map<UID, Interface> interfaces,
std::map<UID, Request> requests,
std::vector<REPLY_TYPE(Request)>* replies) {
Future<Void> getBatchReplies(RequestStream<Request> Interface::*channel, std::map<UID, Interface> interfaces,
std::map<UID, Request> requests, std::vector<REPLY_TYPE(Request)>* replies) {
if ( requests.empty() ) {
if (requests.empty()) {
return Void();
}
loop{
try {
loop {
try {
std::vector<Future<REPLY_TYPE(Request)>> cmdReplies;
for(auto& request : requests) {
RequestStream<Request> const* stream = & (interfaces[request.first].*channel);
cmdReplies.push_back( stream->getReply(request.second) );
for (auto& request : requests) {
RequestStream<Request> const* stream = &(interfaces[request.first].*channel);
cmdReplies.push_back(stream->getReply(request.second));
}
// Alex: Unless you want to do some action when it timeout multiple times, you should use timout. Otherwise, getReply will automatically keep retrying for you.
std::vector<REPLY_TYPE(Request)> reps = wait( timeoutError(getAll(cmdReplies), SERVER_KNOBS->FASTRESTORE_FAILURE_TIMEOUT) ); //tryGetReply. Use GetReply. // Alex: you probably do NOT need the timeoutError.
// Alex: Unless you want to do some action when it timeout multiple times, you should use timout. Otherwise,
// getReply will automatically keep retrying for you.
std::vector<REPLY_TYPE(Request)> reps = wait(
timeoutError(getAll(cmdReplies), SERVER_KNOBS->FASTRESTORE_FAILURE_TIMEOUT));
*replies = reps;
break;
} catch (Error &e) {
if ( e.code() == error_code_operation_cancelled ) break;
} catch (Error& e) {
if (e.code() == error_code_operation_cancelled) break;
fprintf(stdout, "getBatchReplies Error code:%d, error message:%s\n", e.code(), e.what());
}
}
return Void();
}
}
#include "flow/unactorcompiler.h"
#endif

178
fdbserver/fdbserver.actor.cpp
View File

@ -82,12 +82,57 @@
#include "flow/actorcompiler.h" // This must be the last #include.
enum {
OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_NEWCONSOLE,
OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID,
OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACECLOCK,
OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
OPT_TRACE_FORMAT, OPT_USE_OBJECT_SERIALIZER, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE
OPT_CONNFILE,
OPT_SEEDCONNFILE,
OPT_SEEDCONNSTRING,
OPT_ROLE,
OPT_LISTEN,
OPT_PUBLICADDR,
OPT_DATAFOLDER,
OPT_LOGFOLDER,
OPT_PARENTPID,
OPT_NEWCONSOLE,
OPT_NOBOX,
OPT_TESTFILE,
OPT_RESTARTING,
OPT_RESTORING,
OPT_RANDOMSEED,
OPT_KEY,
OPT_MEMLIMIT,
OPT_STORAGEMEMLIMIT,
OPT_CACHEMEMLIMIT,
OPT_MACHINEID,
OPT_DCID,
OPT_MACHINE_CLASS,
OPT_BUGGIFY,
OPT_VERSION,
OPT_CRASHONERROR,
OPT_HELP,
OPT_NETWORKIMPL,
OPT_NOBUFSTDOUT,
OPT_BUFSTDOUTERR,
OPT_TRACECLOCK,
OPT_NUMTESTERS,
OPT_DEVHELP,
OPT_ROLLSIZE,
OPT_MAXLOGS,
OPT_MAXLOGSSIZE,
OPT_KNOB,
OPT_TESTSERVERS,
OPT_TEST_ON_SERVERS,
OPT_METRICSCONNFILE,
OPT_METRICSPREFIX,
OPT_LOGGROUP,
OPT_LOCALITY,
OPT_IO_TRUST_SECONDS,
OPT_IO_TRUST_WARN_ONLY,
OPT_FILESYSTEM,
OPT_PROFILER_RSS_SIZE,
OPT_KVFILE,
OPT_TRACE_FORMAT,
OPT_USE_OBJECT_SERIALIZER,
OPT_WHITELIST_BINPATH,
OPT_BLOB_CREDENTIAL_FILE
};
CSimpleOpt::SOption g_rgOptions[] = {
@ -970,7 +1015,7 @@ int main(int argc, char* argv[]) {
double fileIoTimeout = 0.0;
bool fileIoWarnOnly = false;
std::vector<std::string> blobCredentials; // used for fast restore workers
// const char *blobCredsFromENV = nullptr;
// const char *blobCredsFromENV = nullptr;
uint64_t rsssize = -1;
bool useObjectSerializer = true;
@ -1031,10 +1076,10 @@ int main(int argc, char* argv[]) {
flushAndExit(FDB_EXIT_ERROR);
}
syn = syn.substr(7);
knobs.push_back(std::make_pair(syn, args.OptionArg()));
break;
}
case OPT_LOCALITY: {
knobs.push_back(std::make_pair(syn, args.OptionArg()));
break;
}
case OPT_LOCALITY: {
std::string syn = args.OptionSyntax();
if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) {
fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", syn.c_str());
@ -1044,8 +1089,8 @@ int main(int argc, char* argv[]) {
std::transform(syn.begin(), syn.end(), syn.begin(), ::tolower);
localities.set(Standalone<StringRef>(syn), Standalone<StringRef>(std::string(args.OptionArg())));
break;
}
case OPT_VERSION:
}
case OPT_VERSION:
printVersion();
flushAndExit(FDB_EXIT_SUCCESS);
break;
@ -1126,22 +1171,21 @@ int main(int argc, char* argv[]) {
logFolder = args.OptionArg();
break;
case OPT_NETWORKIMPL: {
const char *a = args.OptionArg();
if (!strcmp(a, "net2")) useNet2 = true;
else if (!strcmp(a, "net2-threadpool")) {
useNet2 = true;
useThreadPool = true;
}
else {
fprintf(stderr, "ERROR: Unknown network implementation `%s'\n", a);
const char* a = args.OptionArg();
if (!strcmp(a, "net2")) useNet2 = true;
else if (!strcmp(a, "net2-threadpool")) {
useNet2 = true;
useThreadPool = true;
} else {
fprintf(stderr, "ERROR: Unknown network implementation `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
break;
}
case OPT_TRACECLOCK: {
const char *a = args.OptionArg();
if (!strcmp(a, "realtime")) g_trace_clock = TRACE_CLOCK_REALTIME;
const char* a = args.OptionArg();
if (!strcmp(a, "realtime")) g_trace_clock = TRACE_CLOCK_REALTIME;
else if (!strcmp(a, "now")) g_trace_clock = TRACE_CLOCK_NOW;
else {
fprintf(stderr, "ERROR: Unknown clock source `%s'\n", a);
@ -1151,17 +1195,17 @@ int main(int argc, char* argv[]) {
break;
}
case OPT_NUMTESTERS: {
const char *a = args.OptionArg();
if (!sscanf(a, "%d", &minTesterCount)) {
fprintf(stderr, "ERROR: Could not parse numtesters `%s'\n", a);
const char* a = args.OptionArg();
if (!sscanf(a, "%d", &minTesterCount)) {
fprintf(stderr, "ERROR: Could not parse numtesters `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
break;
}
case OPT_ROLLSIZE: {
const char *a = args.OptionArg();
ti = parse_with_suffix(a);
const char* a = args.OptionArg();
ti = parse_with_suffix(a);
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse logsize `%s'\n", a);
printHelpTeaser(argv[0]);
@ -1186,12 +1230,12 @@ int main(int argc, char* argv[]) {
const char *a = args.OptionArg();
char *end;
maxLogs = strtoull(a, &end, 10);
if (*end) {
fprintf(stderr, "ERROR: Unrecognized maximum number of logs `%s'\n", a);
if (*end) {
fprintf(stderr, "ERROR: Unrecognized maximum number of logs `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
maxLogsSet = true;
}
maxLogsSet = true;
break;
}
#ifdef _WIN32
@ -1255,32 +1299,32 @@ int main(int argc, char* argv[]) {
}
case OPT_MACHINE_CLASS:
sRole = args.OptionArg();
processClass = ProcessClass(sRole, ProcessClass::CommandLineSource);
if (processClass == ProcessClass::InvalidClass) {
processClass = ProcessClass(sRole, ProcessClass::CommandLineSource);
if (processClass == ProcessClass::InvalidClass) {
fprintf(stderr, "ERROR: Unknown machine class `%s'\n", sRole);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
case OPT_BLOB_CREDENTIAL_FILE: {
//Add blob credential following backup agent example
blobCredentials.push_back(args.OptionArg());
printf("blob credential file:%s\n", blobCredentials.back().c_str());
case OPT_BLOB_CREDENTIAL_FILE: {
// Add blob credential following backup agent example
blobCredentials.push_back(args.OptionArg());
printf("blob credential file:%s\n", blobCredentials.back().c_str());
//
// blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS");
// if (blobCredsFromENV != nullptr) {
// printf("[WARNING] set blob credetial via env variable is not tested\n");
// StringRef t((uint8_t *) blobCredsFromENV, strlen(blobCredsFromENV));
// do {
// StringRef file = t.eat(":");
// if (file.size() != 0)
// blobCredentials.push_back(file.toString());
// } while (t.size() != 0);
// }
break;
}
case OPT_KEY:
//
// blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS");
// if (blobCredsFromENV != nullptr) {
// printf("[WARNING] set blob credetial via env variable is not tested\n");
// StringRef t((uint8_t *) blobCredsFromENV, strlen(blobCredsFromENV));
// do {
// StringRef file = t.eat(":");
// if (file.size() != 0)
// blobCredentials.push_back(file.toString());
// } while (t.size() != 0);
// }
break;
}
case OPT_KEY:
targetKey = args.OptionArg();
break;
case OPT_MEMLIMIT:
@ -1830,31 +1874,33 @@ int main(int argc, char* argv[]) {
g_simulator.run();
} else if (role == FDBD) {
// Call fast restore for the class FastRestoreClass. This is a short-cut to run fast restore in circus
if ( processClass == ProcessClass::FastRestoreClass) {
if (processClass == ProcessClass::FastRestoreClass) {
printf("Run as fast restore worker\n");
// Update the global blob credential files list
std::vector<std::string> *pFiles = (std::vector<std::string> *) g_network->global(INetwork::enBlobCredentialFiles);
std::vector<std::string>* pFiles =
(std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
if (pFiles != nullptr) {
for (auto &f : blobCredentials) {
for (auto& f : blobCredentials) {
pFiles->push_back(f);
}
}
f = stopAfter( restoreWorker(connectionFile, localities) );
f = stopAfter(restoreWorker(connectionFile, localities));
g_network->run();
} else {
ASSERT( connectionFile );
ASSERT(connectionFile);
setupSlowTaskProfiler();
if (!dataFolder.size())
dataFolder = format("fdb/%d/", publicAddresses.address.port); // SOMEDAY: Better default
dataFolder = format("fdb/%d/", publicAddresses.address.port); // SOMEDAY: Better default
vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
actors.push_back( fdbd(connectionFile, localities, processClass, dataFolder, dataFolder, storageMemLimit, metricsConnFile, metricsPrefix, rsssize, whitelistBinPaths) );
//actors.push_back( recurring( []{}, .001 ) ); // for ASIO latency measurement
actors.push_back(fdbd(connectionFile, localities, processClass, dataFolder, dataFolder, storageMemLimit,
metricsConnFile, metricsPrefix, rsssize, whitelistBinPaths));
// actors.push_back( recurring( []{}, .001 ) ); // for ASIO latency measurement
f = stopAfter( waitForAll(actors) );
f = stopAfter(waitForAll(actors));
g_network->run();
}
} else if (role == MultiTester) {

14
fdbserver/sqlite/btree.c
View File

@ -4549,13 +4549,13 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked(
MemPage *pPage = pCur->apPage[pCur->iPage];
int c;
/* pPage->nCell must be greater than zero. If this is the root-page
** the cursor would have been Invalid above and this for(;;) loop
** not run. If this is not the root-page, then the moveToChild() routine
** would have already detected db corruption. Similarly, pPage must
** be the right kind (index or table) of b-tree page. Otherwise
** a moveToChild() or moveToRoot() call would have detected corruption. */
assert( pPage->nCell>0 );
/* pPage->nCell must be greater than zero. If this is the root-page
** the cursor would have been Invalid above and this for(;;) loop
** not run. If this is not the root-page, then the moveToChild() routine
** would have already detected db corruption. Similarly, pPage must
** be the right kind (index or table) of b-tree page. Otherwise
** a moveToChild() or moveToRoot() call would have detected corruption. */
assert( pPage->nCell>0 );
assert( pPage->intKey==(pIdxKey==0) );
lwr = 0;
upr = pPage->nCell-1;

2
fdbserver/tester.actor.cpp
View File

@ -703,7 +703,7 @@ ACTOR Future<DistributedTestResults> runWorkload( Database cx, std::vector< Test
wait( waitForAll( checks ) );
printf("checking tests DONE num_workloads:%d\n", workloads.size());
throwIfError(checks, "CheckFailedForWorkload" + printable(spec.title));
for(int i = 0; i < checks.size(); i++) {

600
fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
View File

File diff suppressed because it is too large Load Diff

7
fdbserver/workloads/BackupCorrectness.actor.cpp
View File

@ -468,7 +468,9 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
auto range = self->restoreRanges[restoreIndex];
Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" + std::to_string(restoreIndex));
restoreTags.push_back(restoreTag);
printf("BackupCorrectness, restore for each range: backupAgent.restore is called for restoreIndex:%d tag:%s ranges:%s\n", restoreIndex, range.toString().c_str(), restoreTag.toString().c_str());
printf("BackupCorrectness, restore for each range: backupAgent.restore is called for "
"restoreIndex:%d tag:%s ranges:%s\n",
restoreIndex, range.toString().c_str(), restoreTag.toString().c_str());
restores.push_back(backupAgent.restore(cx, cx, restoreTag, KeyRef(lastBackupContainer->getURL()), true, targetVersion, true, range, Key(), Key(), self->locked));
}
}
@ -476,7 +478,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
multipleRangesInOneTag = true;
Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" + std::to_string(restoreIndex));
restoreTags.push_back(restoreTag);
printf("BackupCorrectness, backupAgent.restore is called for restoreIndex:%d tag:%s\n", restoreIndex, restoreTag.toString().c_str());
printf("BackupCorrectness, backupAgent.restore is called for restoreIndex:%d tag:%s\n",
restoreIndex, restoreTag.toString().c_str());
restores.push_back(backupAgent.restore(cx, cx, restoreTag, KeyRef(lastBackupContainer->getURL()), self->restoreRanges, true, targetVersion, true, Key(), Key(), self->locked));
}

12
fdbserver/workloads/Cycle.actor.cpp
View File

@ -144,8 +144,11 @@ struct CycleWorkload : TestWorkload {
void logTestData(const VectorRef<KeyValueRef>& data) {
TraceEvent("MXTestFailureDetail");
int index = 0;
for(auto &entry : data) {
TraceEvent("CurrentDataEntry").detail("Index", index).detail("Key", entry.key.toString()).detail("Value", entry.value.toString());
for (auto& entry : data) {
TraceEvent("CurrentDataEntry")
.detail("Index", index)
.detail("Key", entry.key.toString())
.detail("Value", entry.value.toString());
index++;
}
}
@ -154,7 +157,10 @@ struct CycleWorkload : TestWorkload {
if (data.size() != nodeCount) {
logTestData(data);
TraceEvent(SevError, "TestFailure").detail("Reason", "Node count changed").detail("Before", nodeCount).detail("After", data.size()).detail("Version", v).detail("KeyPrefix", keyPrefix.printable());
TraceEvent(SevError, "TestFailureInfo").detail("DataSize", data.size()).detail("NodeCount", nodeCount).detail("Workload", description());
TraceEvent(SevError, "TestFailureInfo")
.detail("DataSize", data.size())
.detail("NodeCount", nodeCount)
.detail("Workload", description());
return false;
}
int i=0;

23
fdbserver/workloads/ParallelRestore.actor.cpp
View File

@ -24,24 +24,18 @@
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/workloads/BulkSetup.actor.h"
#include "fdbserver/RestoreWorkerInterface.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
//A workload which test the correctness of backup and restore process
// A workload which test the correctness of backup and restore process
struct RunRestoreWorkerWorkload : TestWorkload {
Future<Void> worker;
RunRestoreWorkerWorkload(WorkloadContext const& wcx)
: TestWorkload(wcx) {
RunRestoreWorkerWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
TraceEvent("RunRestoreWorkerWorkloadMX");
}
virtual std::string description() {
return "RunRestoreWorkerWorkload";
}
virtual std::string description() { return "RunRestoreWorkerWorkload"; }
virtual Future<Void> setup(Database const& cx) {
return Void();
}
virtual Future<Void> setup(Database const& cx) { return Void(); }
virtual Future<Void> start(Database const& cx) {
int num_myWorkers = 3;
@ -57,12 +51,9 @@ struct RunRestoreWorkerWorkload : TestWorkload {
return Void();
}
virtual Future<bool> check(Database const& cx) {
return true;
}
virtual Future<bool> check(Database const& cx) { return true; }
virtual void getMetrics(vector<PerfMetric>& m) {
}
virtual void getMetrics(vector<PerfMetric>& m) {}
};
WorkloadFactory<RunRestoreWorkerWorkload> RunRestoreWorkerWorkloadFactory("RunRestoreWorkerWorkload");

3
fdbserver/workloads/workloads.actor.h
View File

@ -158,7 +158,8 @@ public:
startDelay = 30.0;
phases = TestWorkload::SETUP | TestWorkload::EXECUTION | TestWorkload::CHECK | TestWorkload::METRICS;
timeout = g_network->isSimulated() ? 15000 : 1500;
//timeout = g_network->isSimulated() ? 150000 : 15000; // MX: increase the timeout to avoid false positive error in test
// timeout = g_network->isSimulated() ? 150000 : 15000; // MX: increase the timeout to avoid false positive
// error in test
databasePingDelay = g_network->isSimulated() ? 0.0 : 15.0;
runConsistencyCheck = g_network->isSimulated();
waitForQuiescenceBegin = true;