wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FastRestore:Apply clang-format

This commit is contained in:
Meng Xu 2020-01-07 11:50:29 -08:00
parent 67e913c3d5
commit 9df02512ab
4 changed files with 31 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
fdbclient/RestoreWorkerInterface.actor.h
View File

@ -211,9 +211,7 @@ struct RestoreAsset {
UID uid;
RestoreAsset() {
uid = deterministicRandom()->randomUniqueID();
}
RestoreAsset() { uid = deterministicRandom()->randomUniqueID(); }
bool operator==(const RestoreAsset& r) const {
return fileIndex == r.fileIndex && filename == r.filename && offset == r.offset && len == r.len &&
@ -236,8 +234,9 @@ struct RestoreAsset {
std::string toString() {
std::stringstream ss;
ss << "UID:" << uid.toString() << " begin:" << beginVersion << " end:" << endVersion << " range:" << range.toString()
<< " filename:" << filename << " fileIndex:" << fileIndex << " offset:" << offset << " len:" << len;
ss << "UID:" << uid.toString() << " begin:" << beginVersion << " end:" << endVersion
<< " range:" << range.toString() << " filename:" << filename << " fileIndex:" << fileIndex
<< " offset:" << offset << " len:" << len;
return ss.str();
}
@ -256,7 +255,7 @@ struct LoadingParam {
Key url;
Version prevVersion;
Version rangeVersion; // range file's version
//Version endVersion; // range file's mutations are all at the endVersion
// Version endVersion; // range file's mutations are all at the endVersion
int64_t blockSize;
RestoreAsset asset;
@ -272,7 +271,7 @@ struct LoadingParam {
template <class Ar>
void serialize(Ar& ar) {
//serializer(ar, isRangeFile, url, prevVersion, endVersion, blockSize, asset);
// serializer(ar, isRangeFile, url, prevVersion, endVersion, blockSize, asset);
serializer(ar, isRangeFile, url, prevVersion, rangeVersion, blockSize, asset);
}
@ -281,7 +280,7 @@ struct LoadingParam {
// str << "isRangeFile:" << isRangeFile << " url:" << url.toString() << " prevVersion:" << prevVersion
// << " endVersion:" << endVersion << " blockSize:" << blockSize << " RestoreAsset:" << asset.toString();
str << "isRangeFile:" << isRangeFile << " url:" << url.toString() << " prevVersion:" << prevVersion
<< " rangeVersion:" << rangeVersion << " blockSize:" << blockSize << " RestoreAsset:" << asset.toString();
<< " rangeVersion:" << rangeVersion << " blockSize:" << blockSize << " RestoreAsset:" << asset.toString();
return str.str();
}
};

8
fdbserver/RestoreLoader.actor.cpp
View File

@ -153,8 +153,8 @@ ACTOR Future<Void> _processLoadingParam(LoadingParam param, Reference<RestoreLoa
subAsset.offset = j;
subAsset.len = std::min<int64_t>(param.blockSize, param.asset.len - j);
if (param.isRangeFile) {
fileParserFutures.push_back(
_parseRangeFileToMutationsOnLoader(kvOpsPerLPIter, samplesIter, self->bc, param.rangeVersion, subAsset));
fileParserFutures.push_back(_parseRangeFileToMutationsOnLoader(kvOpsPerLPIter, samplesIter, self->bc,
param.rangeVersion, subAsset));
} else {
// TODO: Sanity check the log file's range is overlapped with the restored version range
fileParserFutures.push_back(_parseLogFileToMutationsOnLoader(&processedFileOffset, &mutationMap,
@ -246,7 +246,9 @@ ACTOR Future<Void> sendMutationsToApplier(Reference<RestoreLoaderData> self, Ver
}
Version commitVersion = kvOp->first;
if (!(commitVersion >= asset.beginVersion && commitVersion <= asset.endVersion)) { // Debug purpose
TraceEvent(SevError, "FastRestore_SendMutationsToApplier").detail("CommitVersion", commitVersion).detail("RestoreAsset", asset.toString());
TraceEvent(SevError, "FastRestore_SendMutationsToApplier")
.detail("CommitVersion", commitVersion)
.detail("RestoreAsset", asset.toString());
}
ASSERT(commitVersion >= asset.beginVersion);
ASSERT(commitVersion <= asset.endVersion); // endVersion is an empty commit to ensure progress

9
fdbserver/RestoreMaster.actor.cpp
View File

@ -37,8 +37,7 @@
ACTOR static Future<Void> clearDB(Database cx);
ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
std::vector<RestoreFileFR>* logFiles, Database cx,
RestoreRequest request);
std::vector<RestoreFileFR>* logFiles, Database cx, RestoreRequest request);
ACTOR static Future<Version> processRestoreRequest(Reference<RestoreMasterData> self, Database cx, RestoreRequest request);
ACTOR static Future<Void> startProcessRestoreRequests(Reference<RestoreMasterData> self, Database cx);
@ -288,7 +287,7 @@ ACTOR static Future<Void> loadFilesOnLoaders(Reference<RestoreMasterData> self,
LoadingParam param;
param.prevVersion = 0; // Each file's NotifiedVersion starts from 0
//param.endVersion = file.isRange ? file.version : file.endVersion;
// param.endVersion = file.isRange ? file.version : file.endVersion;
param.url = request.url;
param.isRangeFile = file.isRange;
param.rangeVersion = file.isRange ? file.version : -1;
@ -453,8 +452,8 @@ ACTOR static Future<Standalone<VectorRef<RestoreRequest>>> collectRestoreRequest
// Collect the backup files' description into output_files by reading the backupContainer bc.
ACTOR static Future<Void> collectBackupFiles(Reference<IBackupContainer> bc, std::vector<RestoreFileFR>* rangeFiles,
std::vector<RestoreFileFR>* logFiles, Database cx,
RestoreRequest request) {
std::vector<RestoreFileFR>* logFiles, Database cx,
RestoreRequest request) {
state BackupDescription desc = wait(bc->describeBackup());
// Convert version to real time for operators to read the BackupDescription desc.

27
fdbserver/RestoreMaster.actor.h
View File

@ -58,9 +58,7 @@ struct VersionBatch {
}
// RestoreAsset and VersionBatch both use endVersion as exclusive in version range
bool isInVersionRange(Version version) const {
return version >= beginVersion && version < endVersion;
}
bool isInVersionRange(Version version) const { return version >= beginVersion && version < endVersion; }
};
struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMasterData> {
@ -106,17 +104,22 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
int i = 0;
for (auto& vb : versionBatches) {
TraceEvent("FastRestoreVersionBatches")
.detail("BatchIndex", i)
.detail("BatchIndex", i)
.detail("BeginVersion", vb.second.beginVersion)
.detail("EndVersion", vb.second.endVersion)
.detail("Size", vb.second.size);
for (auto& f : vb.second.rangeFiles) {
bool invalidVersion = (f.beginVersion != f.endVersion) || (f.beginVersion >= vb.second.endVersion || f.beginVersion < vb.second.beginVersion);
TraceEvent(invalidVersion ? SevError : SevInfo, "FastRestoreVersionBatches").detail("BatchIndex", i).detail("RangeFile", f.toString());
bool invalidVersion = (f.beginVersion != f.endVersion) || (f.beginVersion >= vb.second.endVersion ||
f.beginVersion < vb.second.beginVersion);
TraceEvent(invalidVersion ? SevError : SevInfo, "FastRestoreVersionBatches")
.detail("BatchIndex", i)
.detail("RangeFile", f.toString());
}
for (auto& f : vb.second.logFiles) {
bool outOfRange = (f.beginVersion >= vb.second.endVersion || f.endVersion <= vb.second.beginVersion);
TraceEvent(outOfRange ? SevError : SevInfo, "FastRestoreVersionBatches").detail("BatchIndex", i).detail("LogFile", f.toString());
TraceEvent(outOfRange ? SevError : SevInfo, "FastRestoreVersionBatches")
.detail("BatchIndex", i)
.detail("LogFile", f.toString());
}
++i;
}
@ -171,7 +174,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
// Assumption 1: input files has no empty files
// Assumption 2: range files at one version > batchSizeThreshold
void buildVersionBatches(const std::vector<RestoreFileFR>& rangeFiles, const std::vector<RestoreFileFR>& logFiles,
std::map<Version, VersionBatch>* versionBatches) {
std::map<Version, VersionBatch>* versionBatches) {
// Version batch range [beginVersion, endVersion)
Version beginVersion = 0;
Version endVersion = 0;
@ -217,8 +220,8 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
getVersionSize(prevEndVersion, nextVersion, rangeFiles, rangeIdx, logFiles);
TraceEvent("FastRestoreBuildVersionBatch")
.detail("VersionBatchBeginVersion", vb.beginVersion)
.detail("PrevEndVersion", prevEndVersion)
.detail("VersionBatchBeginVersion", vb.beginVersion)
.detail("PrevEndVersion", prevEndVersion)
.detail("NextVersion", nextVersion)
.detail("RangeIndex", rangeIdx)
.detail("RangeFiles", rangeFiles.size())
@ -228,7 +231,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
.detail("BatchSize", batchSize)
.detail("NextVersionSize", nextVersionSize)
.detail("NextRangeIndex", nextRangeIdx)
.detail("UsedLogFiles", curLogFiles.size());
.detail("UsedLogFiles", curLogFiles.size());
if (batchSize + nextVersionSize <= opConfig.batchSizeThreshold) {
// nextVersion should be included in this batch
@ -270,7 +273,7 @@ struct RestoreMasterData : RestoreRoleData, public ReferenceCounted<RestoreMaste
continue;
}
// Finalize the current version batch
//vb.endVersion = nextVersion;
// vb.endVersion = nextVersion;
vb.size = batchSize;
versionBatches->emplace(vb.beginVersion, vb); // copy vb to versionBatch
// start finding the next version batch