FastRestore: Rename RestoreFile to RestoreFileFR to avoid weird running error
When two struct have the same name but never used in the same scope, the compiler will NOT report any error in compilation, but the program will arbitrarily choose one of the struct at the linker time, and experience weird error in running time. The runtime error is caused by the corrrupted memory when we assign a struct content to a different struct type.
This commit is contained in:
Meng Xu
parent
d68c9ec09e
commit
068ba2e082
|
|
@ -192,6 +192,7 @@ public:
|
|||
Version endVersion; // not meaningful for range files
|
||||
|
||||
Tuple pack() const {
|
||||
fprintf(stderr, "Filename:%s\n", fileName.c_str());
|
||||
return Tuple()
|
||||
.append(version)
|
||||
.append(StringRef(fileName))
|
||||
|
|
@ -357,7 +358,8 @@ ACTOR Future<std::string> RestoreConfig::getProgress_impl(RestoreConfig restore,
|
|||
.detail("FileBlocksInProgress", fileBlocksDispatched.get() - fileBlocksFinished.get())
|
||||
.detail("BytesWritten", bytesWritten.get())
|
||||
.detail("ApplyLag", lag.get())
|
||||
.detail("TaskInstance", THIS_ADDR);
|
||||
.detail("TaskInstance", THIS_ADDR)
|
||||
.backtrace();
|
||||
|
||||
|
||||
return format("Tag: %s UID: %s State: %s Blocks: %lld/%lld BlocksInProgress: %lld Files: %lld BytesWritten: %lld ApplyVersionLag: %lld LastError: %s",
|
||||
|
|
@ -3398,12 +3400,18 @@ namespace fileBackup {
|
|||
|
||||
printf("restorable.get() ranges:%d logs:%d\n", restorable.get().ranges.size(), restorable.get().logs.size());
|
||||
for(const RangeFile &f : restorable.get().ranges) {
|
||||
printf("Add file:%s\n", f.toString().c_str());
|
||||
files.push_back({f.version, f.fileName, true, f.blockSize, f.fileSize});
|
||||
printf("Add file:%s, filename:%s\n", f.toString().c_str(), f.fileName.c_str());
|
||||
RestoreConfig::RestoreFile tmpFile = {f.version, f.fileName, true, f.blockSize, f.fileSize, -1};
|
||||
files.push_back(tmpFile);
|
||||
}
|
||||
for(const LogFile &f : restorable.get().logs) {
|
||||
printf("Add file:%s\n", f.toString().c_str());
|
||||
files.push_back({f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion});
|
||||
printf("Add file:%s filename:%s\n", f.toString().c_str(), f.fileName.c_str());
|
||||
RestoreConfig::RestoreFile tmpFile = {f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion};
|
||||
files.push_back(tmpFile);
|
||||
}
|
||||
|
||||
for (auto& testfile : files) {
|
||||
printf("Files: filename:%d\n", testfile.fileName.c_str());
|
||||
}
|
||||
|
||||
state std::vector<RestoreConfig::RestoreFile>::iterator start = files.begin();
|
||||
|
|
@ -3415,9 +3423,9 @@ namespace fileBackup {
|
|||
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
|
||||
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
|
||||
|
||||
printf("taskBucket->keepRunning start\n");
|
||||
fprintf(stderr, "taskBucket->keepRunning start\n");
|
||||
wait(taskBucket->keepRunning(tr, task));
|
||||
printf("taskBucket->keepRunning end\n");
|
||||
fprintf(stderr, "taskBucket->keepRunning end\n");
|
||||
|
||||
state std::vector<RestoreConfig::RestoreFile>::iterator i = start;
|
||||
|
||||
|
|
@ -3426,16 +3434,19 @@ namespace fileBackup {
|
|||
state int nFiles = 0;
|
||||
auto fileSet = restore.fileSet();
|
||||
for(; i != end && txBytes < 1e6; ++i) {
|
||||
fprintf(stderr, "txBytes:%d\n", txBytes);
|
||||
txBytes += fileSet.insert(tr, *i);
|
||||
nFileBlocks += (i->fileSize + i->blockSize - 1) / i->blockSize;
|
||||
++nFiles;
|
||||
}
|
||||
|
||||
fprintf(stderr, "nFiles:%d nFileBlocks:%d\n", nFiles, nFileBlocks);
|
||||
// Increment counts
|
||||
restore.fileCount().atomicOp(tr, nFiles, MutationRef::Type::AddValue);
|
||||
restore.fileBlockCount().atomicOp(tr, nFileBlocks, MutationRef::Type::AddValue);
|
||||
|
||||
wait(tr->commit());
|
||||
fprintf(stderr, "nFiles:%d nFileBlocks:%d committed\n", nFiles, nFileBlocks);
|
||||
|
||||
TraceEvent("FileRestoreLoadedFiles")
|
||||
.detail("RestoreUID", restore.getUid())
|
||||
|
|
@ -3447,6 +3458,7 @@ namespace fileBackup {
|
|||
start = i;
|
||||
tr->reset();
|
||||
} catch(Error &e) {
|
||||
fprintf(stderr, "Error at FileRestoreLoadedFiles. Error:%s\n", e.what());
|
||||
wait(tr->onError(e));
|
||||
}
|
||||
}
|
||||
|
|
@ -3689,7 +3701,7 @@ public:
|
|||
|
||||
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
|
||||
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
|
||||
printf("[Debug] submitRestore\n");
|
||||
printf("[Debug] submitRestore tag:%s, uid:%s\n", tagName.toString().c_str(), uid.toString().c_str());
|
||||
|
||||
// Get old restore config for this tag
|
||||
state KeyBackedTag tag = makeRestoreTag(tagName.toString());
|
||||
|
|
@ -3735,6 +3747,7 @@ public:
|
|||
Reference<IBackupContainer> bc = IBackupContainer::openContainer(backupURL.toString());
|
||||
|
||||
// Configure the new restore
|
||||
TraceEvent("BARW_RestoreDebug").detail("TagName", tagName.toString()).detail("RestoreUID", uid);
|
||||
restore.tag().set(tr, tagName.toString());
|
||||
restore.sourceContainer().set(tr, bc);
|
||||
restore.stateEnum().set(tr, ERestoreState::QUEUED);
|
||||
|
|
@ -3748,8 +3761,9 @@ public:
|
|||
// this also sets restore.add/removePrefix.
|
||||
restore.initApplyMutations(tr, addPrefix, removePrefix);
|
||||
|
||||
printf("fileBackup::StartFullRestoreTaskFunc::addTask uid:%s\n", uid.toString().c_str());
|
||||
printf("fileBackup::StartFullRestoreTaskFunc::addTask uid:%s starts\n", uid.toString().c_str());
|
||||
Key taskKey = wait(fileBackup::StartFullRestoreTaskFunc::addTask(tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal()));
|
||||
printf("fileBackup::StartFullRestoreTaskFunc::addTask uid:%s finishes\n", uid.toString().c_str());
|
||||
|
||||
if (lockDB)
|
||||
wait(lockDatabase(tr, uid));
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ public:
|
|||
|
||||
// Describes a file to load blocks from during restore. Ordered by version and then fileName to enable
|
||||
// incrementally advancing through the map, saving the version and path of the next starting point.
|
||||
struct RestoreFile {
|
||||
struct RestoreFileFR {
|
||||
Version version;
|
||||
std::string fileName;
|
||||
bool isRange; // false for log file
|
||||
|
|
@ -224,6 +224,7 @@ public:
|
|||
int64_t cursor; //The start block location to be restored. All blocks before cursor have been scheduled to load and restore
|
||||
|
||||
Tuple pack() const {
|
||||
fprintf(stderr, "MyRestoreFile, filename:%s\n", fileName.c_str());
|
||||
return Tuple()
|
||||
.append(version)
|
||||
.append(StringRef(fileName))
|
||||
|
|
@ -234,8 +235,8 @@ public:
|
|||
.append(beginVersion)
|
||||
.append(cursor);
|
||||
}
|
||||
static RestoreFile unpack(Tuple const &t) {
|
||||
RestoreFile r;
|
||||
static RestoreFileFR unpack(Tuple const &t) {
|
||||
RestoreFileFR r;
|
||||
int i = 0;
|
||||
r.version = t.getInt(i++);
|
||||
r.fileName = t.getString(i++).toString();
|
||||
|
|
@ -248,7 +249,7 @@ public:
|
|||
return r;
|
||||
}
|
||||
|
||||
bool operator<(const RestoreFile& rhs) const { return endVersion < rhs.endVersion; }
|
||||
bool operator<(const RestoreFileFR& rhs) const { return endVersion < rhs.endVersion; }
|
||||
|
||||
std::string toString() const {
|
||||
// return "UNSET4TestHardness";
|
||||
|
|
@ -258,7 +259,7 @@ public:
|
|||
}
|
||||
};
|
||||
|
||||
typedef KeyBackedSet<RestoreFile> FileSetT;
|
||||
typedef KeyBackedSet<RestoreFileFR> FileSetT;
|
||||
FileSetT fileSet() {
|
||||
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
||||
}
|
||||
|
|
@ -380,7 +381,7 @@ public:
|
|||
|
||||
};
|
||||
|
||||
typedef RestoreConfig::RestoreFile RestoreFile;
|
||||
typedef RestoreConfig::RestoreFileFR RestoreFileFR;
|
||||
|
||||
// parallelFileRestore is copied from FileBackupAgent.actor.cpp for the same reason as RestoreConfig is copied
|
||||
namespace parallelFileRestore {
|
||||
|
|
@ -746,14 +747,14 @@ struct RestoreData : NonCopyable, public ReferenceCounted<RestoreData> {
|
|||
// TODO: RestoreStatus
|
||||
// Information of the backup files to be restored, and the restore progress
|
||||
struct LoadingStatus {
|
||||
RestoreFile file;
|
||||
RestoreFileFR file;
|
||||
int64_t start; // Starting point of the block in the file to load
|
||||
int64_t length;// Length of block to load
|
||||
LoadingState state; // Loading state of the particular file block
|
||||
UID node; // The loader node ID that responsible for the file block
|
||||
|
||||
explicit LoadingStatus() {}
|
||||
explicit LoadingStatus(RestoreFile file, int64_t start, int64_t length, UID node): file(file), start(start), length(length), state(LoadingState::Init), node(node) {}
|
||||
explicit LoadingStatus(RestoreFileFR file, int64_t start, int64_t length, UID node): file(file), start(start), length(length), state(LoadingState::Init), node(node) {}
|
||||
};
|
||||
std::map<int64_t, LoadingStatus> loadingStatus; // first is the global index of the loading cmd, starting from 0
|
||||
|
||||
|
|
@ -762,8 +763,8 @@ struct RestoreData : NonCopyable, public ReferenceCounted<RestoreData> {
|
|||
std::map<CMDUID, int> processedCmd;
|
||||
|
||||
|
||||
std::vector<RestoreFile> allFiles; // All backup files to be processed in all version batches
|
||||
std::vector<RestoreFile> files; // Backup files to be parsed and applied: range and log files in 1 version batch
|
||||
std::vector<RestoreFileFR> allFiles; // All backup files to be processed in all version batches
|
||||
std::vector<RestoreFileFR> files; // Backup files to be parsed and applied: range and log files in 1 version batch
|
||||
std::map<Version, Version> forbiddenVersions; // forbidden version range [first, second)
|
||||
|
||||
// Temporary data structure for parsing range and log files into (version, <K, V, mutationType>)
|
||||
|
|
@ -1115,7 +1116,7 @@ ACTOR static Future<Void> prepareRestoreFilesV2(Reference<RestoreData> rd, Datab
|
|||
throw restore_missing_data();
|
||||
}
|
||||
|
||||
// state std::vector<RestoreFile> files;
|
||||
// state std::vector<RestoreFileFR> files;
|
||||
if (!rd->files.empty()) {
|
||||
printf("[WARNING] global files are not empty! files.size()=%d. We forcely clear files\n", rd->files.size());
|
||||
rd->files.clear();
|
||||
|
|
@ -1126,13 +1127,13 @@ ACTOR static Future<Void> prepareRestoreFilesV2(Reference<RestoreData> rd, Datab
|
|||
for(const RangeFile &f : restorable.get().ranges) {
|
||||
// TraceEvent("FoundRangeFileMX").detail("FileInfo", f.toString());
|
||||
printf("[INFO] FoundRangeFile, fileInfo:%s\n", f.toString().c_str());
|
||||
RestoreFile file = {f.version, f.fileName, true, f.blockSize, f.fileSize};
|
||||
RestoreFileFR file = {f.version, f.fileName, true, f.blockSize, f.fileSize};
|
||||
rd->files.push_back(file);
|
||||
}
|
||||
for(const LogFile &f : restorable.get().logs) {
|
||||
// TraceEvent("FoundLogFileMX").detail("FileInfo", f.toString());
|
||||
printf("[INFO] FoundLogFile, fileInfo:%s\n", f.toString().c_str());
|
||||
RestoreFile file = {f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion};
|
||||
RestoreFileFR file = {f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion};
|
||||
rd->files.push_back(file);
|
||||
}
|
||||
|
||||
|
|
@ -2404,8 +2405,8 @@ void printRestorableFileSet(Optional<RestorableFileSet> files) {
|
|||
return;
|
||||
}
|
||||
|
||||
std::vector<RestoreFile> getRestoreFiles(Optional<RestorableFileSet> fileSet) {
|
||||
std::vector<RestoreFile> files;
|
||||
std::vector<RestoreFileFR> getRestoreFiles(Optional<RestorableFileSet> fileSet) {
|
||||
std::vector<RestoreFileFR> files;
|
||||
|
||||
for(const RangeFile &f : fileSet.get().ranges) {
|
||||
files.push_back({f.version, f.fileName, true, f.blockSize, f.fileSize});
|
||||
|
|
@ -2429,7 +2430,7 @@ ACTOR static Future<Void> collectBackupFiles(Reference<RestoreData> rd, Database
|
|||
state Key removePrefix = request.removePrefix;
|
||||
state bool lockDB = request.lockDB;
|
||||
state UID randomUid = request.randomUid;
|
||||
//state VectorRef<RestoreFile> files; // return result
|
||||
//state VectorRef<RestoreFileFR> files; // return result
|
||||
|
||||
ASSERT( lockDB == true );
|
||||
|
||||
|
|
@ -2460,13 +2461,13 @@ ACTOR static Future<Void> collectBackupFiles(Reference<RestoreData> rd, Database
|
|||
for(const RangeFile &f : restorable.get().ranges) {
|
||||
TraceEvent("FoundRangeFileMX").detail("FileInfo", f.toString());
|
||||
printf("[INFO] FoundRangeFile, fileInfo:%s\n", f.toString().c_str());
|
||||
RestoreFile file = {f.version, f.fileName, true, f.blockSize, f.fileSize, 0};
|
||||
RestoreFileFR file = {f.version, f.fileName, true, f.blockSize, f.fileSize, 0};
|
||||
rd->files.push_back(file);
|
||||
}
|
||||
for(const LogFile &f : restorable.get().logs) {
|
||||
TraceEvent("FoundLogFileMX").detail("FileInfo", f.toString());
|
||||
printf("[INFO] FoundLogFile, fileInfo:%s\n", f.toString().c_str());
|
||||
RestoreFile file = {f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion, 0};
|
||||
RestoreFileFR file = {f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion, 0};
|
||||
rd->files.push_back(file);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -561,8 +561,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
|
|||
}));
|
||||
|
||||
// restore database
|
||||
TraceEvent("BARW_Restore", randomID).detail("LastBackupContainer", lastBackupContainer->getURL()).detail("RestoreAfter", self->restoreAfter).detail("BackupTag", printable(self->backupTag));
|
||||
printf("MX:BARW_Restore, LastBackupContainer url:%s BackupTag:%s\n",lastBackupContainer->getURL().c_str(), printable(self->backupTag).c_str() );
|
||||
TraceEvent("BAFRW_Restore", randomID).detail("LastBackupContainer", lastBackupContainer->getURL()).detail("RestoreAfter", self->restoreAfter).detail("BackupTag", printable(self->backupTag));
|
||||
printf("MX:BAFRW_Restore, LastBackupContainer url:%s BackupTag:%s\n",lastBackupContainer->getURL().c_str(), printable(self->backupTag).c_str() );
|
||||
|
||||
auto container = IBackupContainer::openContainer(lastBackupContainer->getURL());
|
||||
BackupDescription desc = wait( container->describeBackup() );
|
||||
|
|
|
|||
|
|
@ -457,6 +457,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
|
|||
}
|
||||
}
|
||||
|
||||
TraceEvent("BARW_RestoreDebug").detail("TargetVersion", targetVersion);
|
||||
|
||||
state std::vector<Future<Version>> restores;
|
||||
state std::vector<Standalone<StringRef>> restoreTags;
|
||||
state bool multipleRangesInOneTag = false;
|
||||
|
|
@ -466,6 +468,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
|
|||
auto range = self->restoreRanges[restoreIndex];
|
||||
Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" + std::to_string(restoreIndex));
|
||||
restoreTags.push_back(restoreTag);
|
||||
printf("BackupCorrectness, restore for each range: backupAgent.restore is called for restoreIndex:%d tag:%s ranges:%s\n", restoreIndex, range.toString().c_str(), restoreTag.toString().c_str());
|
||||
restores.push_back(backupAgent.restore(cx, cx, restoreTag, KeyRef(lastBackupContainer->getURL()), true, targetVersion, true, range, Key(), Key(), self->locked));
|
||||
}
|
||||
}
|
||||
|
|
@ -473,7 +476,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
|
|||
multipleRangesInOneTag = true;
|
||||
Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" + std::to_string(restoreIndex));
|
||||
restoreTags.push_back(restoreTag);
|
||||
printf("BackupCorrectness, backupAgent.restore is called for restoreIndex:%d\n", restoreIndex);
|
||||
printf("BackupCorrectness, backupAgent.restore is called for restoreIndex:%d tag:%s\n", restoreIndex, restoreTag.toString().c_str());
|
||||
restores.push_back(backupAgent.restore(cx, cx, restoreTag, KeyRef(lastBackupContainer->getURL()), self->restoreRanges, true, targetVersion, true, Key(), Key(), self->locked));
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue