Updates lastest backup worker progress after all previous epochs are done

If workers for previous epochs are still ongoing, we may end up with a
container that miss mutations in previous epochs. So the update only happens
after there are only current epoch's backup workers.
This commit is contained in:
Jingyu Zhou 2020-03-12 20:51:10 -07:00
parent 4c75c61f39
commit 9ea549ba7d
1 changed files with 81 additions and 74 deletions

View File

@ -330,23 +330,31 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
// log saved version > snapshot version.
ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
loop {
wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger());
if (self->backups.empty()) {
continue;
while (self->backups.empty() || !self->logSystem.get()) {
wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger() ||
self->logSystem.onChange());
}
// check all workers have started by checking their progress is larger
// than the backup's start version.
state Reference<BackupProgress> progress(new BackupProgress(self->myId, {}));
state Reference<BackupProgress> progress(
new BackupProgress(self->myId, self->logSystem.get()->getOldEpochTagsVersionsInfo()));
wait(getBackupProgress(self->cx, self->myId, progress));
std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
progress->getUnfinishedBackup();
bool finishedPreviousEpochs =
toRecruit.empty() || std::get<0>(toRecruit.begin()->first) == self->recruitedEpoch;
state std::vector<UID> ready;
state std::map<UID, Version> savedLogVersions;
if (tagVersions.size() == self->logSystem.get()->getLogRouterTags()) {
if (tagVersions.size() != self->logSystem.get()->getLogRouterTags()) {
continue;
}
// Check every version is larger than backup's startVersion
for (auto& [uid, info] : self->backups) {
if (info.allWorkerStarted) {
if (info.allWorkerStarted && finishedPreviousEpochs) {
// update update progress so far
Version v = std::numeric_limits<Version>::max();
for (const auto [tag, version] : tagVersions) {
@ -369,7 +377,7 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
}
if (ready.empty() && savedLogVersions.empty()) continue;
// Set "allWorkerStarted" key for ready backups
// Set "allWorkerStarted" and "latestBackupWorkerSavedVersion" key for backups
loop {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
try {
@ -421,7 +429,6 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
}
}
}
}
}
ACTOR Future<Void> saveProgress(BackupData* self, Version backupVersion) {