Updates lastest backup worker progress after all previous epochs are done
If workers for previous epochs are still ongoing, we may end up with a container that miss mutations in previous epochs. So the update only happens after there are only current epoch's backup workers.
This commit is contained in:
parent
4c75c61f39
commit
9ea549ba7d
|
@ -330,23 +330,31 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool started
|
|||
// log saved version > snapshot version.
|
||||
ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
|
||||
loop {
|
||||
wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger());
|
||||
if (self->backups.empty()) {
|
||||
continue;
|
||||
while (self->backups.empty() || !self->logSystem.get()) {
|
||||
wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0) || self->changedTrigger.onTrigger() ||
|
||||
self->logSystem.onChange());
|
||||
}
|
||||
|
||||
// check all workers have started by checking their progress is larger
|
||||
// than the backup's start version.
|
||||
state Reference<BackupProgress> progress(new BackupProgress(self->myId, {}));
|
||||
state Reference<BackupProgress> progress(
|
||||
new BackupProgress(self->myId, self->logSystem.get()->getOldEpochTagsVersionsInfo()));
|
||||
wait(getBackupProgress(self->cx, self->myId, progress));
|
||||
std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
|
||||
std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
|
||||
progress->getUnfinishedBackup();
|
||||
bool finishedPreviousEpochs =
|
||||
toRecruit.empty() || std::get<0>(toRecruit.begin()->first) == self->recruitedEpoch;
|
||||
|
||||
state std::vector<UID> ready;
|
||||
state std::map<UID, Version> savedLogVersions;
|
||||
if (tagVersions.size() == self->logSystem.get()->getLogRouterTags()) {
|
||||
if (tagVersions.size() != self->logSystem.get()->getLogRouterTags()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check every version is larger than backup's startVersion
|
||||
for (auto& [uid, info] : self->backups) {
|
||||
if (info.allWorkerStarted) {
|
||||
if (info.allWorkerStarted && finishedPreviousEpochs) {
|
||||
// update update progress so far
|
||||
Version v = std::numeric_limits<Version>::max();
|
||||
for (const auto [tag, version] : tagVersions) {
|
||||
|
@ -369,7 +377,7 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
|
|||
}
|
||||
if (ready.empty() && savedLogVersions.empty()) continue;
|
||||
|
||||
// Set "allWorkerStarted" key for ready backups
|
||||
// Set "allWorkerStarted" and "latestBackupWorkerSavedVersion" key for backups
|
||||
loop {
|
||||
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
|
||||
try {
|
||||
|
@ -421,7 +429,6 @@ ACTOR Future<Void> monitorAllWorkerProgress(BackupData* self) {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ACTOR Future<Void> saveProgress(BackupData* self, Version backupVersion) {
|
||||
|
|
Loading…
Reference in New Issue