Merge pull request #11309 from apple/sevwarn

Raise visibility of gray failure actions
This commit is contained in:
Jingyu Zhou 2024-04-17 09:08:40 -07:00 committed by GitHub
commit 84d6b86715
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 4 deletions

View File

@ -2989,19 +2989,19 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
self->excludedDegradedServers = self->degradationInfo.degradedServers;
self->excludedDegradedServers.insert(self->degradationInfo.disconnectedServers.begin(),
self->degradationInfo.disconnectedServers.end());
TraceEvent("DegradedServerDetectedAndTriggerRecovery")
TraceEvent(SevWarnAlways, "DegradedServerDetectedAndTriggerRecovery")
.detail("RecentRecoveryCountDueToHealth", self->recentRecoveryCountDueToHealth());
self->db.forceMasterFailure.trigger();
}
} else {
self->excludedDegradedServers.clear();
TraceEvent("DegradedServerDetectedAndSuggestRecovery").log();
TraceEvent(SevWarn, "DegradedServerDetectedAndSuggestRecovery").log();
}
} else if (self->shouldTriggerFailoverDueToDegradedServers()) {
double ccUpTime = now() - machineStartTime();
if (SERVER_KNOBS->CC_HEALTH_TRIGGER_FAILOVER &&
ccUpTime > SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY) {
TraceEvent("DegradedServerDetectedAndTriggerFailover").log();
TraceEvent(SevWarn, "DegradedServerDetectedAndTriggerFailover").log();
std::vector<Optional<Key>> dcPriority;
auto remoteDcId = self->db.config.regions[0].dcId == self->clusterControllerDcId.get()
? self->db.config.regions[1].dcId
@ -3013,7 +3013,7 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
dcPriority.push_back(self->clusterControllerDcId);
self->desiredDcIds.set(dcPriority);
} else {
TraceEvent("DegradedServerDetectedAndSuggestFailover").detail("CCUpTime", ccUpTime);
TraceEvent(SevWarn, "DegradedServerDetectedAndSuggestFailover").detail("CCUpTime", ccUpTime);
}
}
}