diff --git a/fdbclient/include/fdbclient/Audit.h b/fdbclient/include/fdbclient/Audit.h index b366a604e2..55c4fb368c 100644 --- a/fdbclient/include/fdbclient/Audit.h +++ b/fdbclient/include/fdbclient/Audit.h @@ -73,12 +73,13 @@ struct AuditStorageRequest { template void serialize(Ar& ar) { - serializer(ar, id, range, type, reply); + serializer(ar, id, range, type, targetServers, reply); } UID id; KeyRange range; uint8_t type; + std::vector targetServers; ReplyPromise reply; }; diff --git a/fdbserver/DDTxnProcessor.actor.cpp b/fdbserver/DDTxnProcessor.actor.cpp index d9ca9aeca5..104a32451b 100644 --- a/fdbserver/DDTxnProcessor.actor.cpp +++ b/fdbserver/DDTxnProcessor.actor.cpp @@ -132,7 +132,7 @@ class DDTxnProcessorImpl { continue; } const StorageServerInterface ssi = decodeServerListValue(serverListValue.get()); - current.servers.push_back(ssi); + current.servers[ssi.locality.describeDcId()].push_back(ssi); } res.push_back(current); } diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index 67321f024f..76197d1355 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -1344,7 +1344,7 @@ ACTOR Future scheduleAuditForRange(Reference self, UID auditId, KeyRange range, AuditType type) { - TraceEvent(SevDebug, "DDScheduleAuditBegin", auditId).detail("Range", range).detail("AuditType", type); + TraceEvent(SevDebug, "DDScheduleAuditForRangeBegin", auditId).detail("Range", range).detail("AuditType", type); // TODO(heliu): Load the audit map for `range`. state Key begin = range.begin; state KeyRange currentRange = range; @@ -1370,9 +1370,18 @@ ACTOR Future scheduleAuditForRange(Reference self, state int i = 0; for (i = 0; i < rangeLocations.size(); ++i) { - const AuditStorageRequest req(auditId, rangeLocations[i].range, type); - const int idx = deterministicRandom()->randomInt(0, rangeLocations[i].servers.size()); - actors->add(doAuditStorage(self, actors, auditMap, rangeLocations[i].servers[idx], req)); + AuditStorageRequest req(auditId, rangeLocations[i].range, type); + if (type == AuditType::ValidateHA && rangeLocations[i].servers.size() >= 2) { + auto it = rangeLocations[i].servers.begin(); + const int idx = deterministicRandom()->randomInt(0, it->second.size()); + StorageServerInterface& targetServer = it->second[idx]; + ++it; + for (; it != rangeLocations[i].servers.end(); ++it) { + const int idx = deterministicRandom()->randomInt(0, it->second.size()); + req.targetServers.push_back(it->second[idx].id()); + } + actors->add(doAuditStorage(self, actors, auditMap, targetServer, req)); + } begin = rangeLocations[i].range.end; wait(delay(0.01)); } @@ -1395,7 +1404,8 @@ ACTOR Future doAuditStorage(Reference self, TraceEvent(SevDebug, "DDAuditStorageBegin", req.id) .detail("Range", req.range) .detail("AuditType", req.type) - .detail("StorageServer", ssi.toString()); + .detail("StorageServer", ssi.toString()) + .detail("TargetServers", describe(req.targetServers)); try { auditMap->insert(req.range, AuditPhase::Running); diff --git a/fdbserver/include/fdbserver/DDTxnProcessor.h b/fdbserver/include/fdbserver/DDTxnProcessor.h index bf9fb36ad8..24fc34e0ac 100644 --- a/fdbserver/include/fdbserver/DDTxnProcessor.h +++ b/fdbserver/include/fdbserver/DDTxnProcessor.h @@ -41,7 +41,8 @@ public: StorageServersForRange() = default; StorageServersForRange(KeyRangeRef range) : range(range) {} - std::vector servers; + // A map of dcId : list of servers + std::map> servers; KeyRange range; };