wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove getMinReplicasRemaining PromiseStream. 

Instead, in order to enforce the maximum fault tolerance for snapshots,
update getStorageWorkers to return the number of unavailable storage
servers (instead of throwing an error when unavailable storage servers
exist).
This commit is contained in:
sfc-gh-tclinkenbeard 2022-04-07 23:23:23 -07:00
parent 70f378bacc
commit 91930b8040
6 changed files with 36 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
fdbserver/DataDistribution.actor.cpp
View File

@ -516,7 +516,6 @@ ACTOR Future<Void> monitorBatchLimitedTime(Reference<AsyncVar<ServerDBInfo> cons
// Runs the data distribution algorithm for FDB, including the DD Queue, DD tracker, and DD team collection // Runs the data distribution algorithm for FDB, including the DD Queue, DD tracker, and DD team collection
ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self, ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
PromiseStream<GetMetricsListRequest> getShardMetricsList, PromiseStream<GetMetricsListRequest> getShardMetricsList,
FutureStream<Promise<Optional<int>>> getMinReplicasRemaining,
const DDEnabledState* ddEnabledState) { const DDEnabledState* ddEnabledState) {
state double lastLimited = 0; state double lastLimited = 0;
self->addActor.send(monitorBatchLimitedTime(self->dbInfo, &lastLimited)); self->addActor.send(monitorBatchLimitedTime(self->dbInfo, &lastLimited));
@ -755,7 +754,6 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
lock, lock,
getAverageShardBytes, getAverageShardBytes,
getUnhealthyRelocationCount.getFuture(), getUnhealthyRelocationCount.getFuture(),
getMinReplicasRemaining,
self->ddId, self->ddId,
storageTeamSize, storageTeamSize,
configuration.storageTeamSize, configuration.storageTeamSize,
@ -915,24 +913,6 @@ Future<ErrorOr<Void>> trySendSnapReq(RequestStream<Req> stream, Req req) {
return ErrorOr<Void>(Void()); return ErrorOr<Void>(Void());
} }
// Returns the number of storage snapshot failures we can ignore while still still successfully snapshotting storage
// servers
ACTOR static Future<int> getTolerableFailedStorageSnapshots(
Database cx,
PromiseStream<Promise<Optional<int>>> getMinReplicasRemaining) {
Promise<Optional<int>> minReplicasRemainingPromise;
getMinReplicasRemaining.send(minReplicasRemainingPromise);
state Optional<int> minReplicasRemaining = wait(minReplicasRemainingPromise.getFuture());
DatabaseConfiguration configuration = wait(getDatabaseConfiguration(cx));
auto faultTolerance =
std::min<int>(SERVER_KNOBS->MAX_STORAGE_SNAPSHOT_FAULT_TOLERANCE, configuration.storageTeamSize);
if (minReplicasRemaining.present()) {
TEST(minReplicasRemaining.get() == 0); // Some data has 0 replicas across all storage servers
return std::min<int>(faultTolerance, std::max(minReplicasRemaining.get() - 1, 0));
}
return faultTolerance;
}
ACTOR static Future<Void> waitForMost(std::vector<Future<ErrorOr<Void>>> futures, ACTOR static Future<Void> waitForMost(std::vector<Future<ErrorOr<Void>>> futures,
int faultTolerance, int faultTolerance,
Error e, Error e,
@ -951,9 +931,7 @@ ACTOR static Future<Void> waitForMost(std::vector<Future<ErrorOr<Void>>> futures
return Void(); return Void();
} }
ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<AsyncVar<ServerDBInfo> const> db) {
Reference<AsyncVar<ServerDBInfo> const> db,
PromiseStream<Promise<Optional<int>>> getMinReplicasRemaining) {
state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True); state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True);
state ReadYourWritesTransaction tr(cx); state ReadYourWritesTransaction tr(cx);
@ -990,9 +968,16 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq,
.detail("SnapPayload", snapReq.snapPayload) .detail("SnapPayload", snapReq.snapPayload)
.detail("SnapUID", snapReq.snapUID); .detail("SnapUID", snapReq.snapUID);
// snap local storage nodes // snap local storage nodes
state int storageFaultTolerance = wait(getTolerableFailedStorageSnapshots(cx, getMinReplicasRemaining)); state DatabaseConfiguration configuration = wait(getDatabaseConfiguration(cx));
std::vector<WorkerInterface> storageWorkers = std::pair<std::vector<WorkerInterface>, int> storageWorkersAndFailures =
wait(transformErrors(getStorageWorkers(cx, db, true /* localOnly */), snap_storage_failed())); wait(transformErrors(getStorageWorkers(cx, db, true /* localOnly */), snap_storage_failed()));
const auto& [storageWorkers, storageFailures] = storageWorkersAndFailures;
auto const storageFaultTolerance =
static_cast<int>(SERVER_KNOBS->MAX_STORAGE_SNAPSHOT_FAULT_TOLERANCE) - storageFailures;
if (storageFaultTolerance < 0) {
TEST(true); // Too many failed storage servers to complete snapshot
throw snap_storage_failed();
}
TraceEvent("SnapDataDistributor_GotStorageWorkers") TraceEvent("SnapDataDistributor_GotStorageWorkers")
.detail("SnapPayload", snapReq.snapPayload) .detail("SnapPayload", snapReq.snapPayload)
.detail("SnapUID", snapReq.snapUID); .detail("SnapUID", snapReq.snapUID);
@ -1095,7 +1080,6 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq,
ACTOR Future<Void> ddSnapCreate(DistributorSnapRequest snapReq, ACTOR Future<Void> ddSnapCreate(DistributorSnapRequest snapReq,
Reference<AsyncVar<ServerDBInfo> const> db, Reference<AsyncVar<ServerDBInfo> const> db,
PromiseStream<Promise<Optional<int>>> getMinReplicasRemaining,
DDEnabledState* ddEnabledState) { DDEnabledState* ddEnabledState) {
state Future<Void> dbInfoChange = db->onChange(); state Future<Void> dbInfoChange = db->onChange();
if (!ddEnabledState->setDDEnabled(false, snapReq.snapUID)) { if (!ddEnabledState->setDDEnabled(false, snapReq.snapUID)) {
@ -1114,7 +1098,7 @@ ACTOR Future<Void> ddSnapCreate(DistributorSnapRequest snapReq,
.detail("SnapUID", snapReq.snapUID); .detail("SnapUID", snapReq.snapUID);
snapReq.reply.sendError(snap_with_recovery_unsupported()); snapReq.reply.sendError(snap_with_recovery_unsupported());
} }
when(wait(ddSnapCreateCore(snapReq, db, getMinReplicasRemaining))) { when(wait(ddSnapCreateCore(snapReq, db))) {
TraceEvent("SnapDDCreateSuccess") TraceEvent("SnapDDCreateSuccess")
.detail("SnapPayload", snapReq.snapPayload) .detail("SnapPayload", snapReq.snapPayload)
.detail("SnapUID", snapReq.snapUID); .detail("SnapUID", snapReq.snapUID);
@ -1269,7 +1253,6 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
state Reference<DataDistributorData> self(new DataDistributorData(db, di.id())); state Reference<DataDistributorData> self(new DataDistributorData(db, di.id()));
state Future<Void> collection = actorCollection(self->addActor.getFuture()); state Future<Void> collection = actorCollection(self->addActor.getFuture());
state PromiseStream<GetMetricsListRequest> getShardMetricsList; state PromiseStream<GetMetricsListRequest> getShardMetricsList;
state PromiseStream<Promise<Optional<int>>> getMinReplicasRemaining;
state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True); state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True);
state ActorCollection actors(false); state ActorCollection actors(false);
state DDEnabledState ddEnabledState; state DDEnabledState ddEnabledState;
@ -1280,11 +1263,11 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
TraceEvent("DataDistributorRunning", di.id()); TraceEvent("DataDistributorRunning", di.id());
self->addActor.send(waitFailureServer(di.waitFailure.getFuture())); self->addActor.send(waitFailureServer(di.waitFailure.getFuture()));
self->addActor.send(cacheServerWatcher(&cx)); self->addActor.send(cacheServerWatcher(&cx));
state Future<Void> distributor = reportErrorsExcept( state Future<Void> distributor =
dataDistribution(self, getShardMetricsList, getMinReplicasRemaining.getFuture(), &ddEnabledState), reportErrorsExcept(dataDistribution(self, getShardMetricsList, &ddEnabledState),
"DataDistribution", "DataDistribution",
di.id(), di.id(),
&normalDataDistributorErrors()); &normalDataDistributorErrors());
loop choose { loop choose {
when(wait(distributor || collection)) { when(wait(distributor || collection)) {
@ -1300,7 +1283,7 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
actors.add(ddGetMetrics(req, getShardMetricsList)); actors.add(ddGetMetrics(req, getShardMetricsList));
} }
when(DistributorSnapRequest snapReq = waitNext(di.distributorSnapReq.getFuture())) { when(DistributorSnapRequest snapReq = waitNext(di.distributorSnapReq.getFuture())) {
actors.add(ddSnapCreate(snapReq, db, getMinReplicasRemaining, &ddEnabledState)); actors.add(ddSnapCreate(snapReq, db, &ddEnabledState));
} }
when(DistributorExclusionSafetyCheckRequest exclCheckReq = when(DistributorExclusionSafetyCheckRequest exclCheckReq =
waitNext(di.distributorExclCheckReq.getFuture())) { waitNext(di.distributorExclCheckReq.getFuture())) {

1
fdbserver/DataDistribution.actor.h
View File

@ -268,7 +268,6 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
MoveKeysLock lock, MoveKeysLock lock,
PromiseStream<Promise<int64_t>> getAverageShardBytes, PromiseStream<Promise<int64_t>> getAverageShardBytes,
FutureStream<Promise<int>> getUnhealthyRelocationCount, FutureStream<Promise<int>> getUnhealthyRelocationCount,
FutureStream<Promise<Optional<int>>> getMinRemainingReplicas,
UID distributorId, UID distributorId,
int teamSize, int teamSize,
int singleRegionTeamSize, int singleRegionTeamSize,

21
fdbserver/DataDistributionQueue.actor.cpp
View File

@ -1037,21 +1037,6 @@ struct DDQueueData {
} }
return highestPriority; return highestPriority;
} }
// Returns the minimum number of replicas remaining on any team.
// If no replicas are missing, return Optional<int>{}, indicating that
// we can fall back to DatabaseConfiguration::storageTeamSize
Optional<int> getMinReplicasRemaining() const {
auto const highestPriority = getHighestPriorityRelocation();
if (highestPriority >= SERVER_KNOBS->PRIORITY_TEAM_0_LEFT) {
return 0;
} else if (highestPriority >= SERVER_KNOBS->PRIORITY_TEAM_1_LEFT) {
return 1;
} else if (highestPriority >= SERVER_KNOBS->PRIORITY_TEAM_2_LEFT) {
return 2;
}
return {};
}
}; };
static std::string destServersString(std::vector<std::pair<Reference<IDataDistributionTeam>, bool>> const& bestTeams) { static std::string destServersString(std::vector<std::pair<Reference<IDataDistributionTeam>, bool>> const& bestTeams) {
@ -1723,7 +1708,6 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
MoveKeysLock lock, MoveKeysLock lock,
PromiseStream<Promise<int64_t>> getAverageShardBytes, PromiseStream<Promise<int64_t>> getAverageShardBytes,
FutureStream<Promise<int>> getUnhealthyRelocationCount, FutureStream<Promise<int>> getUnhealthyRelocationCount,
FutureStream<Promise<Optional<int>>> getMinReplicasRemaining,
UID distributorId, UID distributorId,
int teamSize, int teamSize,
int singleRegionTeamSize, int singleRegionTeamSize,
@ -1851,12 +1835,9 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
// DataDistributorData::movingDataEventHolder. The track latest key // DataDistributorData::movingDataEventHolder. The track latest key
// we use here must match the key used in the holder. // we use here must match the key used in the holder.
} }
when(Promise<Optional<int>> r = waitNext(getMinReplicasRemaining)) {
r.send(self.getMinReplicasRemaining());
}
when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) { r.send(self.unhealthyRelocations); }
when(wait(self.error.getFuture())) {} // Propagate errors from dataDistributionRelocator when(wait(self.error.getFuture())) {} // Propagate errors from dataDistributionRelocator
when(wait(waitForAll(balancingFutures))) {} when(wait(waitForAll(balancingFutures))) {}
when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) { r.send(self.unhealthyRelocations); }
} }
} }
} catch (Error& e) { } catch (Error& e) {

13
fdbserver/QuietDatabase.actor.cpp
View File

@ -277,9 +277,8 @@ ACTOR Future<std::vector<StorageServerInterface>> getStorageServers(Database cx,
} }
} }
ACTOR Future<std::vector<WorkerInterface>> getStorageWorkers(Database cx, ACTOR Future<std::pair<std::vector<WorkerInterface>, int>>
Reference<AsyncVar<ServerDBInfo> const> dbInfo, getStorageWorkers(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo, bool localOnly) {
bool localOnly) {
state std::vector<StorageServerInterface> servers = wait(getStorageServers(cx)); state std::vector<StorageServerInterface> servers = wait(getStorageServers(cx));
state std::map<NetworkAddress, WorkerInterface> workersMap; state std::map<NetworkAddress, WorkerInterface> workersMap;
std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo)); std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo));
@ -299,7 +298,9 @@ ACTOR Future<std::vector<WorkerInterface>> getStorageWorkers(Database cx,
} }
auto masterDcId = dbInfo->get().master.locality.dcId(); auto masterDcId = dbInfo->get().master.locality.dcId();
std::vector<WorkerInterface> result; std::pair<std::vector<WorkerInterface>, int> result;
auto& [workerInterfaces, failures] = result;
failures = 0;
for (const auto& server : servers) { for (const auto& server : servers) {
TraceEvent(SevDebug, "DcIdInfo") TraceEvent(SevDebug, "DcIdInfo")
.detail("ServerLocalityID", server.locality.dcId()) .detail("ServerLocalityID", server.locality.dcId())
@ -310,9 +311,9 @@ ACTOR Future<std::vector<WorkerInterface>> getStorageWorkers(Database cx,
TraceEvent(SevWarn, "GetStorageWorkers") TraceEvent(SevWarn, "GetStorageWorkers")
.detail("Reason", "Could not find worker for storage server") .detail("Reason", "Could not find worker for storage server")
.detail("SS", server.id()); .detail("SS", server.id());
throw operation_failed(); ++failures;
} }
result.push_back(itr->second); workerInterfaces.push_back(itr->second);
} }
} }
return result; return result;

8
fdbserver/QuietDatabase.h
View File

@ -46,9 +46,11 @@ Future<WorkerInterface> getMasterWorker(Database const& cx, Reference<AsyncVar<S
Future<Void> repairDeadDatacenter(Database const& cx, Future<Void> repairDeadDatacenter(Database const& cx,
Reference<AsyncVar<ServerDBInfo> const> const& dbInfo, Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
std::string const& context); std::string const& context);
Future<std::vector<WorkerInterface>> getStorageWorkers(Database const& cx,
Reference<AsyncVar<ServerDBInfo> const> const& dbInfo, // Returns list of worker interfaces for available storage servers and the number of unavailable
bool const& localOnly); // storage servers
Future<std::pair<std::vector<WorkerInterface>, int>>
getStorageWorkers(Database const& cx, Reference<AsyncVar<ServerDBInfo> const> const& dbInfo, bool const& localOnly);
Future<std::vector<WorkerInterface>> getCoordWorkers(Database const& cx, Future<std::vector<WorkerInterface>> getCoordWorkers(Database const& cx,
Reference<AsyncVar<ServerDBInfo> const> const& dbInfo); Reference<AsyncVar<ServerDBInfo> const> const& dbInfo);

8
fdbserver/workloads/DiskFailureInjection.actor.cpp
View File

@ -154,9 +154,13 @@ struct DiskFailureInjectionWorkload : TestWorkload {
loop { loop {
wait(poisson(&lastTime, 1)); wait(poisson(&lastTime, 1));
try { try {
wait(store(machines, getStorageWorkers(cx, self->dbInfo, false))); std::pair<std::vector<W>, int> m = wait(getStorageWorkers(cx, self->dbInfo, false));
if (m.second > 0) {
throw operation_failed();
}
machines = std::move(m.first);
} catch (Error& e) { } catch (Error& e) {
// If we failed to get a list of storage servers, we can't inject failure events // If we failed to get a complete list of storage servers, we can't inject failure events
// But don't throw the error in that case // But don't throw the error in that case
continue; continue;
} }