wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove getMinReplicasRemaining PromiseStream. 

Instead, in order to enforce the maximum fault tolerance for snapshots,
update getStorageWorkers to return the number of unavailable storage
servers (instead of throwing an error when unavailable storage servers
exist).
This commit is contained in:
sfc-gh-tclinkenbeard 2022-04-07 23:23:23 -07:00
parent 70f378bacc
commit 91930b8040
6 changed files with 36 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
fdbserver/DataDistribution.actor.cpp
View File

@ -516,7 +516,6 @@ ACTOR Future<Void> monitorBatchLimitedTime(Reference<AsyncVar<ServerDBInfo> cons
// Runs the data distribution algorithm for FDB, including the DD Queue, DD tracker, and DD team collection
ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
PromiseStream<GetMetricsListRequest> getShardMetricsList,
FutureStream<Promise<Optional<int>>> getMinReplicasRemaining,
const DDEnabledState* ddEnabledState) {
state double lastLimited = 0;
self->addActor.send(monitorBatchLimitedTime(self->dbInfo, &lastLimited));
@ -755,7 +754,6 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
lock,
getAverageShardBytes,
getUnhealthyRelocationCount.getFuture(),
getMinReplicasRemaining,
self->ddId,
storageTeamSize,
configuration.storageTeamSize,
@ -915,24 +913,6 @@ Future<ErrorOr<Void>> trySendSnapReq(RequestStream<Req> stream, Req req) {
return ErrorOr<Void>(Void());
}
// Returns the number of storage snapshot failures we can ignore while still still successfully snapshotting storage
// servers
ACTOR static Future<int> getTolerableFailedStorageSnapshots(
Database cx,
PromiseStream<Promise<Optional<int>>> getMinReplicasRemaining) {
Promise<Optional<int>> minReplicasRemainingPromise;
getMinReplicasRemaining.send(minReplicasRemainingPromise);
state Optional<int> minReplicasRemaining = wait(minReplicasRemainingPromise.getFuture());
DatabaseConfiguration configuration = wait(getDatabaseConfiguration(cx));
auto faultTolerance =
std::min<int>(SERVER_KNOBS->MAX_STORAGE_SNAPSHOT_FAULT_TOLERANCE, configuration.storageTeamSize);
if (minReplicasRemaining.present()) {
TEST(minReplicasRemaining.get() == 0); // Some data has 0 replicas across all storage servers
return std::min<int>(faultTolerance, std::max(minReplicasRemaining.get() - 1, 0));
}
return faultTolerance;
}
ACTOR static Future<Void> waitForMost(std::vector<Future<ErrorOr<Void>>> futures,
int faultTolerance,
Error e,
@ -951,9 +931,7 @@ ACTOR static Future<Void> waitForMost(std::vector<Future<ErrorOr<Void>>> futures
return Void();
}
ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq,
Reference<AsyncVar<ServerDBInfo> const> db,
PromiseStream<Promise<Optional<int>>> getMinReplicasRemaining) {
ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<AsyncVar<ServerDBInfo> const> db) {
state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True);
state ReadYourWritesTransaction tr(cx);
@ -990,9 +968,16 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq,
.detail("SnapPayload", snapReq.snapPayload)
.detail("SnapUID", snapReq.snapUID);
// snap local storage nodes
state int storageFaultTolerance = wait(getTolerableFailedStorageSnapshots(cx, getMinReplicasRemaining));
std::vector<WorkerInterface> storageWorkers =
state DatabaseConfiguration configuration = wait(getDatabaseConfiguration(cx));
std::pair<std::vector<WorkerInterface>, int> storageWorkersAndFailures =
wait(transformErrors(getStorageWorkers(cx, db, true /* localOnly */), snap_storage_failed()));
const auto& [storageWorkers, storageFailures] = storageWorkersAndFailures;
auto const storageFaultTolerance =
static_cast<int>(SERVER_KNOBS->MAX_STORAGE_SNAPSHOT_FAULT_TOLERANCE) - storageFailures;
if (storageFaultTolerance < 0) {
TEST(true); // Too many failed storage servers to complete snapshot
throw snap_storage_failed();
}
TraceEvent("SnapDataDistributor_GotStorageWorkers")
.detail("SnapPayload", snapReq.snapPayload)
.detail("SnapUID", snapReq.snapUID);
@ -1095,7 +1080,6 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq,
ACTOR Future<Void> ddSnapCreate(DistributorSnapRequest snapReq,
Reference<AsyncVar<ServerDBInfo> const> db,
PromiseStream<Promise<Optional<int>>> getMinReplicasRemaining,
DDEnabledState* ddEnabledState) {
state Future<Void> dbInfoChange = db->onChange();
if (!ddEnabledState->setDDEnabled(false, snapReq.snapUID)) {
@ -1114,7 +1098,7 @@ ACTOR Future<Void> ddSnapCreate(DistributorSnapRequest snapReq,
.detail("SnapUID", snapReq.snapUID);
snapReq.reply.sendError(snap_with_recovery_unsupported());
}
when(wait(ddSnapCreateCore(snapReq, db, getMinReplicasRemaining))) {
when(wait(ddSnapCreateCore(snapReq, db))) {
TraceEvent("SnapDDCreateSuccess")
.detail("SnapPayload", snapReq.snapPayload)
.detail("SnapUID", snapReq.snapUID);
@ -1269,7 +1253,6 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
state Reference<DataDistributorData> self(new DataDistributorData(db, di.id()));
state Future<Void> collection = actorCollection(self->addActor.getFuture());
state PromiseStream<GetMetricsListRequest> getShardMetricsList;
state PromiseStream<Promise<Optional<int>>> getMinReplicasRemaining;
state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True);
state ActorCollection actors(false);
state DDEnabledState ddEnabledState;
@ -1280,11 +1263,11 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
TraceEvent("DataDistributorRunning", di.id());
self->addActor.send(waitFailureServer(di.waitFailure.getFuture()));
self->addActor.send(cacheServerWatcher(&cx));
state Future<Void> distributor = reportErrorsExcept(
dataDistribution(self, getShardMetricsList, getMinReplicasRemaining.getFuture(), &ddEnabledState),
"DataDistribution",
di.id(),
&normalDataDistributorErrors());
state Future<Void> distributor =
reportErrorsExcept(dataDistribution(self, getShardMetricsList, &ddEnabledState),
"DataDistribution",
di.id(),
&normalDataDistributorErrors());
loop choose {
when(wait(distributor || collection)) {
@ -1300,7 +1283,7 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
actors.add(ddGetMetrics(req, getShardMetricsList));
}
when(DistributorSnapRequest snapReq = waitNext(di.distributorSnapReq.getFuture())) {
actors.add(ddSnapCreate(snapReq, db, getMinReplicasRemaining, &ddEnabledState));
actors.add(ddSnapCreate(snapReq, db, &ddEnabledState));
}
when(DistributorExclusionSafetyCheckRequest exclCheckReq =
waitNext(di.distributorExclCheckReq.getFuture())) {

1
fdbserver/DataDistribution.actor.h
View File

@ -268,7 +268,6 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
MoveKeysLock lock,
PromiseStream<Promise<int64_t>> getAverageShardBytes,
FutureStream<Promise<int>> getUnhealthyRelocationCount,
FutureStream<Promise<Optional<int>>> getMinRemainingReplicas,
UID distributorId,
int teamSize,
int singleRegionTeamSize,

21
fdbserver/DataDistributionQueue.actor.cpp
View File

@ -1037,21 +1037,6 @@ struct DDQueueData {
}
return highestPriority;
}
// Returns the minimum number of replicas remaining on any team.
// If no replicas are missing, return Optional<int>{}, indicating that
// we can fall back to DatabaseConfiguration::storageTeamSize
Optional<int> getMinReplicasRemaining() const {
auto const highestPriority = getHighestPriorityRelocation();
if (highestPriority >= SERVER_KNOBS->PRIORITY_TEAM_0_LEFT) {
return 0;
} else if (highestPriority >= SERVER_KNOBS->PRIORITY_TEAM_1_LEFT) {
return 1;
} else if (highestPriority >= SERVER_KNOBS->PRIORITY_TEAM_2_LEFT) {
return 2;
}
return {};
}
};
static std::string destServersString(std::vector<std::pair<Reference<IDataDistributionTeam>, bool>> const& bestTeams) {
@ -1723,7 +1708,6 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
MoveKeysLock lock,
PromiseStream<Promise<int64_t>> getAverageShardBytes,
FutureStream<Promise<int>> getUnhealthyRelocationCount,
FutureStream<Promise<Optional<int>>> getMinReplicasRemaining,
UID distributorId,
int teamSize,
int singleRegionTeamSize,
@ -1851,12 +1835,9 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
// DataDistributorData::movingDataEventHolder. The track latest key
// we use here must match the key used in the holder.
}
when(Promise<Optional<int>> r = waitNext(getMinReplicasRemaining)) {
r.send(self.getMinReplicasRemaining());
}
when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) { r.send(self.unhealthyRelocations); }
when(wait(self.error.getFuture())) {} // Propagate errors from dataDistributionRelocator
when(wait(waitForAll(balancingFutures))) {}
when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) { r.send(self.unhealthyRelocations); }
}
}
} catch (Error& e) {

13
fdbserver/QuietDatabase.actor.cpp
View File

@ -277,9 +277,8 @@ ACTOR Future<std::vector<StorageServerInterface>> getStorageServers(Database cx,
}
}
ACTOR Future<std::vector<WorkerInterface>> getStorageWorkers(Database cx,
Reference<AsyncVar<ServerDBInfo> const> dbInfo,
bool localOnly) {
ACTOR Future<std::pair<std::vector<WorkerInterface>, int>>
getStorageWorkers(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo, bool localOnly) {
state std::vector<StorageServerInterface> servers = wait(getStorageServers(cx));
state std::map<NetworkAddress, WorkerInterface> workersMap;
std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo));
@ -299,7 +298,9 @@ ACTOR Future<std::vector<WorkerInterface>> getStorageWorkers(Database cx,
}
auto masterDcId = dbInfo->get().master.locality.dcId();
std::vector<WorkerInterface> result;
std::pair<std::vector<WorkerInterface>, int> result;
auto& [workerInterfaces, failures] = result;
failures = 0;
for (const auto& server : servers) {
TraceEvent(SevDebug, "DcIdInfo")
.detail("ServerLocalityID", server.locality.dcId())
@ -310,9 +311,9 @@ ACTOR Future<std::vector<WorkerInterface>> getStorageWorkers(Database cx,
TraceEvent(SevWarn, "GetStorageWorkers")
.detail("Reason", "Could not find worker for storage server")
.detail("SS", server.id());
throw operation_failed();
++failures;
}
result.push_back(itr->second);
workerInterfaces.push_back(itr->second);
}
}
return result;

8
fdbserver/QuietDatabase.h
View File

@ -46,9 +46,11 @@ Future<WorkerInterface> getMasterWorker(Database const& cx, Reference<AsyncVar<S
Future<Void> repairDeadDatacenter(Database const& cx,
Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
std::string const& context);
Future<std::vector<WorkerInterface>> getStorageWorkers(Database const& cx,
Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
bool const& localOnly);
// Returns list of worker interfaces for available storage servers and the number of unavailable
// storage servers
Future<std::pair<std::vector<WorkerInterface>, int>>
getStorageWorkers(Database const& cx, Reference<AsyncVar<ServerDBInfo> const> const& dbInfo, bool const& localOnly);
Future<std::vector<WorkerInterface>> getCoordWorkers(Database const& cx,
Reference<AsyncVar<ServerDBInfo> const> const& dbInfo);

8
fdbserver/workloads/DiskFailureInjection.actor.cpp
View File

@ -154,9 +154,13 @@ struct DiskFailureInjectionWorkload : TestWorkload {
loop {
wait(poisson(&lastTime, 1));
try {
wait(store(machines, getStorageWorkers(cx, self->dbInfo, false)));
std::pair<std::vector<W>, int> m = wait(getStorageWorkers(cx, self->dbInfo, false));
if (m.second > 0) {
throw operation_failed();
}
machines = std::move(m.first);
} catch (Error& e) {
// If we failed to get a list of storage servers, we can't inject failure events
// If we failed to get a complete list of storage servers, we can't inject failure events
// But don't throw the error in that case
continue;
}