diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp index 92797984f2..0117cafb5b 100644 --- a/fdbserver/DDTeamCollection.actor.cpp +++ b/fdbserver/DDTeamCollection.actor.cpp @@ -5076,11 +5076,11 @@ bool DDTeamCollection::exclusionSafetyCheck(std::vector& excludeServerIDs) return true; } -StorageWiggler::State DDTeamCollection::getStorageWigglerState() const { +std::pair DDTeamCollection::getStorageWigglerState() const { if (storageWiggler) { - return storageWiggler->getState(); + return { storageWiggler->getState(), storageWiggler->lastStateChangeTs }; } - return StorageWiggler::INVALID; + return { StorageWiggler::INVALID, 0.0 }; } Future DDTeamCollection::run(Reference teamCollection, diff --git a/fdbserver/DDTeamCollection.h b/fdbserver/DDTeamCollection.h index e3498b886a..26cc791333 100644 --- a/fdbserver/DDTeamCollection.h +++ b/fdbserver/DDTeamCollection.h @@ -664,7 +664,8 @@ public: bool isPrimary() const { return primary; } - StorageWiggler::State getStorageWigglerState() const; + // state and last state change timestamp + std::pair getStorageWigglerState() const; UID getDistributorId() const { return distributorId; } diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index 8f5a6e5932..eae8c91776 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -1230,9 +1230,10 @@ static int64_t getMedianShardSize(VectorRef metricVec) { GetStorageWigglerStateReply getStorageWigglerStates(Reference self) { GetStorageWigglerStateReply reply; if (self->teamCollection) { - reply.primary = self->teamCollection->getStorageWigglerState(); + std::tie(reply.primary, reply.lastStateChangePrimary) = self->teamCollection->getStorageWigglerState(); if (self->teamCollection->teamCollections.size() > 1) { - reply.remote = self->teamCollection->teamCollections[1]->getStorageWigglerState(); + std::tie(reply.remote, reply.lastStateChangeRemote) = + self->teamCollection->teamCollections[1]->getStorageWigglerState(); } } return reply; diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h index fac5d7c83b..043dfb284c 100644 --- a/fdbserver/DataDistribution.actor.h +++ b/fdbserver/DataDistribution.actor.h @@ -402,6 +402,8 @@ struct StorageWiggler : ReferenceCounted { std::unordered_map pq_handles; State _state = State::INVALID; + double lastStateChangeTs = 0.0; // timestamp describes when did the state change + explicit StorageWiggler(DDTeamCollection* collection) : nonEmpty(false), teamCollection(collection){}; // add server to wiggling queue void addServer(const UID& serverId, const StorageMetadataType& metadata); @@ -414,7 +416,12 @@ struct StorageWiggler : ReferenceCounted { Optional getNextServerId(); State getState() const { return _state; } - void setState(State s) { _state = s; } + void setState(State s) { + if (_state != s) { + _state = s; + lastStateChangeTs = g_network->now(); + } + } static std::string getStateStr(State s) { switch (s) { case State::RUN: diff --git a/fdbserver/DataDistributorInterface.h b/fdbserver/DataDistributorInterface.h index 3bb4a10769..a7bfcbe1e6 100644 --- a/fdbserver/DataDistributorInterface.h +++ b/fdbserver/DataDistributorInterface.h @@ -170,6 +170,7 @@ struct DistributorSplitRangeRequest { struct GetStorageWigglerStateReply { constexpr static FileIdentifier file_identifier = 356721; uint8_t primary = 0, remote = 0; // StorageWiggler::State enum + double lastStateChangePrimary = 0.0, lastStateChangeRemote = 0.0; GetStorageWigglerStateReply() {} template