From 4c9c7362532c605a3808eb501b84701e03786773 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Thu, 20 Feb 2020 11:21:03 -0800 Subject: [PATCH] Data distribution uses available space instead of free space when evaluating whether processes are low on space and penalizing them. --- fdbclient/StorageServerInterface.h | 4 +- fdbserver/DataDistribution.actor.cpp | 50 +++++++++++------------ fdbserver/DataDistribution.actor.h | 12 +++--- fdbserver/DataDistributionQueue.actor.cpp | 12 +++--- fdbserver/StorageMetrics.actor.h | 6 +-- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 7e56ef8899..9d054104d4 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -383,7 +383,7 @@ struct SplitMetricsRequest { struct GetStorageMetricsReply { constexpr static FileIdentifier file_identifier = 15491478; StorageMetrics load; - StorageMetrics free; + StorageMetrics available; StorageMetrics capacity; double bytesInputRate; @@ -391,7 +391,7 @@ struct GetStorageMetricsReply { template void serialize(Ar& ar) { - serializer(ar, load, free, capacity, bytesInputRate); + serializer(ar, load, available, capacity, bytesInputRate); } }; diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index 6ff733f113..84cb0bebab 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -238,63 +238,63 @@ public: virtual int64_t getLoadBytes( bool includeInFlight = true, double inflightPenalty = 1.0 ) { int64_t physicalBytes = getLoadAverage(); - double minFreeSpaceRatio = getMinFreeSpaceRatio(includeInFlight); + double minAvailableSpaceRatio = getMinAvailableSpaceRatio(includeInFlight); int64_t inFlightBytes = includeInFlight ? getDataInFlightToTeam() / servers.size() : 0; - double freeSpaceMultiplier = SERVER_KNOBS->FREE_SPACE_RATIO_CUTOFF / ( std::max( std::min( SERVER_KNOBS->FREE_SPACE_RATIO_CUTOFF, minFreeSpaceRatio ), 0.000001 ) ); + double availableSpaceMultiplier = SERVER_KNOBS->FREE_SPACE_RATIO_CUTOFF / ( std::max( std::min( SERVER_KNOBS->FREE_SPACE_RATIO_CUTOFF, minAvailableSpaceRatio ), 0.000001 ) ); - if(freeSpaceMultiplier > 1 && deterministicRandom()->random01() < 0.001) - TraceEvent(SevWarn, "DiskNearCapacity").detail("FreeSpaceRatio", minFreeSpaceRatio); + if(availableSpaceMultiplier > 1 && deterministicRandom()->random01() < 0.001) + TraceEvent(SevWarn, "DiskNearCapacity").detail("AvailableSpaceRatio", minAvailableSpaceRatio); - return (physicalBytes + (inflightPenalty*inFlightBytes)) * freeSpaceMultiplier; + return (physicalBytes + (inflightPenalty*inFlightBytes)) * availableSpaceMultiplier; } - virtual int64_t getMinFreeSpace( bool includeInFlight = true ) { - int64_t minFreeSpace = std::numeric_limits::max(); + virtual int64_t getMinAvailableSpace( bool includeInFlight = true ) { + int64_t minAvailableSpace = std::numeric_limits::max(); for(int i=0; iserverMetrics.present() ) { auto& replyValue = servers[i]->serverMetrics.get(); - ASSERT(replyValue.free.bytes >= 0); + ASSERT(replyValue.available.bytes >= 0); ASSERT(replyValue.capacity.bytes >= 0); - int64_t bytesFree = replyValue.free.bytes; + int64_t bytesAvailable = replyValue.available.bytes; if(includeInFlight) { - bytesFree -= servers[i]->dataInFlightToServer; + bytesAvailable -= servers[i]->dataInFlightToServer; } - minFreeSpace = std::min(bytesFree, minFreeSpace); + minAvailableSpace = std::min(bytesAvailable, minAvailableSpace); } } - return minFreeSpace; // Could be negative + return minAvailableSpace; // Could be negative } - virtual double getMinFreeSpaceRatio( bool includeInFlight = true ) { + virtual double getMinAvailableSpaceRatio( bool includeInFlight = true ) { double minRatio = 1.0; for(int i=0; iserverMetrics.present() ) { auto& replyValue = servers[i]->serverMetrics.get(); - ASSERT(replyValue.free.bytes >= 0); + ASSERT(replyValue.available.bytes >= 0); ASSERT(replyValue.capacity.bytes >= 0); - int64_t bytesFree = replyValue.free.bytes; + int64_t bytesAvailable = replyValue.available.bytes; if(includeInFlight) { - bytesFree = std::max((int64_t)0, bytesFree - servers[i]->dataInFlightToServer); + bytesAvailable = std::max((int64_t)0, bytesAvailable - servers[i]->dataInFlightToServer); } if(replyValue.capacity.bytes == 0) minRatio = 0; else - minRatio = std::min( minRatio, ((double)bytesFree) / replyValue.capacity.bytes ); + minRatio = std::min( minRatio, ((double)bytesAvailable) / replyValue.capacity.bytes ); } } return minRatio; } - virtual bool hasHealthyFreeSpace() { - return getMinFreeSpaceRatio() > SERVER_KNOBS->MIN_FREE_SPACE_RATIO && getMinFreeSpace() > SERVER_KNOBS->MIN_FREE_SPACE; + virtual bool hasHealthyAvailableSpace() { + return getMinAvailableSpaceRatio() > SERVER_KNOBS->MIN_FREE_SPACE_RATIO && getMinAvailableSpace() > SERVER_KNOBS->MIN_FREE_SPACE; } virtual Future updateStorageMetrics() { @@ -775,7 +775,7 @@ struct DDTeamCollection : ReferenceCounted { } if(found && teamList[j]->isHealthy() && (!req.teamMustHaveShards || self->shardsAffectedByTeamFailure->getShardsFor(ShardsAffectedByTeamFailure::Team(teamList[j]->getServerIDs(), self->primary)).size() > 0) && - teamList[j]->getMinFreeSpaceRatio() >= req.minFreeSpaceRatio) + teamList[j]->getMinAvailableSpaceRatio() >= req.minAvailableSpaceRatio) { req.reply.send( teamList[j] ); return Void(); @@ -788,9 +788,9 @@ struct DDTeamCollection : ReferenceCounted { ASSERT( !bestOption.present() ); for( int i = 0; i < self->teams.size(); i++ ) { if (self->teams[i]->isHealthy() && - (!req.preferLowerUtilization || self->teams[i]->hasHealthyFreeSpace()) && + (!req.preferLowerUtilization || self->teams[i]->hasHealthyAvailableSpace()) && (!req.teamMustHaveShards || self->shardsAffectedByTeamFailure->getShardsFor(ShardsAffectedByTeamFailure::Team(self->teams[i]->getServerIDs(), self->primary)) .size() > 0) && - self->teams[i]->getMinFreeSpaceRatio() >= req.minFreeSpaceRatio) + self->teams[i]->getMinAvailableSpaceRatio() >= req.minAvailableSpaceRatio) { int64_t loadBytes = self->teams[i]->getLoadBytes(true, req.inflightPenalty); if( !bestOption.present() || ( req.preferLowerUtilization && loadBytes < bestLoadBytes ) || ( !req.preferLowerUtilization && loadBytes > bestLoadBytes ) ) { @@ -806,9 +806,9 @@ struct DDTeamCollection : ReferenceCounted { Reference dest = deterministicRandom()->randomChoice(self->teams); bool ok = dest->isHealthy() && - (!req.preferLowerUtilization || dest->hasHealthyFreeSpace()) && + (!req.preferLowerUtilization || dest->hasHealthyAvailableSpace()) && (!req.teamMustHaveShards || self->shardsAffectedByTeamFailure->getShardsFor(ShardsAffectedByTeamFailure::Team(dest->getServerIDs(), self->primary)).size() > 0) && - dest->getMinFreeSpaceRatio() >= req.minFreeSpaceRatio; + dest->getMinAvailableSpaceRatio() >= req.minAvailableSpaceRatio; for(int i=0; ok && igetServerIDs() == dest->getServerIDs()) { @@ -845,7 +845,7 @@ struct DDTeamCollection : ReferenceCounted { bool found = true; auto serverIDs = teamList[j]->getServerIDs(); if((req.teamMustHaveShards && self->shardsAffectedByTeamFailure->getShardsFor(ShardsAffectedByTeamFailure::Team(serverIDs, self->primary)).size() == 0) || - teamList[j]->getMinFreeSpaceRatio() < req.minFreeSpaceRatio) + teamList[j]->getMinAvailableSpaceRatio() < req.minAvailableSpaceRatio) { continue; } diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h index d1788601df..2d1bdba2d4 100644 --- a/fdbserver/DataDistribution.actor.h +++ b/fdbserver/DataDistribution.actor.h @@ -45,9 +45,9 @@ struct IDataDistributionTeam { virtual void addDataInFlightToTeam( int64_t delta ) = 0; virtual int64_t getDataInFlightToTeam() = 0; virtual int64_t getLoadBytes( bool includeInFlight = true, double inflightPenalty = 1.0 ) = 0; - virtual int64_t getMinFreeSpace( bool includeInFlight = true ) = 0; - virtual double getMinFreeSpaceRatio( bool includeInFlight = true ) = 0; - virtual bool hasHealthyFreeSpace() = 0; + virtual int64_t getMinAvailableSpace( bool includeInFlight = true ) = 0; + virtual double getMinAvailableSpaceRatio( bool includeInFlight = true ) = 0; + virtual bool hasHealthyAvailableSpace() = 0; virtual Future updateStorageMetrics() = 0; virtual void addref() = 0; virtual void delref() = 0; @@ -76,15 +76,15 @@ struct GetTeamRequest { bool wantsTrueBest; bool preferLowerUtilization; bool teamMustHaveShards; - double minFreeSpaceRatio; + double minAvailableSpaceRatio; double inflightPenalty; std::vector completeSources; Promise< Optional< Reference > > reply; GetTeamRequest() {} - GetTeamRequest( bool wantsNewServers, bool wantsTrueBest, bool preferLowerUtilization, bool teamMustHaveShards, double minFreeSpaceRatio = 0.0, double inflightPenalty = 1.0 ) + GetTeamRequest( bool wantsNewServers, bool wantsTrueBest, bool preferLowerUtilization, bool teamMustHaveShards, double minAvailableSpaceRatio = 0.0, double inflightPenalty = 1.0 ) : wantsNewServers( wantsNewServers ), wantsTrueBest( wantsTrueBest ), preferLowerUtilization( preferLowerUtilization ), teamMustHaveShards( teamMustHaveShards ), - minFreeSpaceRatio( minFreeSpaceRatio ), inflightPenalty( inflightPenalty ) {} + minAvailableSpaceRatio( minAvailableSpaceRatio ), inflightPenalty( inflightPenalty ) {} }; struct GetMetricsRequest { diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp index f774fd7785..168bf0f3d8 100644 --- a/fdbserver/DataDistributionQueue.actor.cpp +++ b/fdbserver/DataDistributionQueue.actor.cpp @@ -170,25 +170,25 @@ public: }); } - virtual int64_t getMinFreeSpace(bool includeInFlight = true) { + virtual int64_t getMinAvailableSpace(bool includeInFlight = true) { int64_t result = std::numeric_limits::max(); for (auto it = teams.begin(); it != teams.end(); it++) { - result = std::min(result, (*it)->getMinFreeSpace(includeInFlight)); + result = std::min(result, (*it)->getMinAvailableSpace(includeInFlight)); } return result; } - virtual double getMinFreeSpaceRatio(bool includeInFlight = true) { + virtual double getMinAvailableSpaceRatio(bool includeInFlight = true) { double result = std::numeric_limits::max(); for (auto it = teams.begin(); it != teams.end(); it++) { - result = std::min(result, (*it)->getMinFreeSpaceRatio(includeInFlight)); + result = std::min(result, (*it)->getMinAvailableSpaceRatio(includeInFlight)); } return result; } - virtual bool hasHealthyFreeSpace() { + virtual bool hasHealthyAvailableSpace() { return all([](Reference team) { - return team->hasHealthyFreeSpace(); + return team->hasHealthyAvailableSpace(); }); } diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h index 4f2d6779e6..3a84063679 100644 --- a/fdbserver/StorageMetrics.actor.h +++ b/fdbserver/StorageMetrics.actor.h @@ -356,9 +356,9 @@ struct StorageServerMetrics { .detail("Load", rep.load.bytes); } - rep.free.bytes = sb.free; - rep.free.iosPerKSecond = 10e6; - rep.free.bytesPerKSecond = 100e9; + rep.available.bytes = sb.available; + rep.available.iosPerKSecond = 10e6; + rep.available.bytesPerKSecond = 100e9; rep.capacity.bytes = sb.total; rep.capacity.iosPerKSecond = 10e6;