wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Send tag throttle updates from ratekeeper to proxy only when they change

This commit is contained in:
A.J. Beamon 2020-04-21 16:33:56 -07:00
parent 444b338f63
commit d5fb4d26fe
3 changed files with 45 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
fdbserver/MasterProxyServer.actor.cpp
View File

@ -216,11 +216,8 @@ ACTOR Future<Void> getRate(UID myID, Reference<AsyncVar<ServerDBInfo>> db, int64
lastDetailedReply = now(); lastDetailedReply = now();
} }
for(auto priorityTags : rep.throttledTags) { if(rep.throttledTags.present()) {
auto &localPriorityTags = (*throttledTags)[priorityTags.first]; *throttledTags = rep.throttledTags;
for(auto tag : priorityTags.second) { // TODO: remove missing tags
localPriorityTags[tag.first] = tag.second;
}
} }
} }
when ( wait( leaseTimeout ) ) { when ( wait( leaseTimeout ) ) {
@ -257,7 +254,7 @@ ACTOR Future<Void> queueTransactionStartRequests(
req.reply.send(rep); req.reply.send(rep);
TraceEvent(SevWarnAlways, "ProxyGRVThresholdExceeded").suppressFor(60); TraceEvent(SevWarnAlways, "ProxyGRVThresholdExceeded").suppressFor(60);
} else { } else {
// TODO: this probably needs to happen outside the high priority path // TODO: check whether this is reasonable to do in the fast path
for(auto tag : req.tags) { for(auto tag : req.tags) {
(*transactionTagCounter)[tag.first] += tag.second; (*transactionTagCounter)[tag.first] += tag.second;
} }
@ -1381,9 +1378,8 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture, std::
reply.tagThrottleInfo.clear(); reply.tagThrottleInfo.clear();
auto& priorityThrottledTags = throttledTags[ThrottleApi::priorityFromReadVersionFlags(request.flags)];
for(auto tag : request.tags) { for(auto tag : request.tags) {
auto& priorityThrottledTags = throttledTags[ThrottleApi::priorityFromReadVersionFlags(request.flags)];
auto tagItr = priorityThrottledTags.find(tag.first); auto tagItr = priorityThrottledTags.find(tag.first);
if(tagItr != priorityThrottledTags.end()) { if(tagItr != priorityThrottledTags.end()) {
reply.tagThrottleInfo[tag.first] = tagItr->second; reply.tagThrottleInfo[tag.first] = tagItr->second;

74
fdbserver/Ratekeeper.actor.cpp
View File

@ -284,13 +284,14 @@ struct RatekeeperLimits {
{} {}
}; };
struct TransactionCounts { struct ProxyInfo {
int64_t total; int64_t totalTransactions;
int64_t batch; int64_t batchTransactions;
uint64_t lastThrottledTagChangeId;
double time; double lastUpdateTime;
TransactionCounts() : total(0), batch(0), time(0) {} ProxyInfo() : totalTransactions(0), batchTransactions(0), lastUpdateTime(0), lastThrottledTagChangeId(0) {}
}; };
struct RatekeeperData { struct RatekeeperData {
@ -299,7 +300,7 @@ struct RatekeeperData {
Map<UID, StorageQueueInfo> storageQueueInfo; Map<UID, StorageQueueInfo> storageQueueInfo;
Map<UID, TLogQueueInfo> tlogQueueInfo; Map<UID, TLogQueueInfo> tlogQueueInfo;
std::map<UID, TransactionCounts> proxy_transactionCounts; std::map<UID, ProxyInfo> proxyInfo;
Smoother smoothReleasedTransactions, smoothBatchReleasedTransactions, smoothTotalDurableBytes; Smoother smoothReleasedTransactions, smoothBatchReleasedTransactions, smoothTotalDurableBytes;
HealthMetrics healthMetrics; HealthMetrics healthMetrics;
DatabaseConfiguration configuration; DatabaseConfiguration configuration;
@ -311,6 +312,7 @@ struct RatekeeperData {
double lastSSListFetchedTimestamp; double lastSSListFetchedTimestamp;
TransactionTagMap<RkTagThrottleData> throttledTags; TransactionTagMap<RkTagThrottleData> throttledTags;
uint64_t throttledTagChangeId;
RatekeeperLimits normalLimits; RatekeeperLimits normalLimits;
RatekeeperLimits batchLimits; RatekeeperLimits batchLimits;
@ -322,7 +324,7 @@ struct RatekeeperData {
RatekeeperData(Database db) : db(db), smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT), RatekeeperData(Database db) : db(db), smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
actualTpsMetric(LiteralStringRef("Ratekeeper.ActualTPS")), actualTpsMetric(LiteralStringRef("Ratekeeper.ActualTPS")),
lastWarning(0), lastSSListFetchedTimestamp(now()), lastWarning(0), lastSSListFetchedTimestamp(now()), throttledTagChangeId(0),
normalLimits("", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER, SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER, SERVER_KNOBS->TARGET_BYTES_PER_TLOG, SERVER_KNOBS->SPRING_BYTES_TLOG, SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE, SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS), normalLimits("", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER, SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER, SERVER_KNOBS->TARGET_BYTES_PER_TLOG, SERVER_KNOBS->SPRING_BYTES_TLOG, SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE, SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS),
batchLimits("Batch", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER_BATCH, SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER_BATCH, SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH, SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH, SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH, SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH), batchLimits("Batch", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER_BATCH, SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER_BATCH, SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH, SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH, SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH, SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH),
autoThrottlingEnabled(false) autoThrottlingEnabled(false)
@ -573,6 +575,7 @@ ACTOR Future<Void> monitorThrottlingChanges(RatekeeperData *self) {
} }
self->throttledTags = std::move(updatedTagThrottles); self->throttledTags = std::move(updatedTagThrottles);
++self->throttledTagChangeId;
state Future<Void> watchFuture = tr.watch(tagThrottleSignalKey); state Future<Void> watchFuture = tr.watch(tagThrottleSignalKey);
wait(tr.commit()); wait(tr.commit());
@ -931,7 +934,7 @@ void updateRate(RatekeeperData* self, RatekeeperLimits* limits, TransactionTagMa
.detail("ReleasedBatchTPS", self->smoothBatchReleasedTransactions.smoothRate()) .detail("ReleasedBatchTPS", self->smoothBatchReleasedTransactions.smoothRate())
.detail("TPSBasis", actualTps) .detail("TPSBasis", actualTps)
.detail("StorageServers", sscount) .detail("StorageServers", sscount)
.detail("Proxies", self->proxy_transactionCounts.size()) .detail("Proxies", self->proxyInfo.size())
.detail("TLogs", tlcount) .detail("TLogs", tlcount)
.detail("WorstFreeSpaceStorageServer", worstFreeSpaceStorageServer) .detail("WorstFreeSpaceStorageServer", worstFreeSpaceStorageServer)
.detail("WorstFreeSpaceTLog", worstFreeSpaceTLog) .detail("WorstFreeSpaceTLog", worstFreeSpaceTLog)
@ -1011,9 +1014,9 @@ ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<S
lastLimited = self.smoothReleasedTransactions.smoothRate() > SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit; lastLimited = self.smoothReleasedTransactions.smoothRate() > SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit;
double tooOld = now() - 1.0; double tooOld = now() - 1.0;
for(auto p=self.proxy_transactionCounts.begin(); p!=self.proxy_transactionCounts.end(); ) { for(auto p=self.proxyInfo.begin(); p!=self.proxyInfo.end(); ) {
if (p->second.time < tooOld) if (p->second.lastUpdateTime < tooOld)
p = self.proxy_transactionCounts.erase(p); p = self.proxyInfo.erase(p);
else else
++p; ++p;
} }
@ -1022,10 +1025,10 @@ ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<S
when (GetRateInfoRequest req = waitNext(rkInterf.getRateInfo.getFuture())) { when (GetRateInfoRequest req = waitNext(rkInterf.getRateInfo.getFuture())) {
GetRateInfoReply reply; GetRateInfoReply reply;
auto& p = self.proxy_transactionCounts[ req.requesterID ]; auto& p = self.proxyInfo[ req.requesterID ];
//TraceEvent("RKMPU", req.requesterID).detail("TRT", req.totalReleasedTransactions).detail("Last", p.first).detail("Delta", req.totalReleasedTransactions - p.first); //TraceEvent("RKMPU", req.requesterID).detail("TRT", req.totalReleasedTransactions).detail("Last", p.first).detail("Delta", req.totalReleasedTransactions - p.first);
if (p.total > 0) { if (p.totalTransactions > 0) {
self.smoothReleasedTransactions.addDelta( req.totalReleasedTransactions - p.total ); self.smoothReleasedTransactions.addDelta( req.totalReleasedTransactions - p.totalTransactions );
for(auto tag : req.throttledTagCounts) { for(auto tag : req.throttledTagCounts) {
auto itr = self.throttledTags.find(tag.first); auto itr = self.throttledTags.find(tag.first);
@ -1034,33 +1037,36 @@ ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<S
} }
} }
} }
if(p.batch > 0) { if(p.batchTransactions > 0) {
self.smoothBatchReleasedTransactions.addDelta( req.batchReleasedTransactions - p.batch ); self.smoothBatchReleasedTransactions.addDelta( req.batchReleasedTransactions - p.batchTransactions );
} }
p.total = req.totalReleasedTransactions; p.totalTransactions = req.totalReleasedTransactions;
p.batch = req.batchReleasedTransactions; p.batchTransactions = req.batchReleasedTransactions;
p.time = now(); p.lastUpdateTime = now();
reply.transactionRate = self.normalLimits.tpsLimit / self.proxy_transactionCounts.size(); reply.transactionRate = self.normalLimits.tpsLimit / self.proxyInfo.size();
reply.batchTransactionRate = self.batchLimits.tpsLimit / self.proxy_transactionCounts.size(); reply.batchTransactionRate = self.batchLimits.tpsLimit / self.proxyInfo.size();
reply.leaseDuration = SERVER_KNOBS->METRIC_UPDATE_RATE; reply.leaseDuration = SERVER_KNOBS->METRIC_UPDATE_RATE;
// TODO: avoid iteration every time if(p.lastThrottledTagChangeId != self.throttledTagChangeId) {
for(auto itr = self.throttledTags.begin(); itr != self.throttledTags.end();) { p.lastThrottledTagChangeId = self.throttedTagChangeId;
for(auto &priorityItr : itr->second.throttleData) { reply.throttledTags = PrioritizedTransactionTagMap<ClientTagThrottleLimits>();
auto &priorityTags = reply.throttledTags[priorityItr.first]; // TODO: report all priorities, not just those at the throttle priority for(auto itr = self.throttledTags.begin(); itr != self.throttledTags.end();) {
Optional<std::pair<double, double>> clientRate = itr->second.getClientRate(priorityItr.first); for(auto &priorityItr : itr->second.throttleData) {
if(clientRate.present()) { auto &priorityTags = reply.throttledTags.get()[priorityItr.first]; // TODO: report all priorities, not just those at the throttle priority
priorityTags.try_emplace(itr->first, ClientTagThrottleLimits(clientRate.get().first, clientRate.get().second)); Optional<std::pair<double, double>> clientRate = itr->second.getClientRate(priorityItr.first);
if(clientRate.present()) {
priorityTags.try_emplace(itr->first, ClientTagThrottleLimits(clientRate.get().first, clientRate.get().second));
}
else if(priorityItr.first == ThrottleApi::Priority::BATCH) {
itr = self.throttledTags.erase(itr);
break;
}
} }
else if(priorityItr.first == ThrottleApi::Priority::BATCH) {
itr = self.throttledTags.erase(itr);
break;
}
}
++itr; ++itr;
}
} }
reply.healthMetrics.update(self.healthMetrics, true, req.detailed); reply.healthMetrics.update(self.healthMetrics, true, req.detailed);

2
fdbserver/RatekeeperInterface.h
View File

@ -72,7 +72,7 @@ struct GetRateInfoReply {
double leaseDuration; double leaseDuration;
HealthMetrics healthMetrics; HealthMetrics healthMetrics;
PrioritizedTransactionTagMap<ClientTagThrottleLimits> throttledTags; Optional<PrioritizedTransactionTagMap<ClientTagThrottleLimits>> throttledTags;
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {