Add limiting storage queue and durability lag to health metrics

fdbclient/FDBTypes.h

@@ -997,7 +997,9 @@ struct HealthMetrics {
 	};
 
 	int64_t worstStorageQueue;
+	int64_t limitingStorageQueue;
 	int64_t worstStorageDurabilityLag;
+	int64_t limitingStorageDurabilityLag;
 	int64_t worstTLogQueue;
 	double tpsLimit;
 	bool batchLimited;
@@ -1005,17 +1007,15 @@ struct HealthMetrics {
 	std::map<UID, int64_t> tLogQueue;
 
 	HealthMetrics()
-		: worstStorageQueue(0)
-		, worstStorageDurabilityLag(0)
-		, worstTLogQueue(0)
-		, tpsLimit(0.0)
-		, batchLimited(false)
-	{}
+	  : worstStorageQueue(0), limitingStorageQueue(0), worstStorageDurabilityLag(0), limitingStorageDurabilityLag(0),
+	    worstTLogQueue(0), tpsLimit(0.0), batchLimited(false) {}
 
 	void update(const HealthMetrics& hm, bool detailedInput, bool detailedOutput)
 	{
 		worstStorageQueue = hm.worstStorageQueue;
+		limitingStorageQueue = hm.limitingStorageQueue;
 		worstStorageDurabilityLag = hm.worstStorageDurabilityLag;
+		limitingStorageDurabilityLag = hm.limitingStorageDurabilityLag;
 		worstTLogQueue = hm.worstTLogQueue;
 		tpsLimit = hm.tpsLimit;
 		batchLimited = hm.batchLimited;
@@ -1030,19 +1030,16 @@ struct HealthMetrics {
 	}
 
 	bool operator==(HealthMetrics const& r) const {
-		return (
-			worstStorageQueue == r.worstStorageQueue &&
-			worstStorageDurabilityLag == r.worstStorageDurabilityLag &&
-			worstTLogQueue == r.worstTLogQueue &&
-			storageStats == r.storageStats &&
-			tLogQueue == r.tLogQueue &&
-			batchLimited == r.batchLimited
-		);
+		return (worstStorageQueue == r.worstStorageQueue && limitingStorageQueue == r.limitingStorageQueue &&
+		        worstStorageDurabilityLag == r.worstStorageDurabilityLag &&
+		        limitingStorageDurabilityLag == r.limitingStorageDurabilityLag && worstTLogQueue == r.worstTLogQueue &&
+		        storageStats == r.storageStats && tLogQueue == r.tLogQueue && batchLimited == r.batchLimited);
 	}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, worstStorageQueue, worstStorageDurabilityLag, worstTLogQueue, tpsLimit, batchLimited, storageStats, tLogQueue);
+		serializer(ar, worstStorageQueue, worstStorageDurabilityLag, worstTLogQueue, tpsLimit, batchLimited,
+		           storageStats, tLogQueue, limitingStorageQueue, limitingStorageDurabilityLag);
 	}
 };
 

fdbclient/NativeAPI.actor.cpp

@@ -765,7 +765,9 @@ static Standalone<RangeResultRef> healthMetricsToKVPairs(const HealthMetrics& me
 		statsObj["batch_limited"] = metrics.batchLimited;
 		statsObj["tps_limit"] = metrics.tpsLimit;
 		statsObj["worst_storage_durability_lag"] = metrics.worstStorageDurabilityLag;
+		statsObj["limiting_storage_durability_lag"] = metrics.limitingStorageDurabilityLag;
 		statsObj["worst_storage_queue"] = metrics.worstStorageQueue;
+		statsObj["limiting_storage_queue"] = metrics.limitingStorageQueue;
 		statsObj["worst_log_queue"] = metrics.worstTLogQueue;
 		std::string statsString =
 		    json_spirit::write_string(json_spirit::mValue(statsObj), json_spirit::Output_options::raw_utf8);

fdbclient/Schemas.cpp

@@ -939,6 +939,8 @@ const KeyRef JSONSchemas::storageHealthSchema = LiteralStringRef(R"""(
 const KeyRef JSONSchemas::aggregateHealthSchema = LiteralStringRef(R"""(
 {
   "batch_limited": false,
+  "limiting_storage_durability_lag": 5050809,
+  "limiting_storage_queue": 2030,
   "tps_limit": 457082.8105811302,
   "worst_storage_durability_lag": 5050809,
   "worst_storage_queue": 2030,

fdbserver/Ratekeeper.actor.cpp

@@ -1118,7 +1118,9 @@ void updateRate(RatekeeperData* self, RatekeeperLimits* limits) {
 	}
 
 	self->healthMetrics.worstStorageQueue = worstStorageQueueStorageServer;
+	self->healthMetrics.limitingStorageQueue = limitingStorageQueueStorageServer;
 	self->healthMetrics.worstStorageDurabilityLag = worstDurabilityLag;
+	self->healthMetrics.limitingStorageDurabilityLag = limitingDurabilityLag;
 
 	double writeToReadLatencyLimit = 0;
 	Version worstVersionLag = 0;

fdbserver/workloads/HealthMetricsApi.actor.cpp

@@ -28,7 +28,9 @@
 struct HealthMetricsApiWorkload : TestWorkload {
 	// Performance Metrics
 	int64_t worstStorageQueue = 0;
+	int64_t worstLimitingStorageQueue = 0;
 	int64_t worstStorageDurabilityLag = 0;
+	int64_t worstLimitingStorageDurabilityLag = 0;
 	int64_t worstTLogQueue = 0;
 	int64_t detailedWorstStorageQueue = 0;
 	int64_t detailedWorstStorageDurabilityLag = 0;
@@ -77,7 +79,8 @@ struct HealthMetricsApiWorkload : TestWorkload {
 			return false;
 		}
 		bool correctHealthMetricsState = true;
-		if (worstStorageQueue == 0 || worstStorageDurabilityLag == 0 || worstTLogQueue == 0)
+		if (worstStorageQueue == 0 || worstLimitingStorageQueue == 0 || worstStorageDurabilityLag == 0 ||
+		    worstLimitingStorageDurabilityLag == 0 || worstTLogQueue == 0)
 			correctHealthMetricsState = false;
 		if (sendDetailedHealthMetrics) {
 			if (detailedWorstStorageQueue == 0 || detailedWorstStorageDurabilityLag == 0 ||
@@ -91,7 +94,9 @@ struct HealthMetricsApiWorkload : TestWorkload {
 		if (!correctHealthMetricsState) {
 			TraceEvent(SevError, "IncorrectHealthMetricsState")
 			    .detail("WorstStorageQueue", worstStorageQueue)
+			    .detail("WorstLimitingStorageQueue", worstLimitingStorageQueue)
 			    .detail("WorstStorageDurabilityLag", worstStorageDurabilityLag)
+			    .detail("WorstLimitingStorageDurabilityLag", worstLimitingStorageDurabilityLag)
 			    .detail("WorstTLogQueue", worstTLogQueue)
 			    .detail("DetailedWorstStorageQueue", detailedWorstStorageQueue)
 			    .detail("DetailedWorstStorageDurabilityLag", detailedWorstStorageDurabilityLag)
@@ -128,13 +133,19 @@ struct HealthMetricsApiWorkload : TestWorkload {
 			healthMetrics = newHealthMetrics;
 
 			self->worstStorageQueue = std::max(self->worstStorageQueue, healthMetrics.worstStorageQueue);
+			self->worstLimitingStorageQueue =
+			    std::max(self->worstLimitingStorageQueue, healthMetrics.limitingStorageQueue);
 			self->worstStorageDurabilityLag =
 			    std::max(self->worstStorageDurabilityLag, healthMetrics.worstStorageDurabilityLag);
+			self->worstLimitingStorageDurabilityLag =
+			    std::max(self->worstLimitingStorageDurabilityLag, healthMetrics.limitingStorageDurabilityLag);
 			self->worstTLogQueue = std::max(self->worstTLogQueue, healthMetrics.worstTLogQueue);
 
 			TraceEvent("HealthMetrics")
 			    .detail("WorstStorageQueue", healthMetrics.worstStorageQueue)
+			    .detail("LimitingStorageQueue", healthMetrics.limitingStorageQueue)
 			    .detail("WorstStorageDurabilityLag", healthMetrics.worstStorageDurabilityLag)
+			    .detail("LimitingStorageDurabilityLag", healthMetrics.limitingStorageDurabilityLag)
 			    .detail("WorstTLogQueue", healthMetrics.worstTLogQueue)
 			    .detail("TpsLimit", healthMetrics.tpsLimit);