Merge pull request #6399 from sfc-gh-bvr/fdb#4271

Introduce a new server knob and use it to test if storage servers are…

fdbclient/ServerKnobs.cpp

@@ -582,6 +582,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	init( MIN_AVAILABLE_SPACE,                                   1e8 );
 	init( MIN_AVAILABLE_SPACE_RATIO,                            0.05 );
+	init( MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER,              0.01 );
 	init( TARGET_AVAILABLE_SPACE_RATIO,                         0.30 );
 	init( AVAILABLE_SPACE_UPDATE_DELAY,                          5.0 );
 

fdbclient/ServerKnobs.h

@@ -529,6 +529,7 @@ public:
 
 	int64_t MIN_AVAILABLE_SPACE;
 	double MIN_AVAILABLE_SPACE_RATIO;
+	double MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER;
 	double TARGET_AVAILABLE_SPACE_RATIO;
 	double AVAILABLE_SPACE_UPDATE_DELAY;
 

fdbserver/DDTeamCollection.actor.cpp

@@ -5640,11 +5640,7 @@ public:
 
 		std::pair<Optional<Reference<IDataDistributionTeam>>, bool> resTeam = req.reply.getFuture().get();
 
-		std::set<UID> expectedServers{ UID(2, 0), UID(3, 0), UID(4, 0) };
-		ASSERT(resTeam.first.present());
-		auto servers = resTeam.first.get()->getServerIDs();
-		const std::set<UID> selectedServers(servers.begin(), servers.end());
-		ASSERT(expectedServers == selectedServers);
+		ASSERT(!resTeam.first.present());
 
 		return Void();
 	}

fdbserver/TCInfo.actor.cpp

@@ -137,6 +137,23 @@ TCServerInfo::TCServerInfo(StorageServerInterface ssi,
 	}
 }
 
+bool TCServerInfo::hasHealthyAvailableSpace(double minAvailableSpaceRatio) const {
+	ASSERT(serverMetricsPresent());
+
+	auto& metrics = getServerMetrics();
+	ASSERT(metrics.available.bytes >= 0);
+	ASSERT(metrics.capacity.bytes >= 0);
+
+	double availableSpaceRatio;
+	if (metrics.capacity.bytes == 0) {
+		availableSpaceRatio = 0;
+	} else {
+		availableSpaceRatio = (((double)metrics.available.bytes) / metrics.capacity.bytes);
+	}
+
+	return availableSpaceRatio >= minAvailableSpaceRatio;
+}
+
 Future<Void> TCServerInfo::updateServerMetrics() {
 	return TCServerInfoImpl::updateServerMetrics(this);
 }
@@ -381,8 +398,23 @@ double TCTeamInfo::getMinAvailableSpaceRatio(bool includeInFlight) const {
 	return minRatio;
 }
 
+bool TCTeamInfo::allServersHaveHealthyAvailableSpace() const {
+	bool result = true;
+	double minAvailableSpaceRatio =
+	    SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO + SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER;
+	for (const auto& server : servers) {
+		if (!server->serverMetricsPresent() || !server->hasHealthyAvailableSpace(minAvailableSpaceRatio)) {
+			result = false;
+			break;
+		}
+	}
+
+	return result;
+}
+
 bool TCTeamInfo::hasHealthyAvailableSpace(double minRatio) const {
-	return getMinAvailableSpaceRatio() >= minRatio && getMinAvailableSpace() > SERVER_KNOBS->MIN_AVAILABLE_SPACE;
+	return getMinAvailableSpaceRatio() >= minRatio && getMinAvailableSpace() > SERVER_KNOBS->MIN_AVAILABLE_SPACE &&
+	       allServersHaveHealthyAvailableSpace();
 }
 
 bool TCTeamInfo::isOptimal() const {

fdbserver/TCInfo.h

@@ -93,6 +93,8 @@ public:
 		return (storeType == configStoreType || storeType == KeyValueStoreType::END);
 	}
 
+	bool hasHealthyAvailableSpace(double minAvailableSpaceRatio) const;
+
 	Future<Void> updateServerMetrics();
 	static Future<Void> updateServerMetrics(Reference<TCServerInfo> server);
 	Future<Void> serverMetricsPolling();
@@ -218,4 +220,6 @@ private:
 	// Calculate an "average" of the metrics replies that we received.  Penalize teams from which we did not receive all
 	// replies.
 	int64_t getLoadAverage() const;
+
+	bool allServersHaveHealthyAvailableSpace() const;
 };