add basic mock global state utils

fdbserver/DataDistributionTracker.actor.cpp

@@ -22,6 +22,7 @@
 #include "fdbclient/SystemData.h"
 #include "fdbserver/DataDistribution.actor.h"
 #include "fdbserver/Knobs.h"
+#include "fdbserver/workloads/workloads.actor.h"
 #include "fdbclient/DatabaseContext.h"
 #include "flow/ActorCollection.h"
 #include "flow/FastRef.h"
@@ -844,8 +845,8 @@ ACTOR Future<Void> fetchTopKShardMetrics_impl(DataDistributionTracker* self, Get
 		loop {
 			onChange = Future<Void>();
 			returnMetrics.clear();
-			state int64_t minReadLoad = std::numeric_limits<int64_t>::max();
+			state int64_t minReadLoad = -1;
-			state int64_t maxReadLoad = std::numeric_limits<int64_t>::min();
+			state int64_t maxReadLoad = -1;
 			state int i;
 			for (i = 0; i < SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT && i < req.keys.size(); ++i) {
 				auto range = req.keys[i];
@@ -865,7 +866,7 @@ ACTOR Future<Void> fetchTopKShardMetrics_impl(DataDistributionTracker* self, Get
 				}
 
 				if (metrics.bytesReadPerKSecond > 0) {
-					minReadLoad = std::min(metrics.bytesReadPerKSecond, minReadLoad);
+					minReadLoad = std::min(metrics.bytesReadPerKSecond, std::max((decltype(minReadLoad))0, minReadLoad));
 					maxReadLoad = std::max(metrics.bytesReadPerKSecond, maxReadLoad);
 					if (req.minBytesReadPerKSecond <= metrics.bytesReadPerKSecond &&
 					    metrics.bytesReadPerKSecond <= req.maxBytesReadPerKSecond) {
@@ -1224,9 +1225,29 @@ void ShardsAffectedByTeamFailure::check() const {
 }
 
 namespace data_distribution_test {
-DataDistributionTracker createDDTrackerForUnitTest() {}
 } // namespace data_distribution_test
 TEST_CASE("/DataDistributor/Tracker/FetchTopK") {
-	// state DataDistributionTracker self;
+	state DataDistributionTracker self;
+	state GetTopKMetricsRequest req;
+	req.topK = 3;
+	for(int i = 1; i <= 10; i += 2) {
+		KeyRange keys(KeyRangeRef(doubleToTestKey(i), doubleToTestKey(i+2)));
+		req.keys.push_back(keys);
+		// std::cout << "here: " << req.keys.back().begin.toString() << "\n";
+	}
+	req.minBytesReadPerKSecond = 1000;
+	req.minBytesReadPerKSecond = 10000;
+
+	self.shards = std::make_shared<KeyRangeMap<ShardTrackedData>>();
+	double targetDensities[10] = {2, 1, 3, 5, 4, 10, 6, 8, 7, 0};
+	for(int i = 0; i <= 5; ++ i) {
+
+	}
+	wait(fetchTopKShardMetrics_impl(&self, req));
+	auto& reply = req.reply.getFuture().get();
+	ASSERT(reply.shardMetrics.empty());
+	ASSERT(reply.maxReadLoad == -1);
+	ASSERT(reply.minReadLoad == -1);
+
 	return Void();
 }

fdbserver/MockGlobalState.cpp

@@ -0,0 +1,33 @@
+/*
+ * MockGlobalState.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/MockGlobalState.h"
+
+void MockGlobalState::initialAsEmptyDatabaseMGS(const DatabaseConfiguration& conf, uint64_t defaultDiskSpace) {
+	ASSERT(conf.storageTeamSize > 0);
+	configuration = conf;
+	std::vector<UID> allServers;
+	for(int i = 1; i <= conf.storageTeamSize; ++ i) {
+		allServers.emplace_back(UID(i, 0));
+		servers[allServers.back()] = MockStorageServer(allServers.back(), defaultDiskSpace);
+	}
+	keyServers.insert(allKeys.begin, allServers);
+}
+

fdbserver/SimulatedCluster.actor.cpp

@@ -48,6 +48,7 @@
 #include "flow/FaultInjection.h"
 #include "flow/CodeProbeUtils.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/SimulatedCluster.h"
 
 #undef max
 #undef min
@@ -80,7 +81,7 @@ bool destructed = false;
 
 // Configuration details specified in workload test files that change the simulation
 // environment details
-class TestConfig {
+class TestConfig: public BasicTestConfig {
 	class ConfigBuilder {
 		using value_type = toml::basic_value<toml::discard_comments>;
 		using base_variant = std::variant<int, bool, std::string, std::vector<int>, ConfigDBType>;
@@ -289,11 +290,9 @@ class TestConfig {
 
 public:
 	int extraDB = 0;
-	int minimumReplication = 0;
 	int minimumRegions = 0;
 	bool configureLocked = false;
 	bool startIncompatibleProcess = false;
-	int logAntiQuorum = -1;
 	bool isFirstTestInRestart = false;
 	// 7.0 cannot be downgraded to 6.3 after enabling TSS, so disable TSS for 6.3 downgrade tests
 	bool disableTss = false;
@@ -312,17 +311,15 @@ public:
 	//	5 = "ssd-sharded-rocksdb"
 	// Requires a comma-separated list of numbers WITHOUT whitespaces
 	std::vector<int> storageEngineExcludeTypes;
+	Optional<int> datacenters, stderrSeverity, processesPerMachine;
 	// Set the maximum TLog version that can be selected for a test
 	// Refer to FDBTypes.h::TLogVersion. Defaults to the maximum supported version.
 	int maxTLogVersion = TLogVersion::MAX_SUPPORTED;
-	// Set true to simplify simulation configs for easier debugging
-	bool simpleConfig = false;
 	int extraMachineCountDC = 0;
+
 	Optional<bool> generateFearless, buggify;
-	Optional<int> datacenters, desiredTLogCount, commitProxyCount, grvProxyCount, resolverCount, storageEngineType,
-	    stderrSeverity, machineCount, processesPerMachine, coordinators;
-	bool blobGranulesEnabled = false;
 	Optional<std::string> config;
+	bool blobGranulesEnabled = false;
 	bool randomlyRenameZoneId = false;
 
 	bool allowDefaultTenant = true;
@@ -2516,3 +2513,10 @@ ACTOR void setupAndRun(std::string dataFolder,
 	wait(Never());
 	ASSERT(false);
 }
+
+DatabaseConfiguration generateNormalDatabaseConfiguration(const BasicTestConfig& testConfig, uint64_t defaultDiskSpace) {
+	TestConfig config;
+	config.BasicTestConfig::operator=(testConfig);
+	SimulationConfig simConf(config);
+	return simConf.db;
+}

fdbserver/include/fdbserver/MockGlobalState.h

@@ -0,0 +1,62 @@
+/*
+ * MockGlobalState.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOUNDATIONDB_MOCKGLOBALSTATE_H_H
+#define FOUNDATIONDB_MOCKGLOBALSTATE_H_H
+
+#include "StorageMetrics.h"
+#include "fdbclient/KeyRangeMap.h"
+#include "fdbclient/StorageServerInterface.h"
+#include "fdbclient/DatabaseConfiguration.h"
+#include "SimulatedCluster.h"
+
+class MockStorageServer {
+public:
+	// control plane statistics associated with a real storage server
+	uint64_t usedDiskSpace = 0, availableDiskSpace;
+	KeyRangeMap<KeyRange, uint64_t> shardTotalBytes; // randomly generated in setup phase
+
+	// sampled metrics
+	StorageServerMetrics metrics;
+	CoalescedKeyRangeMap<bool, int64_t, KeyBytesMetric<int64_t>> byteSampleClears;
+
+	StorageServerInterface ssi; // serve RPC requests
+	UID id;
+
+	MockStorageServer() = default;
+	MockStorageServer(const UID& id, uint64_t availableDiskSpace, uint64_t usedDiskSpace = 0)
+	  : usedDiskSpace(usedDiskSpace), availableDiskSpace(availableDiskSpace), id(id) {}
+};
+
+class MockGlobalState {
+public:
+	KeyRangeMap<std::vector<UID>> keyServers; // a shard belongs to which servers
+	std::map<UID, MockStorageServer> servers; // all mock servers
+	DatabaseConfiguration configuration;
+
+	// user defined parameters for mock workload purpose
+	double emptyProb; // probability of doing an empty read
+	uint32_t minByteSize, maxByteSize; // the size band of a point data operation
+
+	void initialAsEmptyDatabaseMGS(const DatabaseConfiguration& conf,
+	                               uint64_t defaultDiskSpace = 1000LL * 1024 * 1024 * 1024);
+};
+
+#endif // FOUNDATIONDB_MOCKGLOBALSTATE_H_H

fdbserver/include/fdbserver/SimulatedCluster.h

@@ -28,4 +28,16 @@ void setupAndRun(std::string const& dataFolder,
                  bool const& restoring,
                  std::string const& whitelistBinPath);
 
+class BasicTestConfig {
+public:
+	int minimumReplication = 0;
+	int logAntiQuorum = -1;
+	// Set true to simplify simulation configs for easier debugging
+	bool simpleConfig = false;
+	Optional<int> desiredTLogCount, commitProxyCount, grvProxyCount, resolverCount, storageEngineType, machineCount,
+	    coordinators;
+};
+
+DatabaseConfiguration generateNormalDatabaseConfiguration(const BasicTestConfig& testConfig);
+
 #endif