Merge pull request #4019 from sfc-gh-anoyes/anoyes/deterministic-status

Make status tests deterministic

fdbclient/StatusClient.actor.cpp

@@ -463,7 +463,7 @@ ACTOR Future<StatusObject> statusFetcherImpl( Reference<ClusterConnectionFile> f
 	state int coordinatorsFaultTolerance = 0;
 
 	try {
-		state int64_t clientTime = time(0);
+		state int64_t clientTime = g_network->timer();
 
 		StatusObject _statusObjClient = wait(clientStatusFetcher(f, &clientMessages, &quorum_reachable, &coordinatorsFaultTolerance));
 		statusObjClient = _statusObjClient;

fdbrpc/FlowTests.actor.cpp

@@ -236,6 +236,7 @@ struct YieldMockNetwork : INetwork, ReferenceCounted<YieldMockNetwork> {
 	virtual void setCurrentTask(TaskPriority taskID) { baseNetwork->setCurrentTask(taskID); }
 	virtual double now() { return baseNetwork->now(); }
 	virtual double timer() { return baseNetwork->timer(); }
+	double timer_monotonic() override { return baseNetwork->timer_monotonic(); }
 	virtual void stop() { return baseNetwork->stop(); }
 	virtual void addStopCallback( std::function<void()> fn ) { ASSERT(false); return; }
 	virtual bool isSimulated() const { return baseNetwork->isSimulated(); }

fdbrpc/sim2.actor.cpp

@@ -751,7 +751,9 @@ public:
 		return timerTime; 
 	}
 
-	virtual Future<class Void> delay( double seconds, TaskPriority taskID ) {
+	double timer_monotonic() override { return timer(); }
+
+	Future<class Void> delay(double seconds, TaskPriority taskID) override {
 		ASSERT(taskID >= TaskPriority::Min && taskID <= TaskPriority::Max);
 		return delay( seconds, taskID, currentProcess );
 	}

fdbserver/Status.actor.cpp

@@ -1055,7 +1055,7 @@ ACTOR static Future<JsonBuilderObject> recoveryStateStatusFetcher(WorkerDetails
 }
 
 ACTOR static Future<double> doGrvProbe(Transaction *tr, Optional<FDBTransactionOptions::Option> priority = Optional<FDBTransactionOptions::Option>()) {
-	state double start = timer_monotonic();
+	state double start = g_network->timer_monotonic();
 
 	loop {
 		try {
@@ -1065,7 +1065,7 @@ ACTOR static Future<double> doGrvProbe(Transaction *tr, Optional<FDBTransactionO
 			}
 
 			wait(success(tr->getReadVersion()));
-			return timer_monotonic() - start;
+			return g_network->timer_monotonic() - start;
 		}
 		catch(Error &e) {
 			wait(tr->onError(e));
@@ -1079,13 +1079,13 @@ ACTOR static Future<double> doReadProbe(Future<double> grvProbe, Transaction *tr
 		throw grv.getError();
 	}
 
-	state double start = timer_monotonic();
+	state double start = g_network->timer_monotonic();
 
 	loop {
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		try {
 			Optional<Standalone<StringRef> > _ = wait(tr->get(LiteralStringRef("\xff/StatusJsonTestKey62793")));
-			return timer_monotonic() - start;
+			return g_network->timer_monotonic() - start;
 		}
 		catch(Error &e) {
 			wait(tr->onError(e));
@@ -1103,7 +1103,7 @@ ACTOR static Future<double> doCommitProbe(Future<double> grvProbe, Transaction *
 	ASSERT(sourceTr->getReadVersion().isReady());
 	tr->setVersion(sourceTr->getReadVersion().get());
 
-	state double start = timer_monotonic();
+	state double start = g_network->timer_monotonic();
 
 	loop {
 		try {
@@ -1111,7 +1111,7 @@ ACTOR static Future<double> doCommitProbe(Future<double> grvProbe, Transaction *
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr->makeSelfConflicting();
 			wait(tr->commit());
-			return timer_monotonic() - start;
+			return g_network->timer_monotonic() - start;
 		}
 		catch(Error &e) {
 			wait(tr->onError(e));
@@ -2606,7 +2606,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 		statusObj["messages"] = messages;
 
-		int64_t clusterTime = time(0);
+		int64_t clusterTime = g_network->timer();
 		if (clusterTime != -1){
 			statusObj["cluster_controller_timestamp"] = clusterTime;
 		}

fdbserver/storageserver.actor.cpp

@@ -1001,7 +1001,7 @@ ACTOR Future<Void> getValueQ( StorageServer* data, GetValueRequest req ) {
 	++data->counters.finishedQueries;
 	--data->readQueueSizeMetric;
 
-	double duration = timer() - req.requestTime();
+	double duration = g_network->timer() - req.requestTime();
 	data->counters.readLatencySample.addMeasurement(duration);
 	if(data->latencyBandConfig.present()) {
 		int maxReadBytes = data->latencyBandConfig.get().readConfig.maxReadBytes.orDefault(std::numeric_limits<int>::max());
@@ -1599,7 +1599,7 @@ ACTOR Future<Void> getKeyValuesQ( StorageServer* data, GetKeyValuesRequest req )
 	++data->counters.finishedQueries;
 	--data->readQueueSizeMetric;
 
-	double duration = timer() - req.requestTime();
+	double duration = g_network->timer() - req.requestTime();
 	data->counters.readLatencySample.addMeasurement(duration);
 	if(data->latencyBandConfig.present()) {
 		int maxReadBytes = data->latencyBandConfig.get().readConfig.maxReadBytes.orDefault(std::numeric_limits<int>::max());
@@ -1665,7 +1665,7 @@ ACTOR Future<Void> getKeyQ( StorageServer* data, GetKeyRequest req ) {
 	++data->counters.finishedQueries;
 	--data->readQueueSizeMetric;
 
-	double duration = timer() - req.requestTime();
+	double duration = g_network->timer() - req.requestTime();
 	data->counters.readLatencySample.addMeasurement(duration);
 	if(data->latencyBandConfig.present()) {
 		int maxReadBytes = data->latencyBandConfig.get().readConfig.maxReadBytes.orDefault(std::numeric_limits<int>::max());

fdbserver/workloads/StatusWorkload.actor.cpp

@@ -27,8 +27,6 @@
 #include "fdbclient/ManagementAPI.actor.h"
 #include "flow/actorcompiler.h"  // This must be the last #include.
 
-extern bool noUnseed;
-
 struct StatusWorkload : TestWorkload {
 	double testDuration, requestsPerSecond;
 	bool enableLatencyBands;
@@ -53,8 +51,6 @@ struct StatusWorkload : TestWorkload {
 			// This is sort of a hack, but generate code coverage *requirements* for everything in schema
 			schemaCoverageRequirements(parsedSchema.get());
 		}
-
-		noUnseed = true;
 	}
 
 	virtual std::string description() { return "StatusWorkload"; }

flow/Net2.actor.cpp

@@ -131,6 +131,7 @@ public:
 	// INetwork interface
 	virtual double now() { return currentTime; };
 	virtual double timer() { return ::timer(); };
+	double timer_monotonic() override { return ::timer_monotonic(); };
 	virtual Future<Void> delay( double seconds, TaskPriority taskId );
 	virtual Future<class Void> yield( TaskPriority taskID );
 	virtual bool check_yield(TaskPriority taskId);

flow/SystemMonitor.cpp

@@ -57,7 +57,7 @@ SystemStatistics customSystemMonitor(std::string eventName, StatisticsState *sta
 	                                                    &ipAddr, &statState->systemState, true);
 	NetworkData netData;
 	netData.init();
-	if (!DEBUG_DETERMINISM && currentStats.initialized) {
+	if (!g_network->isSimulated() && currentStats.initialized) {
 		{
 			TraceEvent(eventName.c_str())
 				.detail("Elapsed", currentStats.elapsed)

flow/network.h

@@ -462,6 +462,9 @@ public:
 	// A wrapper for directly getting the system time. The time returned by now() only updates in the run loop, 
 	// so it cannot be used to measure times of functions that do not have wait statements.
 
+	virtual double timer_monotonic() = 0;
+	// Similar to timer, but monotonic
+
 	virtual Future<class Void> delay( double seconds, TaskPriority taskID ) = 0;
 	// The given future will be set after seconds have elapsed