Merge pull request #8298 from sfc-gh-huliu/testfix

disallow MoveKeysWorkload running in parallel

fdbserver/include/fdbserver/workloads/workloads.actor.h

@@ -101,8 +101,8 @@ struct NoOptions {};
 struct FailureInjectionWorkload : TestWorkload {
 	FailureInjectionWorkload(WorkloadContext const&);
 	virtual ~FailureInjectionWorkload() {}
-	virtual bool add(DeterministicRandom& random, WorkloadRequest const& work, CompoundWorkload const& workload);
+	virtual void initFailureInjectionMode(DeterministicRandom& random);
-	virtual void initFailureInjectionMode(DeterministicRandom& random, unsigned count);
+	virtual bool shouldInject(DeterministicRandom& random, const WorkloadRequest& work, const unsigned count) const;
 
 	Future<Void> setupInjectionWorkload(Database const& cx, Future<Void> done);
 	Future<Void> startInjectionWorkload(Database const& cx, Future<Void> done);
@@ -137,6 +137,9 @@ struct CompoundWorkload : TestWorkload {
 	CompoundWorkload(WorkloadContext& wcx);
 	CompoundWorkload* add(Reference<TestWorkload>&& w);
 	void addFailureInjection(WorkloadRequest& work);
+	bool shouldInjectFailure(DeterministicRandom& random,
+	                         const WorkloadRequest& work,
+	                         Reference<FailureInjectionWorkload> failureInjection) const;
 
 	std::string description() const override;
 

fdbserver/tester.actor.cpp

@@ -399,13 +399,25 @@ void CompoundWorkload::addFailureInjection(WorkloadRequest& work) {
 		if (disabledWorkloads.count(workload->description()) > 0) {
 			continue;
 		}
-		while (workload->add(random, work, *this)) {
+		while (shouldInjectFailure(random, work, workload)) {
+			workload->initFailureInjectionMode(random);
 			failureInjection.push_back(workload);
 			workload = factory->create(*this);
 		}
 	}
 }
 
+bool CompoundWorkload::shouldInjectFailure(DeterministicRandom& random,
+                                           const WorkloadRequest& work,
+                                           Reference<FailureInjectionWorkload> failure) const {
+	auto desc = failure->description();
+	unsigned alreadyAdded =
+	    std::count_if(workloads.begin(), workloads.end(), [&desc](auto const& w) { return w->description() == desc; });
+	alreadyAdded += std::count_if(
+	    failureInjection.begin(), failureInjection.end(), [&desc](auto const& w) { return w->description() == desc; });
+	return failure->shouldInject(random, work, alreadyAdded);
+}
+
 Future<std::vector<PerfMetric>> CompoundWorkload::getMetrics() {
 	return getMetricsCompoundWorkload(this);
 }
@@ -425,24 +437,13 @@ void TestWorkload::disableFailureInjectionWorkloads(std::set<std::string>& out)
 
 FailureInjectionWorkload::FailureInjectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
 
-bool FailureInjectionWorkload::add(DeterministicRandom& random,
+void FailureInjectionWorkload::initFailureInjectionMode(DeterministicRandom& random) {}
-                                   const WorkloadRequest& work,
-                                   const CompoundWorkload& workload) {
-	auto desc = description();
-	unsigned alreadyAdded = std::count_if(workload.workloads.begin(), workload.workloads.end(), [&desc](auto const& w) {
-		return w->description() == desc;
-	});
-	alreadyAdded += std::count_if(workload.failureInjection.begin(),
-	                              workload.failureInjection.end(),
-	                              [&desc](auto const& w) { return w->description() == desc; });
-	bool willAdd = alreadyAdded < 3 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
-	if (willAdd) {
-		initFailureInjectionMode(random, alreadyAdded);
-	}
-	return willAdd;
-}
 
-void FailureInjectionWorkload::initFailureInjectionMode(DeterministicRandom& random, unsigned count) {}
+bool FailureInjectionWorkload::shouldInject(DeterministicRandom& random,
+                                            const WorkloadRequest& work,
+                                            const unsigned alreadyAdded) const {
+	return alreadyAdded < 3 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
+}
 
 Future<Void> FailureInjectionWorkload::setupInjectionWorkload(const Database& cx, Future<Void> done) {
 	return holdWhile(this->setup(cx), done);

fdbserver/workloads/DiskFailureInjection.actor.cpp

@@ -66,7 +66,7 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
 		periodicBroadcastInterval = getOption(options, "periodicBroadcastInterval"_sr, periodicBroadcastInterval);
 	}
 
-	void initFailureInjectionMode(DeterministicRandom& random, unsigned count) override { enabled = clientId == 0; }
+	void initFailureInjectionMode(DeterministicRandom& random) override { enabled = clientId == 0; }
 
 	std::string description() const override {
 		if (g_simulator == g_network)

fdbserver/workloads/MachineAttrition.actor.cpp

@@ -112,26 +112,10 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 		allowFaultInjection = getOption(options, "allowFaultInjection"_sr, allowFaultInjection);
 	}
 
-	bool add(DeterministicRandom& random, WorkloadRequest const& work, CompoundWorkload const& workload) override {
+	bool shouldInject(DeterministicRandom& random,
-		auto desc = this->description();
+	                  const WorkloadRequest& work,
-		unsigned alreadyAdded = std::count_if(workload.workloads.begin(),
+	                  const unsigned alreadyAdded) const override {
-		                                      workload.workloads.end(),
+		return work.useDatabase && random.random01() < 1.0 / (2.0 + alreadyAdded);
-		                                      [&desc](auto const& w) { return w->description() == desc; });
-		alreadyAdded += std::count_if(workload.failureInjection.begin(),
-		                              workload.failureInjection.end(),
-		                              [&desc](auto const& w) { return w->description() == desc; });
-		auto res = work.useDatabase && random.random01() < 1.0 / (2.0 + alreadyAdded);
-		if (res) {
-			initializeForInjection(random);
-		}
-		TraceEvent("AddingFailureInjection")
-		    .detail("Reboot", reboot)
-		    .detail("Replacement", replacement)
-		    .detail("AllowFaultInjection", allowFaultInjection)
-		    .detail("KillDC", killDc)
-		    .detail("KillDataHall", killDatahall)
-		    .detail("KillZone", killZone);
-		return res;
 	}
 
 	void initializeForInjection(DeterministicRandom& random) {
@@ -152,6 +136,13 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 			killDatahall = dataHalls.size() > 0 && killDc && random.random01() < 0.5;
 			killZone = zones.size() > 0 && random.random01() < 0.2;
 		}
+		TraceEvent("AddingFailureInjection")
+		    .detail("Reboot", reboot)
+		    .detail("Replacement", replacement)
+		    .detail("AllowFaultInjection", allowFaultInjection)
+		    .detail("KillDC", killDc)
+		    .detail("KillDataHall", killDatahall)
+		    .detail("KillZone", killZone);
 	}
 
 	static std::vector<ISimulator::ProcessInfo*> getServers() {

fdbserver/workloads/RandomClogging.actor.cpp

@@ -43,23 +43,18 @@ struct RandomCloggingWorkload : FailureInjectionWorkload {
 		swizzleClog = getOption(options, "swizzle"_sr, swizzleClog);
 	}
 
-	bool add(DeterministicRandom& random, WorkloadRequest const& work, CompoundWorkload const& workload) override {
+	bool shouldInject(DeterministicRandom& random,
-		auto desc = description();
+	                  const WorkloadRequest& work,
-		unsigned alreadyAdded = std::count_if(workload.workloads.begin(),
+	                  const unsigned alreadyAdded) const override {
-		                                      workload.workloads.end(),
+		return work.useDatabase && 0.25 / (1 + alreadyAdded) > random.random01();
-		                                      [&desc](auto const& w) { return w->description() == desc; });
+	}
-		alreadyAdded += std::count_if(workload.failureInjection.begin(),
+
-		                              workload.failureInjection.end(),
+	void initFailureInjectionMode(DeterministicRandom& random) override {
-		                              [&desc](auto const& w) { return w->description() == desc; });
+		enabled = this->clientId == 0;
-		bool willAdd = work.useDatabase && 0.25 / (1 + alreadyAdded) > random.random01();
+		scale = std::max(random.random01(), 0.1);
-		if (willAdd) {
+		clogginess = std::max(random.random01(), 0.1);
-			enabled = this->clientId == 0;
+		swizzleClog = random.random01() < 0.3;
-			scale = std::max(random.random01(), 0.1);
+		iterate = random.random01() < 0.5;
-			clogginess = std::max(random.random01(), 0.1);
-			swizzleClog = random.random01() < 0.3;
-			iterate = random.random01() < 0.5;
-		}
-		return willAdd;
 	}
 
 	std::string description() const override {

fdbserver/workloads/RandomMoveKeys.actor.cpp

@@ -27,6 +27,7 @@
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/ServerDBInfo.h"
 #include "fdbserver/QuietDatabase.h"
+#include "flow/DeterministicRandom.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct MoveKeysWorkload : FailureInjectionWorkload {
@@ -50,6 +51,12 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 	Future<Void> setup(Database const& cx) override { return Void(); }
 	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 
+	bool shouldInject(DeterministicRandom& random,
+	                  const WorkloadRequest& work,
+	                  const unsigned alreadyAdded) const override {
+		return alreadyAdded < 1 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
+	}
+
 	ACTOR Future<Void> _start(Database cx, MoveKeysWorkload* self) {
 		if (self->enabled) {
 			// Get the database configuration so as to use proper team size

fdbserver/workloads/Rollback.actor.cpp

@@ -47,7 +47,7 @@ struct RollbackWorkload : FailureInjectionWorkload {
 		multiple = getOption(options, "multiple"_sr, multiple);
 	}
 
-	void initFailureInjectionMode(DeterministicRandom& random, unsigned count) override {
+	void initFailureInjectionMode(DeterministicRandom& random) override {
 		enabled = clientId == 0;
 		multiple = random.coinflip();
 		enableFailures = random.random01() < 0.2;