diff --git a/fdbserver/include/fdbserver/workloads/workloads.actor.h b/fdbserver/include/fdbserver/workloads/workloads.actor.h index 854963bb93..6896e100f8 100644 --- a/fdbserver/include/fdbserver/workloads/workloads.actor.h +++ b/fdbserver/include/fdbserver/workloads/workloads.actor.h @@ -101,8 +101,8 @@ struct NoOptions {}; struct FailureInjectionWorkload : TestWorkload { FailureInjectionWorkload(WorkloadContext const&); virtual ~FailureInjectionWorkload() {} - virtual bool add(DeterministicRandom& random, WorkloadRequest const& work, CompoundWorkload const& workload); - virtual void initFailureInjectionMode(DeterministicRandom& random, unsigned count); + virtual void initFailureInjectionMode(DeterministicRandom& random); + virtual bool shouldInject(DeterministicRandom& random, const WorkloadRequest& work, const unsigned count) const; Future setupInjectionWorkload(Database const& cx, Future done); Future startInjectionWorkload(Database const& cx, Future done); @@ -137,6 +137,9 @@ struct CompoundWorkload : TestWorkload { CompoundWorkload(WorkloadContext& wcx); CompoundWorkload* add(Reference&& w); void addFailureInjection(WorkloadRequest& work); + bool shouldInjectFailure(DeterministicRandom& random, + const WorkloadRequest& work, + Reference failureInjection) const; std::string description() const override; diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 9d96e14e84..92889bb596 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -399,13 +399,25 @@ void CompoundWorkload::addFailureInjection(WorkloadRequest& work) { if (disabledWorkloads.count(workload->description()) > 0) { continue; } - while (workload->add(random, work, *this)) { + while (shouldInjectFailure(random, work, workload)) { + workload->initFailureInjectionMode(random); failureInjection.push_back(workload); workload = factory->create(*this); } } } +bool CompoundWorkload::shouldInjectFailure(DeterministicRandom& random, + const WorkloadRequest& work, + Reference failure) const { + auto desc = failure->description(); + unsigned alreadyAdded = + std::count_if(workloads.begin(), workloads.end(), [&desc](auto const& w) { return w->description() == desc; }); + alreadyAdded += std::count_if( + failureInjection.begin(), failureInjection.end(), [&desc](auto const& w) { return w->description() == desc; }); + return failure->shouldInject(random, work, alreadyAdded); +} + Future> CompoundWorkload::getMetrics() { return getMetricsCompoundWorkload(this); } @@ -425,24 +437,13 @@ void TestWorkload::disableFailureInjectionWorkloads(std::set& out) FailureInjectionWorkload::FailureInjectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {} -bool FailureInjectionWorkload::add(DeterministicRandom& random, - const WorkloadRequest& work, - const CompoundWorkload& workload) { - auto desc = description(); - unsigned alreadyAdded = std::count_if(workload.workloads.begin(), workload.workloads.end(), [&desc](auto const& w) { - return w->description() == desc; - }); - alreadyAdded += std::count_if(workload.failureInjection.begin(), - workload.failureInjection.end(), - [&desc](auto const& w) { return w->description() == desc; }); - bool willAdd = alreadyAdded < 3 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01(); - if (willAdd) { - initFailureInjectionMode(random, alreadyAdded); - } - return willAdd; -} +void FailureInjectionWorkload::initFailureInjectionMode(DeterministicRandom& random) {} -void FailureInjectionWorkload::initFailureInjectionMode(DeterministicRandom& random, unsigned count) {} +bool FailureInjectionWorkload::shouldInject(DeterministicRandom& random, + const WorkloadRequest& work, + const unsigned alreadyAdded) const { + return alreadyAdded < 3 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01(); +} Future FailureInjectionWorkload::setupInjectionWorkload(const Database& cx, Future done) { return holdWhile(this->setup(cx), done); diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index 2a832674bd..63dd4063fb 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -66,7 +66,7 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload { periodicBroadcastInterval = getOption(options, "periodicBroadcastInterval"_sr, periodicBroadcastInterval); } - void initFailureInjectionMode(DeterministicRandom& random, unsigned count) override { enabled = clientId == 0; } + void initFailureInjectionMode(DeterministicRandom& random) override { enabled = clientId == 0; } std::string description() const override { if (g_simulator == g_network) diff --git a/fdbserver/workloads/MachineAttrition.actor.cpp b/fdbserver/workloads/MachineAttrition.actor.cpp index fb6679feb3..6953a9541c 100644 --- a/fdbserver/workloads/MachineAttrition.actor.cpp +++ b/fdbserver/workloads/MachineAttrition.actor.cpp @@ -112,26 +112,10 @@ struct MachineAttritionWorkload : FailureInjectionWorkload { allowFaultInjection = getOption(options, "allowFaultInjection"_sr, allowFaultInjection); } - bool add(DeterministicRandom& random, WorkloadRequest const& work, CompoundWorkload const& workload) override { - auto desc = this->description(); - unsigned alreadyAdded = std::count_if(workload.workloads.begin(), - workload.workloads.end(), - [&desc](auto const& w) { return w->description() == desc; }); - alreadyAdded += std::count_if(workload.failureInjection.begin(), - workload.failureInjection.end(), - [&desc](auto const& w) { return w->description() == desc; }); - auto res = work.useDatabase && random.random01() < 1.0 / (2.0 + alreadyAdded); - if (res) { - initializeForInjection(random); - } - TraceEvent("AddingFailureInjection") - .detail("Reboot", reboot) - .detail("Replacement", replacement) - .detail("AllowFaultInjection", allowFaultInjection) - .detail("KillDC", killDc) - .detail("KillDataHall", killDatahall) - .detail("KillZone", killZone); - return res; + bool shouldInject(DeterministicRandom& random, + const WorkloadRequest& work, + const unsigned alreadyAdded) const override { + return work.useDatabase && random.random01() < 1.0 / (2.0 + alreadyAdded); } void initializeForInjection(DeterministicRandom& random) { @@ -152,6 +136,13 @@ struct MachineAttritionWorkload : FailureInjectionWorkload { killDatahall = dataHalls.size() > 0 && killDc && random.random01() < 0.5; killZone = zones.size() > 0 && random.random01() < 0.2; } + TraceEvent("AddingFailureInjection") + .detail("Reboot", reboot) + .detail("Replacement", replacement) + .detail("AllowFaultInjection", allowFaultInjection) + .detail("KillDC", killDc) + .detail("KillDataHall", killDatahall) + .detail("KillZone", killZone); } static std::vector getServers() { diff --git a/fdbserver/workloads/RandomClogging.actor.cpp b/fdbserver/workloads/RandomClogging.actor.cpp index f842983234..0cf6796fea 100644 --- a/fdbserver/workloads/RandomClogging.actor.cpp +++ b/fdbserver/workloads/RandomClogging.actor.cpp @@ -43,23 +43,18 @@ struct RandomCloggingWorkload : FailureInjectionWorkload { swizzleClog = getOption(options, "swizzle"_sr, swizzleClog); } - bool add(DeterministicRandom& random, WorkloadRequest const& work, CompoundWorkload const& workload) override { - auto desc = description(); - unsigned alreadyAdded = std::count_if(workload.workloads.begin(), - workload.workloads.end(), - [&desc](auto const& w) { return w->description() == desc; }); - alreadyAdded += std::count_if(workload.failureInjection.begin(), - workload.failureInjection.end(), - [&desc](auto const& w) { return w->description() == desc; }); - bool willAdd = work.useDatabase && 0.25 / (1 + alreadyAdded) > random.random01(); - if (willAdd) { - enabled = this->clientId == 0; - scale = std::max(random.random01(), 0.1); - clogginess = std::max(random.random01(), 0.1); - swizzleClog = random.random01() < 0.3; - iterate = random.random01() < 0.5; - } - return willAdd; + bool shouldInject(DeterministicRandom& random, + const WorkloadRequest& work, + const unsigned alreadyAdded) const override { + return work.useDatabase && 0.25 / (1 + alreadyAdded) > random.random01(); + } + + void initFailureInjectionMode(DeterministicRandom& random) override { + enabled = this->clientId == 0; + scale = std::max(random.random01(), 0.1); + clogginess = std::max(random.random01(), 0.1); + swizzleClog = random.random01() < 0.3; + iterate = random.random01() < 0.5; } std::string description() const override { diff --git a/fdbserver/workloads/RandomMoveKeys.actor.cpp b/fdbserver/workloads/RandomMoveKeys.actor.cpp index fc407efb1a..05b4ffd5c6 100644 --- a/fdbserver/workloads/RandomMoveKeys.actor.cpp +++ b/fdbserver/workloads/RandomMoveKeys.actor.cpp @@ -27,6 +27,7 @@ #include "fdbserver/workloads/workloads.actor.h" #include "fdbserver/ServerDBInfo.h" #include "fdbserver/QuietDatabase.h" +#include "flow/DeterministicRandom.h" #include "flow/actorcompiler.h" // This must be the last #include. struct MoveKeysWorkload : FailureInjectionWorkload { @@ -50,6 +51,12 @@ struct MoveKeysWorkload : FailureInjectionWorkload { Future setup(Database const& cx) override { return Void(); } Future start(Database const& cx) override { return _start(cx, this); } + bool shouldInject(DeterministicRandom& random, + const WorkloadRequest& work, + const unsigned alreadyAdded) const override { + return alreadyAdded < 1 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01(); + } + ACTOR Future _start(Database cx, MoveKeysWorkload* self) { if (self->enabled) { // Get the database configuration so as to use proper team size diff --git a/fdbserver/workloads/Rollback.actor.cpp b/fdbserver/workloads/Rollback.actor.cpp index cadff1a1f4..30f4d6e0dc 100644 --- a/fdbserver/workloads/Rollback.actor.cpp +++ b/fdbserver/workloads/Rollback.actor.cpp @@ -47,7 +47,7 @@ struct RollbackWorkload : FailureInjectionWorkload { multiple = getOption(options, "multiple"_sr, multiple); } - void initFailureInjectionMode(DeterministicRandom& random, unsigned count) override { + void initFailureInjectionMode(DeterministicRandom& random) override { enabled = clientId == 0; multiple = random.coinflip(); enableFailures = random.random01() < 0.2;