wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8298 from sfc-gh-huliu/testfix 

disallow MoveKeysWorkload running in parallel
This commit is contained in:
Markus Pilman 2022-09-23 10:11:08 -06:00 committed by GitHub
parent 993e30e563 c9b4fc5761
commit eaccfc1923
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 56 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
fdbserver/include/fdbserver/workloads/workloads.actor.h
View File

@ -101,8 +101,8 @@ struct NoOptions {};
struct FailureInjectionWorkload : TestWorkload {
FailureInjectionWorkload(WorkloadContext const&);
virtual ~FailureInjectionWorkload() {}
virtual bool add(DeterministicRandom& random, WorkloadRequest const& work, CompoundWorkload const& workload);
virtual void initFailureInjectionMode(DeterministicRandom& random, unsigned count);
virtual void initFailureInjectionMode(DeterministicRandom& random);
virtual bool shouldInject(DeterministicRandom& random, const WorkloadRequest& work, const unsigned count) const;
Future<Void> setupInjectionWorkload(Database const& cx, Future<Void> done);
Future<Void> startInjectionWorkload(Database const& cx, Future<Void> done);
@ -137,6 +137,9 @@ struct CompoundWorkload : TestWorkload {
CompoundWorkload(WorkloadContext& wcx);
CompoundWorkload* add(Reference<TestWorkload>&& w);
void addFailureInjection(WorkloadRequest& work);
bool shouldInjectFailure(DeterministicRandom& random,
const WorkloadRequest& work,
Reference<FailureInjectionWorkload> failureInjection) const;
std::string description() const override;

37
fdbserver/tester.actor.cpp
View File

@ -399,13 +399,25 @@ void CompoundWorkload::addFailureInjection(WorkloadRequest& work) {
if (disabledWorkloads.count(workload->description()) > 0) {
continue;
}
while (workload->add(random, work, *this)) {
while (shouldInjectFailure(random, work, workload)) {
workload->initFailureInjectionMode(random);
failureInjection.push_back(workload);
workload = factory->create(*this);
}
}
}
bool CompoundWorkload::shouldInjectFailure(DeterministicRandom& random,
const WorkloadRequest& work,
Reference<FailureInjectionWorkload> failure) const {
auto desc = failure->description();
unsigned alreadyAdded =
std::count_if(workloads.begin(), workloads.end(), [&desc](auto const& w) { return w->description() == desc; });
alreadyAdded += std::count_if(
failureInjection.begin(), failureInjection.end(), [&desc](auto const& w) { return w->description() == desc; });
return failure->shouldInject(random, work, alreadyAdded);
}
Future<std::vector<PerfMetric>> CompoundWorkload::getMetrics() {
return getMetricsCompoundWorkload(this);
}
@ -425,24 +437,13 @@ void TestWorkload::disableFailureInjectionWorkloads(std::set<std::string>& out)
FailureInjectionWorkload::FailureInjectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
bool FailureInjectionWorkload::add(DeterministicRandom& random,
const WorkloadRequest& work,
const CompoundWorkload& workload) {
auto desc = description();
unsigned alreadyAdded = std::count_if(workload.workloads.begin(), workload.workloads.end(), [&desc](auto const& w) {
return w->description() == desc;
});
alreadyAdded += std::count_if(workload.failureInjection.begin(),
workload.failureInjection.end(),
[&desc](auto const& w) { return w->description() == desc; });
bool willAdd = alreadyAdded < 3 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
if (willAdd) {
initFailureInjectionMode(random, alreadyAdded);
}
return willAdd;
}
void FailureInjectionWorkload::initFailureInjectionMode(DeterministicRandom& random) {}
void FailureInjectionWorkload::initFailureInjectionMode(DeterministicRandom& random, unsigned count) {}
bool FailureInjectionWorkload::shouldInject(DeterministicRandom& random,
const WorkloadRequest& work,
const unsigned alreadyAdded) const {
return alreadyAdded < 3 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
}
Future<Void> FailureInjectionWorkload::setupInjectionWorkload(const Database& cx, Future<Void> done) {
return holdWhile(this->setup(cx), done);

2
fdbserver/workloads/DiskFailureInjection.actor.cpp
View File

@ -66,7 +66,7 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
periodicBroadcastInterval = getOption(options, "periodicBroadcastInterval"_sr, periodicBroadcastInterval);
}
void initFailureInjectionMode(DeterministicRandom& random, unsigned count) override { enabled = clientId == 0; }
void initFailureInjectionMode(DeterministicRandom& random) override { enabled = clientId == 0; }
std::string description() const override {
if (g_simulator == g_network)

31
fdbserver/workloads/MachineAttrition.actor.cpp
View File

@ -112,26 +112,10 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
allowFaultInjection = getOption(options, "allowFaultInjection"_sr, allowFaultInjection);
}
bool add(DeterministicRandom& random, WorkloadRequest const& work, CompoundWorkload const& workload) override {
auto desc = this->description();
unsigned alreadyAdded = std::count_if(workload.workloads.begin(),
workload.workloads.end(),
[&desc](auto const& w) { return w->description() == desc; });
alreadyAdded += std::count_if(workload.failureInjection.begin(),
workload.failureInjection.end(),
[&desc](auto const& w) { return w->description() == desc; });
auto res = work.useDatabase && random.random01() < 1.0 / (2.0 + alreadyAdded);
if (res) {
initializeForInjection(random);
}
TraceEvent("AddingFailureInjection")
.detail("Reboot", reboot)
.detail("Replacement", replacement)
.detail("AllowFaultInjection", allowFaultInjection)
.detail("KillDC", killDc)
.detail("KillDataHall", killDatahall)
.detail("KillZone", killZone);
return res;
bool shouldInject(DeterministicRandom& random,
const WorkloadRequest& work,
const unsigned alreadyAdded) const override {
return work.useDatabase && random.random01() < 1.0 / (2.0 + alreadyAdded);
}
void initializeForInjection(DeterministicRandom& random) {
@ -152,6 +136,13 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
killDatahall = dataHalls.size() > 0 && killDc && random.random01() < 0.5;
killZone = zones.size() > 0 && random.random01() < 0.2;
}
TraceEvent("AddingFailureInjection")
.detail("Reboot", reboot)
.detail("Replacement", replacement)
.detail("AllowFaultInjection", allowFaultInjection)
.detail("KillDC", killDc)
.detail("KillDataHall", killDatahall)
.detail("KillZone", killZone);
}
static std::vector<ISimulator::ProcessInfo*> getServers() {

19
fdbserver/workloads/RandomClogging.actor.cpp
View File

@ -43,24 +43,19 @@ struct RandomCloggingWorkload : FailureInjectionWorkload {
swizzleClog = getOption(options, "swizzle"_sr, swizzleClog);
}
bool add(DeterministicRandom& random, WorkloadRequest const& work, CompoundWorkload const& workload) override {
auto desc = description();
unsigned alreadyAdded = std::count_if(workload.workloads.begin(),
workload.workloads.end(),
[&desc](auto const& w) { return w->description() == desc; });
alreadyAdded += std::count_if(workload.failureInjection.begin(),
workload.failureInjection.end(),
[&desc](auto const& w) { return w->description() == desc; });
bool willAdd = work.useDatabase && 0.25 / (1 + alreadyAdded) > random.random01();
if (willAdd) {
bool shouldInject(DeterministicRandom& random,
const WorkloadRequest& work,
const unsigned alreadyAdded) const override {
return work.useDatabase && 0.25 / (1 + alreadyAdded) > random.random01();
}
void initFailureInjectionMode(DeterministicRandom& random) override {
enabled = this->clientId == 0;
scale = std::max(random.random01(), 0.1);
clogginess = std::max(random.random01(), 0.1);
swizzleClog = random.random01() < 0.3;
iterate = random.random01() < 0.5;
}
return willAdd;
}
std::string description() const override {
if (g_simulator == g_network)

7
fdbserver/workloads/RandomMoveKeys.actor.cpp
View File

@ -27,6 +27,7 @@
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/QuietDatabase.h"
#include "flow/DeterministicRandom.h"
#include "flow/actorcompiler.h" // This must be the last #include.
struct MoveKeysWorkload : FailureInjectionWorkload {
@ -50,6 +51,12 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
Future<Void> setup(Database const& cx) override { return Void(); }
Future<Void> start(Database const& cx) override { return _start(cx, this); }
bool shouldInject(DeterministicRandom& random,
const WorkloadRequest& work,
const unsigned alreadyAdded) const override {
return alreadyAdded < 1 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
}
ACTOR Future<Void> _start(Database cx, MoveKeysWorkload* self) {
if (self->enabled) {
// Get the database configuration so as to use proper team size

2
fdbserver/workloads/Rollback.actor.cpp
View File

@ -47,7 +47,7 @@ struct RollbackWorkload : FailureInjectionWorkload {
multiple = getOption(options, "multiple"_sr, multiple);
}
void initFailureInjectionMode(DeterministicRandom& random, unsigned count) override {
void initFailureInjectionMode(DeterministicRandom& random) override {
enabled = clientId == 0;
multiple = random.coinflip();
enableFailures = random.random01() < 0.2;