143 lines
5.1 KiB
C++
143 lines
5.1 KiB
C++
/*
|
|
* LocalRatekeeper.actor.cpp
|
|
*
|
|
* This source file is part of the FoundationDB open source project
|
|
*
|
|
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
#include "fdbserver/workloads/workloads.actor.h"
|
|
#include <fdbserver/Knobs.h>
|
|
#include <flow/actorcompiler.h>
|
|
|
|
namespace {
|
|
|
|
ACTOR Future<StorageServerInterface> getRandomStorage(Database cx) {
|
|
state Transaction tr(cx);
|
|
loop {
|
|
try {
|
|
tr.reset();
|
|
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
|
|
RangeResult range = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
|
|
if (range.size() > 0) {
|
|
auto idx = deterministicRandom()->randomInt(0, range.size());
|
|
return decodeServerListValue(range[idx].value);
|
|
} else {
|
|
wait(delay(1.0));
|
|
}
|
|
} catch (Error& e) {
|
|
wait(tr.onError(e));
|
|
}
|
|
}
|
|
}
|
|
|
|
struct LocalRatekeeperWorkload : TestWorkload {
|
|
|
|
double startAfter = 0.0;
|
|
double blockWritesFor;
|
|
bool testFailed = false;
|
|
|
|
LocalRatekeeperWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
|
|
startAfter = getOption(options, LiteralStringRef("startAfter"), startAfter);
|
|
blockWritesFor = getOption(options,
|
|
LiteralStringRef("blockWritesFor"),
|
|
double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_HARD_MAX) / double(1e6));
|
|
}
|
|
std::string description() const override { return "LocalRatekeeperWorkload"; }
|
|
|
|
ACTOR static Future<Void> testStorage(LocalRatekeeperWorkload* self, Database cx, StorageServerInterface ssi) {
|
|
state Transaction tr(cx);
|
|
state std::vector<Future<GetValueReply>> requests;
|
|
requests.reserve(100);
|
|
loop {
|
|
state StorageQueuingMetricsReply metrics =
|
|
wait(brokenPromiseToNever(ssi.getQueuingMetrics.getReply(StorageQueuingMetricsRequest{})));
|
|
auto durabilityLag = metrics.version - metrics.durableVersion;
|
|
double expectedRateLimit = 1.0;
|
|
if (durabilityLag >= SERVER_KNOBS->STORAGE_DURABILITY_LAG_HARD_MAX) {
|
|
expectedRateLimit = 0.0;
|
|
} else if (durabilityLag > SERVER_KNOBS->STORAGE_DURABILITY_LAG_SOFT_MAX) {
|
|
expectedRateLimit = 1.0 - double(durabilityLag - SERVER_KNOBS->STORAGE_DURABILITY_LAG_SOFT_MAX) /
|
|
double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_HARD_MAX -
|
|
SERVER_KNOBS->STORAGE_DURABILITY_LAG_SOFT_MAX);
|
|
}
|
|
if (expectedRateLimit < metrics.localRateLimit - 0.01 ||
|
|
expectedRateLimit > metrics.localRateLimit + 0.01) {
|
|
self->testFailed = true;
|
|
TraceEvent(SevError, "StorageRateLimitTooFarOff")
|
|
.detail("Storage", ssi.id())
|
|
.detail("Expected", expectedRateLimit)
|
|
.detail("Actual", metrics.localRateLimit);
|
|
}
|
|
tr.reset();
|
|
Version readVersion = wait(tr.getReadVersion());
|
|
requests.clear();
|
|
// we send 100 requests to this storage node and count how many of those get rejected
|
|
for (int i = 0; i < 100; ++i) {
|
|
GetValueRequest req;
|
|
req.version = readVersion;
|
|
// we don't care about the value
|
|
req.key = LiteralStringRef("/lkfs");
|
|
requests.emplace_back(brokenPromiseToNever(ssi.getValue.getReply(req)));
|
|
}
|
|
wait(waitForAllReady(requests));
|
|
int failedRequests = 0;
|
|
int errors = 0;
|
|
for (const auto& resp : requests) {
|
|
if (resp.isError()) {
|
|
self->testFailed = true;
|
|
++errors;
|
|
TraceEvent(SevError, "LoadBalancedResponseReturnedError").error(resp.getError());
|
|
} else if (resp.get().error.present() && resp.get().error.get().code() == error_code_future_version) {
|
|
++failedRequests;
|
|
}
|
|
if (errors > 9) {
|
|
break;
|
|
}
|
|
}
|
|
TraceEvent("RejectedVersions").detail("NumRejected", failedRequests);
|
|
if (self->testFailed) {
|
|
return Void();
|
|
}
|
|
wait(delay(5.0));
|
|
}
|
|
}
|
|
|
|
ACTOR static Future<Void> _start(LocalRatekeeperWorkload* self, Database cx) {
|
|
wait(delay(self->startAfter));
|
|
state StorageServerInterface ssi = wait(getRandomStorage(cx));
|
|
g_simulator.disableFor(format("%s/updateStorage", ssi.id().toString().c_str()), now() + self->blockWritesFor);
|
|
state Future<Void> done = delay(self->blockWritesFor);
|
|
// not much will happen until the storage goes over the soft limit
|
|
wait(delay(double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_SOFT_MAX / 1e6)));
|
|
wait(testStorage(self, cx, ssi) || done);
|
|
return Void();
|
|
}
|
|
|
|
Future<Void> start(Database const& cx) override {
|
|
// we run this only on one client
|
|
if (clientId != 0 || !g_network->isSimulated()) {
|
|
return Void();
|
|
}
|
|
return _start(this, cx);
|
|
}
|
|
Future<bool> check(Database const& cx) override { return !testFailed; }
|
|
void getMetrics(std::vector<PerfMetric>& m) override {}
|
|
};
|
|
|
|
} // namespace
|
|
|
|
WorkloadFactory<LocalRatekeeperWorkload> LocalRatekeeperWorkloadFactory("LocalRatekeeper");
|