wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into arena-get-size-boolean-param

This commit is contained in:
sfc-gh-tclinkenbeard 2022-03-17 23:12:54 -07:00
parent 44b0e05538 3520d7f73c
commit 2c52d28ae2
70 changed files with 1842 additions and 389 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
bindings/c/test/unit/fdb_api.cpp
View File

@ -130,8 +130,15 @@ EmptyFuture Database::create_snapshot(FDBDatabase* db,
return EmptyFuture(fdb_database_create_snapshot(db, uid, uid_length, snap_command, snap_command_length));
}
// Transaction
// Tenant
Tenant::Tenant(FDBDatabase* db, const uint8_t* name, int name_length) {
if (fdb_error_t err = fdb_database_open_tenant(db, name, name_length, &tenant)) {
std::cerr << fdb_get_error(err) << std::endl;
std::abort();
}
}
// Transaction
Transaction::Transaction(FDBDatabase* db) {
if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) {
std::cerr << fdb_get_error(err) << std::endl;
@ -139,6 +146,13 @@ Transaction::Transaction(FDBDatabase* db) {
}
}
Transaction::Transaction(Tenant tenant) {
if (fdb_error_t err = fdb_tenant_create_transaction(tenant.tenant, &tr_)) {
std::cerr << fdb_get_error(err) << std::endl;
std::abort();
}
}
Transaction::~Transaction() {
fdb_transaction_destroy(tr_);
}

10
bindings/c/test/unit/fdb_api.hpp
View File

@ -203,6 +203,15 @@ public:
int snap_command_length);
};
class Tenant final {
public:
Tenant(FDBDatabase* db, const uint8_t* name, int name_length);
private:
friend class Transaction;
FDBTenant* tenant;
};
// Wrapper around FDBTransaction, providing the same set of calls as the C API.
// Handles cleanup of memory, removing the need to call
// fdb_transaction_destroy.
@ -210,6 +219,7 @@ class Transaction final {
public:
// Given an FDBDatabase, initializes a new transaction.
Transaction(FDBDatabase* db);
Transaction(Tenant tenant);
~Transaction();
// Wrapper around fdb_transaction_reset.

95
bindings/c/test/unit/unit_tests.cpp
View File

@ -2410,6 +2410,101 @@ TEST_CASE("Fast alloc thread cleanup") {
}
}
TEST_CASE("Tenant create, access, and delete") {
std::string tenantName = "tenant";
std::string testKey = "foo";
std::string testValue = "bar";
fdb::Transaction tr(db);
while (1) {
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, nullptr, 0));
tr.set("\xff\xff/management/tenant_map/" + tenantName, "");
fdb::EmptyFuture commitFuture = tr.commit();
fdb_error_t err = wait_future(commitFuture);
if (err) {
fdb::EmptyFuture f = tr.on_error(err);
fdb_check(wait_future(f));
continue;
}
tr.reset();
break;
}
fdb::Tenant tenant(db, reinterpret_cast<const uint8_t*>(tenantName.c_str()), tenantName.size());
fdb::Transaction tr2(tenant);
while (1) {
tr2.set(testKey, testValue);
fdb::EmptyFuture commitFuture = tr2.commit();
fdb_error_t err = wait_future(commitFuture);
if (err) {
fdb::EmptyFuture f = tr2.on_error(err);
fdb_check(wait_future(f));
continue;
}
tr2.reset();
break;
}
while (1) {
fdb::ValueFuture f1 = tr2.get(testKey, false);
fdb_error_t err = wait_future(f1);
if (err) {
fdb::EmptyFuture f2 = tr.on_error(err);
fdb_check(wait_future(f2));
continue;
}
int out_present;
char* val;
int vallen;
fdb_check(f1.get(&out_present, (const uint8_t**)&val, &vallen));
CHECK(out_present == 1);
CHECK(vallen == testValue.size());
CHECK(testValue == val);
tr2.clear(testKey);
fdb::EmptyFuture commitFuture = tr2.commit();
err = wait_future(commitFuture);
if (err) {
fdb::EmptyFuture f = tr2.on_error(err);
fdb_check(wait_future(f));
continue;
}
tr2.reset();
break;
}
while (1) {
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, nullptr, 0));
tr.clear("\xff\xff/management/tenant_map/" + tenantName);
fdb::EmptyFuture commitFuture = tr.commit();
fdb_error_t err = wait_future(commitFuture);
if (err) {
fdb::EmptyFuture f = tr.on_error(err);
fdb_check(wait_future(f));
continue;
}
tr.reset();
break;
}
while (1) {
fdb::ValueFuture f1 = tr2.get(testKey, false);
fdb_error_t err = wait_future(f1);
if (err == error_code_tenant_not_found) {
tr2.reset();
break;
}
if (err) {
fdb::EmptyFuture f2 = tr.on_error(err);
fdb_check(wait_future(f2));
continue;
}
}
}
int main(int argc, char** argv) {
if (argc < 3) {
std::cout << "Unit tests for the FoundationDB C API.\n"

9
cmake/ConfigureCompiler.cmake
View File

@ -5,7 +5,6 @@ env_set(USE_DTRACE ON BOOL "Enable dtrace probes on supported platforms")
env_set(USE_VALGRIND OFF BOOL "Compile for valgrind usage")
env_set(USE_VALGRIND_FOR_CTEST ${USE_VALGRIND} BOOL "Use valgrind for ctest")
env_set(ALLOC_INSTRUMENTATION OFF BOOL "Instrument alloc")
env_set(USE_JEMALLOC ON BOOL "Link with jemalloc")
env_set(USE_ASAN OFF BOOL "Compile with address sanitizer")
env_set(USE_GCOV OFF BOOL "Compile with gcov instrumentation")
env_set(USE_MSAN OFF BOOL "Compile with memory sanitizer. To avoid false positives you need to dynamically link to a msan-instrumented libc++ and libc++abi, which you must compile separately. See https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo#instrumented-libc.")
@ -32,6 +31,14 @@ if(USE_ASAN OR USE_VALGRIND OR USE_MSAN OR USE_TSAN OR USE_UBSAN)
set(USE_SANITIZER ON)
endif()
set(jemalloc_default ON)
# We don't want to use jemalloc on Windows
# Nor on FreeBSD, where jemalloc is the default system allocator
if(USE_SANITIZER OR WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") OR APPLE)
set(jemalloc_default OFF)
endif()
env_set(USE_JEMALLOC ${jemalloc_default} BOOL "Link with jemalloc")
if(USE_LIBCXX AND STATIC_LINK_LIBCXX AND NOT USE_LD STREQUAL "LLD")
message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX with libc++ only works if USE_LD=LLD")
endif()

7
cmake/Jemalloc.cmake
View File

@ -1,12 +1,5 @@
add_library(jemalloc INTERFACE)
# We don't want to use jemalloc on Windows
# Nor on FreeBSD, where jemalloc is the default system allocator
if(USE_SANITIZER OR WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") OR APPLE)
set(USE_JEMALLOC OFF)
return()
endif()
if(NOT USE_JEMALLOC)
return()
endif()

9
contrib/TestHarness/Program.cs
View File

@ -359,7 +359,7 @@ namespace SummarizeTest
}
int result = 0;
bool unseedCheck = random.NextDouble() < unseedRatio;
bool unseedCheck = !noSim && random.NextDouble() < unseedRatio;
for (int i = 0; i < maxTries; ++i)
{
bool logOnRetryableError = i == maxTries - 1;
@ -829,17 +829,12 @@ namespace SummarizeTest
if (ev.DDetails.ContainsKey("FaultInjectionEnabled"))
xout.Add(new XAttribute("FaultInjectionEnabled", ev.Details.FaultInjectionEnabled));
}
if (ev.Type == "Simulation")
if (ev.Type == "Simulation" || ev.Type == "NonSimulationTest")
{
xout.Add(
new XAttribute("TestFile", ev.Details.TestFile));
testFile = ev.Details.TestFile.Substring(ev.Details.TestFile.IndexOf("tests"));
}
if (ev.Type == "ActualRun")
{
xout.Add(
new XAttribute("TestFile", ev.Details.RunID));
}
if (ev.Type == "ElapsedTime" && !testEndFound)
{
testEndFound = true;

1
fdbclient/CMakeLists.txt
View File

@ -139,6 +139,7 @@ set(FDBCLIENT_SRCS
TagThrottle.actor.h
TaskBucket.actor.cpp
TaskBucket.h
Tenant.cpp
Tenant.h
TestKnobCollection.cpp
TestKnobCollection.h

5
fdbclient/ClientKnobCollection.h
View File

@ -18,6 +18,9 @@
* limitations under the License.
*/
#ifndef FDBCLIENT_CLIENTKNOBCOLLECTION_H
#define FDBCLIENT_CLIENTKNOBCOLLECTION_H
#pragma once
#include "fdbclient/ClientKnobs.h"
@ -45,3 +48,5 @@ public:
bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) override;
bool isAtomic(std::string const& knobName) const override;
};
#endif // FDBCLIENT_CLIENTKNOBCCOLLECTION_H

3
fdbclient/DatabaseBackupAgent.actor.cpp
View File

@ -32,10 +32,11 @@
#include <numeric>
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbclient/KeyBackedTypes.h"
#include "flow/actorcompiler.h" // has to be last include
#include <inttypes.h>
#include <map>
#include "flow/actorcompiler.h" // has to be last include
const Key DatabaseBackupAgent::keyAddPrefix = LiteralStringRef("add_prefix");
const Key DatabaseBackupAgent::keyRemovePrefix = LiteralStringRef("remove_prefix");
const Key DatabaseBackupAgent::keyRangeVersions = LiteralStringRef("range_versions");

10
fdbclient/DatabaseContext.h
View File

@ -246,7 +246,8 @@ public:
lockAware,
internal,
apiVersion,
switchable));
switchable,
defaultTenant));
}
Optional<KeyRangeLocationInfo> getCachedLocation(const Optional<TenantName>& tenant,
@ -370,7 +371,8 @@ public:
LockAware,
IsInternal = IsInternal::True,
int apiVersion = Database::API_VERSION_LATEST,
IsSwitchable = IsSwitchable::False);
IsSwitchable = IsSwitchable::False,
Optional<TenantName> defaultTenant = Optional<TenantName>());
explicit DatabaseContext(const Error& err);
@ -392,6 +394,10 @@ public:
QueueModel queueModel;
EnableLocalityLoadBalance enableLocalityLoadBalance{ EnableLocalityLoadBalance::False };
// The tenant used when none is specified for a transaction. Ordinarily this is unspecified, in which case the raw
// key-space is used.
Optional<TenantName> defaultTenant;
struct VersionRequest {
SpanID spanContext;
Promise<GetReadVersionReply> reply;

14
fdbclient/GenericManagementAPI.actor.h
View File

@ -633,7 +633,7 @@ ACTOR template <class Transaction>
Future<Optional<TenantMapEntry>> tryGetTenantTransaction(Transaction tr, TenantName name) {
state Key tenantMapKey = name.withPrefix(tenantMapPrefix);
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
Optional<Value> val = wait(safeThreadFutureToFuture(tr->get(tenantMapKey)));
@ -646,6 +646,7 @@ Future<Optional<TenantMapEntry>> tryGetTenant(Reference<DB> db, TenantName name)
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
Optional<TenantMapEntry> entry = wait(tryGetTenantTransaction(tr, name));
return entry;
} catch (Error& e) {
@ -683,7 +684,7 @@ Future<Optional<TenantMapEntry>> createTenantTransaction(Transaction tr, TenantN
throw invalid_tenant_name();
}
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state Future<Optional<TenantMapEntry>> tenantEntryFuture = tryGetTenantTransaction(tr, name);
@ -725,6 +726,8 @@ Future<Void> createTenant(Reference<DB> db, TenantName name) {
state bool firstTry = true;
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (firstTry) {
Optional<TenantMapEntry> entry = wait(tryGetTenantTransaction(tr, name));
if (entry.present()) {
@ -763,7 +766,7 @@ ACTOR template <class Transaction>
Future<Void> deleteTenantTransaction(Transaction tr, TenantNameRef name) {
state Key tenantMapKey = name.withPrefix(tenantMapPrefix);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state Optional<TenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, name));
@ -788,6 +791,8 @@ Future<Void> deleteTenant(Reference<DB> db, TenantName name) {
state bool firstTry = true;
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (firstTry) {
Optional<TenantMapEntry> entry = wait(tryGetTenantTransaction(tr, name));
if (!entry.present()) {
@ -824,7 +829,7 @@ Future<std::map<TenantName, TenantMapEntry>> listTenantsTransaction(Transaction
int limit) {
state KeyRange range = KeyRangeRef(begin, end).withPrefix(tenantMapPrefix);
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
RangeResult results = wait(safeThreadFutureToFuture(
@ -847,6 +852,7 @@ Future<std::map<TenantName, TenantMapEntry>> listTenants(Reference<DB> db,
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
std::map<TenantName, TenantMapEntry> tenants = wait(listTenantsTransaction(tr, begin, end, limit));
return tenants;
} catch (Error& e) {

5
fdbclient/IKnobCollection.h
View File

@ -18,6 +18,9 @@
* limitations under the License.
*/
#ifndef FDBCLIENT_IKNOBCOLLECTION_H
#define FDBCLIENT_IKNOBCOLLECTION_H
#pragma once
#include <memory>
@ -69,3 +72,5 @@ public:
static ConfigMutationRef createSetMutation(Arena, KeyRef, ValueRef);
static ConfigMutationRef createClearMutation(Arena, KeyRef);
};
#endif // FDBCLIENT_IKNOBCOLLECTION_H

3
fdbclient/ManagementAPI.actor.cpp
View File

@ -41,9 +41,10 @@
#include "flow/UnitTest.h"
#include "fdbrpc/ReplicationPolicy.h"
#include "fdbrpc/Replication.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "fdbclient/Schemas.h"
#include "flow/actorcompiler.h" // This must be the last #include.
bool isInteger(const std::string& s) {
if (s.empty())
return false;

130
fdbclient/NativeAPI.actor.cpp
View File

@ -1314,10 +1314,11 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
LockAware lockAware,
IsInternal internal,
int apiVersion,
IsSwitchable switchable)
IsSwitchable switchable,
Optional<TenantName> defaultTenant)
: lockAware(lockAware), switchable(switchable), connectionRecord(connectionRecord), proxyProvisional(false),
clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), internal(internal),
cc("TransactionMetrics"), transactionReadVersions("ReadVersions", cc),
clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), defaultTenant(defaultTenant),
internal(internal), cc("TransactionMetrics"), transactionReadVersions("ReadVersions", cc),
transactionReadVersionsThrottled("ReadVersionsThrottled", cc),
transactionReadVersionsCompleted("ReadVersionsCompleted", cc),
transactionReadVersionBatches("ReadVersionBatches", cc),
@ -2795,7 +2796,7 @@ Future<KeyRangeLocationInfo> getKeyLocation(Reference<TransactionState> trState,
UseTenant useTenant,
Version version) {
auto f = getKeyLocation(trState->cx,
useTenant ? trState->tenant : Optional<TenantName>(),
useTenant ? trState->tenant() : Optional<TenantName>(),
key,
member,
trState->spanID,
@ -2804,7 +2805,7 @@ Future<KeyRangeLocationInfo> getKeyLocation(Reference<TransactionState> trState,
isBackward,
version);
if (trState->tenant.present() && useTenant) {
if (trState->tenant().present() && useTenant) {
return map(f, [trState](const KeyRangeLocationInfo& locationInfo) {
trState->tenantId = locationInfo.tenantEntry.id;
return locationInfo;
@ -2939,7 +2940,7 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Reference<Transac
UseTenant useTenant,
Version version) {
auto f = getKeyRangeLocations(trState->cx,
useTenant ? trState->tenant : Optional<TenantName>(),
useTenant ? trState->tenant() : Optional<TenantName>(),
keys,
limit,
reverse,
@ -2949,7 +2950,7 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Reference<Transac
trState->useProvisionalProxies,
version);
if (trState->tenant.present() && useTenant) {
if (trState->tenant().present() && useTenant) {
return map(f, [trState](const std::vector<KeyRangeLocationInfo>& locationInfo) {
ASSERT(!locationInfo.empty());
trState->tenantId = locationInfo[0].tenantEntry.id;
@ -2969,7 +2970,7 @@ ACTOR Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange
loop {
std::vector<KeyRangeLocationInfo> locations =
wait(getKeyRangeLocations_internal(trState->cx,
trState->tenant,
trState->tenant(),
keys,
CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT,
Reverse::False,
@ -2987,7 +2988,7 @@ ACTOR Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange
if (totalRequests % 20 == 0) {
// To avoid blocking the proxies from starting other transactions, occasionally get a read version.
state Transaction tr(trState->cx, trState->tenant);
state Transaction tr(trState->cx, trState->tenant());
loop {
try {
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@ -3027,14 +3028,15 @@ TransactionState::TransactionState(Database cx,
TaskPriority taskID,
SpanID spanID,
Reference<TransactionLogInfo> trLogInfo)
: cx(cx), tenant(tenant), trLogInfo(trLogInfo), options(cx), taskID(taskID), spanID(spanID) {}
: cx(cx), trLogInfo(trLogInfo), options(cx), taskID(taskID), spanID(spanID), tenant_(tenant),
tenantSet(tenant.present()) {}
Reference<TransactionState> TransactionState::cloneAndReset(Reference<TransactionLogInfo> newTrLogInfo,
bool generateNewSpan) const {
SpanID newSpanID = generateNewSpan ? generateSpanID(cx->transactionTracingSample) : spanID;
Reference<TransactionState> newState =
makeReference<TransactionState>(cx, tenant, cx->taskID, newSpanID, newTrLogInfo);
makeReference<TransactionState>(cx, tenant_, cx->taskID, newSpanID, newTrLogInfo);
if (!cx->apiVersionAtLeast(16)) {
newState->options = options;
@ -3044,22 +3046,42 @@ Reference<TransactionState> TransactionState::cloneAndReset(Reference<Transactio
newState->startTime = startTime;
newState->committedVersion = committedVersion;
newState->conflictingKeys = conflictingKeys;
newState->tenantSet = tenantSet;
return newState;
}
TenantInfo TransactionState::getTenantInfo() const {
if (!cx->internal && !options.rawAccess && cx->clientInfo->get().tenantMode == TenantMode::REQUIRED &&
!tenant.present()) {
throw tenant_name_required();
} else if (options.rawAccess || !tenant.present()) {
TenantInfo TransactionState::getTenantInfo() {
Optional<TenantName> const& t = tenant();
if (options.rawAccess) {
return TenantInfo();
} else if (cx->clientInfo->get().tenantMode == TenantMode::DISABLED && tenant.present()) {
} else if (!cx->internal && cx->clientInfo->get().tenantMode == TenantMode::REQUIRED && !t.present()) {
throw tenant_name_required();
} else if (!t.present()) {
return TenantInfo();
} else if (cx->clientInfo->get().tenantMode == TenantMode::DISABLED && t.present()) {
throw tenants_disabled();
}
ASSERT(tenantId != TenantInfo::INVALID_TENANT);
return TenantInfo(tenant.get(), tenantId);
return TenantInfo(t.get(), tenantId);
}
Optional<TenantName> const& TransactionState::tenant() {
if (tenantSet) {
return tenant_;
} else {
if (!tenant_.present() && !options.rawAccess) {
tenant_ = cx->defaultTenant;
}
tenantSet = true;
return tenant_;
}
}
bool TransactionState::hasTenant() const {
return tenantSet && tenant_.present();
}
Future<Void> Transaction::warmRange(KeyRange keys) {
@ -3073,8 +3095,8 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
TransactionRecordLogInfo recordLogInfo) {
state Version ver = wait(version);
state Span span("NAPI:getValue"_loc, trState->spanID);
if (useTenant && trState->tenant.present()) {
span.addTag("tenant"_sr, trState->tenant.get());
if (useTenant && trState->tenant().present()) {
span.addTag("tenant"_sr, trState->tenant().get());
}
span.addTag("key"_sr, key);
@ -3142,7 +3164,7 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
if (trState->trLogInfo && recordLogInfo) {
int valueSize = reply.value.present() ? reply.value.get().size() : 0;
trState->trLogInfo->addLog(FdbClientLogEvents::EventGet(
startTimeD, trState->cx->clientLocality.dcId(), latency, valueSize, key, trState->tenant));
startTimeD, trState->cx->clientLocality.dcId(), latency, valueSize, key, trState->tenant()));
}
trState->cx->getValueCompleted->latency = timer_int() - startTime;
trState->cx->getValueCompleted->log();
@ -3177,8 +3199,8 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
trState->cx->invalidateCache(locationInfo.tenantEntry.prefix, key);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
} else if (e.code() == error_code_unknown_tenant) {
ASSERT(useTenant && trState->tenant.present());
trState->cx->invalidateCachedTenant(trState->tenant.get());
ASSERT(useTenant && trState->tenant().present());
trState->cx->invalidateCachedTenant(trState->tenant().get());
wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
} else {
if (trState->trLogInfo && recordLogInfo)
@ -3186,7 +3208,7 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
trState->cx->clientLocality.dcId(),
static_cast<int>(e.code()),
key,
trState->tenant));
trState->tenant()));
throw e;
}
}
@ -3284,8 +3306,8 @@ ACTOR Future<Key> getKey(Reference<TransactionState> trState,
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
} else if (e.code() == error_code_unknown_tenant) {
ASSERT(useTenant && trState->tenant.present());
trState->cx->invalidateCachedTenant(trState->tenant.get());
ASSERT(useTenant && trState->tenant().present());
trState->cx->invalidateCachedTenant(trState->tenant().get());
wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
} else {
TraceEvent(SevInfo, "GetKeyError").error(e).detail("AtKey", k.getKey()).detail("Offset", k.offset);
@ -3630,8 +3652,8 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
state RangeResultFamily output;
state Span span("NAPI:getExactRange"_loc, trState->spanID);
if (useTenant && trState->tenant.present()) {
span.addTag("tenant"_sr, trState->tenant.get());
if (useTenant && trState->tenant().present()) {
span.addTag("tenant"_sr, trState->tenant().get());
}
// printf("getExactRange( '%s', '%s' )\n", keys.begin.toString().c_str(), keys.end.toString().c_str());
@ -3792,14 +3814,14 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
break;
} else if (e.code() == error_code_unknown_tenant) {
ASSERT(useTenant && trState->tenant.present());
trState->cx->invalidateCachedTenant(trState->tenant.get());
ASSERT(useTenant && trState->tenant().present());
trState->cx->invalidateCachedTenant(trState->tenant().get());
wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
break;
} else {
TraceEvent(SevInfo, "GetExactRangeError")
.error(e)
.detail("Tenant", trState->tenant)
.detail("Tenant", trState->tenant())
.detail("ShardBegin", locations[shard].range.begin)
.detail("ShardEnd", locations[shard].range.end);
throw;
@ -3859,6 +3881,12 @@ Future<RangeResultFamily> getRangeFallback(Reference<TransactionState> trState,
if (b == allKeys.begin && ((reverse && !r.more) || !reverse))
r.readToBegin = true;
// TODO: this currently causes us to have a conflict range that is too large if our end key resolves to the
// key after the last key in the database. In that case, we don't need a conflict between the last key and
// the end of the database.
//
// If fixed, the ConflictRange test can be updated to stop checking for this condition.
if (e == allKeys.end && ((!reverse && !r.more) || reverse))
r.readThroughEnd = true;
@ -3924,7 +3952,7 @@ void getRangeFinished(Reference<TransactionState> trState,
bytes,
begin.getKey(),
end.getKey(),
trState->tenant));
trState->tenant()));
}
if (!snapshot) {
@ -3981,8 +4009,8 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
state KeySelector originalEnd = end;
state RangeResultFamily output;
state Span span("NAPI:getRange"_loc, trState->spanID);
if (useTenant && trState->tenant.present()) {
span.addTag("tenant"_sr, trState->tenant.get());
if (useTenant && trState->tenant().present()) {
span.addTag("tenant"_sr, trState->tenant().get());
}
try {
@ -4248,8 +4276,8 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
} else if (e.code() == error_code_unknown_tenant) {
ASSERT(useTenant && trState->tenant.present());
trState->cx->invalidateCachedTenant(trState->tenant.get());
ASSERT(useTenant && trState->tenant().present());
trState->cx->invalidateCachedTenant(trState->tenant().get());
wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
} else {
if (trState->trLogInfo)
@ -4259,7 +4287,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
static_cast<int>(e.code()),
begin.getKey(),
end.getKey(),
trState->tenant));
trState->tenant()));
throw e;
}
@ -4704,8 +4732,8 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
break;
} else if (e.code() == error_code_unknown_tenant) {
ASSERT(trState->tenant.present());
trState->cx->invalidateCachedTenant(trState->tenant.get());
ASSERT(trState->tenant().present());
trState->cx->invalidateCachedTenant(trState->tenant().get());
wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
break;
} else {
@ -4979,7 +5007,7 @@ ACTOR Future<TenantInfo> getTenantMetadata(Reference<TransactionState> trState,
}
Future<TenantInfo> populateAndGetTenant(Reference<TransactionState> trState, Key const& key, Version version) {
if (!trState->tenant.present() || key == metadataVersionKey) {
if (!trState->tenant().present() || key == metadataVersionKey) {
return TenantInfo();
} else if (trState->tenantId != TenantInfo::INVALID_TENANT) {
return trState->getTenantInfo();
@ -5066,7 +5094,7 @@ ACTOR Future<Standalone<VectorRef<const char*>>> getAddressesForKeyActor(Referen
state std::vector<StorageServerInterface> ssi;
state Key resolvedKey = key;
if (trState->tenant.present()) {
if (trState->tenant().present()) {
state Version version = wait(ver);
KeyRangeLocationInfo locationInfo = wait(getKeyLocation(
trState, ""_sr, &StorageServerInterface::getValue, Reverse::False, UseTenant::True, version));
@ -5864,7 +5892,7 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
}
state Key tenantPrefix;
if (trState->tenant.present()) {
if (trState->tenant().present()) {
KeyRangeLocationInfo locationInfo = wait(getKeyLocation(trState,
""_sr,
&StorageServerInterface::getValue,
@ -5951,7 +5979,7 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
req.transaction.mutations.expectedSize(),
ci.version,
req,
trState->tenant));
trState->tenant()));
return Void();
} else {
// clear the RYW transaction which contains previous conflicting keys
@ -6009,8 +6037,8 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
// retry it anyway (relying on transaction idempotence) but a client might do something else.
throw commit_unknown_result();
} else if (e.code() == error_code_unknown_tenant) {
ASSERT(trState->tenant.present());
trState->cx->invalidateCachedTenant(trState->tenant.get());
ASSERT(trState->tenant().present());
trState->cx->invalidateCachedTenant(trState->tenant().get());
throw;
} else {
if (e.code() != error_code_transaction_too_old && e.code() != error_code_not_committed &&
@ -6022,7 +6050,7 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
}
if (trState->trLogInfo)
trState->trLogInfo->addLog(FdbClientLogEvents::EventCommitError(
startTime, trState->cx->clientLocality.dcId(), static_cast<int>(e.code()), req, trState->tenant));
startTime, trState->cx->clientLocality.dcId(), static_cast<int>(e.code()), req, trState->tenant()));
throw;
}
}
@ -6368,9 +6396,9 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
// System key access implies raw access. Native API handles the raw access,
// system key access is handled in RYW.
validateOptionValueNotPresent(value);
if (trState->tenant.present()) {
if (trState->hasTenant()) {
Error e = invalid_option();
TraceEvent(SevWarn, "TenantTransactionRawAccess").error(e).detail("Tenant", trState->tenant);
TraceEvent(SevWarn, "TenantTransactionRawAccess").error(e).detail("Tenant", trState->tenant());
throw e;
}
trState->options.rawAccess = true;
@ -6563,7 +6591,7 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
latency,
trState->options.priority,
rep.version,
trState->tenant));
trState->tenant()));
if (rep.locked && !trState->options.lockAware)
throw database_locked();
@ -7235,8 +7263,8 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(Reference<Transa
trState->cx->invalidateCache(locations[0].tenantEntry.prefix, keys);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
} else if (e.code() == error_code_unknown_tenant) {
ASSERT(trState->tenant.present());
trState->cx->invalidateCachedTenant(trState->tenant.get());
ASSERT(trState->tenant().present());
trState->cx->invalidateCachedTenant(trState->tenant().get());
wait(delay(CLIENT_KNOBS->UNKNOWN_TENANT_RETRY_DELAY, trState->taskID));
} else {
TraceEvent(SevError, "GetRangeSplitPoints").error(e);

14
fdbclient/NativeAPI.actor.h
View File

@ -237,7 +237,6 @@ struct Watch : public ReferenceCounted<Watch>, NonCopyable {
struct TransactionState : ReferenceCounted<TransactionState> {
Database cx;
Optional<TenantName> tenant;
int64_t tenantId = TenantInfo::INVALID_TENANT;
Reference<TransactionLogInfo> trLogInfo;
TransactionOptions options;
@ -259,7 +258,7 @@ struct TransactionState : ReferenceCounted<TransactionState> {
std::shared_ptr<CoalescedKeyRangeMap<Value>> conflictingKeys;
// Only available so that Transaction can have a default constructor, for use in state variables
TransactionState(TaskPriority taskID, SpanID spanID) : taskID(taskID), spanID(spanID) {}
TransactionState(TaskPriority taskID, SpanID spanID) : taskID(taskID), spanID(spanID), tenantSet(false) {}
TransactionState(Database cx,
Optional<TenantName> tenant,
@ -268,7 +267,14 @@ struct TransactionState : ReferenceCounted<TransactionState> {
Reference<TransactionLogInfo> trLogInfo);
Reference<TransactionState> cloneAndReset(Reference<TransactionLogInfo> newTrLogInfo, bool generateNewSpan) const;
TenantInfo getTenantInfo() const;
TenantInfo getTenantInfo();
Optional<TenantName> const& tenant();
bool hasTenant() const;
private:
Optional<TenantName> tenant_;
bool tenantSet;
};
class Transaction : NonCopyable {
@ -447,7 +453,7 @@ public:
return Standalone<VectorRef<KeyRangeRef>>(tr.transaction.write_conflict_ranges, tr.arena);
}
Optional<TenantName> getTenant() { return trState->tenant; }
Optional<TenantName> getTenant() { return trState->tenant(); }
Reference<TransactionState> trState;
std::vector<Reference<Watch>> watches;

5
fdbclient/ServerKnobCollection.h
View File

@ -18,6 +18,9 @@
* limitations under the License.
*/
#ifndef FDBCLIENT_SERVERKNOBCOLLECTION_H
#define FDBCLIENT_SERVERKNOBCOLLECTION_H
#pragma once
#include "fdbclient/ClientKnobCollection.h"
@ -43,3 +46,5 @@ public:
bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) override;
bool isAtomic(std::string const& knobName) const override;
};
#endif // FDBCLIENT_SERVERKNOBCOLLECTION_H

56
fdbclient/SpecialKeySpace.actor.cpp
View File

@ -847,7 +847,7 @@ ExcludeServersRangeImpl::ExcludeServersRangeImpl(KeyRangeRef kr) : SpecialKeyRan
Future<RangeResult> ExcludeServersRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr,
GetRangeLimits limitsHint) const {
ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
return rwModuleWithMappingGetRangeActor(ryw, this, kr);
}
@ -1026,7 +1026,7 @@ void includeServers(ReadYourWritesTransaction* ryw) {
ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
ryw->setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
// includeServers might be used in an emergency transaction, so make sure it is retry-self-conflicting and
// CAUSAL_WRITE_RISKY
ryw->setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
@ -1090,7 +1090,7 @@ FailedServersRangeImpl::FailedServersRangeImpl(KeyRangeRef kr) : SpecialKeyRange
Future<RangeResult> FailedServersRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr,
GetRangeLimits limitsHint) const {
ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
return rwModuleWithMappingGetRangeActor(ryw, this, kr);
}
@ -1116,7 +1116,7 @@ Future<Optional<std::string>> FailedServersRangeImpl::commit(ReadYourWritesTrans
ACTOR Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
state RangeResult result;
state Transaction& tr = ryw->getTransaction();
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@ -1177,7 +1177,7 @@ Future<RangeResult> ExclusionInProgressRangeImpl::getRange(ReadYourWritesTransac
}
ACTOR Future<RangeResult> getProcessClassActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
std::vector<ProcessData> _workers = wait(getWorkers(&ryw->getTransaction()));
auto workers = _workers; // strip const
// Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5
@ -1201,7 +1201,7 @@ ACTOR Future<Optional<std::string>> processClassCommitActor(ReadYourWritesTransa
ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
ryw->setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
std::vector<ProcessData> workers = wait(
getWorkers(&ryw->getTransaction())); // make sure we use the Transaction object to avoid used_during_commit()
@ -1295,7 +1295,7 @@ void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef&
}
ACTOR Future<RangeResult> getProcessClassSourceActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
std::vector<ProcessData> _workers = wait(getWorkers(&ryw->getTransaction()));
auto workers = _workers; // strip const
// Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5
@ -1326,7 +1326,7 @@ Future<RangeResult> ProcessClassSourceRangeImpl::getRange(ReadYourWritesTransact
ACTOR Future<RangeResult> getLockedKeyActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->getTransaction().setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(ryw->getTransaction().get(databaseLockedKey));
RangeResult result;
if (val.present()) {
@ -1359,7 +1359,7 @@ Future<RangeResult> LockDatabaseImpl::getRange(ReadYourWritesTransaction* ryw,
ACTOR Future<Optional<std::string>> lockDatabaseCommitActor(ReadYourWritesTransaction* ryw, UID uid) {
state Optional<std::string> msg;
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->getTransaction().setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(ryw->getTransaction().get(databaseLockedKey));
if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != uid) {
@ -1381,7 +1381,7 @@ ACTOR Future<Optional<std::string>> lockDatabaseCommitActor(ReadYourWritesTransa
ACTOR Future<Optional<std::string>> unlockDatabaseCommitActor(ReadYourWritesTransaction* ryw) {
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->getTransaction().setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(ryw->getTransaction().get(databaseLockedKey));
if (val.present()) {
ryw->getTransaction().clear(singleKeyRange(databaseLockedKey));
@ -1408,7 +1408,7 @@ Future<Optional<std::string>> LockDatabaseImpl::commit(ReadYourWritesTransaction
ACTOR Future<RangeResult> getConsistencyCheckKeyActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->getTransaction().setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
ryw->getTransaction().setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
Optional<Value> val = wait(ryw->getTransaction().get(fdbShouldConsistencyCheckBeSuspended));
bool ccSuspendSetting = val.present() ? BinaryReader::fromStringRef<bool>(val.get(), Unversioned()) : false;
@ -1444,7 +1444,7 @@ Future<Optional<std::string>> ConsistencyCheckImpl::commit(ReadYourWritesTransac
ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck")].second;
ryw->getTransaction().setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->getTransaction().setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
ryw->getTransaction().set(fdbShouldConsistencyCheckBeSuspended,
BinaryWriter::toValue(entry.present(), Unversioned()));
return Optional<std::string>();
@ -1503,7 +1503,7 @@ void GlobalConfigImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, co
ACTOR Future<Optional<std::string>> globalConfigCommitActor(GlobalConfigImpl* globalConfig,
ReadYourWritesTransaction* ryw) {
state Transaction& tr = ryw->getTransaction();
ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
// History should only contain three most recent updates. If it currently
// has three items, remove the oldest to make room for a new item.
@ -1787,7 +1787,7 @@ ACTOR static Future<RangeResult> CoordinatorsAutoImplActor(ReadYourWritesTransac
state Transaction& tr = ryw->getTransaction();
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
Optional<Value> currentKey = wait(tr.get(coordinatorsKey));
@ -1834,7 +1834,7 @@ Future<RangeResult> CoordinatorsAutoImpl::getRange(ReadYourWritesTransaction* ry
ACTOR static Future<RangeResult> getMinCommitVersionActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->getTransaction().setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(ryw->getTransaction().get(minRequiredCommitVersionKey));
RangeResult result;
if (val.present()) {
@ -1872,7 +1872,7 @@ ACTOR static Future<Optional<std::string>> advanceVersionCommitActor(ReadYourWri
std::numeric_limits<int64_t>::max() - 1 - CLIENT_KNOBS->VERSIONS_PER_SECOND * 3600 * 24 * 365 * 1000;
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->getTransaction().setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
TraceEvent(SevDebug, "AdvanceVersion").detail("MaxAllowedVersion", maxAllowedVerion);
if (v > maxAllowedVerion) {
return ManagementAPIError::toJsonString(
@ -1891,7 +1891,7 @@ ACTOR static Future<Optional<std::string>> advanceVersionCommitActor(ReadYourWri
}
Future<Optional<std::string>> AdvanceVersionImpl::commit(ReadYourWritesTransaction* ryw) {
ryw->getTransaction().setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
auto minCommitVersion =
ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("advanceversion")].second;
if (minCommitVersion.present()) {
@ -1918,7 +1918,7 @@ ACTOR static Future<RangeResult> ClientProfilingGetRangeActor(ReadYourWritesTran
// client_txn_sample_rate
state Key sampleRateKey = LiteralStringRef("client_txn_sample_rate").withPrefix(prefix);
ryw->getTransaction().setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
if (kr.contains(sampleRateKey)) {
auto entry = ryw->getSpecialKeySpaceWriteMap()[sampleRateKey];
@ -1969,7 +1969,7 @@ Future<RangeResult> ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw
}
Future<Optional<std::string>> ClientProfilingImpl::commit(ReadYourWritesTransaction* ryw) {
ryw->getTransaction().setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
// client_txn_sample_rate
Key sampleRateKey = LiteralStringRef("client_txn_sample_rate").withPrefix(getKeyRange().begin);
@ -2337,7 +2337,7 @@ ACTOR static Future<RangeResult> MaintenanceGetRangeActor(ReadYourWritesTransact
state RangeResult result;
// zoneId
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->getTransaction().setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(ryw->getTransaction().get(healthyZoneKey));
if (val.present()) {
auto healthyZone = decodeHealthyZoneValue(val.get());
@ -2371,7 +2371,7 @@ Future<RangeResult> MaintenanceImpl::getRange(ReadYourWritesTransaction* ryw,
ACTOR static Future<Optional<std::string>> maintenanceCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
// read
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->getTransaction().setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
ryw->getTransaction().setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
Optional<Value> val = wait(ryw->getTransaction().get(healthyZoneKey));
Optional<std::pair<Key, Version>> healthyZone =
@ -2436,7 +2436,7 @@ ACTOR static Future<RangeResult> DataDistributionGetRangeActor(ReadYourWritesTra
// dataDistributionModeKey
state Key modeKey = LiteralStringRef("mode").withPrefix(prefix);
ryw->getTransaction().setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
if (kr.contains(modeKey)) {
auto entry = ryw->getSpecialKeySpaceWriteMap()[modeKey];
@ -2473,7 +2473,7 @@ Future<Optional<std::string>> DataDistributionImpl::commit(ReadYourWritesTransac
// there are two valid keys in the range
// <prefix>/mode -> dataDistributionModeKey, the value is only allowed to be set as "0"(disable) or "1"(enable)
// <prefix>/rebalance_ignored -> rebalanceDDIgnoreKey, value is unused thus empty
ryw->getTransaction().setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<std::string> msg;
KeyRangeRef kr = getKeyRange();
@ -2542,7 +2542,7 @@ Future<Optional<std::string>> DataDistributionImpl::commit(ReadYourWritesTransac
void includeLocalities(ReadYourWritesTransaction* ryw) {
ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
ryw->setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
// includeLocalities might be used in an emergency transaction, so make sure it is retry-self-conflicting and
// CAUSAL_WRITE_RISKY
@ -2624,7 +2624,7 @@ ACTOR Future<Optional<std::string>> excludeLocalityCommitActor(ReadYourWritesTra
state std::vector<AddressExclusion> addresses;
state std::set<AddressExclusion> exclusions;
ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
state std::vector<ProcessData> workers = wait(getWorkers(&ryw->getTransaction()));
if (!parseLocalitiesFromKeys(ryw, failed, localities, addresses, exclusions, workers, result))
@ -2650,7 +2650,7 @@ ExcludedLocalitiesRangeImpl::ExcludedLocalitiesRangeImpl(KeyRangeRef kr) : Speci
Future<RangeResult> ExcludedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr,
GetRangeLimits limitsHint) const {
ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
return rwModuleWithMappingGetRangeActor(ryw, this, kr);
}
@ -2679,7 +2679,7 @@ FailedLocalitiesRangeImpl::FailedLocalitiesRangeImpl(KeyRangeRef kr) : SpecialKe
Future<RangeResult> FailedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr,
GetRangeLimits limitsHint) const {
ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
return rwModuleWithMappingGetRangeActor(ryw, this, kr);
}
@ -2713,7 +2713,7 @@ ACTOR Future<RangeResult> getTenantList(ReadYourWritesTransaction* ryw, KeyRange
TenantMapRangeImpl::submoduleRange.begin.size());
std::map<TenantName, TenantMapEntry> tenants = wait(ManagementAPI::listTenantsTransaction(
Reference<ReadYourWritesTransaction>::addRef(ryw), tenantRange.begin, tenantRange.end, limitsHint.rows));
&ryw->getTransaction(), tenantRange.begin, tenantRange.end, limitsHint.rows));
RangeResult results;
for (auto tenant : tenants) {

3
fdbclient/StatusClient.actor.cpp
View File

@ -27,9 +27,10 @@
#include "fdbclient/json_spirit/json_spirit_writer_template.h"
#include "fdbclient/json_spirit/json_spirit_reader_template.h"
#include "fdbrpc/genericactors.actor.h"
#include "flow/actorcompiler.h" // has to be last include
#include <cstdint>
#include "flow/actorcompiler.h" // has to be last include
json_spirit::mValue readJSONStrictly(const std::string& s) {
json_spirit::mValue val;
std::string::const_iterator i = s.begin();

57
fdbclient/Tenant.cpp Normal file
View File

@ -0,0 +1,57 @@
/*
* Tenant.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbclient/SystemData.h"
#include "fdbclient/Tenant.h"
#include "flow/UnitTest.h"
TEST_CASE("/fdbclient/TenantMapEntry/Serialization") {
TenantMapEntry entry1(1, ""_sr);
ASSERT(entry1.prefix == "\x00\x00\x00\x00\x00\x00\x00\x01"_sr);
TenantMapEntry entry2 = decodeTenantEntry(encodeTenantEntry(entry1));
ASSERT(entry1.id == entry2.id && entry1.prefix == entry2.prefix);
TenantMapEntry entry3(std::numeric_limits<int64_t>::max(), "foo"_sr);
ASSERT(entry3.prefix == "foo\xfe\xff\xff\xff\xff\xff\xff\xff"_sr);
TenantMapEntry entry4 = decodeTenantEntry(encodeTenantEntry(entry3));
ASSERT(entry3.id == entry4.id && entry3.prefix == entry4.prefix);
for (int i = 0; i < 100; ++i) {
int bits = deterministicRandom()->randomInt(1, 64);
int64_t min = bits == 1 ? 0 : (1 << (bits - 1));
int64_t maxPlusOne = std::min<uint64_t>(1 << bits, std::numeric_limits<int64_t>::max());
int64_t id = deterministicRandom()->randomInt64(min, maxPlusOne);
int subspaceLength = deterministicRandom()->randomInt(0, 20);
Standalone<StringRef> subspace = makeString(subspaceLength);
generateRandomData(mutateString(subspace), subspaceLength);
TenantMapEntry entry(id, subspace);
int64_t bigEndianId = bigEndian64(id);
ASSERT(entry.id == id && entry.prefix.startsWith(subspace) &&
entry.prefix.endsWith(StringRef(reinterpret_cast<uint8_t*>(&bigEndianId), 8)) &&
entry.prefix.size() == subspaceLength + 8);
TenantMapEntry decodedEntry = decodeTenantEntry(encodeTenantEntry(entry));
ASSERT(decodedEntry.id = entry.id && decodedEntry.prefix == entry.prefix);
}
return Void();
}

5
fdbclient/TestKnobCollection.h
View File

@ -18,6 +18,9 @@
* limitations under the License.
*/
#ifndef FDBCLIENT_TESTKNOBCOLLECTION_H
#define FDBCLIENT_TESTKNOBCOLLECTION_H
#pragma once
#include "fdbclient/IKnobCollection.h"
@ -67,3 +70,5 @@ public:
bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) override;
bool isAtomic(std::string const& knobName) const override;
};
#endif // FDBCLIENT_TESTKNOBCOLLECTION_H

6
fdbrpc/AsyncFileCached.actor.h
View File

@ -84,6 +84,9 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
#else
page->data = pageSize == 4096 ? FastAllocator<4096>::allocate() : aligned_alloc(4096, pageSize);
#endif
if (page->data == nullptr) {
platform::outOfMemory();
}
if (RANDOM == cacheEvictionType) {
page->index = pages.size();
pages.push_back(page);
@ -396,6 +399,9 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
#else
data = pageCache->pageSize == 4096 ? FastAllocator<4096>::allocate() : aligned_alloc(4096, pageCache->pageSize);
#endif
if (data == nullptr) {
platform::outOfMemory();
}
}
Future<Void> write(void const* data, int length, int offset) {

105
fdbserver/CMakeLists.txt
View File

@ -1,6 +1,6 @@
set(FDBSERVER_SRCS
ApplyMetadataMutation.h
ApplyMetadataMutation.cpp
ApplyMetadataMutation.h
BackupInterface.h
BackupProgress.actor.cpp
BackupProgress.actor.h
@ -8,10 +8,11 @@ set(FDBSERVER_SRCS
BlobManager.actor.cpp
BlobManagerInterface.h
BlobWorker.actor.cpp
ClusterController.actor.h
ClusterController.actor.cpp
ClusterRecovery.actor.h
ClusterController.actor.h
ClusterRecovery.actor.cpp
ClusterRecovery.actor.h
CommitProxyServer.actor.cpp
ConfigBroadcaster.actor.cpp
ConfigBroadcaster.h
ConfigDatabaseUnitTests.actor.cpp
@ -34,11 +35,11 @@ set(FDBSERVER_SRCS
DDTeamCollection.actor.cpp
DDTeamCollection.h
DiskQueue.actor.cpp
EncryptKeyProxyInterface.h
EncryptKeyProxy.actor.cpp
fdbserver.actor.cpp
EncryptKeyProxyInterface.h
FDBExecHelper.actor.cpp
FDBExecHelper.actor.h
fdbserver.actor.cpp
GrvProxyServer.actor.cpp
IConfigConsumer.cpp
IConfigConsumer.h
@ -50,10 +51,8 @@ set(FDBSERVER_SRCS
KeyValueStoreMemory.actor.cpp
KeyValueStoreRocksDB.actor.cpp
KeyValueStoreSQLite.actor.cpp
ServerCheckpoint.actor.cpp
ServerCheckpoint.actor.h
RocksDBCheckpointUtils.actor.cpp
RocksDBCheckpointUtils.actor.h
KnobProtectiveGroups.cpp
KnobProtectiveGroups.h
Knobs.h
LatencyBandConfig.cpp
LatencyBandConfig.h
@ -71,14 +70,13 @@ set(FDBSERVER_SRCS
LogSystemDiskQueueAdapter.h
LogSystemPeekCursor.actor.cpp
MasterInterface.h
masterserver.actor.cpp
MetricLogger.actor.cpp
MetricLogger.actor.h
CommitProxyServer.actor.cpp
masterserver.actor.cpp
MutationTracking.h
MutationTracking.cpp
MoveKeys.actor.h
MoveKeys.actor.cpp
MoveKeys.actor.h
MutationTracking.cpp
MutationTracking.h
networktest.actor.cpp
NetworkTest.h
OldTLogServer_4_6.actor.cpp
@ -94,35 +92,39 @@ set(FDBSERVER_SRCS
QuietDatabase.actor.cpp
QuietDatabase.h
RadixTree.h
Ratekeeper.h
Ratekeeper.actor.cpp
Ratekeeper.h
RatekeeperInterface.h
RecoveryState.h
RestoreCommon.actor.h
RestoreCommon.actor.cpp
RestoreUtil.h
RestoreUtil.actor.cpp
RestoreRoleCommon.actor.h
RestoreRoleCommon.actor.cpp
RestoreController.actor.h
RestoreController.actor.cpp
RestoreApplier.actor.h
RestoreApplier.actor.cpp
RestoreLoader.actor.h
RestoreLoader.actor.cpp
RestoreWorker.actor.h
RestoreWorker.actor.cpp
RestoreWorkerInterface.actor.cpp
RestoreWorkerInterface.actor.h
Resolver.actor.cpp
ResolverInterface.h
RoleLineage.actor.h
RestoreApplier.actor.cpp
RestoreApplier.actor.h
RestoreCommon.actor.cpp
RestoreCommon.actor.h
RestoreController.actor.cpp
RestoreController.actor.h
RestoreLoader.actor.cpp
RestoreLoader.actor.h
RestoreRoleCommon.actor.cpp
RestoreRoleCommon.actor.h
RestoreUtil.actor.cpp
RestoreUtil.h
RestoreWorker.actor.cpp
RestoreWorker.actor.h
RestoreWorkerInterface.actor.cpp
RestoreWorkerInterface.actor.h
RocksDBCheckpointUtils.actor.cpp
RocksDBCheckpointUtils.actor.h
RoleLineage.actor.cpp
RoleLineage.actor.h
ServerCheckpoint.actor.cpp
ServerCheckpoint.actor.h
ServerDBInfo.actor.h
ServerDBInfo.h
SigStack.cpp
SimEncryptVaultProxy.actor.h
SimEncryptVaultProxy.actor.cpp
SimEncryptVaultProxy.actor.h
SimpleConfigConsumer.actor.cpp
SimpleConfigConsumer.h
SimulatedCluster.actor.cpp
@ -139,18 +141,18 @@ set(FDBSERVER_SRCS
TagPartitionedLogSystem.actor.h
TagThrottler.actor.cpp
TagThrottler.h
template_fdb.h
TCInfo.actor.cpp
TCInfo.h
template_fdb.h
tester.actor.cpp
TesterInterface.actor.h
TLogInterface.h
TLogServer.actor.cpp
TSSMappingUtil.actor.h
TSSMappingUtil.actor.cpp
TSSMappingUtil.actor.h
VersionedBTree.actor.cpp
VFSAsync.h
VFSAsync.cpp
VFSAsync.h
WaitFailure.actor.cpp
WaitFailure.h
worker.actor.cpp
@ -168,37 +170,35 @@ set(FDBSERVER_SRCS
workloads/AtomicRestore.actor.cpp
workloads/AtomicSwitchover.actor.cpp
workloads/BackgroundSelectors.actor.cpp
workloads/BackupCorrectness.actor.cpp
workloads/BackupAndParallelRestoreCorrectness.actor.cpp
workloads/ClogSingleConnection.actor.cpp
workloads/ConfigIncrement.actor.cpp
workloads/BackupCorrectness.actor.cpp
workloads/BackupToBlob.actor.cpp
workloads/BackupToDBAbort.actor.cpp
workloads/BackupToDBCorrectness.actor.cpp
workloads/BackupToDBUpgrade.actor.cpp
workloads/BlobStoreWorkload.h
workloads/BlobGranuleVerifier.actor.cpp
workloads/BlobStoreWorkload.h
workloads/BulkLoad.actor.cpp
workloads/BulkSetup.actor.h
workloads/Cache.actor.cpp
workloads/ChangeConfig.actor.cpp
workloads/ChangeFeeds.actor.cpp
workloads/ClearSingleRange.actor.cpp
workloads/ClientTransactionProfileCorrectness.actor.cpp
workloads/TriggerRecovery.actor.cpp
workloads/SuspendProcesses.actor.cpp
workloads/ClogSingleConnection.actor.cpp
workloads/CommitBugCheck.actor.cpp
workloads/ConfigIncrement.actor.cpp
workloads/ConfigureDatabase.actor.cpp
workloads/ConflictRange.actor.cpp
workloads/ConsistencyCheck.actor.cpp
workloads/CpuProfiler.actor.cpp
workloads/Cycle.actor.cpp
workloads/ChangeFeeds.actor.cpp
workloads/DataDistributionMetrics.actor.cpp
workloads/DataLossRecovery.actor.cpp
workloads/PhysicalShardMove.actor.cpp
workloads/DDBalance.actor.cpp
workloads/DDMetrics.actor.cpp
workloads/DDMetricsExclude.actor.cpp
workloads/DifferentClustersSameRV.actor.cpp
workloads/DiskDurability.actor.cpp
workloads/DiskDurabilityTest.actor.cpp
workloads/DiskFailureInjection.actor.cpp
@ -209,18 +209,19 @@ set(FDBSERVER_SRCS
workloads/FileSystem.actor.cpp
workloads/Fuzz.cpp
workloads/FuzzApiCorrectness.actor.cpp
workloads/GetMappedRange.actor.cpp
workloads/GetRangeStream.actor.cpp
workloads/HealthMetricsApi.actor.cpp
workloads/HighContentionPrefixAllocatorWorkload.actor.cpp
workloads/IncrementalBackup.actor.cpp
workloads/Increment.actor.cpp
workloads/IncrementalBackup.actor.cpp
workloads/IndexScan.actor.cpp
workloads/Inventory.actor.cpp
workloads/KVStoreTest.actor.cpp
workloads/KillRegion.actor.cpp
workloads/KVStoreTest.actor.cpp
workloads/LocalRatekeeper.actor.cpp
workloads/LockDatabase.actor.cpp
workloads/LockDatabaseFrequently.actor.cpp
workloads/LocalRatekeeper.actor.cpp
workloads/LogMetrics.actor.cpp
workloads/LowLatency.actor.cpp
workloads/MachineAttrition.actor.cpp
@ -231,9 +232,9 @@ set(FDBSERVER_SRCS
workloads/MetricLogging.actor.cpp
workloads/MiniCycle.actor.cpp
workloads/MutationLogReaderCorrectness.actor.cpp
workloads/GetMappedRange.actor.cpp
workloads/ParallelRestore.actor.cpp
workloads/Performance.actor.cpp
workloads/PhysicalShardMove.actor.cpp
workloads/Ping.actor.cpp
workloads/PopulateTPCC.actor.cpp
workloads/ProtocolVersion.actor.cpp
@ -262,25 +263,27 @@ set(FDBSERVER_SRCS
workloads/SlowTaskWorkload.actor.cpp
workloads/SnapTest.actor.cpp
workloads/SpecialKeySpaceCorrectness.actor.cpp
workloads/StreamingRangeRead.actor.cpp
workloads/StatusWorkload.actor.cpp
workloads/Storefront.actor.cpp
workloads/StreamingRangeRead.actor.cpp
workloads/StreamingRead.actor.cpp
workloads/SubmitBackup.actor.cpp
workloads/SuspendProcesses.actor.cpp
workloads/TagThrottleApi.actor.cpp
workloads/TargetedKill.actor.cpp
workloads/TaskBucketCorrectness.actor.cpp
workloads/TenantManagement.actor.cpp
workloads/ThreadSafety.actor.cpp
workloads/Throttling.actor.cpp
workloads/Throughput.actor.cpp
workloads/TimeKeeperCorrectness.actor.cpp
workloads/TPCC.actor.cpp
workloads/TPCCWorkload.h
workloads/DifferentClustersSameRV.actor.cpp
workloads/TriggerRecovery.actor.cpp
workloads/UDPWorkload.actor.cpp
workloads/UnitPerf.actor.cpp
workloads/UnitTests.actor.cpp
workloads/Unreadable.actor.cpp
workloads/UDPWorkload.actor.cpp
workloads/VersionStamp.actor.cpp
workloads/WatchAndWait.actor.cpp
workloads/Watches.actor.cpp

6
fdbserver/Coordination.actor.cpp
View File

@ -18,6 +18,8 @@
* limitations under the License.
*/
#include <cstdint>
#include "fdbclient/ConfigTransactionInterface.h"
#include "fdbserver/CoordinationInterface.h"
#include "fdbserver/ConfigNode.h"
@ -31,9 +33,9 @@
#include "flow/UnitTest.h"
#include "flow/IndexedSet.h"
#include "fdbclient/MonitorLeader.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/network.h"
#include <cstdint>
#include "flow/actorcompiler.h" // This must be the last #include.
// This module implements coordinationServer() and the interfaces in CoordinationInterface.h

3
fdbserver/DiskQueue.actor.cpp
View File

@ -24,9 +24,10 @@
#include "fdbrpc/simulator.h"
#include "flow/crc32c.h"
#include "flow/genericactors.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/xxhash.h"
#include "flow/actorcompiler.h" // This must be the last #include.
typedef bool (*compare_pages)(void*, void*);
typedef int64_t loc_t;

20
fdbserver/FDBExecHelper.actor.cpp
View File

@ -1,3 +1,23 @@
/*
* FDBExecHelper.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#if !defined(_WIN32) && !defined(__APPLE__) && !defined(__INTEL_COMPILER)
#define BOOST_SYSTEM_NO_LIB
#define BOOST_DATE_TIME_NO_LIB

23
fdbserver/FDBExecHelper.actor.h
View File

@ -1,3 +1,23 @@
/*
* FDBExecHelper.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_EXEC_HELPER_ACTOR_G_H)
#define FDBSERVER_EXEC_HELPER_ACTOR_G_H
@ -10,9 +30,10 @@
#include <map>
#include "flow/Arena.h"
#include "flow/flow.h"
#include "flow/actorcompiler.h"
#include "fdbclient/FDBTypes.h"
#include "flow/actorcompiler.h" // This must be the last #include.
// execute/snapshot command takes two arguments: <param1> <param2>
// param1 - represents the command type/name
// param2 - takes a binary path followed by a set of arguments in the following

73
fdbserver/KnobProtectiveGroups.cpp Normal file
View File

@ -0,0 +1,73 @@
/*
* KnobProtectiveGroups.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbserver/KnobProtectiveGroups.h"
#include <array>
#include "fdbclient/Knobs.h"
#include "fdbclient/ServerKnobCollection.h"
#include "fdbserver/Knobs.h"
void KnobKeyValuePairs::set(const std::string& name, const ParsedKnobValue value) {
ASSERT(knobs.count(name) == 0);
knobs[name] = value;
}
const KnobKeyValuePairs::container_t& KnobKeyValuePairs::getKnobs() const {
return knobs;
}
KnobProtectiveGroup::KnobProtectiveGroup(const KnobKeyValuePairs& overriddenKnobKeyValuePairs_)
: overriddenKnobs(overriddenKnobKeyValuePairs_) {
snapshotOriginalKnobs();
assignKnobs(overriddenKnobs);
}
KnobProtectiveGroup::~KnobProtectiveGroup() {
assignKnobs(originalKnobs);
}
void KnobProtectiveGroup::snapshotOriginalKnobs() {
for (const auto& [name, _] : overriddenKnobs.getKnobs()) {
ParsedKnobValue value = CLIENT_KNOBS->getKnob(name);
if (std::get_if<NoKnobFound>(&value)) {
value = SERVER_KNOBS->getKnob(name);
}
if (std::get_if<NoKnobFound>(&value)) {
ASSERT(false);
}
originalKnobs.set(name, value);
TraceEvent(SevInfo, "SnapshotKnobValue")
.detail("KnobName", name)
.detail("KnobValue", KnobValueRef::create(value).toString());
}
}
void KnobProtectiveGroup::assignKnobs(const KnobKeyValuePairs& overrideKnobs) {
auto& mutableServerKnobs = dynamic_cast<ServerKnobCollection&>(IKnobCollection::getMutableGlobalKnobCollection());
for (const auto& [name, value] : overrideKnobs.getKnobs()) {
Standalone<KnobValueRef> valueRef = KnobValueRef::create(value);
ASSERT(mutableServerKnobs.trySetKnob(name, valueRef));
TraceEvent(SevInfo, "AssignKnobValue").detail("KnobName", name).detail("KnobValue", valueRef.toString());
}
}

61
fdbserver/KnobProtectiveGroups.h Normal file
View File

@ -0,0 +1,61 @@
/*
* KnobProtectiveGroups.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBSERVER_KNOBPROTECTIVEGROUPS_H
#define FDBSERVER_KNOBPROTECTIVEGROUPS_H
#include <array>
#include <unordered_map>
#include "fdbclient/IKnobCollection.h"
// A list of knob key value pairs
class KnobKeyValuePairs {
public:
using container_t = std::unordered_map<std::string, ParsedKnobValue>;
private:
// Here the knob value is directly stored, unlike KnobValue, for simplicity
container_t knobs;
public:
// Sets a value for a given knob
void set(const std::string& name, const ParsedKnobValue value);
// Gets a list of knobs for given type
const container_t& getKnobs() const;
};
// For knobs, temporarily change the values, the original values will be recovered
class KnobProtectiveGroup {
KnobKeyValuePairs overriddenKnobs;
KnobKeyValuePairs originalKnobs;
// Snapshots the current knob values base on those knob keys in overriddenKnobs
void snapshotOriginalKnobs();
void assignKnobs(const KnobKeyValuePairs& overrideKnobs);
public:
KnobProtectiveGroup(const KnobKeyValuePairs& overridenKnobs_);
~KnobProtectiveGroup();
};
#endif // FDBSERVER_KNOBPROTECTIVEGROUPS_H

5
fdbserver/Knobs.h
View File

@ -18,8 +18,13 @@
* limitations under the License.
*/
#ifndef FDBSERVER_KNOBS_H
#define FDBSERVER_KNOBS_H
#pragma once
#include "fdbclient/IKnobCollection.h"
#define SERVER_KNOBS (&IKnobCollection::getGlobalKnobCollection().getServerKnobs())
#endif // FDBSERVER_KNOBS_H

3
fdbserver/RestoreApplier.actor.cpp
View File

@ -33,9 +33,10 @@
#include "fdbserver/RestoreRoleCommon.actor.h"
#include "fdbserver/RestoreApplier.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/network.h"
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMutationsRequest req,
Reference<RestoreApplierData> self);
ACTOR static Future<Void> handleApplyToDBRequest(RestoreVersionBatchRequest req,

1
fdbserver/RocksDBCheckpointUtils.actor.cpp
View File

@ -140,6 +140,7 @@ ACTOR Future<Void> fetchCheckpointFile(Database cx,
state StorageServerInterface ssi;
loop {
try {
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
Optional<Value> ss = wait(tr.get(serverListKeyFor(ssID)));
if (!ss.present()) {
throw checkpoint_not_found();

54
fdbserver/SimulatedCluster.actor.cpp
View File

@ -300,6 +300,9 @@ public:
stderrSeverity, machineCount, processesPerMachine, coordinators;
Optional<std::string> config;
bool allowDefaultTenant = true;
bool allowDisablingTenants = true;
ConfigDBType getConfigDBType() const { return configDBType; }
bool tomlKeyPresent(const toml::value& data, std::string key) {
@ -350,7 +353,9 @@ public:
.add("processesPerMachine", &processesPerMachine)
.add("coordinators", &coordinators)
.add("configDB", &configDBType)
.add("extraMachineCountDC", &extraMachineCountDC);
.add("extraMachineCountDC", &extraMachineCountDC)
.add("allowDefaultTenant", &allowDefaultTenant)
.add("allowDisablingTenants", &allowDisablingTenants);
try {
auto file = toml::parse(testFile);
if (file.contains("configuration") && toml::find(file, "configuration").is_table()) {
@ -1786,12 +1791,16 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
Standalone<StringRef>* pStartingConfiguration,
std::string whitelistBinPaths,
TestConfig testConfig,
ProtocolVersion protocolVersion) {
ProtocolVersion protocolVersion,
TenantMode tenantMode) {
// SOMEDAY: this does not test multi-interface configurations
SimulationConfig simconfig(testConfig);
if (testConfig.logAntiQuorum != -1) {
simconfig.db.tLogWriteAntiQuorum = testConfig.logAntiQuorum;
}
simconfig.db.tenantMode = tenantMode;
StatusObject startingConfigJSON = simconfig.db.toJSON(true);
std::string startingConfigString = "new";
if (testConfig.configureLocked) {
@ -1884,7 +1893,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
ASSERT(g_simulator.storagePolicy && g_simulator.tLogPolicy);
ASSERT(!g_simulator.hasSatelliteReplication || g_simulator.satelliteTLogPolicy);
TraceEvent("SimulatorConfig").detail("ConfigString", StringRef(startingConfigString));
TraceEvent("SimulatorConfig").setMaxFieldLength(10000).detail("ConfigString", StringRef(startingConfigString));
const int dataCenters = simconfig.datacenters;
const int machineCount = simconfig.machine_count;
@ -2271,11 +2280,18 @@ ACTOR void setupAndRun(std::string dataFolder,
g_simulator.hasDiffProtocolProcess = testConfig.startIncompatibleProcess;
g_simulator.setDiffProtocol = false;
state bool allowDefaultTenant = testConfig.allowDefaultTenant;
state bool allowDisablingTenants = testConfig.allowDisablingTenants;
// The RocksDB storage engine does not support the restarting tests because you cannot consistently get a clean
// snapshot of the storage engine without a snapshotting file system.
// https://github.com/apple/foundationdb/issues/5155
if (std::string_view(testFile).find("restarting") != std::string_view::npos) {
testConfig.storageEngineExcludeTypes.push_back(4);
// Disable the default tenant in restarting tests for now
// TODO: persist the chosen default tenant in the restartInfo.ini file for the second test
allowDefaultTenant = false;
}
// TODO: Currently backup and restore related simulation tests are failing when run with rocksDB storage engine
@ -2285,6 +2301,13 @@ ACTOR void setupAndRun(std::string dataFolder,
testConfig.storageEngineExcludeTypes.push_back(4);
}
// Disable the default tenant in backup and DR tests for now. This is because backup does not currently duplicate
// the tenant map and related state.
// TODO: reenable when backup/DR supports tenants.
if (std::string_view(testFile).find("Backup") != std::string_view::npos || testConfig.extraDB != 0) {
allowDefaultTenant = false;
}
// The RocksDB engine is not always built with the rest of fdbserver. Don't try to use it if it is not included
// in the build.
if (!rocksDBEnabled) {
@ -2319,6 +2342,23 @@ ACTOR void setupAndRun(std::string dataFolder,
FlowTransport::createInstance(true, 1, WLTOKEN_RESERVED_COUNT);
TEST(true); // Simulation start
state Optional<TenantName> defaultTenant;
state TenantMode tenantMode = TenantMode::DISABLED;
if (allowDefaultTenant && deterministicRandom()->random01() < 0.5) {
defaultTenant = "SimulatedDefaultTenant"_sr;
if (deterministicRandom()->random01() < 0.9) {
tenantMode = TenantMode::REQUIRED;
} else {
tenantMode = TenantMode::OPTIONAL_TENANT;
}
} else if (!allowDisablingTenants || deterministicRandom()->random01() < 0.5) {
tenantMode = TenantMode::OPTIONAL_TENANT;
}
TraceEvent("SimulatedClusterTenantMode")
.detail("UsingTenant", defaultTenant)
.detail("TenantRequired", tenantMode.toString());
try {
// systemActors.push_back( startSystemMonitor(dataFolder) );
if (rebooting) {
@ -2344,7 +2384,8 @@ ACTOR void setupAndRun(std::string dataFolder,
&startingConfiguration,
whitelistBinPaths,
testConfig,
protocolVersion);
protocolVersion,
tenantMode);
wait(delay(1.0)); // FIXME: WHY!!! //wait for machines to boot
}
std::string clusterFileDir = joinPath(dataFolder, deterministicRandom()->randomUniqueID().toString());
@ -2355,7 +2396,10 @@ ACTOR void setupAndRun(std::string dataFolder,
TEST_ON_TESTERS,
testerCount,
testFile,
startingConfiguration),
startingConfiguration,
LocalityData(),
UnitTestParameters(),
defaultTenant),
isBuggifyEnabled(BuggifyType::General) ? 36000.0 : 5400.0));
} catch (Error& e) {
TraceEvent(SevError, "SetupAndRunError").error(e);

5
fdbserver/TesterInterface.actor.h
View File

@ -66,6 +66,7 @@ struct WorkloadRequest {
double databasePingDelay;
int64_t sharedRandomNumber;
bool useDatabase;
Optional<TenantNameRef> defaultTenant;
// The vector of option lists are to construct compound workloads. If there
// is only one workload to be run...pass just one list of options!
@ -96,6 +97,7 @@ struct WorkloadRequest {
clientId,
clientCount,
reply,
defaultTenant,
arena);
}
};
@ -127,7 +129,8 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
std::string fileName = std::string(),
StringRef startingConfiguration = StringRef(),
LocalityData locality = LocalityData(),
UnitTestParameters testOptions = UnitTestParameters());
UnitTestParameters testOptions = UnitTestParameters(),
Optional<TenantName> defaultTenant = Optional<TenantName>());
#include "flow/unactorcompiler.h"
#endif

2
fdbserver/VersionedBTree.actor.cpp
View File

@ -45,9 +45,9 @@
#include "fdbserver/IKeyValueStore.h"
#include "fdbserver/DeltaTree.h"
#include <string.h>
#include "flow/actorcompiler.h"
#include <cinttypes>
#include <boost/intrusive/list.hpp>
#include "flow/actorcompiler.h" // must be last include
#define REDWOOD_DEBUG 0

1
fdbserver/fdbserver.actor.cpp
View File

@ -2085,6 +2085,7 @@ int main(int argc, char* argv[]) {
opts.localities));
g_network->run();
} else if (role == ServerRole::Test) {
TraceEvent("NonSimulationTest").detail("TestFile", opts.testFile);
setupRunLoopProfiler();
auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
f = stopAfter(runTests(

3
fdbserver/networktest.actor.cpp
View File

@ -21,11 +21,12 @@
#include "contrib/fmt-8.1.1/include/fmt/format.h"
#include "fdbserver/NetworkTest.h"
#include "flow/Knobs.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/ActorCollection.h"
#include "flow/UnitTest.h"
#include <inttypes.h>
#include "flow/actorcompiler.h" // This must be the last #include.
constexpr int WLTOKEN_NETWORKTEST = WLTOKEN_FIRST_AVAILABLE;
struct LatencyStats {

12
fdbserver/storageserver.actor.cpp
View File

@ -2469,7 +2469,17 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
else
readBegin = range.begin;
vCurrent = view.lower_bound(readBegin);
if (vCurrent) {
// We can get first greater or equal from the result of lastLessOrEqual
if (vCurrent.key() != readBegin) {
++vCurrent;
}
} else {
// There's nothing less than or equal to readBegin in view, so
// begin() is the first thing greater than readBegin, or end().
// Either way that's the correct result for lower_bound.
vCurrent = view.begin();
}
while (limit > 0 && *pLimitBytes > 0 && readBegin < range.end) {
ASSERT(!vCurrent || vCurrent.key() >= readBegin);

71
fdbserver/tester.actor.cpp
View File

@ -31,6 +31,7 @@
#include "fdbclient/ClusterInterface.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbserver/KnobProtectiveGroups.h"
#include "fdbserver/TesterInterface.actor.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
@ -642,6 +643,7 @@ ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
if (work.useDatabase) {
cx = Database::createDatabase(ccr, -1, IsInternal::True, locality);
cx->defaultTenant = work.defaultTenant.castTo<TenantName>();
wait(delay(1.0));
}
@ -779,7 +781,10 @@ void throwIfError(const std::vector<Future<ErrorOr<T>>>& futures, std::string er
}
}
ACTOR Future<DistributedTestResults> runWorkload(Database cx, std::vector<TesterInterface> testers, TestSpec spec) {
ACTOR Future<DistributedTestResults> runWorkload(Database cx,
std::vector<TesterInterface> testers,
TestSpec spec,
Optional<TenantName> defaultTenant) {
TraceEvent("TestRunning")
.detail("WorkloadTitle", spec.title)
.detail("TesterCount", testers.size())
@ -803,6 +808,7 @@ ACTOR Future<DistributedTestResults> runWorkload(Database cx, std::vector<Tester
req.clientId = i;
req.clientCount = testers.size();
req.sharedRandomNumber = sharedRandom;
req.defaultTenant = defaultTenant.castTo<TenantNameRef>();
workRequests.push_back(testers[i].recruitments.getReply(req));
}
@ -894,7 +900,7 @@ ACTOR Future<Void> changeConfiguration(Database cx, std::vector<TesterInterface>
options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("configMode"), configMode));
spec.options.push_back_deep(spec.options.arena(), options);
DistributedTestResults testResults = wait(runWorkload(cx, testers, spec));
DistributedTestResults testResults = wait(runWorkload(cx, testers, spec, Optional<TenantName>()));
return Void();
}
@ -949,7 +955,7 @@ ACTOR Future<Void> checkConsistency(Database cx,
state double start = now();
state bool lastRun = false;
loop {
DistributedTestResults testResults = wait(runWorkload(cx, testers, spec));
DistributedTestResults testResults = wait(runWorkload(cx, testers, spec, Optional<TenantName>()));
if (testResults.ok() || lastRun) {
if (g_network->isSimulated()) {
g_simulator.connectionFailuresDisableDuration = connectionFailures;
@ -969,11 +975,12 @@ ACTOR Future<Void> checkConsistency(Database cx,
ACTOR Future<bool> runTest(Database cx,
std::vector<TesterInterface> testers,
TestSpec spec,
Reference<AsyncVar<ServerDBInfo>> dbInfo) {
Reference<AsyncVar<ServerDBInfo>> dbInfo,
Optional<TenantName> defaultTenant) {
state DistributedTestResults testResults;
try {
Future<DistributedTestResults> fTestResults = runWorkload(cx, testers, spec);
Future<DistributedTestResults> fTestResults = runWorkload(cx, testers, spec, defaultTenant);
if (spec.timeout > 0) {
fTestResults = timeoutError(fTestResults, spec.timeout);
}
@ -1312,7 +1319,7 @@ std::vector<TestSpec> readTOMLTests_(std::string fileName) {
// First handle all test-level settings
for (const auto& [k, v] : test.as_table()) {
if (k == "workload") {
if (k == "workload" || k == "knobs") {
continue;
}
if (testSpecTestKeys.find(k) != testSpecTestKeys.end()) {
@ -1338,6 +1345,29 @@ std::vector<TestSpec> readTOMLTests_(std::string fileName) {
spec.options.push_back_deep(spec.options.arena(), workloadOptions);
}
// And then copy the knob attributes to spec.overrideKnobs
try {
const toml::array& overrideKnobs = toml::find(test, "knobs").as_array();
for (const toml::value& knob : overrideKnobs) {
for (const auto& [key, value_] : knob.as_table()) {
const std::string& value = toml_to_string(value_);
ParsedKnobValue parsedValue = CLIENT_KNOBS->parseKnobValue(key, value);
if (std::get_if<NoKnobFound>(&parsedValue)) {
parsedValue = SERVER_KNOBS->parseKnobValue(key, value);
}
if (std::get_if<NoKnobFound>(&parsedValue)) {
TraceEvent(SevError, "TestSpecUnrecognizedKnob")
.detail("KnobName", key)
.detail("OverrideValue", value);
continue;
}
spec.overrideKnobs.set(key, parsedValue);
}
}
} catch (const std::out_of_range&) {
// no knob overridden
}
result.push_back(spec);
}
@ -1418,7 +1448,8 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
std::vector<TesterInterface> testers,
std::vector<TestSpec> tests,
StringRef startingConfiguration,
LocalityData locality) {
LocalityData locality,
Optional<TenantName> defaultTenant) {
state Database cx;
state Reference<AsyncVar<ServerDBInfo>> dbInfo(new AsyncVar<ServerDBInfo>);
state Future<Void> ccMonitor = monitorServerDBInfo(cc, LocalityData(), dbInfo); // FIXME: locality
@ -1466,6 +1497,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
if (useDB) {
cx = openDBOnServer(dbInfo);
cx->defaultTenant = defaultTenant;
}
state Future<Void> disabler = disableConnectionFailuresAfter(FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, "Tester");
@ -1493,6 +1525,11 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
}
}
if (useDB && defaultTenant.present()) {
TraceEvent("CreatingDefaultTenant").detail("Tenant", defaultTenant.get());
wait(ManagementAPI::createTenant(cx.getReference(), defaultTenant.get()));
}
if (useDB && waitForQuiescenceBegin) {
TraceEvent("TesterStartingPreTestChecks")
.detail("DatabasePingDelay", databasePingDelay)
@ -1516,9 +1553,12 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
TraceEvent("TestsExpectedToPass").detail("Count", tests.size());
state int idx = 0;
state std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup;
for (; idx < tests.size(); idx++) {
printf("Run test:%s start\n", tests[idx].title.toString().c_str());
wait(success(runTest(cx, testers, tests[idx], dbInfo)));
knobProtectiveGroup = std::make_unique<KnobProtectiveGroup>(tests[idx].overrideKnobs);
wait(success(runTest(cx, testers, tests[idx], dbInfo, defaultTenant)));
knobProtectiveGroup.reset(nullptr);
printf("Run test:%s Done.\n", tests[idx].title.toString().c_str());
// do we handle a failure here?
}
@ -1568,7 +1608,8 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
test_location_t at,
int minTestersExpected,
StringRef startingConfiguration,
LocalityData locality) {
LocalityData locality,
Optional<TenantName> defaultTenant) {
state int flags = (at == TEST_ON_SERVERS ? 0 : GetWorkersRequest::TESTER_CLASS_ONLY) |
GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
state Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
@ -1599,7 +1640,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
for (int i = 0; i < workers.size(); i++)
ts.push_back(workers[i].interf.testerInterface);
wait(runTests(cc, ci, ts, tests, startingConfiguration, locality));
wait(runTests(cc, ci, ts, tests, startingConfiguration, locality, defaultTenant));
return Void();
}
@ -1636,7 +1677,8 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
std::string fileName,
StringRef startingConfiguration,
LocalityData locality,
UnitTestParameters testOptions) {
UnitTestParameters testOptions,
Optional<TenantName> defaultTenant) {
state std::vector<TestSpec> testSpecs;
auto cc = makeReference<AsyncVar<Optional<ClusterControllerFullInterface>>>();
auto ci = makeReference<AsyncVar<Optional<ClusterInterface>>>();
@ -1718,10 +1760,11 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
actors.push_back(
reportErrors(monitorServerDBInfo(cc, LocalityData(), db), "MonitorServerDBInfo")); // FIXME: Locality
actors.push_back(reportErrors(testerServerCore(iTesters[0], connRecord, db, locality), "TesterServerCore"));
tests = runTests(cc, ci, iTesters, testSpecs, startingConfiguration, locality);
tests = runTests(cc, ci, iTesters, testSpecs, startingConfiguration, locality, defaultTenant);
} else {
tests = reportErrors(runTests(cc, ci, testSpecs, at, minTestersExpected, startingConfiguration, locality),
"RunTests");
tests = reportErrors(
runTests(cc, ci, testSpecs, at, minTestersExpected, startingConfiguration, locality, defaultTenant),
"RunTests");
}
choose {

3
fdbserver/workloads/ChangeFeeds.actor.cpp
View File

@ -28,11 +28,12 @@
#include "flow/Arena.h"
#include "flow/IRandom.h"
#include "flow/Trace.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/serialize.h"
#include <cstring>
#include <limits>
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR Future<std::pair<Standalone<VectorRef<KeyValueRef>>, Version>> readDatabase(Database cx) {
state Transaction tr(cx);
loop {

11
fdbserver/workloads/ConflictRange.actor.cpp
View File

@ -93,6 +93,10 @@ struct ConflictRangeWorkload : TestWorkload {
wait(timeKeeperSetDisable(cx));
}
// Set one key after the end of the tested range. If this key is included in the result, then
// we may have drifted into the system key-space and cannot evaluate the result.
state Key sentinelKey = StringRef(format("%010d", self->maxKeySpace));
loop {
randomSets = !randomSets;
@ -127,6 +131,8 @@ struct ConflictRangeWorkload : TestWorkload {
}
}
tr0.set(sentinelKey, deterministicRandom()->randomUniqueID().toString());
wait(tr0.commit());
break;
} catch (Error& e) {
@ -177,7 +183,6 @@ struct ConflictRangeWorkload : TestWorkload {
if (self->testReadYourWrites) {
trRYOW.setVersion(readVersion);
trRYOW.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
} else
tr3.setVersion(readVersion);
@ -262,7 +267,7 @@ struct ConflictRangeWorkload : TestWorkload {
throw not_committed();
}
if (originalResults[originalResults.size() - 1].key >= LiteralStringRef("\xff")) {
if (originalResults[originalResults.size() - 1].key >= sentinelKey) {
// Results go into server keyspace, so if a key selector does not fully resolve offset, a
// change won't effect results
throw not_committed();
@ -316,7 +321,7 @@ struct ConflictRangeWorkload : TestWorkload {
allKeyEntries += printable(res[i].key) + " ";
}
TraceEvent("ConflictRangeDump").detail("Keys", allKeyEntries);
TraceEvent("ConflictRangeDump").setMaxFieldLength(10000).detail("Keys", allKeyEntries);
}
throw not_committed();
} else {

3
fdbserver/workloads/ConsistencyCheck.actor.cpp
View File

@ -36,9 +36,10 @@
#include "flow/DeterministicRandom.h"
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbclient/StorageServerInterface.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/network.h"
#include "flow/actorcompiler.h" // This must be the last #include.
//#define SevCCheckInfo SevVerbose
#define SevCCheckInfo SevInfo

6
fdbserver/workloads/Cycle.actor.cpp
View File

@ -18,6 +18,8 @@
* limitations under the License.
*/
#include <cstring>
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbserver/TesterInterface.actor.h"
@ -26,9 +28,9 @@
#include "flow/Arena.h"
#include "flow/IRandom.h"
#include "flow/Trace.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/serialize.h"
#include <cstring>
#include "flow/actorcompiler.h" // This must be the last #include.
struct CycleWorkload : TestWorkload {
int actorCount, nodeCount;

5
fdbserver/workloads/DataLossRecovery.actor.cpp
View File

@ -20,6 +20,7 @@
#include <cstdint>
#include <limits>
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbserver/MoveKeys.actor.h"
@ -105,6 +106,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
state Optional<Value> res = wait(timeoutError(tr.get(key), 30.0));
const bool equal = !expectedValue.isError() && res == expectedValue.get();
if (!equal) {
@ -126,6 +128,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
state Transaction tr(cx);
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
if (value.present()) {
tr.set(key, value.get());
} else {
@ -193,6 +196,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
loop {
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
BinaryWriter wrMyOwner(Unversioned());
wrMyOwner << owner;
tr.set(moveKeysLockOwnerKey, wrMyOwner.toValue());
@ -228,6 +232,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
state Transaction validateTr(cx);
loop {
try {
validateTr.setOption(FDBTransactionOptions::RAW_ACCESS);
Standalone<VectorRef<const char*>> addresses = wait(validateTr.getAddressesForKey(keys.begin));
// The move function is not what we are testing here, crash the test if the move fails.
ASSERT(addresses.size() == 1);

424
fdbserver/workloads/FuzzApiCorrectness.actor.cpp
View File

@ -23,7 +23,9 @@
#include <functional>
#include <sstream>
#include "fdbclient/FDBOptions.g.h"
#include "fdbserver/TesterInterface.actor.h"
#include "fdbclient/GenericManagementAPI.actor.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "flow/ActorCollection.h"
#include "fdbserver/workloads/workloads.actor.h"
@ -53,7 +55,7 @@ struct ExceptionContract {
static occurance_t requiredIf(bool in) { return in ? Always : Never; }
static occurance_t possibleIf(bool in) { return in ? Possible : Never; }
void handleException(const Error& e) const {
void handleException(const Error& e, Reference<ITransaction> tr) const {
// We should always ignore these.
if (e.code() == error_code_used_during_commit || e.code() == error_code_transaction_too_old ||
e.code() == error_code_future_version || e.code() == error_code_transaction_cancelled ||
@ -70,6 +72,7 @@ struct ExceptionContract {
evt.error(e)
.detail("Thrown", true)
.detail("Expected", i->second == Possible ? "possible" : "always")
.detail("Tenant", tr->getTenant())
.backtrace();
if (augment)
augment(evt);
@ -77,20 +80,21 @@ struct ExceptionContract {
}
TraceEvent evt(SevError, func.c_str());
evt.error(e).detail("Thrown", true).detail("Expected", "never").backtrace();
evt.error(e).detail("Thrown", true).detail("Expected", "never").detail("Tenant", tr->getTenant()).backtrace();
if (augment)
augment(evt);
throw e;
}
// Return true if we should have thrown, but didn't.
void handleNotThrown() const {
void handleNotThrown(Reference<ITransaction> tr) const {
for (auto i : expected) {
if (i.second == Always) {
TraceEvent evt(SevError, func.c_str());
evt.error(Error::fromUnvalidatedCode(i.first))
.detail("Thrown", false)
.detail("Expected", "always")
.detail("Tenant", tr->getTenant())
.backtrace();
if (augment)
augment(evt);
@ -113,7 +117,6 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
int maxClearSize;
double initialKeyDensity;
bool useSystemKeys;
std::string keyPrefix;
KeyRange conflictRange;
unsigned int operationId;
int64_t maximumTotalData;
@ -122,6 +125,15 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
bool success;
Reference<IDatabase> db;
std::vector<Reference<ITenant>> tenants;
std::set<TenantName> createdTenants;
int numTenants;
// Map from tenant number to key prefix
std::map<int, std::string> keyPrefixes;
FuzzApiCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), operationId(0), success(true) {
std::call_once(onceFlag, [&]() { addTestCases(); });
@ -138,6 +150,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
// Only enable special keys writes when allowed to access system keys
specialKeysWritesEnabled = useSystemKeys && deterministicRandom()->coinflip();
int maxTenants = getOption(options, "numTenants"_sr, 4);
numTenants = deterministicRandom()->randomInt(0, maxTenants + 1);
// See https://github.com/apple/foundationdb/issues/2424
if (BUGGIFY) {
enableBuggify(true, BuggifyType::Client);
@ -150,18 +165,20 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
nodes = deterministicRandom()->randomInt(1, 4 << deterministicRandom()->randomInt(0, 20));
}
int newNodes = std::min<int>(nodes, maximumTotalData / (getKeyForIndex(nodes).size() + valueSizeRange.second));
int newNodes =
std::min<int>(nodes, maximumTotalData / (getKeyForIndex(-1, nodes).size() + valueSizeRange.second));
minNode = std::max(minNode, nodes - newNodes);
nodes = newNodes;
if (useSystemKeys && deterministicRandom()->coinflip()) {
keyPrefix = "\xff\x01";
keyPrefixes[-1] = "\xff\x01";
}
maxClearSize = 1 << deterministicRandom()->randomInt(0, 20);
conflictRange = KeyRangeRef(LiteralStringRef("\xfe"), LiteralStringRef("\xfe\x00"));
TraceEvent("FuzzApiCorrectnessConfiguration")
.detail("Nodes", nodes)
.detail("NumTenants", numTenants)
.detail("InitialKeyDensity", initialKeyDensity)
.detail("AdjacentKeys", adjacentKeys)
.detail("ValueSizeMin", valueSizeRange.first)
@ -187,23 +204,59 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
std::string description() const override { return "FuzzApiCorrectness"; }
static TenantName getTenant(int num) { return TenantNameRef(format("tenant_%d", num)); }
bool canUseTenant(Optional<TenantName> tenant) { return !tenant.present() || createdTenants.count(tenant.get()); }
Future<Void> setup(Database const& cx) override {
if (clientId == 0) {
return _setup(cx, this);
}
return Void();
}
ACTOR Future<Void> _setup(Database cx, FuzzApiCorrectnessWorkload* self) {
Reference<IDatabase> db = wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
self->db = db;
std::vector<Future<Void>> tenantFutures;
for (int i = 0; i < self->numTenants + 1; ++i) {
TenantName tenantName = getTenant(i);
self->tenants.push_back(self->db->openTenant(tenantName));
// The last tenant will not be created
if (i < self->numTenants) {
tenantFutures.push_back(ManagementAPI::createTenant(cx.getReference(), tenantName));
self->createdTenants.insert(tenantName);
}
}
wait(waitForAll(tenantFutures));
return Void();
}
Future<Void> start(Database const& cx) override {
if (clientId == 0) {
return loadAndRun(cx, this);
return loadAndRun(this);
}
return Void();
}
Future<bool> check(Database const& cx) override { return success; }
Key getRandomKey() const { return getKeyForIndex(deterministicRandom()->randomInt(0, nodes)); }
Key getKeyForIndex(int idx) const {
Key getKeyForIndex(int tenantNum, int idx) {
idx += minNode;
if (adjacentKeys) {
return Key(keyPrefix + std::string(idx, '\x00'));
return Key(keyPrefixes[tenantNum] + std::string(idx, '\x00'));
} else {
return Key(keyPrefix + format("%010d", idx));
return Key(keyPrefixes[tenantNum] + format("%010d", idx));
}
}
KeyRef getMaxKey(Reference<ITransaction> tr) const {
if (useSystemKeys && !tr->getTenant().present()) {
return systemKeys.end;
} else {
return normalKeys.end;
}
}
@ -217,65 +270,88 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
// m.push_back( retries.getMetric() );
}
ACTOR Future<Void> loadAndRun(Database db, FuzzApiCorrectnessWorkload* self) {
// Prevent a write only transaction whose commit was previously cancelled from being reordered after this
// transaction
ACTOR Future<Void> writeBarrier(Reference<IDatabase> db) {
state Reference<ITransaction> tr = db->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
// Write-only transactions have a self-conflict in the system keys
tr->addWriteConflictRange(allKeys);
tr->clear(normalKeys);
wait(unsafeThreadFutureToFuture(tr->commit()));
return Void();
} catch (Error& e) {
wait(unsafeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR Future<Void> loadAndRun(FuzzApiCorrectnessWorkload* self) {
state double startTime = now();
state Reference<IDatabase> cx =
wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(db)));
state int nodesPerTenant = std::max<int>(1, self->nodes / (self->numTenants + 1));
state int keysPerBatch =
std::min<int64_t>(1000,
1 + CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT / 2 /
(self->getKeyForIndex(-1, nodesPerTenant).size() + self->valueSizeRange.second));
try {
loop {
state int i = 0;
state int keysPerBatch =
std::min<int64_t>(1000,
1 + CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT / 2 /
(self->getKeyForIndex(self->nodes).size() + self->valueSizeRange.second));
for (; i < self->nodes; i += keysPerBatch) {
state Reference<ITransaction> tr = cx->createTransaction();
loop {
if (now() - startTime > self->testDuration)
return Void();
try {
if (i == 0) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->addWriteConflictRange(
allKeys); // To prevent a write only transaction whose commit was previously
// cancelled from being reordered after this transaction
tr->clear(normalKeys);
}
if (self->useSystemKeys)
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (self->specialKeysRelaxed)
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED);
if (self->specialKeysWritesEnabled)
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
state int tenantNum = -1;
for (; tenantNum < self->numTenants; ++tenantNum) {
state int i = 0;
wait(self->writeBarrier(self->db));
for (; i < nodesPerTenant; i += keysPerBatch) {
state Reference<ITransaction> tr = tenantNum < 0
? self->db->createTransaction()
: self->tenants[tenantNum]->createTransaction();
loop {
if (now() - startTime > self->testDuration)
return Void();
try {
if (self->useSystemKeys && tenantNum == -1) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
}
if (self->specialKeysRelaxed)
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED);
if (self->specialKeysWritesEnabled)
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
int end = std::min(self->nodes, i + keysPerBatch);
tr->clear(KeyRangeRef(self->getKeyForIndex(i), self->getKeyForIndex(end)));
if (i == 0) {
tr->clear(normalKeys);
}
for (int j = i; j < end; j++) {
if (deterministicRandom()->random01() < self->initialKeyDensity) {
Key key = self->getKeyForIndex(j);
if (key.size() <= (key.startsWith(systemKeys.begin)
? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
: CLIENT_KNOBS->KEY_SIZE_LIMIT)) {
Value value = self->getRandomValue();
value = value.substr(
0, std::min<int>(value.size(), CLIENT_KNOBS->VALUE_SIZE_LIMIT));
tr->set(key, value);
int end = std::min(nodesPerTenant, i + keysPerBatch);
tr->clear(KeyRangeRef(self->getKeyForIndex(tenantNum, i),
self->getKeyForIndex(tenantNum, end)));
for (int j = i; j < end; j++) {
if (deterministicRandom()->random01() < self->initialKeyDensity) {
Key key = self->getKeyForIndex(tenantNum, j);
if (key.size() <= (key.startsWith(systemKeys.begin)
? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
: CLIENT_KNOBS->KEY_SIZE_LIMIT)) {
Value value = self->getRandomValue();
value = value.substr(
0, std::min<int>(value.size(), CLIENT_KNOBS->VALUE_SIZE_LIMIT));
tr->set(key, value);
}
}
}
wait(unsafeThreadFutureToFuture(tr->commit()));
//TraceEvent("WDRInitBatch").detail("I", i).detail("CommittedVersion", tr->getCommittedVersion());
break;
} catch (Error& e) {
wait(unsafeThreadFutureToFuture(tr->onError(e)));
}
wait(unsafeThreadFutureToFuture(tr->commit()));
//TraceEvent("WDRInitBatch").detail("I", i).detail("CommittedVersion", tr->getCommittedVersion());
break;
} catch (Error& e) {
wait(unsafeThreadFutureToFuture(tr->onError(e)));
}
}
}
loop {
try {
wait(self->randomTransaction(cx, self) && delay(self->numOps * .001));
wait(self->randomTransaction(self) && delay(self->numOps * .001));
} catch (Error& e) {
if (e.code() != error_code_not_committed)
throw e;
@ -291,20 +367,29 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
}
}
ACTOR Future<Void> randomTransaction(Reference<IDatabase> cx, FuzzApiCorrectnessWorkload* self) {
state Reference<ITransaction> tr = cx->createTransaction();
ACTOR Future<Void> randomTransaction(FuzzApiCorrectnessWorkload* self) {
state Reference<ITransaction> tr;
state bool readYourWritesDisabled = deterministicRandom()->coinflip();
state bool readAheadDisabled = deterministicRandom()->coinflip();
state std::vector<Future<Void>> operations;
state int waitLocation = 0;
state int tenantNum = deterministicRandom()->randomInt(-1, self->tenants.size());
if (tenantNum == -1) {
tr = self->db->createTransaction();
} else {
tr = self->tenants[tenantNum]->createTransaction();
}
state bool rawAccess = tenantNum == -1 && deterministicRandom()->coinflip();
loop {
state bool cancelled = false;
if (readYourWritesDisabled)
tr->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
if (readAheadDisabled)
tr->setOption(FDBTransactionOptions::READ_AHEAD_DISABLE);
if (self->useSystemKeys) {
if (self->useSystemKeys && tenantNum == -1) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
}
if (self->specialKeysRelaxed) {
@ -313,6 +398,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
if (self->specialKeysWritesEnabled) {
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
}
if (rawAccess) {
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
}
tr->addWriteConflictRange(self->conflictRange);
try {
@ -350,7 +438,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
wait(timeoutError(unsafeThreadFutureToFuture(tr->commit()), 30));
} catch (Error& e) {
if (e.code() == error_code_client_invalid_operation ||
e.code() == error_code_transaction_too_large) {
e.code() == error_code_transaction_too_large || e.code() == error_code_unknown_tenant ||
e.code() == error_code_invalid_option) {
throw not_committed();
}
}
@ -380,14 +469,14 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef T value_type;
ACTOR static Future<Void> runTest(unsigned int id, FuzzApiCorrectnessWorkload* wl, Reference<ITransaction> tr) {
state Subclass self(id, wl);
state Subclass self(id, wl, tr);
try {
value_type result = wait(timeoutError(BaseTest::runTest2(tr, &self), 1000));
self.contract.handleNotThrown();
self.contract.handleNotThrown(tr);
return self.errorCheck(tr, result);
} catch (Error& e) {
self.contract.handleException(e);
self.contract.handleException(e, tr);
}
return Void();
}
@ -404,7 +493,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
value_type result = wait(future);
if (future.isError()) {
self->contract.handleException(future.getError());
self->contract.handleException(future.getError(), tr);
} else {
ASSERT(future.isValid());
}
@ -588,7 +677,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestSetVersion, Version> base_type;
Version v;
TestSetVersion(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestSetVersion(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTest(id, workload, "TestSetVersion") {
if (deterministicRandom()->coinflip())
v = deterministicRandom()->randomInt64(INT64_MIN, 0);
@ -620,13 +709,13 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestGet, Optional<Value>> base_type;
Key key;
TestGet(unsigned int id, FuzzApiCorrectnessWorkload* workload) : BaseTest(id, workload, "TestGet") {
TestGet(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTest(id, workload, "TestGet") {
key = makeKey();
contract = {
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
(key >= (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) &&
!specialKeys.contains(key))),
std::make_pair(
error_code_key_outside_legal_range,
ExceptionContract::requiredIf((key >= workload->getMaxKey(tr)) && !specialKeys.contains(key))),
std::make_pair(error_code_client_invalid_operation, ExceptionContract::Possible),
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible),
std::make_pair(
@ -641,7 +730,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
ExceptionContract::possibleIf(
key ==
LiteralStringRef("auto_coordinators")
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)))
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin))),
std::make_pair(error_code_tenant_not_found,
ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
std::make_pair(error_code_invalid_option,
ExceptionContract::possibleIf(tr->getTenant().present() && specialKeys.contains(key)))
};
}
@ -651,7 +744,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key));
e.detail("Key", key);
e.detail("Size", key.size());
}
};
@ -659,14 +753,15 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestGetKey, Key> base_type;
KeySelector keysel;
TestGetKey(unsigned int id, FuzzApiCorrectnessWorkload* workload) : BaseTest(id, workload, "TestGetKey") {
TestGetKey(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTest(id, workload, "TestGetKey") {
keysel = makeKeySel();
contract = { std::make_pair(
error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
(keysel.getKey() > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)))),
contract = { std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf((keysel.getKey() > workload->getMaxKey(tr)))),
std::make_pair(error_code_client_invalid_operation, ExceptionContract::Possible),
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible) };
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible),
std::make_pair(error_code_tenant_not_found,
ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))) };
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
@ -684,7 +779,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
KeySelector keysel1, keysel2;
int limit;
TestGetRange0(unsigned int id, FuzzApiCorrectnessWorkload* workload) : BaseTest(id, workload, "TestGetRange0") {
TestGetRange0(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTest(id, workload, "TestGetRange0") {
keysel1 = makeKeySel();
keysel2 = makeKeySel();
limit = 0;
@ -704,10 +800,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
std::make_pair(error_code_range_limits_invalid, ExceptionContract::possibleButRequiredIf(limit < 0)),
std::make_pair(error_code_client_invalid_operation, ExceptionContract::Possible),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((keysel1.getKey() > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) ||
(keysel2.getKey() > (workload->useSystemKeys ? systemKeys.end : normalKeys.end))) &&
!isSpecialKeyRange)),
ExceptionContract::requiredIf(((keysel1.getKey() > workload->getMaxKey(tr)) ||
(keysel2.getKey() > workload->getMaxKey(tr))) &&
!isSpecialKeyRange)),
std::make_pair(error_code_special_keys_cross_module_read,
ExceptionContract::possibleIf(isSpecialKeyRange && !workload->specialKeysRelaxed)),
std::make_pair(error_code_special_keys_no_module_found,
@ -715,7 +810,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
// Read some special keys, e.g. status/json, can throw timed_out
std::make_pair(error_code_timed_out, ExceptionContract::possibleIf(isSpecialKeyRange)),
std::make_pair(error_code_special_keys_api_failure, ExceptionContract::possibleIf(isSpecialKeyRange)),
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible)
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible),
std::make_pair(error_code_tenant_not_found,
ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
std::make_pair(error_code_invalid_option,
ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange))
};
}
@ -735,7 +834,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
KeySelector keysel1, keysel2;
GetRangeLimits limits;
TestGetRange1(unsigned int id, FuzzApiCorrectnessWorkload* workload) : BaseTest(id, workload, "TestGetRange1") {
TestGetRange1(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTest(id, workload, "TestGetRange1") {
keysel1 = makeKeySel();
keysel2 = makeKeySel();
limits = makeRangeLimits();
@ -748,17 +848,20 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
ExceptionContract::possibleButRequiredIf(!limits.isReached() && !limits.isValid())),
std::make_pair(error_code_client_invalid_operation, ExceptionContract::Possible),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((keysel1.getKey() > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) ||
(keysel2.getKey() > (workload->useSystemKeys ? systemKeys.end : normalKeys.end))) &&
!isSpecialKeyRange)),
ExceptionContract::requiredIf(((keysel1.getKey() > workload->getMaxKey(tr)) ||
(keysel2.getKey() > workload->getMaxKey(tr))) &&
!isSpecialKeyRange)),
std::make_pair(error_code_special_keys_cross_module_read,
ExceptionContract::possibleIf(isSpecialKeyRange && !workload->specialKeysRelaxed)),
std::make_pair(error_code_special_keys_no_module_found,
ExceptionContract::possibleIf(isSpecialKeyRange && !workload->specialKeysRelaxed)),
std::make_pair(error_code_timed_out, ExceptionContract::possibleIf(isSpecialKeyRange)),
std::make_pair(error_code_special_keys_api_failure, ExceptionContract::possibleIf(isSpecialKeyRange)),
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible)
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible),
std::make_pair(error_code_tenant_not_found,
ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
std::make_pair(error_code_invalid_option,
ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange))
};
}
@ -781,7 +884,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
Key key1, key2;
int limit;
TestGetRange2(unsigned int id, FuzzApiCorrectnessWorkload* workload) : BaseTest(id, workload, "TestGetRange2") {
TestGetRange2(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTest(id, workload, "TestGetRange2") {
key1 = makeKey();
key2 = makeKey();
limit = 0;
@ -806,11 +910,10 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
std::make_pair(error_code_inverted_range, ExceptionContract::requiredIf(key1 > key2)),
std::make_pair(error_code_range_limits_invalid, ExceptionContract::possibleButRequiredIf(limit < 0)),
std::make_pair(error_code_client_invalid_operation, ExceptionContract::Possible),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((key1 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) ||
(key2 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end))) &&
!isSpecialKeyRange)),
std::make_pair(
error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((key1 > workload->getMaxKey(tr)) || (key2 > workload->getMaxKey(tr))) && !isSpecialKeyRange)),
std::make_pair(error_code_special_keys_cross_module_read,
ExceptionContract::possibleIf(isSpecialKeyRange && !workload->specialKeysRelaxed)),
std::make_pair(error_code_special_keys_no_module_found,
@ -821,7 +924,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
std::make_pair(error_code_special_keys_api_failure,
ExceptionContract::possibleIf(key1 <= autoCoordinatorSpecialKey &&
autoCoordinatorSpecialKey < key2)),
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible)
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible),
std::make_pair(error_code_tenant_not_found,
ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
std::make_pair(error_code_invalid_option,
ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange))
};
}
@ -832,7 +939,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2)).detail("Limit", limit);
e.detail("Key1", key1).detail("Key2", key2).detail("Limit", limit);
}
};
@ -841,7 +948,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
Key key1, key2;
GetRangeLimits limits;
TestGetRange3(unsigned int id, FuzzApiCorrectnessWorkload* workload) : BaseTest(id, workload, "TestGetRange3") {
TestGetRange3(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTest(id, workload, "TestGetRange3") {
key1 = makeKey();
key2 = makeKey();
limits = makeRangeLimits();
@ -857,11 +965,10 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
std::make_pair(error_code_range_limits_invalid,
ExceptionContract::possibleButRequiredIf(!limits.isReached() && !limits.isValid())),
std::make_pair(error_code_client_invalid_operation, ExceptionContract::Possible),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((key1 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) ||
(key2 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end))) &&
!isSpecialKeyRange)),
std::make_pair(
error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((key1 > workload->getMaxKey(tr)) || (key2 > workload->getMaxKey(tr))) && !isSpecialKeyRange)),
std::make_pair(error_code_special_keys_cross_module_read,
ExceptionContract::possibleIf(isSpecialKeyRange && !workload->specialKeysRelaxed)),
std::make_pair(error_code_special_keys_no_module_found,
@ -872,7 +979,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
std::make_pair(error_code_special_keys_api_failure,
ExceptionContract::possibleIf((key1 <= autoCoordinatorSpecialKey) &&
(autoCoordinatorSpecialKey < key2))),
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible)
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible),
std::make_pair(error_code_tenant_not_found,
ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))),
std::make_pair(error_code_invalid_option,
ExceptionContract::possibleIf(tr->getTenant().present() && isSpecialKeyRange))
};
}
@ -883,7 +994,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
e.detail("Key1", key1).detail("Key2", key2);
std::stringstream ss;
ss << "(" << limits.rows << ", " << limits.minRows << ", " << limits.bytes << ")";
e.detail("Limits", ss.str());
@ -894,10 +1005,12 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestGetAddressesForKey, Standalone<VectorRef<const char*>>> base_type;
Key key;
TestGetAddressesForKey(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestGetAddressesForKey(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTest(id, workload, "TestGetAddressesForKey") {
key = makeKey();
contract = { std::make_pair(error_code_client_invalid_operation, ExceptionContract::Possible) };
contract = { std::make_pair(error_code_client_invalid_operation, ExceptionContract::Possible),
std::make_pair(error_code_tenant_not_found,
ExceptionContract::requiredIf(!workload->canUseTenant(tr->getTenant()))) };
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
@ -906,7 +1019,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key));
e.detail("Key", key);
}
};
@ -914,22 +1027,21 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestAddReadConflictRange, Void> base_type;
Key key1, key2;
TestAddReadConflictRange(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestAddReadConflictRange(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTestCallback(id, workload, "TestAddReadConflictRange") {
key1 = makeKey();
key2 = makeKey();
contract = { std::make_pair(error_code_inverted_range, ExceptionContract::requiredIf(key1 > key2)),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
(key1 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) ||
(key2 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)))) };
ExceptionContract::requiredIf((key1 > workload->getMaxKey(tr)) ||
(key2 > workload->getMaxKey(tr)))) };
}
void callback(Reference<ITransaction> tr) override { tr->addReadConflictRange(KeyRangeRef(key1, key2)); }
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
e.detail("Key1", key1).detail("Key2", key2);
}
};
@ -940,7 +1052,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
uint8_t op;
int32_t pos;
TestAtomicOp(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestAtomicOp(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTestCallback(id, workload, "TestAtomicOp") {
key = makeKey();
while (isProtectedKey(key)) {
@ -990,8 +1102,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
error_code_invalid_mutation_type,
ExceptionContract::requiredIf(!isValidMutationType(op) || !isAtomicOp((MutationRef::Type)op))),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
(key >= (workload->useSystemKeys ? systemKeys.end : normalKeys.end)))),
ExceptionContract::requiredIf((key >= workload->getMaxKey(tr)))),
std::make_pair(
error_code_client_invalid_operation,
ExceptionContract::requiredIf(
@ -1004,7 +1115,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key)).detail("Value", printable(value)).detail("Op", op).detail("Pos", pos);
e.detail("Key", key).detail("Value", value).detail("Op", op).detail("Pos", pos);
}
};
@ -1013,7 +1124,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
Key key;
Value value;
TestSet(unsigned int id, FuzzApiCorrectnessWorkload* workload) : BaseTestCallback(id, workload, "TestSet") {
TestSet(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTestCallback(id, workload, "TestSet") {
key = makeKey();
while (isProtectedKey(key)) {
key = makeKey();
@ -1027,9 +1139,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
std::make_pair(error_code_value_too_large,
ExceptionContract::requiredIf(value.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT)),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
(key >= (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) &&
!specialKeys.contains(key))),
ExceptionContract::requiredIf((key >= workload->getMaxKey(tr)) &&
!specialKeys.contains(key))),
std::make_pair(error_code_special_keys_write_disabled,
ExceptionContract::requiredIf(specialKeys.contains(key) &&
!workload->specialKeysWritesEnabled)),
@ -1042,7 +1153,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key)).detail("Value", printable(value));
e.detail("Key", key).detail("Value", value);
}
};
@ -1050,7 +1161,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestClear0, Void> base_type;
Key key1, key2;
TestClear0(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestClear0(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTestCallback(id, workload, "TestClear0") {
key1 = makeKey();
key2 = makeKey();
@ -1063,11 +1174,10 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
contract = {
std::make_pair(error_code_inverted_range, ExceptionContract::requiredIf(key1 > key2)),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((key1 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) ||
(key2 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end))) &&
!isSpecialKeyRange)),
std::make_pair(
error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((key1 > workload->getMaxKey(tr)) || (key2 > workload->getMaxKey(tr))) && !isSpecialKeyRange)),
std::make_pair(error_code_special_keys_write_disabled,
ExceptionContract::requiredIf(isSpecialKeyRange && !workload->specialKeysWritesEnabled)),
std::make_pair(error_code_special_keys_cross_module_clear,
@ -1081,7 +1191,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
e.detail("Key1", key1).detail("Key2", key2);
}
};
@ -1089,7 +1199,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestClear1, Void> base_type;
Key key1, key2;
TestClear1(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestClear1(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTestCallback(id, workload, "TestClear1") {
key1 = makeKey();
key2 = makeKey();
@ -1102,11 +1212,10 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
contract = {
std::make_pair(error_code_inverted_range, ExceptionContract::requiredIf(key1 > key2)),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((key1 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) ||
(key2 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end))) &&
!isSpecialKeyRange)),
std::make_pair(
error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
((key1 > workload->getMaxKey(tr)) || (key2 > workload->getMaxKey(tr))) && !isSpecialKeyRange)),
std::make_pair(error_code_special_keys_write_disabled,
ExceptionContract::requiredIf(isSpecialKeyRange && !workload->specialKeysWritesEnabled)),
std::make_pair(error_code_special_keys_cross_module_clear,
@ -1120,7 +1229,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
e.detail("Key1", key1).detail("Key2", key2);
}
};
@ -1128,15 +1237,14 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestClear2, Void> base_type;
Key key;
TestClear2(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestClear2(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTestCallback(id, workload, "TestClear2") {
key = makeKey();
while (isProtectedKey(key)) {
key = makeKey();
}
contract = { std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
key >= (workload->useSystemKeys ? systemKeys.end : normalKeys.end))),
ExceptionContract::requiredIf(key >= workload->getMaxKey(tr))),
std::make_pair(error_code_special_keys_write_disabled,
ExceptionContract::requiredIf(specialKeys.contains(key) &&
!workload->specialKeysWritesEnabled)),
@ -1149,7 +1257,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key));
e.detail("Key", key);
}
};
@ -1157,7 +1265,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestWatch, Void> base_type;
Key key;
TestWatch(unsigned int id, FuzzApiCorrectnessWorkload* workload) : BaseTest(id, workload, "TestWatch") {
TestWatch(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTest(id, workload, "TestWatch") {
key = makeKey();
contract = { std::make_pair(
error_code_key_too_large,
@ -1166,18 +1275,19 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
: CLIENT_KNOBS->KEY_SIZE_LIMIT))),
std::make_pair(error_code_watches_disabled, ExceptionContract::Possible),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
(key >= (workload->useSystemKeys ? systemKeys.end : normalKeys.end)))),
ExceptionContract::requiredIf((key >= workload->getMaxKey(tr)))),
std::make_pair(error_code_client_invalid_operation, ExceptionContract::Possible),
std::make_pair(error_code_timed_out, ExceptionContract::Possible),
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible) };
std::make_pair(error_code_accessed_unreadable, ExceptionContract::Possible),
std::make_pair(error_code_tenant_not_found,
ExceptionContract::possibleIf(!workload->canUseTenant(tr->getTenant()))) };
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override { return tr->watch(key); }
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key));
e.detail("Key", key);
}
};
@ -1185,22 +1295,21 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestAddWriteConflictRange, Void> base_type;
Key key1, key2;
TestAddWriteConflictRange(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestAddWriteConflictRange(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTestCallback(id, workload, "TestAddWriteConflictRange") {
key1 = makeKey();
key2 = makeKey();
contract = { std::make_pair(error_code_inverted_range, ExceptionContract::requiredIf(key1 > key2)),
std::make_pair(error_code_key_outside_legal_range,
ExceptionContract::requiredIf(
(key1 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)) ||
(key2 > (workload->useSystemKeys ? systemKeys.end : normalKeys.end)))) };
ExceptionContract::requiredIf((key1 > workload->getMaxKey(tr)) ||
(key2 > workload->getMaxKey(tr)))) };
}
void callback(Reference<ITransaction> tr) override { tr->addWriteConflictRange(KeyRangeRef(key1, key2)); }
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
e.detail("Key1", key1).detail("Key2", key2);
}
};
@ -1209,7 +1318,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
int op;
Optional<Standalone<StringRef>> val;
TestSetOption(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestSetOption(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTestCallback(id, workload, "TestSetOption") {
double arv = deterministicRandom()->random01();
if (arv < 0.25) {
@ -1240,7 +1349,8 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
// do not test the following options since they are actually used by the workload
if (op == FDBTransactionOptions::ACCESS_SYSTEM_KEYS || op == FDBTransactionOptions::READ_SYSTEM_KEYS ||
op == FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES) {
op == FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES ||
op == FDBTransactionOptions::RAW_ACCESS) {
op = -1;
}
@ -1285,7 +1395,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Op", op).detail("Val", printable(val));
e.detail("Op", op).detail("Val", val);
}
};
@ -1293,7 +1403,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
typedef BaseTest<TestOnError, Void> base_type;
int errorcode;
TestOnError(unsigned int id, FuzzApiCorrectnessWorkload* workload)
TestOnError(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
: BaseTestCallback(id, workload, "TestOnError") {
errorcode = 0;
double erv = deterministicRandom()->random01();
@ -1309,7 +1419,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
tr->onError(Error::fromUnvalidatedCode(errorcode));
// This is necessary here, as onError will have reset this
// value, we will be looking at the wrong thing.
if (workload->useSystemKeys)
if (workload->useSystemKeys && !tr->getTenant().present())
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
}

3
fdbserver/workloads/LockDatabase.actor.cpp
View File

@ -18,6 +18,7 @@
* limitations under the License.
*/
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbserver/TesterInterface.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
@ -54,6 +55,7 @@ struct LockDatabaseWorkload : TestWorkload {
state Transaction tr(cx);
loop {
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
wait(lockDatabase(&tr, lockID));
state RangeResult data = wait(tr.getRange(normalKeys, 50000));
ASSERT(!data.more);
@ -70,6 +72,7 @@ struct LockDatabaseWorkload : TestWorkload {
loop {
try {
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
Optional<Value> val = wait(tr.get(databaseLockedKey));
if (!val.present())
return Void();

2
fdbserver/workloads/MachineAttrition.actor.cpp
View File

@ -18,6 +18,7 @@
* limitations under the License.
*/
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/CoordinationInterface.h"
#include "fdbserver/TesterInterface.actor.h"
@ -48,6 +49,7 @@ ACTOR Future<bool> ignoreSSFailuresForDuration(Database cx, double duration) {
loop {
try {
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.clear(healthyZoneKey);
wait(tr.commit());
TraceEvent("IgnoreSSFailureComplete").log();

3
fdbserver/workloads/MiniCycle.actor.cpp
View File

@ -26,10 +26,11 @@
#include "flow/Arena.h"
#include "flow/IRandom.h"
#include "flow/Trace.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/serialize.h"
#include <cstring>
#include "flow/actorcompiler.h" // This must be the last #include.
struct MiniCycleWorkload : TestWorkload {
int actorCount, nodeCount;
double testDuration, transactionsPerSecond, minExpectedTransactionsPerSecond, traceParentProbability;

4
fdbserver/workloads/Performance.actor.cpp
View File

@ -137,7 +137,7 @@ struct PerformanceWorkload : TestWorkload {
TestSpec spec(LiteralStringRef("PerformanceSetup"), false, false);
spec.options = options;
spec.phases = TestWorkload::SETUP;
DistributedTestResults results = wait(runWorkload(cx, testers, spec));
DistributedTestResults results = wait(runWorkload(cx, testers, spec, Optional<TenantName>()));
return Void();
}
@ -172,7 +172,7 @@ struct PerformanceWorkload : TestWorkload {
TestSpec spec(LiteralStringRef("PerformanceRun"), false, false);
spec.phases = TestWorkload::EXECUTION | TestWorkload::METRICS;
spec.options = options;
DistributedTestResults r = wait(runWorkload(cx, self->testers, spec));
DistributedTestResults r = wait(runWorkload(cx, self->testers, spec, Optional<TenantName>()));
results = r;
} catch (Error& e) {
TraceEvent("PerformanceRunError")

10
fdbserver/workloads/PhysicalShardMove.actor.cpp
View File

@ -77,11 +77,11 @@ struct SSCheckpointWorkload : TestWorkload {
// Create checkpoint.
state Transaction tr(cx);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state CheckpointFormat format = RocksDBColumnFamily;
loop {
try {
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
wait(createCheckpoint(&tr, KeyRangeRef(key, endKey), format));
wait(tr.commit());
version = tr.getCommittedVersion();
@ -157,9 +157,10 @@ struct SSCheckpointWorkload : TestWorkload {
// Compare the keyrange between the original database and the one restored from checkpoint.
// For now, it should have been a single key.
tr.reset();
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
loop {
try {
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
state RangeResult res = wait(tr.getRange(KeyRangeRef(key, endKey), CLIENT_KNOBS->TOO_MANY));
break;
} catch (Error& e) {
@ -182,10 +183,10 @@ struct SSCheckpointWorkload : TestWorkload {
Key key,
ErrorOr<Optional<Value>> expectedValue) {
state Transaction tr(cx);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
state Optional<Value> res = wait(timeoutError(tr.get(key), 30.0));
const bool equal = !expectedValue.isError() && res == expectedValue.get();
if (!equal) {
@ -208,6 +209,7 @@ struct SSCheckpointWorkload : TestWorkload {
state Version version;
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
if (value.present()) {
tr.set(key, value.get());
} else {

2
fdbserver/workloads/SelectorCorrectness.actor.cpp
View File

@ -118,8 +118,6 @@ struct SelectorCorrectnessWorkload : TestWorkload {
state Transaction tr(cx);
state ReadYourWritesTransaction trRYOW(cx);
trRYOW.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
if (self->testReadYourWrites) {
myValue = StringRef(format("%010d", deterministicRandom()->randomInt(0, 10000000)));
for (int i = 2; i < self->maxKeySpace; i += 4)

46
fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
View File

@ -70,6 +70,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
Future<Void> _setup(Database cx, SpecialKeySpaceCorrectnessWorkload* self) {
cx->specialKeySpace = std::make_unique<SpecialKeySpace>();
self->ryw = makeReference<ReadYourWritesTransaction>(cx);
self->ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
self->ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED);
self->ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
self->ryw->setVersion(100);
@ -291,6 +292,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
// begin key outside module range
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(
KeyRangeRef(LiteralStringRef("\xff\xff/transactio"), LiteralStringRef("\xff\xff/transaction0")),
CLIENT_KNOBS->TOO_MANY)));
@ -303,6 +305,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// end key outside module range
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(
KeyRangeRef(LiteralStringRef("\xff\xff/transaction/"), LiteralStringRef("\xff\xff/transaction1")),
CLIENT_KNOBS->TOO_MANY)));
@ -315,6 +318,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// both begin and end outside module range
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(
KeyRangeRef(LiteralStringRef("\xff\xff/transaction"), LiteralStringRef("\xff\xff/transaction1")),
CLIENT_KNOBS->TOO_MANY)));
@ -327,6 +331,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// legal range read using the module range
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(
KeyRangeRef(LiteralStringRef("\xff\xff/transaction/"), LiteralStringRef("\xff\xff/transaction0")),
CLIENT_KNOBS->TOO_MANY)));
@ -337,6 +342,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// cross module read with option turned on
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED);
const KeyRef startKey = LiteralStringRef("\xff\xff/transactio");
const KeyRef endKey = LiteralStringRef("\xff\xff/transaction1");
@ -350,6 +356,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// end keySelector inside module range, *** a tricky corner case ***
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->addReadConflictRange(singleKeyRange(LiteralStringRef("testKey")));
KeySelector begin = KeySelectorRef(readConflictRangeKeysRange.begin, false, 1);
KeySelector end = KeySelectorRef(LiteralStringRef("\xff\xff/transaction0"), false, 0);
@ -361,6 +368,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// No module found error case with keys
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/A_no_module_related_prefix"),
LiteralStringRef("\xff\xff/I_am_also_not_in_any_module")),
CLIENT_KNOBS->TOO_MANY)));
@ -373,6 +381,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// No module found error with KeySelectors, *** a tricky corner case ***
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
KeySelector begin = KeySelectorRef(LiteralStringRef("\xff\xff/zzz_i_am_not_a_module"), false, 1);
KeySelector end = KeySelectorRef(LiteralStringRef("\xff\xff/zzz_to_be_the_final_one"), false, 2);
wait(success(tx->getRange(begin, end, CLIENT_KNOBS->TOO_MANY)));
@ -385,6 +394,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// begin and end keySelectors clamp up to the boundary of the module
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
const KeyRef key = LiteralStringRef("\xff\xff/cluster_file_path");
KeySelector begin = KeySelectorRef(key, false, 0);
KeySelector end = KeySelectorRef(keyAfter(key), false, 2);
@ -395,6 +405,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
throw;
}
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->addReadConflictRange(singleKeyRange(LiteralStringRef("readKey")));
const KeyRef key = LiteralStringRef("\xff\xff/transaction/a_to_be_the_first");
KeySelector begin = KeySelectorRef(key, false, 0);
@ -408,6 +419,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// Errors introduced by SpecialKeyRangeRWImpl
// Writes are disabled by default
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->set(LiteralStringRef("\xff\xff/I_am_not_a_range_can_be_written"), ValueRef());
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled)
@ -417,6 +429,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// The special key is not in a range that can be called with set
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(LiteralStringRef("\xff\xff/I_am_not_a_range_can_be_written"), ValueRef());
ASSERT(false);
@ -428,6 +441,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// A clear cross two ranges are forbidden
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->clear(KeyRangeRef(SpecialKeySpace::getManagementApiCommandRange("exclude").begin,
SpecialKeySpace::getManagementApiCommandRange("failed").end));
@ -440,6 +454,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// base key of the end key selector not in (\xff\xff, \xff\xff\xff), throw key_outside_legal_range()
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
const KeySelector startKeySelector = KeySelectorRef(LiteralStringRef("\xff\xff/test"), true, -200);
const KeySelector endKeySelector = KeySelectorRef(LiteralStringRef("test"), true, -10);
RangeResult result =
@ -453,6 +468,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// test case when registered range is the same as the underlying module
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
state RangeResult result = wait(tx->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
LiteralStringRef("\xff\xff/worker_interfaces0")),
CLIENT_KNOBS->TOO_MANY));
@ -480,12 +496,13 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
state Reference<ReadYourWritesTransaction> referenceTx = makeReference<ReadYourWritesTransaction>(cx);
state bool ryw = deterministicRandom()->coinflip();
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
if (!ryw) {
tx->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
}
referenceTx->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
referenceTx->setVersion(100); // Prevent this from doing a GRV or committing
referenceTx->clear(normalKeys);
referenceTx->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
int numKeys = deterministicRandom()->randomInt(1, self->conflictRangeSizeFactor) * 4;
state std::vector<std::string> keys; // Must all be distinct
keys.resize(numKeys);
@ -630,6 +647,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
// test ordered option keys
{
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
for (const std::string& option : SpecialKeySpace::getManagementApiOptionsSet()) {
tx->set(
@ -648,6 +666,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// "exclude" error message shema check
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(LiteralStringRef("Invalid_Network_Address")
.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("exclude")),
@ -676,6 +695,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// "setclass"
{
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// test getRange
state RangeResult result = wait(tx->getRange(
@ -747,6 +767,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
{
try {
// test getRange
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
state RangeResult class_source_result = wait(tx->getRange(
KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0"))
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin),
@ -784,6 +805,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// class source will be changed
wait(tx->commit());
tx->reset();
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> class_source = wait(tx->get(
Key("process/class_source/" + address)
.withPrefix(
@ -807,6 +829,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// maske sure we lock the database
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// lock the database
UID uid = deterministicRandom()->randomUniqueID();
@ -842,6 +865,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// if database locked, fdb read should get database_locked error
try {
tx->reset();
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
RangeResult res = wait(tx->getRange(normalKeys, 1));
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled)
@ -853,6 +877,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
loop {
try {
tx->reset();
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// unlock the database
tx->clear(SpecialKeySpace::getManagementApiCommandPrefix("lock"));
@ -860,6 +885,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
TraceEvent(SevDebug, "DatabaseUnlocked").log();
tx->reset();
// read should be successful
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
RangeResult res = wait(tx->getRange(normalKeys, 1));
tx->reset();
break;
@ -904,6 +930,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
{
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->clear(SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck"));
wait(tx->commit());
@ -1001,6 +1028,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
loop {
try {
std::string new_processes_key(new_coordinator_process);
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
for (const auto& address : old_coordinators_processes) {
new_processes_key += "," + address;
@ -1071,6 +1099,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
loop {
try {
std::string new_processes_key;
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
for (const auto& address : old_coordinators_processes) {
new_processes_key += new_processes_key.size() ? "," : "";
@ -1116,6 +1145,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// advanceversion
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
Version v1 = wait(tx->getReadVersion());
TraceEvent(SevDebug, "InitialReadVersion").detail("Version", v1);
state Version v2 = 2 * v1;
@ -1127,6 +1157,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
TraceEvent(SevDebug, "AdvanceVersionSuccess").detail("Version", v3);
break;
}
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// force the cluster to recover at v2
tx->set(SpecialKeySpace::getManagementApiCommandPrefix("advanceversion"), std::to_string(v2));
@ -1192,6 +1223,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// update the sample rate and size limit
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(LiteralStringRef("client_txn_sample_rate")
.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile")),
@ -1225,6 +1257,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// Change back to default
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(LiteralStringRef("client_txn_sample_rate")
.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile")),
@ -1242,6 +1275,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// Test invalid values
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set((deterministicRandom()->coinflip() ? LiteralStringRef("client_txn_sample_rate")
: LiteralStringRef("client_txn_size_limit"))
@ -1297,6 +1331,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// Make sure setting more than one zone as maintenance will fail
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(Key(deterministicRandom()->randomAlphaNumeric(8))
.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("maintenance")),
@ -1333,6 +1368,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
state int ignoreSSFailuresRetry = 0;
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(ignoreSSFailuresZoneString.withPrefix(
SpecialKeySpace::getManagementApiCommandPrefix("maintenance")),
@ -1371,6 +1407,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// set dd mode to 0 and disable DD for rebalance
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");
tx->set(LiteralStringRef("mode").withPrefix(ddPrefix), LiteralStringRef("0"));
@ -1410,6 +1447,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// then, clear all changes
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->clear(ignoreSSFailuresZoneString.withPrefix(
SpecialKeySpace::getManagementApiCommandPrefix("maintenance")));
@ -1450,10 +1488,12 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(cx));
loop {
try {
Version readVersion = wait(tr1->getReadVersion());
tr2->setVersion(readVersion);
tr1->setOption(FDBTransactionOptions::RAW_ACCESS);
tr1->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tr2->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
Version readVersion = wait(tr1->getReadVersion());
tr2->setVersion(readVersion);
KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");
tr1->set(LiteralStringRef("mode").withPrefix(ddPrefix), LiteralStringRef("1"));
wait(tr1->commit());

3
fdbserver/workloads/StreamingRangeRead.actor.cpp
View File

@ -27,10 +27,11 @@
#include "flow/Error.h"
#include "flow/IRandom.h"
#include "flow/Trace.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/serialize.h"
#include <cstring>
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR Future<Void> streamUsingGetRange(PromiseStream<RangeResult> results, Transaction* tr, KeyRange keys) {
state KeySelectorRef begin = firstGreaterOrEqual(keys.begin);
state KeySelectorRef end = firstGreaterOrEqual(keys.end);

583
fdbserver/workloads/TenantManagement.actor.cpp Normal file
View File

@ -0,0 +1,583 @@
/*
* TenantManagement.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <cstdint>
#include <limits>
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/GenericManagementAPI.actor.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/Knobs.h"
#include "flow/Error.h"
#include "flow/IRandom.h"
#include "flow/flow.h"
#include "flow/actorcompiler.h" // This must be the last #include.
struct TenantManagementWorkload : TestWorkload {
struct TenantState {
int64_t id;
bool empty;
TenantState() : id(-1), empty(true) {}
TenantState(int64_t id, bool empty) : id(id), empty(empty) {}
};
std::map<TenantName, TenantState> createdTenants;
int64_t maxId = -1;
Key tenantSubspace;
const Key keyName = "key"_sr;
const Key tenantSubspaceKey = "tenant_subspace"_sr;
const Value noTenantValue = "no_tenant"_sr;
const TenantName tenantNamePrefix = "tenant_management_workload_"_sr;
TenantName localTenantNamePrefix;
const Key specialKeysTenantMapPrefix = TenantMapRangeImpl::submoduleRange.begin.withPrefix(
SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
int maxTenants;
double testDuration;
enum class OperationType { SPECIAL_KEYS, MANAGEMENT_DATABASE, MANAGEMENT_TRANSACTION };
static OperationType randomOperationType() {
int randomNum = deterministicRandom()->randomInt(0, 3);
if (randomNum == 0) {
return OperationType::SPECIAL_KEYS;
} else if (randomNum == 1) {
return OperationType::MANAGEMENT_DATABASE;
} else {
return OperationType::MANAGEMENT_TRANSACTION;
}
}
TenantManagementWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
maxTenants = std::min<int>(1e8 - 1, getOption(options, "maxTenants"_sr, 1000));
testDuration = getOption(options, "testDuration"_sr, 60.0);
localTenantNamePrefix = format("%stenant_%d_", tenantNamePrefix.toString().c_str(), clientId);
}
std::string description() const override { return "TenantManagement"; }
Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
ACTOR Future<Void> _setup(Database cx, TenantManagementWorkload* self) {
state Transaction tr(cx);
if (self->clientId == 0) {
self->tenantSubspace = makeString(deterministicRandom()->randomInt(0, 10));
generateRandomData(mutateString(self->tenantSubspace), self->tenantSubspace.size());
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
tr.set(self->keyName, self->noTenantValue);
tr.set(self->tenantSubspaceKey, self->tenantSubspace);
tr.set(tenantDataPrefixKey, self->tenantSubspace);
wait(tr.commit());
break;
} catch (Error& e) {
wait(tr.onError(e));
}
}
} else {
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(tr.get(self->tenantSubspaceKey));
if (val.present()) {
self->tenantSubspace = val.get();
break;
}
wait(delay(1.0));
} catch (Error& e) {
wait(tr.onError(e));
}
}
}
return Void();
}
TenantName chooseTenantName(bool allowSystemTenant) {
TenantName tenant(format(
"%s%08d", localTenantNamePrefix.toString().c_str(), deterministicRandom()->randomInt(0, maxTenants)));
if (allowSystemTenant && deterministicRandom()->random01() < 0.02) {
tenant = tenant.withPrefix("\xff"_sr);
}
return tenant;
}
ACTOR Future<Void> createTenant(Database cx, TenantManagementWorkload* self) {
state TenantName tenant = self->chooseTenantName(true);
state bool alreadyExists = self->createdTenants.count(tenant);
state OperationType operationType = TenantManagementWorkload::randomOperationType();
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
loop {
try {
if (operationType == OperationType::SPECIAL_KEYS) {
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
Key key = self->specialKeysTenantMapPrefix.withSuffix(tenant);
tr->set(key, ""_sr);
wait(tr->commit());
} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
wait(ManagementAPI::createTenant(cx.getReference(), tenant));
} else {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
Optional<TenantMapEntry> _ = wait(ManagementAPI::createTenantTransaction(tr, tenant));
wait(tr->commit());
}
if (operationType != OperationType::MANAGEMENT_DATABASE && alreadyExists) {
return Void();
}
ASSERT(!alreadyExists);
ASSERT(!tenant.startsWith("\xff"_sr));
state Optional<TenantMapEntry> entry = wait(ManagementAPI::tryGetTenant(cx.getReference(), tenant));
ASSERT(entry.present());
ASSERT(entry.get().id > self->maxId);
ASSERT(entry.get().prefix.startsWith(self->tenantSubspace));
self->maxId = entry.get().id;
self->createdTenants[tenant] = TenantState(entry.get().id, true);
state bool insertData = deterministicRandom()->random01() < 0.5;
if (insertData) {
state Transaction insertTr(cx, tenant);
loop {
try {
insertTr.set(self->keyName, tenant);
wait(insertTr.commit());
break;
} catch (Error& e) {
wait(insertTr.onError(e));
}
}
self->createdTenants[tenant].empty = false;
state Transaction checkTr(cx);
loop {
try {
checkTr.setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(checkTr.get(self->keyName.withPrefix(entry.get().prefix)));
ASSERT(val.present());
ASSERT(val.get() == tenant);
break;
} catch (Error& e) {
wait(checkTr.onError(e));
}
}
}
wait(self->checkTenant(cx, self, tenant, self->createdTenants[tenant]));
return Void();
} catch (Error& e) {
if (e.code() == error_code_invalid_tenant_name) {
ASSERT(tenant.startsWith("\xff"_sr));
return Void();
} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
if (e.code() == error_code_tenant_already_exists) {
ASSERT(alreadyExists && operationType == OperationType::MANAGEMENT_DATABASE);
} else {
TraceEvent(SevError, "CreateTenantFailure").error(e).detail("TenantName", tenant);
}
return Void();
} else {
try {
wait(tr->onError(e));
} catch (Error& e) {
TraceEvent(SevError, "CreateTenantFailure").error(e).detail("TenantName", tenant);
return Void();
}
}
}
}
}
ACTOR Future<Void> deleteTenant(Database cx, TenantManagementWorkload* self) {
state TenantName tenant = self->chooseTenantName(true);
state OperationType operationType = TenantManagementWorkload::randomOperationType();
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
state Optional<TenantName> endTenant = operationType != OperationType::MANAGEMENT_DATABASE &&
!tenant.startsWith("\xff"_sr) &&
deterministicRandom()->random01() < 0.2
? Optional<TenantName>(self->chooseTenantName(false))
: Optional<TenantName>();
if (endTenant.present() && endTenant < tenant) {
TenantName temp = tenant;
tenant = endTenant.get();
endTenant = temp;
}
auto itr = self->createdTenants.find(tenant);
state bool alreadyExists = itr != self->createdTenants.end();
state bool isEmpty = true;
state std::vector<TenantName> tenants;
if (!endTenant.present()) {
tenants.push_back(tenant);
} else if (endTenant.present()) {
for (auto itr = self->createdTenants.lower_bound(tenant);
itr != self->createdTenants.end() && itr->first < endTenant.get();
++itr) {
tenants.push_back(itr->first);
}
}
try {
if (alreadyExists || endTenant.present()) {
state int tenantIndex = 0;
for (; tenantIndex < tenants.size(); ++tenantIndex) {
if (deterministicRandom()->random01() < 0.9) {
state Transaction clearTr(cx, tenants[tenantIndex]);
loop {
try {
clearTr.clear(self->keyName);
wait(clearTr.commit());
auto itr = self->createdTenants.find(tenants[tenantIndex]);
ASSERT(itr != self->createdTenants.end());
itr->second.empty = true;
break;
} catch (Error& e) {
wait(clearTr.onError(e));
}
}
} else {
auto itr = self->createdTenants.find(tenants[tenantIndex]);
ASSERT(itr != self->createdTenants.end());
isEmpty = isEmpty && itr->second.empty;
}
}
}
} catch (Error& e) {
TraceEvent(SevError, "DeleteTenantFailure")
.error(e)
.detail("TenantName", tenant)
.detail("EndTenant", endTenant);
return Void();
}
loop {
try {
if (operationType == OperationType::SPECIAL_KEYS) {
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
Key key = self->specialKeysTenantMapPrefix.withSuffix(tenant);
if (endTenant.present()) {
tr->clear(KeyRangeRef(key, self->specialKeysTenantMapPrefix.withSuffix(endTenant.get())));
} else {
tr->clear(key);
}
wait(tr->commit());
} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
ASSERT(tenants.size() == 1);
for (auto tenant : tenants) {
wait(ManagementAPI::deleteTenant(cx.getReference(), tenant));
}
} else {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
for (auto tenant : tenants) {
wait(ManagementAPI::deleteTenantTransaction(tr, tenant));
}
wait(tr->commit());
}
if (!alreadyExists && !endTenant.present() && operationType != OperationType::MANAGEMENT_DATABASE) {
return Void();
}
ASSERT(alreadyExists || endTenant.present());
ASSERT(isEmpty);
for (auto tenant : tenants) {
self->createdTenants.erase(tenant);
}
return Void();
} catch (Error& e) {
if (e.code() == error_code_tenant_not_empty) {
ASSERT(!isEmpty);
return Void();
} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
if (e.code() == error_code_tenant_not_found) {
ASSERT(!alreadyExists && !endTenant.present());
} else {
TraceEvent(SevError, "DeleteTenantFailure")
.error(e)
.detail("TenantName", tenant)
.detail("EndTenant", endTenant);
}
return Void();
} else {
try {
wait(tr->onError(e));
} catch (Error& e) {
TraceEvent(SevError, "DeleteTenantFailure")
.error(e)
.detail("TenantName", tenant)
.detail("EndTenant", endTenant);
return Void();
}
}
}
}
}
ACTOR Future<Void> checkTenant(Database cx,
TenantManagementWorkload* self,
TenantName tenant,
TenantState tenantState) {
state Transaction tr(cx, tenant);
loop {
try {
state RangeResult result = wait(tr.getRange(KeyRangeRef(""_sr, "\xff"_sr), 2));
if (tenantState.empty) {
ASSERT(result.size() == 0);
} else {
ASSERT(result.size() == 1);
ASSERT(result[0].key == self->keyName);
ASSERT(result[0].value == tenant);
}
break;
} catch (Error& e) {
wait(tr.onError(e));
}
}
return Void();
}
static TenantMapEntry jsonToTenantMapEntry(ValueRef tenantJson) {
json_spirit::mValue jsonObject;
json_spirit::read_string(tenantJson.toString(), jsonObject);
JSONDoc jsonDoc(jsonObject);
int64_t id;
std::string prefix;
jsonDoc.get("id", id);
jsonDoc.get("prefix", prefix);
Key prefixKey = KeyRef(prefix);
TenantMapEntry entry(id, prefixKey.substr(0, prefixKey.size() - 8));
ASSERT(entry.prefix == prefixKey);
return entry;
}
ACTOR Future<Void> getTenant(Database cx, TenantManagementWorkload* self) {
state TenantName tenant = self->chooseTenantName(true);
auto itr = self->createdTenants.find(tenant);
state bool alreadyExists = itr != self->createdTenants.end();
state TenantState tenantState = itr->second;
state OperationType operationType = TenantManagementWorkload::randomOperationType();
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
loop {
try {
state TenantMapEntry entry;
if (operationType == OperationType::SPECIAL_KEYS) {
Key key = self->specialKeysTenantMapPrefix.withSuffix(tenant);
Optional<Value> value = wait(tr->get(key));
if (!value.present()) {
throw tenant_not_found();
}
entry = TenantManagementWorkload::jsonToTenantMapEntry(value.get());
} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
TenantMapEntry _entry = wait(ManagementAPI::getTenant(cx.getReference(), tenant));
entry = _entry;
} else {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
TenantMapEntry _entry = wait(ManagementAPI::getTenantTransaction(tr, tenant));
entry = _entry;
}
ASSERT(alreadyExists);
ASSERT(entry.id == tenantState.id);
wait(self->checkTenant(cx, self, tenant, tenantState));
return Void();
} catch (Error& e) {
state bool retry = true;
state Error error = e;
if (e.code() == error_code_tenant_not_found) {
ASSERT(!alreadyExists);
return Void();
} else if (operationType != OperationType::MANAGEMENT_DATABASE) {
try {
wait(tr->onError(e));
} catch (Error& e) {
error = e;
retry = false;
}
}
if (!retry) {
TraceEvent(SevError, "GetTenantFailure").error(error).detail("TenantName", tenant);
return Void();
}
}
}
}
ACTOR Future<Void> listTenants(Database cx, TenantManagementWorkload* self) {
state TenantName beginTenant = self->chooseTenantName(false);
state TenantName endTenant = self->chooseTenantName(false);
state int limit = std::min(CLIENT_KNOBS->TOO_MANY, deterministicRandom()->randomInt(1, self->maxTenants * 2));
state OperationType operationType = TenantManagementWorkload::randomOperationType();
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
if (beginTenant > endTenant) {
std::swap(beginTenant, endTenant);
}
loop {
try {
state std::map<TenantName, TenantMapEntry> tenants;
if (operationType == OperationType::SPECIAL_KEYS) {
KeyRange range = KeyRangeRef(beginTenant, endTenant).withPrefix(self->specialKeysTenantMapPrefix);
RangeResult results = wait(tr->getRange(range, limit));
for (auto result : results) {
tenants[result.key.removePrefix(self->specialKeysTenantMapPrefix)] =
TenantManagementWorkload::jsonToTenantMapEntry(result.value);
}
} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
std::map<TenantName, TenantMapEntry> _tenants =
wait(ManagementAPI::listTenants(cx.getReference(), beginTenant, endTenant, limit));
tenants = _tenants;
} else {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
std::map<TenantName, TenantMapEntry> _tenants =
wait(ManagementAPI::listTenantsTransaction(tr, beginTenant, endTenant, limit));
tenants = _tenants;
}
ASSERT(tenants.size() <= limit);
auto localItr = self->createdTenants.lower_bound(beginTenant);
auto tenantMapItr = tenants.begin();
for (; tenantMapItr != tenants.end(); ++tenantMapItr, ++localItr) {
ASSERT(localItr != self->createdTenants.end());
ASSERT(localItr->first == tenantMapItr->first);
}
if (!(tenants.size() == limit || localItr == self->createdTenants.end())) {
for (auto tenant : self->createdTenants) {
TraceEvent("ExistingTenant").detail("Tenant", tenant.first);
}
}
ASSERT(tenants.size() == limit || localItr == self->createdTenants.end() ||
localItr->first >= endTenant);
return Void();
} catch (Error& e) {
state bool retry = true;
state Error error = e;
if (operationType != OperationType::MANAGEMENT_DATABASE) {
try {
wait(tr->onError(e));
} catch (Error& e) {
error = e;
retry = false;
}
}
if (!retry) {
TraceEvent(SevError, "ListTenantFailure")
.error(error)
.detail("BeginTenant", beginTenant)
.detail("EndTenant", endTenant);
return Void();
}
}
}
}
Future<Void> start(Database const& cx) override { return _start(cx, this); }
ACTOR Future<Void> _start(Database cx, TenantManagementWorkload* self) {
state double start = now();
while (now() < start + self->testDuration) {
state int operation = deterministicRandom()->randomInt(0, 4);
if (operation == 0) {
wait(self->createTenant(cx, self));
} else if (operation == 1) {
wait(self->deleteTenant(cx, self));
} else if (operation == 2) {
wait(self->getTenant(cx, self));
} else {
wait(self->listTenants(cx, self));
}
}
return Void();
}
Future<bool> check(Database const& cx) override { return _check(cx, this); }
ACTOR Future<bool> _check(Database cx, TenantManagementWorkload* self) {
state Transaction tr(cx);
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(tr.get(self->keyName));
ASSERT(val.present() && val.get() == self->noTenantValue);
break;
} catch (Error& e) {
wait(tr.onError(e));
}
}
state std::map<TenantName, TenantState>::iterator itr = self->createdTenants.begin();
state std::vector<Future<Void>> checkTenants;
state TenantName beginTenant = ""_sr.withPrefix(self->localTenantNamePrefix);
state TenantName endTenant = "\xff\xff"_sr.withPrefix(self->localTenantNamePrefix);
loop {
std::map<TenantName, TenantMapEntry> tenants =
wait(ManagementAPI::listTenants(cx.getReference(), beginTenant, endTenant, 1000));
TenantNameRef lastTenant;
for (auto tenant : tenants) {
ASSERT(itr != self->createdTenants.end());
ASSERT(tenant.first == itr->first);
checkTenants.push_back(self->checkTenant(cx, self, tenant.first, itr->second));
lastTenant = tenant.first;
++itr;
}
if (tenants.size() < 1000) {
break;
} else {
beginTenant = keyAfter(lastTenant);
}
}
ASSERT(itr == self->createdTenants.end());
wait(waitForAll(checkTenants));
return true;
}
void getMetrics(std::vector<PerfMetric>& m) override {}
};
WorkloadFactory<TenantManagementWorkload> TenantManagementWorkload("TenantManagement");

3
fdbserver/workloads/UDPWorkload.actor.cpp
View File

@ -27,7 +27,6 @@
#include "flow/IRandom.h"
#include "flow/flow.h"
#include "flow/network.h"
#include "flow/actorcompiler.h" // has to be last include
#include "flow/serialize.h"
#include <functional>
#include <limits>
@ -36,6 +35,8 @@
#include <memory>
#include <functional>
#include "flow/actorcompiler.h" // has to be last include
namespace {
struct UDPWorkload : TestWorkload {

20
fdbserver/workloads/VersionStamp.actor.cpp
View File

@ -42,6 +42,7 @@ struct VersionStampWorkload : TestWorkload {
std::map<Key, std::vector<std::pair<Version, Standalone<StringRef>>>> versionStampKey_commit;
int apiVersion;
bool soleOwnerOfMetadataVersionKey;
bool allowMetadataVersionKey;
VersionStampWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0);
@ -74,6 +75,9 @@ struct VersionStampWorkload : TestWorkload {
apiVersion = Database::API_VERSION_LATEST;
}
TraceEvent("VersionStampApiVersion").detail("ApiVersion", apiVersion);
allowMetadataVersionKey = apiVersion >= 610 || apiVersion == Database::API_VERSION_LATEST;
cx->apiVersion = apiVersion;
if (clientId == 0)
return _start(cx, this, 1 / transactionsPerSecond);
@ -81,7 +85,7 @@ struct VersionStampWorkload : TestWorkload {
}
Key keyForIndex(uint64_t index) {
if ((apiVersion >= 610 || apiVersion == Database::API_VERSION_LATEST) && index == 0) {
if (allowMetadataVersionKey && index == 0) {
return metadataVersionKey;
}
@ -191,8 +195,7 @@ struct VersionStampWorkload : TestWorkload {
RangeResult result_ = wait(tr.getRange(
KeyRangeRef(self->vsValuePrefix, endOfRange(self->vsValuePrefix)), self->nodeCount + 1));
result = result_;
if ((self->apiVersion >= 610 || self->apiVersion == Database::API_VERSION_LATEST) &&
self->key_commit.count(metadataVersionKey)) {
if (self->allowMetadataVersionKey && self->key_commit.count(metadataVersionKey)) {
Optional<Value> mVal = wait(tr.get(metadataVersionKey));
if (mVal.present()) {
result.push_back_deep(result.arena(), KeyValueRef(metadataVersionKey, mVal.get()));
@ -314,6 +317,7 @@ struct VersionStampWorkload : TestWorkload {
extraDB = Database::createDatabase(extraFile, -1);
}
state Future<Void> metadataWatch = Void();
loop {
wait(poisson(&lastTime, delay));
bool oldVSFormat = !cx->apiVersionAtLeast(520);
@ -350,6 +354,7 @@ struct VersionStampWorkload : TestWorkload {
state Error err;
//TraceEvent("VST_CommitBegin").detail("Key", printable(key)).detail("VsKey", printable(versionStampKey)).detail("Clear", printable(range));
state Key testKey;
state Future<Void> nextMetadataWatch;
try {
tr.atomicOp(key, versionStampValue, MutationRef::SetVersionstampedValue);
if (key == metadataVersionKey) {
@ -358,12 +363,21 @@ struct VersionStampWorkload : TestWorkload {
}
tr.clear(range);
tr.atomicOp(versionStampKey, value, MutationRef::SetVersionstampedKey);
if (key == metadataVersionKey) {
nextMetadataWatch = tr.watch(versionStampKey);
}
state Future<Standalone<StringRef>> fTrVs = tr.getVersionstamp();
wait(tr.commit());
committedVersion = tr.getCommittedVersion();
Standalone<StringRef> committedVersionStamp_ = wait(fTrVs);
committedVersionStamp = committedVersionStamp_;
if (key == metadataVersionKey) {
wait(timeoutError(metadataWatch, 30));
nextMetadataWatch = metadataWatch;
}
} catch (Error& e) {
err = e;
if (err.code() == error_code_database_locked && g_simulator.extraDB != nullptr) {

35
fdbserver/workloads/WriteDuringRead.actor.cpp
View File

@ -583,9 +583,29 @@ struct WriteDuringReadWorkload : TestWorkload {
std::string(deterministicRandom()->randomInt(valueSizeRange.first, valueSizeRange.second + 1), 'x'));
}
// Prevent a write only transaction whose commit was previously cancelled from being reordered after this
// transaction
ACTOR Future<Void> writeBarrier(Database cx) {
state Transaction tr(cx);
loop {
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
// Write-only transactions have a self-conflict in the system keys
tr.addWriteConflictRange(allKeys);
wait(tr.commit());
return Void();
} catch (Error& e) {
wait(tr.onError(e));
}
}
}
ACTOR Future<Void> loadAndRun(Database cx, WriteDuringReadWorkload* self) {
state double startTime = now();
loop {
wait(self->writeBarrier(cx));
state int i = 0;
state int keysPerBatch =
std::min<int64_t>(1000,
@ -595,19 +615,16 @@ struct WriteDuringReadWorkload : TestWorkload {
for (; i < self->nodes; i += keysPerBatch) {
state Transaction tr(cx);
loop {
if (now() - startTime > self->testDuration)
return Void();
try {
if (i == 0) {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.addWriteConflictRange(
allKeys); // To prevent a write only transaction whose commit was previously cancelled
// from being reordered after this transaction
tr.clear(normalKeys);
}
if (now() - startTime > self->testDuration)
return Void();
if (self->useSystemKeys)
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (i == 0) {
tr.clear(normalKeys);
}
int end = std::min(self->nodes, i + keysPerBatch);
tr.clear(KeyRangeRef(self->getKeyForIndex(i), self->getKeyForIndex(end)));
self->memoryDatabase.erase(self->memoryDatabase.lower_bound(self->getKeyForIndex(i)),

8
fdbserver/workloads/workloads.actor.h
View File

@ -27,6 +27,7 @@
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/DatabaseContext.h" // for clone()
#include "fdbserver/KnobProtectiveGroups.h"
#include "fdbserver/TesterInterface.actor.h"
#include "fdbrpc/simulator.h"
#include "flow/actorcompiler.h"
@ -205,9 +206,14 @@ public:
ISimulator::BackupAgentType simBackupAgents; // If set to true, then the simulation runs backup agents on the
// workers. Can only be used in simulation.
ISimulator::BackupAgentType simDrAgents;
KnobKeyValuePairs overrideKnobs;
};
ACTOR Future<DistributedTestResults> runWorkload(Database cx, std::vector<TesterInterface> testers, TestSpec spec);
ACTOR Future<DistributedTestResults> runWorkload(Database cx,
std::vector<TesterInterface> testers,
TestSpec spec,
Optional<TenantName> defaultTenant);
void logMetrics(std::vector<PerfMetric> metrics);

12
flow/Deque.h
View File

@ -44,7 +44,9 @@ public:
Deque(Deque const& r) : arr(nullptr), begin(0), end(r.size()), mask(r.mask) {
if (r.capacity() > 0) {
arr = (T*)aligned_alloc(std::max(__alignof(T), sizeof(void*)), capacity() * sizeof(T));
ASSERT(arr != nullptr);
if (arr == nullptr) {
platform::outOfMemory();
}
}
ASSERT(capacity() >= end || end == 0);
if (r.end < r.capacity()) {
@ -67,7 +69,9 @@ public:
mask = r.mask;
if (r.capacity() > 0) {
arr = (T*)aligned_alloc(std::max(__alignof(T), sizeof(void*)), capacity() * sizeof(T));
ASSERT(arr != nullptr);
if (arr == nullptr) {
platform::outOfMemory();
}
}
ASSERT(capacity() >= end || end == 0);
if (r.end < r.capacity()) {
@ -193,7 +197,9 @@ private:
// printf("Growing to %lld (%u-%u mask %u)\n", (long long)newSize, begin, end, mask);
T* newArr = (T*)aligned_alloc(std::max(__alignof(T), sizeof(void*)),
newSize * sizeof(T)); // SOMEDAY: FastAllocator
ASSERT(newArr != nullptr);
if (newArr == nullptr) {
platform::outOfMemory();
}
for (int i = begin; i != end; i++) {
try {
new (&newArr[i - begin]) T(std::move_if_noexcept(arr[i & mask]));

20
flow/FastAlloc.cpp
View File

@ -593,15 +593,25 @@ template class FastAllocator<16384>;
#ifdef USE_JEMALLOC
#include <jemalloc/jemalloc.h>
TEST_CASE("/jemalloc/4k_aligned_usable_size") {
for (int i = 1; i < 4; ++i) {
auto* ptr = aligned_alloc(4096, i * 4096);
try {
void* ptr;
try {
// Check that we can allocate 4k aligned up to 16k with no internal
// fragmentation
for (int i = 1; i < 4; ++i) {
ptr = aligned_alloc(4096, i * 4096);
ASSERT_EQ(malloc_usable_size(ptr), i * 4096);
} catch (...) {
aligned_free(ptr);
throw;
ptr = nullptr;
}
// Also check that we can allocate magazines with no internal
// fragmentation, should we decide to do that.
ptr = aligned_alloc(4096, kFastAllocMagazineBytes);
ASSERT_EQ(malloc_usable_size(ptr), kFastAllocMagazineBytes);
aligned_free(ptr);
ptr = nullptr;
} catch (...) {
aligned_free(ptr);
throw;
}
return Void();
}

10
flow/FastAlloc.h
View File

@ -103,6 +103,8 @@ void recordAllocation(void* ptr, size_t size);
void recordDeallocation(void* ptr);
#endif
inline constexpr auto kFastAllocMagazineBytes = 128 << 10;
template <int Size>
class FastAllocator {
public:
@ -125,7 +127,7 @@ private:
static unsigned long vLock;
#endif
static const int magazine_size = (128 << 10) / Size;
static const int magazine_size = kFastAllocMagazineBytes / Size;
static const int PSize = Size / sizeof(void*);
struct GlobalData;
struct ThreadData {
@ -287,7 +289,11 @@ inline void freeFast(int size, void* ptr) {
if (size <= 16384)
return FastAllocator<16384>::allocate();
#endif
return aligned_alloc(4096, size);
auto* result = aligned_alloc(4096, size);
if (result == nullptr) {
platform::outOfMemory();
}
return result;
}
inline void freeFast4kAligned(int size, void* ptr) {

20
flow/Knobs.cpp
View File

@ -346,6 +346,26 @@ bool Knobs::setKnob(std::string const& knob, std::string const& value) {
return true;
}
ParsedKnobValue Knobs::getKnob(const std::string& name) const {
if (double_knobs.count(name) > 0) {
return ParsedKnobValue{ *double_knobs.at(name).value };
}
if (int64_knobs.count(name) > 0) {
return ParsedKnobValue{ *int64_knobs.at(name).value };
}
if (int_knobs.count(name) > 0) {
return ParsedKnobValue{ *int_knobs.at(name).value };
}
if (string_knobs.count(name) > 0) {
return ParsedKnobValue{ *string_knobs.at(name).value };
}
if (bool_knobs.count(name) > 0) {
return ParsedKnobValue{ *bool_knobs.at(name).value };
}
return ParsedKnobValue{ NoKnobFound() };
}
bool Knobs::isAtomic(std::string const& knob) const {
if (double_knobs.count(knob)) {
return double_knobs.find(knob)->second.atomic == Atomic::YES;

13
flow/Knobs.h
View File

@ -76,11 +76,24 @@ protected:
std::set<std::string> explicitlySetKnobs;
public:
// Sets an integer value to an integer knob, returns false if the knob does not exist or type mismatch
bool setKnob(std::string const& name, int value);
// Sets a boolean value to a bool knob, returns false if the knob does not exist or type mismatch
bool setKnob(std::string const& name, bool value);
// Sets an int64_t value to an int64_t knob, returns false if the knob does not exist or type mismatch
bool setKnob(std::string const& name, int64_t value);
// Sets a double value to a double knob, returns false if the knob does not exist or type mismatch
bool setKnob(std::string const& name, double value);
// Sets a string value to a string knob, returns false if the knob does not exist or type mismatch
bool setKnob(std::string const& name, std::string const& value);
// Gets the value of knob
ParsedKnobValue getKnob(const std::string& name) const;
ParsedKnobValue parseKnobValue(std::string const& name, std::string const& value) const;
bool isAtomic(std::string const& knob) const;
void trace() const;

2
tests/CMakeLists.txt
View File

@ -300,6 +300,8 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES slow/SwizzledDdBalance.toml)
add_fdb_test(TEST_FILES slow/SwizzledRollbackTimeLapse.toml)
add_fdb_test(TEST_FILES slow/SwizzledRollbackTimeLapseIncrement.toml)
add_fdb_test(TEST_FILES slow/SwizzledTenantManagement.toml)
add_fdb_test(TEST_FILES slow/TenantManagement.toml)
add_fdb_test(TEST_FILES slow/VersionStampBackupToDB.toml)
add_fdb_test(TEST_FILES slow/VersionStampSwitchover.toml)
add_fdb_test(TEST_FILES slow/WriteDuringReadAtomicRestore.toml)

2
tests/TestRunner/local_cluster.py
View File

@ -136,5 +136,5 @@ logdir = {logdir}
def create_database(self, storage='ssd'):
args = [self.fdbcli_binary, '-C', self.etc.joinpath('fdb.cluster'), '--exec',
'configure new single {}'.format(storage)]
'configure new single {} tenant_mode=optional_experimental'.format(storage)]
subprocess.run(args)

3
tests/fast/ChangeFeeds.toml
View File

@ -1,3 +1,6 @@
[configuration]
allowDefaultTenant = false
[[test]]
testTitle = 'ChangeFeed'

2
tests/fast/FuzzApiCorrectness.toml
View File

@ -1,5 +1,7 @@
[configuration]
StderrSeverity = 30
allowDisablingTenants = false
allowDefaultTenant = false
[[test]]
testTitle = 'FuzzApiCorrectness'

2
tests/fast/FuzzApiCorrectnessClean.toml
View File

@ -1,5 +1,7 @@
[configuration]
StderrSeverity = 30
allowDisablingTenants = false
allowDefaultTenant = false
[[test]]
testTitle = 'FuzzApiCorrectness'

46
tests/slow/SwizzledTenantManagement.toml Normal file
View File

@ -0,0 +1,46 @@
[configuration]
allowDisablingTenants = false
[[test]]
testTitle = 'TenantManagementTest'
clearAfterTest = true
timeout = 2100
runSetup = true
[[test.workload]]
testName = 'TenantManagement'
maxTenants = 1000
testDuration = 60
[[test.workload]]
testName = 'RandomClogging'
testDuration = 120.0
swizzle = 1
[[test.workload]]
testName = 'Rollback'
testDuration = 120.0
meanDelay = 10.0
[[test.workload]]
testName = 'Attrition'
testDuration = 120.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 120.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 120.0
[[test.workload]]
testName = 'ChangeConfig'
maxDelayBeforeChange = 120.0
coordinators = 'auto'

13
tests/slow/TenantManagement.toml Normal file
View File

@ -0,0 +1,13 @@
[configuration]
allowDisablingTenants = false
[[test]]
testTitle = 'TenantManagementTest'
clearAfterTest = true
timeout = 2100
runSetup = true
[[test.workload]]
testName = 'TenantManagement'
maxTenants = 1000
testDuration = 60

1
tests/slow/WriteDuringReadAtomicRestore.toml
View File

@ -1,5 +1,6 @@
[configuration]
StderrSeverity = 30
allowDefaultTenant = false
[[test]]
testTitle = 'WriteDuringReadTest'