wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9346 from sfc-gh-nwijetunga/nim/global-tenant-ids 

Support for Two Byte Prefix for Tenant IDs
This commit is contained in:
A.J. Beamon 2023-02-11 11:31:24 -08:00 committed by GitHub
parent b32eebac30 640f1afd77
commit ee1b48323d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 377 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
fdbcli/MetaclusterCommands.actor.cpp
View File

@ -23,6 +23,7 @@
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/IClientApi.h"
#include "fdbclient/Knobs.h"
#include "fdbclient/Metacluster.h"
#include "fdbclient/MetaclusterManagement.actor.h"
#include "fdbclient/Schemas.h"
@ -30,6 +31,7 @@
#include "flow/FastRef.h"
#include "flow/ThreadHelper.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include <string>
namespace fdb_cli {
@ -83,14 +85,23 @@ void printMetaclusterConfigureOptionsUsage() {
// metacluster create command
ACTOR Future<bool> metaclusterCreateCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
if (tokens.size() != 3) {
fmt::print("Usage: metacluster create_experimental <NAME>\n\n");
if (tokens.size() != 4) {
fmt::print("Usage: metacluster create_experimental <NAME> <TENANT_ID_PREFIX>\n\n");
fmt::print("Configures the cluster to be a management cluster in a metacluster.\n");
fmt::print("NAME is an identifier used to distinguish this metacluster from other metaclusters.\n");
fmt::print("TENANT_ID_PREFIX is an integer in the range [0,32767] inclusive which is prepended to all tenant "
"ids in the metacluster.\n");
return false;
}
Optional<std::string> errorStr = wait(MetaclusterAPI::createMetacluster(db, tokens[2]));
int64_t tenantIdPrefix = std::stoi(tokens[3].toString());
if (tenantIdPrefix < TenantAPI::TENANT_ID_PREFIX_MIN_VALUE ||
tenantIdPrefix > TenantAPI::TENANT_ID_PREFIX_MAX_VALUE) {
fmt::print("TENANT_ID_PREFIX must be in the range [0,32767] inclusive\n");
return false;
}
Optional<std::string> errorStr = wait(MetaclusterAPI::createMetacluster(db, tokens[2], tenantIdPrefix));
if (errorStr.present()) {
fmt::print("ERROR: {}.\n", errorStr.get());
} else {
@ -420,7 +431,7 @@ std::vector<const char*> metaclusterHintGenerator(std::vector<StringRef> const&
if (tokens.size() == 1) {
return { "<create_experimental|decommission|register|remove|configure|list|get|status>", "[ARGS]" };
} else if (tokencmp(tokens[1], "create_experimental")) {
return { "<NAME>" };
return { "<NAME> <TENANT_ID_PREFIX>" };
} else if (tokencmp(tokens[1], "decommission")) {
return {};
} else if (tokencmp(tokens[1], "register") && tokens.size() < 5) {

8
fdbcli/tests/metacluster_fdbcli_tests.py
View File

@ -2,6 +2,7 @@
import argparse
import os
import subprocess
import random
from argparse import RawDescriptionHelpFormatter
@ -34,8 +35,8 @@ def get_cluster_connection_str(cluster_file_path):
return conn_str
def metacluster_create(cluster_file, name):
return run_fdbcli_command(cluster_file, "metacluster create_experimental", name)
def metacluster_create(cluster_file, name, tenant_id_prefix):
return run_fdbcli_command(cluster_file, "metacluster create_experimental", name, str(tenant_id_prefix))
def metacluster_register(management_cluster_file, data_cluster_file, name):
@ -75,7 +76,8 @@ if __name__ == "__main__":
names = ['meta_mgmt']
names.extend(['data{}'.format(i) for i in range(1, num_clusters)])
metacluster_create(cluster_files[0], names[0])
tenant_id_prefix = random.randint(0, 32767)
metacluster_create(cluster_files[0], names[0], tenant_id_prefix)
for (cf, name) in zip(cluster_files[1:], names[1:]):
output = metacluster_register(cluster_files[0], cf, name)

5
fdbclient/Tenant.cpp
View File

@ -196,6 +196,11 @@ Key TenantMetadata::tenantMapPrivatePrefix() {
return _prefix;
}
KeyBackedProperty<int64_t>& TenantMetadata::tenantIdPrefix() {
static KeyBackedProperty<int64_t> instance(TenantMetadata::instance().subspace.withSuffix("idPrefix"_sr));
return instance;
}
TEST_CASE("/fdbclient/libb64/base64decoder") {
Standalone<StringRef> buf = makeString(100);
for (int i = 0; i < 1000; ++i) {

27
fdbclient/TenantManagement.actor.cpp
View File

@ -24,6 +24,7 @@
#include "fdbclient/SystemData.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/Tuple.h"
#include "flow/Trace.h"
#include "flow/actorcompiler.h" // has to be last include
namespace TenantAPI {
@ -66,4 +67,30 @@ int64_t extractTenantIdFromKeyRef(StringRef s) {
return TenantAPI::prefixToId(prefix, EnforceValidTenantId::False);
}
// validates whether the lastTenantId and the nextTenantId share the same 2 byte prefix
bool nextTenantIdPrefixMatches(int64_t lastTenantId, int64_t nextTenantId) {
if (getTenantIdPrefix(nextTenantId) != getTenantIdPrefix(lastTenantId)) {
TraceEvent(g_network->isSimulated() ? SevWarnAlways : SevError, "TenantIdPrefixMismatch")
.detail("CurrentTenantId", lastTenantId)
.detail("NewTenantId", nextTenantId)
.detail("CurrentTenantIdPrefix", getTenantIdPrefix(lastTenantId))
.detail("NewTenantIdPrefix", getTenantIdPrefix(nextTenantId));
return false;
}
return true;
}
// returns the maximum allowable tenant id in which the 2 byte prefix is not overriden
int64_t getMaxAllowableTenantId(int64_t curTenantId) {
// The maximum tenant id allowed is 1 for the first 48 bits (6 bytes) with the first 16 bits (2 bytes) being the
// tenant prefix
int64_t maxTenantId = curTenantId | 0xFFFFFFFFFFFFLL;
ASSERT(maxTenantId > 0);
return maxTenantId;
}
int64_t getTenantIdPrefix(int64_t tenantId) {
return tenantId >> 48;
}
} // namespace TenantAPI

26
fdbclient/include/fdbclient/MetaclusterManagement.actor.h
View File

@ -451,9 +451,11 @@ Future<Void> managementClusterCheckEmpty(Transaction tr) {
}
ACTOR template <class DB>
Future<Optional<std::string>> createMetacluster(Reference<DB> db, ClusterName name) {
Future<Optional<std::string>> createMetacluster(Reference<DB> db, ClusterName name, int64_t tenantIdPrefix) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
state Optional<UID> metaclusterUid;
ASSERT(tenantIdPrefix >= TenantAPI::TENANT_ID_PREFIX_MIN_VALUE &&
tenantIdPrefix <= TenantAPI::TENANT_ID_PREFIX_MAX_VALUE);
loop {
try {
@ -462,7 +464,7 @@ Future<Optional<std::string>> createMetacluster(Reference<DB> db, ClusterName na
state Future<Optional<MetaclusterRegistrationEntry>> metaclusterRegistrationFuture =
MetaclusterMetadata::metaclusterRegistration().get(tr);
wait(managementClusterCheckEmpty(tr));
state Future<Void> metaclusterEmptinessCheck = managementClusterCheckEmpty(tr);
Optional<MetaclusterRegistrationEntry> existingRegistration = wait(metaclusterRegistrationFuture);
if (existingRegistration.present()) {
@ -477,6 +479,8 @@ Future<Optional<std::string>> createMetacluster(Reference<DB> db, ClusterName na
}
}
wait(metaclusterEmptinessCheck);
if (!metaclusterUid.present()) {
metaclusterUid = deterministicRandom()->randomUniqueID();
}
@ -484,6 +488,8 @@ Future<Optional<std::string>> createMetacluster(Reference<DB> db, ClusterName na
MetaclusterMetadata::metaclusterRegistration().set(
tr, MetaclusterRegistrationEntry(name, metaclusterUid.get()));
TenantMetadata::tenantIdPrefix().set(tr, tenantIdPrefix);
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
break;
} catch (Error& e) {
@ -491,6 +497,8 @@ Future<Optional<std::string>> createMetacluster(Reference<DB> db, ClusterName na
}
}
TraceEvent("CreatedMetacluster").detail("Name", name).detail("Prefix", tenantIdPrefix);
return Optional<std::string>();
}
@ -1266,8 +1274,18 @@ struct CreateTenantImpl {
state Future<Void> setClusterFuture = self->ctx.setCluster(tr, assignment.first);
// Create a tenant entry in the management cluster
Optional<int64_t> lastId = wait(ManagementClusterMetadata::tenantMetadata().lastTenantId.get(tr));
self->tenantEntry.setId(lastId.orDefault(-1) + 1);
state Optional<int64_t> lastId = wait(ManagementClusterMetadata::tenantMetadata().lastTenantId.get(tr));
// If the last tenant id is not present fetch the prefix from system keys and make it the prefix for the next
// allocated tenant id
if (!lastId.present()) {
Optional<int64_t> tenantIdPrefix = wait(TenantMetadata::tenantIdPrefix().get(tr));
ASSERT(tenantIdPrefix.present());
lastId = tenantIdPrefix.get() << 48;
}
if (!TenantAPI::nextTenantIdPrefixMatches(lastId.get(), lastId.get() + 1)) {
throw cluster_no_capacity();
}
self->tenantEntry.setId(lastId.get() + 1);
ManagementClusterMetadata::tenantMetadata().lastTenantId.set(tr, self->tenantEntry.id);
self->tenantEntry.tenantState = TenantState::REGISTERING;

3
fdbclient/include/fdbclient/Tenant.h
View File

@ -231,6 +231,9 @@ struct TenantMetadata {
static inline auto& tenantGroupMap() { return instance().tenantGroupMap; }
static inline auto& storageQuota() { return instance().storageQuota; }
static inline auto& lastTenantModification() { return instance().lastTenantModification; }
// This system keys stores the tenant id prefix that is used during metacluster/standalone cluster creation. If the
// key is not present then we will assume the prefix to be 0
static KeyBackedProperty<int64_t>& tenantIdPrefix();
static Key tenantMapPrivatePrefix();
};

22
fdbclient/include/fdbclient/TenantManagement.actor.h
View File

@ -20,8 +20,11 @@
#pragma once
#include "fdbclient/ClientBooleanParams.h"
#include "fdbclient/Knobs.h"
#include "fdbclient/Tenant.h"
#include "flow/IRandom.h"
#include "flow/ThreadHelper.actor.h"
#include <algorithm>
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_TENANT_MANAGEMENT_ACTOR_G_H)
#define FDBCLIENT_TENANT_MANAGEMENT_ACTOR_G_H
#include "fdbclient/TenantManagement.actor.g.h"
@ -37,6 +40,9 @@
namespace TenantAPI {
static const int TENANT_ID_PREFIX_MIN_VALUE = 0;
static const int TENANT_ID_PREFIX_MAX_VALUE = 32767;
template <class Transaction>
Future<Optional<TenantMapEntry>> tryGetTenantTransaction(Transaction tr, int64_t tenantId) {
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
@ -120,6 +126,9 @@ Future<Void> checkTenantMode(Transaction tr, ClusterType expectedClusterType) {
TenantMode tenantModeForClusterType(ClusterType clusterType, TenantMode tenantMode);
int64_t extractTenantIdFromMutation(MutationRef m);
int64_t extractTenantIdFromKeyRef(StringRef s);
bool nextTenantIdPrefixMatches(int64_t lastTenantId, int64_t nextTenantId);
int64_t getMaxAllowableTenantId(int64_t curTenantId);
int64_t getTenantIdPrefix(int64_t tenantId);
// Returns true if the specified ID has already been deleted and false if not. If the ID is old enough
// that we no longer keep tombstones for it, an error is thrown.
@ -216,11 +225,20 @@ createTenantTransaction(Transaction tr, TenantMapEntry tenantEntry, ClusterType
ACTOR template <class Transaction>
Future<int64_t> getNextTenantId(Transaction tr) {
Optional<int64_t> lastId = wait(TenantMetadata::lastTenantId().get(tr));
int64_t tenantId = lastId.orDefault(-1) + 1;
state Optional<int64_t> lastId = wait(TenantMetadata::lastTenantId().get(tr));
if (!lastId.present()) {
// If the last tenant id is not present fetch the tenantIdPrefix (if any) and initalize the lastId
int64_t tenantIdPrefix = wait(TenantMetadata::tenantIdPrefix().getD(tr, Snapshot::False, 0));
// Shift by 6 bytes to make the prefix the first two bytes of the tenant id
lastId = tenantIdPrefix << 48;
}
int64_t tenantId = lastId.get() + 1;
if (BUGGIFY) {
tenantId += deterministicRandom()->randomSkewedUInt32(1, 1e9);
}
if (!TenantAPI::nextTenantIdPrefixMatches(lastId.get(), tenantId)) {
throw cluster_no_capacity();
}
return tenantId;
}

4
fdbclient/include/fdbclient/TenantSpecialKeys.actor.h
View File

@ -134,9 +134,13 @@ private:
TenantMetadata::tenantCount().getD(&ryw->getTransaction(), Snapshot::False, 0);
int64_t _nextId = wait(TenantAPI::getNextTenantId(&ryw->getTransaction()));
state int64_t nextId = _nextId;
ASSERT(nextId > 0);
state std::vector<Future<bool>> createFutures;
for (auto const& [tenant, config] : tenants) {
if (!TenantAPI::nextTenantIdPrefixMatches(nextId - 1, nextId)) {
throw cluster_no_capacity();
}
createFutures.push_back(createTenant(ryw, tenant, config, nextId++, tenantGroupNetTenantDelta));
}

87
fdbserver/include/fdbserver/workloads/MetaclusterConsistency.actor.h
View File

@ -24,6 +24,8 @@
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantManagement.actor.h"
#include "flow/BooleanParam.h"
#if defined(NO_INTELLISENSE) && !defined(WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_G_H)
#define WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_G_H
@ -60,6 +62,8 @@ private:
int64_t tenantCount;
RangeResult systemTenantSubspaceKeys;
Optional<int64_t> tenantIdPrefix;
Optional<int64_t> lastTenantId;
};
ManagementClusterData managementMetadata;
@ -76,36 +80,41 @@ private:
try {
managementTr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
state typename transaction_future_type<typename DB::TransactionT, RangeResult>::type
systemTenantSubspaceKeysFuture = managementTr->getRange(prefixRange(TenantMetadata::subspace()), 1);
systemTenantSubspaceKeysFuture = managementTr->getRange(prefixRange(TenantMetadata::subspace()), 2);
wait(store(self->managementMetadata.metaclusterRegistration,
MetaclusterMetadata::metaclusterRegistration().get(managementTr)) &&
store(self->managementMetadata.dataClusters,
MetaclusterAPI::listClustersTransaction(
managementTr, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS + 1)) &&
store(self->managementMetadata.clusterCapacityTuples,
MetaclusterAPI::ManagementClusterMetadata::clusterCapacityIndex.getRange(
managementTr, {}, {}, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
store(self->managementMetadata.clusterTenantCounts,
MetaclusterAPI::ManagementClusterMetadata::clusterTenantCount.getRange(
managementTr, {}, {}, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
store(self->managementMetadata.clusterTenantTuples,
MetaclusterAPI::ManagementClusterMetadata::clusterTenantIndex.getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
store(self->managementMetadata.clusterTenantGroupTuples,
MetaclusterAPI::ManagementClusterMetadata::clusterTenantGroupIndex.getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
store(self->managementMetadata.tenantCount,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantCount.getD(
managementTr, Snapshot::False, 0)) &&
store(tenantList,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantMap.getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
store(self->managementMetadata.tenantGroups,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantGroupMap.getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
store(self->managementMetadata.systemTenantSubspaceKeys,
safeThreadFutureToFuture(systemTenantSubspaceKeysFuture)));
wait(
store(self->managementMetadata.tenantIdPrefix,
TenantMetadata::tenantIdPrefix().get(managementTr)) &&
store(self->managementMetadata.lastTenantId,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().lastTenantId.get(managementTr)) &&
store(self->managementMetadata.metaclusterRegistration,
MetaclusterMetadata::metaclusterRegistration().get(managementTr)) &&
store(self->managementMetadata.dataClusters,
MetaclusterAPI::listClustersTransaction(
managementTr, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS + 1)) &&
store(self->managementMetadata.clusterCapacityTuples,
MetaclusterAPI::ManagementClusterMetadata::clusterCapacityIndex.getRange(
managementTr, {}, {}, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
store(self->managementMetadata.clusterTenantCounts,
MetaclusterAPI::ManagementClusterMetadata::clusterTenantCount.getRange(
managementTr, {}, {}, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
store(self->managementMetadata.clusterTenantTuples,
MetaclusterAPI::ManagementClusterMetadata::clusterTenantIndex.getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
store(self->managementMetadata.clusterTenantGroupTuples,
MetaclusterAPI::ManagementClusterMetadata::clusterTenantGroupIndex.getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
store(self->managementMetadata.tenantCount,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantCount.getD(
managementTr, Snapshot::False, 0)) &&
store(tenantList,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantMap.getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
store(self->managementMetadata.tenantGroups,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantGroupMap.getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
store(self->managementMetadata.systemTenantSubspaceKeys,
safeThreadFutureToFuture(systemTenantSubspaceKeysFuture)));
break;
} catch (Error& e) {
@ -155,6 +164,12 @@ private:
!managementMetadata.tenantGroups.more);
ASSERT_EQ(managementMetadata.clusterTenantGroupTuples.results.size(),
managementMetadata.tenantGroups.results.size());
ASSERT(managementMetadata.tenantIdPrefix.present());
if (managementMetadata.lastTenantId.present()) {
ASSERT(TenantAPI::getTenantIdPrefix(managementMetadata.lastTenantId.get()) ==
managementMetadata.tenantIdPrefix.get());
}
// Parse the cluster capacity index. Check that no cluster is represented in the index more than once.
std::map<ClusterName, int64_t> clusterAllocatedMap;
@ -203,6 +218,7 @@ private:
std::set<TenantGroupName> processedTenantGroups;
for (auto [tenantId, entry] : managementMetadata.tenantMap) {
ASSERT(entry.assignedCluster.present());
ASSERT(TenantAPI::getTenantIdPrefix(tenantId) == managementMetadata.tenantIdPrefix.get());
// Each tenant should be assigned to the same cluster where it is stored in the cluster tenant index
auto clusterItr = managementMetadata.clusterTenantMap.find(entry.assignedCluster.get());
@ -241,8 +257,8 @@ private:
ASSERT(clusterItr->second.count(name));
}
// We should not be storing any data in the `\xff` tenant subspace.
ASSERT(managementMetadata.systemTenantSubspaceKeys.empty());
// The only key in the `\xff` tenant subspace should be the tenant id prefix
ASSERT(managementMetadata.systemTenantSubspaceKeys.size() == 1);
}
ACTOR static Future<Void> validateDataCluster(MetaclusterConsistencyCheck* self,
@ -257,11 +273,13 @@ private:
state TenantConsistencyCheck<IDatabase> tenantConsistencyCheck(dataDb);
wait(tenantConsistencyCheck.run());
state Optional<int64_t> lastTenantId;
loop {
try {
dataTr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
wait(store(dataClusterRegistration, MetaclusterMetadata::metaclusterRegistration().get(dataTr)) &&
wait(store(lastTenantId, TenantMetadata::lastTenantId().get(dataTr)) &&
store(dataClusterRegistration, MetaclusterMetadata::metaclusterRegistration().get(dataTr)) &&
store(dataClusterTenantList,
TenantMetadata::tenantMap().getRange(
dataTr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)) &&
@ -275,6 +293,10 @@ private:
}
}
if (lastTenantId.present()) {
ASSERT(TenantAPI::getTenantIdPrefix(lastTenantId.get()) == self->managementMetadata.tenantIdPrefix.get());
}
state std::map<int64_t, TenantMapEntry> dataClusterTenantMap(dataClusterTenantList.results.begin(),
dataClusterTenantList.results.end());
state std::map<TenantGroupName, TenantGroupEntry> dataClusterTenantGroupMap(
@ -312,6 +334,7 @@ private:
TenantMapEntry const& metaclusterEntry = self->managementMetadata.tenantMap[tenantId];
ASSERT(!entry.assignedCluster.present());
ASSERT_EQ(entry.id, metaclusterEntry.id);
ASSERT(TenantAPI::getTenantIdPrefix(entry.id) == self->managementMetadata.tenantIdPrefix.get());
ASSERT(entry.tenantName == metaclusterEntry.tenantName);
ASSERT_EQ(entry.tenantState, TenantState::READY);

10
fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp
View File

@ -86,12 +86,17 @@ struct MetaclusterManagementWorkload : TestWorkload {
int maxTenants;
int maxTenantGroups;
int64_t tenantIdPrefix;
double testDuration;
MetaclusterManagementWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
maxTenants = std::min<int>(1e8 - 1, getOption(options, "maxTenants"_sr, 1000));
maxTenantGroups = std::min<int>(2 * maxTenants, getOption(options, "maxTenantGroups"_sr, 20));
testDuration = getOption(options, "testDuration"_sr, 120.0);
tenantIdPrefix = getOption(options,
"tenantIdPrefix"_sr,
deterministicRandom()->randomInt(TenantAPI::TENANT_ID_PREFIX_MIN_VALUE,
TenantAPI::TENANT_ID_PREFIX_MAX_VALUE + 1));
}
void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("Attrition"); }
@ -121,8 +126,8 @@ struct MetaclusterManagementWorkload : TestWorkload {
self->dataDbs[self->dataDbIndex.back()] =
DataClusterData(Database::createSimulatedExtraDatabase(connectionString, cx->defaultTenant));
}
wait(success(MetaclusterAPI::createMetacluster(cx.getReference(), "management_cluster"_sr)));
wait(success(
MetaclusterAPI::createMetacluster(cx.getReference(), "management_cluster"_sr, self->tenantIdPrefix)));
return Void();
}
@ -502,6 +507,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
ASSERT(hasCapacity);
ASSERT(entry.assignedCluster.present());
ASSERT(entry.tenantGroup == tenantGroup);
ASSERT(TenantAPI::getTenantIdPrefix(entry.id) == self->tenantIdPrefix);
if (tenantGroup.present()) {
auto tenantGroupData =

188
fdbserver/workloads/TenantCapacityLimits.actor.cpp Normal file
View File

@ -0,0 +1,188 @@
/*
* TenantCapacityLimits.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <cstdint>
#include <limits>
#include "fdbclient/ClusterConnectionMemoryRecord.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/GenericManagementAPI.actor.h"
#include "fdbclient/Metacluster.h"
#include "fdbclient/MetaclusterManagement.actor.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/RunRYWTransaction.actor.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/TenantSpecialKeys.actor.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/Knobs.h"
#include "flow/BooleanParam.h"
#include "flow/Error.h"
#include "flow/IRandom.h"
#include "flow/ThreadHelper.actor.h"
#include "flow/Trace.h"
#include "flow/flow.h"
#include "flow/actorcompiler.h" // This must be the last #include.
struct TenantCapacityLimits : TestWorkload {
static constexpr auto NAME = "TenantCapacityLimits";
Reference<IDatabase> managementDb;
Database dataDb;
int64_t tenantIdPrefix;
bool useMetacluster = false;
const Key specialKeysTenantMapPrefix = SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT)
.begin.withSuffix(TenantRangeImpl::submoduleRange.begin)
.withSuffix(TenantRangeImpl::mapSubRange.begin);
TenantCapacityLimits(WorkloadContext const& wcx) : TestWorkload(wcx) {
tenantIdPrefix = getOption(options,
"tenantIdPrefix"_sr,
deterministicRandom()->randomInt(TenantAPI::TENANT_ID_PREFIX_MIN_VALUE,
TenantAPI::TENANT_ID_PREFIX_MAX_VALUE + 1));
if (clientId == 0) {
useMetacluster = deterministicRandom()->coinflip();
}
}
Future<Void> setup(Database const& cx) override {
if (clientId == 0) {
return _setup(cx, this);
} else {
return Void();
}
}
ACTOR static Future<Void> _setup(Database cx, TenantCapacityLimits* self) {
if (self->useMetacluster) {
Reference<IDatabase> threadSafeHandle =
wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
self->managementDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
wait(success(
MetaclusterAPI::createMetacluster(cx.getReference(), "management_cluster"_sr, self->tenantIdPrefix)));
DataClusterEntry entry;
entry.capacity.numTenantGroups = 1e9;
wait(MetaclusterAPI::registerCluster(
self->managementDb, "test_data_cluster"_sr, g_simulator->extraDatabases[0], entry));
ASSERT(g_simulator->extraDatabases.size() == 1);
self->dataDb = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], cx->defaultTenant);
// wait for tenant mode change on dataDB
wait(success(self->waitDataDbTenantModeChange()));
} else {
self->dataDb = cx;
}
return Void();
}
Future<Optional<Key>> waitDataDbTenantModeChange() const {
return runRYWTransaction(dataDb, [](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
return tr->get("\xff"_sr); // just a meaningless read
});
}
Future<Void> start(Database const& cx) override {
if (clientId == 0) {
return _start(cx, this);
} else {
return Void();
}
}
ACTOR static Future<Void> _start(Database cx, TenantCapacityLimits* self) {
if (self->useMetacluster) {
// Set the max tenant id for the metacluster
state Reference<ITransaction> tr = self->managementDb->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
int64_t maxTenantId = TenantAPI::getMaxAllowableTenantId(self->tenantIdPrefix << 48);
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().lastTenantId.set(tr, maxTenantId);
wait(safeThreadFutureToFuture(tr->commit()));
break;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
// Attempt to create a tenant on the metacluster which should fail since the cluster is at capacity
try {
TenantMapEntry entry;
entry.tenantName = "test_tenant_metacluster"_sr;
wait(MetaclusterAPI::createTenant(self->managementDb, entry, AssignClusterAutomatically::True));
ASSERT(false);
} catch (Error& e) {
ASSERT(e.code() == error_code_cluster_no_capacity);
}
} else {
// set the max tenant id for the standalone cluster
state Reference<ReadYourWritesTransaction> dataTr = makeReference<ReadYourWritesTransaction>(self->dataDb);
loop {
try {
dataTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
int64_t maxTenantId = TenantAPI::getMaxAllowableTenantId(0);
TenantMetadata::lastTenantId().set(dataTr, maxTenantId);
wait(dataTr->commit());
break;
} catch (Error& e) {
wait(dataTr->onError(e));
}
}
// Use the management database api to create a tenant which should fail since the cluster is at capacity
try {
wait(success(TenantAPI::createTenant(self->dataDb.getReference(), "test_tenant_management_api"_sr)));
ASSERT(false);
} catch (Error& e) {
ASSERT(e.code() == error_code_cluster_no_capacity);
}
// use special keys to create a tenant which should fail since the cluster is at capacity
loop {
try {
dataTr->reset();
dataTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
dataTr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
TenantMapEntry entry;
dataTr->set(self->specialKeysTenantMapPrefix.withSuffix("test_tenant_special_keys"_sr), ""_sr);
wait(dataTr->commit());
ASSERT(false);
} catch (Error& e) {
if (e.code() == error_code_cluster_no_capacity) {
break;
}
wait(dataTr->onError(e));
}
}
}
return Void();
}
Future<bool> check(Database const& cx) override { return true; }
void getMetrics(std::vector<PerfMetric>& m) override {}
};
WorkloadFactory<TenantCapacityLimits> TenantCapacityLimitsFactory;

6
fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp
View File

@ -101,7 +101,11 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
self->mvDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
if (self->useMetacluster && self->clientId == 0) {
wait(success(MetaclusterAPI::createMetacluster(cx.getReference(), "management_cluster"_sr)));
wait(success(MetaclusterAPI::createMetacluster(
cx.getReference(),
"management_cluster"_sr,
deterministicRandom()->randomInt(TenantAPI::TENANT_ID_PREFIX_MIN_VALUE,
TenantAPI::TENANT_ID_PREFIX_MAX_VALUE + 1))));
DataClusterEntry entry;
entry.capacity.numTenantGroups = 1e9;

6
fdbserver/workloads/TenantManagementWorkload.actor.cpp
View File

@ -103,6 +103,7 @@ struct TenantManagementWorkload : TestWorkload {
Reference<IDatabase> mvDb;
Database dataDb;
bool hasNoTenantKey = false; // whether this workload has non-tenant key
int64_t tenantIdPrefix = 0;
// This test exercises multiple different ways to work with tenants
enum class OperationType {
@ -236,7 +237,8 @@ struct TenantManagementWorkload : TestWorkload {
if (self->useMetacluster) {
fmt::print("Create metacluster and register data cluster ... \n");
// Configure the metacluster (this changes the tenant mode)
wait(success(MetaclusterAPI::createMetacluster(cx.getReference(), "management_cluster"_sr)));
wait(success(
MetaclusterAPI::createMetacluster(cx.getReference(), "management_cluster"_sr, self->tenantIdPrefix)));
DataClusterEntry entry;
entry.capacity.numTenantGroups = 1e9;
@ -549,6 +551,7 @@ struct TenantManagementWorkload : TestWorkload {
ASSERT(entry.present());
ASSERT(entry.get().id > self->maxId);
ASSERT(TenantAPI::getTenantIdPrefix(entry.get().id) == self->tenantIdPrefix);
ASSERT(entry.get().tenantGroup == tenantItr->second.tenantGroup);
ASSERT(entry.get().tenantState == TenantState::READY);
@ -561,6 +564,7 @@ struct TenantManagementWorkload : TestWorkload {
wait(TenantAPI::tryGetTenant(self->dataDb.getReference(), tenantItr->first));
ASSERT(dataEntry.present());
ASSERT(dataEntry.get().id == entry.get().id);
ASSERT(TenantAPI::getTenantIdPrefix(dataEntry.get().id) == self->tenantIdPrefix);
ASSERT(dataEntry.get().tenantGroup == entry.get().tenantGroup);
ASSERT(dataEntry.get().tenantState == TenantState::READY);
}

1
tests/CMakeLists.txt
View File

@ -417,6 +417,7 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES slow/SwizzledRollbackTimeLapseIncrement.toml)
add_fdb_test(TEST_FILES slow/SwizzledTenantManagement.toml)
add_fdb_test(TEST_FILES slow/SwizzledTenantManagementMetacluster.toml)
add_fdb_test(TEST_FILES slow/TenantCapacityLimits.toml)
add_fdb_test(TEST_FILES slow/TenantManagement.toml)
add_fdb_test(TEST_FILES slow/TenantManagementConcurrency.toml)
add_fdb_test(TEST_FILES slow/VersionStampBackupToDB.toml)

14
tests/slow/TenantCapacityLimits.toml Normal file
View File

@ -0,0 +1,14 @@
[configuration]
allowDefaultTenant = false
tenantModes = ['optional', 'required']
allowCreatingTenants = false
extraDatabaseMode = 'Single'
[[test]]
testTitle = 'TenantCapacityLimitTest'
clearAfterTest = true
timeout = 2100
runSetup = true
[[test.workload]]
testName = 'TenantCapacityLimits'