wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add first implementation of tenant creation and deletion in a metacluster

This commit is contained in:
A.J. Beamon 2022-05-20 15:21:21 -07:00
parent 69261f9f10
commit d784173f7f
18 changed files with 511 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
fdbcli/TenantCommands.actor.cpp
View File

@ -24,6 +24,7 @@
#include "fdbclient/IClientApi.h"
#include "fdbclient/Knobs.h"
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbclient/MetaclusterManagement.actor.h"
#include "fdbclient/Schemas.h"
#include "flow/Arena.h"
@ -89,7 +90,12 @@ ACTOR Future<bool> createTenantCommandActor(Reference<IDatabase> db, std::vector
loop {
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
try {
state ThreadFuture<Optional<Value>> tenantModeFuture =
tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr));
if (!doneExistenceCheck) {
// Hold the reference to the standalone's memory
state ThreadFuture<Optional<Value>> existingTenantFuture = tr->get(tenantNameKey);
@ -100,11 +106,17 @@ ACTOR Future<bool> createTenantCommandActor(Reference<IDatabase> db, std::vector
doneExistenceCheck = true;
}
tr->set(tenantNameKey, ValueRef());
if (configuration.second.tenantGroup.present()) {
tr->set(makeConfigKey("tenant_group"_sr, tokens[1]), configuration.second.tenantGroup.get());
Optional<Value> tenantMode = wait(safeThreadFutureToFuture(tenantModeFuture));
if (TenantMode::fromValue(tenantMode.castTo<ValueRef>()) == TenantMode::MANAGEMENT) {
wait(MetaclusterAPI::createTenant(db, tokens[1], configuration.second));
} else {
tr->set(tenantNameKey, ValueRef());
if (configuration.second.tenantGroup.present()) {
tr->set(makeConfigKey("tenant_group"_sr, tokens[1]), configuration.second.tenantGroup.get());
}
wait(safeThreadFutureToFuture(tr->commit()));
}
wait(safeThreadFutureToFuture(tr->commit()));
break;
} catch (Error& e) {
state Error err(e);
@ -141,7 +153,11 @@ ACTOR Future<bool> deleteTenantCommandActor(Reference<IDatabase> db, std::vector
loop {
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
state ThreadFuture<Optional<Value>> tenantModeFuture =
tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr));
if (!doneExistenceCheck) {
// Hold the reference to the standalone's memory
state ThreadFuture<Optional<Value>> existingTenantFuture = tr->get(tenantNameKey);
@ -152,8 +168,13 @@ ACTOR Future<bool> deleteTenantCommandActor(Reference<IDatabase> db, std::vector
doneExistenceCheck = true;
}
tr->clear(tenantNameKey);
wait(safeThreadFutureToFuture(tr->commit()));
Optional<Value> tenantMode = wait(safeThreadFutureToFuture(tenantModeFuture));
if (TenantMode::fromValue(tenantMode.castTo<ValueRef>()) == TenantMode::MANAGEMENT) {
wait(MetaclusterAPI::deleteTenant(db, tokens[1]));
} else {
tr->clear(tenantNameKey);
wait(safeThreadFutureToFuture(tr->commit()));
}
break;
} catch (Error& e) {
state Error err(e);
@ -286,13 +307,22 @@ ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<St
int64_t id;
std::string prefix;
std::string tenantState;
std::string assignedCluster;
std::string tenantGroup;
doc.get("id", id);
doc.get("prefix", prefix);
doc.get("tenant_state", tenantState);
bool hasAssignedCluster = doc.tryGet("assigned_cluster", assignedCluster);
bool hasTenantGroup = doc.tryGet("tenant_group", tenantGroup);
printf(" id: %" PRId64 "\n", id);
printf(" prefix: %s\n", printable(prefix).c_str());
printf(" tenant state: %s\n", printable(tenantState).c_str());
if (hasAssignedCluster) {
printf(" assigned cluster: %s\n", printable(assignedCluster).c_str());
}
if (hasTenantGroup) {
printf(" tenant group: %s\n", printable(tenantGroup).c_str());
}

1
fdbclient/CMakeLists.txt
View File

@ -143,6 +143,7 @@ set(FDBCLIENT_SRCS
TaskBucket.h
Tenant.cpp
Tenant.h
TenantManagement.actor.cpp
TenantManagement.actor.h
TenantSpecialKeys.actor.cpp
TestKnobCollection.cpp

3
fdbclient/DatabaseConfiguration.cpp
View File

@ -637,8 +637,7 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
parse((&type), value);
storageMigrationType = (StorageMigrationType::MigrationType)type;
} else if (ck == LiteralStringRef("tenant_mode")) {
parse((&type), value);
tenantMode = (TenantMode::Mode)type;
tenantMode = TenantMode::fromValue(value);
} else if (ck == LiteralStringRef("proxies")) {
overwriteProxiesCount();
} else if (ck == LiteralStringRef("blob_granules_enabled")) {

16
fdbclient/FDBTypes.h
View File

@ -1375,6 +1375,22 @@ struct TenantMode {
return "";
}
Value toValue() const { return ValueRef(format("%d", (int)mode)); }
static TenantMode fromValue(Optional<ValueRef> val) {
if (!val.present()) {
return DISABLED;
}
// A failed parsing returns 0 (DISABLED)
int num = atoi(val.get().toString().c_str());
if (num < 0 || num >= END) {
return DISABLED;
}
return static_cast<Mode>(num);
}
uint32_t mode;
};
struct GRVCacheSpace {

3
fdbclient/Metacluster.h
View File

@ -41,6 +41,7 @@ struct ClusterUsage {
bool operator==(const ClusterUsage& other) const noexcept { return numTenantGroups == other.numTenantGroups; }
bool operator!=(const ClusterUsage& other) const noexcept { return !(*this == other); }
bool operator<(const ClusterUsage& other) const noexcept { return numTenantGroups < other.numTenantGroups; }
template <class Ar>
void serialize(Ar& ar) {
@ -72,6 +73,8 @@ struct DataClusterEntry {
return id == other.id && capacity == other.capacity;
}
bool hasCapacity() const { return allocated < capacity; }
Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion(ProtocolVersion::withMetacluster())); }
static DataClusterEntry decode(ValueRef const& value) {
DataClusterEntry entry;

210
fdbclient/MetaclusterManagement.actor.h
View File

@ -92,11 +92,19 @@ Future<Optional<DataClusterMetadata>> tryGetClusterTransaction(Transaction tr, C
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantModeFuture =
tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr));
state typename transaction_future_type<Transaction, Optional<Value>>::type metadataFuture =
tr->get(dataClusterMetadataKey);
state typename transaction_future_type<Transaction, Optional<Value>>::type connectionRecordFuture =
tr->get(dataClusterConnectionRecordKey);
Optional<Value> tenantModeVal = wait(safeThreadFutureToFuture(tenantModeFuture));
if (TenantMode::fromValue(tenantModeVal.castTo<ValueRef>()) != TenantMode::MANAGEMENT) {
throw invalid_metacluster_operation();
}
state Optional<Value> metadata = wait(safeThreadFutureToFuture(metadataFuture));
Optional<Value> connectionString = wait(safeThreadFutureToFuture(connectionRecordFuture));
@ -295,13 +303,6 @@ Future<Void> registerCluster(Reference<DB> db,
throw commit_unknown_result();
}
TraceEvent("RegisteredDataCluster")
.detail("ClusterName", name)
.detail("ClusterID", entry.id)
.detail("Capacity", entry.capacity)
.detail("Version", precheckTr->getCommittedVersion())
.detail("ConnectionString", connectionString.toString());
break;
} catch (Error& e) {
wait(safeThreadFutureToFuture(precheckTr->onError(e)));
@ -359,7 +360,7 @@ Future<Void> registerCluster(Reference<DB> db,
throw commit_unknown_result();
}
TraceEvent("FinalizedDataCluster")
TraceEvent("RegisteredDataCluster")
.detail("ClusterName", name)
.detail("ClusterID", entry.id)
.detail("Capacity", entry.capacity)
@ -560,11 +561,19 @@ Future<std::map<ClusterName, DataClusterMetadata>> listClustersTransaction(Trans
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantModeFuture =
tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr));
state typename transaction_future_type<Transaction, RangeResult>::type metadataFuture =
tr->getRange(firstGreaterOrEqual(metadataRange.begin), firstGreaterOrEqual(metadataRange.end), limit);
state typename transaction_future_type<Transaction, RangeResult>::type connectionStringFuture = tr->getRange(
firstGreaterOrEqual(connectionStringRange.begin), firstGreaterOrEqual(connectionStringRange.end), limit);
Optional<Value> tenantModeVal = wait(safeThreadFutureToFuture(tenantModeFuture));
if (TenantMode::fromValue(tenantModeVal.castTo<ValueRef>()) != TenantMode::MANAGEMENT) {
throw invalid_metacluster_operation();
}
state RangeResult metadata = wait(safeThreadFutureToFuture(metadataFuture));
RangeResult connectionStrings = wait(safeThreadFutureToFuture(connectionStringFuture));
@ -598,6 +607,191 @@ Future<std::map<ClusterName, DataClusterMetadata>> listClusters(Reference<DB> db
}
}
}
ACTOR template <class Transaction>
Future<std::pair<ClusterName, DataClusterMetadata>> assignTenant(Transaction tr, TenantMapEntry tenantEntry) {
// TODO: more efficient
// TODO: account for tenant groups
std::map<ClusterName, DataClusterMetadata> clusters =
wait(listClustersTransaction(tr, ""_sr, "\xff"_sr, CLIENT_KNOBS->TOO_MANY));
for (auto c : clusters) {
if (c.second.entry.hasCapacity()) {
// TODO: check that the chosen cluster is available, otherwise we can try another
return c;
}
}
throw metacluster_no_capacity();
}
ACTOR template <class DB>
Future<Void> createTenant(Reference<DB> db, TenantName name, TenantMapEntry tenantEntry) {
state DataClusterMetadata clusterMetadata;
state TenantMapEntry createdTenant;
// Step 1: assign the tenant and record its details in the management cluster
state Reference<typename DB::TransactionT> assignTr = db->createTransaction();
loop {
try {
assignTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
std::pair<ClusterName, DataClusterMetadata> assignment = wait(assignTenant(assignTr, tenantEntry));
tenantEntry.assignedCluster = assignment.first;
clusterMetadata = assignment.second;
std::pair<TenantMapEntry, bool> result = wait(ManagementAPI::createTenantTransaction(
assignTr, name, tenantEntry, ManagementAPI::TenantOperationType::MANAGEMENT_CLUSTER));
createdTenant = result.first;
if (!result.second) {
if (!result.first.matchesConfiguration(tenantEntry)) {
throw tenant_already_exists();
} else if (tenantEntry.assignedCluster != createdTenant.assignedCluster) {
if (!result.first.assignedCluster.present()) {
// This is an unexpected state in a metacluster, but if it happens then it wasn't created here
throw tenant_already_exists();
}
Optional<DataClusterMetadata> actualMetadata =
wait(tryGetClusterTransaction(assignTr, createdTenant.assignedCluster.get()));
// TODO: move the tenant to an error state?
ASSERT(actualMetadata.present());
clusterMetadata = actualMetadata.get();
}
} else {
// TODO: this should actually track the groups, not the tenants
++clusterMetadata.entry.allocated.numTenantGroups;
updateClusterMetadata(assignTr,
createdTenant.assignedCluster.get(),
Optional<ClusterConnectionString>(),
clusterMetadata.entry);
}
wait(safeThreadFutureToFuture(assignTr->commit()));
break;
} catch (Error& e) {
wait(safeThreadFutureToFuture(assignTr->onError(e)));
}
}
// Step 2: store the tenant info in the data cluster
state Reference<IDatabase> dataClusterDb = MultiVersionApi::api->createDatabase(
makeReference<ClusterConnectionMemoryRecord>(clusterMetadata.connectionString));
createdTenant.tenantState = TenantState::READY;
TenantMapEntry _ = wait(ManagementAPI::createTenant(
dataClusterDb, name, createdTenant, ManagementAPI::TenantOperationType::DATA_CLUSTER));
// Step 3: mark the tenant as ready in the management cluster
state Reference<typename DB::TransactionT> finalizeTr = db->createTransaction();
loop {
try {
finalizeTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
Optional<TenantMapEntry> managementEntry = wait(ManagementAPI::tryGetTenantTransaction(finalizeTr, name));
if (!managementEntry.present()) {
throw tenant_removed();
} else if (managementEntry.get().id != createdTenant.id) {
throw tenant_already_exists();
}
if (managementEntry.get().tenantState == TenantState::REGISTERING) {
TenantMapEntry updatedEntry = managementEntry.get();
updatedEntry.tenantState = TenantState::READY;
ManagementAPI::configureTenantTransaction(finalizeTr, name, updatedEntry);
wait(safeThreadFutureToFuture(finalizeTr->commit()));
}
break;
} catch (Error& e) {
wait(safeThreadFutureToFuture(finalizeTr->onError(e)));
}
}
return Void();
}
ACTOR template <class DB>
Future<Void> deleteTenant(Reference<DB> db, TenantName name) {
state Optional<DataClusterMetadata> clusterMetadata;
// Step 1: record that we are removing the tenant in the management cluster
state Reference<typename DB::TransactionT> startTr = db->createTransaction();
loop {
try {
startTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Optional<TenantMapEntry> managementEntry =
wait(ManagementAPI::tryGetTenantTransaction(startTr, name));
if (!managementEntry.present()) {
throw tenant_not_found();
}
if (managementEntry.get().assignedCluster.present()) {
Optional<DataClusterMetadata> _clusterMetadata =
wait(tryGetClusterTransaction(startTr, managementEntry.get().assignedCluster.get()));
clusterMetadata = _clusterMetadata;
if (managementEntry.get().tenantState != TenantState::REMOVING) {
TenantMapEntry updatedEntry = managementEntry.get();
updatedEntry.tenantState = TenantState::REMOVING;
ManagementAPI::configureTenantTransaction(startTr, name, updatedEntry);
wait(safeThreadFutureToFuture(startTr->commit()));
}
}
break;
} catch (Error& e) {
wait(safeThreadFutureToFuture(startTr->onError(e)));
}
}
// Step 2: remove the tenant from the data cluster
if (clusterMetadata.present()) {
state Reference<IDatabase> dataClusterDb = MultiVersionApi::api->createDatabase(
makeReference<ClusterConnectionMemoryRecord>(clusterMetadata.get().connectionString));
wait(ManagementAPI::deleteTenant(dataClusterDb, name, ManagementAPI::TenantOperationType::DATA_CLUSTER));
}
// Step 3: mark the tenant as ready in the management cluster
state Reference<typename DB::TransactionT> removeTr = db->createTransaction();
loop {
try {
removeTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Optional<TenantMapEntry> finalEntry = wait(ManagementAPI::tryGetTenantTransaction(removeTr, name));
if (finalEntry.present() && finalEntry.get().assignedCluster.present()) {
state Optional<DataClusterMetadata> finalClusterMetadata =
wait(tryGetClusterTransaction(removeTr, finalEntry.get().assignedCluster.get()));
if (finalClusterMetadata.present()) {
DataClusterEntry updatedEntry = finalClusterMetadata.get().entry;
// TODO: this should actually track the groups, not the tenants
--updatedEntry.allocated.numTenantGroups;
updateClusterMetadata(removeTr,
finalEntry.get().assignedCluster.get(),
Optional<ClusterConnectionString>(),
updatedEntry);
}
}
wait(ManagementAPI::deleteTenantTransaction(
removeTr, name, ManagementAPI::TenantOperationType::MANAGEMENT_CLUSTER));
wait(safeThreadFutureToFuture(removeTr->commit()));
break;
} catch (Error& e) {
wait(safeThreadFutureToFuture(removeTr->onError(e)));
}
}
return Void();
}
}; // namespace MetaclusterAPI
#include "flow/unactorcompiler.h"

2
fdbclient/NativeAPI.actor.cpp
View File

@ -3190,6 +3190,8 @@ TenantInfo TransactionState::getTenantInfo() {
if (options.rawAccess) {
return TenantInfo();
} else if (!cx->internal && cx->clientInfo->get().tenantMode == TenantMode::MANAGEMENT) {
throw management_cluster_invalid_access();
} else if (!cx->internal && cx->clientInfo->get().tenantMode == TenantMode::REQUIRED && !t.present()) {
throw tenant_name_required();
} else if (!t.present()) {

6
fdbclient/Tenant.cpp
View File

@ -23,12 +23,12 @@
#include "flow/UnitTest.h"
TEST_CASE("/fdbclient/TenantMapEntry/Serialization") {
TenantMapEntry entry1(1, ""_sr);
TenantMapEntry entry1(1, ""_sr, TenantState::READY);
ASSERT(entry1.prefix == "\x00\x00\x00\x00\x00\x00\x00\x01"_sr);
TenantMapEntry entry2 = decodeTenantEntry(encodeTenantEntry(entry1));
ASSERT(entry1.id == entry2.id && entry1.prefix == entry2.prefix);
TenantMapEntry entry3(std::numeric_limits<int64_t>::max(), "foo"_sr);
TenantMapEntry entry3(std::numeric_limits<int64_t>::max(), "foo"_sr, TenantState::READY);
ASSERT(entry3.prefix == "foo\xfe\xff\xff\xff\xff\xff\xff\xff"_sr);
TenantMapEntry entry4 = decodeTenantEntry(encodeTenantEntry(entry3));
ASSERT(entry3.id == entry4.id && entry3.prefix == entry4.prefix);
@ -43,7 +43,7 @@ TEST_CASE("/fdbclient/TenantMapEntry/Serialization") {
Standalone<StringRef> subspace = makeString(subspaceLength);
generateRandomData(mutateString(subspace), subspaceLength);
TenantMapEntry entry(id, subspace);
TenantMapEntry entry(id, subspace, TenantState::READY);
int64_t bigEndianId = bigEndian64(id);
ASSERT(entry.id == id && entry.prefix.startsWith(subspace) &&
entry.prefix.endsWith(StringRef(reinterpret_cast<uint8_t*>(&bigEndianId), 8)) &&

49
fdbclient/Tenant.h
View File

@ -31,6 +31,8 @@ typedef Standalone<TenantNameRef> TenantName;
typedef StringRef TenantGroupNameRef;
typedef Standalone<TenantGroupNameRef> TenantGroupName;
enum class TenantState { REGISTERING, READY, REMOVING, ERROR };
struct TenantMapEntry {
constexpr static FileIdentifier file_identifier = 12247338;
@ -47,10 +49,43 @@ struct TenantMapEntry {
return id;
}
static std::string tenantStateToString(TenantState tenantState) {
switch (tenantState) {
case TenantState::REGISTERING:
return "registering";
case TenantState::READY:
return "ready";
case TenantState::REMOVING:
return "removing";
case TenantState::ERROR:
return "error";
default:
ASSERT(false);
}
}
static TenantState stringToTenantState(std::string stateStr) {
if (stateStr == "registering") {
return TenantState::REGISTERING;
} else if (stateStr == "ready") {
return TenantState::READY;
} else if (stateStr == "removing") {
return TenantState::REMOVING;
} else if (stateStr == "error") {
return TenantState::ERROR;
}
ASSERT(false);
throw internal_error();
}
Arena arena;
int64_t id;
Key prefix;
Optional<TenantGroupName> tenantGroup;
TenantState tenantState;
// TODO: fix this
Optional<Standalone<StringRef>> assignedCluster;
constexpr static int ROOT_PREFIX_SIZE = sizeof(id);
@ -66,17 +101,21 @@ struct TenantMapEntry {
}
TenantMapEntry() : id(-1) {}
TenantMapEntry(int64_t id, KeyRef subspace) : id(id) { setSubspace(subspace); }
TenantMapEntry(int64_t id, KeyRef subspace, Optional<TenantGroupName> tenantGroup)
: id(id), tenantGroup(tenantGroup) {
TenantMapEntry(int64_t id, KeyRef subspace, TenantState tenantState) : id(id), tenantState(tenantState) {
setSubspace(subspace);
}
TenantMapEntry(int64_t id, KeyRef subspace, Optional<TenantGroupName> tenantGroup, TenantState tenantState)
: id(id), tenantGroup(tenantGroup), tenantState(tenantState) {
setSubspace(subspace);
}
bool matchesConfiguration(TenantMapEntry const& other) const { return tenantGroup == other.tenantGroup; }
template <class Ar>
void serialize(Ar& ar) {
KeyRef subspace;
if (ar.isDeserializing) {
serializer(ar, id, subspace, tenantGroup);
serializer(ar, id, subspace, tenantGroup, tenantState, assignedCluster);
if (id >= 0) {
setSubspace(subspace);
}
@ -85,7 +124,7 @@ struct TenantMapEntry {
if (!prefix.empty()) {
subspace = prefix.substr(0, prefix.size() - 8);
}
serializer(ar, id, subspace, tenantGroup);
serializer(ar, id, subspace, tenantGroup, tenantState, assignedCluster);
}
}
};

47
fdbclient/TenantManagement.actor.cpp Normal file
View File

@ -0,0 +1,47 @@
/*
* TenantManagement.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <string>
#include <map>
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/SystemData.h"
#include "flow/actorcompiler.h" // has to be last include
namespace ManagementAPI {
bool checkTenantMode(Optional<Value> tenantModeValue, bool isDataCluster, TenantOperationType operationType) {
TenantMode tenantMode = TenantMode::fromValue(tenantModeValue.castTo<ValueRef>());
if (tenantMode == TenantMode::DISABLED) {
return false;
} else if (operationType == TenantOperationType::MANAGEMENT_CLUSTER && tenantMode != TenantMode::MANAGEMENT) {
return false;
} else if (operationType == TenantOperationType::DATA_CLUSTER &&
(tenantMode != TenantMode::REQUIRED || !isDataCluster)) {
return false;
} else if (operationType == TenantOperationType::STANDALONE_CLUSTER &&
(tenantMode == TenantMode::MANAGEMENT || isDataCluster)) {
return false;
}
return true;
}
} // namespace ManagementAPI

177
fdbclient/TenantManagement.actor.h
View File

@ -32,12 +32,15 @@
#include "flow/actorcompiler.h" // has to be last include
namespace ManagementAPI {
enum class TenantOperationType { STANDALONE_CLUSTER, MANAGEMENT_CLUSTER, DATA_CLUSTER };
ACTOR template <class Transaction>
Future<Optional<TenantMapEntry>> tryGetTenantTransaction(Transaction tr, TenantName name) {
state Key tenantMapKey = name.withPrefix(tenantMapPrefix);
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantFuture = tr->get(tenantMapKey);
Optional<Value> val = wait(safeThreadFutureToFuture(tenantFuture));
@ -79,13 +82,22 @@ Future<TenantMapEntry> getTenant(Reference<DB> db, TenantName name) {
return entry.get();
}
bool checkTenantMode(Optional<Value> tenantModeValue, bool isDataCluster, TenantOperationType operationType);
// Creates a tenant with the given name. If the tenant already exists, an empty optional will be returned.
ACTOR template <class Transaction>
Future<Optional<TenantMapEntry>> createTenantTransaction(Transaction tr,
TenantNameRef name,
TenantMapEntry tenantEntry = TenantMapEntry()) {
Future<std::pair<TenantMapEntry, bool>> createTenantTransaction(
Transaction tr,
TenantNameRef name,
TenantMapEntry tenantEntry = TenantMapEntry(),
TenantOperationType operationType = TenantOperationType::STANDALONE_CLUSTER) {
state Key tenantMapKey = name.withPrefix(tenantMapPrefix);
state bool generateId = operationType != TenantOperationType::DATA_CLUSTER;
state bool allowSubspace = operationType == TenantOperationType::STANDALONE_CLUSTER;
ASSERT(operationType != TenantOperationType::MANAGEMENT_CLUSTER || tenantEntry.assignedCluster.present());
if (name.startsWith("\xff"_sr)) {
throw invalid_tenant_name();
}
@ -94,57 +106,78 @@ Future<Optional<TenantMapEntry>> createTenantTransaction(Transaction tr,
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state Future<Optional<TenantMapEntry>> existingEntryFuture = tryGetTenantTransaction(tr, name);
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantDataPrefixFuture =
tr->get(tenantDataPrefixKey);
state typename transaction_future_type<Transaction, Optional<Value>>::type lastIdFuture = tr->get(tenantLastIdKey);
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantDataPrefixFuture;
if (allowSubspace) {
tenantDataPrefixFuture = tr->get(tenantDataPrefixKey);
}
state typename transaction_future_type<Transaction, Optional<Value>>::type lastIdFuture;
if (generateId) {
lastIdFuture = tr->get(tenantLastIdKey);
}
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantModeFuture =
tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr));
state typename transaction_future_type<Transaction, Optional<Value>>::type metaclusterRegistrationFuture =
tr->get(dataClusterRegistrationKey);
Optional<Value> tenantMode = wait(safeThreadFutureToFuture(tenantModeFuture));
state Optional<Value> tenantMode = wait(safeThreadFutureToFuture(tenantModeFuture));
Optional<Value> metaclusterRegistration = wait(safeThreadFutureToFuture(metaclusterRegistrationFuture));
// TODO: disallow tenant creation directly on clusters in a metacluster
if (!tenantMode.present() || tenantMode.get() == StringRef(format("%d", TenantMode::DISABLED))) {
if (!checkTenantMode(tenantMode, metaclusterRegistration.present(), operationType)) {
throw tenants_disabled();
}
Optional<TenantMapEntry> existingEntry = wait(existingEntryFuture);
if (existingEntry.present()) {
return Optional<TenantMapEntry>();
return std::make_pair(existingEntry.get(), false);
}
state Optional<Value> lastIdVal = wait(safeThreadFutureToFuture(lastIdFuture));
Optional<Value> tenantDataPrefix = wait(safeThreadFutureToFuture(tenantDataPrefixFuture));
if (tenantDataPrefix.present() &&
tenantDataPrefix.get().size() + TenantMapEntry::ROOT_PREFIX_SIZE > CLIENT_KNOBS->TENANT_PREFIX_SIZE_LIMIT) {
TraceEvent(SevWarnAlways, "TenantPrefixTooLarge")
.detail("TenantSubspace", tenantDataPrefix.get())
.detail("TenantSubspaceLength", tenantDataPrefix.get().size())
.detail("RootPrefixLength", TenantMapEntry::ROOT_PREFIX_SIZE)
.detail("MaxTenantPrefixSize", CLIENT_KNOBS->TENANT_PREFIX_SIZE_LIMIT);
throw client_invalid_operation();
if (generateId) {
state Optional<Value> lastIdVal = wait(safeThreadFutureToFuture(lastIdFuture));
tenantEntry.id = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) + 1 : 0;
tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(tenantEntry.id));
}
tenantEntry.id = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) + 1 : 0;
tenantEntry.setSubspace(tenantDataPrefix.present() ? (KeyRef)tenantDataPrefix.get() : ""_sr);
if (allowSubspace) {
Optional<Value> tenantDataPrefix = wait(safeThreadFutureToFuture(tenantDataPrefixFuture));
if (tenantDataPrefix.present() &&
tenantDataPrefix.get().size() + TenantMapEntry::ROOT_PREFIX_SIZE > CLIENT_KNOBS->TENANT_PREFIX_SIZE_LIMIT) {
TraceEvent(SevWarnAlways, "TenantPrefixTooLarge")
.detail("TenantSubspace", tenantDataPrefix.get())
.detail("TenantSubspaceLength", tenantDataPrefix.get().size())
.detail("RootPrefixLength", TenantMapEntry::ROOT_PREFIX_SIZE)
.detail("MaxTenantPrefixSize", CLIENT_KNOBS->TENANT_PREFIX_SIZE_LIMIT);
state typename transaction_future_type<Transaction, RangeResult>::type prefixRangeFuture =
tr->getRange(prefixRange(tenantEntry.prefix), 1);
RangeResult contents = wait(safeThreadFutureToFuture(prefixRangeFuture));
if (!contents.empty()) {
throw tenant_prefix_allocator_conflict();
throw client_invalid_operation();
}
tenantEntry.setSubspace(tenantDataPrefix.present() ? (KeyRef)tenantDataPrefix.get() : ""_sr);
} else {
tenantEntry.setSubspace(""_sr);
}
if (operationType != TenantOperationType::MANAGEMENT_CLUSTER) {
state typename transaction_future_type<Transaction, RangeResult>::type prefixRangeFuture =
tr->getRange(prefixRange(tenantEntry.prefix), 1);
RangeResult contents = wait(safeThreadFutureToFuture(prefixRangeFuture));
if (!contents.empty()) {
throw tenant_prefix_allocator_conflict();
}
}
// We don't store some metadata in the tenant entries on data clusters
if (operationType == TenantOperationType::DATA_CLUSTER) {
tenantEntry.assignedCluster = Optional<ClusterName>();
}
tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(tenantEntry.id));
tr->set(tenantMapKey, encodeTenantEntry(tenantEntry));
return tenantEntry;
return std::make_pair(tenantEntry, true);
}
ACTOR template <class DB>
Future<Void> createTenant(Reference<DB> db, TenantName name, TenantMapEntry tenantEntry = TenantMapEntry()) {
Future<TenantMapEntry> createTenant(Reference<DB> db,
TenantName name,
TenantMapEntry tenantEntry = TenantMapEntry(),
TenantOperationType operationType = TenantOperationType::STANDALONE_CLUSTER) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
state bool firstTry = true;
@ -161,26 +194,29 @@ Future<Void> createTenant(Reference<DB> db, TenantName name, TenantMapEntry tena
firstTry = false;
}
state Optional<TenantMapEntry> newTenant = wait(createTenantTransaction(tr, name, tenantEntry));
state std::pair<TenantMapEntry, bool> newTenant =
wait(createTenantTransaction(tr, name, tenantEntry, operationType));
if (BUGGIFY) {
throw commit_unknown_result();
if (newTenant.second) {
if (BUGGIFY) {
throw commit_unknown_result();
}
wait(safeThreadFutureToFuture(tr->commit()));
if (BUGGIFY) {
throw commit_unknown_result();
}
TraceEvent("CreatedTenant")
.detail("Tenant", name)
.detail("TenantId", newTenant.first.id)
.detail("Prefix", newTenant.first.prefix)
.detail("TenantGroup", tenantEntry.tenantGroup)
.detail("Version", tr->getCommittedVersion());
}
wait(safeThreadFutureToFuture(tr->commit()));
if (BUGGIFY) {
throw commit_unknown_result();
}
TraceEvent("CreatedTenant")
.detail("Tenant", name)
.detail("TenantId", newTenant.present() ? newTenant.get().id : -1)
.detail("Prefix", newTenant.present() ? (StringRef)newTenant.get().prefix : "Unknown"_sr)
.detail("TenantGroup", tenantEntry.tenantGroup)
.detail("Version", tr->getCommittedVersion());
return Void();
return newTenant.first;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
@ -188,12 +224,19 @@ Future<Void> createTenant(Reference<DB> db, TenantName name, TenantMapEntry tena
}
ACTOR template <class Transaction>
Future<Void> deleteTenantTransaction(Transaction tr, TenantNameRef name) {
Future<Void> deleteTenantTransaction(Transaction tr,
TenantNameRef name,
TenantOperationType operationType = TenantOperationType::STANDALONE_CLUSTER) {
state Key tenantMapKey = name.withPrefix(tenantMapPrefix);
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantModeFuture =
tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr));
state typename transaction_future_type<Transaction, Optional<Value>>::type metaclusterRegistrationFuture =
tr->get(dataClusterRegistrationKey);
state Optional<TenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, name));
if (!tenantEntry.present()) {
return Void();
@ -201,6 +244,20 @@ Future<Void> deleteTenantTransaction(Transaction tr, TenantNameRef name) {
state typename transaction_future_type<Transaction, RangeResult>::type prefixRangeFuture =
tr->getRange(prefixRange(tenantEntry.get().prefix), 1);
state Optional<Value> tenantMode = wait(safeThreadFutureToFuture(tenantModeFuture));
Optional<Value> metaclusterRegistration = wait(safeThreadFutureToFuture(metaclusterRegistrationFuture));
if (!checkTenantMode(tenantMode, metaclusterRegistration.present(), operationType)) {
throw tenants_disabled();
}
if (operationType == TenantOperationType::MANAGEMENT_CLUSTER &&
tenantEntry.get().tenantState != TenantState::REMOVING) {
// TODO: better error
throw operation_failed();
}
RangeResult contents = wait(safeThreadFutureToFuture(prefixRangeFuture));
if (!contents.empty()) {
throw tenant_not_empty();
@ -212,24 +269,26 @@ Future<Void> deleteTenantTransaction(Transaction tr, TenantNameRef name) {
}
ACTOR template <class DB>
Future<Void> deleteTenant(Reference<DB> db, TenantName name) {
Future<Void> deleteTenant(Reference<DB> db,
TenantName name,
TenantOperationType operationType = TenantOperationType::STANDALONE_CLUSTER) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
state bool firstTry = true;
state bool checkExistence = operationType == TenantOperationType::STANDALONE_CLUSTER;
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (firstTry) {
if (checkExistence) {
Optional<TenantMapEntry> entry = wait(tryGetTenantTransaction(tr, name));
if (!entry.present()) {
throw tenant_not_found();
}
firstTry = false;
checkExistence = false;
}
wait(deleteTenantTransaction(tr, name));
wait(deleteTenantTransaction(tr, name, operationType));
if (BUGGIFY) {
throw commit_unknown_result();
@ -266,7 +325,7 @@ Future<std::map<TenantName, TenantMapEntry>> listTenantsTransaction(Transaction
state KeyRange range = KeyRangeRef(begin, end).withPrefix(tenantMapPrefix);
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state typename transaction_future_type<Transaction, RangeResult>::type listFuture =
tr->getRange(firstGreaterOrEqual(range.begin), firstGreaterOrEqual(range.end), limit);

6
fdbclient/TenantSpecialKeys.actor.cpp
View File

@ -77,6 +77,10 @@ ACTOR Future<Void> getTenantList(ReadYourWritesTransaction* ryw,
json_spirit::mObject tenantEntry;
tenantEntry["id"] = tenant.second.id;
tenantEntry["prefix"] = tenant.second.prefix.toString();
tenantEntry["tenant_state"] = TenantMapEntry::tenantStateToString(tenant.second.tenantState);
if (tenant.second.assignedCluster.present()) {
tenantEntry["assigned_cluster"] = tenant.second.assignedCluster.get().toString();
}
if (tenant.second.tenantGroup.present()) {
tenantEntry["tenant_group"] = tenant.second.tenantGroup.get().toString();
}
@ -208,7 +212,7 @@ ACTOR Future<Void> createTenant(ReadYourWritesTransaction* ryw,
wait(applyTenantConfig(ryw, tenantName, configMutations.get(), &tenantEntry, true));
}
Optional<TenantMapEntry> entry =
std::pair<TenantMapEntry, bool> entry =
wait(ManagementAPI::createTenantTransaction(&ryw->getTransaction(), tenantName, tenantEntry));
return Void();

2
fdbserver/CMakeLists.txt
View File

@ -295,7 +295,7 @@ set(FDBSERVER_SRCS
workloads/TagThrottleApi.actor.cpp
workloads/TargetedKill.actor.cpp
workloads/TaskBucketCorrectness.actor.cpp
workloads/TenantManagement.actor.cpp
workloads/TenantManagementWorkload.actor.cpp
workloads/ThreadSafety.actor.cpp
workloads/Throttling.actor.cpp
workloads/Throughput.actor.cpp

12
fdbserver/storageserver.actor.cpp
View File

@ -3809,11 +3809,13 @@ bool rangeIntersectsAnyTenant(TenantPrefixIndex& prefixIndex, KeyRangeRef range,
}
TEST_CASE("/fdbserver/storageserver/rangeIntersectsAnyTenant") {
std::map<TenantName, TenantMapEntry> entries = { std::make_pair("tenant0"_sr, TenantMapEntry(0, ""_sr)),
std::make_pair("tenant2"_sr, TenantMapEntry(2, ""_sr)),
std::make_pair("tenant3"_sr, TenantMapEntry(3, ""_sr)),
std::make_pair("tenant4"_sr, TenantMapEntry(4, ""_sr)),
std::make_pair("tenant6"_sr, TenantMapEntry(6, ""_sr)) };
std::map<TenantName, TenantMapEntry> entries = {
std::make_pair("tenant0"_sr, TenantMapEntry(0, ""_sr, TenantState::READY)),
std::make_pair("tenant2"_sr, TenantMapEntry(2, ""_sr, TenantState::READY)),
std::make_pair("tenant3"_sr, TenantMapEntry(3, ""_sr, TenantState::READY)),
std::make_pair("tenant4"_sr, TenantMapEntry(4, ""_sr, TenantState::READY)),
std::make_pair("tenant6"_sr, TenantMapEntry(6, ""_sr, TenantState::READY))
};
TenantPrefixIndex index;
index.createNewVersion(1);
for (auto entry : entries) {

2
fdbserver/tester.actor.cpp
View File

@ -1626,7 +1626,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
}
if (useDB) {
std::vector<Future<Void>> tenantFutures;
std::vector<Future<TenantMapEntry>> tenantFutures;
for (auto tenant : tenantsToCreate) {
TenantMapEntry entry;
if (deterministicRandom()->coinflip()) {

2
fdbserver/workloads/FuzzApiCorrectness.actor.cpp
View File

@ -230,7 +230,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
Reference<IDatabase> db = wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
self->db = db;
std::vector<Future<Void>> tenantFutures;
std::vector<Future<TenantMapEntry>> tenantFutures;
for (int i = 0; i < self->numTenants + 1; ++i) {
TenantName tenantName = getTenant(i);
self->tenants.push_back(self->db->openTenant(tenantName));

19
fdbserver/workloads/TenantManagement.actor.cpp → fdbserver/workloads/TenantManagementWorkload.actor.cpp
View File

@ -1,5 +1,5 @@
/*
* TenantManagement.actor.cpp
* TenantManagementWorkload.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
@ -168,12 +168,12 @@ struct TenantManagementWorkload : TestWorkload {
} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
TenantMapEntry entry;
entry.tenantGroup = tenantGroup;
wait(ManagementAPI::createTenant(cx.getReference(), tenant, entry));
TenantMapEntry result = wait(ManagementAPI::createTenant(cx.getReference(), tenant, entry));
} else {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
TenantMapEntry entry;
entry.tenantGroup = tenantGroup;
Optional<TenantMapEntry> _ = wait(ManagementAPI::createTenantTransaction(tr, tenant, entry));
std::pair<TenantMapEntry, bool> _ = wait(ManagementAPI::createTenantTransaction(tr, tenant, entry));
wait(tr->commit());
}
@ -408,9 +408,17 @@ struct TenantManagementWorkload : TestWorkload {
int64_t id;
std::string prefix;
std::string tenantStateStr;
std::string assignedClusterStr;
std::string tenantGroupStr;
jsonDoc.get("id", id);
jsonDoc.get("prefix", prefix);
jsonDoc.get("tenant_state", tenantStateStr);
Optional<ClusterName> assignedCluster;
if (jsonDoc.tryGet("assigned_cluster", assignedClusterStr)) {
assignedCluster = ClusterNameRef(assignedClusterStr);
}
Optional<TenantGroupName> tenantGroup;
if (jsonDoc.tryGet("tenant_group", tenantGroupStr)) {
@ -418,7 +426,10 @@ struct TenantManagementWorkload : TestWorkload {
}
Key prefixKey = KeyRef(prefix);
TenantMapEntry entry(id, prefixKey.substr(0, prefixKey.size() - 8), tenantGroup);
TenantMapEntry entry(id,
prefixKey.substr(0, prefixKey.size() - 8),
tenantGroup,
TenantMapEntry::stringToTenantState(tenantStateStr));
ASSERT(entry.prefix == prefixKey);
return entry;

13
flow/error_definitions.h
View File

@ -225,11 +225,12 @@ ERROR( tenant_name_required, 2130, "Tenant name must be specified to access data
ERROR( tenant_not_found, 2131, "Tenant does not exist" )
ERROR( tenant_already_exists, 2132, "A tenant with the given name already exists" )
ERROR( tenant_not_empty, 2133, "Cannot delete a non-empty tenant" )
ERROR( invalid_tenant_name, 2134, "Tenant name cannot begin with \\xff");
ERROR( tenant_prefix_allocator_conflict, 2135, "The database already has keys stored at the prefix allocated for the tenant");
ERROR( tenants_disabled, 2136, "Tenants have been disabled in the cluster");
ERROR( unknown_tenant, 2137, "Tenant is not available from this server")
ERROR( illegal_tenant_access, 2138, "Illegal tenant access")
ERROR( invalid_tenant_name, 2134, "Tenant name cannot begin with \\xff" )
ERROR( tenant_prefix_allocator_conflict, 2135, "The database already has keys stored at the prefix allocated for the tenant" )
ERROR( tenants_disabled, 2136, "Tenants have been disabled in the cluster" )
ERROR( unknown_tenant, 2137, "Tenant is not available from this server" )
ERROR( illegal_tenant_access, 2138, "Illegal tenant access" )
ERROR( tenant_removed, 2139, "The tenant was removed" )
ERROR( invalid_cluster_name, 2150, "Data cluster name cannot begin with \\xff" )
ERROR( invalid_metacluster_operation, 2151, "Metacluster operation performed on non-metacluster" )
@ -238,6 +239,8 @@ ERROR( cluster_not_found, 2153, "Data cluster does not exist" )
ERROR( cluster_not_empty, 2154, "Data cluster must be empty" )
ERROR( cluster_configuration_failure, 2155, "Could not configure cluster" )
ERROR( cluster_already_registered, 2156, "Data cluster is already registered with a metacluster" )
ERROR( metacluster_no_capacity, 2157, "Metacluster does not have capacity to create new tenants" )
ERROR( management_cluster_invalid_access, 2158, "Standard transactions cannot be run against the management cluster" )
// 2200 - errors from bindings and official APIs
ERROR( api_version_unset, 2200, "API version is not set" )