From cd19f2cdc2370a5e5c282e7cd7da901cb287c45c Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Thu, 28 Jul 2022 15:03:38 -0700 Subject: [PATCH] Add metacluster awareness of the tenant count limit --- fdbclient/Metacluster.cpp | 1 - fdbclient/MetaclusterManagement.actor.cpp | 2 + fdbclient/include/fdbclient/KeyBackedTypes.h | 23 ++++++++++ .../fdbclient/MetaclusterManagement.actor.h | 27 +++++++++++- ...antManagementConcurrencyWorkload.actor.cpp | 2 +- .../TenantManagementWorkload.actor.cpp | 43 +++++++++++++++---- 6 files changed, 86 insertions(+), 12 deletions(-) diff --git a/fdbclient/Metacluster.cpp b/fdbclient/Metacluster.cpp index 6482d9eef8..733b73bfde 100644 --- a/fdbclient/Metacluster.cpp +++ b/fdbclient/Metacluster.cpp @@ -29,4 +29,3 @@ json_spirit::mObject ClusterUsage::toJson() const { obj["num_tenant_groups"] = numTenantGroups; return obj; } - diff --git a/fdbclient/MetaclusterManagement.actor.cpp b/fdbclient/MetaclusterManagement.actor.cpp index 8a42e6a82a..c8ea6032b6 100644 --- a/fdbclient/MetaclusterManagement.actor.cpp +++ b/fdbclient/MetaclusterManagement.actor.cpp @@ -51,6 +51,8 @@ KeyBackedMap ManagementClusterMetadata::clusterCapacityIndex("metacluster/clusterCapacityIndex/"_sr); +KeyBackedMap, BinaryCodec> + ManagementClusterMetadata::clusterTenantCount("metacluster/clusterTenantCount/"_sr); KeyBackedSet ManagementClusterMetadata::clusterTenantIndex("metacluster/dataCluster/tenantMap/"_sr); KeyBackedSet ManagementClusterMetadata::clusterTenantGroupIndex("metacluster/dataCluster/tenantGroupMap/"_sr); diff --git a/fdbclient/include/fdbclient/KeyBackedTypes.h b/fdbclient/include/fdbclient/KeyBackedTypes.h index 2977262b6c..aaf6e8ba3a 100644 --- a/fdbclient/include/fdbclient/KeyBackedTypes.h +++ b/fdbclient/include/fdbclient/KeyBackedTypes.h @@ -156,6 +156,12 @@ struct NullCodec { static Standalone unpack(Standalone val) { return val; } }; +template +struct BinaryCodec { + static Standalone pack(T val) { return BinaryWriter::toValue(val, Unversioned()); } + static T unpack(Standalone val) { return BinaryReader::fromStringRef(val, Unversioned()); } +}; + template struct KeyBackedRangeResult { std::vector results; @@ -364,6 +370,16 @@ public: })); } + // Get key's value or defaultValue if it doesn't exist + template + Future getD(Transaction tr, + KeyType const& key, + Snapshot snapshot = Snapshot::False, + ValueType defaultValue = ValueType()) const { + return map(get(tr, key, snapshot), + [=](Optional val) -> ValueType { return val.orDefault(defaultValue); }); + } + // Returns a Property that can be get/set that represents key's entry in this this. KeyBackedProperty getProperty(KeyType const& key) const { return subspace.begin.withSuffix(KeyCodec::pack(key)); @@ -378,6 +394,13 @@ public: return k.expectedSize() + v.expectedSize(); } + template + void atomicOp(Transaction tr, KeyType const& key, ValueType const& val, MutationRef::Type type) { + Key k = subspace.begin.withSuffix(KeyCodec::pack(key)); + Value v = ValueCodec::pack(val); + tr->atomicOp(k, v, type); + } + template void erase(Transaction tr, KeyType const& key) { tr->clear(subspace.begin.withSuffix(KeyCodec::pack(key))); diff --git a/fdbclient/include/fdbclient/MetaclusterManagement.actor.h b/fdbclient/include/fdbclient/MetaclusterManagement.actor.h index ec9e92945f..a4abc77a56 100644 --- a/fdbclient/include/fdbclient/MetaclusterManagement.actor.h +++ b/fdbclient/include/fdbclient/MetaclusterManagement.actor.h @@ -106,6 +106,9 @@ struct ManagementClusterMetadata { // A set of non-full clusters where the key is the tuple (num tenant groups allocated, cluster name). static KeyBackedSet clusterCapacityIndex; + // A map from cluster name to a count of tenants + static KeyBackedMap, BinaryCodec> clusterTenantCount; + // A set of cluster/tenant pairings ordered by cluster static KeyBackedSet clusterTenantIndex; @@ -634,6 +637,11 @@ Future managementClusterPurgeDataCluster(Reference db, ClusterNameRef ManagementClusterMetadata::tenantMetadata.tenantMap.erase(tr, entry.getString(1)); } + ManagementClusterMetadata::tenantMetadata.tenantCount.atomicOp( + tr, -tenantEntries.results.size(), MutationRef::AddValue); + ManagementClusterMetadata::clusterTenantCount.atomicOp( + tr, name, -tenantEntries.results.size(), MutationRef::AddValue); + // Erase all of the tenants processed in this transaction from the cluster tenant index ManagementClusterMetadata::clusterTenantIndex.erase( tr, @@ -1042,7 +1050,6 @@ struct CreateTenantImpl { if (self->tenantName.startsWith("\xff"_sr)) { throw invalid_tenant_name(); } - state Future> existingEntryFuture = tryGetTenantTransaction(tr, self->tenantName); Optional existingEntry = wait(existingEntryFuture); @@ -1058,6 +1065,19 @@ struct CreateTenantImpl { self->tenantEntry.tenantState = TenantState::REGISTERING; ManagementClusterMetadata::tenantMetadata.tenantMap.set(tr, self->tenantName, self->tenantEntry); + if (!existingEntry.present()) { + ManagementClusterMetadata::tenantMetadata.tenantCount.atomicOp(tr, 1, MutationRef::AddValue); + ManagementClusterMetadata::clusterTenantCount.atomicOp( + tr, self->tenantEntry.assignedCluster.get(), 1, MutationRef::AddValue); + + int64_t clusterTenantCount = wait(ManagementClusterMetadata::clusterTenantCount.getD( + tr, self->tenantEntry.assignedCluster.get(), Snapshot::False, 0)); + + if (clusterTenantCount > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER) { + throw cluster_no_capacity(); + } + } + return std::make_pair(self->tenantEntry, true); } @@ -1213,6 +1233,11 @@ struct DeleteTenantImpl { // Erase the tenant entry itself ManagementClusterMetadata::tenantMetadata.tenantMap.erase(tr, tenantName); + // This is idempotent because this function is only called if the tenant is in the map + ManagementClusterMetadata::tenantMetadata.tenantCount.atomicOp(tr, -1, MutationRef::AddValue); + ManagementClusterMetadata::clusterTenantCount.atomicOp( + tr, tenantEntry.assignedCluster.get(), -1, MutationRef::AddValue); + // Clean up cluster based tenant indices and remove the tenant group if it is empty state DataClusterMetadata clusterMetadata = wait(getClusterTransaction(tr, tenantEntry.assignedCluster.get())); diff --git a/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp b/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp index a3b2f35a8b..a12c361b1f 100644 --- a/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp +++ b/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp @@ -195,7 +195,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload { return Void(); } catch (Error& e) { - if (e.code() != error_code_tenant_already_exists) { + if (e.code() != error_code_tenant_already_exists && e.code() != error_code_cluster_no_capacity) { TraceEvent(SevError, "CreateTenantFailure").error(e).detail("TenantName", tenant); } diff --git a/fdbserver/workloads/TenantManagementWorkload.actor.cpp b/fdbserver/workloads/TenantManagementWorkload.actor.cpp index 26a5198521..c1aea4605e 100644 --- a/fdbserver/workloads/TenantManagementWorkload.actor.cpp +++ b/fdbserver/workloads/TenantManagementWorkload.actor.cpp @@ -64,6 +64,7 @@ struct TenantManagementWorkload : TestWorkload { const Key testParametersKey = "test_parameters"_sr; const Value noTenantValue = "no_tenant"_sr; const TenantName tenantNamePrefix = "tenant_management_workload_"_sr; + const ClusterName dataClusterName = "cluster1"_sr; TenantName localTenantNamePrefix; TenantName localTenantGroupNamePrefix; @@ -165,7 +166,8 @@ struct TenantManagementWorkload : TestWorkload { DataClusterEntry entry; entry.capacity.numTenantGroups = 1e9; - wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, g_simulator.extraDatabases[0], entry)); + wait(MetaclusterAPI::registerCluster( + self->mvDb, self->dataClusterName, g_simulator.extraDatabases[0], entry)); } state Transaction tr(cx); @@ -1397,20 +1399,40 @@ struct TenantManagementWorkload : TestWorkload { // Verify that the tenant count matches the actual number of tenants in the cluster and that we haven't created too // many - ACTOR static Future checkTenantCount(Database cx) { - state Reference tr = cx->createTransaction(); + ACTOR template + static Future checkTenantCount(TenantManagementWorkload* self, Reference db) { + state Reference tr = db->createTransaction(); loop { try { tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - state int64_t tenantCount = wait(TenantMetadata::tenantCount.getD(tr, Snapshot::False, 0)); - KeyBackedRangeResult> tenants = - wait(TenantMetadata::tenantMap.getRange(tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)); + state int64_t tenantCount; + state KeyBackedRangeResult> tenants; + + if (UseMetacluster) { + wait(store(tenantCount, + MetaclusterAPI::ManagementClusterMetadata::tenantMetadata.tenantCount.getD( + tr, Snapshot::False, 0))); + wait(store(tenants, + MetaclusterAPI::ManagementClusterMetadata::tenantMetadata.tenantMap.getRange( + tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1))); + + int64_t clusterTenantCount = + wait(MetaclusterAPI::ManagementClusterMetadata::clusterTenantCount.getD( + tr, self->dataClusterName, Snapshot::False, 0)); + + ASSERT(clusterTenantCount == tenantCount); + } else { + wait(store(tenantCount, TenantMetadata::tenantCount.getD(tr, Snapshot::False, 0))); + wait(store( + tenants, + TenantMetadata::tenantMap.getRange(tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1))); + ASSERT(tenantCount <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER); + } ASSERT(tenants.results.size() == tenantCount && !tenants.more); - ASSERT(tenantCount <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER); return Void(); } catch (Error& e) { - wait(tr->onError(e)); + wait(safeThreadFutureToFuture(tr->onError(e))); } } } @@ -1647,7 +1669,10 @@ struct TenantManagementWorkload : TestWorkload { } if (self->clientId == 0) { - wait(checkTenantCount(cx)); + if (self->useMetacluster) { + wait(checkTenantCount(self, self->mvDb)); + } + wait(checkTenantCount(self, self->dataDb.getReference())); } wait(compareTenants(self) && compareTenantGroups(self) && checkTenantTombstones(self));