wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add metacluster awareness of the tenant count limit

This commit is contained in:
A.J. Beamon 2022-07-28 15:03:38 -07:00
parent 620467a91b
commit cd19f2cdc2
6 changed files with 86 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fdbclient/Metacluster.cpp
View File

@ -29,4 +29,3 @@ json_spirit::mObject ClusterUsage::toJson() const {
obj["num_tenant_groups"] = numTenantGroups;
return obj;
}

2
fdbclient/MetaclusterManagement.actor.cpp
View File

@ -51,6 +51,8 @@ KeyBackedMap<ClusterName,
ManagementClusterMetadata::dataClusterConnectionRecords("metacluster/dataCluster/connectionString/"_sr);
KeyBackedSet<Tuple> ManagementClusterMetadata::clusterCapacityIndex("metacluster/clusterCapacityIndex/"_sr);
KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>>
ManagementClusterMetadata::clusterTenantCount("metacluster/clusterTenantCount/"_sr);
KeyBackedSet<Tuple> ManagementClusterMetadata::clusterTenantIndex("metacluster/dataCluster/tenantMap/"_sr);
KeyBackedSet<Tuple> ManagementClusterMetadata::clusterTenantGroupIndex("metacluster/dataCluster/tenantGroupMap/"_sr);

23
fdbclient/include/fdbclient/KeyBackedTypes.h
View File

@ -156,6 +156,12 @@ struct NullCodec {
static Standalone<StringRef> unpack(Standalone<StringRef> val) { return val; }
};
template <class T>
struct BinaryCodec {
static Standalone<StringRef> pack(T val) { return BinaryWriter::toValue<T>(val, Unversioned()); }
static T unpack(Standalone<StringRef> val) { return BinaryReader::fromStringRef<T>(val, Unversioned()); }
};
template <typename ResultType>
struct KeyBackedRangeResult {
std::vector<ResultType> results;
@ -364,6 +370,16 @@ public:
}));
}
// Get key's value or defaultValue if it doesn't exist
template <class Transaction>
Future<ValueType> getD(Transaction tr,
KeyType const& key,
Snapshot snapshot = Snapshot::False,
ValueType defaultValue = ValueType()) const {
return map(get(tr, key, snapshot),
[=](Optional<ValueType> val) -> ValueType { return val.orDefault(defaultValue); });
}
// Returns a Property that can be get/set that represents key's entry in this this.
KeyBackedProperty<ValueType> getProperty(KeyType const& key) const {
return subspace.begin.withSuffix(KeyCodec::pack(key));
@ -378,6 +394,13 @@ public:
return k.expectedSize() + v.expectedSize();
}
template <class Transaction>
void atomicOp(Transaction tr, KeyType const& key, ValueType const& val, MutationRef::Type type) {
Key k = subspace.begin.withSuffix(KeyCodec::pack(key));
Value v = ValueCodec::pack(val);
tr->atomicOp(k, v, type);
}
template <class Transaction>
void erase(Transaction tr, KeyType const& key) {
tr->clear(subspace.begin.withSuffix(KeyCodec::pack(key)));

27
fdbclient/include/fdbclient/MetaclusterManagement.actor.h
View File

@ -106,6 +106,9 @@ struct ManagementClusterMetadata {
// A set of non-full clusters where the key is the tuple (num tenant groups allocated, cluster name).
static KeyBackedSet<Tuple> clusterCapacityIndex;
// A map from cluster name to a count of tenants
static KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>> clusterTenantCount;
// A set of cluster/tenant pairings ordered by cluster
static KeyBackedSet<Tuple> clusterTenantIndex;
@ -634,6 +637,11 @@ Future<Void> managementClusterPurgeDataCluster(Reference<DB> db, ClusterNameRef
ManagementClusterMetadata::tenantMetadata.tenantMap.erase(tr, entry.getString(1));
}
ManagementClusterMetadata::tenantMetadata.tenantCount.atomicOp(
tr, -tenantEntries.results.size(), MutationRef::AddValue);
ManagementClusterMetadata::clusterTenantCount.atomicOp(
tr, name, -tenantEntries.results.size(), MutationRef::AddValue);
// Erase all of the tenants processed in this transaction from the cluster tenant index
ManagementClusterMetadata::clusterTenantIndex.erase(
tr,
@ -1042,7 +1050,6 @@ struct CreateTenantImpl {
if (self->tenantName.startsWith("\xff"_sr)) {
throw invalid_tenant_name();
}
state Future<Optional<TenantMapEntry>> existingEntryFuture = tryGetTenantTransaction(tr, self->tenantName);
Optional<TenantMapEntry> existingEntry = wait(existingEntryFuture);
@ -1058,6 +1065,19 @@ struct CreateTenantImpl {
self->tenantEntry.tenantState = TenantState::REGISTERING;
ManagementClusterMetadata::tenantMetadata.tenantMap.set(tr, self->tenantName, self->tenantEntry);
if (!existingEntry.present()) {
ManagementClusterMetadata::tenantMetadata.tenantCount.atomicOp(tr, 1, MutationRef::AddValue);
ManagementClusterMetadata::clusterTenantCount.atomicOp(
tr, self->tenantEntry.assignedCluster.get(), 1, MutationRef::AddValue);
int64_t clusterTenantCount = wait(ManagementClusterMetadata::clusterTenantCount.getD(
tr, self->tenantEntry.assignedCluster.get(), Snapshot::False, 0));
if (clusterTenantCount > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER) {
throw cluster_no_capacity();
}
}
return std::make_pair(self->tenantEntry, true);
}
@ -1213,6 +1233,11 @@ struct DeleteTenantImpl {
// Erase the tenant entry itself
ManagementClusterMetadata::tenantMetadata.tenantMap.erase(tr, tenantName);
// This is idempotent because this function is only called if the tenant is in the map
ManagementClusterMetadata::tenantMetadata.tenantCount.atomicOp(tr, -1, MutationRef::AddValue);
ManagementClusterMetadata::clusterTenantCount.atomicOp(
tr, tenantEntry.assignedCluster.get(), -1, MutationRef::AddValue);
// Clean up cluster based tenant indices and remove the tenant group if it is empty
state DataClusterMetadata clusterMetadata = wait(getClusterTransaction(tr, tenantEntry.assignedCluster.get()));

2
fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp
View File

@ -195,7 +195,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
return Void();
} catch (Error& e) {
if (e.code() != error_code_tenant_already_exists) {
if (e.code() != error_code_tenant_already_exists && e.code() != error_code_cluster_no_capacity) {
TraceEvent(SevError, "CreateTenantFailure").error(e).detail("TenantName", tenant);
}

43
fdbserver/workloads/TenantManagementWorkload.actor.cpp
View File

@ -64,6 +64,7 @@ struct TenantManagementWorkload : TestWorkload {
const Key testParametersKey = "test_parameters"_sr;
const Value noTenantValue = "no_tenant"_sr;
const TenantName tenantNamePrefix = "tenant_management_workload_"_sr;
const ClusterName dataClusterName = "cluster1"_sr;
TenantName localTenantNamePrefix;
TenantName localTenantGroupNamePrefix;
@ -165,7 +166,8 @@ struct TenantManagementWorkload : TestWorkload {
DataClusterEntry entry;
entry.capacity.numTenantGroups = 1e9;
wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, g_simulator.extraDatabases[0], entry));
wait(MetaclusterAPI::registerCluster(
self->mvDb, self->dataClusterName, g_simulator.extraDatabases[0], entry));
}
state Transaction tr(cx);
@ -1397,20 +1399,40 @@ struct TenantManagementWorkload : TestWorkload {
// Verify that the tenant count matches the actual number of tenants in the cluster and that we haven't created too
// many
ACTOR static Future<Void> checkTenantCount(Database cx) {
state Reference<ReadYourWritesTransaction> tr = cx->createTransaction();
ACTOR template <bool UseMetacluster, class DB>
static Future<Void> checkTenantCount(TenantManagementWorkload* self, Reference<DB> db) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
state int64_t tenantCount = wait(TenantMetadata::tenantCount.getD(tr, Snapshot::False, 0));
KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenants =
wait(TenantMetadata::tenantMap.getRange(tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1));
state int64_t tenantCount;
state KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenants;
if (UseMetacluster) {
wait(store(tenantCount,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata.tenantCount.getD(
tr, Snapshot::False, 0)));
wait(store(tenants,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata.tenantMap.getRange(
tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)));
int64_t clusterTenantCount =
wait(MetaclusterAPI::ManagementClusterMetadata::clusterTenantCount.getD(
tr, self->dataClusterName, Snapshot::False, 0));
ASSERT(clusterTenantCount == tenantCount);
} else {
wait(store(tenantCount, TenantMetadata::tenantCount.getD(tr, Snapshot::False, 0)));
wait(store(
tenants,
TenantMetadata::tenantMap.getRange(tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)));
ASSERT(tenantCount <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
}
ASSERT(tenants.results.size() == tenantCount && !tenants.more);
ASSERT(tenantCount <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
return Void();
} catch (Error& e) {
wait(tr->onError(e));
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
@ -1647,7 +1669,10 @@ struct TenantManagementWorkload : TestWorkload {
}
if (self->clientId == 0) {
wait(checkTenantCount(cx));
if (self->useMetacluster) {
wait(checkTenantCount<true>(self, self->mvDb));
}
wait(checkTenantCount<false>(self, self->dataDb.getReference()));
}
wait(compareTenants(self) && compareTenantGroups(self) && checkTenantTombstones(self));